yindo/posthog
1
0
Fork 0
mirror of https://github.com/BillyOutlast/posthog.git synced 2026-02-04 03:01:23 +01:00
Code Issues Packages Projects Releases Wiki Activity

feat(err): pipeline support for hermes frames (#37539)

Browse Source
This commit is contained in:
Oliver Browne
2025-09-16 13:42:11 +03:00
committed by GitHub
parent a6f83972e5
commit 56fa23f873
37 changed files with 753 additions and 184 deletions
Download Patch File Download Diff File Expand all files Collapse all files

89
cli/Cargo.lock generated
View File

@@ -239,9 +239,9 @@ dependencies = [
[[package]]
name = "cc"
version = "1.2.36"
version = "1.2.37"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5252b3d2648e5eedbc1a6f501e3c795e07025c1e93bbf8bbdd6eef7f447a6d54"
checksum = "65193589c6404eb80b450d618eaf9a2cafaaafd57ecce47370519ef674a7bd44"
dependencies = [
"find-msvc-tools",
"shlex",
@@ -725,7 +725,7 @@ dependencies = [
"js-sys",
"libc",
"r-efi",
"wasi 0.14.5+wasi-0.2.4",
"wasi 0.14.7+wasi-0.2.4",
"wasm-bindgen",
]
@@ -930,9 +930,9 @@ dependencies = [
[[package]]
name = "hyper-util"
version = "0.1.16"
version = "0.1.17"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8d9b05277c7e8da2c93a568989bb6207bef0112e8d17df7a6eda4a3cf143bc5e"
checksum = "3c6995591a8f1380fcb4ba966a252a4b29188d51d2b89e3a252f5305be65aea8"
dependencies = [
"base64 0.22.1",
"bytes",
@@ -954,9 +954,9 @@ dependencies = [
[[package]]
name = "iana-time-zone"
version = "0.1.63"
version = "0.1.64"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b0c919e5debc312ad217002b8048a17b7d83f80703865bbfcfebb0458b0b27d8"
checksum = "33e57f83510bb73707521ebaffa789ec8caf86f9657cad665b092b581d40e9fb"
dependencies = [
"android_system_properties",
"core-foundation-sys",
@@ -1097,9 +1097,9 @@ checksum = "cd62e6b5e86ea8eeeb8db1de02880a6abc01a397b2ebb64b5d74ac255318f5cb"
[[package]]
name = "indexmap"
version = "2.11.1"
version = "2.11.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "206a8042aec68fa4a62e8d3f7aa4ceb508177d9324faf261e1959e495b7a1921"
checksum = "92119844f513ffa41556430369ab02c295a3578af21cf945caa3e9e0c2481ac3"
dependencies = [
"equivalent",
"hashbrown",
@@ -1219,9 +1219,9 @@ checksum = "6a82ae493e598baaea5209805c49bbf2ea7de956d50d7da0da1164f9c6d28543"
[[package]]
name = "libredox"
version = "0.1.9"
version = "0.1.10"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "391290121bad3d37fbddad76d8f5d1c1c314cfc646d143d7e07a3086ddff0ce3"
checksum = "416f7e718bdb06000964960ffa43b4335ad4012ae8b99060261aa4a8088d5ccb"
dependencies = [
"bitflags 2.9.4",
"libc",
@@ -1543,9 +1543,9 @@ dependencies = [
[[package]]
name = "posthog-symbol-data"
version = "0.1.0"
version = "0.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "155615f55dabf848e8e69d7081412e424afbea5060a49bb9582f48d0f8141f64"
checksum = "ed6c8edcab40fb71e1eb6b6af8961b3d0d04fbcaa841331036c964868d5dcb22"
dependencies = [
"serde",
"thiserror 1.0.69",
@@ -1908,7 +1908,7 @@ dependencies = [
"once_cell",
"ring",
"rustls-pki-types",
"rustls-webpki 0.103.5",
"rustls-webpki 0.103.6",
"subtle",
"zeroize",
]
@@ -1944,9 +1944,9 @@ dependencies = [
[[package]]
name = "rustls-webpki"
version = "0.103.5"
version = "0.103.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b5a37813727b78798e53c2bec3f5e8fe12a6d6f8389bf9ca7802add4c9905ad8"
checksum = "8572f3c2cb9934231157b45499fc41e1f58c589fdfb81a844ba873265e80f8eb"
dependencies = [
"ring",
"rustls-pki-types",
@@ -1992,24 +1992,34 @@ dependencies = [
[[package]]
name = "semver"
version = "1.0.26"
version = "1.0.27"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "56e6fa9c48d24d85fb3de5ad847117517440f6beceb7798af16b4a87d616b8d0"
checksum = "d767eb0aabc880b29956c35734170f26ed551a859dbd361d140cdbeca61ab1e2"
[[package]]
name = "serde"
version = "1.0.219"
version = "1.0.225"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5f0e2c6ed6606019b4e29e69dbaba95b11854410e5347d525002456dbbb786b6"
checksum = "fd6c24dee235d0da097043389623fb913daddf92c76e9f5a1db88607a0bcbd1d"
dependencies = [
"serde_core",
"serde_derive",
]
[[package]]
name = "serde_core"
version = "1.0.225"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "659356f9a0cb1e529b24c01e43ad2bdf520ec4ceaf83047b83ddcc2251f96383"
dependencies = [
"serde_derive",
]
[[package]]
name = "serde_derive"
version = "1.0.219"
version = "1.0.225"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5b0276cf7f2c73365f7157c8123c21cd9a50fbbd844757af28ca1f5925fc2a00"
checksum = "0ea936adf78b1f766949a4977b91d2f5595825bd6ec079aa9543ad2685fc4516"
dependencies = [
"proc-macro2",
"quote",
@@ -2018,14 +2028,15 @@ dependencies = [
[[package]]
name = "serde_json"
version = "1.0.143"
version = "1.0.145"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d401abef1d108fbd9cbaebc3e46611f4b1021f714a0597a71f41ee463f5f4a5a"
checksum = "402a6f66d8c709116cf22f558eab210f5a50187f702eb4d7e5ef38d9a7f1c79c"
dependencies = [
"itoa",
"memchr",
"ryu",
"serde",
"serde_core",
]
[[package]]
@@ -2725,18 +2736,18 @@ checksum = "ccf3ec651a847eb01de73ccad15eb7d99f80485de043efb2f370cd654f4ea44b"
[[package]]
name = "wasi"
version = "0.14.5+wasi-0.2.4"
version = "0.14.7+wasi-0.2.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a4494f6290a82f5fe584817a676a34b9d6763e8d9d18204009fb31dceca98fd4"
checksum = "883478de20367e224c0090af9cf5f9fa85bed63a95c1abf3afc5c083ebc06e8c"
dependencies = [
"wasip2",
]
[[package]]
name = "wasip2"
version = "1.0.0+wasi-0.2.4"
version = "1.0.1+wasi-0.2.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "03fa2761397e5bd52002cd7e73110c71af2109aca4e521a9f40473fe685b0a24"
checksum = "0562428422c63773dad2c345a1882263bbf4d65cf3f42e90921f787ef5ad58e7"
dependencies = [
"wit-bindgen",
]
@@ -2881,13 +2892,13 @@ checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f"
[[package]]
name = "windows-core"
version = "0.61.2"
version = "0.62.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c0fdd3ddb90610c7638aa2b3a3ab2904fb9e5cdbecc643ddb3647212781c4ae3"
checksum = "57fe7168f7de578d2d8a05b07fd61870d2e73b4020e9f49aa00da8471723497c"
dependencies = [
"windows-implement",
"windows-interface",
"windows-link 0.1.3",
"windows-link 0.2.0",
"windows-result",
"windows-strings",
]
@@ -2928,20 +2939,20 @@ checksum = "45e46c0661abb7180e7b9c281db115305d49ca1709ab8242adf09666d2173c65"
[[package]]
name = "windows-result"
version = "0.3.4"
version = "0.4.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "56f42bd332cc6c8eac5af113fc0c1fd6a8fd2aa08a0119358686e5160d0586c6"
checksum = "7084dcc306f89883455a206237404d3eaf961e5bd7e0f312f7c91f57eb44167f"
dependencies = [
"windows-link 0.1.3",
"windows-link 0.2.0",
]
[[package]]
name = "windows-strings"
version = "0.4.2"
version = "0.5.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "56e6c93f3a0c3b36176cb1327a4958a0353d5d166c2a35cb268ace15e91d3b57"
checksum = "7218c655a553b0bed4426cf54b20d7ba363ef543b52d515b3e48d7fd55318dda"
dependencies = [
"windows-link 0.1.3",
"windows-link 0.2.0",
]
[[package]]
@@ -3187,9 +3198,9 @@ dependencies = [
[[package]]
name = "wit-bindgen"
version = "0.45.1"
version = "0.46.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5c573471f125075647d03df72e026074b7203790d41351cd6edc96f46bcccd36"
checksum = "f17a85883d4e6d00e8a97c586de764dabcc06133f7f1d55dce5cdc070ad7fe59"
[[package]]
name = "writeable"

2
cli/Cargo.toml
View File

@@ -22,7 +22,7 @@ path = "src/main.rs"
clap = { version = "4.5.31", features = ["derive"] }
dirs = "6.0.0"
inquire = "0.7.5"
posthog-symbol-data = "0.1.0"
posthog-symbol-data = "0.2.0"
walkdir = "2.5.0"
globset = "0.4"
ratatui = "0.29.0"

8
rust/Cargo.lock generated
View File

@@ -2033,7 +2033,7 @@ dependencies = [
"moka",
"once_cell",
"posthog-rs",
"posthog-symbol-data 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)",
"posthog-symbol-data 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)",
"rdkafka",
"regex",
"reqwest 0.12.3",
@@ -5154,7 +5154,7 @@ dependencies = [
[[package]]
name = "posthog-symbol-data"
version = "0.1.0"
version = "0.2.0"
dependencies = [
"serde",
"thiserror 1.0.69",
@@ -5162,9 +5162,9 @@ dependencies = [
[[package]]
name = "posthog-symbol-data"
version = "0.1.0"
version = "0.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "155615f55dabf848e8e69d7081412e424afbea5060a49bb9582f48d0f8141f64"
checksum = "ed6c8edcab40fb71e1eb6b6af8961b3d0d04fbcaa841331036c964868d5dcb22"
dependencies = [
"serde",
"thiserror 1.0.69",

2
rust/common/symbol_data/Cargo.toml
View File

@@ -1,6 +1,6 @@
[package]
name = "posthog-symbol-data"
version = "0.1.0"
version = "0.2.0"
authors = [
"David <david@posthog.com>",
"Olly <oliver@posthog.com>",

22
rust/common/symbol_data/src/data_types/hermesmap.rs Normal file
View File

@@ -0,0 +1,22 @@
use crate::symbol_data::{SymbolData, SymbolDataType};
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct HermesMap {
pub sourcemap: String, // JSON content of sourcemap
}
impl SymbolData for HermesMap {
fn from_bytes(data: Vec<u8>) -> Result<Self, crate::SymbolDataError> {
Ok(Self {
sourcemap: String::from_utf8(data)?,
})
}
fn into_bytes(self) -> Vec<u8> {
self.sourcemap.into_bytes()
}
fn data_type() -> crate::symbol_data::SymbolDataType {
SymbolDataType::HermesMap
}
}

1
rust/common/symbol_data/src/data_types/mod.rs
View File

@@ -1 +1,2 @@
pub mod hermesmap;
pub mod sourcemap;

3
rust/common/symbol_data/src/lib.rs
View File

@@ -12,3 +12,6 @@ pub use symbol_data::write as write_symbol_data;
// Javascript
pub use data_types::sourcemap::SourceAndMap;
// Hermes
pub use data_types::hermesmap::HermesMap;

1
rust/common/symbol_data/src/symbol_data.rs
View File

@@ -6,6 +6,7 @@ const VERSION: u32 = 1;
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum SymbolDataType {
SourceAndMap = 2,
HermesMap = 3,
}
pub trait SymbolData: Sized {

2
rust/cymbal/Cargo.toml
View File

@@ -19,7 +19,7 @@ common-types = { path = "../common/types" }
common-dns = { path = "../common/dns" }
common-redis = { path = "../common/redis" }
limiters = { path = "../common/limiters" }
posthog-symbol-data = "0.1.0"
posthog-symbol-data = "0.2.0"
common-geoip = { path = "../common/geoip" }
hogvm = { path = "../common/hogvm" }
thiserror = { workspace = true }

25
rust/cymbal/src/app_context.rs
View File

@@ -23,6 +23,7 @@ use crate::{
caching::{Caching, SymbolSetCache},
chunk_id::ChunkIdFetcher,
concurrency,
hermesmap::HermesMapProvider,
saving::Saving,
sourcemap::SourcemapProvider,
Catalog, S3Client,
@@ -107,34 +108,42 @@ impl AppContext {
)));
let smp = SourcemapProvider::new(config);
let chunk_layer = ChunkIdFetcher::new(
let smp_chunk = ChunkIdFetcher::new(
smp,
s3_client.clone(),
pool.clone(),
config.object_storage_bucket.clone(),
);
let saving_layer = Saving::new(
chunk_layer,
let smp_saving = Saving::new(
smp_chunk,
pool.clone(),
s3_client.clone(),
config.object_storage_bucket.clone(),
config.ss_prefix.clone(),
);
let caching_layer = Caching::new(saving_layer, ss_cache.clone());
let smp_caching = Caching::new(smp_saving, ss_cache.clone());
// We want to fetch each sourcemap from the outside world
// exactly once, and if it isn't in the cache, load/parse
// it from s3 exactly once too. Limiting the per symbol set
// reference concurrency to 1 ensures this.
let limited_layer = concurrency::AtMostOne::new(caching_layer);
let smp_atmostonce = concurrency::AtMostOne::new(smp_caching);
let hmp = HermesMapProvider {};
let hmp_chunk = ChunkIdFetcher::new(
hmp,
s3_client.clone(),
pool.clone(),
config.object_storage_bucket.clone(),
);
let hmp_caching = Caching::new(hmp_chunk, ss_cache.clone());
// We skip the saving layer for HermesMapProvider, since it'll never fetch something from the outside world.
info!(
"AppContext initialized, subscribed to topic {}",
config.consumer.kafka_consumer_topic
);
let catalog = Catalog::new(limited_layer);
let catalog = Catalog::new(smp_atmostonce, hmp_caching);
let resolver = Resolver::new(config);
let team_manager = TeamManager::new(config);

36
rust/cymbal/src/bin/hermes.rs Normal file
View File

@@ -0,0 +1,36 @@
use regex::Regex;
#[tokio::main]
async fn main() {
let composed_map = sourcemap::decode_slice(
include_str!("../../tests/static/hermes/composed_example.map").as_bytes(),
)
.unwrap();
let raw_stack = include_str!("../../tests/static/hermes/raw_stack.txt");
let frame_regex = Regex::new(r"at\s+(\S+)\s+\(address at\s+[^:]+:(\d+):(\d+)\)").unwrap();
let expected_names = [
"c",
"b",
"a",
"loadModuleImplementation",
"guardedLoadModule",
"metroRequire",
"global",
];
for (captures, expected) in frame_regex
.captures_iter(raw_stack)
.zip(expected_names.iter())
{
let line: u32 = captures[2].parse().unwrap();
let col: u32 = captures[3].parse().unwrap();
composed_map.lookup_token(line - 1, col).unwrap();
let resolved = composed_map
.get_original_function_name(line - 1, col, Some(&captures[1]), None)
.unwrap_or(&captures[1]);
assert_eq!(resolved, *expected);
}
}

36
rust/cymbal/src/error.rs
View File

@@ -10,13 +10,11 @@ use thiserror::Error;
use uuid::Uuid;
#[derive(Debug, Error)]
pub enum Error {
pub enum ResolveError {
#[error(transparent)]
UnhandledError(#[from] UnhandledError),
#[error(transparent)]
ResolutionError(#[from] FrameError),
#[error(transparent)]
EventError(#[from] EventError),
}
// An unhandled failure at some stage of the event pipeline, as
@@ -68,6 +66,8 @@ pub enum UnhandledError {
pub enum FrameError {
#[error(transparent)]
JavaScript(#[from] JsResolveErr),
#[error(transparent)]
Hermes(#[from] HermesError),
#[error("No symbol set for chunk id: {0}")]
MissingChunkIdData(String),
}
@@ -124,6 +124,20 @@ pub enum JsResolveErr {
NoSourcemapUploaded(String),
}
#[derive(Debug, Error, Serialize, Deserialize)]
pub enum HermesError {
#[error("Data error: {0}")]
DataError(#[from] SymbolDataError),
#[error("Invalid map: {0}")]
InvalidMap(String),
#[error("No sourcemap uploaded for chunk id: {0}")]
NoSourcemapUploaded(String),
#[error("No chunk id sent with frame")]
NoChunkId,
#[error("No token for column {0} on chunk {1}")]
NoTokenForColumn(u32, String),
}
#[derive(Debug, Error, Clone)]
pub enum EventError {
#[error("Wrong event type: {0} for event {1}")]
@@ -146,12 +160,26 @@ pub enum EventError {
FilteredByTeamId,
}
impl From<JsResolveErr> for Error {
impl From<JsResolveErr> for ResolveError {
fn from(e: JsResolveErr) -> Self {
FrameError::JavaScript(e).into()
}
}
impl From<HermesError> for ResolveError {
fn from(e: HermesError) -> Self {
FrameError::Hermes(e).into()
}
}
impl From<FrameError> for UnhandledError {
fn from(e: FrameError) -> Self {
// TODO - this should be unreachable, but I need to reconsider the error enum structure to make it possible to assert that
// at the type level. Leaving for a later refactor for now.
UnhandledError::Other(format!("Unhandled resolution error: {e}"))
}
}
impl From<reqwest::Error> for JsResolveErr {
fn from(e: reqwest::Error) -> Self {
if e.is_timeout() {

22
rust/cymbal/src/fingerprinting/mod.rs
View File

@@ -109,12 +109,16 @@ mod test {
use uuid::Uuid;
use crate::{frames::Frame, types::Stacktrace};
use crate::{
frames::{Frame, FrameId},
types::Stacktrace,
};
use super::*;
#[test]
fn test_some_resolved_frames() {
let team_id = 1;
let mut exception = Exception {
exception_id: None,
exception_type: "TypeError".to_string(),
@@ -127,7 +131,7 @@ mod test {
let mut resolved_frames = vec![
Frame {
raw_id: String::new(),
raw_id: FrameId::new(String::new(), team_id),
mangled_name: "foo".to_string(),
line: Some(10),
column: Some(5),
@@ -143,7 +147,7 @@ mod test {
synthetic: false,
},
Frame {
raw_id: String::new(),
raw_id: FrameId::new(String::new(), team_id),
mangled_name: "bar".to_string(),
line: Some(20),
column: Some(15),
@@ -161,7 +165,7 @@ mod test {
];
let unresolved_frame = Frame {
raw_id: String::new(),
raw_id: FrameId::new(String::new(), team_id),
mangled_name: "xyz".to_string(),
line: Some(30),
column: Some(25),
@@ -209,7 +213,7 @@ mod test {
let resolved_frames = vec![
Frame {
raw_id: String::new(),
raw_id: FrameId::new(String::new(), 1),
mangled_name: "foo".to_string(),
line: Some(10),
column: Some(5),
@@ -225,7 +229,7 @@ mod test {
synthetic: false,
},
Frame {
raw_id: String::new(),
raw_id: FrameId::new(String::new(), 1),
mangled_name: "bar".to_string(),
line: Some(20),
column: Some(15),
@@ -241,7 +245,7 @@ mod test {
synthetic: false,
},
Frame {
raw_id: String::new(),
raw_id: FrameId::new(String::new(), 1),
mangled_name: "xyz".to_string(),
line: Some(30),
column: Some(25),
@@ -283,7 +287,7 @@ mod test {
};
let mut resolved_frames = vec![Frame {
raw_id: String::new(),
raw_id: FrameId::new(String::new(), 1),
mangled_name: "foo".to_string(),
line: Some(10),
column: Some(5),
@@ -300,7 +304,7 @@ mod test {
}];
let non_app_frame = Frame {
raw_id: String::new(),
raw_id: FrameId::new(String::new(), 1),
mangled_name: "bar".to_string(),
line: Some(20),
column: Some(15),

48
rust/cymbal/src/frames/mod.rs
View File

@@ -8,8 +8,8 @@ use crate::{
error::UnhandledError,
fingerprinting::{FingerprintBuilder, FingerprintComponent, FingerprintRecordPart},
langs::{
custom::CustomFrame, go::RawGoFrame, js::RawJSFrame, node::RawNodeFrame,
python::RawPythonFrame,
custom::CustomFrame, go::RawGoFrame, hermes::RawHermesFrame, js::RawJSFrame,
node::RawNodeFrame, python::RawPythonFrame,
},
metric_consts::PER_FRAME_TIME,
sanitize_string,
@@ -33,6 +33,8 @@ pub enum RawFrame {
JavaScriptNode(RawNodeFrame),
#[serde(rename = "go")]
Go(RawGoFrame),
#[serde(rename = "hermes")]
Hermes(RawHermesFrame),
// TODO - remove once we're happy no clients are using this anymore
#[serde(rename = "javascript")]
LegacyJS(RawJSFrame),
@@ -40,6 +42,26 @@ pub enum RawFrame {
Custom(CustomFrame),
}
#[derive(Debug, Deserialize, Serialize, Clone, Hash, Eq, PartialEq)]
pub struct FrameId {
pub raw_id: String,
#[serde(skip)]
pub team_id: i32,
}
impl FrameId {
pub fn new(raw_id: String, team_id: i32) -> Self {
FrameId { raw_id, team_id }
}
pub fn placeholder() -> Self {
FrameId {
raw_id: "placeholder".to_string(),
team_id: 0,
}
}
}
impl RawFrame {
pub async fn resolve(&self, team_id: i32, catalog: &Catalog) -> Result<Frame, UnhandledError> {
let frame_resolve_time = common_metrics::timing_guard(PER_FRAME_TIME, &[]);
@@ -53,11 +75,12 @@ impl RawFrame {
RawFrame::Python(frame) => (Ok(frame.into()), "python"),
RawFrame::Custom(frame) => (Ok(frame.into()), "custom"),
RawFrame::Go(frame) => (Ok(frame.into()), "go"),
RawFrame::Hermes(frame) => (frame.resolve(team_id, catalog).await, "hermes"),
};
// The raw id of the frame is set after it's resolved
let res = res.map(|mut f| {
f.raw_id = self.frame_id();
f.raw_id = self.frame_id(team_id);
f
});
@@ -76,6 +99,7 @@ impl RawFrame {
match self {
RawFrame::JavaScriptWeb(frame) | RawFrame::LegacyJS(frame) => frame.symbol_set_ref(),
RawFrame::JavaScriptNode(frame) => frame.chunk_id.clone(),
RawFrame::Hermes(frame) => frame.chunk_id.clone(),
// TODO - Python and Go frames don't use symbol sets for frame resolution, but could still use "marker" symbol set
// to associate a given frame with a given release (basically, a symbol set with no data, just some id,
// which we'd then use to do a join on the releases table to get release information)
@@ -84,14 +108,17 @@ impl RawFrame {
}
}
pub fn frame_id(&self) -> String {
match self {
pub fn frame_id(&self, team_id: i32) -> FrameId {
let hash_id = match self {
RawFrame::JavaScriptWeb(raw) | RawFrame::LegacyJS(raw) => raw.frame_id(),
RawFrame::JavaScriptNode(raw) => raw.frame_id(),
RawFrame::Python(raw) => raw.frame_id(),
RawFrame::Go(raw) => raw.frame_id(),
RawFrame::Custom(raw) => raw.frame_id(),
}
RawFrame::Hermes(raw) => raw.frame_id(),
};
FrameId::new(hash_id, team_id)
}
}
@@ -99,7 +126,8 @@ impl RawFrame {
#[derive(Debug, Clone, Serialize, Deserialize, Eq, PartialEq)]
pub struct Frame {
// Properties used in processing
pub raw_id: String, // The raw frame id this was resolved from
#[serde(flatten)]
pub raw_id: FrameId, // The raw frame id this was resolved from
pub mangled_name: String, // Mangled name of the function
#[serde(skip_serializing_if = "Option::is_none")]
pub line: Option<u32>, // Line the function is define on, if known
@@ -146,8 +174,8 @@ pub struct ContextLine {
impl FingerprintComponent for Frame {
fn update(&self, fp: &mut FingerprintBuilder) {
let get_part = |s: &str, p: Vec<&str>| FingerprintRecordPart::Frame {
raw_id: s.to_string(),
let get_part = |s: &FrameId, p: Vec<&str>| FingerprintRecordPart::Frame {
raw_id: s.raw_id.to_string(),
pieces: p.into_iter().map(String::from).collect(),
};
@@ -220,7 +248,7 @@ impl ContextLine {
impl std::fmt::Display for Frame {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
writeln!(f, "Frame {}:", self.raw_id)?;
writeln!(f, "Frame {}:", self.raw_id.raw_id)?;
// Function name and location
write!(

25
rust/cymbal/src/frames/records.rs
View File

@@ -4,14 +4,13 @@ use serde_json::Value;
use sqlx::Executor;
use uuid::Uuid;
use crate::error::UnhandledError;
use crate::{error::UnhandledError, frames::FrameId};
use super::{Context, Frame};
#[derive(Debug, Serialize, Deserialize, Clone)]
pub struct ErrorTrackingStackFrame {
pub raw_id: String,
pub team_id: i32,
pub id: FrameId,
pub created_at: DateTime<Utc>,
pub symbol_set_id: Option<Uuid>,
pub contents: Frame,
@@ -21,16 +20,14 @@ pub struct ErrorTrackingStackFrame {
impl ErrorTrackingStackFrame {
pub fn new(
raw_id: String,
team_id: i32,
id: FrameId,
symbol_set_id: Option<Uuid>,
contents: Frame,
resolved: bool,
context: Option<Context>,
) -> Self {
Self {
raw_id,
team_id,
id,
symbol_set_id,
contents,
resolved,
@@ -59,8 +56,8 @@ impl ErrorTrackingStackFrame {
resolved = $6,
context = $8
"#,
self.raw_id,
self.team_id,
self.id.raw_id,
self.id.team_id,
self.created_at,
self.symbol_set_id,
serde_json::to_value(&self.contents)?,
@@ -73,8 +70,7 @@ impl ErrorTrackingStackFrame {
pub async fn load<'c, E>(
e: E,
team_id: i32,
raw_id: &str,
id: &FrameId,
result_ttl: Duration,
) -> Result<Option<Self>, UnhandledError>
where
@@ -96,8 +92,8 @@ impl ErrorTrackingStackFrame {
FROM posthog_errortrackingstackframe
WHERE raw_id = $1 AND team_id = $2
"#,
raw_id,
team_id
id.raw_id,
id.team_id
)
.fetch_optional(e)
.await?;
@@ -127,8 +123,7 @@ impl ErrorTrackingStackFrame {
frame.context = context.clone();
Ok(Some(Self {
raw_id: found.raw_id,
team_id: found.team_id,
id: FrameId::new(found.raw_id, found.team_id),
created_at: found.created_at,
symbol_set_id: found.symbol_set_id,
contents: frame,

35
rust/cymbal/src/frames/resolver.rs
View File

@@ -6,6 +6,7 @@ use sqlx::PgPool;
use crate::{
config::Config,
error::UnhandledError,
frames::FrameId,
metric_consts::{FRAME_CACHE_HITS, FRAME_CACHE_MISSES, FRAME_DB_HITS, FRAME_DB_MISSES},
symbol_store::{saving::SymbolSetRecord, Catalog},
};
@@ -13,7 +14,7 @@ use crate::{
use super::{records::ErrorTrackingStackFrame, releases::ReleaseRecord, Frame, RawFrame};
pub struct Resolver {
cache: Cache<String, ErrorTrackingStackFrame>,
cache: Cache<FrameId, ErrorTrackingStackFrame>,
result_ttl: chrono::Duration,
}
@@ -35,19 +36,19 @@ impl Resolver {
pool: &PgPool,
catalog: &Catalog,
) -> Result<Frame, UnhandledError> {
if let Some(result) = self.cache.get(&frame.frame_id()) {
if let Some(result) = self.cache.get(&frame.frame_id(team_id)) {
metrics::counter!(FRAME_CACHE_HITS).increment(1);
return Ok(result.contents);
}
metrics::counter!(FRAME_CACHE_MISSES).increment(1);
if let Some(mut result) =
ErrorTrackingStackFrame::load(pool, team_id, &frame.frame_id(), self.result_ttl).await?
ErrorTrackingStackFrame::load(pool, &frame.frame_id(team_id), self.result_ttl).await?
{
// We don't serialise release information on the frame, so we have to reload it if we fetched
// the saved result from the DB
result.contents = add_release_info(pool, result.contents, frame, team_id).await?;
self.cache.insert(frame.frame_id(), result.clone());
self.cache.insert(frame.frame_id(team_id), result.clone());
metrics::counter!(FRAME_DB_HITS).increment(1);
return Ok(result.contents);
}
@@ -64,8 +65,7 @@ impl Resolver {
};
let record = ErrorTrackingStackFrame::new(
frame.frame_id(),
team_id,
frame.frame_id(team_id),
set.map(|s| s.id),
resolved.clone(),
resolved.resolved,
@@ -74,7 +74,7 @@ impl Resolver {
record.save(pool).await?;
self.cache.insert(frame.frame_id(), record);
self.cache.insert(frame.frame_id(team_id), record);
Ok(resolved)
}
}
@@ -107,6 +107,7 @@ mod test {
frames::{records::ErrorTrackingStackFrame, resolver::Resolver, RawFrame},
symbol_store::{
chunk_id::ChunkIdFetcher,
hermesmap::HermesMapProvider,
saving::{Saving, SymbolSetRecord},
sourcemap::SourcemapProvider,
Catalog, S3Client,
@@ -161,7 +162,14 @@ mod test {
config.ss_prefix.clone(),
);
let catalog = Catalog::new(saving_smp);
let hmp = ChunkIdFetcher::new(
HermesMapProvider {},
client.clone(),
pool.clone(),
config.object_storage_bucket.clone(),
);
let catalog = Catalog::new(saving_smp, hmp);
(config, catalog, server)
}
@@ -272,12 +280,11 @@ mod test {
.unwrap();
// get the frame
let frame_id = frame.frame_id();
let frame =
ErrorTrackingStackFrame::load(&pool, 0, &frame_id, chrono::Duration::minutes(30))
.await
.unwrap()
.unwrap();
let frame_id = frame.frame_id(0);
let frame = ErrorTrackingStackFrame::load(&pool, &frame_id, chrono::Duration::minutes(30))
.await
.unwrap()
.unwrap();
assert_eq!(frame.symbol_set_id.unwrap(), set.id);

4
rust/cymbal/src/langs/custom.rs
View File

@@ -2,7 +2,7 @@ use serde::{Deserialize, Serialize};
use sha2::{Digest, Sha512};
use crate::{
frames::{Context, ContextLine, Frame},
frames::{Context, ContextLine, Frame, FrameId},
langs::CommonFrameMetadata,
};
@@ -87,7 +87,7 @@ impl CustomFrame {
impl From<&CustomFrame> for Frame {
fn from(value: &CustomFrame) -> Self {
Frame {
raw_id: String::new(),
raw_id: FrameId::placeholder(),
mangled_name: value.function.clone(),
line: value.lineno,
column: value.colno,

7
rust/cymbal/src/langs/go.rs
View File

@@ -1,7 +1,10 @@
use serde::{Deserialize, Serialize};
use sha2::{Digest, Sha512};
use crate::{frames::Frame, langs::CommonFrameMetadata};
use crate::{
frames::{Frame, FrameId},
langs::CommonFrameMetadata,
};
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct RawGoFrame {
@@ -25,7 +28,7 @@ impl RawGoFrame {
impl From<&RawGoFrame> for Frame {
fn from(frame: &RawGoFrame) -> Self {
Frame {
raw_id: frame.frame_id(),
raw_id: FrameId::placeholder(),
mangled_name: frame.function.clone(),
line: Some(frame.lineno),
column: None,

292
rust/cymbal/src/langs/hermes.rs Normal file
View File

@@ -0,0 +1,292 @@
use std::fmt::Display;
use serde::{Deserialize, Serialize};
use sha2::{Digest, Sha512};
use sourcemap::Token;
use crate::{
error::{FrameError, HermesError, ResolveError, UnhandledError},
frames::{Frame, FrameId},
langs::{
utils::{add_raw_to_junk, get_token_context},
CommonFrameMetadata,
},
sanitize_string,
symbol_store::{chunk_id::OrChunkId, hermesmap::ParsedHermesMap, SymbolCatalog},
};
#[derive(Debug, Clone, Deserialize, Serialize)]
pub struct RawHermesFrame {
#[serde(rename = "colno")]
pub column: u32, // Hermes frames don't have a line number
#[serde(rename = "filename")]
pub source: String, // This will /usually/ be meaningless
#[serde(rename = "function")]
pub fn_name: String, // Mangled function name - sometimes, but not always, the same as the demangled function name
#[serde(rename = "chunkId", skip_serializing_if = "Option::is_none")]
pub chunk_id: Option<String>, // Hermes frames are required to provide a chunk ID, or they cannot be resolved
#[serde(flatten)]
pub meta: CommonFrameMetadata,
}
// This is an enum it's impossible to construct an instance of. We use it here, along with OrChunkId, to represent that the hermes frames
// will always have a chunk ID - this lets us assert the OrChunkId variant will always be OrChunkId::ChunkId, because the R in this case
// is impossible to construct. Change to a never type once that's stable - https://doc.rust-lang.org/std/primitive.never.html
#[derive(Debug, Clone)]
pub enum HermesRef {}
impl RawHermesFrame {
pub async fn resolve<C>(&self, team_id: i32, catalog: &C) -> Result<Frame, UnhandledError>
where
C: SymbolCatalog<OrChunkId<HermesRef>, ParsedHermesMap>,
{
match self.resolve_impl(team_id, catalog).await {
Ok(frame) => Ok(frame),
Err(ResolveError::ResolutionError(FrameError::Hermes(e))) => {
Ok(self.handle_resolution_error(e))
}
Err(ResolveError::ResolutionError(FrameError::MissingChunkIdData(chunk_id))) => {
Ok(self.handle_resolution_error(HermesError::NoSourcemapUploaded(chunk_id)))
}
Err(ResolveError::ResolutionError(FrameError::JavaScript(e))) => {
// TODO - should be unreachable, specialize ResolveError to encode that
Err(UnhandledError::from(FrameError::from(e)))
}
Err(ResolveError::UnhandledError(e)) => Err(e),
}
}
async fn resolve_impl<C>(&self, team_id: i32, catalog: &C) -> Result<Frame, ResolveError>
where
C: SymbolCatalog<OrChunkId<HermesRef>, ParsedHermesMap>,
{
let r = self.get_ref()?;
let sourcemap = catalog.lookup(team_id, r.clone()).await?;
let sourcemap = &sourcemap.map;
let Some(token) = sourcemap.lookup_token(0, self.column) else {
return Err(HermesError::NoTokenForColumn(self.column, r.to_string()).into());
};
let resolved_name = sourcemap
.get_original_function_name(self.column)
.map(|s| s.to_string());
Ok((self, token, resolved_name).into())
}
pub fn frame_id(&self) -> String {
let mut hasher = Sha512::new();
hasher.update(self.fn_name.as_bytes());
hasher.update(self.source.as_bytes());
hasher.update(self.column.to_string().as_bytes());
if let Some(chunk_id) = &self.chunk_id {
hasher.update(chunk_id.as_bytes());
}
format!("{:x}", hasher.finalize())
}
pub fn symbol_set_ref(&self) -> Option<String> {
self.get_ref().ok().map(|r| r.to_string())
}
fn get_ref(&self) -> Result<OrChunkId<HermesRef>, HermesError> {
self.chunk_id
.as_ref()
.map(|id| OrChunkId::chunk_id(id.clone()))
.ok_or(HermesError::NoChunkId)
}
fn handle_resolution_error(&self, err: HermesError) -> Frame {
(self, err).into()
}
}
impl Display for HermesRef {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
write!(f, "HermesRef")
}
}
impl From<(&RawHermesFrame, HermesError)> for Frame {
fn from((frame, err): (&RawHermesFrame, HermesError)) -> Self {
let mut res = Self {
raw_id: FrameId::placeholder(),
mangled_name: frame.fn_name.clone(),
line: Some(1), // Hermes frames are 1-indexed and always 1
column: Some(frame.column),
source: Some(frame.source.clone()),
in_app: frame.meta.in_app,
resolved_name: None,
lang: "hermes-js".to_string(),
resolved: false,
resolve_failure: Some(err.to_string()),
synthetic: frame.meta.synthetic,
junk_drawer: None,
context: None,
release: None,
};
add_raw_to_junk(&mut res, frame);
res
}
}
impl From<(&RawHermesFrame, Token<'_>, Option<String>)> for Frame {
fn from((frame, token, resolved_name): (&RawHermesFrame, Token<'_>, Option<String>)) -> Self {
let source = token.get_source().map(|s| sanitize_string(s.to_string()));
let in_app = source
.as_ref()
.map(|s| !s.contains("node_modules"))
.unwrap_or(frame.meta.in_app);
let mut res = Self {
raw_id: FrameId::placeholder(),
mangled_name: frame.fn_name.clone(),
line: Some(token.get_src_line()),
column: Some(token.get_src_col()),
source,
in_app,
resolved_name,
lang: "hermes-js".to_string(),
resolved: true,
resolve_failure: None,
synthetic: frame.meta.synthetic,
junk_drawer: None,
context: get_token_context(&token, token.get_src_line() as usize),
release: None,
};
add_raw_to_junk(&mut res, frame);
res
}
}
#[cfg(test)]
mod test {
use std::sync::Arc;
use chrono::Utc;
use mockall::predicate;
use posthog_symbol_data::write_symbol_data;
use regex::Regex;
use sqlx::PgPool;
use uuid::Uuid;
use crate::{
config::Config,
frames::RawFrame,
langs::{hermes::RawHermesFrame, CommonFrameMetadata},
symbol_store::{
chunk_id::ChunkIdFetcher, hermesmap::HermesMapProvider, saving::SymbolSetRecord,
sourcemap::SourcemapProvider, Catalog, S3Client,
},
};
const HERMES_MAP: &str = include_str!("../../tests/static/hermes/composed_example.map");
const RAW_STACK: &str = include_str!("../../tests/static/hermes/raw_stack.txt");
#[sqlx::test(migrations = "./tests/test_migrations")]
async fn test_hermes_resolution(db: PgPool) {
let team_id = 1;
let mut config = Config::init_with_defaults().unwrap();
config.object_storage_bucket = "test-bucket".to_string();
let chunk_id = Uuid::now_v7().to_string();
let mut record = SymbolSetRecord {
id: Uuid::now_v7(),
team_id,
set_ref: chunk_id.clone(),
storage_ptr: Some(chunk_id.clone()),
failure_reason: None,
created_at: Utc::now(),
content_hash: Some("fake-hash".to_string()),
last_used: Some(Utc::now()),
};
record.save(&db).await.unwrap();
let mut client = S3Client::default();
client
.expect_get()
.with(
predicate::eq(config.object_storage_bucket.clone()),
predicate::eq(chunk_id.clone()), // We set the chunk id as the storage ptr above, in production it will be a different value with a prefix
)
.returning(|_, _| Ok(get_symbol_data_bytes()));
let client = Arc::new(client);
let hmp = HermesMapProvider {};
let hmp = ChunkIdFetcher::new(
hmp,
client.clone(),
db.clone(),
config.object_storage_bucket.clone(),
);
let smp = SourcemapProvider::new(&config);
let smp = ChunkIdFetcher::new(
smp,
client.clone(),
db.clone(),
config.object_storage_bucket.clone(),
);
let c = Catalog::new(smp, hmp);
for (raw_frame, expected_name) in get_frames(chunk_id) {
let res = raw_frame.resolve(team_id, &c).await.unwrap();
println!("GOT FRAME: {}", serde_json::to_string_pretty(&res).unwrap());
assert!(res.resolved);
assert_eq!(res.resolved_name, expected_name)
}
}
fn get_frames(chunk_id: String) -> Vec<(RawFrame, Option<String>)> {
let frame_regex = Regex::new(r"at\s+(\S+)\s+\(address at\s+[^:]+:(\d+):(\d+)\)").unwrap();
let mut frames = Vec::new();
let expected_names = [
Some("c"),
Some("b"),
Some("a"),
Some("loadModuleImplementation"),
Some("guardedLoadModule"),
Some("metroRequire"),
None,
];
for (captures, expected) in frame_regex
.captures_iter(RAW_STACK)
.zip(expected_names.iter())
{
let name = &captures[1];
let _line: u32 = captures[2].parse().unwrap();
let col: u32 = captures[3].parse().unwrap();
let frame = RawHermesFrame {
column: col,
source: String::new(),
fn_name: name.to_string(),
chunk_id: Some(chunk_id.clone()),
meta: CommonFrameMetadata::default(),
};
frames.push((RawFrame::Hermes(frame), expected.map(String::from)));
}
frames
}
fn get_symbol_data_bytes() -> Vec<u8> {
write_symbol_data(posthog_symbol_data::HermesMap {
sourcemap: HERMES_MAP.to_string(),
})
.unwrap()
}
}

38
rust/cymbal/src/langs/js.rs
View File

@@ -4,15 +4,15 @@ use sha2::{Digest, Sha512};
use symbolic::sourcemapcache::{ScopeLookupResult, SourceLocation, SourcePosition};
use crate::{
error::{Error, FrameError, JsResolveErr, UnhandledError},
frames::Frame,
error::{FrameError, JsResolveErr, ResolveError, UnhandledError},
frames::{Frame, FrameId},
langs::CommonFrameMetadata,
metric_consts::{FRAME_NOT_RESOLVED, FRAME_RESOLVED},
sanitize_string,
symbol_store::{chunk_id::OrChunkId, sourcemap::OwnedSourceMapCache, SymbolCatalog},
};
use super::utils::{add_raw_to_junk, get_context};
use super::utils::{add_raw_to_junk, get_sourcelocation_context};
// A minifed JS stack frame. Just the minimal information needed to lookup some
// sourcemap for it and produce a "real" stack frame.
@@ -45,18 +45,21 @@ impl RawJSFrame {
{
match self.resolve_impl(team_id, catalog).await {
Ok(frame) => Ok(frame),
Err(Error::ResolutionError(FrameError::JavaScript(e))) => {
Err(ResolveError::ResolutionError(FrameError::JavaScript(e))) => {
Ok(self.handle_resolution_error(e))
}
Err(Error::ResolutionError(FrameError::MissingChunkIdData(chunk_id))) => {
Err(ResolveError::ResolutionError(FrameError::MissingChunkIdData(chunk_id))) => {
Ok(self.handle_resolution_error(JsResolveErr::NoSourcemapUploaded(chunk_id)))
}
Err(Error::UnhandledError(e)) => Err(e),
Err(Error::EventError(_)) => unreachable!(),
Err(ResolveError::ResolutionError(FrameError::Hermes(e))) => {
// TODO - should be unreachable, specialize ResolveError to encode that
Err(UnhandledError::from(FrameError::from(e)))
}
Err(ResolveError::UnhandledError(e)) => Err(e),
}
}
async fn resolve_impl<C>(&self, team_id: i32, catalog: &C) -> Result<Frame, Error>
async fn resolve_impl<C>(&self, team_id: i32, catalog: &C) -> Result<Frame, ResolveError>
where
C: SymbolCatalog<OrChunkId<Url>, OwnedSourceMapCache>,
{
@@ -176,28 +179,29 @@ impl From<(&RawJSFrame, SourceLocation<'_>)> for Frame {
ScopeLookupResult::Unknown => None,
};
let source = token.file().and_then(|f| f.name()).map(|s| s.to_string());
let source = token
.file()
.and_then(|f| f.name())
.map(|s| sanitize_string(s.to_string()));
let in_app = source
.as_ref()
.map(|s| !s.contains("node_modules"))
.unwrap_or(raw_frame.meta.in_app);
let mut res = Self {
raw_id: String::new(), // We use placeholders here, as they're overriden at the RawFrame level
raw_id: FrameId::placeholder(), // We use placeholders here, as they're overriden at the RawFrame level
mangled_name: raw_frame.fn_name.clone(),
line: Some(token.line()),
column: Some(token.column()),
source: token
.file()
.and_then(|f| f.name())
.map(|s| sanitize_string(s.to_string())),
source,
in_app,
resolved_name,
lang: "javascript".to_string(),
resolved: true,
resolve_failure: None,
junk_drawer: None,
context: get_context(&token),
context: get_sourcelocation_context(&token),
release: None,
synthetic: raw_frame.meta.synthetic,
};
@@ -229,7 +233,7 @@ impl From<(&RawJSFrame, JsResolveErr, &FrameLocation)> for Frame {
};
let mut res = Self {
raw_id: String::new(),
raw_id: FrameId::placeholder(),
mangled_name: raw_frame.fn_name.clone(),
line: Some(location.line),
column: Some(location.column),
@@ -270,7 +274,7 @@ impl From<&RawJSFrame> for Frame {
let in_app = raw_frame.meta.in_app && !is_anon;
let mut res = Self {
raw_id: String::new(),
raw_id: FrameId::placeholder(),
mangled_name: raw_frame.fn_name.clone(),
line: None,
column: None,

1
rust/cymbal/src/langs/mod.rs
View File

@@ -2,6 +2,7 @@ use serde::{Deserialize, Serialize};
pub mod custom;
pub mod go;
pub mod hermes;
pub mod js;
pub mod node;
pub mod python;

27
rust/cymbal/src/langs/node.rs
View File

@@ -1,6 +1,6 @@
use crate::{
error::{Error, FrameError, JsResolveErr, UnhandledError},
frames::{Context, ContextLine, Frame},
error::{FrameError, JsResolveErr, ResolveError, UnhandledError},
frames::{Context, ContextLine, Frame, FrameId},
langs::CommonFrameMetadata,
metric_consts::{FRAME_NOT_RESOLVED, FRAME_RESOLVED},
sanitize_string,
@@ -13,7 +13,7 @@ use symbolic::sourcemapcache::{ScopeLookupResult, SourceLocation, SourcePosition
use super::{
js::FrameLocation,
utils::{add_raw_to_junk, get_context},
utils::{add_raw_to_junk, get_sourcelocation_context},
};
#[derive(Debug, Clone, Deserialize, Serialize)]
@@ -45,12 +45,15 @@ impl RawNodeFrame {
match self.resolve_impl(team_id, catalog, chunk_id.clone()).await {
Ok(frame) => Ok(frame),
Err(Error::ResolutionError(FrameError::JavaScript(e))) => Ok((self, e).into()),
Err(Error::ResolutionError(FrameError::MissingChunkIdData(chunk_id))) => {
Err(ResolveError::ResolutionError(FrameError::JavaScript(e))) => Ok((self, e).into()),
Err(ResolveError::ResolutionError(FrameError::MissingChunkIdData(chunk_id))) => {
Ok((self, JsResolveErr::NoSourcemapUploaded(chunk_id)).into())
}
Err(Error::UnhandledError(e)) => Err(e),
Err(Error::EventError(_)) => unreachable!(),
Err(ResolveError::ResolutionError(FrameError::Hermes(e))) => {
// TODO - should be unreachable, specialize Error to encode that
Err(UnhandledError::from(FrameError::from(e)))
}
Err(ResolveError::UnhandledError(e)) => Err(e),
}
}
@@ -59,7 +62,7 @@ impl RawNodeFrame {
team_id: i32,
catalog: &C,
chunk_id: String,
) -> Result<Frame, Error>
) -> Result<Frame, ResolveError>
where
C: SymbolCatalog<OrChunkId<Url>, OwnedSourceMapCache>,
{
@@ -146,7 +149,7 @@ impl RawNodeFrame {
impl From<&RawNodeFrame> for Frame {
fn from(raw: &RawNodeFrame) -> Self {
Frame {
raw_id: String::new(),
raw_id: FrameId::placeholder(),
mangled_name: raw.function.clone(),
line: raw.lineno,
column: None,
@@ -184,7 +187,7 @@ impl From<(&RawNodeFrame, SourceLocation<'_>)> for Frame {
.unwrap_or(raw_frame.meta.in_app);
let mut res = Self {
raw_id: String::new(), // We use placeholders here, as they're overriden at the RawFrame level
raw_id: FrameId::placeholder(),
mangled_name: raw_frame.function.clone(),
line: Some(location.line()),
column: Some(location.column()),
@@ -198,7 +201,7 @@ impl From<(&RawNodeFrame, SourceLocation<'_>)> for Frame {
resolved: true,
resolve_failure: None,
junk_drawer: None,
context: get_context(&location),
context: get_sourcelocation_context(&location),
release: None,
synthetic: raw_frame.meta.synthetic,
};
@@ -226,7 +229,7 @@ impl From<(&RawNodeFrame, JsResolveErr)> for Frame {
};
let mut res = Self {
raw_id: String::new(),
raw_id: FrameId::placeholder(),
mangled_name: raw_frame.function.clone(),
line: raw_frame.lineno,
column: raw_frame.colno,

4
rust/cymbal/src/langs/python.rs
View File

@@ -2,7 +2,7 @@ use serde::{Deserialize, Serialize};
use sha2::{Digest, Sha512};
use crate::{
frames::{Context, ContextLine, Frame},
frames::{Context, ContextLine, Frame, FrameId},
langs::CommonFrameMetadata,
};
@@ -79,7 +79,7 @@ impl RawPythonFrame {
impl From<&RawPythonFrame> for Frame {
fn from(raw: &RawPythonFrame) -> Self {
Frame {
raw_id: String::new(),
raw_id: FrameId::placeholder(),
mangled_name: raw.function.clone(),
line: raw.lineno,
column: None,

20
rust/cymbal/src/langs/utils.rs
View File

@@ -1,6 +1,7 @@
use std::sync::atomic::Ordering;
use serde::Serialize;
use sourcemap::Token;
use symbolic::sourcemapcache::SourceLocation;
use crate::{
@@ -13,19 +14,30 @@ pub fn add_raw_to_junk<T: Serialize + Clone>(frame: &mut Frame, raw: &T) {
frame.add_junk("raw_frame", raw.clone()).unwrap();
}
pub fn get_context(token: &SourceLocation) -> Option<Context> {
pub fn get_sourcelocation_context(token: &SourceLocation) -> Option<Context> {
let file = token.file()?;
let token_line_num = token.line();
let src = file.source()?;
let line_limit = FRAME_CONTEXT_LINES.load(Ordering::Relaxed);
get_context_lines(src, token_line_num as usize, line_limit)
get_context_lines(src.lines(), token_line_num as usize, line_limit)
}
fn get_context_lines(src: &str, line: usize, context_len: usize) -> Option<Context> {
pub fn get_token_context(token: &Token<'_>, line: usize) -> Option<Context> {
let src = token.get_source_view()?;
let lines = src.lines();
let line_limit = FRAME_CONTEXT_LINES.load(Ordering::Relaxed);
get_context_lines(lines, line, line_limit)
}
fn get_context_lines<'a, L>(lines: L, line: usize, context_len: usize) -> Option<Context>
where
L: Iterator<Item = &'a str>,
{
let start = line.saturating_sub(context_len).saturating_sub(1);
let mut lines = src.lines().enumerate().skip(start);
let mut lines = lines.enumerate().skip(start);
let before = (&mut lines)
.take(line - start)
.map(|(number, line)| ContextLine::new(number as u32, line))

19
rust/cymbal/src/pipeline/exception/stack_processing.rs
View File

@@ -6,6 +6,7 @@ use crate::{
app_context::AppContext,
error::{PipelineResult, UnhandledError},
fingerprinting::resolve_fingerprint,
frames::FrameId,
metric_consts::{FINGERPRINT_BATCH_TIME, FRAME_BATCH_TIME, FRAME_RESOLUTION},
types::{FingerprintedErrProps, RawErrProps, Stacktrace},
};
@@ -43,7 +44,7 @@ pub async fn do_stack_processing(
};
for frame in frames.iter() {
let id = frame.frame_id();
let id = frame.frame_id(team_id);
if frame_resolve_handles.contains_key(&id) {
// We've already spawned a task to resolve this frame, so we don't need to do it again.
continue;
@@ -88,14 +89,20 @@ pub async fn do_stack_processing(
let fingerprint_timer = common_metrics::timing_guard(FINGERPRINT_BATCH_TIME, &[]);
let mut indexed_fingerprinted = Vec::new();
for (index, mut props) in indexed_props.into_iter() {
let team_id = events[index]
.as_ref()
.expect("no events have been dropped since indexed-property gathering")
.team_id;
for exception in props.exception_list.iter_mut() {
exception.stack = exception
.stack
.take()
.map(|s| {
s.resolve(&frame_lookup_table).ok_or(UnhandledError::Other(
"Stacktrace::resolve returned None".to_string(),
))
s.resolve(team_id, &frame_lookup_table)
.ok_or(UnhandledError::Other(
"Stacktrace::resolve returned None".to_string(),
))
})
.transpose()
.map_err(|e| (index, e))?
@@ -124,12 +131,12 @@ pub async fn do_stack_processing(
Ok(indexed_fingerprinted)
}
fn find_index_with_matching_frame_id(id: &str, list: &[(usize, RawErrProps)]) -> usize {
fn find_index_with_matching_frame_id(id: &FrameId, list: &[(usize, RawErrProps)]) -> usize {
for (index, props) in list.iter() {
for exception in props.exception_list.iter() {
if let Some(Stacktrace::Raw { frames }) = &exception.stack {
for frame in frames {
if frame.frame_id() == id {
if frame.frame_id(id.team_id) == *id {
return *index;
}
}

22
rust/cymbal/src/symbol_store/chunk_id.rs
View File

@@ -204,11 +204,12 @@ mod test {
use crate::{
config::Config,
error::Error,
error::ResolveError,
frames::RawFrame,
langs::js::RawJSFrame,
symbol_store::{
chunk_id::{ChunkIdFetcher, OrChunkId},
hermesmap::HermesMapProvider,
saving::SymbolSetRecord,
sourcemap::{OwnedSourceMapCache, SourcemapProvider},
Catalog, Provider, S3Client,
@@ -226,7 +227,7 @@ mod test {
impl Provider for UnimplementedProvider {
type Ref = Url;
type Set = OwnedSourceMapCache;
type Err = Error;
type Err = ResolveError;
async fn lookup(&self, _team_id: i32, _r: Self::Ref) -> Result<Arc<Self::Set>, Self::Err> {
unimplemented!()
@@ -331,10 +332,21 @@ mod test {
let client = Arc::new(client);
let smp = SourcemapProvider::new(&config);
let chunk_id_fetcher =
ChunkIdFetcher::new(smp, client, db.clone(), config.object_storage_bucket);
let chunk_id_fetcher = ChunkIdFetcher::new(
smp,
client.clone(),
db.clone(),
config.object_storage_bucket.clone(),
);
let catalog = Catalog::new(chunk_id_fetcher);
let hermes_map_fetcher = ChunkIdFetcher::new(
HermesMapProvider {},
client.clone(),
db.clone(),
config.object_storage_bucket,
);
let catalog = Catalog::new(chunk_id_fetcher, hermes_map_fetcher);
let mut frame = get_example_frame();
frame.chunk_id = Some(chunk_id.clone());

46
rust/cymbal/src/symbol_store/hermesmap.rs Normal file
View File

@@ -0,0 +1,46 @@
use axum::async_trait;
use posthog_symbol_data::{read_symbol_data, HermesMap};
use crate::{
error::{HermesError, ResolveError},
langs::hermes::HermesRef,
symbol_store::{Fetcher, Parser},
};
pub struct ParsedHermesMap {
pub map: sourcemap::SourceMapHermes,
}
pub struct HermesMapProvider {}
#[async_trait]
impl Fetcher for HermesMapProvider {
type Ref = HermesRef;
type Fetched = Vec<u8>;
type Err = ResolveError;
async fn fetch(&self, _: i32, _: HermesRef) -> Result<Vec<u8>, Self::Err> {
unreachable!("HermesRef is impossible to construct, so cannot be passed")
}
}
#[async_trait]
impl Parser for HermesMapProvider {
type Source = Vec<u8>;
type Set = ParsedHermesMap;
type Err = ResolveError;
async fn parse(&self, source: Vec<u8>) -> Result<ParsedHermesMap, Self::Err> {
let map: HermesMap = read_symbol_data(source).map_err(HermesError::DataError)?;
Ok(ParsedHermesMap::parse(map)?)
}
}
impl ParsedHermesMap {
pub fn parse(map: HermesMap) -> Result<ParsedHermesMap, HermesError> {
Ok(ParsedHermesMap {
map: sourcemap::SourceMapHermes::from_reader(map.sourcemap.as_bytes())
.map_err(|e| HermesError::InvalidMap(e.to_string()))?,
})
}
}

35
rust/cymbal/src/symbol_store/mod.rs
View File

@@ -6,11 +6,14 @@ use chunk_id::OrChunkId;
use reqwest::Url;
use sourcemap::OwnedSourceMapCache;
use crate::error::Error;
use crate::{
error::ResolveError, langs::hermes::HermesRef, symbol_store::hermesmap::ParsedHermesMap,
};
pub mod caching;
pub mod chunk_id;
pub mod concurrency;
pub mod hermesmap;
pub mod saving;
pub mod sourcemap;
@@ -25,7 +28,7 @@ pub trait SymbolCatalog<Ref, Set>: Send + Sync + 'static {
// TODO - this doesn't actually need to return an Arc, but it does for now, because I'd
// need to re-write the cache to let it return &'s instead, and the Arc overhead is not
// going to be super critical right now
async fn lookup(&self, team_id: i32, r: Ref) -> Result<Arc<Set>, Error>;
async fn lookup(&self, team_id: i32, r: Ref) -> Result<Arc<Set>, ResolveError>;
}
#[async_trait]
@@ -55,20 +58,27 @@ pub trait Provider: Send + Sync + 'static {
pub struct Catalog {
// "source map provider"
pub smp: Box<dyn Provider<Ref = OrChunkId<Url>, Set = OwnedSourceMapCache, Err = Error>>,
pub smp: Box<dyn Provider<Ref = OrChunkId<Url>, Set = OwnedSourceMapCache, Err = ResolveError>>,
// Hermes map provider
pub hmp:
Box<dyn Provider<Ref = OrChunkId<HermesRef>, Set = ParsedHermesMap, Err = ResolveError>>,
}
impl Catalog {
pub fn new(
smp: impl Provider<Ref = OrChunkId<Url>, Set = OwnedSourceMapCache, Err = Error>,
smp: impl Provider<Ref = OrChunkId<Url>, Set = OwnedSourceMapCache, Err = ResolveError>,
hmp: impl Provider<Ref = OrChunkId<HermesRef>, Set = ParsedHermesMap, Err = ResolveError>,
) -> Self {
Self { smp: Box::new(smp) }
Self {
smp: Box::new(smp),
hmp: Box::new(hmp),
}
}
}
#[async_trait]
impl SymbolCatalog<Url, OwnedSourceMapCache> for Catalog {
async fn lookup(&self, team_id: i32, r: Url) -> Result<Arc<OwnedSourceMapCache>, Error> {
async fn lookup(&self, team_id: i32, r: Url) -> Result<Arc<OwnedSourceMapCache>, ResolveError> {
let r = OrChunkId::inner(r);
self.smp.lookup(team_id, r).await
}
@@ -80,11 +90,22 @@ impl SymbolCatalog<OrChunkId<Url>, OwnedSourceMapCache> for Catalog {
&self,
team_id: i32,
r: OrChunkId<Url>,
) -> Result<Arc<OwnedSourceMapCache>, Error> {
) -> Result<Arc<OwnedSourceMapCache>, ResolveError> {
self.smp.lookup(team_id, r).await
}
}
#[async_trait]
impl SymbolCatalog<OrChunkId<HermesRef>, ParsedHermesMap> for Catalog {
async fn lookup(
&self,
team_id: i32,
r: OrChunkId<HermesRef>,
) -> Result<Arc<ParsedHermesMap>, ResolveError> {
self.hmp.lookup(team_id, r).await
}
}
#[async_trait]
impl<T> Provider for T
where

16
rust/cymbal/src/symbol_store/saving.rs
View File

@@ -9,7 +9,7 @@ use tracing::{error, info};
use uuid::Uuid;
use crate::{
error::{Error, FrameError, UnhandledError},
error::{FrameError, ResolveError, UnhandledError},
metric_consts::{
FRAME_RESOLUTION_RESULTS_DELETED, SAVED_SYMBOL_SET_ERROR_RETURNED, SAVED_SYMBOL_SET_LOADED,
SAVE_SYMBOL_SET, SYMBOL_SET_DB_FETCHES, SYMBOL_SET_DB_HITS, SYMBOL_SET_DB_MISSES,
@@ -153,7 +153,7 @@ impl<F> Saving<F> {
#[async_trait]
impl<F> Fetcher for Saving<F>
where
F: Fetcher<Fetched = Vec<u8>, Err = Error>,
F: Fetcher<Fetched = Vec<u8>, Err = ResolveError>,
F::Ref: ToString + Send,
{
type Ref = F::Ref;
@@ -200,7 +200,7 @@ where
// case, there is no saved data).
let error = serde_json::from_str(&record.failure_reason.unwrap())
.map_err(UnhandledError::from)?;
return Err(Error::ResolutionError(error));
return Err(ResolveError::ResolutionError(error));
}
info!("Found stale symbol set error for {}", set_ref);
// We last tried to get the symbol set more than a day ago, so we should try again
@@ -220,10 +220,10 @@ where
set_ref,
})
}
Err(Error::ResolutionError(e)) => {
Err(ResolveError::ResolutionError(e)) => {
// But if we failed to get any data, we save that fact
self.save_no_data(team_id, set_ref, &e).await?;
return Err(Error::ResolutionError(e));
return Err(ResolveError::ResolutionError(e));
}
Err(e) => Err(e), // If some non-resolution error occurred, we just bail out
}
@@ -233,7 +233,7 @@ where
#[async_trait]
impl<F> Parser for Saving<F>
where
F: Parser<Source = Vec<u8>, Err = Error>,
F: Parser<Source = Vec<u8>, Err = ResolveError>,
F::Set: Send,
{
type Source = Saveable;
@@ -251,11 +251,11 @@ where
}
return Ok(s);
}
Err(Error::ResolutionError(e)) => {
Err(ResolveError::ResolutionError(e)) => {
info!("Failed to parse symbol set data for {}", data.set_ref);
// We save the no-data case here, to prevent us from fetching again for day
self.save_no_data(data.team_id, data.set_ref, &e).await?;
return Err(Error::ResolutionError(e));
return Err(ResolveError::ResolutionError(e));
}
Err(e) => return Err(e),
}

18
rust/cymbal/src/symbol_store/sourcemap.rs
View File

@@ -9,7 +9,7 @@ use tracing::{info, warn};
use crate::{
config::Config,
error::{Error, JsResolveErr},
error::{JsResolveErr, ResolveError},
metric_consts::{
SOURCEMAP_BODY_FETCHES, SOURCEMAP_BODY_REF_FOUND, SOURCEMAP_FETCH, SOURCEMAP_HEADER_FOUND,
SOURCEMAP_NOT_FOUND, SOURCEMAP_PARSE,
@@ -89,7 +89,7 @@ impl From<Url> for SourceMappingUrl {
impl Fetcher for SourcemapProvider {
type Ref = Url;
type Fetched = Vec<u8>;
type Err = Error;
type Err = ResolveError;
async fn fetch(&self, _: i32, r: Url) -> Result<Vec<u8>, Self::Err> {
let start = common_metrics::timing_guard(SOURCEMAP_FETCH, &[]);
let (smu, minified_source) = find_sourcemap_url(&self.client, r).await?;
@@ -122,7 +122,7 @@ impl Fetcher for SourcemapProvider {
impl Parser for SourcemapProvider {
type Source = Vec<u8>;
type Set = OwnedSourceMapCache;
type Err = Error;
type Err = ResolveError;
async fn parse(&self, data: Vec<u8>) -> Result<Self::Set, Self::Err> {
let start = common_metrics::timing_guard(SOURCEMAP_PARSE, &[]);
let sam: SourceAndMap = read_symbol_data(data).map_err(JsResolveErr::JSDataError)?;
@@ -137,7 +137,7 @@ impl Parser for SourcemapProvider {
async fn find_sourcemap_url(
client: &reqwest::Client,
start: Url,
) -> Result<(SourceMappingUrl, String), Error> {
) -> Result<(SourceMappingUrl, String), ResolveError> {
info!("Fetching script source from {}", start);
// If this request fails, we cannot resolve the frame, and hand this error to the frames
@@ -244,7 +244,7 @@ async fn find_sourcemap_url(
Err(JsResolveErr::NoSourcemap(final_url.to_string()).into())
}
async fn fetch_source_map(client: &reqwest::Client, url: Url) -> Result<String, Error> {
async fn fetch_source_map(client: &reqwest::Client, url: Url) -> Result<String, ResolveError> {
metrics::counter!(SOURCEMAP_BODY_FETCHES).increment(1);
let res = client.get(url).send().await.map_err(JsResolveErr::from)?;
res.error_for_status_ref().map_err(JsResolveErr::from)?;
@@ -264,8 +264,8 @@ struct DataUrlContent {
fn maybe_as_data_url<T>(
source_url: &str,
data: &str,
parse_fn: impl FnOnce(DataUrlContent) -> Result<T, Error>,
) -> Result<Option<T>, Error> {
parse_fn: impl FnOnce(DataUrlContent) -> Result<T, ResolveError>,
) -> Result<Option<T>, ResolveError> {
if !data.starts_with(DATA_SCHEME) {
return Ok(None);
}
@@ -343,7 +343,7 @@ fn maybe_as_data_url<T>(
Ok(Some(parse_fn(content)?))
}
fn data_url_to_json_str(content: DataUrlContent) -> Result<String, Error> {
fn data_url_to_json_str(content: DataUrlContent) -> Result<String, ResolveError> {
if !content.mime_type.starts_with("application/json") {
return Err(JsResolveErr::InvalidDataUrl(
"data".into(),
@@ -362,7 +362,7 @@ fn data_url_to_json_str(content: DataUrlContent) -> Result<String, Error> {
Ok(data.to_string())
}
fn assert_is_sourcemap(data: &str) -> Result<(), Error> {
fn assert_is_sourcemap(data: &str) -> Result<(), ResolveError> {
if let Err(e) = sourcemap::decode_slice(data.as_bytes()) {
return Err(JsResolveErr::InvalidSourceMap(e.to_string()).into());
}

6
rust/cymbal/src/types/mod.rs
View File

@@ -10,7 +10,7 @@ use crate::fingerprinting::{
Fingerprint, FingerprintBuilder, FingerprintComponent, FingerprintRecordPart,
};
use crate::frames::releases::{ReleaseInfo, ReleaseRecord};
use crate::frames::{Frame, RawFrame};
use crate::frames::{Frame, FrameId, RawFrame};
#[derive(Debug, Deserialize, Serialize, Clone)]
pub struct Mechanism {
@@ -320,14 +320,14 @@ impl OutputErrProps {
}
impl Stacktrace {
pub fn resolve(&self, lookup_table: &HashMap<String, Frame>) -> Option<Self> {
pub fn resolve(&self, team_id: i32, lookup_table: &HashMap<FrameId, Frame>) -> Option<Self> {
let Stacktrace::Raw { frames } = self else {
return Some(self.clone());
};
let mut resolved_frames = Vec::with_capacity(frames.len());
for frame in frames {
match lookup_table.get(&frame.frame_id()) {
match lookup_table.get(&frame.frame_id(team_id)) {
Some(resolved_frame) => resolved_frames.push(resolved_frame.clone()),
None => return None,
}

6
rust/cymbal/tests/resolve.rs
View File

@@ -9,6 +9,7 @@ use cymbal::{
symbol_store::{
caching::{Caching, SymbolSetCache},
chunk_id::OrChunkId,
hermesmap::HermesMapProvider,
sourcemap::{OwnedSourceMapCache, SourcemapProvider},
Catalog, Fetcher, Parser,
},
@@ -116,8 +117,11 @@ async fn end_to_end_resolver_test() {
)));
let wrapped = NoOpChunkIdFetcher { inner: sourcemap };
let hmp = NoOpChunkIdFetcher {
inner: HermesMapProvider {},
};
let catalog = Catalog::new(Caching::new(wrapped, cache));
let catalog = Catalog::new(Caching::new(wrapped, cache), hmp);
let mut resolved_frames = Vec::new();
for frame in test_stack {

1
rust/cymbal/tests/static/hermes/composed_example.map Normal file
View File

File diff suppressed because one or more lines are too long

8
rust/cymbal/tests/static/hermes/final_stack.txt Normal file
View File

@@ -0,0 +1,8 @@
Uncaught Error: kaboom from Hermes
at c (address at /Users/olly/Documents/work/test-rn/HermesMapsDemo/mini-entry.js:1:c)
at b (address at /Users/olly/Documents/work/test-rn/HermesMapsDemo/mini-entry.js:2:b)
at anonymous (address at /Users/olly/Documents/work/test-rn/HermesMapsDemo/mini-entry.js:3:a)
at loadModuleImplementation (address at /Users/olly/Documents/work/test-rn/HermesMapsDemo/node_modules/metro-runtime/src/polyfills/require.js:285:loadModuleImplementation)
at guardedLoadModule (address at /Users/olly/Documents/work/test-rn/HermesMapsDemo/node_modules/metro-runtime/src/polyfills/require.js:183:guardedLoadModule)
at metroRequire (address at /Users/olly/Documents/work/test-rn/HermesMapsDemo/node_modules/metro-runtime/src/polyfills/require.js:98:metroRequire)
at global (address at null:null:null)

1
rust/cymbal/tests/static/hermes/hermes_example.map Normal file
View File

File diff suppressed because one or more lines are too long

1
rust/cymbal/tests/static/hermes/metro_example.map Normal file
View File

File diff suppressed because one or more lines are too long

8
rust/cymbal/tests/static/hermes/raw_stack.txt Normal file
View File

@@ -0,0 +1,8 @@
Uncaught Error: kaboom from Hermes
at c (address at ./build/android/index.android.bundle.hbc:1:8258)
at b (address at ./build/android/index.android.bundle.hbc:1:8277)
at anonymous (address at ./build/android/index.android.bundle.hbc:1:8228)
at loadModuleImplementation (address at ./build/android/index.android.bundle.hbc:1:1755)
at guardedLoadModule (address at ./build/android/index.android.bundle.hbc:1:1249)
at metroRequire (address at ./build/android/index.android.bundle.hbc:1:894)
at global (address at ./build/android/index.android.bundle.hbc:1:425)