yindo/posthog
1
0
Fork 0
mirror of https://github.com/BillyOutlast/posthog.git synced 2026-02-04 03:01:23 +01:00
Code Issues Packages Projects Releases Wiki Activity

feat(cli): hermes support (#39998)

Browse Source
This commit is contained in:
Oliver Browne
2025-10-24 01:06:38 +03:00
committed by GitHub
parent 8bcf1ff059
commit ae39132bc1
19 changed files with 723 additions and 360 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
cli/CHANGELOG.md
View File

@@ -1,5 +1,9 @@
# posthog-cli
# 0.5.6
- Adding experimental support for hermes sourcemaps
# 0.5.5
- When running inject command multiple times, we only update chunk ids when releases are different

14
cli/Cargo.lock generated
View File

@@ -762,9 +762,9 @@ checksum = "e629b9b98ef3dd8afe6ca2bd0f89306cec16d43d907889945bc5d6687f2f13c7"
[[package]]
name = "globset"
version = "0.4.17"
version = "0.4.18"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "eab69130804d941f8075cfd713bf8848a2c3b3f201a9457a11e6f87e1ab62305"
checksum = "52dfc19153a48bde0cbd630453615c8151bce3a5adfac7a0aebfbf0a1e1f57e3"
dependencies = [
"aho-corasick",
"bstr",
@@ -1520,7 +1520,7 @@ checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184"
[[package]]
name = "posthog-cli"
version = "0.5.5"
version = "0.5.6"
dependencies = [
"anyhow",
"chrono",
@@ -1595,9 +1595,9 @@ dependencies = [
[[package]]
name = "proc-macro2"
version = "1.0.101"
version = "1.0.102"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "89ae43fd86e4158d6db51ad8e2b80f313af9cc74f5c0e03ccb87de09998732de"
checksum = "8e0f6df8eaa422d97d72edcd152e1451618fed47fabbdbd5a8864167b1d4aff7"
dependencies = [
"unicode-ident",
]
@@ -2271,9 +2271,9 @@ checksum = "b7401a30af6cb5818bb64852270bb722533397edcfc7344954a38f420819ece2"
[[package]]
name = "syn"
version = "2.0.107"
version = "2.0.108"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "2a26dbd934e5451d21ef060c018dae56fc073894c5a7896f882928a76e6d081b"
checksum = "da58917d35242480a05c2897064da0a80589a2a0476c9a3f2fdc83b53502e917"
dependencies = [
"proc-macro2",
"quote",

2
cli/Cargo.toml
View File

@@ -1,6 +1,6 @@
[package]
name = "posthog-cli"
version = "0.5.5"
version = "0.5.6"
authors = [
"David <david@posthog.com>",
"Olly <oliver@posthog.com>",

5
cli/src/api/symbol_sets.rs
View File

@@ -152,10 +152,7 @@ fn upload_to_s3(presigned_url: PresignedUrl, data: &[u8]) -> Result<()> {
}
}
if attempt < 3 {
warn!(
"Upload attempt {} failed, retrying in {:?}...",
attempt, delay
);
warn!("Upload attempt {attempt} failed, retrying in {delay:?}...",);
std::thread::sleep(delay);
delay *= 2;
}

40
cli/src/commands.rs
View File

@@ -5,7 +5,7 @@ use crate::{
error::CapturedError,
experimental::{query::command::QueryCommand, tasks::TaskCommand},
invocation_context::{context, init_context},
sourcemaps::SourcemapCommand,
sourcemaps::{hermes::HermesSubcommand, plain::SourcemapCommand},
};
#[derive(Parser)]
@@ -18,6 +18,8 @@ pub struct Cli {
/// Disable non-zero exit codes on errors. Use with caution.
#[arg(long, default_value = "false")]
no_fail: bool,
#[arg(long, default_value = "false")]
skip_ssl_verification: bool,
#[command(subcommand)]
command: Commands,
@@ -60,6 +62,12 @@ pub enum ExpCommand {
#[command(subcommand)]
cmd: QueryCommand,
},
#[command(about = "Upload hermes sourcemaps to PostHog")]
Hermes {
#[command(subcommand)]
cmd: HermesSubcommand,
},
}
impl Cli {
@@ -85,6 +93,10 @@ impl Cli {
}
fn run_impl(self) -> Result<(), CapturedError> {
if !matches!(self.command, Commands::Login) {
init_context(self.host.clone(), self.skip_ssl_verification)?;
}
match self.command {
Commands::Login => {
// Notably login doesn't have a context set up going it - it sets one up
@@ -92,32 +104,38 @@ impl Cli {
}
Commands::Sourcemap { cmd } => match cmd {
SourcemapCommand::Inject(input_args) => {
init_context(self.host.clone(), false)?;
crate::sourcemaps::inject::inject(&input_args)?;
crate::sourcemaps::plain::inject::inject(&input_args)?;
}
SourcemapCommand::Upload(upload_args) => {
init_context(self.host.clone(), upload_args.skip_ssl_verification)?;
crate::sourcemaps::upload::upload_cmd(upload_args.clone())?;
crate::sourcemaps::plain::upload::upload(&upload_args)?;
}
SourcemapCommand::Process(args) => {
init_context(self.host.clone(), args.skip_ssl_verification)?;
let (inject, upload) = args.into();
crate::sourcemaps::inject::inject(&inject)?;
crate::sourcemaps::upload::upload_cmd(upload)?;
crate::sourcemaps::plain::inject::inject(&inject)?;
crate::sourcemaps::plain::upload::upload(&upload)?;
}
},
Commands::Exp { cmd } => match cmd {
ExpCommand::Task {
cmd,
skip_ssl_verification,
skip_ssl_verification: _,
} => {
init_context(self.host.clone(), skip_ssl_verification)?;
cmd.run()?;
}
ExpCommand::Query { cmd } => {
init_context(self.host.clone(), false)?;
crate::experimental::query::command::query_command(&cmd)?
}
ExpCommand::Hermes { cmd } => match cmd {
HermesSubcommand::Inject(args) => {
crate::sourcemaps::hermes::inject::inject(&args)?;
}
HermesSubcommand::Upload(args) => {
crate::sourcemaps::hermes::upload::upload(&args)?;
}
HermesSubcommand::Clone(args) => {
crate::sourcemaps::hermes::clone::clone(&args)?;
}
},
},
}

5
cli/src/main.rs
View File

@@ -6,8 +6,9 @@ fn main() {
let subscriber = tracing_subscriber::fmt()
.with_writer(std::io::stderr)
.with_env_filter(
tracing_subscriber::EnvFilter::from_default_env()
.add_directive(tracing::Level::INFO.into()),
tracing_subscriber::EnvFilter::builder()
.with_default_directive(tracing::Level::INFO.into())
.from_env_lossy(),
)
.finish();

235
cli/src/sourcemaps/source_pair.rs → cli/src/sourcemaps/content.rs
View File

@@ -1,19 +1,17 @@
use anyhow::{anyhow, bail, Result};
use magic_string::{GenerateDecodedMapOptions, MagicString};
use posthog_symbol_data::{write_symbol_data, HermesMap};
use serde::{Deserialize, Serialize};
use serde_json::Value;
use sourcemap::SourceMap;
use std::{collections::BTreeMap, path::PathBuf};
use tracing::info;
use crate::{
api::symbol_sets::SymbolSetUpload,
sourcemaps::constant::{CHUNKID_COMMENT_PREFIX, CHUNKID_PLACEHOLDER, CODE_SNIPPET_TEMPLATE},
utils::files::{is_javascript_file, SourceFile},
utils::files::SourceFile,
};
use anyhow::{anyhow, bail, Context, Result};
use globset::{Glob, GlobSetBuilder};
use magic_string::{GenerateDecodedMapOptions, MagicString};
use posthog_symbol_data::{write_symbol_data, SourceAndMap};
use serde::{Deserialize, Serialize};
use serde_json::Value;
use sourcemap::SourceMap;
use tracing::{info, warn};
use walkdir::WalkDir;
#[derive(Debug, Serialize, Deserialize, PartialEq, Eq)]
pub struct SourceMapContent {
@@ -33,154 +31,6 @@ pub struct MinifiedSourceFile {
pub inner: SourceFile<String>,
}
// Source pairs are the fundamental unit of a frontend symbol set
pub struct SourcePair {
pub source: MinifiedSourceFile,
pub sourcemap: SourceMapFile,
}
impl SourcePair {
pub fn has_chunk_id(&self) -> bool {
// Minified chunks are the source of truth for their ID's, not sourcemaps,
// because sometimes sourcemaps are shared across multiple chunks.
self.get_chunk_id().is_some()
}
pub fn get_chunk_id(&self) -> Option<String> {
self.source.get_chunk_id()
}
pub fn has_release_id(&self) -> bool {
self.get_release_id().is_some()
}
pub fn remove_chunk_id(&mut self, chunk_id: String) -> Result<()> {
if self.get_chunk_id().as_ref() != Some(&chunk_id) {
return Err(anyhow!("Chunk ID mismatch"));
}
let adjustment = self.source.remove_chunk_id(chunk_id)?;
self.sourcemap.apply_adjustment(adjustment)?;
self.sourcemap.set_chunk_id(None);
Ok(())
}
pub fn update_chunk_id(
&mut self,
previous_chunk_id: String,
new_chunk_id: String,
) -> Result<()> {
self.remove_chunk_id(previous_chunk_id)?;
self.add_chunk_id(new_chunk_id)?;
Ok(())
}
pub fn add_chunk_id(&mut self, chunk_id: String) -> Result<()> {
if self.has_chunk_id() {
return Err(anyhow!("Chunk ID already set"));
}
let adjustment = self.source.set_chunk_id(&chunk_id)?;
// In cases where sourcemaps are shared across multiple chunks,
// we should only apply the adjustment if the sourcemap doesn't
// have a chunk ID set (since otherwise, it's already been adjusted)
if self.sourcemap.get_chunk_id().is_none() {
self.sourcemap.apply_adjustment(adjustment)?;
self.sourcemap.set_chunk_id(Some(chunk_id));
}
Ok(())
}
pub fn set_release_id(&mut self, release_id: Option<String>) {
self.sourcemap.set_release_id(release_id);
}
pub fn save(&self) -> Result<()> {
self.source.save()?;
self.sourcemap.save()?;
Ok(())
}
pub(crate) fn get_release_id(&self) -> Option<String> {
self.sourcemap.get_release_id()
}
}
pub fn read_pairs(
directory: &PathBuf,
ignore_globs: &[String],
strip_prefix: &Option<String>,
) -> Result<Vec<SourcePair>> {
// Make sure the directory exists
if !directory.exists() {
bail!("Directory does not exist");
}
let mut builder = GlobSetBuilder::new();
for glob in ignore_globs {
builder.add(Glob::new(glob)?);
}
let set: globset::GlobSet = builder.build()?;
let mut pairs = Vec::new();
for entry_path in WalkDir::new(directory)
.into_iter()
.filter_map(|e| e.ok())
.filter(is_javascript_file)
.map(|e| e.path().canonicalize())
{
let entry_path = entry_path?;
if set.is_match(&entry_path) {
info!(
"Skipping because it matches an ignored glob: {}",
entry_path.display()
);
continue;
}
info!("Processing file: {}", entry_path.display());
let source = MinifiedSourceFile::load(&entry_path)?;
let sourcemap_path = source.get_sourcemap_path(strip_prefix)?;
let Some(path) = sourcemap_path else {
warn!(
"No sourcemap file found for file {}, skipping",
entry_path.display()
);
continue;
};
let sourcemap = SourceMapFile::load(&path).context(format!("reading {path:?}"))?;
pairs.push(SourcePair { source, sourcemap });
}
Ok(pairs)
}
impl TryInto<SymbolSetUpload> for SourcePair {
type Error = anyhow::Error;
fn try_into(self) -> Result<SymbolSetUpload> {
let chunk_id = self
.get_chunk_id()
.ok_or_else(|| anyhow!("Chunk ID not found"))?;
let source_content = self.source.inner.content;
let sourcemap_content = serde_json::to_string(&self.sourcemap.inner.content)?;
let data = SourceAndMap {
minified_source: source_content,
sourcemap: sourcemap_content,
};
let data = write_symbol_data(data)?;
Ok(SymbolSetUpload {
chunk_id,
data,
release_id: self.sourcemap.get_release_id(),
})
}
}
impl SourceMapFile {
pub fn load(path: &PathBuf) -> Result<Self> {
let inner = SourceFile::load(path)?;
@@ -203,26 +53,38 @@ impl SourceMapFile {
pub fn apply_adjustment(&mut self, adjustment: SourceMap) -> Result<()> {
let new_content = {
let content = serde_json::to_string(&self.inner.content)?.into_bytes();
let mut original_sourcemap = match sourcemap::decode_slice(content.as_slice())
.map_err(|err| anyhow!("Failed to parse sourcemap: {err}"))?
{
sourcemap::DecodedMap::Regular(map) => map,
sourcemap::DecodedMap::Index(index_map) => index_map
let mut map = sourcemap::decode_slice(content.as_slice())
.map_err(|err| anyhow!("Failed to parse sourcemap: {err}"))?;
// This looks weird. The reason we do it, is that we want `original` below
// to be a &mut SourceMap. This is easy to do if it's a Regular, or Hermes
// map, but if it's an Index map (Regular is already a SourceMap, so just
// taking the &mut works, and Hermes maps impl DerefMut<Target = SourceMap>),
// but for index maps, we have to flatten first, and that necessitates a Clone.
// Doing that Clone in the match below and then trying to borrow a &mut to the
// result of the Clone causes us to try and borrow something we immediately drop,
// (the clone is done in the match arm scope, and then a ref to a local in that
// scope is returned to the outer scope), so instead, we do the clone here if
// we need to, and declare the index branch unreachable below.
if let sourcemap::DecodedMap::Index(indexed) = &mut map {
let replacement = indexed
.flatten()
.map_err(|err| anyhow!("Failed to parse sourcemap: {err}"))?,
sourcemap::DecodedMap::Hermes(_) => {
// TODO(olly) - YES THEY ARE!!!!! WOOOOOOO!!!!! YIPEEEEEEEE!!!
anyhow::bail!("Hermes source maps are not supported")
}
.map_err(|err| anyhow!("Failed to flatten sourcemap: {err}"))?;
map = sourcemap::DecodedMap::Regular(replacement);
};
original_sourcemap.adjust_mappings(&adjustment);
let original = match &mut map {
sourcemap::DecodedMap::Regular(m) => m,
sourcemap::DecodedMap::Hermes(m) => m,
sourcemap::DecodedMap::Index(_) => unreachable!(),
};
original.adjust_mappings(&adjustment);
// I mean if we've got the bytes allocated already, why not use 'em
let mut content = content;
content.clear();
original_sourcemap.to_writer(&mut content)?;
original.to_writer(&mut content)?;
serde_json::from_slice(&content)?
};
@@ -368,7 +230,7 @@ impl MinifiedSourceFile {
None
}
fn remove_chunk_id(&mut self, chunk_id: String) -> Result<SourceMap> {
pub fn remove_chunk_id(&mut self, chunk_id: String) -> Result<SourceMap> {
let (new_source_content, source_adjustment) = {
// Update source content with chunk ID
let source_content = &self.inner.content;
@@ -412,3 +274,30 @@ impl MinifiedSourceFile {
Ok(source_adjustment)
}
}
impl TryInto<SymbolSetUpload> for SourceMapFile {
type Error = anyhow::Error;
fn try_into(self) -> Result<SymbolSetUpload> {
let chunk_id = self
.get_chunk_id()
.ok_or_else(|| anyhow!("Chunk ID not found"))?;
let release_id = self.get_release_id();
let sourcemap = self.inner.content;
let content = serde_json::to_string(&sourcemap)?;
if !sourcemap.fields.contains_key("x_hermes_function_offsets") {
bail!("Map is not a hermes sourcemap - missing key x_hermes_function_offsets");
}
let data = HermesMap { sourcemap: content };
let data = write_symbol_data(data)?;
Ok(SymbolSetUpload {
chunk_id,
release_id,
data,
})
}
}

119
cli/src/sourcemaps/hermes/clone.rs Normal file
View File

@@ -0,0 +1,119 @@
use std::path::PathBuf;
use anyhow::{anyhow, bail, Result};
use tracing::{info, warn};
use crate::{
invocation_context::context,
sourcemaps::{
content::SourceMapFile,
hermes::{get_composed_map, inject::is_metro_bundle},
inject::get_release_for_pairs,
source_pairs::read_pairs,
},
};
#[derive(clap::Args)]
pub struct CloneArgs {
/// The directory containing the bundled chunks
#[arg(short, long)]
pub directory: PathBuf,
/// One or more directory glob patterns to ignore
#[arg(short, long)]
pub ignore: Vec<String>,
/// The project name associated with the uploaded chunks. Required to have the uploaded chunks associated with
/// a specific release. We will try to auto-derive this from git information if not provided. Strongly recommended
/// to be set explicitly during release CD workflows. Only necessary if no project was provided during injection.
#[arg(long)]
pub project: Option<String>,
/// The version of the project - this can be a version number, semantic version, or a git commit hash. Required
/// to have the uploaded chunks associated with a specific release.
#[arg(long)]
pub version: Option<String>,
}
pub fn clone(args: &CloneArgs) -> Result<()> {
context().capture_command_invoked("hermes_clone");
let CloneArgs {
directory,
ignore,
project,
version,
} = args;
let directory = directory.canonicalize().map_err(|e| {
anyhow!(
"Directory '{}' not found or inaccessible: {}",
directory.display(),
e
)
})?;
info!("Processing directory: {}", directory.display());
let pairs = read_pairs(&directory, ignore, is_metro_bundle, &None)?;
if pairs.is_empty() {
bail!("No source files found");
}
info!("Found {} pairs", pairs.len());
let release_id =
get_release_for_pairs(&directory, project, version, &pairs)?.map(|r| r.id.to_string());
// The flow here differs from plain sourcemap injection a bit - here, we don't ever
// overwrite the chunk ID, because at this point in the build process, we no longer
// control what chunk ID is inside the compiled hermes byte code bundle. So, instead,
// we permit e.g. uploading the same chunk ID's to two different posthog envs with two
// different release ID's, or arbitrarily re-running the upload command, but if someone
// tries to run `clone` twice, changing release but not posthog env, we'll error out. The
// correct way to upload the same set of artefacts to the same posthog env as part of
// two different releases is, 1, not to, but failing that, 2, to re-run the bundling process
let mut pairs = pairs;
for pair in &mut pairs {
if !pair.has_release_id() || pair.get_release_id() != release_id {
pair.set_release_id(release_id.clone());
pair.save()?;
}
}
let pairs = pairs;
let maps: Result<Vec<(&SourceMapFile, Option<SourceMapFile>)>> = pairs
.iter()
.map(|p| get_composed_map(p).map(|c| (&p.sourcemap, c)))
.collect();
let maps = maps?;
for (minified, composed) in maps {
let Some(mut composed) = composed else {
warn!(
"Could not find composed map for minified sourcemap {}",
minified.inner.path.display()
);
continue;
};
// Copy metadata from source map to composed map
if let Some(chunk_id) = minified.get_chunk_id() {
composed.set_chunk_id(Some(chunk_id));
}
if let Some(release_id) = minified.get_release_id() {
composed.set_release_id(Some(release_id));
}
composed.save()?;
info!(
"Successfully cloned metadata to {}",
composed.inner.path.display()
);
}
info!("Finished cloning metadata");
Ok(())
}

24
cli/src/sourcemaps/hermes/inject.rs Normal file
View File

@@ -0,0 +1,24 @@
// Inject-minified is identical to injecting web-facing bundles, just with slightly different search parameters
// It's intended as an escape hatch for people rolling their own build pipeline - we expect most users to be
// using the metro plugin for injecting, and then calling clone
use anyhow::Result;
use walkdir::DirEntry;
use crate::{
invocation_context::context,
sourcemaps::inject::{inject_impl, InjectArgs},
};
pub fn inject(args: &InjectArgs) -> Result<()> {
context().capture_command_invoked("hermes_inject");
inject_impl(args, is_metro_bundle)
}
pub fn is_metro_bundle(entry: &DirEntry) -> bool {
entry.file_type().is_file()
&& entry
.path()
.extension()
.is_some_and(|ext| ext == "bundle" || ext == "jsbundle")
}

56
cli/src/sourcemaps/hermes/mod.rs Normal file
View File

@@ -0,0 +1,56 @@
use std::path::PathBuf;
use anyhow::{anyhow, Context, Result};
use clap::Subcommand;
use tracing::info;
use crate::sourcemaps::{content::SourceMapFile, inject::InjectArgs, source_pairs::SourcePair};
pub mod clone;
pub mod inject;
pub mod upload;
#[derive(Subcommand)]
pub enum HermesSubcommand {
/// Inject your bundled chunk with a posthog chunk ID
Inject(InjectArgs),
/// Upload the bundled chunk to PostHog
Upload(upload::Args),
/// Clone chunk_id and release_id metadata from bundle maps to composed maps
Clone(clone::CloneArgs),
}
pub fn get_composed_map(pair: &SourcePair) -> Result<Option<SourceMapFile>> {
let sourcemap_path = &pair.sourcemap.inner.path;
// Look for composed map: change .bundle.map to .bundle.hbc.composed.map
let composed_path = sourcemap_path
.to_str()
.and_then(|s| {
if s.ends_with(".bundle.map") {
Some(PathBuf::from(
s.replace(".bundle.map", ".bundle.hbc.composed.map"),
))
} else if s.ends_with(".jsbundle.map") {
Some(PathBuf::from(
s.replace(".jsbundle.map", ".jsbundle.hbc.composed.map"),
))
} else {
None
}
})
.ok_or_else(|| anyhow!("Could not determine composed map path for {sourcemap_path:?}"))?;
if !composed_path.exists() {
info!(
"Skipping {} - no composed map found at {}",
sourcemap_path.display(),
composed_path.display()
);
return Ok(None);
}
Ok(Some(SourceMapFile::load(&composed_path).context(
format!("reading composed map at {composed_path:?}"),
)?))
}

61
cli/src/sourcemaps/hermes/upload.rs Normal file
View File

@@ -0,0 +1,61 @@
use std::path::PathBuf;
use anyhow::{anyhow, Ok, Result};
use tracing::{info, warn};
use crate::api::symbol_sets::{self, SymbolSetUpload};
use crate::invocation_context::context;
use crate::sourcemaps::hermes::get_composed_map;
use crate::sourcemaps::hermes::inject::is_metro_bundle;
use crate::sourcemaps::source_pairs::read_pairs;
#[derive(clap::Args, Clone)]
pub struct Args {
/// The directory containing the bundled chunks
#[arg(short, long)]
pub directory: PathBuf,
/// One or more directory glob patterns to ignore
#[arg(short, long)]
pub ignore: Vec<String>,
}
pub fn upload(args: &Args) -> Result<()> {
context().capture_command_invoked("hermes_upload");
let Args { directory, ignore } = args;
let directory = directory.canonicalize().map_err(|e| {
anyhow!(
"Directory '{}' not found or inaccessible: {}",
directory.display(),
e
)
})?;
info!("Processing directory: {}", directory.display());
let pairs = read_pairs(&directory, ignore, is_metro_bundle, &None)?;
let maps: Result<Vec<_>> = pairs.iter().map(get_composed_map).collect();
let maps = maps?;
let mut uploads: Vec<SymbolSetUpload> = Vec::new();
for map in maps.into_iter() {
let Some(map) = map else {
continue;
};
if map.get_chunk_id().is_none() {
warn!("Skipping map {}, no chunk ID", map.inner.path.display());
continue;
}
uploads.push(map.try_into()?);
}
info!("Found {} bundles to upload", uploads.len());
symbol_sets::upload(&uploads, 100)?;
Ok(())
}

79
cli/src/sourcemaps/inject.rs
View File

@@ -1,12 +1,11 @@
use anyhow::{anyhow, bail, Ok, Result};
use std::path::PathBuf;
use anyhow::{anyhow, bail, Result};
use std::path::{Path, PathBuf};
use tracing::info;
use uuid;
use walkdir::DirEntry;
use crate::{
api::releases::ReleaseBuilder,
invocation_context::context,
sourcemaps::source_pair::{read_pairs, SourcePair},
api::releases::{Release, ReleaseBuilder},
sourcemaps::source_pairs::{read_pairs, SourcePair},
utils::git::get_git_info,
};
@@ -33,13 +32,12 @@ pub struct InjectArgs {
pub project: Option<String>,
/// The version of the project - this can be a version number, semantic version, or a git commit hash. Required
/// to have the uploaded chunks associated with a specific release. Overrides release information set during
/// injection. Strongly prefer setting release information during injection.
/// to have the uploaded chunks associated with a specific release.
#[arg(long)]
pub version: Option<String>,
}
pub fn inject(args: &InjectArgs) -> Result<()> {
pub fn inject_impl(args: &InjectArgs, matcher: impl Fn(&DirEntry) -> bool) -> Result<()> {
let InjectArgs {
directory,
public_path_prefix,
@@ -48,8 +46,6 @@ pub fn inject(args: &InjectArgs) -> Result<()> {
version,
} = args;
context().capture_command_invoked("sourcemap_inject");
let directory = directory.canonicalize().map_err(|e| {
anyhow!(
"Directory '{}' not found or inaccessible: {}",
@@ -59,36 +55,15 @@ pub fn inject(args: &InjectArgs) -> Result<()> {
})?;
info!("Processing directory: {}", directory.display());
let mut pairs = read_pairs(&directory, ignore, public_path_prefix)?;
let mut pairs = read_pairs(&directory, ignore, matcher, public_path_prefix)?;
if pairs.is_empty() {
bail!("No source files found");
}
info!("Found {} pairs", pairs.len());
// We need to fetch or create a release if: the user specified one, any pair is missing one, or the user
// forced release overriding
let needs_release =
project.is_some() || version.is_some() || pairs.iter().any(|p| !p.has_release_id());
let mut created_release = None;
if needs_release {
let mut builder = get_git_info(Some(directory))?
.map(ReleaseBuilder::init_from_git)
.unwrap_or_default();
if let Some(project) = project {
builder.with_project(project);
}
if let Some(version) = version {
builder.with_version(version);
}
if builder.can_create() {
created_release = Some(builder.fetch_or_create()?);
}
}
let created_release_id = created_release.as_ref().map(|r| r.id.to_string());
let created_release_id = get_release_for_pairs(&directory, project, version, &pairs)?
.as_ref()
.map(|r| r.id.to_string());
pairs = inject_pairs(pairs, created_release_id)?;
@@ -121,3 +96,35 @@ pub fn inject_pairs(
Ok(pairs)
}
pub fn get_release_for_pairs<'a>(
directory: &Path,
project: &Option<String>,
version: &Option<String>,
pairs: impl IntoIterator<Item = &'a SourcePair>,
) -> Result<Option<Release>> {
// We need to fetch or create a release if: the user specified one, any pair is missing one, or the user
// forced release overriding
let needs_release =
project.is_some() || version.is_some() || pairs.into_iter().any(|p| !p.has_release_id());
let mut created_release = None;
if needs_release {
let mut builder = get_git_info(Some(directory.to_path_buf()))?
.map(ReleaseBuilder::init_from_git)
.unwrap_or_default();
if let Some(project) = project {
builder.with_project(project);
}
if let Some(version) = version {
builder.with_version(version);
}
if builder.can_create() {
created_release = Some(builder.fetch_or_create()?);
}
}
Ok(created_release)
}

90
cli/src/sourcemaps/mod.rs
View File

@@ -1,88 +1,6 @@
use clap::Subcommand;
use core::str;
use std::path::PathBuf;
pub mod constant;
pub mod content;
pub mod hermes;
pub mod inject;
pub mod source_pair;
pub mod upload;
use crate::sourcemaps::inject::InjectArgs;
use crate::sourcemaps::upload::UploadArgs;
#[derive(clap::Args)]
pub struct ProcessArgs {
/// The directory containing the bundled chunks
#[arg(short, long)]
pub directory: PathBuf,
/// If your bundler adds a public path prefix to sourcemap URLs,
/// we need to ignore it while searching for them
/// For use alongside e.g. esbuilds "publicPath" config setting.
#[arg(short, long)]
pub public_path_prefix: Option<String>,
/// One or more directory glob patterns to ignore
#[arg(short, long)]
pub ignore: Vec<String>,
/// The project name associated with the uploaded chunks. Required to have the uploaded chunks associated with
/// a specific release. We will try to auto-derive this from git information if not provided. Strongly recommended
/// to be set explicitly during release CD workflows.
#[arg(long)]
pub project: Option<String>,
/// The version of the project - this can be a version number, semantic version, or a git commit hash. Required
/// to have the uploaded chunks associated with a specific release. Overrides release information set during
/// injection. Strongly prefer setting release information during injection.
#[arg(long)]
pub version: Option<String>,
/// Whether to delete the source map files after uploading them
#[arg(long, default_value = "false")]
pub delete_after: bool,
/// Whether to skip SSL verification when uploading chunks - only use when using self-signed certificates for
/// self-deployed instances
#[arg(long, default_value = "false")]
pub skip_ssl_verification: bool,
/// The maximum number of chunks to upload in a single batch
#[arg(long, default_value = "50")]
pub batch_size: usize,
}
#[derive(Subcommand)]
pub enum SourcemapCommand {
/// Inject each bundled chunk with a posthog chunk ID
Inject(InjectArgs),
/// Upload the bundled chunks to PostHog
Upload(UploadArgs),
/// Run inject and upload in one command
Process(ProcessArgs),
}
impl From<ProcessArgs> for (InjectArgs, UploadArgs) {
fn from(args: ProcessArgs) -> Self {
let inject_args = InjectArgs {
directory: args.directory.clone(),
ignore: args.ignore.clone(),
project: args.project,
version: args.version,
public_path_prefix: args.public_path_prefix.clone(),
};
let upload_args = UploadArgs {
directory: args.directory,
ignore: args.ignore,
delete_after: args.delete_after,
skip_ssl_verification: args.skip_ssl_verification,
batch_size: args.batch_size,
project: None,
version: None,
public_path_prefix: args.public_path_prefix,
};
(inject_args, upload_args)
}
}
pub mod plain;
pub mod source_pairs;

21
cli/src/sourcemaps/plain/inject.rs Normal file
View File

@@ -0,0 +1,21 @@
use anyhow::Result;
use walkdir::DirEntry;
use crate::{
invocation_context::context,
sourcemaps::inject::{inject_impl, InjectArgs},
};
pub fn inject(args: &InjectArgs) -> Result<()> {
context().capture_command_invoked("sourcemap_inject");
inject_impl(args, is_javascript_file)
}
pub fn is_javascript_file(entry: &DirEntry) -> bool {
entry.file_type().is_file()
&& entry
.path()
.extension()
.is_some_and(|ext| ext == "js" || ext == "mjs" || ext == "cjs")
}

79
cli/src/sourcemaps/plain/mod.rs Normal file
View File

@@ -0,0 +1,79 @@
use std::path::PathBuf;
use clap::Subcommand;
use crate::sourcemaps::inject::InjectArgs;
pub mod inject;
pub mod upload;
#[derive(Subcommand)]
pub enum SourcemapCommand {
/// Inject each bundled chunk with a posthog chunk ID
Inject(InjectArgs),
/// Upload the bundled chunks to PostHog
Upload(upload::Args),
/// Run inject and upload in one command
Process(ProcessArgs),
}
#[derive(clap::Args)]
pub struct ProcessArgs {
/// The directory containing the bundled chunks
#[arg(short, long)]
pub directory: PathBuf,
/// One or more directory glob patterns to ignore
#[arg(short, long)]
pub ignore: Vec<String>,
/// If your bundler adds a public path prefix to sourcemap URLs,
/// we need to ignore it while searching for them
/// For use alongside e.g. esbuilds "publicPath" config setting.
#[arg(short, long)]
pub public_path_prefix: Option<String>,
/// The project name associated with the uploaded chunks. Required to have the uploaded chunks associated with
/// a specific release. We will try to auto-derive this from git information if not provided. Strongly recommended
/// to be set explicitly during release CD workflows.
#[arg(long)]
pub project: Option<String>,
/// The version of the project - this can be a version number, semantic version, or a git commit hash. Required
/// to have the uploaded chunks associated with a specific release. Overrides release information set during
/// injection. Strongly prefer setting release information during injection.
#[arg(long)]
pub version: Option<String>,
/// Whether to delete the source map files after uploading them
#[arg(long, default_value = "false")]
pub delete_after: bool,
/// The maximum number of chunks to upload in a single batch
#[arg(long, default_value = "50")]
pub batch_size: usize,
}
impl From<ProcessArgs> for (InjectArgs, upload::Args) {
fn from(args: ProcessArgs) -> Self {
let inject_args = InjectArgs {
directory: args.directory.clone(),
ignore: args.ignore.clone(),
project: args.project,
version: args.version,
public_path_prefix: args.public_path_prefix.clone(),
};
let upload_args = upload::Args {
directory: args.directory,
public_path_prefix: args.public_path_prefix,
ignore: args.ignore,
delete_after: args.delete_after,
skip_ssl_verification: false,
batch_size: args.batch_size,
project: None,
version: None,
};
(inject_args, upload_args)
}
}

57
cli/src/sourcemaps/upload.rs → cli/src/sourcemaps/plain/upload.rs
View File

@@ -1,16 +1,17 @@
use std::path::PathBuf;
use anyhow::{Context, Ok, Result};
use anyhow::{Context, Result};
use tracing::{info, warn};
use crate::api::symbol_sets::{upload, SymbolSetUpload};
use crate::invocation_context::context;
use crate::sourcemaps::source_pair::read_pairs;
use crate::utils::files::delete_files;
use crate::{
api::symbol_sets::{self, SymbolSetUpload},
invocation_context::context,
sourcemaps::{plain::inject::is_javascript_file, source_pairs::read_pairs},
utils::files::delete_files,
};
#[derive(clap::Args, Clone)]
pub struct UploadArgs {
pub struct Args {
/// The directory containing the bundled chunks
#[arg(short, long)]
pub directory: PathBuf,
@@ -29,11 +30,6 @@ pub struct UploadArgs {
#[arg(long, default_value = "false")]
pub delete_after: bool,
/// Whether to skip SSL verification when uploading chunks - only use when using self-signed certificates for
/// self-deployed instances
#[arg(long, default_value = "false")]
pub skip_ssl_verification: bool,
/// The maximum number of chunks to upload in a single batch
#[arg(long, default_value = "50")]
pub batch_size: usize,
@@ -45,27 +41,32 @@ pub struct UploadArgs {
/// DEPRECATED: Does nothing. Set version during `inject` instead
#[arg(long)]
pub version: Option<String>,
/// DEPRECATED - use top-level `--skip-ssl-verification` instead
#[arg(long, default_value = "false")]
pub skip_ssl_verification: bool,
}
pub fn upload_cmd(args: UploadArgs) -> Result<()> {
let UploadArgs {
directory,
public_path_prefix,
ignore,
delete_after,
skip_ssl_verification: _,
batch_size,
project: p,
version: v,
} = args;
if p.is_some() || v.is_some() {
pub fn upload_cmd(args: &Args) -> Result<()> {
if args.project.is_some() || args.version.is_some() {
warn!("`--project` and `--version` are deprecated and do nothing. Set project and version during `inject` instead.");
}
context().capture_command_invoked("sourcemap_upload");
upload(args)
}
let pairs = read_pairs(&directory, &ignore, &public_path_prefix)?;
pub fn upload(args: &Args) -> Result<()> {
if args.project.is_some() || args.version.is_some() {
warn!("`--project` and `--version` are deprecated and do nothing. Set project and version during `inject` instead.");
}
let pairs = read_pairs(
&args.directory,
&args.ignore,
is_javascript_file,
&args.public_path_prefix,
)?;
let sourcemap_paths = pairs
.iter()
.map(|pair| pair.sourcemap.inner.path.clone())
@@ -78,9 +79,9 @@ pub fn upload_cmd(args: UploadArgs) -> Result<()> {
.collect::<Result<Vec<SymbolSetUpload>>>()
.context("While preparing files for upload")?;
upload(&uploads, batch_size)?;
symbol_sets::upload(&uploads, args.batch_size)?;
if delete_after {
if args.delete_after {
delete_files(sourcemap_paths).context("While deleting sourcemaps")?;
}

162
cli/src/sourcemaps/source_pairs.rs Normal file
View File

@@ -0,0 +1,162 @@
use std::path::PathBuf;
use crate::{
api::symbol_sets::SymbolSetUpload,
sourcemaps::content::{MinifiedSourceFile, SourceMapFile},
};
use anyhow::{anyhow, bail, Context, Result};
use globset::{Glob, GlobSetBuilder};
use posthog_symbol_data::{write_symbol_data, SourceAndMap};
use tracing::{info, warn};
use walkdir::{DirEntry, WalkDir};
// Source pairs are the fundamental unit of a frontend symbol set
pub struct SourcePair {
pub source: MinifiedSourceFile,
pub sourcemap: SourceMapFile,
}
impl SourcePair {
pub fn has_chunk_id(&self) -> bool {
// Minified chunks are the source of truth for their ID's, not sourcemaps,
// because sometimes sourcemaps are shared across multiple chunks.
self.get_chunk_id().is_some()
}
pub fn get_chunk_id(&self) -> Option<String> {
self.source.get_chunk_id()
}
pub fn has_release_id(&self) -> bool {
self.get_release_id().is_some()
}
pub fn get_release_id(&self) -> Option<String> {
self.sourcemap.get_release_id()
}
pub fn remove_chunk_id(&mut self, chunk_id: String) -> Result<()> {
if self.get_chunk_id().as_ref() != Some(&chunk_id) {
return Err(anyhow!("Chunk ID mismatch"));
}
let adjustment = self.source.remove_chunk_id(chunk_id)?;
self.sourcemap.apply_adjustment(adjustment)?;
self.sourcemap.set_chunk_id(None);
Ok(())
}
pub fn update_chunk_id(
&mut self,
previous_chunk_id: String,
new_chunk_id: String,
) -> Result<()> {
self.remove_chunk_id(previous_chunk_id)?;
self.add_chunk_id(new_chunk_id)?;
Ok(())
}
pub fn add_chunk_id(&mut self, chunk_id: String) -> Result<()> {
if self.has_chunk_id() {
return Err(anyhow!("Chunk ID already set"));
}
let adjustment = self.source.set_chunk_id(&chunk_id)?;
// In cases where sourcemaps are shared across multiple chunks,
// we should only apply the adjustment if the sourcemap doesn't
// have a chunk ID set (since otherwise, it's already been adjusted)
if self.sourcemap.get_chunk_id().is_none() {
self.sourcemap.apply_adjustment(adjustment)?;
self.sourcemap.set_chunk_id(Some(chunk_id));
}
Ok(())
}
pub fn set_release_id(&mut self, release_id: Option<String>) {
self.sourcemap.set_release_id(release_id);
}
pub fn save(&self) -> Result<()> {
self.source.save()?;
self.sourcemap.save()?;
Ok(())
}
}
pub fn read_pairs(
directory: &PathBuf,
ignore_globs: &[String],
matcher: impl Fn(&DirEntry) -> bool,
prefix: &Option<String>,
) -> Result<Vec<SourcePair>> {
// Make sure the directory exists
if !directory.exists() {
bail!("Directory does not exist");
}
let mut builder = GlobSetBuilder::new();
for glob in ignore_globs {
builder.add(Glob::new(glob)?);
}
let set: globset::GlobSet = builder.build()?;
let mut pairs = Vec::new();
for entry_path in WalkDir::new(directory)
.into_iter()
.filter_map(|e| e.ok())
.filter(matcher)
.map(|e| e.path().canonicalize())
{
let entry_path = entry_path?;
if set.is_match(&entry_path) {
info!(
"Skipping because it matches an ignored glob: {}",
entry_path.display()
);
continue;
}
info!("Processing file: {}", entry_path.display());
let source = MinifiedSourceFile::load(&entry_path)?;
let sourcemap_path = source.get_sourcemap_path(prefix)?;
let Some(path) = sourcemap_path else {
warn!(
"No sourcemap file found for file {}, skipping",
entry_path.display()
);
continue;
};
let sourcemap = SourceMapFile::load(&path).context(format!("reading {path:?}"))?;
pairs.push(SourcePair { source, sourcemap });
}
Ok(pairs)
}
impl TryInto<SymbolSetUpload> for SourcePair {
type Error = anyhow::Error;
fn try_into(self) -> Result<SymbolSetUpload> {
let chunk_id = self
.sourcemap
.get_chunk_id()
.ok_or_else(|| anyhow!("Chunk ID not found"))?;
let source_content = self.source.inner.content;
let sourcemap_content = serde_json::to_string(&self.sourcemap.inner.content)?;
let data = SourceAndMap {
minified_source: source_content,
sourcemap: sourcemap_content,
};
let data = write_symbol_data(data)?;
Ok(SymbolSetUpload {
chunk_id,
data,
release_id: self.sourcemap.get_release_id(),
})
}
}

11
cli/src/utils/files.rs
View File

@@ -1,9 +1,8 @@
use anyhow::{Context, Result};
use sha2::Digest;
use std::path::PathBuf;
use walkdir::DirEntry;
use crate::sourcemaps::source_pair::SourceMapContent;
use crate::sourcemaps::content::SourceMapContent;
pub struct SourceFile<T: SourceContent> {
pub path: PathBuf,
@@ -89,11 +88,3 @@ where
}
format!("{:x}", hasher.finalize())
}
pub fn is_javascript_file(entry: &DirEntry) -> bool {
entry.file_type().is_file()
&& entry
.path()
.extension()
.is_some_and(|ext| ext == "js" || ext == "mjs" || ext == "cjs")
}

19
cli/tests/sourcemap.rs
View File

@@ -1,8 +1,10 @@
use posthog_cli::sourcemaps::{
inject::inject_pairs,
source_pair::{read_pairs, SourceMapContent},
content::SourceMapContent, inject::inject_pairs, plain::inject::is_javascript_file,
source_pairs::SourcePair,
};
use anyhow::Result;
use std::{
fs,
path::{Path, PathBuf},
@@ -27,6 +29,19 @@ fn assert_file_eq(base_path: &Path, path: &str, actual: impl Into<String>) {
assert_eq!(expected, actual.into());
}
pub fn read_pairs(
directory: &PathBuf,
ignore_globs: &[String],
prefix: &Option<String>,
) -> Result<Vec<SourcePair>> {
posthog_cli::sourcemaps::source_pairs::read_pairs(
directory,
ignore_globs,
is_javascript_file,
prefix,
)
}
#[test]
fn test_search_without_prefix() {
let pairs =