feat(core): Emit opt-in start marker spans for workflow and node executions (#40762)

Co-authored-by: Claude Opus 5.5 <[email protected]>
This commit is contained in:
ocdejong
2026-10-09 14:09:39 +00:00
committed by GitHub
co-authored by Claude Opus 5.5
parent 88a2531558
commit c7135451eb
31 changed files with 546 additions and 42 deletions
@@ -40,6 +40,26 @@ describe('UpdateOtelSettingsDto', () => {
expect(result.data.exporterProtocol).toBe('http/protobuf');
});
it('defaults the start span flags to false when omitted (body predates the fields)', () => {
const result = UpdateOtelSettingsDto.safeParse(validSettings);
assert(result.success, 'Expected a body without the start span flags to stay valid');
expect(result.data.emitWorkflowStartSpan).toBe(false);
expect(result.data.emitNodeStartSpan).toBe(false);
});
it('accepts the start span flags', () => {
const result = UpdateOtelSettingsDto.safeParse({
...validSettings,
emitWorkflowStartSpan: true,
emitNodeStartSpan: false,
});
assert(result.success, 'Expected the start span flags to be valid');
expect(result.data.emitWorkflowStartSpan).toBe(true);
expect(result.data.emitNodeStartSpan).toBe(false);
});
it('accepts a full body', () => {
const result = UpdateOtelSettingsDto.safeParse(validSettings);
expect(result.success).toBe(true);
@@ -20,6 +20,8 @@ export class OtelSettingsPublicDto extends Z.class(
tracesSampleRate: z.number().openapi(docs.tracesSampleRate),
startupConnectivityTimeoutMs: z.number().openapi(docs.startupConnectivityTimeoutMs),
includeNodeSpans: z.boolean().openapi(docs.includeNodeSpans),
emitWorkflowStartSpan: z.boolean().openapi(docs.emitWorkflowStartSpan),
emitNodeStartSpan: z.boolean().openapi(docs.emitNodeStartSpan),
injectOutbound: z.boolean().openapi(docs.injectOutbound),
productionExecutionsOnly: z.boolean().openapi(docs.productionExecutionsOnly),
},
@@ -43,6 +45,8 @@ export class UpdateOtelSettingsPublicDto extends Z.class(
docs.startupConnectivityTimeoutMs,
),
includeNodeSpans: fields.includeNodeSpans.openapi(docs.includeNodeSpans),
emitWorkflowStartSpan: fields.emitWorkflowStartSpan.openapi(docs.emitWorkflowStartSpan),
emitNodeStartSpan: fields.emitNodeStartSpan.openapi(docs.emitNodeStartSpan),
injectOutbound: fields.injectOutbound.openapi(docs.injectOutbound),
productionExecutionsOnly: fields.productionExecutionsOnly.openapi(
docs.productionExecutionsOnly,
@@ -3,7 +3,8 @@ import type { ZodOpenAPIMetadata } from '@asteasolutions/zod-to-openapi';
export const otelSettingsPublicDescription =
'The OpenTelemetry configuration, matching the fields exposed in the UI. On a write this is a ' +
'full replacement: every field must be provided, except `exporterProtocol`, which defaults to ' +
'`http/protobuf` when omitted. Fields managed declaratively via environment variables are ' +
'`http/protobuf` when omitted, and `emitWorkflowStartSpan` and `emitNodeStartSpan`, which ' +
'default to `false` when omitted. Fields managed declaratively via environment variables are ' +
'returned with their effective value. A write can repeat these values, but changing them returns 409.';
export const otelSettingsPublicFieldDocs = {
@@ -63,6 +64,21 @@ export const otelSettingsPublicFieldDocs = {
description: 'Whether to emit a span for each node execution in addition to the workflow span.',
example: true,
},
emitWorkflowStartSpan: {
description:
'Whether to emit a `workflow.execute.started` span with a duration of 0 ms when each ' +
'execution or resumed segment starts. Optional: a write that omits it selects `false`, but an ' +
'instance that sets this field to `true` with an environment variable rejects that write with 409.',
example: false,
},
emitNodeStartSpan: {
description:
'Whether to emit a `node.execute.started` span with a duration of 0 ms when each node span ' +
'starts. This doubles the number of node spans. Has no effect when `includeNodeSpans` is ' +
'false. Optional: a write that omits it selects `false`, but an instance that sets this field ' +
'to `true` with an environment variable rejects that write with 409.',
example: false,
},
injectOutbound: {
description:
'Whether to inject trace context headers into outbound HTTP requests made by nodes.',
@@ -25,6 +25,9 @@ export class UpdateOtelSettingsDto extends Z.class({
tracesSampleRate: z.number().min(0).max(1),
startupConnectivityTimeoutMs: z.number().int().nonnegative(),
includeNodeSpans: z.boolean(),
// Defaulted so a body from before these fields existed still parses.
emitWorkflowStartSpan: z.boolean().default(false),
emitNodeStartSpan: z.boolean().default(false),
injectOutbound: z.boolean(),
productionExecutionsOnly: z.boolean(),
}) {}
+25
View File
@@ -62,6 +62,31 @@ project name holds the user's name and email.
This key is n8n's single source of truth for the execution identity. Inbound and outbound
baggage (GOV-143) decides what it reads from this key and what it adds to baggage.
### Start marker spans
A span leaves n8n only when it ends. A marker span tells a trace backend that an execution
or a node started, before the real span ends.
| Environment variable | Default | Marker |
| ------------------------------------------ | ------- | -------------------------------------------- |
| `N8N_OTEL_TRACES_EMIT_WORKFLOW_START_SPAN` | `false` | `workflow.execute.started` under each workflow span |
| `N8N_OTEL_TRACES_EMIT_NODE_START_SPAN` | `false` | `node.execute.started` under each node span |
- A marker is a child of the span that it announces. Its duration is 0 ms.
- A marker has the start attributes of its parent and the execution identity. The node
marker does not have node custom tags, because n8n sets them when the node ends.
- Each `workflow.execute` span gets one workflow marker. A resume after a Wait node starts a
new segment with its own `workflow.execute` span.
- In queue mode, main and the worker each send a `workflow.execute` span for each segment,
so each segment gets two workflow markers. The marker from main shows when main queued the
execution. The marker from the worker shows when the worker started it. The resource
attribute `n8n.instance.role` (`main` or `worker`) tells them apart.
- The node marker follows the rules of `node.execute`. With
`N8N_OTEL_TRACES_INCLUDE_NODE_SPANS=false`, no node marker exists. The node flag doubles
the number of node spans.
- Both flags follow `N8N_OTEL_TRACES_PRODUCTION_ONLY`. Set them on workers too.
- A marker that fails logs a warning. It does not stop the real span or the execution.
### Module architecture
```mermaid
graph TD
@@ -1,5 +1,5 @@
import type { Logger } from '@n8n/backend-common';
import type { TextMapPropagator } from '@opentelemetry/api';
import type { TextMapPropagator, Tracer } from '@opentelemetry/api';
import { context, propagation, SpanStatusCode, trace } from '@opentelemetry/api';
import { hrTimeToMilliseconds } from '@opentelemetry/core';
import type { ExecutionStatus } from 'n8n-workflow';
@@ -10,6 +10,7 @@ import { ExecutionLevelTracer } from '../execution-level-tracer';
import type { StartWorkflowParams } from '../execution-level-tracer.types';
import type { OtelSettingsService } from '../otel-settings.service';
import type { OtelConfig } from '../otel.config';
import type { OtelService } from '../otel.service';
import { OtelTestProvider } from './support/otel-test-provider';
describe('ExecutionLevelTracer', () => {
@@ -1323,4 +1324,145 @@ describe('ExecutionLevelTracer', () => {
expect(headers).not.toHaveProperty('baggage');
});
});
describe('start marker spans', () => {
const node = { id: 'n1', name: 'MyNode', type: 'test', typeVersion: 1 };
const project = { id: 'proj-1', name: 'Finance', customAttributes: { env: 'prod' } };
const workflow = { ...defaultWorkflow, customAttributes: { team: 'ops' } };
const identity = {
'n8n.execution.id': 'exec-marker',
'n8n.workflow.id': 'wf-1',
'n8n.workflow.name': 'Test',
'n8n.project.id': 'proj-1',
'n8n.project.name': 'Finance',
};
const runSegment = (emitStartSpan: boolean, params: Partial<StartWorkflowParams> = {}) => {
tracer.startWorkflow({
executionId: 'exec-marker',
workflow,
project,
emitStartSpan,
...params,
});
tracer.startNode({ executionId: 'exec-marker', node, emitStartSpan });
tracer.endNode({ executionId: 'exec-marker', node, inputItemCount: 1, outputItemCount: 1 });
tracer.endWorkflow({
executionId: 'exec-marker',
status: 'success',
mode: 'trigger',
isRetry: false,
});
};
const spansNamed = (name: string) => otel.getFinishedSpans().filter((s) => s.name === name);
it('should emit no marker when the flags are off', () => {
runSegment(false);
expect(otel.getFinishedSpans().map((s) => s.name)).toEqual([
'node.execute',
'workflow.execute',
]);
});
it('should emit one zero-duration workflow marker as a child of the workflow span', () => {
runSegment(true);
const [workflowSpan] = spansNamed('workflow.execute');
const markers = spansNamed('workflow.execute.started');
expect(markers).toHaveLength(1);
const [marker] = markers;
expect(marker.parentSpanContext?.spanId).toBe(workflowSpan.spanContext().spanId);
expect(marker.spanContext().traceId).toBe(workflowSpan.spanContext().traceId);
expect(marker.duration).toEqual([0, 0]);
expect(marker.startTime).toEqual(workflowSpan.startTime);
expect(marker.endTime).toEqual(workflowSpan.startTime);
});
it('should give the workflow marker the start attributes and identity of the workflow span', () => {
runSegment(true);
const [marker] = spansNamed('workflow.execute.started');
expect(marker.attributes).toEqual({
'n8n.workflow.version_id': 'v1',
'n8n.workflow.node_count': 2,
'n8n.workflow.custom.team': 'ops',
'n8n.project.custom.env': 'prod',
...identity,
});
});
it('should emit one zero-duration node marker as a child of each node span', () => {
runSegment(true);
const [nodeSpan] = spansNamed('node.execute');
const markers = spansNamed('node.execute.started');
expect(markers).toHaveLength(1);
const [marker] = markers;
expect(marker.parentSpanContext?.spanId).toBe(nodeSpan.spanContext().spanId);
expect(marker.duration).toEqual([0, 0]);
expect(marker.startTime).toEqual(nodeSpan.startTime);
expect(marker.endTime).toEqual(nodeSpan.startTime);
});
it('should give the node marker the node attributes and identity, without node custom tags', () => {
runSegment(true);
const [marker] = spansNamed('node.execute.started');
expect(marker.attributes).toEqual({
'n8n.node.id': 'n1',
'n8n.node.name': 'MyNode',
'n8n.node.type': 'test',
'n8n.node.type_version': 1,
'n8n.project.custom.env': 'prod',
...identity,
});
});
it('should emit one workflow marker for each segment', () => {
runSegment(true);
runSegment(true, { workflow: { ...workflow, name: 'Renamed' } });
const workflowSpans = spansNamed('workflow.execute');
const markers = spansNamed('workflow.execute.started');
expect(markers).toHaveLength(2);
expect(markers.map((m) => m.parentSpanContext?.spanId)).toEqual(
workflowSpans.map((s) => s.spanContext().spanId),
);
expect(markers.map((m) => m.attributes['n8n.workflow.name'])).toEqual(['Test', 'Renamed']);
});
it('should keep the real spans and only warn when a marker fails', () => {
const failingMarkers = mock<OtelService>({
getTracer: (name: string) => {
const real = otel.provider.getTracer(name);
return {
startSpan: (spanName, options, ctx) => {
if (spanName.endsWith('.started')) throw new Error('marker failed');
return real.startSpan(spanName, options, ctx);
},
} as Tracer;
},
});
tracer = new ExecutionLevelTracer(failingMarkers, makeOtelSettingsService(), logger);
expect(() => runSegment(true)).not.toThrow();
expect(otel.getFinishedSpans().map((s) => s.name)).toEqual([
'node.execute',
'workflow.execute',
]);
expect(logger.warn).toHaveBeenCalledWith(
'Failed to emit start marker span',
expect.objectContaining({ spanName: 'workflow.execute.started', error: 'marker failed' }),
);
expect(logger.warn).toHaveBeenCalledWith(
'Failed to emit start marker span',
expect.objectContaining({ spanName: 'node.execute.started', error: 'marker failed' }),
);
});
});
});
@@ -47,6 +47,8 @@ function makeOtelSettingsService(
enabled: true,
productionExecutionsOnly: false,
includeNodeSpans: true,
emitWorkflowStartSpan: false,
emitNodeStartSpan: false,
exporterProtocol: 'http/protobuf',
exporterEndpoint: 'http://localhost:4318',
exporterTracingPath: '/v1/traces',
@@ -374,6 +376,38 @@ describe('OtelLifecycleHandler', () => {
expect(traceContextService.get).toHaveBeenCalledWith('exec-sub');
});
it('should not ask for a workflow start span when the flag is off', async () => {
traceContextService.get.mockResolvedValueOnce(undefined);
await handler.onWorkflowStart(baseCtx);
expect(tracer.startWorkflow).toHaveBeenCalledWith(
expect.objectContaining({ emitStartSpan: false }),
);
});
it('should ask for a workflow start span when the flag is on', async () => {
otelSettingsService._settings.emitWorkflowStartSpan = true;
traceContextService.get.mockResolvedValueOnce(undefined);
await handler.onWorkflowStart(baseCtx);
expect(tracer.startWorkflow).toHaveBeenCalledWith(
expect.objectContaining({ emitStartSpan: true }),
);
});
it('should ask for a workflow start span without a workflow instance (queue-mode main)', async () => {
otelSettingsService._settings.emitWorkflowStartSpan = true;
traceContextService.get.mockResolvedValueOnce(undefined);
await handler.onWorkflowStart({ ...baseCtx, workflowInstance: undefined });
expect(tracer.startWorkflow).toHaveBeenCalledWith(
expect.objectContaining({ emitStartSpan: true }),
);
});
it('should always persist generated spanContext', async () => {
traceContextService.get.mockResolvedValueOnce(parentTracingContext);
@@ -706,6 +740,23 @@ describe('OtelLifecycleHandler', () => {
);
});
it('should ask for a workflow start span on resume when the flag is on', async () => {
otelSettingsService._settings.emitWorkflowStartSpan = true;
traceContextService.get.mockResolvedValueOnce(prePauseContext);
await handler.onWorkflowResume({
type: 'workflowExecuteResume',
workflow: { id: 'wf-1', name: 'Test', versionId: 'v1', nodes: [], connections: {} },
workflowInstance: createWorkflowInstance(),
executionData: undefined as never,
executionId: 'exec-resume',
} as never);
expect(tracer.startWorkflow).toHaveBeenCalledWith(
expect.objectContaining({ emitStartSpan: true }),
);
});
it('should start a root span when no pre-wait context is persisted', async () => {
traceContextService.get.mockResolvedValueOnce(undefined);
@@ -1030,7 +1081,21 @@ describe('OtelLifecycleHandler', () => {
it('should call tracer.startNode with the resolved node', () => {
handler.onNodeStart(makeStartCtx());
expect(tracer.startNode).toHaveBeenCalledWith({ executionId: 'exec-1', node });
expect(tracer.startNode).toHaveBeenCalledWith({
executionId: 'exec-1',
node,
emitStartSpan: false,
});
});
it('should ask for a node start span when the flag is on', () => {
otelSettingsService._settings.emitNodeStartSpan = true;
handler.onNodeStart(makeStartCtx());
expect(tracer.startNode).toHaveBeenCalledWith(
expect.objectContaining({ emitStartSpan: true }),
);
});
it('should not call tracer.startNode when node is not found in workflow', () => {
@@ -27,6 +27,8 @@ const baseSettings: OtelConfig = {
tracesSampleRate: 1,
startupConnectivityTimeoutMs: 2_000,
includeNodeSpans: true,
emitWorkflowStartSpan: false,
emitNodeStartSpan: false,
injectOutbound: true,
productionExecutionsOnly: false,
};
@@ -88,6 +88,8 @@ describe('OtelSettingsService', () => {
tracesSampleRate: 1.0,
startupConnectivityTimeoutMs: 2_000,
includeNodeSpans: true,
emitWorkflowStartSpan: false,
emitNodeStartSpan: false,
injectOutbound: true,
productionExecutionsOnly: true,
envManagedFields: [],
@@ -308,6 +310,8 @@ describe('OtelSettingsService', () => {
tracesSampleRate: 0.7,
startupConnectivityTimeoutMs: 5_000,
includeNodeSpans: false,
emitWorkflowStartSpan: false,
emitNodeStartSpan: false,
injectOutbound: true,
productionExecutionsOnly: false,
};
@@ -25,6 +25,8 @@ import {
createFailingWorkflowFixture,
createWaitWorkflowFixture,
} from './support/otel-workflow-fixtures';
import { OtelSettingsService } from '../otel-settings.service';
import { OtelConfig } from '../otel.config';
let otel: OtelTestProvider;
let workflowRunner: WorkflowRunner;
@@ -219,6 +221,67 @@ describe('OTEL Workflow Tracing Integration', () => {
});
});
describe('Start marker spans', () => {
const setStartSpanFlags = async (enabled: boolean) => {
const config = Container.get(OtelConfig);
config.emitWorkflowStartSpan = enabled;
config.emitNodeStartSpan = enabled;
await Container.get(OtelSettingsService).loadSettings();
};
beforeAll(async () => await setStartSpanFlags(true));
afterAll(async () => await setStartSpanFlags(false));
it('should emit one workflow marker and one marker for each node span', async () => {
const project = await createTeamProject();
const workflow = await createWorkflow(createMultiNodeWorkflowFixture(), project);
const executionId = await executeWorkflow(workflowRunner, workflow, project.id);
await waitForExecution(executionRepository, executionId);
const spans = otel.getFinishedSpans();
const workflowSpan = spans.find((s) => s.name === 'workflow.execute')!;
const nodeSpanIds = spans
.filter((s) => s.name === 'node.execute')
.map((s) => s.spanContext().spanId);
const workflowMarkers = spans.filter((s) => s.name === 'workflow.execute.started');
const nodeMarkers = spans.filter((s) => s.name === 'node.execute.started');
expect(workflowMarkers).toHaveLength(1);
expect(workflowMarkers[0].parentSpanContext?.spanId).toBe(workflowSpan.spanContext().spanId);
expect(workflowMarkers[0].attributes['n8n.execution.id']).toBe(executionId);
expect(nodeMarkers.map((m) => m.parentSpanContext?.spanId).sort()).toEqual(nodeSpanIds.sort());
for (const marker of [...workflowMarkers, ...nodeMarkers]) {
expect(marker.duration).toEqual([0, 0]);
}
});
it('should emit a workflow marker for each segment of a resumed execution', async () => {
const project = await createTeamProject();
const workflow = await createWorkflow(createWaitWorkflowFixture(), project);
const executionId = await executeWorkflow(workflowRunner, workflow, project.id);
await waitForExecutionStatus(executionRepository, executionId, 'waiting');
await Container.get(WorkflowRepository).update(workflow.id, { name: 'Renamed during wait' });
await Container.get(WaitTracker).startExecution(executionId);
await waitForExecutionStatus(executionRepository, executionId, 'success');
const spans = otel.getFinishedSpans();
const parked = spans.find((s) => s.attributes['n8n.execution.status'] === 'waiting')!;
const resumed = spans.find((s) => s.attributes['n8n.execution.status'] === 'success')!;
const markerOf = (segment: typeof parked) =>
spans.filter(
(s) =>
s.name === 'workflow.execute.started' &&
s.parentSpanContext?.spanId === segment.spanContext().spanId,
);
expect(spans.filter((s) => s.name === 'workflow.execute.started')).toHaveLength(2);
expect(markerOf(parked)).toHaveLength(1);
expect(markerOf(resumed)).toHaveLength(1);
expect(markerOf(resumed)[0].attributes['n8n.workflow.name']).toBe('Renamed during wait');
});
});
describe('Custom Telemetry Tags', () => {
const createWorkflowWithCustomTagsFixture = () => ({
nodes: [
@@ -54,6 +54,8 @@ const enabledSettings: OtelConfig = {
tracesSampleRate: 1,
startupConnectivityTimeoutMs: 2_000,
includeNodeSpans: true,
emitWorkflowStartSpan: false,
emitNodeStartSpan: false,
injectOutbound: true,
productionExecutionsOnly: true,
};
@@ -162,6 +162,8 @@ const enabledSettings: OtelConfig = {
tracesSampleRate: 1,
startupConnectivityTimeoutMs: 2_000,
includeNodeSpans: true,
emitWorkflowStartSpan: false,
emitNodeStartSpan: false,
injectOutbound: true,
productionExecutionsOnly: true,
};
@@ -1,6 +1,6 @@
import { Logger } from '@n8n/backend-common';
import { Service } from '@n8n/di';
import type { Context, Exception, Span } from '@opentelemetry/api';
import type { Attributes, Context, Exception, Span, TimeInput } from '@opentelemetry/api';
import {
context,
defaultTextMapGetter,
@@ -9,7 +9,7 @@ import {
SpanStatusCode,
trace,
} from '@opentelemetry/api';
import { W3CTraceContextPropagator } from '@opentelemetry/core';
import { isTimeInputHrTime, W3CTraceContextPropagator } from '@opentelemetry/core';
import type { ExecutionStatus } from 'n8n-workflow';
import { WorkflowCrashedError } from '@/errors/workflow-crashed.error';
@@ -76,31 +76,34 @@ export class ExecutionLevelTracer {
params.project?.customAttributes,
);
const span = this.tracer.startSpan(
'workflow.execute',
{
attributes: {
[ATTR.WORKFLOW_NAME]: params.workflow.name,
[ATTR.WORKFLOW_VERSION_ID]: params.workflow.versionId ?? '',
[ATTR.WORKFLOW_NODE_COUNT]: params.workflow.nodeCount,
...buildCustomAttributes(
ATTR.WORKFLOW_CUSTOM_PREFIX,
params.workflow?.customAttributes,
),
...(params.project && { [ATTR.PROJECT_ID]: params.project.id }),
...projectCustomAttributes,
...identity,
},
links,
},
parentCtx,
);
const attributes = {
[ATTR.WORKFLOW_NAME]: params.workflow.name,
[ATTR.WORKFLOW_VERSION_ID]: params.workflow.versionId ?? '',
[ATTR.WORKFLOW_NODE_COUNT]: params.workflow.nodeCount,
...buildCustomAttributes(ATTR.WORKFLOW_CUSTOM_PREFIX, params.workflow?.customAttributes),
...(params.project && { [ATTR.PROJECT_ID]: params.project.id }),
...projectCustomAttributes,
...identity,
};
const span = this.tracer.startSpan('workflow.execute', { attributes, links }, parentCtx);
const workflowContext = trace.setSpan(parentCtx, span);
this.activeWorkflowSpans.set(params.executionId, {
span,
context: trace.setSpan(parentCtx, span),
context: workflowContext,
projectCustomAttributes,
});
if (params.emitStartSpan) {
this.emitStartMarker(
'workflow.execute.started',
attributes,
workflowContext,
params.executionId,
);
}
return toTracingParentContext(span, identity);
} catch (error) {
this.logger.warn('Failed to start workflow span', {
@@ -207,19 +210,24 @@ export class ExecutionLevelTracer {
return;
}
const span = this.tracer.startSpan(
'node.execute',
{
attributes: {
[ATTR.NODE_ID]: params.node.id,
[ATTR.NODE_NAME]: params.node.name,
[ATTR.NODE_TYPE]: params.node.type,
[ATTR.NODE_TYPE_VERSION]: params.node.typeVersion,
...tracked.projectCustomAttributes,
},
},
tracked.context,
);
const attributes = {
[ATTR.NODE_ID]: params.node.id,
[ATTR.NODE_NAME]: params.node.name,
[ATTR.NODE_TYPE]: params.node.type,
[ATTR.NODE_TYPE_VERSION]: params.node.typeVersion,
...tracked.projectCustomAttributes,
};
const span = this.tracer.startSpan('node.execute', { attributes }, tracked.context);
if (params.emitStartSpan) {
this.emitStartMarker(
'node.execute.started',
attributes,
trace.setSpan(tracked.context, span),
params.executionId,
);
}
let executionNodes = this.activeNodeSpansByExecutionId.get(params.executionId);
@@ -306,6 +314,24 @@ export class ExecutionLevelTracer {
}
}
private emitStartMarker(
name: string,
attributes: Attributes,
parentCtx: Context,
executionId: string,
): void {
try {
const startTime = getStartTime(trace.getSpan(parentCtx));
this.tracer.startSpan(name, { startTime, attributes }, parentCtx).end(startTime);
} catch (error) {
this.logger.warn('Failed to emit start marker span', {
executionId,
spanName: name,
error: error instanceof Error ? error.message : String(error),
});
}
}
private parseTraceParentHeaders(tracingContext?: TracingContext): Context {
if (!tracingContext) return ROOT_CONTEXT;
return propagator.extract(ROOT_CONTEXT, tracingContext, defaultTextMapGetter);
@@ -394,6 +420,12 @@ function toTracingParentContext(span: Span, identity: ExecutionIdentity): Tracin
return { traceparent: carrier.traceparent, tracestate: carrier.tracestate, identity };
}
// Only a recorded SDK span has a start time. The API span type does not expose it.
function getStartTime(span: Span | undefined): TimeInput {
const startTime = span && 'startTime' in span ? span.startTime : undefined;
return isTimeInputHrTime(startTime) ? startTime : new Date();
}
function terminateSpan(span: Span, reason: string): void {
span.setAttribute(ATTR.NODE_TERMINATION_REASON, reason);
span.setStatus({ code: SpanStatusCode.ERROR });
@@ -28,6 +28,7 @@ export type StartWorkflowParams = {
savedIdentity?: ExecutionIdentity;
workflow: WorkflowContext;
project?: ProjectContext;
emitStartSpan?: boolean;
};
export type EndWorkflowParams = {
@@ -59,6 +60,7 @@ type NodeTracingParams = Pick<INode, 'id' | 'name' | 'type' | 'typeVersion'>;
export type StartNodeParams = {
executionId: string;
node: NodeTracingParams;
emitStartSpan?: boolean;
};
type EndNodeError = {
@@ -108,6 +108,7 @@ export class OtelLifecycleHandler {
executionId: ctx.executionId,
tracingContext,
savedIdentity,
emitStartSpan: this.otelSettingsService.getSettings().emitWorkflowStartSpan,
project: this.buildProjectContext(project),
workflow: {
id: ctx.workflow.id,
@@ -151,6 +152,7 @@ export class OtelLifecycleHandler {
tracingContext: previousWorkflowExecution,
linkTo: previousWorkflowExecution,
savedIdentity: toExecutionIdentity(previousWorkflowExecution?.identity),
emitStartSpan: this.otelSettingsService.getSettings().emitWorkflowStartSpan,
project: this.buildProjectContext(project),
workflow: {
id: ctx.workflow.id,
@@ -214,6 +216,7 @@ export class OtelLifecycleHandler {
this.tracer.startNode({
executionId: ctx.executionId,
node,
emitStartSpan: this.otelSettingsService.getSettings().emitNodeStartSpan,
});
}
@@ -207,6 +207,8 @@ export class OtelSettingsService {
tracesSampleRate: pick('tracesSampleRate'),
startupConnectivityTimeoutMs: pick('startupConnectivityTimeoutMs'),
includeNodeSpans: pick('includeNodeSpans'),
emitWorkflowStartSpan: pick('emitWorkflowStartSpan'),
emitNodeStartSpan: pick('emitNodeStartSpan'),
injectOutbound: pick('injectOutbound'),
productionExecutionsOnly: pick('productionExecutionsOnly'),
};
@@ -33,6 +33,12 @@ export class OtelConfig {
@Env(OTEL_ENV_VARS.includeNodeSpans)
includeNodeSpans: boolean = true;
@Env(OTEL_ENV_VARS.emitWorkflowStartSpan)
emitWorkflowStartSpan: boolean = false;
@Env(OTEL_ENV_VARS.emitNodeStartSpan)
emitNodeStartSpan: boolean = false;
@Env(OTEL_ENV_VARS.injectOutbound)
injectOutbound: boolean = true;
@@ -13,6 +13,8 @@ export const OTEL_ENV_VARS = {
tracesSampleRate: 'N8N_OTEL_TRACES_SAMPLE_RATE',
startupConnectivityTimeoutMs: 'N8N_OTEL_STARTUP_CONNECTIVITY_TIMEOUT_MS',
includeNodeSpans: 'N8N_OTEL_TRACES_INCLUDE_NODE_SPANS',
emitWorkflowStartSpan: 'N8N_OTEL_TRACES_EMIT_WORKFLOW_START_SPAN',
emitNodeStartSpan: 'N8N_OTEL_TRACES_EMIT_NODE_START_SPAN',
injectOutbound: 'N8N_OTEL_TRACES_INJECT_OUTBOUND',
productionExecutionsOnly: 'N8N_OTEL_TRACES_PRODUCTION_ONLY',
} as const;
@@ -63,7 +63,7 @@ export class OtelPublicController {
@ApiKeyScope('otel:manage')
@ApiSummary('Set the OpenTelemetry configuration')
@ApiDescription(
'Set the OpenTelemetry configuration. This is a full replacement: every field must be provided, and a partial body is rejected. The one exception is `exporterProtocol`, which defaults to `http/protobuf` when omitted. The update takes effect exactly as it would from the UI, using the same validation, and is applied to the running instance immediately. Fields managed declaratively via environment variables are read-only: attempting to change one is rejected with 409, while re-submitting its current value (as returned by GET) is accepted. Requires the `otel:manage` scope.',
'Set the OpenTelemetry configuration. This is a full replacement: every field must be provided, and a partial body is rejected. The exceptions are `exporterProtocol`, which defaults to `http/protobuf` when omitted, and `emitWorkflowStartSpan` and `emitNodeStartSpan`, which default to `false` when omitted. The update takes effect exactly as it would from the UI, using the same validation, and is applied to the running instance immediately. Fields managed declaratively via environment variables are read-only: attempting to change one is rejected with 409, while re-submitting its current value (as returned by GET) is accepted. Requires the `otel:manage` scope.',
)
@ApiTags(tags)
@ApiResponse(200, OtelSettingsPublicDto)
@@ -2,7 +2,7 @@ operationId: updateOtelSettings
tags:
- SettingsOtel
summary: Set the OpenTelemetry configuration
description: 'Set the OpenTelemetry configuration. This is a full replacement: every field must be provided, and a partial body is rejected. The one exception is `exporterProtocol`, which defaults to `http/protobuf` when omitted. The update takes effect exactly as it would from the UI, using the same validation, and is applied to the running instance immediately. Fields managed declaratively via environment variables are read-only: attempting to change one is rejected with 409, while re-submitting its current value (as returned by GET) is accepted. Requires the `otel:manage` scope.'
description: 'Set the OpenTelemetry configuration. This is a full replacement: every field must be provided, and a partial body is rejected. The exceptions are `exporterProtocol`, which defaults to `http/protobuf` when omitted, and `emitWorkflowStartSpan` and `emitNodeStartSpan`, which default to `false` when omitted. The update takes effect exactly as it would from the UI, using the same validation, and is applied to the running instance immediately. Fields managed declaratively via environment variables are read-only: attempting to change one is rejected with 409, while re-submitting its current value (as returned by GET) is accepted. Requires the `otel:manage` scope.'
x-required-scope: otel:manage
x-eov-operation-id: unreachable
x-eov-operation-handler: v1/handlers/decorator-routed.handler
@@ -60,6 +60,16 @@ requestBody:
type: boolean
description: Whether to emit a span for each node execution in addition to the workflow span.
example: true
emitWorkflowStartSpan:
type: boolean
default: false
description: 'Whether to emit a `workflow.execute.started` span with a duration of 0 ms when each execution or resumed segment starts. Optional: a write that omits it selects `false`, but an instance that sets this field to `true` with an environment variable rejects that write with 409.'
example: false
emitNodeStartSpan:
type: boolean
default: false
description: 'Whether to emit a `node.execute.started` span with a duration of 0 ms when each node span starts. This doubles the number of node spans. Has no effect when `includeNodeSpans` is false. Optional: a write that omits it selects `false`, but an instance that sets this field to `true` with an environment variable rejects that write with 409.'
example: false
injectOutbound:
type: boolean
description: Whether to inject trace context headers into outbound HTTP requests made by nodes.
@@ -80,7 +90,7 @@ requestBody:
- injectOutbound
- productionExecutionsOnly
additionalProperties: false
description: 'The OpenTelemetry configuration, matching the fields exposed in the UI. On a write this is a full replacement: every field must be provided, except `exporterProtocol`, which defaults to `http/protobuf` when omitted. Fields managed declaratively via environment variables are returned with their effective value. A write can repeat these values, but changing them returns 409.'
description: 'The OpenTelemetry configuration, matching the fields exposed in the UI. On a write this is a full replacement: every field must be provided, except `exporterProtocol`, which defaults to `http/protobuf` when omitted, and `emitWorkflowStartSpan` and `emitNodeStartSpan`, which default to `false` when omitted. Fields managed declaratively via environment variables are returned with their effective value. A write can repeat these values, but changing them returns 409.'
responses:
'200':
description: Operation successful.
@@ -13,6 +13,8 @@ export function toOtelSettingsResponse(config: OtelConfig): OtelSettingsResponse
tracesSampleRate: config.tracesSampleRate,
startupConnectivityTimeoutMs: config.startupConnectivityTimeoutMs,
includeNodeSpans: config.includeNodeSpans,
emitWorkflowStartSpan: config.emitWorkflowStartSpan,
emitNodeStartSpan: config.emitNodeStartSpan,
injectOutbound: config.injectOutbound,
productionExecutionsOnly: config.productionExecutionsOnly,
};
@@ -45,6 +45,14 @@ properties:
type: boolean
description: Whether to emit a span for each node execution in addition to the workflow span.
example: true
emitWorkflowStartSpan:
type: boolean
description: 'Whether to emit a `workflow.execute.started` span with a duration of 0 ms when each execution or resumed segment starts. Optional: a write that omits it selects `false`, but an instance that sets this field to `true` with an environment variable rejects that write with 409.'
example: false
emitNodeStartSpan:
type: boolean
description: 'Whether to emit a `node.execute.started` span with a duration of 0 ms when each node span starts. This doubles the number of node spans. Has no effect when `includeNodeSpans` is false. Optional: a write that omits it selects `false`, but an instance that sets this field to `true` with an environment variable rejects that write with 409.'
example: false
injectOutbound:
type: boolean
description: Whether to inject trace context headers into outbound HTTP requests made by nodes.
@@ -63,7 +71,9 @@ required:
- tracesSampleRate
- startupConnectivityTimeoutMs
- includeNodeSpans
- emitWorkflowStartSpan
- emitNodeStartSpan
- injectOutbound
- productionExecutionsOnly
additionalProperties: false
description: 'The OpenTelemetry configuration, matching the fields exposed in the UI. On a write this is a full replacement: every field must be provided, except `exporterProtocol`, which defaults to `http/protobuf` when omitted. Fields managed declaratively via environment variables are returned with their effective value. A write can repeat these values, but changing them returns 409.'
description: 'The OpenTelemetry configuration, matching the fields exposed in the UI. On a write this is a full replacement: every field must be provided, except `exporterProtocol`, which defaults to `http/protobuf` when omitted, and `emitWorkflowStartSpan` and `emitNodeStartSpan`, which default to `false` when omitted. Fields managed declaratively via environment variables are returned with their effective value. A write can repeat these values, but changing them returns 409.'
@@ -21,6 +21,8 @@ const validSettings = {
tracesSampleRate: 0.5,
startupConnectivityTimeoutMs: 3_000,
includeNodeSpans: false,
emitWorkflowStartSpan: true,
emitNodeStartSpan: true,
injectOutbound: false,
productionExecutionsOnly: false,
};
@@ -101,6 +103,8 @@ describe('OpenTelemetry settings in Public API', () => {
'tracesSampleRate',
'startupConnectivityTimeoutMs',
'includeNodeSpans',
'emitWorkflowStartSpan',
'emitNodeStartSpan',
'injectOutbound',
'productionExecutionsOnly',
].sort(),
@@ -248,6 +252,24 @@ describe('OpenTelemetry settings in Public API', () => {
expect(replaced.body.exporterProtocol).toBe('http/protobuf');
});
it('resets omitted start span flags to false (PUT is a full replacement)', async () => {
await testServer.publicApiAgentFor(owner).put('/settings/otel').send(validSettings);
const {
emitWorkflowStartSpan: _workflow,
emitNodeStartSpan: _node,
...bodyWithoutFlags
} = validSettings;
const response = await testServer
.publicApiAgentFor(owner)
.put('/settings/otel')
.send(bodyWithoutFlags);
expect(response.status).toBe(200);
expect(response.body.emitWorkflowStartSpan).toBe(false);
expect(response.body.emitNodeStartSpan).toBe(false);
});
it('stores the exporter endpoint without surrounding whitespace', async () => {
const response = await testServer
.publicApiAgentFor(owner)
@@ -3557,6 +3557,10 @@
"settings.opentelemetry.tracesSampleRate.slug": "of {max}",
"settings.opentelemetry.includeNodeSpans.label": "Include node spans",
"settings.opentelemetry.includeNodeSpans.description": "One span per node, or workflow-level spans only.",
"settings.opentelemetry.emitWorkflowStartSpan.label": "Send workflow start spans",
"settings.opentelemetry.emitWorkflowStartSpan.description": "Send a 0 ms span when a run starts, so it shows before the run ends.",
"settings.opentelemetry.emitNodeStartSpan.label": "Send node start spans",
"settings.opentelemetry.emitNodeStartSpan.description": "Send a 0 ms span when each node starts.",
"settings.opentelemetry.injectOutbound.label": "Inject outbound traceparent",
"settings.opentelemetry.injectOutbound.description": "Add a W3C traceparent header to outbound HTTP so downstream services join the same trace.",
"settings.opentelemetry.productionExecutionsOnly.label": "Track published workflows only",
@@ -59,6 +59,8 @@ const makeSettings = (overrides: Partial<OtelSettingsResponse> = {}): OtelSettin
tracesSampleRate: 1.0,
startupConnectivityTimeoutMs: 2000,
includeNodeSpans: true,
emitWorkflowStartSpan: false,
emitNodeStartSpan: false,
injectOutbound: true,
productionExecutionsOnly: true,
envManagedFields: [],
@@ -238,6 +240,19 @@ describe('SettingsOpenTelemetryView', () => {
await waitFor(() => expect(getByTestId('settings-save-bar-save')).toBeInTheDocument());
});
it('disables the node start span checkbox while node spans are off', async () => {
getOtelSettingsMock.mockResolvedValue(makeSettings({ includeNodeSpans: false }));
const { getByTestId } = render();
await waitFor(() => expect(getByTestId('otel-emit-node-start-span')).toBeInTheDocument());
expect(getByTestId('otel-emit-node-start-span')).toBeDisabled();
await userEvent.click(getByTestId('otel-include-node-spans'));
await waitFor(() => expect(getByTestId('otel-emit-node-start-span')).toBeEnabled());
});
// ── save ──────────────────────────────────────────────────────────────────
it('calls updateOtelSettings and shows success toast when saving', async () => {
@@ -440,6 +455,8 @@ describe('SettingsOpenTelemetryView', () => {
// Checkboxes
await userEvent.click(getByTestId('otel-include-node-spans'));
await userEvent.click(getByTestId('otel-emit-workflow-start-span'));
await userEvent.click(getByTestId('otel-emit-node-start-span'));
await userEvent.click(getByTestId('otel-inject-outbound'));
await userEvent.click(getByTestId('otel-production-only'));
@@ -458,6 +475,8 @@ describe('SettingsOpenTelemetryView', () => {
startupConnectivityTimeoutMs: 5000,
tracesSampleRate: 0.5,
includeNodeSpans: true,
emitWorkflowStartSpan: true,
emitNodeStartSpan: true,
injectOutbound: true,
productionExecutionsOnly: true,
}),
@@ -703,6 +703,38 @@ watch(
</template>
</OtelSettingsRow>
<OtelSettingsRow
:title="i18n.baseText('settings.opentelemetry.emitWorkflowStartSpan.label')"
:description="i18n.baseText('settings.opentelemetry.emitWorkflowStartSpan.description')"
:env-tooltip="envTooltip('emitWorkflowStartSpan')"
>
<template #action>
<N8nCheckbox
:model-value="otelStore.settings.emitWorkflowStartSpan"
:disabled="isEnvManaged('emitWorkflowStartSpan')"
data-test-id="otel-emit-workflow-start-span"
@update:model-value="otelStore.settings.emitWorkflowStartSpan = Boolean($event)"
/>
</template>
</OtelSettingsRow>
<OtelSettingsRow
:title="i18n.baseText('settings.opentelemetry.emitNodeStartSpan.label')"
:description="i18n.baseText('settings.opentelemetry.emitNodeStartSpan.description')"
:env-tooltip="envTooltip('emitNodeStartSpan')"
>
<template #action>
<N8nCheckbox
:model-value="otelStore.settings.emitNodeStartSpan"
:disabled="
isEnvManaged('emitNodeStartSpan') || !otelStore.settings.includeNodeSpans
"
data-test-id="otel-emit-node-start-span"
@update:model-value="otelStore.settings.emitNodeStartSpan = Boolean($event)"
/>
</template>
</OtelSettingsRow>
<OtelSettingsRow
:title="i18n.baseText('settings.opentelemetry.injectOutbound.label')"
:description="i18n.baseText('settings.opentelemetry.injectOutbound.description')"
@@ -22,6 +22,8 @@ const makeSettings = (overrides: Partial<OtelSettingsResponse> = {}): OtelSettin
tracesSampleRate: 1.0,
startupConnectivityTimeoutMs: 2000,
includeNodeSpans: true,
emitWorkflowStartSpan: false,
emitNodeStartSpan: false,
injectOutbound: true,
productionExecutionsOnly: true,
envManagedFields: [],
@@ -13,6 +13,8 @@ export type OtelSettings = {
tracesSampleRate: number;
startupConnectivityTimeoutMs: number;
includeNodeSpans: boolean;
emitWorkflowStartSpan: boolean;
emitNodeStartSpan: boolean;
injectOutbound: boolean;
productionExecutionsOnly: boolean;
};
@@ -33,6 +33,8 @@ export const OTEL_FIELD_ENV_VARS = {
tracesSampleRate: 'N8N_OTEL_TRACES_SAMPLE_RATE',
startupConnectivityTimeoutMs: 'N8N_OTEL_STARTUP_CONNECTIVITY_TIMEOUT_MS',
includeNodeSpans: 'N8N_OTEL_TRACES_INCLUDE_NODE_SPANS',
emitWorkflowStartSpan: 'N8N_OTEL_TRACES_EMIT_WORKFLOW_START_SPAN',
emitNodeStartSpan: 'N8N_OTEL_TRACES_EMIT_NODE_START_SPAN',
injectOutbound: 'N8N_OTEL_TRACES_INJECT_OUTBOUND',
productionExecutionsOnly: 'N8N_OTEL_TRACES_PRODUCTION_ONLY',
} as const;
@@ -30,6 +30,8 @@ const makeSettings = (overrides: Partial<OtelSettingsResponse> = {}): OtelSettin
tracesSampleRate: 1.0,
startupConnectivityTimeoutMs: 2000,
includeNodeSpans: true,
emitWorkflowStartSpan: false,
emitNodeStartSpan: false,
injectOutbound: true,
productionExecutionsOnly: true,
envManagedFields: [],
@@ -37,6 +37,8 @@ const defaultSettings: OtelSettings = {
tracesSampleRate: 1.0,
startupConnectivityTimeoutMs: 2_000,
includeNodeSpans: true,
emitWorkflowStartSpan: false,
emitNodeStartSpan: false,
injectOutbound: true,
productionExecutionsOnly: true,
};