mirror of
https://github.com/BillyOutlast/posthog.git
synced 2026-02-04 03:01:23 +01:00
refactor: remove ts normalization from node (#40706)
This commit is contained in:
Paweł Ledwoń
@@ -1,6 +1,6 @@
|
||||
import { DateTime } from 'luxon'
|
||||
|
||||
import { EventHeaders } from '../../types'
|
||||
import type { EventHeaders } from '../../types'
|
||||
import { logger } from '../../utils/logger'
|
||||
import { compareTimestamps } from './timestamp-comparison'
|
||||
|
||||
|
||||
@@ -1,116 +1,58 @@
|
||||
import { DateTime, Duration } from 'luxon'
|
||||
import { DateTime } from 'luxon'
|
||||
|
||||
import { PluginEvent } from '@posthog/plugin-scaffold'
|
||||
|
||||
import { logger } from '../../utils/logger'
|
||||
import { captureException } from '../../utils/posthog'
|
||||
|
||||
type IngestionWarningCallback = (type: string, details: Record<string, any>) => void
|
||||
|
||||
const FutureEventHoursCutoffMillis = 23 * 3600 * 1000 // 23 hours
|
||||
|
||||
/**
|
||||
* Parse event timestamp from plugin-server event data.
|
||||
*
|
||||
* NOTE: Timestamp normalization (clock skew adjustment, future event clamping, offset handling)
|
||||
* is now handled in the Rust capture service. This function only parses the timestamp string
|
||||
* that comes from the event data. The timestamp in event data is already normalized by the Rust
|
||||
* capture service (via parse_event_timestamp in rust/common/types/src/timestamp.rs).
|
||||
*
|
||||
* The Rust capture service handles:
|
||||
* - Clock skew adjustment using sent_at and now
|
||||
* - Future event clamping (to now if >23 hours in future)
|
||||
* - Out-of-bounds validation (year < 0 or > 9999, fallback to epoch)
|
||||
* - Offset handling
|
||||
*
|
||||
* This function only needs to parse the string to a DateTime object.
|
||||
*/
|
||||
export function parseEventTimestamp(data: PluginEvent, callback?: IngestionWarningCallback): DateTime {
|
||||
const now = DateTime.fromISO(data['now']).toUTC() // now is set by the capture endpoint and assumed valid
|
||||
// The timestamp has already been normalized by the Rust capture service
|
||||
// Just parse it from the data
|
||||
if (data['timestamp']) {
|
||||
const parsedTs = parseDate(data['timestamp'])
|
||||
|
||||
let sentAt: DateTime | null = null
|
||||
if (!data.properties?.['$ignore_sent_at'] && data['sent_at']) {
|
||||
sentAt = DateTime.fromISO(data['sent_at']).toUTC()
|
||||
if (!sentAt.isValid) {
|
||||
if (!parsedTs.isValid) {
|
||||
callback?.('ignored_invalid_timestamp', {
|
||||
eventUuid: data['uuid'] ?? '',
|
||||
field: 'sent_at',
|
||||
value: data['sent_at'],
|
||||
reason: sentAt.invalidExplanation || 'unknown error',
|
||||
field: 'timestamp',
|
||||
value: data['timestamp'],
|
||||
reason: parsedTs.invalidExplanation || 'unknown error',
|
||||
})
|
||||
sentAt = null
|
||||
}
|
||||
}
|
||||
|
||||
let parsedTs = handleTimestamp(data, now, sentAt, data.team_id)
|
||||
|
||||
// Events in the future would indicate an instrumentation bug, lets' ingest them
|
||||
// but publish an integration warning to help diagnose such issues.
|
||||
// We will also 'fix' the date to be now()
|
||||
const nowDiff = parsedTs.toUTC().diff(now).toMillis()
|
||||
if (nowDiff > FutureEventHoursCutoffMillis) {
|
||||
callback?.('event_timestamp_in_future', {
|
||||
timestamp: data['timestamp'] ?? '',
|
||||
sentAt: data['sent_at'] ?? '',
|
||||
offset: data['offset'] ?? '',
|
||||
now: data['now'],
|
||||
result: parsedTs.toISO(),
|
||||
eventUuid: data['uuid'],
|
||||
eventName: data['event'],
|
||||
})
|
||||
|
||||
parsedTs = now
|
||||
}
|
||||
|
||||
const parsedTsOutOfBounds = parsedTs.year < 0 || parsedTs.year > 9999
|
||||
if (!parsedTs.isValid || parsedTsOutOfBounds) {
|
||||
const details: Record<string, any> = {
|
||||
eventUuid: data['uuid'] ?? '',
|
||||
field: 'timestamp',
|
||||
value: data['timestamp'] ?? '',
|
||||
reason: parsedTs.invalidExplanation || (parsedTsOutOfBounds ? 'out of bounds' : 'unknown error'),
|
||||
return DateTime.utc()
|
||||
}
|
||||
|
||||
const parsedTsOutOfBounds = parsedTs.year < 0 || parsedTs.year > 9999
|
||||
if (parsedTsOutOfBounds) {
|
||||
details['offset'] = data['offset']
|
||||
details['parsed_year'] = parsedTs.year
|
||||
}
|
||||
|
||||
callback?.('ignored_invalid_timestamp', details)
|
||||
return DateTime.utc()
|
||||
}
|
||||
|
||||
return parsedTs
|
||||
}
|
||||
|
||||
function handleTimestamp(data: PluginEvent, now: DateTime, sentAt: DateTime | null, teamId: number): DateTime {
|
||||
let parsedTs: DateTime = now
|
||||
let timestamp: DateTime = now
|
||||
|
||||
if (data['timestamp']) {
|
||||
timestamp = parseDate(data['timestamp'])
|
||||
|
||||
if (!sentAt || !timestamp.isValid) {
|
||||
return timestamp
|
||||
}
|
||||
|
||||
// To handle clock skew between the client and server, we attempt
|
||||
// to compute the skew based on the difference between the
|
||||
// client-generated `sent_at` and the server-generated `now`
|
||||
// filled by the capture endpoint.
|
||||
//
|
||||
// We calculate the skew as:
|
||||
//
|
||||
// skew = sent_at - now
|
||||
//
|
||||
// And adjust the timestamp accordingly.
|
||||
|
||||
// sent_at - timestamp == now - x
|
||||
// x = now + (timestamp - sent_at)
|
||||
try {
|
||||
// timestamp and sent_at must both be in the same format: either both with or both without timezones
|
||||
// otherwise we can't get a diff to add to now
|
||||
parsedTs = now.plus(timestamp.diff(sentAt))
|
||||
} catch (error) {
|
||||
logger.error('⚠️', 'Error when handling timestamp:', { error: error.message })
|
||||
captureException(error, {
|
||||
tags: { team_id: teamId },
|
||||
extra: { data, now, sentAt },
|
||||
callback?.('ignored_invalid_timestamp', {
|
||||
eventUuid: data['uuid'] ?? '',
|
||||
field: 'timestamp',
|
||||
value: data['timestamp'],
|
||||
reason: 'out of bounds',
|
||||
parsed_year: parsedTs.year,
|
||||
})
|
||||
|
||||
return timestamp
|
||||
return DateTime.utc()
|
||||
}
|
||||
|
||||
return parsedTs
|
||||
}
|
||||
|
||||
if (data['offset']) {
|
||||
parsedTs = now.minus(Duration.fromMillis(data['offset']))
|
||||
}
|
||||
|
||||
return parsedTs
|
||||
// Fallback to current time if no timestamp provided
|
||||
return DateTime.utc()
|
||||
}
|
||||
|
||||
export function parseDate(supposedIsoString: string): DateTime {
|
||||
|
||||
@@ -229,34 +229,32 @@ describe('dropOldEventsStep()', () => {
|
||||
})
|
||||
|
||||
describe('timestamp handling', () => {
|
||||
it('handles events with sent_at timestamp', async () => {
|
||||
it('ignores sent_at and uses already-normalized timestamp from Rust', async () => {
|
||||
const team = createTeamWithDropSetting(3600) // 1 hour threshold
|
||||
|
||||
// Test events with sent_at that should pass through (adjusted age under 1 hour)
|
||||
// Note: Timestamp normalization (clock skew with sent_at) is now done in Rust capture service.
|
||||
// The timestamp field contains the already-normalized value.
|
||||
// sent_at is included in test data but should be ignored by plugin-server.
|
||||
|
||||
// Test events that should pass through (under 1 hour old after normalization)
|
||||
const eventsThatShouldPass = [
|
||||
createEvent({
|
||||
now: '2024-01-15T12:00:00.000Z',
|
||||
timestamp: '2024-01-15T11:00:00.000Z',
|
||||
sent_at: '2024-01-15T11:30:00.000Z',
|
||||
eventType: 'sent_at_30min_timestamp_1h',
|
||||
timestamp: '2024-01-15T11:30:00.000Z', // Normalized timestamp: 30 min old
|
||||
sent_at: '2024-01-15T11:00:00.000Z', // This should be IGNORED
|
||||
eventType: 'normalized_30min_old_with_sent_at',
|
||||
}),
|
||||
createEvent({
|
||||
now: '2024-01-15T12:00:00.000Z',
|
||||
timestamp: '2024-01-15T10:00:00.000Z',
|
||||
sent_at: '2024-01-15T11:00:00.000Z',
|
||||
eventType: 'sent_at_1h_timestamp_2h',
|
||||
timestamp: '2024-01-15T11:00:00.000Z', // Normalized timestamp: 1 hour old
|
||||
sent_at: '2024-01-15T10:00:00.000Z', // This should be IGNORED
|
||||
eventType: 'normalized_1h_old_with_sent_at',
|
||||
}),
|
||||
createEvent({
|
||||
now: '2024-01-15T12:00:00.000Z',
|
||||
timestamp: '2024-01-15T11:00:00.000Z',
|
||||
sent_at: '2024-01-15T11:01:00.000Z',
|
||||
eventType: 'sent_at_59min_timestamp_1h',
|
||||
}),
|
||||
createEvent({
|
||||
now: '2024-01-15T12:00:00.000Z',
|
||||
timestamp: '2024-01-15T10:59:50.000Z',
|
||||
sent_at: '2024-01-15T11:00:00.000Z',
|
||||
eventType: 'sent_at_1h_timestamp_1h_minus_10s',
|
||||
timestamp: '2024-01-15T11:00:01.000Z', // Normalized timestamp: just under 1 hour
|
||||
sent_at: '2024-01-15T09:00:00.000Z', // This should be IGNORED
|
||||
eventType: 'normalized_just_under_1h_with_sent_at',
|
||||
}),
|
||||
]
|
||||
|
||||
@@ -265,49 +263,19 @@ describe('dropOldEventsStep()', () => {
|
||||
expect(result).toEqual(event)
|
||||
}
|
||||
|
||||
// Test events with sent_at that should be dropped (adjusted age 1 hour or older)
|
||||
// Test events that should be dropped (over 1 hour old after normalization)
|
||||
const eventsThatShouldBeDropped = [
|
||||
createEvent({
|
||||
now: '2024-01-15T12:00:00.000Z',
|
||||
timestamp: '2024-01-15T09:59:59.000Z',
|
||||
sent_at: '2024-01-15T11:00:00.000Z',
|
||||
eventType: 'sent_at_2h1s_timestamp_1h',
|
||||
timestamp: '2024-01-15T10:59:59.000Z', // Normalized timestamp: just over 1 hour
|
||||
sent_at: '2024-01-15T11:30:00.000Z', // This should be IGNORED (would suggest event is recent if used)
|
||||
eventType: 'normalized_just_over_1h_with_sent_at',
|
||||
}),
|
||||
createEvent({
|
||||
now: '2024-01-15T12:00:00.000Z',
|
||||
timestamp: '2024-01-15T09:00:00.000Z',
|
||||
sent_at: '2024-01-15T11:00:00.000Z',
|
||||
eventType: 'sent_at_3h_timestamp_1h',
|
||||
}),
|
||||
createEvent({
|
||||
now: '2024-01-15T12:00:00.000Z',
|
||||
timestamp: '2024-01-15T08:00:00.000Z',
|
||||
sent_at: '2024-01-15T11:00:00.000Z',
|
||||
eventType: 'sent_at_4h_timestamp_1h',
|
||||
}),
|
||||
createEvent({
|
||||
now: '2024-01-15T12:00:00.000Z',
|
||||
timestamp: '2024-01-14T12:00:00.000Z',
|
||||
sent_at: '2024-01-15T11:00:00.000Z',
|
||||
eventType: 'sent_at_1day_timestamp_1h',
|
||||
}),
|
||||
createEvent({
|
||||
now: '2024-01-15T12:00:00.000Z',
|
||||
timestamp: '2024-01-10T12:00:00.000Z',
|
||||
sent_at: '2024-01-15T11:00:00.000Z',
|
||||
eventType: 'sent_at_5days_timestamp_1h',
|
||||
}),
|
||||
createEvent({
|
||||
now: '2024-01-15T12:00:00.000Z',
|
||||
timestamp: '2023-12-15T12:00:00.000Z',
|
||||
sent_at: '2024-01-15T11:00:00.000Z',
|
||||
eventType: 'sent_at_1month_timestamp_1h',
|
||||
}),
|
||||
createEvent({
|
||||
now: '2024-01-15T12:00:00.000Z',
|
||||
timestamp: '2022-01-15T12:00:00.000Z',
|
||||
sent_at: '2024-01-15T11:00:00.000Z',
|
||||
eventType: 'sent_at_2years_timestamp_1h',
|
||||
timestamp: '2024-01-15T10:00:00.000Z', // Normalized timestamp: 2 hours old
|
||||
sent_at: '2024-01-15T11:45:00.000Z', // This should be IGNORED (would suggest event is recent if used)
|
||||
eventType: 'normalized_2h_old_with_sent_at',
|
||||
}),
|
||||
]
|
||||
|
||||
@@ -317,16 +285,33 @@ describe('dropOldEventsStep()', () => {
|
||||
}
|
||||
})
|
||||
|
||||
it('handles events with offset', async () => {
|
||||
it('ignores offset and uses already-normalized timestamp from Rust', async () => {
|
||||
const team = createTeamWithDropSetting(86400) // 1 day threshold
|
||||
|
||||
// Test events with offset that should pass through (under 1 day old)
|
||||
// Note: Offset normalization is now done in Rust capture service.
|
||||
// The timestamp field contains the already-normalized value (now - offset).
|
||||
// offset is included in test data but should be ignored by plugin-server.
|
||||
|
||||
// Test events that should pass through (under 1 day old after normalization)
|
||||
const eventsThatShouldPass = [
|
||||
createEvent({ now: '2024-01-15T12:00:00.000Z', eventType: 'current_time_no_offset' }),
|
||||
createEvent({ now: '2024-01-15T12:00:00.000Z', offset: 60000, eventType: 'offset_1min' }), // 1 minute old
|
||||
createEvent({ now: '2024-01-15T12:00:00.000Z', offset: 3600000, eventType: 'offset_1h' }), // 1 hour old
|
||||
createEvent({ now: '2024-01-15T12:00:00.000Z', offset: 43200000, eventType: 'offset_12h' }), // 12 hours old
|
||||
createEvent({ now: '2024-01-15T12:00:00.000Z', offset: 86399000, eventType: 'offset_just_under_1day' }), // 23:59:59 old
|
||||
createEvent({
|
||||
now: '2024-01-15T12:00:00.000Z',
|
||||
timestamp: '2024-01-15T11:59:00.000Z', // Normalized: 1 min old
|
||||
offset: 3600000, // This should be IGNORED (would suggest 1 hour old if used)
|
||||
eventType: 'normalized_1min_old_with_offset',
|
||||
}),
|
||||
createEvent({
|
||||
now: '2024-01-15T12:00:00.000Z',
|
||||
timestamp: '2024-01-15T00:00:00.000Z', // Normalized: 12 hours old
|
||||
offset: 86400000, // This should be IGNORED (would suggest 1 day old if used)
|
||||
eventType: 'normalized_12h_old_with_offset',
|
||||
}),
|
||||
createEvent({
|
||||
now: '2024-01-15T12:00:00.000Z',
|
||||
timestamp: '2024-01-14T12:00:01.000Z', // Normalized: just under 1 day
|
||||
offset: 172800000, // This should be IGNORED (would suggest 2 days old if used)
|
||||
eventType: 'normalized_just_under_1day_with_offset',
|
||||
}),
|
||||
]
|
||||
|
||||
for (const event of eventsThatShouldPass) {
|
||||
@@ -334,13 +319,20 @@ describe('dropOldEventsStep()', () => {
|
||||
expect(result).toEqual(event)
|
||||
}
|
||||
|
||||
// Test events with offset that should be dropped (1 day old or older)
|
||||
// Test events that should be dropped (1 day old or older after normalization)
|
||||
const eventsThatShouldBeDropped = [
|
||||
createEvent({ now: '2024-01-15T12:00:00.000Z', offset: 86401000, eventType: 'offset_just_over_1day' }), // 1:00:01 old
|
||||
createEvent({ now: '2024-01-15T12:00:00.000Z', offset: 172800000, eventType: 'offset_2days' }), // 2 days old
|
||||
createEvent({ now: '2024-01-15T12:00:00.000Z', offset: 604800000, eventType: 'offset_1week' }), // 1 week old
|
||||
createEvent({ now: '2024-01-15T12:00:00.000Z', offset: 2592000000, eventType: 'offset_30days' }), // 30 days old
|
||||
createEvent({ now: '2024-01-15T12:00:00.000Z', offset: 31536000000, eventType: 'offset_1year' }), // 1 year old
|
||||
createEvent({
|
||||
now: '2024-01-15T12:00:00.000Z',
|
||||
timestamp: '2024-01-14T11:59:59.000Z', // Normalized: just over 1 day
|
||||
offset: 60000, // This should be IGNORED (would suggest 1 min old if used)
|
||||
eventType: 'normalized_just_over_1day_with_offset',
|
||||
}),
|
||||
createEvent({
|
||||
now: '2024-01-15T12:00:00.000Z',
|
||||
timestamp: '2024-01-13T12:00:00.000Z', // Normalized: 2 days old
|
||||
offset: 3600000, // This should be IGNORED (would suggest 1 hour old if used)
|
||||
eventType: 'normalized_2days_old_with_offset',
|
||||
}),
|
||||
]
|
||||
|
||||
for (const event of eventsThatShouldBeDropped) {
|
||||
@@ -415,9 +407,13 @@ describe('dropOldEventsStep()', () => {
|
||||
}
|
||||
})
|
||||
|
||||
it('handles invalid timestamps, offsets, and sent_at gracefully', async () => {
|
||||
it('handles invalid timestamps gracefully', async () => {
|
||||
const team = createTeamWithDropSetting(3600) // 1 hour threshold
|
||||
|
||||
// Note: Timestamp normalization is done in Rust, but we still need to handle
|
||||
// invalid timestamps gracefully in case they slip through.
|
||||
// Invalid timestamps should not cause the drop logic to fail - events should pass through.
|
||||
|
||||
const invalidTimestampEvents = [
|
||||
createEvent({
|
||||
now: '2024-01-15T12:00:00.000Z',
|
||||
@@ -432,51 +428,11 @@ describe('dropOldEventsStep()', () => {
|
||||
}),
|
||||
]
|
||||
|
||||
// Invalid timestamps should be handled gracefully and events should pass through
|
||||
for (const event of invalidTimestampEvents) {
|
||||
const result = await dropOldEventsStep(mockRunner, event, team)
|
||||
expect(result).toEqual(event)
|
||||
}
|
||||
|
||||
const invalidOffsetEvents = [
|
||||
createEvent({ now: '2024-01-15T12:00:00.000Z', offset: NaN, eventType: 'invalid_offset_nan' }),
|
||||
createEvent({
|
||||
now: '2024-01-15T12:00:00.000Z',
|
||||
offset: Infinity,
|
||||
eventType: 'invalid_offset_infinity',
|
||||
}),
|
||||
createEvent({ now: '2024-01-15T12:00:00.000Z', offset: -1, eventType: 'invalid_offset_negative' }),
|
||||
]
|
||||
|
||||
for (const event of invalidOffsetEvents) {
|
||||
const result = await dropOldEventsStep(mockRunner, event, team)
|
||||
expect(result).toEqual(event)
|
||||
}
|
||||
|
||||
const invalidSentAtEvents = [
|
||||
createEvent({
|
||||
now: '2024-01-15T12:00:00.000Z',
|
||||
timestamp: '2024-01-15T11:00:00.000Z',
|
||||
sent_at: 'invalid-sent-at',
|
||||
eventType: 'invalid_sent_at_event',
|
||||
}),
|
||||
createEvent({
|
||||
now: '2024-01-15T12:00:00.000Z',
|
||||
timestamp: '2024-01-15T11:00:00.000Z',
|
||||
sent_at: '',
|
||||
eventType: 'empty_sent_at_event',
|
||||
}),
|
||||
createEvent({
|
||||
now: '2024-01-15T12:00:00.000Z',
|
||||
timestamp: '2024-01-15T11:00:00.000Z',
|
||||
sent_at: '2024-13-45T25:70:99.999Z',
|
||||
eventType: 'invalid_sent_at_date_event',
|
||||
}),
|
||||
]
|
||||
|
||||
for (const event of invalidSentAtEvents) {
|
||||
const result = await dropOldEventsStep(mockRunner, event, team)
|
||||
expect(result).toEqual(event)
|
||||
}
|
||||
})
|
||||
})
|
||||
|
||||
|
||||
@@ -9,28 +9,44 @@ import {
|
||||
} from '../../../src/worker/ingestion/timestamps'
|
||||
|
||||
describe('parseDate()', () => {
|
||||
// Get local timezone offset for Oct 29, 2021 at midnight
|
||||
const testDate = new Date('2021-10-29T00:00:00')
|
||||
const offsetMinutes = testDate.getTimezoneOffset()
|
||||
const offsetHours = Math.abs(Math.floor(offsetMinutes / 60))
|
||||
const offsetMins = Math.abs(offsetMinutes % 60)
|
||||
const offsetSign = offsetMinutes <= 0 ? '+' : '-'
|
||||
const tzOffset = `${offsetSign}${String(offsetHours).padStart(2, '0')}:${String(offsetMins).padStart(2, '0')}`
|
||||
|
||||
// For timestamps without explicit timezone, they'll be interpreted in local time then converted to UTC
|
||||
// So '2021-10-29 00:00:00' in local time becomes '2021-10-29T00:00:00<local-offset>' in UTC
|
||||
const expectedLocalAsUTC = `2021-10-29T00:00:00.000${tzOffset}`
|
||||
const parsedExpected = parseDate(expectedLocalAsUTC)
|
||||
|
||||
// Note: '2021-10-29' (date-only) is treated as UTC by new Date(), not local time
|
||||
const expectedDateOnly = parseDate('2021-10-29T00:00:00.000Z')
|
||||
|
||||
const timestamps = [
|
||||
'2021-10-29',
|
||||
'2021-10-29 00:00:00',
|
||||
'2021-10-29 00:00:00.000000',
|
||||
'2021-10-29T00:00:00.000Z',
|
||||
'2021-10-29 00:00:00+00:00',
|
||||
'2021-10-29T00:00:00.000-00:00',
|
||||
'2021-10-29T00:00:00.000',
|
||||
'2021-10-29T00:00:00.000+00:00',
|
||||
'2021-W43-5',
|
||||
'2021-302',
|
||||
{ input: '2021-10-29', expected: expectedDateOnly }, // Date-only format is treated as UTC
|
||||
{ input: '2021-10-29 00:00:00', expected: parsedExpected },
|
||||
{ input: '2021-10-29 00:00:00.000000', expected: parsedExpected },
|
||||
{ input: '2021-10-29T00:00:00.000Z', expected: parseDate('2021-10-29T00:00:00.000Z') },
|
||||
{ input: '2021-10-29 00:00:00+00:00', expected: parseDate('2021-10-29T00:00:00.000Z') },
|
||||
{ input: '2021-10-29T00:00:00.000-00:00', expected: parseDate('2021-10-29T00:00:00.000Z') },
|
||||
{ input: '2021-10-29T00:00:00.000', expected: parsedExpected },
|
||||
{ input: '2021-10-29T00:00:00.000+00:00', expected: parseDate('2021-10-29T00:00:00.000Z') },
|
||||
{ input: '2021-W43-5', expected: parsedExpected },
|
||||
{ input: '2021-302', expected: parsedExpected },
|
||||
]
|
||||
|
||||
test.each(timestamps)('parses %s', (timestamp) => {
|
||||
const parsedTimestamp = parseDate(timestamp)
|
||||
expect(parsedTimestamp.year).toBe(2021)
|
||||
expect(parsedTimestamp.month).toBe(10)
|
||||
expect(parsedTimestamp.day).toBe(29)
|
||||
expect(parsedTimestamp.hour).toBe(0)
|
||||
expect(parsedTimestamp.minute).toBe(0)
|
||||
expect(parsedTimestamp.second).toBe(0)
|
||||
expect(parsedTimestamp.millisecond).toBe(0)
|
||||
test.each(timestamps)('parses $input', ({ input, expected }) => {
|
||||
const parsedTimestamp = parseDate(input)
|
||||
expect(parsedTimestamp.year).toBe(expected.year)
|
||||
expect(parsedTimestamp.month).toBe(expected.month)
|
||||
expect(parsedTimestamp.day).toBe(expected.day)
|
||||
expect(parsedTimestamp.hour).toBe(expected.hour)
|
||||
expect(parsedTimestamp.minute).toBe(expected.minute)
|
||||
expect(parsedTimestamp.second).toBe(expected.second)
|
||||
expect(parsedTimestamp.millisecond).toBe(expected.millisecond)
|
||||
})
|
||||
})
|
||||
|
||||
@@ -42,26 +58,11 @@ describe('parseEventTimestamp()', () => {
|
||||
jest.useRealTimers()
|
||||
})
|
||||
|
||||
it('captures sent_at to adjusts timestamp', () => {
|
||||
it('parses a valid timestamp', () => {
|
||||
// Timestamp normalization is now done in Rust capture service
|
||||
// This test verifies we correctly parse the already-normalized timestamp
|
||||
const event = {
|
||||
timestamp: '2021-10-30T03:02:00.000Z',
|
||||
sent_at: '2021-10-30T03:12:00.000Z',
|
||||
now: '2021-10-29T01:44:00.000Z',
|
||||
} as any as PluginEvent
|
||||
|
||||
const callbackMock = jest.fn()
|
||||
const timestamp = parseEventTimestamp(event, callbackMock)
|
||||
expect(callbackMock.mock.calls.length).toEqual(0)
|
||||
|
||||
expect(timestamp.toISO()).toEqual('2021-10-29T01:34:00.000Z')
|
||||
})
|
||||
|
||||
it('Ignores sent_at if $ignore_sent_at set', () => {
|
||||
const event = {
|
||||
properties: { $ignore_sent_at: true },
|
||||
timestamp: '2021-10-30T03:02:00.000Z',
|
||||
sent_at: '2021-10-30T03:12:00.000Z',
|
||||
now: '2021-11-29T01:44:00.000Z',
|
||||
} as any as PluginEvent
|
||||
|
||||
const callbackMock = jest.fn()
|
||||
@@ -71,91 +72,24 @@ describe('parseEventTimestamp()', () => {
|
||||
expect(timestamp.toISO()).toEqual('2021-10-30T03:02:00.000Z')
|
||||
})
|
||||
|
||||
it('ignores and reports invalid sent_at', () => {
|
||||
const event = {
|
||||
timestamp: '2021-10-31T00:44:00.000Z',
|
||||
sent_at: 'invalid',
|
||||
now: '2021-10-30T01:44:00.000Z',
|
||||
uuid: new UUIDT(),
|
||||
} as any as PluginEvent
|
||||
|
||||
const callbackMock = jest.fn()
|
||||
const timestamp = parseEventTimestamp(event, callbackMock)
|
||||
expect(callbackMock.mock.calls).toEqual([
|
||||
[
|
||||
'ignored_invalid_timestamp',
|
||||
{
|
||||
field: 'sent_at',
|
||||
reason: 'the input "invalid" can\'t be parsed as ISO 8601',
|
||||
value: 'invalid',
|
||||
eventUuid: event.uuid,
|
||||
},
|
||||
],
|
||||
])
|
||||
|
||||
expect(timestamp.toISO()).toEqual('2021-10-31T00:44:00.000Z')
|
||||
})
|
||||
|
||||
it('captures sent_at with timezone info', () => {
|
||||
it('parses timestamp with timezone info', () => {
|
||||
const event = {
|
||||
timestamp: '2021-10-30T03:02:00.000+04:00',
|
||||
sent_at: '2021-10-30T03:12:00.000+04:00',
|
||||
now: '2021-10-29T01:44:00.000Z',
|
||||
} as any as PluginEvent
|
||||
|
||||
const callbackMock = jest.fn()
|
||||
const timestamp = parseEventTimestamp(event, callbackMock)
|
||||
expect(callbackMock.mock.calls.length).toEqual(0)
|
||||
|
||||
expect(timestamp.toISO()).toEqual('2021-10-29T01:34:00.000Z')
|
||||
// Should be converted to UTC
|
||||
expect(timestamp.toISO()).toEqual('2021-10-29T23:02:00.000Z')
|
||||
})
|
||||
|
||||
it('captures timestamp with no sent_at', () => {
|
||||
it('handles out of bounds timestamps', () => {
|
||||
// Even though Rust normalizes, we still validate for safety
|
||||
// Year 10000 is out of bounds (> 9999) - luxon can't parse it
|
||||
const event = {
|
||||
timestamp: '2021-10-30T03:02:00.000Z',
|
||||
now: '2021-10-30T01:44:00.000Z',
|
||||
} as any as PluginEvent
|
||||
|
||||
const callbackMock = jest.fn()
|
||||
const timestamp = parseEventTimestamp(event, callbackMock)
|
||||
expect(callbackMock.mock.calls.length).toEqual(0)
|
||||
|
||||
expect(timestamp.toISO()).toEqual(event.timestamp)
|
||||
})
|
||||
|
||||
it('captures with time offset and ignores sent_at', () => {
|
||||
const event = {
|
||||
offset: 6000, // 6 seconds
|
||||
now: '2021-10-29T01:44:00.000Z',
|
||||
sent_at: '2021-10-30T03:12:00.000+04:00', // ignored
|
||||
} as any as PluginEvent
|
||||
|
||||
const callbackMock = jest.fn()
|
||||
const timestamp = parseEventTimestamp(event, callbackMock)
|
||||
expect(callbackMock.mock.calls.length).toEqual(0)
|
||||
|
||||
expect(timestamp.toUTC().toISO()).toEqual('2021-10-29T01:43:54.000Z')
|
||||
})
|
||||
|
||||
it('captures with time offset', () => {
|
||||
const event = {
|
||||
offset: 6000, // 6 seconds
|
||||
now: '2021-10-29T01:44:00.000Z',
|
||||
} as any as PluginEvent
|
||||
|
||||
const callbackMock = jest.fn()
|
||||
const timestamp = parseEventTimestamp(event, callbackMock)
|
||||
expect(callbackMock.mock.calls.length).toEqual(0)
|
||||
|
||||
expect(timestamp.toUTC().toISO()).toEqual('2021-10-29T01:43:54.000Z')
|
||||
})
|
||||
|
||||
it('timestamps adjusted way out of bounds are ignored', () => {
|
||||
const event = {
|
||||
offset: 600000000000000,
|
||||
timestamp: '2021-10-28T01:00:00.000Z',
|
||||
sent_at: '2021-10-28T01:05:00.000Z',
|
||||
now: '2021-10-28T01:10:00.000Z',
|
||||
timestamp: '10000-01-01T00:00:00.000Z',
|
||||
uuid: new UUIDT(),
|
||||
} as any as PluginEvent
|
||||
|
||||
@@ -167,14 +101,13 @@ describe('parseEventTimestamp()', () => {
|
||||
{
|
||||
field: 'timestamp',
|
||||
eventUuid: event.uuid,
|
||||
offset: 600000000000000,
|
||||
parsed_year: -16992,
|
||||
reason: 'out of bounds',
|
||||
value: '2021-10-28T01:00:00.000Z',
|
||||
reason: 'the input "10000-01-01T00:00:00.000Z" can\'t be parsed as ISO 8601',
|
||||
value: '10000-01-01T00:00:00.000Z',
|
||||
},
|
||||
],
|
||||
])
|
||||
|
||||
// Falls back to current time
|
||||
expect(timestamp.toUTC().toISO()).toEqual('2020-08-12T01:02:00.000Z')
|
||||
})
|
||||
|
||||
@@ -182,7 +115,6 @@ describe('parseEventTimestamp()', () => {
|
||||
const event = {
|
||||
team_id: 123,
|
||||
timestamp: 'notISO',
|
||||
now: '2020-01-01T12:00:05.200Z',
|
||||
uuid: new UUIDT(),
|
||||
} as any as PluginEvent
|
||||
|
||||
@@ -203,89 +135,17 @@ describe('parseEventTimestamp()', () => {
|
||||
expect(timestamp.toUTC().toISO()).toEqual('2020-08-12T01:02:00.000Z')
|
||||
})
|
||||
|
||||
it('reports event_timestamp_in_future with sent_at', () => {
|
||||
it('returns current time when no timestamp provided', () => {
|
||||
const event = {
|
||||
timestamp: '2021-10-29T02:30:00.000Z',
|
||||
sent_at: '2021-10-28T01:00:00.000Z',
|
||||
now: '2021-10-29T01:00:00.000Z',
|
||||
event: 'test event name',
|
||||
uuid: '12345678-1234-1234-1234-123456789abc',
|
||||
uuid: new UUIDT(),
|
||||
} as any as PluginEvent
|
||||
|
||||
const callbackMock = jest.fn()
|
||||
const timestamp = parseEventTimestamp(event, callbackMock)
|
||||
expect(callbackMock.mock.calls).toEqual([
|
||||
[
|
||||
'event_timestamp_in_future',
|
||||
{
|
||||
now: '2021-10-29T01:00:00.000Z',
|
||||
offset: '',
|
||||
result: '2021-10-30T02:30:00.000Z',
|
||||
sentAt: '2021-10-28T01:00:00.000Z',
|
||||
timestamp: '2021-10-29T02:30:00.000Z',
|
||||
eventUuid: '12345678-1234-1234-1234-123456789abc',
|
||||
eventName: 'test event name',
|
||||
},
|
||||
],
|
||||
])
|
||||
expect(callbackMock.mock.calls.length).toEqual(0)
|
||||
|
||||
expect(timestamp.toISO()).toEqual('2021-10-29T01:00:00.000Z')
|
||||
})
|
||||
|
||||
it('reports event_timestamp_in_future with $ignore_sent_at', () => {
|
||||
const event = {
|
||||
timestamp: '2021-10-29T02:30:00.000Z',
|
||||
now: '2021-09-29T01:00:00.000Z',
|
||||
event: 'test event name',
|
||||
uuid: '12345678-1234-1234-1234-123456789abc',
|
||||
} as any as PluginEvent
|
||||
|
||||
const callbackMock = jest.fn()
|
||||
const timestamp = parseEventTimestamp(event, callbackMock)
|
||||
expect(callbackMock.mock.calls).toEqual([
|
||||
[
|
||||
'event_timestamp_in_future',
|
||||
{
|
||||
now: '2021-09-29T01:00:00.000Z',
|
||||
offset: '',
|
||||
result: '2021-10-29T02:30:00.000Z',
|
||||
sentAt: '',
|
||||
timestamp: '2021-10-29T02:30:00.000Z',
|
||||
eventUuid: '12345678-1234-1234-1234-123456789abc',
|
||||
eventName: 'test event name',
|
||||
},
|
||||
],
|
||||
])
|
||||
expect(timestamp.toISO()).toEqual('2021-09-29T01:00:00.000Z')
|
||||
})
|
||||
|
||||
it('reports event_timestamp_in_future with negative offset', () => {
|
||||
const event = {
|
||||
offset: -82860000,
|
||||
now: '2021-10-29T01:00:00.000Z',
|
||||
event: 'test event name',
|
||||
uuid: '12345678-1234-1234-1234-123456789abc',
|
||||
} as any as PluginEvent
|
||||
|
||||
const callbackMock = jest.fn()
|
||||
const timestamp = parseEventTimestamp(event, callbackMock)
|
||||
|
||||
expect(callbackMock.mock.calls).toEqual([
|
||||
[
|
||||
'event_timestamp_in_future',
|
||||
{
|
||||
now: '2021-10-29T01:00:00.000Z',
|
||||
offset: -82860000,
|
||||
result: '2021-10-30T00:01:00.000Z',
|
||||
sentAt: '',
|
||||
timestamp: '',
|
||||
eventUuid: '12345678-1234-1234-1234-123456789abc',
|
||||
eventName: 'test event name',
|
||||
},
|
||||
],
|
||||
])
|
||||
|
||||
expect(timestamp.toISO()).toEqual('2021-10-29T01:00:00.000Z')
|
||||
// Should return current UTC time
|
||||
expect(timestamp.toUTC().toISO()).toEqual('2020-08-12T01:02:00.000Z')
|
||||
})
|
||||
})
|
||||
|
||||
|
||||
Reference in New Issue
Block a user