yindo/posthog
1
0
Fork 0
mirror of https://github.com/BillyOutlast/posthog.git synced 2026-02-04 03:01:23 +01:00
Code Issues Packages Projects Releases Wiki Activity

feat(cookieless): Change cookieless constants to assume up to 72 hour ingestion lag (#41489)

Browse Source 
Co-authored-by: greptile-apps[bot] <165735046+greptile-apps[bot]@users.noreply.github.com>
This commit is contained in:
Robbie
2025-11-14 00:43:18 +00:00
committed by GitHub
parent 35cc7e06e5
commit fac3adf0ae
6 changed files with 76 additions and 16 deletions
Download Patch File Download Diff File Expand all files Collapse all files

6
plugin-server/src/config/config.ts
View File

@@ -299,11 +299,11 @@ export function getDefaultConfig(): PluginsServerConfig {
COOKIELESS_FORCE_STATELESS_MODE: false,
COOKIELESS_DISABLED: false,
COOKIELESS_DELETE_EXPIRED_LOCAL_SALTS_INTERVAL_MS: 60 * 60 * 1000, // 1 hour
COOKIELESS_SESSION_TTL_SECONDS: 60 * 60 * 24, // 24 hours
COOKIELESS_SALT_TTL_SECONDS: 60 * 60 * 24, // 24 hours
COOKIELESS_SESSION_TTL_SECONDS: 60 * 60 * (72 + 24), // 96 hours (72 ingestion lag + 24 validity)
COOKIELESS_SALT_TTL_SECONDS: 60 * 60 * (72 + 24), // 96 hours (72 ingestion lag + 24 validity)
COOKIELESS_SESSION_INACTIVITY_MS: 30 * 60 * 1000, // 30 minutes
COOKIELESS_IDENTIFIES_TTL_SECONDS:
(24 + // max supported ingestion lag
(72 + // max supported ingestion lag in hours
12 + // max negative timezone in the world*/
14 + // max positive timezone in the world */
24) * // amount of time salt is valid in one timezone

54
plugin-server/src/ingestion/cookieless/cookieless-manager.test.ts
View File

@@ -22,6 +22,7 @@ import {
extractRootDomain,
getRedisIdentifiesKey,
hashToDistinctId,
isCalendarDateValid,
sessionStateToBuffer,
toYYYYMMDDInTimezoneSafe,
} from './cookieless-manager'
@@ -113,6 +114,59 @@ describe('CookielessManager', () => {
})
})
describe('isCalendarDateValid', () => {
const fixedTime = new Date('2025-11-13T12:00:00Z')
beforeEach(() => {
jest.useFakeTimers({ now: fixedTime })
})
afterEach(() => {
jest.useRealTimers()
})
it('should accept today', () => {
// Fixed time: 2025-11-13 12:00 UTC
expect(isCalendarDateValid('2025-11-13')).toBe(true)
})
it('should accept yesterday', () => {
// Salt window for 2025-11-12: Nov 11 12:00 to Nov 15 14:00
// NOW (Nov 13 12:00) is within window
expect(isCalendarDateValid('2025-11-12')).toBe(true)
})
it('should accept 3 days ago (within 72h + timezone buffer)', () => {
// Salt window for 2025-11-10: Nov 9 12:00 to Nov 13 14:00
// NOW (Nov 13 12:00) is within window
expect(isCalendarDateValid('2025-11-10')).toBe(true)
})
it('should reject 4 days ago (salt window expired)', () => {
// Salt window for 2025-11-09: Nov 8 12:00 to Nov 12 14:00
// NOW (Nov 13 12:00) is after window ended
expect(isCalendarDateValid('2025-11-09')).toBe(false)
})
it('should reject 5 days ago (salt window expired)', () => {
// Salt window for 2025-11-08: Nov 7 12:00 to Nov 11 14:00
// NOW (Nov 13 12:00) is well after window ended
expect(isCalendarDateValid('2025-11-08')).toBe(false)
})
it('should reject tomorrow-ish dates', () => {
// Salt window for 2025-11-08: Nov 7 12:00 to Nov 11 14:00
// NOW (Nov 13 12:00) is well after window ended
expect(isCalendarDateValid('2025-11-15')).toBe(false)
})
it('should reject invalid date format', () => {
expect(isCalendarDateValid('not-a-date')).toBe(false)
expect(isCalendarDateValid('2025/01/01')).toBe(false)
expect(isCalendarDateValid('2025-13-01')).toBe(false)
})
})
describe('pipeline step', () => {
let hub: Hub
let organizationId: string

3
plugin-server/src/ingestion/cookieless/cookieless-manager.ts
View File

@@ -83,6 +83,7 @@ export const COOKIELESS_MODE_FLAG_PROPERTY = '$cookieless_mode'
export const COOKIELESS_EXTRA_HASH_CONTENTS_PROPERTY = '$cookieless_extra'
const MAX_NEGATIVE_TIMEZONE_HOURS = 12
const MAX_POSITIVE_TIMEZONE_HOURS = 14
const MAX_SUPPORTED_INGESTION_LAG_HOURS = 72 // if changing this, you will also need to change the TTLs
interface CookielessConfig {
disabled: boolean
@@ -683,7 +684,7 @@ export function isCalendarDateValid(yyyymmdd: string): boolean {
startOfDayMinus12.setUTCHours(-MAX_NEGATIVE_TIMEZONE_HOURS) // Start at UTC−12
const endOfDayPlus14 = new Date(utcDate)
endOfDayPlus14.setUTCHours(MAX_POSITIVE_TIMEZONE_HOURS + 24) // End at UTC+14
endOfDayPlus14.setUTCHours(MAX_POSITIVE_TIMEZONE_HOURS + MAX_SUPPORTED_INGESTION_LAG_HOURS) // End at UTC+14 (72h ingestion lag buffer)
const isGteMinimum = nowUTC >= startOfDayMinus12
const isLtMaximum = nowUTC < endOfDayPlus14

13
rust/common/cookieless/src/constants.rs
View File

@@ -13,11 +13,12 @@ pub const MAX_NEGATIVE_TIMEZONE_HOURS: i32 = 12; // Baker Island, Howland Island
pub const MAX_POSITIVE_TIMEZONE_HOURS: i32 = 14; // Line Islands (UTC+14)
// Time constants
pub const SALT_TTL_SECONDS: u64 = 60 * 60 * 24; // 24 hours
pub const SESSION_TTL_SECONDS: u64 = 60 * 60 * 24; // 24 hours
pub const MAX_SUPPORTED_INGESTION_LAG_HOURS: i32 = 72;
pub const SALT_TTL_SECONDS: u64 = 60 * 60 * (MAX_SUPPORTED_INGESTION_LAG_HOURS as u64 + 24);
pub const SESSION_TTL_SECONDS: u64 = 60 * 60 * (MAX_SUPPORTED_INGESTION_LAG_HOURS as u64 + 24);
pub const SESSION_INACTIVITY_MS: u64 = 30 * 60 * 1000; // 30 minutes
const IDENTIFIES_TTL_HOURS: u64 = (24 // Time salt is valid within the same time zone
+ 24 // Max supported ingestion lag (hours)
+ MAX_NEGATIVE_TIMEZONE_HOURS
+ MAX_POSITIVE_TIMEZONE_HOURS) as u64;
const IDENTIFIES_TTL_HOURS: u64 = 24 // Time salt is valid within the same time zone
+ MAX_SUPPORTED_INGESTION_LAG_HOURS as u64
+ MAX_NEGATIVE_TIMEZONE_HOURS as u64
+ MAX_POSITIVE_TIMEZONE_HOURS as u64;
pub const IDENTIFIES_TTL_SECONDS: u64 = IDENTIFIES_TTL_HOURS * (60 * 60);

8
rust/common/cookieless/src/salt_cache.rs
View File

@@ -3,7 +3,8 @@ use std::sync::Arc;
use std::time::{Duration, SystemTime, UNIX_EPOCH};
use crate::constants::{
MAX_NEGATIVE_TIMEZONE_HOURS, MAX_POSITIVE_TIMEZONE_HOURS, SALT_TTL_SECONDS,
MAX_NEGATIVE_TIMEZONE_HOURS, MAX_POSITIVE_TIMEZONE_HOURS, MAX_SUPPORTED_INGESTION_LAG_HOURS,
SALT_TTL_SECONDS,
};
use crate::metrics::metrics_consts::{
COOKIELESS_CACHE_HIT_COUNTER, COOKIELESS_CACHE_MISS_COUNTER, COOKIELESS_REDIS_ERROR_COUNTER,
@@ -284,7 +285,10 @@ pub fn is_calendar_date_valid(yyyymmdd: &str) -> bool {
// Define the range of the calendar day in UTC
let start_of_day_minus_12 = utc_date - (i64::from(MAX_NEGATIVE_TIMEZONE_HOURS) * 3600 * 1000);
let end_of_day_plus_14 = utc_date + (i64::from(MAX_POSITIVE_TIMEZONE_HOURS + 24) * 3600 * 1000);
let end_of_day_plus_14 = utc_date
+ (i64::from(MAX_POSITIVE_TIMEZONE_HOURS + MAX_SUPPORTED_INGESTION_LAG_HOURS)
* 3600
* 1000);
// Check if the current UTC time falls within this range
now_utc >= start_of_day_minus_12 && now_utc < end_of_day_plus_14

8
rust/feature-flags/src/config.rs
View File

@@ -348,10 +348,10 @@ pub struct Config {
#[envconfig(from = "COOKIELESS_FORCE_STATELESS", default = "false")]
pub cookieless_force_stateless: bool,
#[envconfig(from = "COOKIELESS_IDENTIFIES_TTL_SECONDS", default = "7200")]
#[envconfig(from = "COOKIELESS_IDENTIFIES_TTL_SECONDS", default = "345600")]
pub cookieless_identifies_ttl_seconds: u64,
#[envconfig(from = "COOKIELESS_SALT_TTL_SECONDS", default = "86400")]
#[envconfig(from = "COOKIELESS_SALT_TTL_SECONDS", default = "345600")]
pub cookieless_salt_ttl_seconds: u64,
#[envconfig(from = "COOKIELESS_REDIS_HOST", default = "localhost")]
@@ -501,8 +501,8 @@ impl Config {
flags_cache_ttl_seconds: 432000,
cookieless_disabled: false,
cookieless_force_stateless: false,
cookieless_identifies_ttl_seconds: 7200,
cookieless_salt_ttl_seconds: 86400,
cookieless_identifies_ttl_seconds: 345600,
cookieless_salt_ttl_seconds: 345600,
cookieless_redis_host: "localhost".to_string(),
cookieless_redis_port: 6379,
new_analytics_capture_endpoint: "/i/v0/e/".to_string(),