refactor: split out persons postgres (#31099)

This commit is contained in:
Paweł Ledwoń
2025-05-01 19:47:46 +01:00
committed by GitHub
parent 88cd944d3a
commit 4e83591d19
18 changed files with 351 additions and 428 deletions

View File

@@ -183,6 +183,7 @@ jobs:
TEST: 'true'
SECRET_KEY: 'abcdef' # unsafe - for testing only
DATABASE_URL: 'postgres://posthog:posthog@localhost:5432/posthog'
PERSONS_DATABASE_URL: 'postgres://posthog:posthog@localhost:5432/posthog'
run: pnpm --filter=@posthog/plugin-server setup:test
- name: Test with Jest
@@ -190,6 +191,7 @@ jobs:
env:
# Below DB name has `test_` prepended, as that's how Django (ran above) creates the test DB
DATABASE_URL: 'postgres://posthog:posthog@localhost:5432/test_posthog'
PERSONS_DATABASE_URL: 'postgres://posthog:posthog@localhost:5432/test_posthog'
REDIS_URL: 'redis://localhost'
NODE_OPTIONS: '--max_old_space_size=4096'
SHARD_INDEX: ${{ matrix.shard }}
@@ -208,6 +210,7 @@ jobs:
CLICKHOUSE_DATABASE: 'posthog_test'
KAFKA_HOSTS: 'kafka:9092'
DATABASE_URL: 'postgres://posthog:posthog@localhost:5432/posthog'
PERSONS_DATABASE_URL: 'postgres://posthog:posthog@localhost:5432/posthog'
RELOAD_PLUGIN_JITTER_MAX_MS: 0
ENCRYPTION_SALT_KEYS: '00beef0000beef0000beef0000beef00'

View File

@@ -235,6 +235,7 @@ services:
restart: on-failure
environment:
DATABASE_URL: 'postgres://posthog:posthog@db:5432/posthog'
PERSONS_DATABASE_URL: 'postgres://posthog:posthog@db:5432/posthog'
KAFKA_HOSTS: 'kafka:9092'
REDIS_URL: 'redis://redis:6379/'
CLICKHOUSE_HOST: 'clickhouse'

View File

@@ -19,7 +19,7 @@ Let's get you developing the plugin server in no time:
1. Make sure that the plugin server is configured correctly (see [Configuration](#Configuration)).
The following settings need to be the same for the plugin server and the main server:
`DATABASE_URL`, `REDIS_URL`, `KAFKA_HOSTS`, `CLICKHOUSE_HOST`, `CLICKHOUSE_DATABASE`,
`DATABASE_URL`, `PERSONS_DATABASE_URL`, `REDIS_URL`, `KAFKA_HOSTS`, `CLICKHOUSE_HOST`, `CLICKHOUSE_DATABASE`,
`CLICKHOUSE_USER`, and `CLICKHOUSE_PASSWORD`. Their default values should work just fine in local
development though.
@@ -59,6 +59,7 @@ testing:
APP_METRICS_FLUSH_FREQUENCY_MS=0 \
CLICKHOUSE_DATABASE='default' \
DATABASE_URL=postgres://posthog:posthog@localhost:5432/test_posthog \
PERSONS_DATABASE_URL=postgres://posthog:posthog@localhost:5432/test_posthog \
PLUGINS_DEFAULT_LOG_LEVEL=0 \
RELOAD_PLUGIN_JITTER_MAX_MS=0 \
PLUGIN_SERVER_MODE=functional-tests \
@@ -68,6 +69,7 @@ testing:
```bash
CLICKHOUSE_DATABASE='default' \
DATABASE_URL=postgres://posthog:posthog@localhost:5432/test_posthog \
PERSONS_DATABASE_URL=postgres://posthog:posthog@localhost:5432/test_posthog \
pnpm functional_tests --watch
```
@@ -107,6 +109,7 @@ There's a multitude of settings you can use to control the plugin server. Use th
| Name | Description | Default value |
| -------------------------------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ------------------------------------- |
| DATABASE_URL | Postgres database URL | `'postgres://localhost:5432/posthog'` |
| PERSONS_DATABASE_URL | Postgres persons database URL | `'postgres://localhost:5432/posthog'` |
| REDIS_URL | Redis store URL | `'redis://localhost'` |
| BASE_DIR | base path for resolving local plugins | `'.'` |
| WORKER_CONCURRENCY | number of concurrent worker threads | `0` all cores |

View File

@@ -17,6 +17,7 @@ export PLUGINS_DEFAULT_LOG_LEVEL=0 # All logs, as debug logs are used in sy
export NODE_ENV=production-functional-tests
export PLUGIN_SERVER_MODE=functional-tests # running all capabilities is too slow
export DATABASE_URL='postgres://posthog:posthog@localhost:5432/posthog'
export PERSONS_DATABASE_URL='postgres://posthog:posthog@localhost:5432/posthog'
# Not important at all, but I like to see nice red/green for tests
export FORCE_COLOR=true
export ENCRYPTION_SALT_KEYS='00beef0000beef0000beef0000beef00'

View File

@@ -318,7 +318,7 @@ export const createGroup = async (
export const fetchPostgresPersons = async (teamId: number) => {
const { rows } = await postgres.query(
PostgresUse.COMMON_WRITE,
PostgresUse.PERSONS_WRITE,
`SELECT *
FROM posthog_person
WHERE team_id = $1`,

View File

@@ -23,6 +23,12 @@ export function getDefaultConfig(): PluginsServerConfig {
: '',
DATABASE_READONLY_URL: '',
PLUGIN_STORAGE_DATABASE_URL: '',
PERSONS_DATABASE_URL: isTestEnv()
? 'postgres://posthog:posthog@localhost:5432/test_posthog'
: isDevEnv()
? 'postgres://posthog:posthog@localhost:5432/posthog'
: '',
PERSONS_READONLY_DATABASE_URL: '',
POSTGRES_CONNECTION_POOL_SIZE: 10,
POSTHOG_DB_NAME: null,
POSTHOG_DB_USER: 'postgres',

View File

@@ -144,6 +144,8 @@ export interface PluginsServerConfig extends CdpConfig, IngestionConsumerConfig
TASK_TIMEOUT: number // how many seconds until tasks are timed out
DATABASE_URL: string // Postgres database URL
DATABASE_READONLY_URL: string // Optional read-only replica to the main Postgres database
PERSONS_DATABASE_URL: string // Optional read-write Postgres database for persons
PERSONS_READONLY_DATABASE_URL: string // Optional read-only replica to the persons Postgres database
PLUGIN_STORAGE_DATABASE_URL: string // Optional read-write Postgres database for plugin storage
POSTGRES_CONNECTION_POOL_SIZE: number
POSTHOG_DB_NAME: string | null

View File

@@ -492,7 +492,7 @@ export class DB {
}) as ClickHousePerson[]
} else if (database === Database.Postgres) {
return await this.postgres
.query<RawPerson>(PostgresUse.COMMON_WRITE, 'SELECT * FROM posthog_person', undefined, 'fetchPersons')
.query<RawPerson>(PostgresUse.PERSONS_WRITE, 'SELECT * FROM posthog_person', undefined, 'fetchPersons')
.then(({ rows }) => rows.map(this.toPerson))
} else {
throw new Error(`Can't fetch persons for database: ${database}`)
@@ -560,7 +560,7 @@ export class DB {
const values = [teamId, distinctId]
const { rows } = await this.postgres.query<RawPerson>(
options.useReadReplica ? PostgresUse.COMMON_READ : PostgresUse.COMMON_WRITE,
options.useReadReplica ? PostgresUse.PERSONS_READ : PostgresUse.PERSONS_WRITE,
queryString,
values,
'fetchPerson'
@@ -593,7 +593,7 @@ export class DB {
const personVersion = 0
const { rows } = await this.postgres.query<RawPerson>(
tx ?? PostgresUse.COMMON_WRITE,
tx ?? PostgresUse.PERSONS_WRITE,
`WITH inserted_person AS (
INSERT INTO posthog_person (
created_at, properties, properties_last_updated_at,
@@ -696,7 +696,7 @@ export class DB {
RETURNING *`
const { rows } = await this.postgres.query<RawPerson>(
tx ?? PostgresUse.COMMON_WRITE,
tx ?? PostgresUse.PERSONS_WRITE,
queryString,
values,
'updatePerson'
@@ -732,7 +732,7 @@ export class DB {
public async deletePerson(person: InternalPerson, tx?: TransactionClient): Promise<TopicMessage[]> {
const { rows } = await this.postgres.query<{ version: string }>(
tx ?? PostgresUse.COMMON_WRITE,
tx ?? PostgresUse.PERSONS_WRITE,
'DELETE FROM posthog_person WHERE team_id = $1 AND id = $2 RETURNING version',
[person.team_id, person.id],
'deletePerson'
@@ -773,7 +773,7 @@ export class DB {
).data as ClickHousePersonDistinctId2[]
} else if (database === Database.Postgres) {
const result = await this.postgres.query(
PostgresUse.COMMON_WRITE, // used in tests only
PostgresUse.PERSONS_WRITE, // used in tests only
'SELECT * FROM posthog_persondistinctid WHERE person_id=$1 AND team_id=$2 ORDER BY id',
[person.id, person.team_id],
'fetchDistinctIds'
@@ -794,7 +794,7 @@ export class DB {
public async addPersonlessDistinctId(teamId: number, distinctId: string): Promise<boolean> {
const result = await this.postgres.query(
PostgresUse.COMMON_WRITE,
PostgresUse.PERSONS_WRITE,
`
INSERT INTO posthog_personlessdistinctid (team_id, distinct_id, is_merged, created_at)
VALUES ($1, $2, false, now())
@@ -811,7 +811,7 @@ export class DB {
// ON CONFLICT ... DO NOTHING won't give us our RETURNING, so we have to do another SELECT
const existingResult = await this.postgres.query(
PostgresUse.COMMON_WRITE,
PostgresUse.PERSONS_WRITE,
`
SELECT is_merged
FROM posthog_personlessdistinctid
@@ -830,7 +830,7 @@ export class DB {
tx?: TransactionClient
): Promise<boolean> {
const result = await this.postgres.query(
tx ?? PostgresUse.COMMON_WRITE,
tx ?? PostgresUse.PERSONS_WRITE,
`
INSERT INTO posthog_personlessdistinctid (team_id, distinct_id, is_merged, created_at)
VALUES ($1, $2, true, now())
@@ -852,7 +852,7 @@ export class DB {
tx?: TransactionClient
): Promise<TopicMessage[]> {
const insertResult = await this.postgres.query(
tx ?? PostgresUse.COMMON_WRITE,
tx ?? PostgresUse.PERSONS_WRITE,
// NOTE: Keep this in sync with the posthog_persondistinctid INSERT in `createPerson`
'INSERT INTO posthog_persondistinctid (distinct_id, person_id, team_id, version) VALUES ($1, $2, $3, $4) RETURNING *',
[distinctId, person.id, person.team_id, version],
@@ -887,7 +887,7 @@ export class DB {
let movedDistinctIdResult: QueryResult<any> | null = null
try {
movedDistinctIdResult = await this.postgres.query(
tx ?? PostgresUse.COMMON_WRITE,
tx ?? PostgresUse.PERSONS_WRITE,
`
UPDATE posthog_persondistinctid
SET person_id = $1, version = COALESCE(version, 0)::numeric + 1

View File

@@ -14,6 +14,8 @@ export enum PostgresUse {
COMMON_READ, // Read replica on the common tables, uses need to account for possible replication delay
COMMON_WRITE, // Main PG master with common tables, we need to move as many queries away from it as possible
PLUGIN_STORAGE_RW, // Plugin Storage table, no read replica for it
PERSONS_READ, // Person database, read replica
PERSONS_WRITE, // Person database, write
}
export class TransactionClient {
@@ -44,6 +46,7 @@ export class PostgresRouter {
[PostgresUse.COMMON_WRITE, commonClient],
[PostgresUse.COMMON_READ, commonClient],
[PostgresUse.PLUGIN_STORAGE_RW, commonClient],
[PostgresUse.PERSONS_WRITE, commonClient],
])
if (serverConfig.DATABASE_READONLY_URL) {
@@ -70,6 +73,33 @@ export class PostgresRouter {
)
logger.info('👍', `Plugin-storage Postgresql ready`)
}
if (serverConfig.PERSONS_DATABASE_URL) {
logger.info('🤔', `Connecting to persons Postgresql...`)
this.pools.set(
PostgresUse.PERSONS_WRITE,
createPostgresPool(
serverConfig.PERSONS_DATABASE_URL,
serverConfig.POSTGRES_CONNECTION_POOL_SIZE,
app_name
)
)
logger.info('👍', `Persons Postgresql ready`)
}
if (serverConfig.PERSONS_READONLY_DATABASE_URL) {
logger.info('🤔', `Connecting to persons read-only Postgresql...`)
this.pools.set(
PostgresUse.PERSONS_READ,
createPostgresPool(
serverConfig.PERSONS_READONLY_DATABASE_URL,
serverConfig.POSTGRES_CONNECTION_POOL_SIZE,
app_name
)
)
logger.info('👍', `Persons read-only Postgresql ready`)
} else {
this.pools.set(PostgresUse.PERSONS_READ, this.pools.get(PostgresUse.PERSONS_WRITE)!)
logger.info('👍', `Using persons write pool for read-only`)
}
}
public async query<R extends QueryResultRow = any, I extends any[] = any[]>(

View File

@@ -18,7 +18,7 @@ import {
PropertyOperator,
StringMatching,
} from '../../types'
import { PostgresRouter, PostgresUse } from '../../utils/db/postgres'
import { PostgresRouter } from '../../utils/db/postgres'
import { stringToBoolean } from '../../utils/env-utils'
import { mutatePostIngestionEventWithElementsList } from '../../utils/event'
import { captureException } from '../../utils/posthog'
@@ -139,12 +139,10 @@ export class ActionMatcher {
}
/** Get all actions matched to the event. */
public async match(event: PostIngestionEvent): Promise<Action[]> {
public match(event: PostIngestionEvent): Action[] {
const matchingStart = new Date()
const teamActions: Action[] = Object.values(this.actionManager.getTeamActions(event.teamId))
const teamActionsMatching: boolean[] = await Promise.all(
teamActions.map((action) => this.checkAction(event, action))
)
const teamActionsMatching: boolean[] = teamActions.map((action) => this.checkAction(event, action))
const matches: Action[] = []
for (let i = 0; i < teamActionsMatching.length; i++) {
if (teamActionsMatching[i]) {
@@ -165,10 +163,10 @@ export class ActionMatcher {
* Return whether the event is a match for the action.
* The event is considered a match if any of the action's steps (match groups) is a match.
*/
public async checkAction(event: PostIngestionEvent, action: Action): Promise<boolean> {
public checkAction(event: PostIngestionEvent, action: Action): boolean {
for (const step of action.steps) {
try {
if (await this.checkStep(event, step)) {
if (this.checkStep(event, step)) {
return true
}
} catch (error) {
@@ -196,13 +194,13 @@ export class ActionMatcher {
* Return whether the event is a match for the step (match group).
* The event is considered a match if no subcheck fails. Many subchecks are usually irrelevant and skipped.
*/
private async checkStep(event: PostIngestionEvent, step: ActionStep): Promise<boolean> {
private checkStep(event: PostIngestionEvent, step: ActionStep): boolean {
return (
this.checkStepUrl(event, step) &&
this.checkStepEvent(event, step) &&
// The below checks are less performant may parse the elements chain or do a database query hence moved to the end
this.checkStepElement(event, step) &&
(await this.checkStepFilters(event, step))
this.checkStepFilters(event, step)
)
}
@@ -287,12 +285,12 @@ export class ActionMatcher {
* Return whether the event is a match for the step's fiter constraints.
* Step property: `properties`.
*/
private async checkStepFilters(event: PostIngestionEvent, step: ActionStep): Promise<boolean> {
private checkStepFilters(event: PostIngestionEvent, step: ActionStep): boolean {
// CHECK CONDITIONS, OTHERWISE SKIPPED, OTHERWISE SKIPPED
if (step.properties && step.properties.length) {
// EVERY FILTER MUST BE A MATCH
for (const filter of step.properties) {
if (!(await this.checkEventAgainstFilterAsync(event, filter))) {
if (!this.checkEventAgainstFilterAsync(event, filter)) {
return false
}
}
@@ -319,7 +317,7 @@ export class ActionMatcher {
/**
* Sublevel 3 of action matching.
*/
private async checkEventAgainstFilterAsync(event: PostIngestionEvent, filter: PropertyFilter): Promise<boolean> {
private checkEventAgainstFilterAsync(event: PostIngestionEvent, filter: PropertyFilter): boolean {
const match = this.checkEventAgainstFilterSync(event, filter)
if (match) {
@@ -328,7 +326,7 @@ export class ActionMatcher {
switch (filter.type) {
case 'cohort':
return await this.checkEventAgainstCohortFilter(event, filter)
return this.checkEventAgainstCohortFilter(event, filter)
default:
return false
}
@@ -368,10 +366,7 @@ export class ActionMatcher {
/**
* Sublevel 4 of action matching.
*/
private async checkEventAgainstCohortFilter(
event: PostIngestionEvent,
filter: CohortPropertyFilter
): Promise<boolean> {
private checkEventAgainstCohortFilter(event: PostIngestionEvent, filter: CohortPropertyFilter): boolean {
let cohortId = filter.value
if (cohortId === 'all') {
// The "All users" cohort matches anyone
@@ -386,25 +381,7 @@ export class ActionMatcher {
if (isNaN(cohortId)) {
throw new Error(`Can't match against invalid cohort ID value "${filter.value}!"`)
}
return await this.doesPersonBelongToCohort(Number(filter.value), event.person_id, event.teamId)
}
public async doesPersonBelongToCohort(cohortId: number, personUuid: string, teamId: number): Promise<boolean> {
const psqlResult = await this.postgres.query(
PostgresUse.COMMON_READ,
`
SELECT count(1) AS count
FROM posthog_cohortpeople
JOIN posthog_cohort ON (posthog_cohort.id = posthog_cohortpeople.cohort_id)
JOIN (SELECT * FROM posthog_person where team_id = $3) AS posthog_person_in_team ON (posthog_cohortpeople.person_id = posthog_person_in_team.id)
WHERE cohort_id=$1
AND posthog_person_in_team.uuid=$2
AND posthog_cohortpeople.version IS NOT DISTINCT FROM posthog_cohort.version
`,
[cohortId, personUuid, teamId],
'doesPersonBelongToCohort'
)
return psqlResult.rows[0].count > 0
return false
}
/**

View File

@@ -37,9 +37,7 @@ export async function processWebhooksStep(
actionMatcher: ActionMatcher,
hookCannon: HookCommander
) {
const actionMatches = await instrumentWebhookStep('actionMatching', async () => {
return await actionMatcher.match(event)
})
const actionMatches = actionMatcher.match(event)
await instrumentWebhookStep('findAndfireHooks', async () => {
await hookCannon.findAndFireHooks(event, actionMatches)
})

View File

@@ -34,23 +34,33 @@ export interface ExtraDatabaseRows {
}
// Reset the tables with some truncated first if we have issues regarding foreign keys
export const POSTGRES_DELETE_TABLES_QUERY = `
DO $$
DECLARE
r RECORD;
export const POSTGRES_DELETE_PERSON_TABLES_QUERY = `
DO $$
BEGIN
DELETE FROM posthog_persondistinctid CASCADE;
DELETE FROM posthog_person CASCADE;
END $$;
`
export const POSTGRES_DELETE_PRE_PERSON_TABLES_QUERY = `
DO $$
BEGIN
-- Delete from tables in order of dependencies
DELETE FROM posthog_featureflaghashkeyoverride CASCADE;
DELETE FROM posthog_cohortpeople CASCADE;
DELETE FROM posthog_cohort CASCADE;
DELETE FROM posthog_featureflag CASCADE;
DELETE FROM posthog_persondistinctid CASCADE;
DELETE FROM posthog_person CASCADE;
END $$;
`
export const POSTGRES_DELETE_OTHER_TABLES_QUERY = `
DO $$
DECLARE
r RECORD;
BEGIN
-- Then handle remaining tables
FOR r IN (
SELECT tablename
FROM pg_tables
SELECT tablename
FROM pg_tables
WHERE schemaname = current_schema()
AND tablename NOT IN ('posthog_persondistinctid', 'posthog_person')
) LOOP
@@ -67,10 +77,30 @@ export async function resetTestDatabase(
): Promise<void> {
const config = { ...defaultConfig, ...extraServerConfig, POSTGRES_CONNECTION_POOL_SIZE: 1 }
const db = new PostgresRouter(config)
await db.query(PostgresUse.COMMON_WRITE, POSTGRES_DELETE_TABLES_QUERY, undefined, 'delete-tables').catch((e) => {
console.error('Error deleting tables', e)
throw e
})
// Delete pre-person tables using COMMON_WRITE
await db
.query(PostgresUse.COMMON_WRITE, POSTGRES_DELETE_PRE_PERSON_TABLES_QUERY, undefined, 'delete-pre-person-tables')
.catch((e) => {
console.error('Error deleting pre-person tables', e)
throw e
})
// Delete person tables using PERSONS_WRITE
await db
.query(PostgresUse.PERSONS_WRITE, POSTGRES_DELETE_PERSON_TABLES_QUERY, undefined, 'delete-person-tables')
.catch((e) => {
console.error('Error deleting person tables', e)
throw e
})
// Delete other tables using COMMON_WRITE
await db
.query(PostgresUse.COMMON_WRITE, POSTGRES_DELETE_OTHER_TABLES_QUERY, undefined, 'delete-other-tables')
.catch((e) => {
console.error('Error deleting other tables', e)
throw e
})
const mocks = makePluginObjects(code)
const teamIds = mocks.pluginConfigRows.map((c) => c.team_id)
@@ -441,7 +471,7 @@ export const createOrganizationMembership = async (pg: PostgresRouter, organizat
export async function fetchPostgresPersons(db: DB, teamId: number) {
const query = `SELECT * FROM posthog_person WHERE team_id = ${teamId} ORDER BY id`
return (await db.postgres.query(PostgresUse.COMMON_READ, query, undefined, 'persons')).rows.map(
return (await db.postgres.query(PostgresUse.PERSONS_READ, query, undefined, 'persons')).rows.map(
// NOTE: we map to update some values here to maintain
// compatibility with `hub.db.fetchPersons`.
// TODO: remove unnecessary property translation operation.

View File

@@ -267,7 +267,7 @@ describe('DB', () => {
async function fetchPersonByPersonId(teamId: number, personId: InternalPerson['id']): Promise<Person | undefined> {
const selectResult = await db.postgres.query(
PostgresUse.COMMON_WRITE,
PostgresUse.PERSONS_WRITE,
`SELECT * FROM posthog_person WHERE team_id = $1 AND id = $2`,
[teamId, personId],
'fetchPersonByPersonId'
@@ -284,7 +284,7 @@ describe('DB', () => {
await db.addPersonlessDistinctId(team.id, 'addPersonlessDistinctId')
const result = await db.postgres.query(
PostgresUse.COMMON_WRITE,
PostgresUse.PERSONS_WRITE,
'SELECT id FROM posthog_personlessdistinctid WHERE team_id = $1 AND distinct_id = $2',
[team.id, 'addPersonlessDistinctId'],
'addPersonlessDistinctId'

View File

@@ -34,7 +34,8 @@ import {
createPlugin,
createPluginConfig,
createTeam,
POSTGRES_DELETE_TABLES_QUERY,
POSTGRES_DELETE_OTHER_TABLES_QUERY,
POSTGRES_DELETE_PERSON_TABLES_QUERY,
} from '../../helpers/sql'
jest.setTimeout(10000)
@@ -56,7 +57,13 @@ describe('runAppsOnEventPipeline()', () => {
jest.useFakeTimers({ advanceTimers: true })
hub = await createHub()
redis = await hub.redisPool.acquire()
await hub.postgres.query(PostgresUse.COMMON_WRITE, POSTGRES_DELETE_TABLES_QUERY, [], 'deleteTables') // Need to clear the DB to avoid unique constraint violations on ids
await hub.postgres.query(
PostgresUse.PERSONS_WRITE,
POSTGRES_DELETE_PERSON_TABLES_QUERY,
[],
'deletePersonTables'
) // Need to clear the DB to avoid unique constraint violations on ids
await hub.postgres.query(PostgresUse.COMMON_WRITE, POSTGRES_DELETE_OTHER_TABLES_QUERY, [], 'deleteTables') // Need to clear the DB to avoid unique constraint violations on ids
})
afterEach(async () => {

View File

@@ -9,7 +9,12 @@ import { closeHub, createHub } from '../../../src/utils/db/hub'
import { PostgresUse } from '../../../src/utils/db/postgres'
import { UUIDT } from '../../../src/utils/utils'
import { EventPipelineRunner } from '../../../src/worker/ingestion/event-pipeline/runner'
import { createOrganization, createTeam, POSTGRES_DELETE_TABLES_QUERY } from '../../helpers/sql'
import {
createOrganization,
createTeam,
POSTGRES_DELETE_OTHER_TABLES_QUERY,
POSTGRES_DELETE_PERSON_TABLES_QUERY,
} from '../../helpers/sql'
describe('workerTasks.runEventPipeline()', () => {
let hub: Hub
@@ -19,7 +24,8 @@ describe('workerTasks.runEventPipeline()', () => {
beforeAll(async () => {
hub = await createHub()
redis = await hub.redisPool.acquire()
await hub.postgres.query(PostgresUse.COMMON_WRITE, POSTGRES_DELETE_TABLES_QUERY, undefined, '') // Need to clear the DB to avoid unique constraint violations on ids
await hub.postgres.query(PostgresUse.PERSONS_WRITE, POSTGRES_DELETE_PERSON_TABLES_QUERY, undefined, '') // Need to clear the DB to avoid unique constraint violations on ids
await hub.postgres.query(PostgresUse.COMMON_WRITE, POSTGRES_DELETE_OTHER_TABLES_QUERY, undefined, '') // Need to clear the DB to avoid unique constraint violations on ids
process.env = { ...OLD_ENV } // Make a copy
})

View File

@@ -3,23 +3,19 @@ import { DateTime } from 'luxon'
import {
Action,
ActionStep,
Cohort,
Element,
Hub,
InternalPerson,
ISOTimestamp,
PostIngestionEvent,
PropertyOperator,
RawAction,
StringMatching,
Team,
} from '../../../src/types'
import { closeHub, createHub } from '../../../src/utils/db/hub'
import { UUIDT } from '../../../src/utils/utils'
import { ActionManager } from '../../../src/worker/ingestion/action-manager'
import { ActionMatcher, castingCompare } from '../../../src/worker/ingestion/action-matcher'
import { commonUserId } from '../../helpers/plugins'
import { getFirstTeam, insertRow, resetTestDatabase } from '../../helpers/sql'
import { insertRow, resetTestDatabase } from '../../helpers/sql'
jest.mock('../../../src/utils/logger')
@@ -111,7 +107,7 @@ describe('ActionMatcher', () => {
const event = createTestEvent()
expect(await actionMatcher.match(event)).toEqual([])
expect(actionMatcher.match(event)).toEqual([])
})
it('returns a match in case of event property operator exact', async () => {
@@ -138,23 +134,20 @@ describe('ActionMatcher', () => {
const eventFooTrue = createTestEvent({ properties: { foo: true } })
const eventFooNull = createTestEvent({ properties: { foo: null } })
expect(await actionMatcher.match(eventFooBar)).toEqual([
expect(actionMatcher.match(eventFooBar)).toEqual([actionDefinitionOpExact, actionDefinitionOpUndefined])
expect(actionMatcher.match(eventFooBarPolPot)).toEqual([
actionDefinitionOpExact,
actionDefinitionOpUndefined,
])
expect(await actionMatcher.match(eventFooBarPolPot)).toEqual([
actionDefinitionOpExact,
actionDefinitionOpUndefined,
])
expect(await actionMatcher.match(eventFooBaR)).toEqual([])
expect(await actionMatcher.match(eventFooBaz)).toEqual([])
expect(await actionMatcher.match(eventFooBarabara)).toEqual([])
expect(await actionMatcher.match(eventFooRabarbar)).toEqual([])
expect(await actionMatcher.match(eventFooNumber)).toEqual([])
expect(await actionMatcher.match(eventNoNothing)).toEqual([])
expect(await actionMatcher.match(eventFigNumber)).toEqual([])
expect(await actionMatcher.match(eventFooTrue)).toEqual([])
expect(await actionMatcher.match(eventFooNull)).toEqual([])
expect(actionMatcher.match(eventFooBaR)).toEqual([])
expect(actionMatcher.match(eventFooBaz)).toEqual([])
expect(actionMatcher.match(eventFooBarabara)).toEqual([])
expect(actionMatcher.match(eventFooRabarbar)).toEqual([])
expect(actionMatcher.match(eventFooNumber)).toEqual([])
expect(actionMatcher.match(eventNoNothing)).toEqual([])
expect(actionMatcher.match(eventFigNumber)).toEqual([])
expect(actionMatcher.match(eventFooTrue)).toEqual([])
expect(actionMatcher.match(eventFooNull)).toEqual([])
})
it('returns a match in case of event property operator is not', async () => {
@@ -176,17 +169,17 @@ describe('ActionMatcher', () => {
const eventFooTrue = createTestEvent({ properties: { foo: true } })
const eventFooNull = createTestEvent({ properties: { foo: null } })
expect(await actionMatcher.match(eventFooBar)).toEqual([])
expect(await actionMatcher.match(eventFooBarPolPot)).toEqual([])
expect(await actionMatcher.match(eventFooBaR)).toEqual([actionDefinitionOpIsNot])
expect(await actionMatcher.match(eventFooBaz)).toEqual([actionDefinitionOpIsNot])
expect(await actionMatcher.match(eventFooBarabara)).toEqual([actionDefinitionOpIsNot])
expect(await actionMatcher.match(eventFooRabarbar)).toEqual([actionDefinitionOpIsNot])
expect(await actionMatcher.match(eventFooNumber)).toEqual([actionDefinitionOpIsNot])
expect(await actionMatcher.match(eventNoNothing)).toEqual([actionDefinitionOpIsNot])
expect(await actionMatcher.match(eventFigNumber)).toEqual([actionDefinitionOpIsNot])
expect(await actionMatcher.match(eventFooTrue)).toEqual([actionDefinitionOpIsNot])
expect(await actionMatcher.match(eventFooNull)).toEqual([actionDefinitionOpIsNot])
expect(actionMatcher.match(eventFooBar)).toEqual([])
expect(actionMatcher.match(eventFooBarPolPot)).toEqual([])
expect(actionMatcher.match(eventFooBaR)).toEqual([actionDefinitionOpIsNot])
expect(actionMatcher.match(eventFooBaz)).toEqual([actionDefinitionOpIsNot])
expect(actionMatcher.match(eventFooBarabara)).toEqual([actionDefinitionOpIsNot])
expect(actionMatcher.match(eventFooRabarbar)).toEqual([actionDefinitionOpIsNot])
expect(actionMatcher.match(eventFooNumber)).toEqual([actionDefinitionOpIsNot])
expect(actionMatcher.match(eventNoNothing)).toEqual([actionDefinitionOpIsNot])
expect(actionMatcher.match(eventFigNumber)).toEqual([actionDefinitionOpIsNot])
expect(actionMatcher.match(eventFooTrue)).toEqual([actionDefinitionOpIsNot])
expect(actionMatcher.match(eventFooNull)).toEqual([actionDefinitionOpIsNot])
})
it('returns a match in case of event property operator contains', async () => {
@@ -210,17 +203,17 @@ describe('ActionMatcher', () => {
const eventFooTrue = createTestEvent({ properties: { foo: true } })
const eventFooNull = createTestEvent({ properties: { foo: null } })
expect(await actionMatcher.match(eventFooBar)).toEqual([actionDefinitionOpContains])
expect(await actionMatcher.match(eventFooBarPolPot)).toEqual([actionDefinitionOpContains])
expect(await actionMatcher.match(eventFooBaR)).toEqual([actionDefinitionOpContains])
expect(await actionMatcher.match(eventFooBaz)).toEqual([])
expect(await actionMatcher.match(eventFooBarabara)).toEqual([actionDefinitionOpContains])
expect(await actionMatcher.match(eventFooRabarbar)).toEqual([actionDefinitionOpContains])
expect(await actionMatcher.match(eventFooNumber)).toEqual([])
expect(await actionMatcher.match(eventNoNothing)).toEqual([])
expect(await actionMatcher.match(eventFigNumber)).toEqual([])
expect(await actionMatcher.match(eventFooTrue)).toEqual([])
expect(await actionMatcher.match(eventFooNull)).toEqual([])
expect(actionMatcher.match(eventFooBar)).toEqual([actionDefinitionOpContains])
expect(actionMatcher.match(eventFooBarPolPot)).toEqual([actionDefinitionOpContains])
expect(actionMatcher.match(eventFooBaR)).toEqual([actionDefinitionOpContains])
expect(actionMatcher.match(eventFooBaz)).toEqual([])
expect(actionMatcher.match(eventFooBarabara)).toEqual([actionDefinitionOpContains])
expect(actionMatcher.match(eventFooRabarbar)).toEqual([actionDefinitionOpContains])
expect(actionMatcher.match(eventFooNumber)).toEqual([])
expect(actionMatcher.match(eventNoNothing)).toEqual([])
expect(actionMatcher.match(eventFigNumber)).toEqual([])
expect(actionMatcher.match(eventFooTrue)).toEqual([])
expect(actionMatcher.match(eventFooNull)).toEqual([])
})
it('returns a match in case of event property operator does not contain', async () => {
@@ -244,17 +237,17 @@ describe('ActionMatcher', () => {
const eventFooTrue = createTestEvent({ properties: { foo: true } })
const eventFooNull = createTestEvent({ properties: { foo: null } })
expect(await actionMatcher.match(eventFooBar)).toEqual([])
expect(await actionMatcher.match(eventFooBarPolPot)).toEqual([])
expect(await actionMatcher.match(eventFooBaR)).toEqual([])
expect(await actionMatcher.match(eventFooBaz)).toEqual([actionDefinitionOpNotContains])
expect(await actionMatcher.match(eventFooBarabara)).toEqual([])
expect(await actionMatcher.match(eventFooRabarbar)).toEqual([])
expect(await actionMatcher.match(eventFooNumber)).toEqual([actionDefinitionOpNotContains])
expect(await actionMatcher.match(eventNoNothing)).toEqual([actionDefinitionOpNotContains])
expect(await actionMatcher.match(eventFigNumber)).toEqual([actionDefinitionOpNotContains])
expect(await actionMatcher.match(eventFooTrue)).toEqual([actionDefinitionOpNotContains])
expect(await actionMatcher.match(eventFooNull)).toEqual([actionDefinitionOpNotContains])
expect(actionMatcher.match(eventFooBar)).toEqual([])
expect(actionMatcher.match(eventFooBarPolPot)).toEqual([])
expect(actionMatcher.match(eventFooBaR)).toEqual([])
expect(actionMatcher.match(eventFooBaz)).toEqual([actionDefinitionOpNotContains])
expect(actionMatcher.match(eventFooBarabara)).toEqual([])
expect(actionMatcher.match(eventFooRabarbar)).toEqual([])
expect(actionMatcher.match(eventFooNumber)).toEqual([actionDefinitionOpNotContains])
expect(actionMatcher.match(eventNoNothing)).toEqual([actionDefinitionOpNotContains])
expect(actionMatcher.match(eventFigNumber)).toEqual([actionDefinitionOpNotContains])
expect(actionMatcher.match(eventFooTrue)).toEqual([actionDefinitionOpNotContains])
expect(actionMatcher.match(eventFooNull)).toEqual([actionDefinitionOpNotContains])
})
it('returns a match in case of event property operator regex', async () => {
@@ -283,20 +276,17 @@ describe('ActionMatcher', () => {
const eventFooTrue = createTestEvent({ properties: { foo: true } })
const eventFooNull = createTestEvent({ properties: { foo: null } })
expect(await actionMatcher.match(eventFooBar)).toEqual([actionDefinitionOpRegex1])
expect(await actionMatcher.match(eventFooBarPolPot)).toEqual([actionDefinitionOpRegex1])
expect(await actionMatcher.match(eventFooBaR)).toEqual([actionDefinitionOpRegex2])
expect(await actionMatcher.match(eventFooBaz)).toEqual([])
expect(await actionMatcher.match(eventFooBarabara)).toEqual([
actionDefinitionOpRegex1,
actionDefinitionOpRegex2,
])
expect(await actionMatcher.match(eventFooRabarbar)).toEqual([actionDefinitionOpRegex2])
expect(await actionMatcher.match(eventFooNumber)).toEqual([])
expect(await actionMatcher.match(eventNoNothing)).toEqual([])
expect(await actionMatcher.match(eventFigNumber)).toEqual([])
expect(await actionMatcher.match(eventFooTrue)).toEqual([])
expect(await actionMatcher.match(eventFooNull)).toEqual([])
expect(actionMatcher.match(eventFooBar)).toEqual([actionDefinitionOpRegex1])
expect(actionMatcher.match(eventFooBarPolPot)).toEqual([actionDefinitionOpRegex1])
expect(actionMatcher.match(eventFooBaR)).toEqual([actionDefinitionOpRegex2])
expect(actionMatcher.match(eventFooBaz)).toEqual([])
expect(actionMatcher.match(eventFooBarabara)).toEqual([actionDefinitionOpRegex1, actionDefinitionOpRegex2])
expect(actionMatcher.match(eventFooRabarbar)).toEqual([actionDefinitionOpRegex2])
expect(actionMatcher.match(eventFooNumber)).toEqual([])
expect(actionMatcher.match(eventNoNothing)).toEqual([])
expect(actionMatcher.match(eventFigNumber)).toEqual([])
expect(actionMatcher.match(eventFooTrue)).toEqual([])
expect(actionMatcher.match(eventFooNull)).toEqual([])
})
it('returns a match in case of event property operator not regex', async () => {
@@ -332,32 +322,29 @@ describe('ActionMatcher', () => {
const eventFooTrue = createTestEvent({ properties: { foo: true } })
const eventFooNull = createTestEvent({ properties: { foo: null } })
expect(await actionMatcher.match(eventFooBar)).toEqual([actionDefinitionOpNotRegex2])
expect(await actionMatcher.match(eventFooBarPolPot)).toEqual([actionDefinitionOpNotRegex2])
expect(await actionMatcher.match(eventFooBaR)).toEqual([actionDefinitionOpNotRegex1])
expect(await actionMatcher.match(eventFooBaz)).toEqual([
expect(actionMatcher.match(eventFooBar)).toEqual([actionDefinitionOpNotRegex2])
expect(actionMatcher.match(eventFooBarPolPot)).toEqual([actionDefinitionOpNotRegex2])
expect(actionMatcher.match(eventFooBaR)).toEqual([actionDefinitionOpNotRegex1])
expect(actionMatcher.match(eventFooBaz)).toEqual([actionDefinitionOpNotRegex1, actionDefinitionOpNotRegex2])
expect(actionMatcher.match(eventFooBarabara)).toEqual([])
expect(actionMatcher.match(eventFooRabarbar)).toEqual([actionDefinitionOpNotRegex1])
expect(actionMatcher.match(eventFooNumber)).toEqual([
actionDefinitionOpNotRegex1,
actionDefinitionOpNotRegex2,
])
expect(await actionMatcher.match(eventFooBarabara)).toEqual([])
expect(await actionMatcher.match(eventFooRabarbar)).toEqual([actionDefinitionOpNotRegex1])
expect(await actionMatcher.match(eventFooNumber)).toEqual([
expect(actionMatcher.match(eventNoNothing)).toEqual([
actionDefinitionOpNotRegex1,
actionDefinitionOpNotRegex2,
])
expect(await actionMatcher.match(eventNoNothing)).toEqual([
expect(actionMatcher.match(eventFigNumber)).toEqual([
actionDefinitionOpNotRegex1,
actionDefinitionOpNotRegex2,
])
expect(await actionMatcher.match(eventFigNumber)).toEqual([
expect(actionMatcher.match(eventFooTrue)).toEqual([
actionDefinitionOpNotRegex1,
actionDefinitionOpNotRegex2,
])
expect(await actionMatcher.match(eventFooTrue)).toEqual([
actionDefinitionOpNotRegex1,
actionDefinitionOpNotRegex2,
])
expect(await actionMatcher.match(eventFooNull)).toEqual([
expect(actionMatcher.match(eventFooNull)).toEqual([
actionDefinitionOpNotRegex1,
actionDefinitionOpNotRegex2,
])
@@ -382,17 +369,17 @@ describe('ActionMatcher', () => {
const eventFooTrue = createTestEvent({ properties: { foo: true } })
const eventFooNull = createTestEvent({ properties: { foo: null } })
expect(await actionMatcher.match(eventFooBar)).toEqual([actionDefinitionOpIsSet])
expect(await actionMatcher.match(eventFooBarPolPot)).toEqual([actionDefinitionOpIsSet])
expect(await actionMatcher.match(eventFooBaR)).toEqual([actionDefinitionOpIsSet])
expect(await actionMatcher.match(eventFooBaz)).toEqual([actionDefinitionOpIsSet])
expect(await actionMatcher.match(eventFooBarabara)).toEqual([actionDefinitionOpIsSet])
expect(await actionMatcher.match(eventFooRabarbar)).toEqual([actionDefinitionOpIsSet])
expect(await actionMatcher.match(eventFooNumber)).toEqual([actionDefinitionOpIsSet])
expect(await actionMatcher.match(eventNoNothing)).toEqual([])
expect(await actionMatcher.match(eventFigNumber)).toEqual([])
expect(await actionMatcher.match(eventFooTrue)).toEqual([actionDefinitionOpIsSet])
expect(await actionMatcher.match(eventFooNull)).toEqual([actionDefinitionOpIsSet])
expect(actionMatcher.match(eventFooBar)).toEqual([actionDefinitionOpIsSet])
expect(actionMatcher.match(eventFooBarPolPot)).toEqual([actionDefinitionOpIsSet])
expect(actionMatcher.match(eventFooBaR)).toEqual([actionDefinitionOpIsSet])
expect(actionMatcher.match(eventFooBaz)).toEqual([actionDefinitionOpIsSet])
expect(actionMatcher.match(eventFooBarabara)).toEqual([actionDefinitionOpIsSet])
expect(actionMatcher.match(eventFooRabarbar)).toEqual([actionDefinitionOpIsSet])
expect(actionMatcher.match(eventFooNumber)).toEqual([actionDefinitionOpIsSet])
expect(actionMatcher.match(eventNoNothing)).toEqual([])
expect(actionMatcher.match(eventFigNumber)).toEqual([])
expect(actionMatcher.match(eventFooTrue)).toEqual([actionDefinitionOpIsSet])
expect(actionMatcher.match(eventFooNull)).toEqual([actionDefinitionOpIsSet])
})
it('returns a match in case of event property operator is not set', async () => {
@@ -414,17 +401,17 @@ describe('ActionMatcher', () => {
const eventFooTrue = createTestEvent({ properties: { foo: true } })
const eventFooNull = createTestEvent({ properties: { foo: null } })
expect(await actionMatcher.match(eventFooBar)).toEqual([])
expect(await actionMatcher.match(eventFooBarPolPot)).toEqual([])
expect(await actionMatcher.match(eventFooBaR)).toEqual([])
expect(await actionMatcher.match(eventFooBaz)).toEqual([])
expect(await actionMatcher.match(eventFooBarabara)).toEqual([])
expect(await actionMatcher.match(eventFooRabarbar)).toEqual([])
expect(await actionMatcher.match(eventFooNumber)).toEqual([])
expect(await actionMatcher.match(eventNoNothing)).toEqual([actionDefinitionOpIsNotSet])
expect(await actionMatcher.match(eventFigNumber)).toEqual([actionDefinitionOpIsNotSet])
expect(await actionMatcher.match(eventFooTrue)).toEqual([])
expect(await actionMatcher.match(eventFooNull)).toEqual([])
expect(actionMatcher.match(eventFooBar)).toEqual([])
expect(actionMatcher.match(eventFooBarPolPot)).toEqual([])
expect(actionMatcher.match(eventFooBaR)).toEqual([])
expect(actionMatcher.match(eventFooBaz)).toEqual([])
expect(actionMatcher.match(eventFooBarabara)).toEqual([])
expect(actionMatcher.match(eventFooRabarbar)).toEqual([])
expect(actionMatcher.match(eventFooNumber)).toEqual([])
expect(actionMatcher.match(eventNoNothing)).toEqual([actionDefinitionOpIsNotSet])
expect(actionMatcher.match(eventFigNumber)).toEqual([actionDefinitionOpIsNotSet])
expect(actionMatcher.match(eventFooTrue)).toEqual([])
expect(actionMatcher.match(eventFooNull)).toEqual([])
})
it('returns a match in case of event property operator greater than', async () => {
@@ -448,19 +435,19 @@ describe('ActionMatcher', () => {
const eventFooTrue = createTestEvent({ properties: { foo: true } })
const eventFooNull = createTestEvent({ properties: { foo: null } })
expect(await actionMatcher.match(eventFooBar)).toEqual([])
expect(await actionMatcher.match(eventFooBarPolPot)).toEqual([])
expect(await actionMatcher.match(eventFooBaR)).toEqual([])
expect(await actionMatcher.match(eventFooBaz)).toEqual([])
expect(await actionMatcher.match(eventFooBarabara)).toEqual([])
expect(await actionMatcher.match(eventFooRabarbar)).toEqual([])
expect(await actionMatcher.match(eventFooNumberMinusOne)).toEqual([])
expect(await actionMatcher.match(eventFooNumberFive)).toEqual([])
expect(await actionMatcher.match(eventFooNumberSevenNines)).toEqual([actionDefinitionOpGreaterThan])
expect(await actionMatcher.match(eventNoNothing)).toEqual([])
expect(await actionMatcher.match(eventFigNumber)).toEqual([])
expect(await actionMatcher.match(eventFooTrue)).toEqual([])
expect(await actionMatcher.match(eventFooNull)).toEqual([])
expect(actionMatcher.match(eventFooBar)).toEqual([])
expect(actionMatcher.match(eventFooBarPolPot)).toEqual([])
expect(actionMatcher.match(eventFooBaR)).toEqual([])
expect(actionMatcher.match(eventFooBaz)).toEqual([])
expect(actionMatcher.match(eventFooBarabara)).toEqual([])
expect(actionMatcher.match(eventFooRabarbar)).toEqual([])
expect(actionMatcher.match(eventFooNumberMinusOne)).toEqual([])
expect(actionMatcher.match(eventFooNumberFive)).toEqual([])
expect(actionMatcher.match(eventFooNumberSevenNines)).toEqual([actionDefinitionOpGreaterThan])
expect(actionMatcher.match(eventNoNothing)).toEqual([])
expect(actionMatcher.match(eventFigNumber)).toEqual([])
expect(actionMatcher.match(eventFooTrue)).toEqual([])
expect(actionMatcher.match(eventFooNull)).toEqual([])
})
it('returns a match in case of event property operator less than', async () => {
@@ -484,19 +471,19 @@ describe('ActionMatcher', () => {
const eventFooTrue = createTestEvent({ properties: { foo: true } })
const eventFooNull = createTestEvent({ properties: { foo: null } })
expect(await actionMatcher.match(eventFooBar)).toEqual([])
expect(await actionMatcher.match(eventFooBarPolPot)).toEqual([])
expect(await actionMatcher.match(eventFooBaR)).toEqual([])
expect(await actionMatcher.match(eventFooBaz)).toEqual([])
expect(await actionMatcher.match(eventFooBarabara)).toEqual([])
expect(await actionMatcher.match(eventFooRabarbar)).toEqual([])
expect(await actionMatcher.match(eventFooNumberMinusOne)).toEqual([actionDefinitionOpLessThan])
expect(await actionMatcher.match(eventFooNumberFive)).toEqual([])
expect(await actionMatcher.match(eventFooNumberSevenNines)).toEqual([])
expect(await actionMatcher.match(eventNoNothing)).toEqual([])
expect(await actionMatcher.match(eventFigNumber)).toEqual([])
expect(await actionMatcher.match(eventFooTrue)).toEqual([actionDefinitionOpLessThan]) // true is a 1
expect(await actionMatcher.match(eventFooNull)).toEqual([])
expect(actionMatcher.match(eventFooBar)).toEqual([])
expect(actionMatcher.match(eventFooBarPolPot)).toEqual([])
expect(actionMatcher.match(eventFooBaR)).toEqual([])
expect(actionMatcher.match(eventFooBaz)).toEqual([])
expect(actionMatcher.match(eventFooBarabara)).toEqual([])
expect(actionMatcher.match(eventFooRabarbar)).toEqual([])
expect(actionMatcher.match(eventFooNumberMinusOne)).toEqual([actionDefinitionOpLessThan])
expect(actionMatcher.match(eventFooNumberFive)).toEqual([])
expect(actionMatcher.match(eventFooNumberSevenNines)).toEqual([])
expect(actionMatcher.match(eventNoNothing)).toEqual([])
expect(actionMatcher.match(eventFigNumber)).toEqual([])
expect(actionMatcher.match(eventFooTrue)).toEqual([actionDefinitionOpLessThan]) // true is a 1
expect(actionMatcher.match(eventFooNull)).toEqual([])
})
it('returns a match in case of URL contains page view', async () => {
@@ -522,8 +509,8 @@ describe('ActionMatcher', () => {
properties: { $current_url: 'https://example.com/' },
})
expect(await actionMatcher.match(eventPosthog)).toEqual([])
expect(await actionMatcher.match(eventExample)).toEqual([actionDefinition, actionDefinitionEmptyMatching])
expect(actionMatcher.match(eventPosthog)).toEqual([])
expect(actionMatcher.match(eventExample)).toEqual([actionDefinition, actionDefinitionEmptyMatching])
})
it('returns a match in case of URL contains page views with % and _', async () => {
@@ -549,11 +536,8 @@ describe('ActionMatcher', () => {
properties: { $current_url: 'https://example.com/index.html' },
})
expect(await actionMatcher.match(eventExample)).toEqual([])
expect(await actionMatcher.match(eventExampleHtml)).toEqual([
actionDefinition,
actionDefinitionEmptyMatching,
])
expect(actionMatcher.match(eventExample)).toEqual([])
expect(actionMatcher.match(eventExampleHtml)).toEqual([actionDefinition, actionDefinitionEmptyMatching])
})
it('returns a match in case of URL matches regex page views', async () => {
@@ -584,12 +568,12 @@ describe('ActionMatcher', () => {
properties: { $current_url: 'https://example.com/1foo/' },
})
expect(await actionMatcher.match(eventExampleOk1)).toEqual([actionDefinition])
expect(await actionMatcher.match(eventExampleOk2)).toEqual([actionDefinition])
expect(await actionMatcher.match(eventExampleBad1)).toEqual([])
expect(await actionMatcher.match(eventExampleBad2)).toEqual([])
expect(await actionMatcher.match(eventExampleBad3)).toEqual([])
expect(await actionMatcher.match(eventExampleBad4)).toEqual([])
expect(actionMatcher.match(eventExampleOk1)).toEqual([actionDefinition])
expect(actionMatcher.match(eventExampleOk2)).toEqual([actionDefinition])
expect(actionMatcher.match(eventExampleBad1)).toEqual([])
expect(actionMatcher.match(eventExampleBad2)).toEqual([])
expect(actionMatcher.match(eventExampleBad3)).toEqual([])
expect(actionMatcher.match(eventExampleBad4)).toEqual([])
})
it('returns a match in case of URL matches exactly page views', async () => {
@@ -611,9 +595,9 @@ describe('ActionMatcher', () => {
properties: { $current_url: 'https://www.mozilla.org/de/firefox/' },
})
expect(await actionMatcher.match(eventExampleOk)).toEqual([actionDefinition])
expect(await actionMatcher.match(eventExampleBad1)).toEqual([])
expect(await actionMatcher.match(eventExampleBad2)).toEqual([])
expect(actionMatcher.match(eventExampleOk)).toEqual([actionDefinition])
expect(actionMatcher.match(eventExampleBad1)).toEqual([])
expect(actionMatcher.match(eventExampleBad2)).toEqual([])
})
it('returns a match in case of exact event name', async () => {
@@ -633,9 +617,9 @@ describe('ActionMatcher', () => {
event: 'WOOF',
})
expect(await actionMatcher.match(eventExampleOk)).toEqual([actionDefinition])
expect(await actionMatcher.match(eventExampleBad1)).toEqual([])
expect(await actionMatcher.match(eventExampleBad2)).toEqual([])
expect(actionMatcher.match(eventExampleOk)).toEqual([actionDefinition])
expect(actionMatcher.match(eventExampleBad1)).toEqual([])
expect(actionMatcher.match(eventExampleBad2)).toEqual([])
})
it('returns a match in case of exact event name AND URL contains', async () => {
@@ -668,11 +652,11 @@ describe('ActionMatcher', () => {
properties: { $current_url: 'https://www.pets.co' },
})
expect(await actionMatcher.match(eventExampleOk1)).toEqual([actionDefinition])
expect(await actionMatcher.match(eventExampleOk2)).toEqual([actionDefinition])
expect(await actionMatcher.match(eventExampleBad1)).toEqual([])
expect(await actionMatcher.match(eventExampleBad2)).toEqual([])
expect(await actionMatcher.match(eventExampleBad3)).toEqual([])
expect(actionMatcher.match(eventExampleOk1)).toEqual([actionDefinition])
expect(actionMatcher.match(eventExampleOk2)).toEqual([actionDefinition])
expect(actionMatcher.match(eventExampleBad1)).toEqual([])
expect(actionMatcher.match(eventExampleBad2)).toEqual([])
expect(actionMatcher.match(eventExampleBad3)).toEqual([])
})
it('returns a match in case of person property operator exact', async () => {
@@ -689,93 +673,23 @@ describe('ActionMatcher', () => {
const event = createTestEvent()
expect(await actionMatcher.match({ ...event, person_properties: { foo: 'bar' } })).toEqual([
expect(actionMatcher.match({ ...event, person_properties: { foo: 'bar' } })).toEqual([
actionDefinitionOpExact,
actionDefinitionOpUndefined,
])
expect(await actionMatcher.match({ ...event, person_properties: { foo: 'bar', pol: 'pot' } })).toEqual([
expect(actionMatcher.match({ ...event, person_properties: { foo: 'bar', pol: 'pot' } })).toEqual([
actionDefinitionOpExact,
actionDefinitionOpUndefined,
])
expect(await actionMatcher.match({ ...event, person_properties: { foo: 'baR' } })).toEqual([])
expect(await actionMatcher.match({ ...event, person_properties: { foo: 'baz' } })).toEqual([])
expect(await actionMatcher.match({ ...event, person_properties: { foo: 'barabara' } })).toEqual([])
expect(await actionMatcher.match({ ...event, person_properties: { foo: 'rabarbar' } })).toEqual([])
expect(await actionMatcher.match({ ...event, person_properties: { foo: 7 } })).toEqual([])
expect(await actionMatcher.match({ ...event, person_properties: {} })).toEqual([])
expect(await actionMatcher.match({ ...event, person_properties: { something_else: 999 } })).toEqual([])
expect(await actionMatcher.match({ ...event, person_properties: { foo: true } })).toEqual([])
expect(await actionMatcher.match({ ...event, person_properties: { foo: null } })).toEqual([])
})
it('returns a match in case of cohort match', async () => {
const testCohort = await hub.db.createCohort({
name: 'Test',
description: 'Test',
created_by_id: commonUserId,
team_id: 2,
})
const actionDefinition: Action = await createTestAction([
{
properties: [{ type: 'cohort', key: 'id', value: testCohort.id }],
},
])
const actionDefinitionAllUsers: Action = await createTestAction([
{
properties: [{ type: 'cohort', key: 'id', value: 'all' }],
},
])
const [nonCohortPerson, kafkaMessagesNonCohort] = await hub.db.createPerson(
DateTime.local(),
{},
{},
{},
actionDefinition.team_id,
null,
true,
new UUIDT().toString(),
[{ distinctId: 'random' }]
)
await hub.db.kafkaProducer.queueMessages(kafkaMessagesNonCohort)
const [cohortPerson, kafkaMessagesCohort] = await hub.db.createPerson(
DateTime.local(),
{},
{},
{},
actionDefinition.team_id,
null,
true,
new UUIDT().toString(),
[{ distinctId: 'cohort' }]
)
await hub.db.kafkaProducer.queueMessages(kafkaMessagesCohort)
await hub.db.addPersonToCohort(testCohort.id, cohortPerson.id, testCohort.version)
const eventExamplePersonBad = createTestEvent({
event: 'meow',
distinctId: 'random',
person_id: nonCohortPerson.uuid,
})
const eventExamplePersonOk = createTestEvent({
event: 'meow',
distinctId: 'cohort',
person_id: cohortPerson.uuid,
})
const eventExamplePersonUnknown = createTestEvent({
event: 'meow',
distinctId: 'unknown',
person_id: undefined,
})
expect(await actionMatcher.match(eventExamplePersonOk)).toEqual([
actionDefinition,
actionDefinitionAllUsers,
])
expect(await actionMatcher.match(eventExamplePersonBad)).toEqual([actionDefinitionAllUsers])
expect(await actionMatcher.match(eventExamplePersonUnknown)).toEqual([actionDefinitionAllUsers])
expect(actionMatcher.match({ ...event, person_properties: { foo: 'baR' } })).toEqual([])
expect(actionMatcher.match({ ...event, person_properties: { foo: 'baz' } })).toEqual([])
expect(actionMatcher.match({ ...event, person_properties: { foo: 'barabara' } })).toEqual([])
expect(actionMatcher.match({ ...event, person_properties: { foo: 'rabarbar' } })).toEqual([])
expect(actionMatcher.match({ ...event, person_properties: { foo: 7 } })).toEqual([])
expect(actionMatcher.match({ ...event, person_properties: {} })).toEqual([])
expect(actionMatcher.match({ ...event, person_properties: { something_else: 999 } })).toEqual([])
expect(actionMatcher.match({ ...event, person_properties: { foo: true } })).toEqual([])
expect(actionMatcher.match({ ...event, person_properties: { foo: null } })).toEqual([])
})
it('returns a match in case of element href equals', async () => {
@@ -802,13 +716,13 @@ describe('ActionMatcher', () => {
{ tag_name: 'main' },
]
expect(await actionMatcher.match({ ...event, elementsList: elementsHrefOuter })).toEqual([
expect(actionMatcher.match({ ...event, elementsList: elementsHrefOuter })).toEqual([
actionDefinitionLinkHref,
])
expect(await actionMatcher.match({ ...event, elementsList: elementsHrefInner })).toEqual([
expect(actionMatcher.match({ ...event, elementsList: elementsHrefInner })).toEqual([
actionDefinitionLinkHref,
])
expect(await actionMatcher.match({ ...event, elementsList: elementsNoHref })).toEqual([])
expect(actionMatcher.match({ ...event, elementsList: elementsNoHref })).toEqual([])
})
it('returns a match in case of element href contains', async () => {
@@ -846,17 +760,17 @@ describe('ActionMatcher', () => {
{ tag_name: 'main' },
]
expect(await actionMatcher.match({ ...event, elementsList: elementsExactHrefOuter })).toEqual([
expect(actionMatcher.match({ ...event, elementsList: elementsExactHrefOuter })).toEqual([
actionDefinitionLinkHref,
])
expect(await actionMatcher.match({ ...event, elementsList: elementsExactHrefInner })).toEqual([
expect(actionMatcher.match({ ...event, elementsList: elementsExactHrefInner })).toEqual([
actionDefinitionLinkHref,
])
expect(await actionMatcher.match({ ...event, elementsList: elementsExtendedHref })).toEqual([
expect(actionMatcher.match({ ...event, elementsList: elementsExtendedHref })).toEqual([
actionDefinitionLinkHref,
])
expect(await actionMatcher.match({ ...event, elementsList: elementsBadHref })).toEqual([])
expect(await actionMatcher.match({ ...event, elementsList: elementsNoHref })).toEqual([])
expect(actionMatcher.match({ ...event, elementsList: elementsBadHref })).toEqual([])
expect(actionMatcher.match({ ...event, elementsList: elementsNoHref })).toEqual([])
})
it('returns a match in case of element href contains, with wildcard', async () => {
@@ -894,13 +808,13 @@ describe('ActionMatcher', () => {
{ tag_name: 'main' },
]
expect(await actionMatcher.match({ ...event, elementsList: elementsExactHrefOuter })).toEqual([])
expect(await actionMatcher.match({ ...event, elementsList: elementsExactHrefInner })).toEqual([])
expect(await actionMatcher.match({ ...event, elementsList: elementsExtendedHref })).toEqual([
expect(actionMatcher.match({ ...event, elementsList: elementsExactHrefOuter })).toEqual([])
expect(actionMatcher.match({ ...event, elementsList: elementsExactHrefInner })).toEqual([])
expect(actionMatcher.match({ ...event, elementsList: elementsExtendedHref })).toEqual([
actionDefinitionLinkHref,
])
expect(await actionMatcher.match({ ...event, elementsList: elementsBadHref })).toEqual([])
expect(await actionMatcher.match({ ...event, elementsList: elementsNoHref })).toEqual([])
expect(actionMatcher.match({ ...event, elementsList: elementsBadHref })).toEqual([])
expect(actionMatcher.match({ ...event, elementsList: elementsNoHref })).toEqual([])
})
it('returns a match in case of element href matches regex', async () => {
@@ -938,13 +852,13 @@ describe('ActionMatcher', () => {
{ tag_name: 'main' },
]
expect(await actionMatcher.match({ ...event, elementsList: elementsExactHrefOuter })).toEqual([])
expect(await actionMatcher.match({ ...event, elementsList: elementsExactHrefInner })).toEqual([])
expect(await actionMatcher.match({ ...event, elementsList: elementsExtendedHref })).toEqual([
expect(actionMatcher.match({ ...event, elementsList: elementsExactHrefOuter })).toEqual([])
expect(actionMatcher.match({ ...event, elementsList: elementsExactHrefInner })).toEqual([])
expect(actionMatcher.match({ ...event, elementsList: elementsExtendedHref })).toEqual([
actionDefinitionLinkHref,
])
expect(await actionMatcher.match({ ...event, elementsList: elementsBadHref })).toEqual([])
expect(await actionMatcher.match({ ...event, elementsList: elementsNoHref })).toEqual([])
expect(actionMatcher.match({ ...event, elementsList: elementsBadHref })).toEqual([])
expect(actionMatcher.match({ ...event, elementsList: elementsNoHref })).toEqual([])
})
it('returns a match in case of element text and tag name equals', async () => {
@@ -978,12 +892,12 @@ describe('ActionMatcher', () => {
{ tag_name: 'main' },
]
expect(await actionMatcher.match({ ...event, elementsList: elementsHrefProper })).toEqual([
expect(actionMatcher.match({ ...event, elementsList: elementsHrefProper })).toEqual([
actionDefinitionLinkHref,
])
expect(await actionMatcher.match({ ...event, elementsList: elementsHrefWrongTag })).toEqual([])
expect(await actionMatcher.match({ ...event, elementsList: elementsHrefWrongText })).toEqual([])
expect(await actionMatcher.match({ ...event, elementsList: elementsHrefWrongLevel })).toEqual([])
expect(actionMatcher.match({ ...event, elementsList: elementsHrefWrongTag })).toEqual([])
expect(actionMatcher.match({ ...event, elementsList: elementsHrefWrongText })).toEqual([])
expect(actionMatcher.match({ ...event, elementsList: elementsHrefWrongLevel })).toEqual([])
})
it('returns a match in case of element text contains', async () => {
@@ -1006,8 +920,8 @@ describe('ActionMatcher', () => {
{ tag_name: 'main' },
]
expect(await actionMatcher.match({ ...event, elementsList: elementsHrefBadText })).toEqual([])
expect(await actionMatcher.match({ ...event, elementsList: elementsHrefGoodText })).toEqual([
expect(actionMatcher.match({ ...event, elementsList: elementsHrefBadText })).toEqual([])
expect(actionMatcher.match({ ...event, elementsList: elementsHrefGoodText })).toEqual([
actionDefinitionLinkHref,
])
})
@@ -1050,15 +964,15 @@ describe('ActionMatcher', () => {
{ tag_name: 'main' },
]
expect(await actionMatcher.match({ ...event, elementsList: elementsHrefProperNondirect })).toEqual([
expect(actionMatcher.match({ ...event, elementsList: elementsHrefProperNondirect })).toEqual([
actionDefinitionAnyDescendant,
actionDefinitionDirectHref,
actionDefinitionArraySelectorProp,
])
expect(await actionMatcher.match({ ...event, elementsList: elementsHrefWrongClassNondirect })).toEqual([
expect(actionMatcher.match({ ...event, elementsList: elementsHrefWrongClassNondirect })).toEqual([
actionDefinitionDirectHref,
])
expect(await actionMatcher.match({ ...event, elementsList: elementsHrefProperDirect })).toEqual([
expect(actionMatcher.match({ ...event, elementsList: elementsHrefProperDirect })).toEqual([
actionDefinitionAnyDescendant,
actionDefinitionDirectDescendant,
actionDefinitionArraySelectorProp,
@@ -1096,7 +1010,7 @@ describe('ActionMatcher', () => {
},
})
expect(await actionMatcher.match(eventExampleOk1)).toEqual([actionDefinition])
expect(actionMatcher.match(eventExampleOk1)).toEqual([actionDefinition])
})
it('properly handles is_not null string coercion', async () => {
@@ -1120,16 +1034,16 @@ describe('ActionMatcher', () => {
const eventFooTrue = createTestEvent({ properties: { foo: true } })
const eventFooNull = createTestEvent({ properties: { foo: null } })
expect(await actionMatcher.match(eventFooBar)).toEqual([actionDefinitionOpIsSet])
expect(await actionMatcher.match(eventFooBarPolPot)).toEqual([])
expect(await actionMatcher.match(eventFooBaR)).toEqual([actionDefinitionOpIsSet])
expect(await actionMatcher.match(eventFooBaz)).toEqual([actionDefinitionOpIsSet])
expect(await actionMatcher.match(eventFooRabarbar)).toEqual([actionDefinitionOpIsSet])
expect(await actionMatcher.match(eventFooNumber)).toEqual([actionDefinitionOpIsSet])
expect(await actionMatcher.match(eventNoNothing)).toEqual([])
expect(await actionMatcher.match(eventFigNumber)).toEqual([])
expect(await actionMatcher.match(eventFooTrue)).toEqual([actionDefinitionOpIsSet])
expect(await actionMatcher.match(eventFooNull)).toEqual([])
expect(actionMatcher.match(eventFooBar)).toEqual([actionDefinitionOpIsSet])
expect(actionMatcher.match(eventFooBarPolPot)).toEqual([])
expect(actionMatcher.match(eventFooBaR)).toEqual([actionDefinitionOpIsSet])
expect(actionMatcher.match(eventFooBaz)).toEqual([actionDefinitionOpIsSet])
expect(actionMatcher.match(eventFooRabarbar)).toEqual([actionDefinitionOpIsSet])
expect(actionMatcher.match(eventFooNumber)).toEqual([actionDefinitionOpIsSet])
expect(actionMatcher.match(eventNoNothing)).toEqual([])
expect(actionMatcher.match(eventFigNumber)).toEqual([])
expect(actionMatcher.match(eventFooTrue)).toEqual([actionDefinitionOpIsSet])
expect(actionMatcher.match(eventFooNull)).toEqual([])
})
it('properly handles exact null string coercion', async () => {
@@ -1150,16 +1064,16 @@ describe('ActionMatcher', () => {
const eventFooTrue = createTestEvent({ properties: { foo: true } })
const eventFooNull = createTestEvent({ properties: { foo: null } })
expect(await actionMatcher.match(eventFooBar)).toEqual([])
expect(await actionMatcher.match(eventFooBarPolPot)).toEqual([actionDefinitionOpIsSet])
expect(await actionMatcher.match(eventFooBaR)).toEqual([])
expect(await actionMatcher.match(eventFooBaz)).toEqual([])
expect(await actionMatcher.match(eventFooRabarbar)).toEqual([])
expect(await actionMatcher.match(eventFooNumber)).toEqual([])
expect(await actionMatcher.match(eventNoNothing)).toEqual([])
expect(await actionMatcher.match(eventFigNumber)).toEqual([])
expect(await actionMatcher.match(eventFooTrue)).toEqual([])
expect(await actionMatcher.match(eventFooNull)).toEqual([actionDefinitionOpIsSet])
expect(actionMatcher.match(eventFooBar)).toEqual([])
expect(actionMatcher.match(eventFooBarPolPot)).toEqual([actionDefinitionOpIsSet])
expect(actionMatcher.match(eventFooBaR)).toEqual([])
expect(actionMatcher.match(eventFooBaz)).toEqual([])
expect(actionMatcher.match(eventFooRabarbar)).toEqual([])
expect(actionMatcher.match(eventFooNumber)).toEqual([])
expect(actionMatcher.match(eventNoNothing)).toEqual([])
expect(actionMatcher.match(eventFigNumber)).toEqual([])
expect(actionMatcher.match(eventFooTrue)).toEqual([])
expect(actionMatcher.match(eventFooNull)).toEqual([actionDefinitionOpIsSet])
})
})
@@ -1277,76 +1191,6 @@ describe('ActionMatcher', () => {
expect(checkElementsAgainstSelector(elements, 'section > span:nth-child(2):nth-of-type(3)')).toBeFalsy()
})
})
describe('doesPersonBelongToCohort()', () => {
let team: Team
let cohort: Cohort
let person: InternalPerson
let personId: InternalPerson['id']
const TIMESTAMP = DateTime.fromISO('2000-10-14T11:42:06.502Z').toUTC()
beforeEach(async () => {
team = await getFirstTeam(hub)
cohort = await hub.db.createCohort({
name: 'testCohort',
description: '',
team_id: team.id,
version: 10,
})
const [personLocal, kafkaMessages] = await hub.db.createPerson(
TIMESTAMP,
{},
{},
{},
team.id,
null,
false,
new UUIDT().toString(),
[]
)
await hub.db.kafkaProducer.queueMessages(kafkaMessages)
person = personLocal
personId = person.id
})
it('returns false if person does not belong to cohort', async () => {
const cohort2 = await hub.db.createCohort({
name: 'testCohort2',
description: '',
team_id: team.id,
})
await hub.db.addPersonToCohort(cohort2.id, personId, cohort.version)
expect(await actionMatcher.doesPersonBelongToCohort(cohort.id, person.uuid, person.team_id)).toEqual(false)
})
it('returns true if person belongs to cohort', async () => {
await hub.db.addPersonToCohort(cohort.id, personId, cohort.version)
expect(await actionMatcher.doesPersonBelongToCohort(cohort.id, person.uuid, person.team_id)).toEqual(true)
})
it('returns false if person does not belong to current version of the cohort', async () => {
await hub.db.addPersonToCohort(cohort.id, personId, -1)
expect(await actionMatcher.doesPersonBelongToCohort(cohort.id, person.uuid, person.team_id)).toEqual(false)
})
it('handles NULL version cohorts', async () => {
const cohort2 = await hub.db.createCohort({
name: 'null_cohort',
description: '',
team_id: team.id,
version: null,
})
expect(await actionMatcher.doesPersonBelongToCohort(cohort2.id, person.uuid, person.team_id)).toEqual(false)
await hub.db.addPersonToCohort(cohort2.id, personId, null)
expect(await actionMatcher.doesPersonBelongToCohort(cohort2.id, person.uuid, person.team_id)).toEqual(true)
})
})
})
describe('castingCompare', () => {

View File

@@ -33,7 +33,7 @@ describe('runAsyncHandlersStep()', () => {
processAsyncOnEventHandlers: true,
},
actionMatcher: {
match: jest.fn().mockResolvedValue(['action1', 'action2']),
match: jest.fn().mockReturnValue(['action1', 'action2']),
},
hookCannon: {
findAndFireHooks: jest.fn().mockResolvedValue(true),

View File

@@ -2,7 +2,11 @@ import { Hub } from '../../src/types'
import { closeHub, createHub } from '../../src/utils/db/hub'
import { PostgresUse } from '../../src/utils/db/postgres'
import { PluginsApiKeyManager } from '../../src/worker/vm/extensions/helpers/api-key-manager'
import { createUserTeamAndOrganization, POSTGRES_DELETE_TABLES_QUERY } from '../helpers/sql'
import {
createUserTeamAndOrganization,
POSTGRES_DELETE_OTHER_TABLES_QUERY,
POSTGRES_DELETE_PERSON_TABLES_QUERY,
} from '../helpers/sql'
const ORG_ID_1 = '0174f81e-36f5-0000-7ef8-cc26c1fbab1c'
const ORG_ID_2 = '4dc8564d-bd82-1065-2f40-97f7c50f67cf'
@@ -14,7 +18,18 @@ describe('PluginsApiKeyManager', () => {
hub = await createHub({
TASK_TIMEOUT: 1,
})
await hub.db.postgres.query(PostgresUse.COMMON_WRITE, POSTGRES_DELETE_TABLES_QUERY, [], 'truncateTablesTest')
await hub.db.postgres.query(
PostgresUse.PERSONS_WRITE,
POSTGRES_DELETE_PERSON_TABLES_QUERY,
[],
'truncatePersonTablesTest'
)
await hub.db.postgres.query(
PostgresUse.COMMON_WRITE,
POSTGRES_DELETE_OTHER_TABLES_QUERY,
[],
'truncateTablesTest'
)
await hub.db.redisExpire(`plugins-api-key-manager/${ORG_ID_1}`, 0)
await hub.db.redisExpire(`plugins-api-key-manager/${ORG_ID_2}`, 0)
})