mirror of
https://github.com/BillyOutlast/posthog.git
synced 2026-02-04 03:01:23 +01:00
style(plugin-server): Ban unused vars (#9922)
* Update .eslintrc.js * Remove unused vars * Add `eslint-plugin-prettier` * Fix missing import
This commit is contained in:
@@ -6,13 +6,19 @@ module.exports = {
|
||||
tsconfigRootDir: __dirname,
|
||||
project: ['./tsconfig.eslint.json'],
|
||||
},
|
||||
plugins: ['@typescript-eslint', 'simple-import-sort'],
|
||||
plugins: ['@typescript-eslint', 'simple-import-sort', 'prettier'],
|
||||
extends: ['plugin:@typescript-eslint/recommended', 'prettier'],
|
||||
ignorePatterns: ['bin', 'dist', 'node_modules'],
|
||||
ignorePatterns: ['bin', 'dist', 'node_modules', '**/protos.d.ts'],
|
||||
rules: {
|
||||
'simple-import-sort/imports': 'error',
|
||||
'simple-import-sort/exports': 'error',
|
||||
'@typescript-eslint/no-unused-vars': 'off',
|
||||
'no-unused-vars': 'off',
|
||||
'@typescript-eslint/no-unused-vars': [
|
||||
'error',
|
||||
{
|
||||
ignoreRestSiblings: true,
|
||||
},
|
||||
],
|
||||
'@typescript-eslint/explicit-function-return-type': 'off',
|
||||
'@typescript-eslint/no-non-null-assertion': 'off',
|
||||
'@typescript-eslint/no-var-requires': 'off',
|
||||
|
||||
@@ -33,8 +33,7 @@ describe('ingestion benchmarks', () => {
|
||||
team.id,
|
||||
now,
|
||||
now,
|
||||
new UUIDT().toString(),
|
||||
'http://example.com'
|
||||
new UUIDT().toString()
|
||||
)
|
||||
}
|
||||
|
||||
|
||||
@@ -116,6 +116,7 @@
|
||||
"eslint-config-prettier": "^8.3.0",
|
||||
"eslint-plugin-import": "^2.23.4",
|
||||
"eslint-plugin-node": "^11.1.0",
|
||||
"eslint-plugin-prettier": "^4.0.0",
|
||||
"eslint-plugin-promise": "^5.1.0",
|
||||
"eslint-plugin-react": "^7.24.0",
|
||||
"eslint-plugin-simple-import-sort": "^7.0.0",
|
||||
|
||||
@@ -1,14 +1,11 @@
|
||||
import { PluginEvent } from '@posthog/plugin-scaffold'
|
||||
import * as Sentry from '@sentry/node'
|
||||
import { Consumer, EachBatchPayload, Kafka, KafkaMessage } from 'kafkajs'
|
||||
import { Consumer, EachBatchPayload, Kafka } from 'kafkajs'
|
||||
|
||||
import { Hub, Queue, WorkerMethods } from '../../types'
|
||||
import { status } from '../../utils/status'
|
||||
import { groupIntoBatches, killGracefully, sanitizeEvent } from '../../utils/utils'
|
||||
import { runInstrumentedFunction } from '../utils'
|
||||
import { killGracefully } from '../../utils/utils'
|
||||
import { KAFKA_BUFFER, KAFKA_EVENTS_JSON } from './../../config/kafka-topics'
|
||||
import { eachBatchBuffer } from './batch-processing/each-batch-buffer'
|
||||
import { eachBatchIngestion, ingestEvent } from './batch-processing/each-batch-ingestion'
|
||||
import { eachBatchIngestion } from './batch-processing/each-batch-ingestion'
|
||||
|
||||
type ConsumerManagementPayload = {
|
||||
topic: string
|
||||
|
||||
@@ -2,7 +2,7 @@ import * as Sentry from '@sentry/node'
|
||||
import { makeWorkerUtils, run, Runner, WorkerUtils } from 'graphile-worker'
|
||||
import { Pool } from 'pg'
|
||||
|
||||
import { EnqueuedJob, JobQueue, OnJobCallback, PluginsServerConfig } from '../../../types'
|
||||
import { EnqueuedJob, PluginsServerConfig } from '../../../types'
|
||||
import { status } from '../../../utils/status'
|
||||
import { createPostgresPool } from '../../../utils/utils'
|
||||
import { JobQueueBase } from '../job-queue-base'
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
import { EnqueuedJob, JobQueue, OnJobCallback, PluginsServerConfig } from '../../types'
|
||||
import { EnqueuedJob, JobQueue, OnJobCallback } from '../../types'
|
||||
|
||||
export class JobQueueBase implements JobQueue {
|
||||
started: boolean
|
||||
@@ -22,7 +22,7 @@ export class JobQueueBase implements JobQueue {
|
||||
}
|
||||
|
||||
enqueue(retry: EnqueuedJob): void
|
||||
// eslint-disable-next-line @typescript-eslint/require-await
|
||||
// eslint-disable-next-line @typescript-eslint/require-await, @typescript-eslint/no-unused-vars
|
||||
async enqueue(retry: EnqueuedJob): Promise<void> {
|
||||
throw new Error('enqueue() not implemented for job queue!')
|
||||
}
|
||||
|
||||
@@ -1,5 +1,4 @@
|
||||
import Piscina from '@posthog/piscina'
|
||||
import * as Sentry from '@sentry/node'
|
||||
|
||||
import { Hub, JobQueueConsumerControl, OnJobCallback } from '../../types'
|
||||
import { killProcess } from '../../utils/kill'
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
import { EnqueuedJob, JobQueue, OnJobCallback } from '../../../types'
|
||||
import { EnqueuedJob, OnJobCallback } from '../../../types'
|
||||
import Timeout = NodeJS.Timeout
|
||||
import * as fs from 'fs'
|
||||
import * as path from 'path'
|
||||
|
||||
@@ -19,7 +19,7 @@ export async function startPluginSchedules(
|
||||
|
||||
// Import this just to trigger build on ts-node-dev
|
||||
// This is a total hack and needs to be fixed - seems to be bug with ts-node-dev
|
||||
const _ = require('../../worker/worker')
|
||||
require('../../worker/worker')
|
||||
|
||||
let stopped = false
|
||||
let weHaveTheLock = false
|
||||
|
||||
@@ -63,7 +63,7 @@ export async function kafkaHealthcheck(
|
||||
let kafkaConsumerWorking = false
|
||||
let timer: Date | null = new Date()
|
||||
const waitForConsumerConnected = new Promise<void>((resolve) => {
|
||||
consumer.on(consumer.events.FETCH_START, (...args) => {
|
||||
consumer.on(consumer.events.FETCH_START, () => {
|
||||
if (timer) {
|
||||
statsd?.timing('kafka_healthcheck_consumer_latency', timer)
|
||||
timer = null
|
||||
|
||||
@@ -45,7 +45,6 @@ import {
|
||||
PluginLogEntrySource,
|
||||
PluginLogEntryType,
|
||||
PluginSourceFileStatus,
|
||||
PluginsServerConfig,
|
||||
PostgresSessionRecordingEvent,
|
||||
PropertiesLastOperation,
|
||||
PropertiesLastUpdatedAt,
|
||||
@@ -70,7 +69,7 @@ import {
|
||||
UUID,
|
||||
UUIDT,
|
||||
} from '../utils'
|
||||
import { OrganizationPluginsAccessLevel, PluginLogLevel } from './../../types'
|
||||
import { OrganizationPluginsAccessLevel } from './../../types'
|
||||
import { KafkaProducerWrapper } from './kafka-producer-wrapper'
|
||||
import { PostgresLogsWrapper } from './postgres-logs-wrapper'
|
||||
import {
|
||||
|
||||
@@ -134,8 +134,7 @@ export function chainToElements(chain: string): Element[] {
|
||||
const tagAndClass = elStringSplit[1].split('.')
|
||||
element.tag_name = tagAndClass[0]
|
||||
if (tagAndClass.length > 1) {
|
||||
const [_, ...rest] = tagAndClass
|
||||
element.attr_class = rest.filter((t) => t)
|
||||
element.attr_class = tagAndClass.slice(1).filter(Boolean)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -203,10 +202,10 @@ const combinedParams = new Set([...campaignParams, ...initialParams])
|
||||
/** If we get new UTM params, make sure we set those **/
|
||||
export function personInitialAndUTMProperties(properties: Properties): Properties {
|
||||
const propertiesCopy = { ...properties }
|
||||
const maybeSet = Object.entries(properties).filter(([key, value]) => campaignParams.has(key))
|
||||
const maybeSet = Object.entries(properties).filter(([key]) => campaignParams.has(key))
|
||||
|
||||
const maybeSetInitial = Object.entries(properties)
|
||||
.filter(([key, value]) => combinedParams.has(key))
|
||||
.filter(([key]) => combinedParams.has(key))
|
||||
.map(([key, value]) => [`$initial_${key.replace('$', '')}`, value])
|
||||
if (Object.keys(maybeSet).length > 0) {
|
||||
propertiesCopy.$set = { ...(properties.$set || {}), ...Object.fromEntries(maybeSet) }
|
||||
@@ -310,7 +309,7 @@ export function shouldStoreLog(
|
||||
// keep in sync with posthog/posthog/api/utils.py::safe_clickhouse_string
|
||||
export function safeClickhouseString(str: string): string {
|
||||
// character is a surrogate
|
||||
return str.replace(/[\ud800-\udfff]/gu, (match, _) => {
|
||||
return str.replace(/[\ud800-\udfff]/gu, (match) => {
|
||||
const res = JSON.stringify(match)
|
||||
return res.slice(1, res.length - 1) + `\\`
|
||||
})
|
||||
|
||||
@@ -1,4 +1,3 @@
|
||||
import * as Sentry from '@sentry/node'
|
||||
import { StatsD, Tags } from 'hot-shots'
|
||||
|
||||
export async function instrumentQuery<T>(
|
||||
|
||||
@@ -520,7 +520,7 @@ export function pluginConfigIdFromStack(
|
||||
// This matches `pluginConfigIdentifier` from worker/vm/vm.ts
|
||||
// For example: "at __asyncGuard__PluginConfig_39_3af03d... (vm.js:11..."
|
||||
const regexp = /at __[a-zA-Z0-9]+__PluginConfig_([0-9]+)_([0-9a-f]+) \(vm\.js\:/
|
||||
const [_, id, hash] =
|
||||
const [, id, hash] =
|
||||
stack
|
||||
.split('\n')
|
||||
.map((l) => l.match(regexp))
|
||||
|
||||
@@ -13,8 +13,7 @@ export async function prepareEventStep(runner: EventPipelineRunner, event: Plugi
|
||||
team_id,
|
||||
DateTime.fromISO(now),
|
||||
sent_at ? DateTime.fromISO(sent_at) : null,
|
||||
uuid!, // it will throw if it's undefined,
|
||||
site_url
|
||||
uuid! // it will throw if it's undefined,
|
||||
)
|
||||
|
||||
await runner.hub.siteUrlManager.updateIngestionSiteUrl(site_url)
|
||||
|
||||
@@ -103,8 +103,7 @@ export class EventsProcessor {
|
||||
teamId: number,
|
||||
now: DateTime,
|
||||
sentAt: DateTime | null,
|
||||
eventUuid: string,
|
||||
siteUrl: string
|
||||
eventUuid: string
|
||||
): Promise<PreIngestionEvent | null> {
|
||||
if (!UUID.validateString(eventUuid, false)) {
|
||||
throw new Error(`Not a valid UUID: "${eventUuid}"`)
|
||||
@@ -187,8 +186,7 @@ export class EventsProcessor {
|
||||
data['event'],
|
||||
distinctId,
|
||||
properties,
|
||||
ts,
|
||||
siteUrl
|
||||
ts
|
||||
)
|
||||
this.pluginsServer.statsd?.timing('kafka_queue.single_save.standard', singleSaveTimer, {
|
||||
team_id: teamId.toString(),
|
||||
@@ -236,8 +234,7 @@ export class EventsProcessor {
|
||||
teamId: number,
|
||||
distinctId: string,
|
||||
properties: Properties,
|
||||
propertiesOnce: Properties,
|
||||
timestamp: DateTime
|
||||
propertiesOnce: Properties
|
||||
): Promise<void> {
|
||||
await this.updatePersonPropertiesDeprecated(teamId, distinctId, properties, propertiesOnce)
|
||||
}
|
||||
@@ -520,8 +517,7 @@ export class EventsProcessor {
|
||||
event: string,
|
||||
distinctId: string,
|
||||
properties: Properties,
|
||||
timestamp: DateTime,
|
||||
siteUrl: string
|
||||
timestamp: DateTime
|
||||
): Promise<PreIngestionEvent> {
|
||||
event = sanitizeEventName(event)
|
||||
const elements: Record<string, any>[] | undefined = properties['$elements']
|
||||
@@ -559,8 +555,7 @@ export class EventsProcessor {
|
||||
team.id,
|
||||
distinctId,
|
||||
properties['$set'] || {},
|
||||
properties['$set_once'] || {},
|
||||
timestamp
|
||||
properties['$set_once'] || {}
|
||||
)
|
||||
}
|
||||
|
||||
@@ -734,9 +729,7 @@ export class EventsProcessor {
|
||||
messages: [{ key: uuid, value: Buffer.from(JSON.stringify(data)) }],
|
||||
})
|
||||
} else {
|
||||
const {
|
||||
rows: [eventCreated],
|
||||
} = await this.db.postgresQuery(
|
||||
await this.db.postgresQuery(
|
||||
'INSERT INTO posthog_sessionrecordingevent (created_at, team_id, distinct_id, session_id, window_id, timestamp, snapshot_data) VALUES ($1, $2, $3, $4, $5, $6, $7) RETURNING *',
|
||||
[
|
||||
data.created_at,
|
||||
|
||||
@@ -1,19 +1,15 @@
|
||||
import { Properties } from '@posthog/plugin-scaffold'
|
||||
import { KafkaMessage, ProducerRecord } from 'kafkajs'
|
||||
import { DateTime } from 'luxon'
|
||||
import { PoolClient, QueryResult } from 'pg'
|
||||
|
||||
import {
|
||||
Group,
|
||||
GroupTypeIndex,
|
||||
Person,
|
||||
PropertiesLastOperation,
|
||||
PropertiesLastUpdatedAt,
|
||||
PropertyUpdateOperation,
|
||||
TeamId,
|
||||
} from '../../types'
|
||||
import { DB } from '../../utils/db/db'
|
||||
import { generateKafkaPersonUpdateMessage } from '../../utils/db/utils'
|
||||
import { RaceConditionError } from '../../utils/utils'
|
||||
|
||||
interface PropertiesUpdate {
|
||||
@@ -128,7 +124,7 @@ export function shouldUpdateProperty(
|
||||
export function calculateUpdateSingleProperty(
|
||||
result: PropertiesUpdate,
|
||||
key: string,
|
||||
value: any,
|
||||
value: any, // eslint-disable-line @typescript-eslint/explicit-module-boundary-types
|
||||
operation: PropertyUpdateOperation,
|
||||
timestamp: DateTime,
|
||||
currentPropertiesLastOperation: PropertiesLastOperation,
|
||||
|
||||
@@ -16,7 +16,7 @@ export async function loadPlugin(hub: Hub, pluginConfig: PluginConfig): Promise<
|
||||
}
|
||||
|
||||
try {
|
||||
let getFile = (file: string): Promise<string | null> => Promise.resolve(null)
|
||||
let getFile: (file: string) => Promise<string | null> = () => Promise.resolve(null)
|
||||
if (plugin.url?.startsWith('file:')) {
|
||||
const pluginPath = path.resolve(hub.BASE_DIR, plugin.url.substring(5))
|
||||
getFile = (file) => {
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
import { PluginEvent, ProcessedPluginEvent } from '@posthog/plugin-scaffold'
|
||||
|
||||
import { Alert, Hub, PluginConfig, PluginTaskType, TeamId } from '../../types'
|
||||
import { Hub, PluginConfig, PluginTaskType } from '../../types'
|
||||
import { processError } from '../../utils/db/error'
|
||||
import { IllegalOperationError } from '../../utils/utils'
|
||||
import { Action } from './../../types'
|
||||
@@ -151,6 +151,7 @@ export async function runPluginTask(
|
||||
const timer = new Date()
|
||||
let response
|
||||
const pluginConfig = server.pluginConfigs.get(pluginConfigId)
|
||||
const teamIdStr = pluginConfig?.team_id.toString() || '?'
|
||||
try {
|
||||
const task = await pluginConfig?.vm?.getTask(taskName, taskType)
|
||||
if (!task) {
|
||||
@@ -161,10 +162,6 @@ export async function runPluginTask(
|
||||
response = await (payload ? task?.exec(payload) : task?.exec())
|
||||
} catch (error) {
|
||||
await processError(server, pluginConfig || null, error)
|
||||
let teamIdStr = '?'
|
||||
if (pluginConfig != null) {
|
||||
teamIdStr = pluginConfig.team_id.toString()
|
||||
}
|
||||
|
||||
server.statsd?.increment(`plugin.task.ERROR`, {
|
||||
taskType: taskType,
|
||||
@@ -173,17 +170,13 @@ export async function runPluginTask(
|
||||
teamId: teamIdStr,
|
||||
})
|
||||
}
|
||||
server.statsd?.timing(`plugin.task`, timer, {
|
||||
plugin: pluginConfig?.plugin?.name ?? '?',
|
||||
teamId: teamIdStr,
|
||||
})
|
||||
return response
|
||||
}
|
||||
|
||||
function getPluginsForTeam(server: Hub, teamId: number): PluginConfig[] {
|
||||
return server.pluginConfigsPerTeam.get(teamId) || []
|
||||
}
|
||||
|
||||
function getPluginsForTeams(server: Hub, teamIds: TeamId[]) {
|
||||
let plugins: PluginConfig[] = []
|
||||
for (const teamId of teamIds) {
|
||||
plugins = plugins.concat(getPluginsForTeam(server, teamId))
|
||||
}
|
||||
return plugins
|
||||
}
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
import { ProcessedPluginEvent } from '@posthog/plugin-scaffold'
|
||||
import { PluginEvent } from '@posthog/plugin-scaffold/src/types'
|
||||
|
||||
import { Action, Alert, EnqueuedJob, Hub, PluginTaskType, PreIngestionEvent, Team } from '../types'
|
||||
import { Action, EnqueuedJob, Hub, PluginTaskType, PreIngestionEvent, Team } from '../types'
|
||||
import { convertToProcessedPluginEvent } from '../utils/event'
|
||||
import { EventPipelineRunner } from './ingestion/event-pipeline/runner'
|
||||
import { runPluginTask, runProcessEvent } from './plugins/run'
|
||||
@@ -34,6 +34,7 @@ export const workerTasks: Record<string, TaskRunner> = {
|
||||
const runner = new EventPipelineRunner(hub, convertToProcessedPluginEvent(args.event))
|
||||
return await runner.runBufferEventPipeline(args.event)
|
||||
},
|
||||
// eslint-disable-next-line @typescript-eslint/no-unused-vars
|
||||
runAsyncHandlersEventPipeline: async (hub, args: { event: ProcessedPluginEvent }) => {
|
||||
await Promise.resolve()
|
||||
return
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
import { Plugin, PluginEvent, PluginMeta, ProcessedPluginEvent, RetryError } from '@posthog/plugin-scaffold'
|
||||
|
||||
import { Hub, MetricMathOperations, PluginConfig, PluginConfigVMInternalResponse, PluginTaskType } from '../../../types'
|
||||
import { Hub, PluginConfig, PluginConfigVMInternalResponse, PluginTaskType } from '../../../types'
|
||||
import { determineNodeEnv, NodeEnv } from '../../../utils/env-utils'
|
||||
import { status } from '../../../utils/status'
|
||||
import { stringClamp } from '../../../utils/utils'
|
||||
|
||||
@@ -2,7 +2,6 @@ import { PluginEvent, PluginMeta, RetryError } from '@posthog/plugin-scaffold'
|
||||
|
||||
import {
|
||||
Hub,
|
||||
MetricMathOperations,
|
||||
PluginConfig,
|
||||
PluginConfigVMInternalResponse,
|
||||
PluginLogEntrySource,
|
||||
@@ -319,10 +318,10 @@ export function addHistoricalEventsExportCapability(
|
||||
const progress = progressDenominator === 0 ? 20 : Math.round(progressNumerator / progressDenominator) * 20
|
||||
|
||||
const progressBarCompleted = Array.from({ length: progress })
|
||||
.map((_) => '■')
|
||||
.map(() => '■')
|
||||
.join('')
|
||||
const progressBarRemaining = Array.from({ length: 20 - progress })
|
||||
.map((_) => '□')
|
||||
.map(() => '□')
|
||||
.join('')
|
||||
createLog(`Export progress: ${progressBarCompleted}${progressBarRemaining}`)
|
||||
}
|
||||
|
||||
@@ -25,7 +25,7 @@ export class ExportEventsBuffer {
|
||||
this.hub = hub
|
||||
}
|
||||
|
||||
public async add(object: any, points = 1): Promise<void> {
|
||||
public async add(object: Record<string, any>, points = 1): Promise<void> {
|
||||
// flush existing if adding would make us go over the limit
|
||||
if (this.points && this.points + points > this.options.limit) {
|
||||
await this.flush()
|
||||
|
||||
@@ -3,7 +3,6 @@ import { Plugin } from '@posthog/plugin-scaffold'
|
||||
import { DateTime } from 'luxon'
|
||||
import { Client } from 'pg'
|
||||
|
||||
import { defaultConfig } from '../../../../config/config'
|
||||
import { ClickHouseEvent, Element, Event, TimestampFormat } from '../../../../types'
|
||||
import { DB } from '../../../../utils/db/db'
|
||||
import { chainToElements, transformPostgresElementsToEventPayloadFormat } from '../../../../utils/db/utils'
|
||||
|
||||
@@ -1,4 +1,3 @@
|
||||
import { StatsD } from 'hot-shots'
|
||||
import { QueryResult } from 'pg'
|
||||
|
||||
import { PluginConfig } from '../../types'
|
||||
@@ -32,7 +31,7 @@ export const postgresSetOnce = async (
|
||||
key: string,
|
||||
value: number
|
||||
): Promise<void> => {
|
||||
const se = await db.postgresQuery(
|
||||
await db.postgresQuery(
|
||||
`
|
||||
INSERT INTO posthog_pluginstorage (plugin_config_id, key, value)
|
||||
VALUES ($1, $2, $3)
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
import { PluginEvent } from '@posthog/plugin-scaffold/src/types'
|
||||
|
||||
import { Hub, LogLevel, PluginsServerConfig } from '../../src/types'
|
||||
import { Hub, LogLevel } from '../../src/types'
|
||||
import { createHub } from '../../src/utils/db/hub'
|
||||
import { UUIDT } from '../../src/utils/utils'
|
||||
import { generateEventDeadLetterQueueMessage } from '../../src/worker/ingestion/utils'
|
||||
@@ -18,7 +18,7 @@ jest.mock('../../src/worker/ingestion/utils', () => {
|
||||
})
|
||||
|
||||
class MockEventsProcessor {
|
||||
public async processEvent(...args: any[]) {
|
||||
public async processEvent() {
|
||||
await new Promise<void>((resolve) => resolve())
|
||||
throw new Error('database unavailable')
|
||||
}
|
||||
|
||||
@@ -1,5 +1,3 @@
|
||||
import Piscina from '@posthog/piscina'
|
||||
|
||||
import { KAFKA_EVENTS_PLUGIN_INGESTION } from '../../../src/config/kafka-topics'
|
||||
import { ServerInstance, startPluginsServer } from '../../../src/main/pluginsServer'
|
||||
import { LogLevel, PluginsServerConfig } from '../../../src/types'
|
||||
@@ -31,7 +29,6 @@ describe.skip('KafkaQueue', () => {
|
||||
let hub: Hub
|
||||
let stopServer: () => Promise<void>
|
||||
let posthog: DummyPostHog
|
||||
let piscina: Piscina
|
||||
let pluginServer: ServerInstance
|
||||
|
||||
beforeAll(async () => {
|
||||
|
||||
@@ -2,7 +2,6 @@ import { Consumer } from 'kafkajs'
|
||||
|
||||
import { KAFKA_HEALTHCHECK } from '../../src/config/kafka-topics'
|
||||
import { kafkaHealthcheck, setupKafkaHealthcheckConsumer } from '../../src/main/utils'
|
||||
import { PluginsServerConfig } from '../../src/types'
|
||||
import { Hub } from '../../src/types'
|
||||
import { createHub } from '../../src/utils/db/hub'
|
||||
import { resetKafka } from '../helpers/kafka'
|
||||
|
||||
@@ -4,7 +4,7 @@ import IORedis from 'ioredis'
|
||||
import { ONE_HOUR } from '../src/config/constants'
|
||||
import { KAFKA_EVENTS_PLUGIN_INGESTION } from '../src/config/kafka-topics'
|
||||
import { startPluginsServer } from '../src/main/pluginsServer'
|
||||
import { AlertLevel, LogLevel, PluginsServerConfig, Service } from '../src/types'
|
||||
import { LogLevel, PluginsServerConfig } from '../src/types'
|
||||
import { Hub } from '../src/types'
|
||||
import { delay, UUIDT } from '../src/utils/utils'
|
||||
import { makePiscina } from '../src/worker/piscina'
|
||||
|
||||
@@ -1,5 +1,3 @@
|
||||
import Piscina from '@posthog/piscina'
|
||||
|
||||
import { KAFKA_EVENTS_PLUGIN_INGESTION } from '../src/config/kafka-topics'
|
||||
import { startPluginsServer } from '../src/main/pluginsServer'
|
||||
import { Hub, LogLevel, PluginsServerConfig } from '../src/types'
|
||||
@@ -24,7 +22,6 @@ describe('e2e ingestion timeout', () => {
|
||||
let hub: Hub
|
||||
let stopServer: () => Promise<void>
|
||||
let posthog: DummyPostHog
|
||||
let piscina: Piscina
|
||||
|
||||
beforeAll(async () => {
|
||||
await resetKafka(extraServerConfig)
|
||||
@@ -45,7 +42,6 @@ describe('e2e ingestion timeout', () => {
|
||||
await resetTestDatabaseClickhouse(extraServerConfig)
|
||||
const startResponse = await startPluginsServer(extraServerConfig, makePiscina)
|
||||
hub = startResponse.hub
|
||||
piscina = startResponse.piscina
|
||||
stopServer = startResponse.stop
|
||||
posthog = createPosthog(hub, pluginConfig39)
|
||||
})
|
||||
|
||||
@@ -2,7 +2,6 @@ import { gzipSync } from 'zlib'
|
||||
|
||||
import { defaultConfig } from '../src/config/config'
|
||||
import { LOCKED_RESOURCE } from '../src/main/job-queues/job-queue-consumer'
|
||||
import { JobQueueManager } from '../src/main/job-queues/job-queue-manager'
|
||||
import { ServerInstance, startPluginsServer } from '../src/main/pluginsServer'
|
||||
import { EnqueuedJob, Hub, LogLevel, PluginsServerConfig } from '../src/types'
|
||||
import { createHub } from '../src/utils/db/hub'
|
||||
@@ -167,7 +166,7 @@ describe.skip('job queues', () => {
|
||||
}
|
||||
|
||||
server.hub.jobQueueManager.enqueue(job)
|
||||
const consumedJob: EnqueuedJob = await new Promise((resolve, reject) => {
|
||||
const consumedJob: EnqueuedJob = await new Promise((resolve) => {
|
||||
server.hub.jobQueueManager.startConsumer((consumedJob) => {
|
||||
resolve(consumedJob[0])
|
||||
})
|
||||
@@ -225,7 +224,6 @@ describe.skip('job queues', () => {
|
||||
})
|
||||
|
||||
describe('s3 queue', () => {
|
||||
let jobQueue: JobQueueManager
|
||||
let hub: Hub
|
||||
let closeHub: () => Promise<void>
|
||||
|
||||
@@ -296,7 +294,7 @@ describe.skip('job queues', () => {
|
||||
Body: gzipSync(Buffer.from(JSON.stringify(job), 'utf8')),
|
||||
})
|
||||
|
||||
const consumedJob: EnqueuedJob = await new Promise((resolve, reject) => {
|
||||
const consumedJob: EnqueuedJob = await new Promise((resolve) => {
|
||||
hub.jobQueueManager.startConsumer((consumedJob) => {
|
||||
resolve(consumedJob[0])
|
||||
})
|
||||
@@ -335,7 +333,7 @@ describe.skip('job queues', () => {
|
||||
Body: gzipSync(Buffer.from(JSON.stringify(job), 'utf8')),
|
||||
})
|
||||
|
||||
const consumedJob: EnqueuedJob = await new Promise((resolve, reject) => {
|
||||
const consumedJob: EnqueuedJob = await new Promise((resolve) => {
|
||||
hub.jobQueueManager.startConsumer((consumedJob) => {
|
||||
resolve(consumedJob[0])
|
||||
})
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
import { Hub } from '../src/types'
|
||||
import { createHub } from '../src/utils/db/hub'
|
||||
import { PluginsApiKeyManager } from '../src/worker/vm/extensions/helpers/api-key-manager'
|
||||
import { createUserTeamAndOrganization, resetTestDatabase } from './helpers/sql'
|
||||
import { createUserTeamAndOrganization } from './helpers/sql'
|
||||
import { POSTGRES_TRUNCATE_TABLES_QUERY } from './helpers/sql'
|
||||
|
||||
describe('PluginsApiKeyManager', () => {
|
||||
|
||||
@@ -120,7 +120,7 @@ describe('plugins', () => {
|
||||
getPluginConfigRows.mockReturnValueOnce([pluginConfig39, { ...pluginConfig39, id: 40, team_id: 1 }])
|
||||
|
||||
await setupPlugins(hub)
|
||||
const { plugins, pluginConfigs } = hub
|
||||
const { pluginConfigs } = hub
|
||||
|
||||
expect(getPluginRows).toHaveBeenCalled()
|
||||
expect(getPluginAttachmentRows).toHaveBeenCalled()
|
||||
|
||||
@@ -71,7 +71,7 @@ describe('teardown', () => {
|
||||
throw new Error('This Happened In The Teardown Palace')
|
||||
}
|
||||
`)
|
||||
const { piscina, stop } = await startPluginsServer(
|
||||
const { stop } = await startPluginsServer(
|
||||
{
|
||||
WORKER_CONCURRENCY: 2,
|
||||
LOG_LEVEL: LogLevel.Log,
|
||||
|
||||
@@ -75,7 +75,6 @@ describe('DB', () => {
|
||||
|
||||
describe('createPerson', () => {
|
||||
let team: Team
|
||||
let person: Person
|
||||
const uuid = new UUIDT().toString()
|
||||
const distinctId = 'distinct_id1'
|
||||
|
||||
|
||||
@@ -114,16 +114,7 @@ async function processEvent(
|
||||
sentAt: DateTime | null,
|
||||
eventUuid: string
|
||||
): Promise<PreIngestionEvent | null> {
|
||||
const response = await eventsProcessor.processEvent(
|
||||
distinctId,
|
||||
ip,
|
||||
data,
|
||||
teamId,
|
||||
now,
|
||||
sentAt,
|
||||
eventUuid,
|
||||
'http://example.com'
|
||||
)
|
||||
const response = await eventsProcessor.processEvent(distinctId, ip, data, teamId, now, sentAt, eventUuid)
|
||||
if (response) {
|
||||
await eventsProcessor.createEvent(response)
|
||||
}
|
||||
@@ -1169,8 +1160,7 @@ it('snapshot event not stored if session recording disabled', async () => {
|
||||
team.id,
|
||||
now,
|
||||
now,
|
||||
new UUIDT().toString(),
|
||||
'http://example.com'
|
||||
new UUIDT().toString()
|
||||
)
|
||||
await delayUntilEventIngested(() => hub.db.fetchSessionRecordingEvents())
|
||||
|
||||
@@ -1192,8 +1182,7 @@ test('snapshot event stored as session_recording_event', async () => {
|
||||
team.id,
|
||||
now,
|
||||
now,
|
||||
new UUIDT().toString(),
|
||||
'http://example.com'
|
||||
new UUIDT().toString()
|
||||
)
|
||||
await delayUntilEventIngested(() => hub.db.fetchSessionRecordingEvents())
|
||||
|
||||
@@ -1220,8 +1209,7 @@ test('$snapshot event creates new person if needed', async () => {
|
||||
team.id,
|
||||
now,
|
||||
now,
|
||||
new UUIDT().toString(),
|
||||
'http://example.com'
|
||||
new UUIDT().toString()
|
||||
)
|
||||
await delayUntilEventIngested(() => hub.db.fetchPersons())
|
||||
|
||||
|
||||
@@ -1,4 +1,3 @@
|
||||
import { PluginsServerConfig } from '../../src/types'
|
||||
import { delay } from '../../src/utils/utils'
|
||||
import { PromiseManager } from '../../src/worker/vm/promise-manager'
|
||||
import { Hub } from './../../src/types'
|
||||
|
||||
@@ -1,11 +1,9 @@
|
||||
import { PluginEvent } from '@posthog/plugin-scaffold/src/types'
|
||||
import fetch from 'node-fetch'
|
||||
import { MockedFunction } from 'ts-jest/dist/utils/testing'
|
||||
|
||||
import { Hook, Hub } from '../../../../src/types'
|
||||
import { createHub } from '../../../../src/utils/db/hub'
|
||||
import { UUIDT } from '../../../../src/utils/utils'
|
||||
import { ActionMatcher } from '../../../../src/worker/ingestion/action-matcher'
|
||||
import { EventPipelineRunner } from '../../../../src/worker/ingestion/event-pipeline/runner'
|
||||
import { commonUserId } from '../../../helpers/plugins'
|
||||
import { insertRow, resetTestDatabase } from '../../../helpers/sql'
|
||||
@@ -13,14 +11,12 @@ import { insertRow, resetTestDatabase } from '../../../helpers/sql'
|
||||
describe('Event Pipeline integration test', () => {
|
||||
let hub: Hub
|
||||
let closeServer: () => Promise<void>
|
||||
let actionMatcher: ActionMatcher
|
||||
|
||||
const ingestEvent = (event: PluginEvent) => new EventPipelineRunner(hub, event).runEventPipeline(event)
|
||||
|
||||
beforeEach(async () => {
|
||||
await resetTestDatabase()
|
||||
;[hub, closeServer] = await createHub()
|
||||
actionMatcher = hub.actionMatcher
|
||||
})
|
||||
|
||||
afterEach(async () => {
|
||||
@@ -116,8 +112,8 @@ describe('Event Pipeline integration test', () => {
|
||||
|
||||
// Using a more verbose way instead of toHaveBeenCalledWith because we need to parse request body
|
||||
// and use expect.any for a few payload properties, which wouldn't be possible in a simpler way
|
||||
expect((fetch as MockedFunction<typeof fetch>).mock.calls[0][0]).toBe('https://rest-hooks.example.com/')
|
||||
const secondArg = (fetch as MockedFunction<typeof fetch>).mock.calls[0][1]
|
||||
expect(jest.mocked(fetch).mock.calls[0][0]).toBe('https://rest-hooks.example.com/')
|
||||
const secondArg = jest.mocked(fetch).mock.calls[0][1]
|
||||
expect(JSON.parse(secondArg!.body as unknown as string)).toStrictEqual(expectedPayload)
|
||||
expect(JSON.parse(secondArg!.body as unknown as string)).toStrictEqual(expectedPayload)
|
||||
expect(secondArg!.headers).toStrictEqual({ 'Content-Type': 'application/json' })
|
||||
|
||||
@@ -1,4 +1,3 @@
|
||||
import { PluginEvent } from '@posthog/plugin-scaffold'
|
||||
import { DateTime } from 'luxon'
|
||||
import * as fetch from 'node-fetch'
|
||||
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
import { Properties } from '@posthog/plugin-scaffold'
|
||||
import { DateTime } from 'luxon'
|
||||
|
||||
import { Group, Hub, Person, Team } from '../../../src/types'
|
||||
import { Group, Hub, Team } from '../../../src/types'
|
||||
import { DB } from '../../../src/utils/db/db'
|
||||
import { createHub } from '../../../src/utils/db/hub'
|
||||
import { UUIDT } from '../../../src/utils/utils'
|
||||
@@ -17,12 +17,10 @@ describe('properties-updater', () => {
|
||||
let db: DB
|
||||
|
||||
let team: Team
|
||||
let person: Person
|
||||
const uuid = new UUIDT().toString()
|
||||
const distinctId = 'distinct_id_update_person_properties'
|
||||
|
||||
const FUTURE_TIMESTAMP = DateTime.fromISO('2050-10-14T11:42:06.502Z')
|
||||
const MIDDLE_TIMESTAMP = DateTime.fromISO('2021-10-14T11:42:06.502Z')
|
||||
const PAST_TIMESTAMP = DateTime.fromISO('2000-10-14T11:42:06.502Z')
|
||||
|
||||
beforeEach(async () => {
|
||||
@@ -31,7 +29,7 @@ describe('properties-updater', () => {
|
||||
db = hub.db
|
||||
|
||||
team = await getFirstTeam(hub)
|
||||
person = await db.createPerson(PAST_TIMESTAMP, {}, {}, {}, team.id, null, false, uuid, [distinctId])
|
||||
await db.createPerson(PAST_TIMESTAMP, {}, {}, {}, team.id, null, false, uuid, [distinctId])
|
||||
})
|
||||
|
||||
afterEach(async () => {
|
||||
|
||||
@@ -4595,6 +4595,13 @@ eslint-plugin-node@^11.1.0:
|
||||
resolve "^1.10.1"
|
||||
semver "^6.1.0"
|
||||
|
||||
eslint-plugin-prettier@^4.0.0:
|
||||
version "4.0.0"
|
||||
resolved "https://registry.yarnpkg.com/eslint-plugin-prettier/-/eslint-plugin-prettier-4.0.0.tgz#8b99d1e4b8b24a762472b4567992023619cb98e0"
|
||||
integrity sha512-98MqmCJ7vJodoQK359bqQWaxOE0CS8paAz/GgjaZLyex4TTk3g9HugoO89EqWCrFiOqn9EVvcoo7gZzONCWVwQ==
|
||||
dependencies:
|
||||
prettier-linter-helpers "^1.0.0"
|
||||
|
||||
eslint-plugin-promise@^5.1.0:
|
||||
version "5.1.0"
|
||||
resolved "https://registry.yarnpkg.com/eslint-plugin-promise/-/eslint-plugin-promise-5.1.0.tgz#fb2188fb734e4557993733b41aa1a688f46c6f24"
|
||||
@@ -4891,6 +4898,11 @@ fast-deep-equal@^3.1.1, fast-deep-equal@^3.1.3:
|
||||
resolved "https://registry.yarnpkg.com/fast-deep-equal/-/fast-deep-equal-3.1.3.tgz#3a7d56b559d6cbc3eb512325244e619a65c6c525"
|
||||
integrity sha512-f3qQ9oQy9j2AhBe/H9VC91wLmKBCCU/gDOnKNAYG5hswO7BLKj09Hc5HYNz9cGI++xlpDCIgDaitVs03ATR84Q==
|
||||
|
||||
fast-diff@^1.1.2:
|
||||
version "1.2.0"
|
||||
resolved "https://registry.yarnpkg.com/fast-diff/-/fast-diff-1.2.0.tgz#73ee11982d86caaf7959828d519cfe927fac5f03"
|
||||
integrity sha512-xJuoT5+L99XlZ8twedaRf6Ax2TgQVxvgZOYoPKqZufmJib0tL2tegPBOZb1pVNgIhlqDlA0eO0c3wBvQcmzx4w==
|
||||
|
||||
fast-glob@^3.1.1:
|
||||
version "3.2.4"
|
||||
resolved "https://registry.yarnpkg.com/fast-glob/-/fast-glob-3.2.4.tgz#d20aefbf99579383e7f3cc66529158c9b98554d3"
|
||||
@@ -7889,6 +7901,13 @@ prelude-ls@~1.1.2:
|
||||
resolved "https://registry.yarnpkg.com/prelude-ls/-/prelude-ls-1.1.2.tgz#21932a549f5e52ffd9a827f570e04be62a97da54"
|
||||
integrity sha1-IZMqVJ9eUv/ZqCf1cOBL5iqX2lQ=
|
||||
|
||||
prettier-linter-helpers@^1.0.0:
|
||||
version "1.0.0"
|
||||
resolved "https://registry.yarnpkg.com/prettier-linter-helpers/-/prettier-linter-helpers-1.0.0.tgz#d23d41fe1375646de2d0104d3454a3008802cf7b"
|
||||
integrity sha512-GbK2cP9nraSSUF9N2XwUwqfzlAFlMNYYl+ShE/V+H8a9uNl/oUqB1w2EL54Jh0OlyRSd8RfWYJ3coVS4TROP2w==
|
||||
dependencies:
|
||||
fast-diff "^1.1.2"
|
||||
|
||||
prettier@^2.3.1:
|
||||
version "2.3.1"
|
||||
resolved "https://registry.yarnpkg.com/prettier/-/prettier-2.3.1.tgz#76903c3f8c4449bc9ac597acefa24dc5ad4cbea6"
|
||||
|
||||
Reference in New Issue
Block a user