diff --git a/frontend/__snapshots__/components-playerinspector-itemevent--page-view-with-current-url--dark.png b/frontend/__snapshots__/components-playerinspector-itemevent--page-view-with-current-url--dark.png index 63fef605af..86c95a0c7e 100644 Binary files a/frontend/__snapshots__/components-playerinspector-itemevent--page-view-with-current-url--dark.png and b/frontend/__snapshots__/components-playerinspector-itemevent--page-view-with-current-url--dark.png differ diff --git a/frontend/__snapshots__/scenes-app-data-pipelines--pipeline-destination-page--light.png b/frontend/__snapshots__/scenes-app-data-pipelines--pipeline-destination-page--light.png index 53f2163297..0d128526cb 100644 Binary files a/frontend/__snapshots__/scenes-app-data-pipelines--pipeline-destination-page--light.png and b/frontend/__snapshots__/scenes-app-data-pipelines--pipeline-destination-page--light.png differ diff --git a/frontend/public/services/customer-io.png b/frontend/public/services/customer-io.png new file mode 100644 index 0000000000..287fd15d5a Binary files /dev/null and b/frontend/public/services/customer-io.png differ diff --git a/frontend/src/queries/schema.json b/frontend/src/queries/schema.json index d9190eb01d..2bf798017e 100644 --- a/frontend/src/queries/schema.json +++ b/frontend/src/queries/schema.json @@ -13387,6 +13387,7 @@ }, "ExternalDataSourceType": { "enum": [ + "CustomerIO", "Github", "Stripe", "Hubspot", diff --git a/frontend/src/queries/schema/schema-general.ts b/frontend/src/queries/schema/schema-general.ts index 85e3503dde..11f5643ba0 100644 --- a/frontend/src/queries/schema/schema-general.ts +++ b/frontend/src/queries/schema/schema-general.ts @@ -4237,6 +4237,7 @@ export interface SourceConfig { } export const externalDataSources = [ + 'CustomerIO', 'Github', 'Stripe', 'Hubspot', diff --git a/frontend/src/scenes/insights/stories/TrendsLine.stories.tsx b/frontend/src/scenes/insights/stories/TrendsLine.stories.tsx index 262ceeac92..846a2bf141 100644 --- a/frontend/src/scenes/insights/stories/TrendsLine.stories.tsx +++ b/frontend/src/scenes/insights/stories/TrendsLine.stories.tsx @@ -43,7 +43,11 @@ export const TrendsLine: Story = createInsightStory( require('../../../mocks/fixtures/api/projects/team_id/insights/trendsLine.json') ) TrendsLine.parameters = { - testOptions: { waitForSelector: '[data-attr=trend-line-graph] > canvas' }, + ...meta.parameters, + testOptions: { + ...meta.parameters?.testOptions, + waitForSelector: '[data-attr=trend-line-graph] > canvas', + }, } // FLAP! @@ -59,28 +63,44 @@ export const TrendsLineMulti: Story = createInsightStory( require('../../../mocks/fixtures/api/projects/team_id/insights/trendsLineMulti.json') ) TrendsLineMulti.parameters = { - testOptions: { waitForSelector: '[data-attr=trend-line-graph] > canvas' }, + ...meta.parameters, + testOptions: { + ...meta.parameters?.testOptions, + waitForSelector: '[data-attr=trend-line-graph] > canvas', + }, } export const TrendsLineMultiEdit: Story = createInsightStory( require('../../../mocks/fixtures/api/projects/team_id/insights/trendsLineMulti.json'), 'edit' ) TrendsLineMultiEdit.parameters = { - testOptions: { waitForSelector: '[data-attr=trend-line-graph] > canvas' }, + ...meta.parameters, + testOptions: { + ...meta.parameters?.testOptions, + waitForSelector: '[data-attr=trend-line-graph] > canvas', + }, } export const TrendsLineBreakdown: Story = createInsightStory( require('../../../mocks/fixtures/api/projects/team_id/insights/trendsLineBreakdown.json') ) TrendsLineBreakdown.parameters = { - testOptions: { waitForSelector: '[data-attr=trend-line-graph] > canvas' }, + ...meta.parameters, + testOptions: { + ...meta.parameters?.testOptions, + waitForSelector: '[data-attr=trend-line-graph] > canvas', + }, } export const TrendsLineBreakdownEdit: Story = createInsightStory( require('../../../mocks/fixtures/api/projects/team_id/insights/trendsLineBreakdown.json'), 'edit' ) TrendsLineBreakdownEdit.parameters = { - testOptions: { waitForSelector: '[data-attr=trend-line-graph] > canvas' }, + ...meta.parameters, + testOptions: { + ...meta.parameters?.testOptions, + waitForSelector: '[data-attr=trend-line-graph] > canvas', + }, } export const TrendsLineBreakdownLabels: Story = createInsightStory( @@ -89,7 +109,11 @@ export const TrendsLineBreakdownLabels: Story = createInsightStory( true ) TrendsLineBreakdownLabels.parameters = { - testOptions: { waitForSelector: '[data-attr=trend-line-graph] > canvas' }, + ...meta.parameters, + testOptions: { + ...meta.parameters?.testOptions, + waitForSelector: '[data-attr=trend-line-graph] > canvas', + }, } // Trends Bar @@ -97,27 +121,43 @@ export const TrendsBar: Story = createInsightStory( require('../../../mocks/fixtures/api/projects/team_id/insights/trendsBar.json') ) TrendsBar.parameters = { - testOptions: { waitForSelector: '[data-attr=trend-line-graph] > canvas' }, + ...meta.parameters, + testOptions: { + ...meta.parameters?.testOptions, + waitForSelector: '[data-attr=trend-line-graph] > canvas', + }, } export const TrendsBarEdit: Story = createInsightStory( require('../../../mocks/fixtures/api/projects/team_id/insights/trendsBar.json'), 'edit' ) TrendsBarEdit.parameters = { - testOptions: { waitForSelector: '[data-attr=trend-line-graph] > canvas' }, + ...meta.parameters, + testOptions: { + ...meta.parameters?.testOptions, + waitForSelector: '[data-attr=trend-line-graph] > canvas', + }, } export const TrendsBarBreakdown: Story = createInsightStory( require('../../../mocks/fixtures/api/projects/team_id/insights/trendsBarBreakdown.json') ) TrendsBarBreakdown.parameters = { - testOptions: { waitForSelector: '[data-attr=trend-line-graph] > canvas' }, + ...meta.parameters, + testOptions: { + ...meta.parameters?.testOptions, + waitForSelector: '[data-attr=trend-line-graph] > canvas', + }, } export const TrendsBarBreakdownEdit: Story = createInsightStory( require('../../../mocks/fixtures/api/projects/team_id/insights/trendsBarBreakdown.json'), 'edit' ) TrendsBarBreakdownEdit.parameters = { - testOptions: { waitForSelector: '[data-attr=trend-line-graph] > canvas' }, + ...meta.parameters, + testOptions: { + ...meta.parameters?.testOptions, + waitForSelector: '[data-attr=trend-line-graph] > canvas', + }, } /* eslint-enable @typescript-eslint/no-var-requires */ diff --git a/frontend/src/scenes/session-recordings/player/inspector/components/ItemEvent.stories.tsx b/frontend/src/scenes/session-recordings/player/inspector/components/ItemEvent.stories.tsx index d758bb225a..9381378f9e 100644 --- a/frontend/src/scenes/session-recordings/player/inspector/components/ItemEvent.stories.tsx +++ b/frontend/src/scenes/session-recordings/player/inspector/components/ItemEvent.stories.tsx @@ -1,6 +1,6 @@ import { Meta, StoryFn, StoryObj } from '@storybook/react' -import { now } from 'lib/dayjs' +import { dayjs } from 'lib/dayjs' import { LemonDivider } from 'lib/lemon-ui/LemonDivider' import { uuid } from 'lib/utils' import { @@ -33,6 +33,7 @@ function makeItem( dataOverrides: Partial = {}, propertiesOverrides: Record = {} ): InspectorListItemEvent { + const mockDate = dayjs('2025-11-04') const data: RecordingEventType = { elements: [], event: '', @@ -40,7 +41,7 @@ function makeItem( id: '', playerTime: 0, - timestamp: now().toISOString(), + timestamp: mockDate.toISOString(), ...dataOverrides, // this is last so that it overrides data overrides sensibly 🙃 properties: { @@ -51,7 +52,7 @@ function makeItem( data: data, search: '', timeInRecording: 0, - timestamp: now(), + timestamp: mockDate, type: 'events', key: `some-key-${uuid()}`, ...itemOverrides, diff --git a/playwright/__snapshots__/pageview-trends-insight.png b/playwright/__snapshots__/pageview-trends-insight.png index b4a1183a25..68307b782e 100644 Binary files a/playwright/__snapshots__/pageview-trends-insight.png and b/playwright/__snapshots__/pageview-trends-insight.png differ diff --git a/posthog/management/commands/create_datastack_source.py b/posthog/management/commands/create_datastack_source.py index 4eccd1280a..f3f9e5f86f 100644 --- a/posthog/management/commands/create_datastack_source.py +++ b/posthog/management/commands/create_datastack_source.py @@ -17,6 +17,7 @@ from products.data_warehouse.backend import types os.environ["DEBUG"] = "1" os.environ["SKIP_ASYNC_MIGRATIONS_SETUP"] = "1" + SOURCE_TEMPLATE = """\ from typing import cast @@ -82,7 +83,7 @@ class Migration(migrations.Migration): model_name="externaldatasource", name="source_type", field=models.CharField( - choices=[ {choices} ], + choices={choices}, max_length=128, ), ), @@ -137,6 +138,7 @@ class Command(BaseCommand): "constant": textcase.constant(name), "caps": textcase.constant(name).replace("_", ""), } + self._fix_common_endings(transforms=name_transforms) logo_filename = f"{name_transforms['kebab']}.png" self._setup_source_structure(repo, transforms=name_transforms) @@ -161,6 +163,12 @@ class Command(BaseCommand): self._schema_build(transforms=name_transforms) self._format_files() + def _fix_common_endings(self, transforms: NameTransforms): + common_endings = ("Io", "Ai", "Db", "Ci") + for end in common_endings: + if transforms["pascal"].endswith(end): + transforms["pascal"] = transforms["pascal"][: -len(end)] + end.upper() + def _setup_source_structure(self, repo: Repo, transforms: NameTransforms): sources_root = os.path.join(repo.working_dir, "posthog", "temporal", "data_imports", "sources") assert os.path.exists(sources_root), f"Sources root {sources_root} not found" diff --git a/posthog/management/commands/test/test_create_datastack_source.py b/posthog/management/commands/test/test_create_datastack_source.py index 8ddba2beb2..b9e54e4a3a 100644 --- a/posthog/management/commands/test/test_create_datastack_source.py +++ b/posthog/management/commands/test/test_create_datastack_source.py @@ -2,7 +2,7 @@ import pathlib import pytest -from posthog.management.commands.create_datastack_source import Command +from posthog.management.commands.create_datastack_source import Command, NameTransforms TEST_FILE_CONTENT = "a b c d e f" TEST_BLOCK = """\ @@ -19,6 +19,11 @@ def command(): return Command() +@pytest.fixture +def transforms(): + return NameTransforms(snake="", kebab="", pascal="", caps="", constant="") + + def test_split_file_by_regex(command: Command, tmp_path: pathlib.Path): path = tmp_path / "test.file" path.write_text(TEST_FILE_CONTENT) @@ -43,3 +48,21 @@ def test_format_file_line(command: Command): expected = " a" assert command._format_file_line("a", indent_level=2, end="") + + +def test_fix_common_endings(command: Command, transforms: NameTransforms): + transforms["pascal"] = "CustomerIo" + command._fix_common_endings(transforms) + assert transforms["pascal"] == "CustomerIO" + + transforms["pascal"] = "Something.Ai" + command._fix_common_endings(transforms) + assert transforms["pascal"] == "Something.AI" + + transforms["pascal"] = "ImpressiveCi" + command._fix_common_endings(transforms) + assert transforms["pascal"] == "ImpressiveCI" + + transforms["pascal"] = "DuckDb" + command._fix_common_endings(transforms) + assert transforms["pascal"] == "DuckDB" diff --git a/posthog/schema.py b/posthog/schema.py index 3f049d80de..24e071dfd4 100644 --- a/posthog/schema.py +++ b/posthog/schema.py @@ -1370,6 +1370,7 @@ class ExperimentVariantTrendsBaseStats(BaseModel): class ExternalDataSourceType(StrEnum): + CUSTOMER_IO = "CustomerIO" GITHUB = "Github" STRIPE = "Stripe" HUBSPOT = "Hubspot" diff --git a/posthog/temporal/data_imports/sources/__init__.py b/posthog/temporal/data_imports/sources/__init__.py index 7e1fe61858..9272e7c30c 100644 --- a/posthog/temporal/data_imports/sources/__init__.py +++ b/posthog/temporal/data_imports/sources/__init__.py @@ -2,6 +2,7 @@ from .bigquery.source import BigQuerySource from .braze.source import BrazeSource from .chargebee.source import ChargebeeSource from .common.registry import SourceRegistry +from .customer_io.source import CustomerIOSource from .doit.source import DoItSource from .github.source import GithubSource from .google_ads.source import GoogleAdsSource @@ -30,6 +31,7 @@ from .vitally.source import VitallySource from .zendesk.source import ZendeskSource __all__ = [ + "CustomerIOSource", "GithubSource", "SourceRegistry", "BigQuerySource", diff --git a/posthog/temporal/data_imports/sources/common/test/__snapshots__/test_source_config_generator.ambr b/posthog/temporal/data_imports/sources/common/test/__snapshots__/test_source_config_generator.ambr index f11b0c84b0..5a3f90a40f 100644 --- a/posthog/temporal/data_imports/sources/common/test/__snapshots__/test_source_config_generator.ambr +++ b/posthog/temporal/data_imports/sources/common/test/__snapshots__/test_source_config_generator.ambr @@ -46,6 +46,7 @@ ExternalDataSourceType.BIGQUERY: BigQuerySourceConfig, ExternalDataSourceType.BRAZE: BrazeSourceConfig, ExternalDataSourceType.CHARGEBEE: ChargebeeSourceConfig, + ExternalDataSourceType.CUSTOMERIO: CustomerIOSourceConfig, ExternalDataSourceType.DOIT: DoItSourceConfig, ExternalDataSourceType.GITHUB: GithubSourceConfig, ExternalDataSourceType.GOOGLEADS: GoogleAdsSourceConfig, diff --git a/posthog/temporal/data_imports/sources/customer_io/source.py b/posthog/temporal/data_imports/sources/customer_io/source.py new file mode 100644 index 0000000000..95d16c389d --- /dev/null +++ b/posthog/temporal/data_imports/sources/customer_io/source.py @@ -0,0 +1,49 @@ +from typing import cast + +from posthog.schema import ( + ExternalDataSourceType as SchemaExternalDataSourceType, + SourceConfig, +) + +from posthog.temporal.data_imports.pipelines.pipeline.typings import SourceInputs, SourceResponse +from posthog.temporal.data_imports.sources.common.base import BaseSource, FieldType +from posthog.temporal.data_imports.sources.common.registry import SourceRegistry +from posthog.temporal.data_imports.sources.common.schema import SourceSchema +from posthog.temporal.data_imports.sources.generated_configs import CustomerIOSourceConfig + +from products.data_warehouse.backend.types import ExternalDataSourceType + +# TODO(Andrew J. McGehee): implement the source logic for CustomerIOSource + + +@SourceRegistry.register +class CustomerIOSource(BaseSource[CustomerIOSourceConfig]): + @property + def source_type(self) -> ExternalDataSourceType: + return ExternalDataSourceType.CUSTOMERIO + + @property + def get_source_config(self) -> SourceConfig: + return SourceConfig( + name=SchemaExternalDataSourceType.CUSTOMER_IO, + iconPath="/static/services/customer-io.png", + label="Customer.io", + caption=None, # only needed if you want to inline docs + docsUrl=None, # TODO(Andrew J. McGehee): link to the docs in the website, full path including https:// + fields=cast(list[FieldType], []), # TODO(Andrew J. McGehee): add source config fields here + unreleasedSource=True, + ) + + def validate_credentials(self, config: CustomerIOSourceConfig, team_id: int) -> tuple[bool, str | None]: + # TODO(Andrew J. McGehee): implement the logic to validate the credentials of your source, + # e.g. check the validity of API keys. returns a tuple of whether the credentials are valid, + # and if not, returns an error message to return to the user + raise NotImplementedError() + + def get_schemas( + self, config: CustomerIOSourceConfig, team_id: int, with_counts: bool = False + ) -> list[SourceSchema]: + raise NotImplementedError() + + def source_for_pipeline(self, config: CustomerIOSourceConfig, inputs: SourceInputs) -> SourceResponse: + raise NotImplementedError() diff --git a/posthog/temporal/data_imports/sources/generated_configs.py b/posthog/temporal/data_imports/sources/generated_configs.py index 9f97466c1e..91fd73dcfc 100644 --- a/posthog/temporal/data_imports/sources/generated_configs.py +++ b/posthog/temporal/data_imports/sources/generated_configs.py @@ -72,6 +72,11 @@ class ChargebeeSourceConfig(config.Config): site_name: str +@config.config +class CustomerIOSourceConfig(config.Config): + pass + + @config.config class DoItSourceConfig(config.Config): api_key: str @@ -249,6 +254,7 @@ def get_config_for_source(source: ExternalDataSourceType): ExternalDataSourceType.BIGQUERY: BigQuerySourceConfig, ExternalDataSourceType.BRAZE: BrazeSourceConfig, ExternalDataSourceType.CHARGEBEE: ChargebeeSourceConfig, + ExternalDataSourceType.CUSTOMERIO: CustomerIOSourceConfig, ExternalDataSourceType.DOIT: DoItSourceConfig, ExternalDataSourceType.GITHUB: GithubSourceConfig, ExternalDataSourceType.GOOGLEADS: GoogleAdsSourceConfig, diff --git a/products/data_warehouse/backend/migrations/0006_alter_externaldatasource_source_type.py b/products/data_warehouse/backend/migrations/0006_alter_externaldatasource_source_type.py new file mode 100644 index 0000000000..c6bb1fa485 --- /dev/null +++ b/products/data_warehouse/backend/migrations/0006_alter_externaldatasource_source_type.py @@ -0,0 +1,51 @@ +# Generated by create_datastack_source command on 2025-10-29 17:07 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + dependencies = [ + ("data_warehouse", "0005_dwh_anchor_time_reset"), + ] + + operations = [ + migrations.AlterField( + model_name="externaldatasource", + name="source_type", + field=models.CharField( + choices=[ + ("CustomerIO", "CustomerIO"), + ("Github", "Github"), + ("Stripe", "Stripe"), + ("Hubspot", "Hubspot"), + ("Postgres", "Postgres"), + ("Zendesk", "Zendesk"), + ("Snowflake", "Snowflake"), + ("Salesforce", "Salesforce"), + ("MySQL", "MySQL"), + ("MongoDB", "MongoDB"), + ("MSSQL", "MSSQL"), + ("Vitally", "Vitally"), + ("BigQuery", "BigQuery"), + ("Chargebee", "Chargebee"), + ("GoogleAds", "GoogleAds"), + ("TemporalIO", "TemporalIO"), + ("DoIt", "DoIt"), + ("GoogleSheets", "GoogleSheets"), + ("MetaAds", "MetaAds"), + ("Klaviyo", "Klaviyo"), + ("Mailchimp", "Mailchimp"), + ("Braze", "Braze"), + ("Mailjet", "Mailjet"), + ("Redshift", "Redshift"), + ("Polar", "Polar"), + ("RevenueCat", "RevenueCat"), + ("LinkedinAds", "LinkedinAds"), + ("RedditAds", "RedditAds"), + ("TikTokAds", "TikTokAds"), + ("Shopify", "Shopify"), + ], + max_length=128, + ), + ), + ] diff --git a/products/data_warehouse/backend/migrations/max_migration.txt b/products/data_warehouse/backend/migrations/max_migration.txt index 18d6fecd34..d81a0e9c89 100644 --- a/products/data_warehouse/backend/migrations/max_migration.txt +++ b/products/data_warehouse/backend/migrations/max_migration.txt @@ -1 +1 @@ -0005_dwh_anchor_time_reset +0006_alter_externaldatasource_source_type diff --git a/products/data_warehouse/backend/types.py b/products/data_warehouse/backend/types.py index ee4fc5e063..90e1d503e2 100644 --- a/products/data_warehouse/backend/types.py +++ b/products/data_warehouse/backend/types.py @@ -34,6 +34,7 @@ class PartitionSettings(typing.NamedTuple): class ExternalDataSourceType(models.TextChoices): + CUSTOMERIO = "CustomerIO", "CustomerIO" GITHUB = "Github", "Github" STRIPE = "Stripe", "Stripe" HUBSPOT = "Hubspot", "Hubspot"