feat(ds): add customer io as unreleased source (#40771)

Co-authored-by: github-actions <41898282+github-actions[bot]@users.noreply.github.com>

frontend/src/queries/schema.json

@@ -13387,6 +13387,7 @@
         },
         "ExternalDataSourceType": {
             "enum": [
+                "CustomerIO",
                 "Github",
                 "Stripe",
                 "Hubspot",

frontend/src/queries/schema/schema-general.ts

@@ -4237,6 +4237,7 @@ export interface SourceConfig {
 }
 
 export const externalDataSources = [
+    'CustomerIO',
     'Github',
     'Stripe',
     'Hubspot',

frontend/src/scenes/insights/stories/TrendsLine.stories.tsx

@@ -43,7 +43,11 @@ export const TrendsLine: Story = createInsightStory(
     require('../../../mocks/fixtures/api/projects/team_id/insights/trendsLine.json')
 )
 TrendsLine.parameters = {
-    testOptions: { waitForSelector: '[data-attr=trend-line-graph] > canvas' },
+    ...meta.parameters,
+    testOptions: {
+        ...meta.parameters?.testOptions,
+        waitForSelector: '[data-attr=trend-line-graph] > canvas',
+    },
 }
 
 // FLAP!
@@ -59,28 +63,44 @@ export const TrendsLineMulti: Story = createInsightStory(
     require('../../../mocks/fixtures/api/projects/team_id/insights/trendsLineMulti.json')
 )
 TrendsLineMulti.parameters = {
-    testOptions: { waitForSelector: '[data-attr=trend-line-graph] > canvas' },
+    ...meta.parameters,
+    testOptions: {
+        ...meta.parameters?.testOptions,
+        waitForSelector: '[data-attr=trend-line-graph] > canvas',
+    },
 }
 export const TrendsLineMultiEdit: Story = createInsightStory(
     require('../../../mocks/fixtures/api/projects/team_id/insights/trendsLineMulti.json'),
     'edit'
 )
 TrendsLineMultiEdit.parameters = {
-    testOptions: { waitForSelector: '[data-attr=trend-line-graph] > canvas' },
+    ...meta.parameters,
+    testOptions: {
+        ...meta.parameters?.testOptions,
+        waitForSelector: '[data-attr=trend-line-graph] > canvas',
+    },
 }
 
 export const TrendsLineBreakdown: Story = createInsightStory(
     require('../../../mocks/fixtures/api/projects/team_id/insights/trendsLineBreakdown.json')
 )
 TrendsLineBreakdown.parameters = {
-    testOptions: { waitForSelector: '[data-attr=trend-line-graph] > canvas' },
+    ...meta.parameters,
+    testOptions: {
+        ...meta.parameters?.testOptions,
+        waitForSelector: '[data-attr=trend-line-graph] > canvas',
+    },
 }
 export const TrendsLineBreakdownEdit: Story = createInsightStory(
     require('../../../mocks/fixtures/api/projects/team_id/insights/trendsLineBreakdown.json'),
     'edit'
 )
 TrendsLineBreakdownEdit.parameters = {
-    testOptions: { waitForSelector: '[data-attr=trend-line-graph] > canvas' },
+    ...meta.parameters,
+    testOptions: {
+        ...meta.parameters?.testOptions,
+        waitForSelector: '[data-attr=trend-line-graph] > canvas',
+    },
 }
 
 export const TrendsLineBreakdownLabels: Story = createInsightStory(
@@ -89,7 +109,11 @@ export const TrendsLineBreakdownLabels: Story = createInsightStory(
     true
 )
 TrendsLineBreakdownLabels.parameters = {
-    testOptions: { waitForSelector: '[data-attr=trend-line-graph] > canvas' },
+    ...meta.parameters,
+    testOptions: {
+        ...meta.parameters?.testOptions,
+        waitForSelector: '[data-attr=trend-line-graph] > canvas',
+    },
 }
 
 // Trends Bar
@@ -97,27 +121,43 @@ export const TrendsBar: Story = createInsightStory(
     require('../../../mocks/fixtures/api/projects/team_id/insights/trendsBar.json')
 )
 TrendsBar.parameters = {
-    testOptions: { waitForSelector: '[data-attr=trend-line-graph] > canvas' },
+    ...meta.parameters,
+    testOptions: {
+        ...meta.parameters?.testOptions,
+        waitForSelector: '[data-attr=trend-line-graph] > canvas',
+    },
 }
 export const TrendsBarEdit: Story = createInsightStory(
     require('../../../mocks/fixtures/api/projects/team_id/insights/trendsBar.json'),
     'edit'
 )
 TrendsBarEdit.parameters = {
-    testOptions: { waitForSelector: '[data-attr=trend-line-graph] > canvas' },
+    ...meta.parameters,
+    testOptions: {
+        ...meta.parameters?.testOptions,
+        waitForSelector: '[data-attr=trend-line-graph] > canvas',
+    },
 }
 
 export const TrendsBarBreakdown: Story = createInsightStory(
     require('../../../mocks/fixtures/api/projects/team_id/insights/trendsBarBreakdown.json')
 )
 TrendsBarBreakdown.parameters = {
-    testOptions: { waitForSelector: '[data-attr=trend-line-graph] > canvas' },
+    ...meta.parameters,
+    testOptions: {
+        ...meta.parameters?.testOptions,
+        waitForSelector: '[data-attr=trend-line-graph] > canvas',
+    },
 }
 export const TrendsBarBreakdownEdit: Story = createInsightStory(
     require('../../../mocks/fixtures/api/projects/team_id/insights/trendsBarBreakdown.json'),
     'edit'
 )
 TrendsBarBreakdownEdit.parameters = {
-    testOptions: { waitForSelector: '[data-attr=trend-line-graph] > canvas' },
+    ...meta.parameters,
+    testOptions: {
+        ...meta.parameters?.testOptions,
+        waitForSelector: '[data-attr=trend-line-graph] > canvas',
+    },
 }
 /* eslint-enable @typescript-eslint/no-var-requires */

frontend/src/scenes/session-recordings/player/inspector/components/ItemEvent.stories.tsx

@@ -1,6 +1,6 @@
 import { Meta, StoryFn, StoryObj } from '@storybook/react'
 
-import { now } from 'lib/dayjs'
+import { dayjs } from 'lib/dayjs'
 import { LemonDivider } from 'lib/lemon-ui/LemonDivider'
 import { uuid } from 'lib/utils'
 import {
@@ -33,6 +33,7 @@ function makeItem(
     dataOverrides: Partial<RecordingEventType> = {},
     propertiesOverrides: Record<string, any> = {}
 ): InspectorListItemEvent {
+    const mockDate = dayjs('2025-11-04')
     const data: RecordingEventType = {
         elements: [],
         event: '',
@@ -40,7 +41,7 @@ function makeItem(
         id: '',
         playerTime: 0,
 
-        timestamp: now().toISOString(),
+        timestamp: mockDate.toISOString(),
         ...dataOverrides,
         // this is last so that it overrides data overrides sensibly 🙃
         properties: {
@@ -51,7 +52,7 @@ function makeItem(
         data: data,
         search: '',
         timeInRecording: 0,
-        timestamp: now(),
+        timestamp: mockDate,
         type: 'events',
         key: `some-key-${uuid()}`,
         ...itemOverrides,

posthog/management/commands/create_datastack_source.py

@@ -17,6 +17,7 @@ from products.data_warehouse.backend import types
 os.environ["DEBUG"] = "1"
 os.environ["SKIP_ASYNC_MIGRATIONS_SETUP"] = "1"
 
+
 SOURCE_TEMPLATE = """\
 from typing import cast
 
@@ -82,7 +83,7 @@ class Migration(migrations.Migration):
             model_name="externaldatasource",
             name="source_type",
             field=models.CharField(
-                choices=[ {choices} ],
+                choices={choices},
                 max_length=128,
             ),
         ),
@@ -137,6 +138,7 @@ class Command(BaseCommand):
             "constant": textcase.constant(name),
             "caps": textcase.constant(name).replace("_", ""),
         }
+        self._fix_common_endings(transforms=name_transforms)
         logo_filename = f"{name_transforms['kebab']}.png"
 
         self._setup_source_structure(repo, transforms=name_transforms)
@@ -161,6 +163,12 @@ class Command(BaseCommand):
         self._schema_build(transforms=name_transforms)
         self._format_files()
 
+    def _fix_common_endings(self, transforms: NameTransforms):
+        common_endings = ("Io", "Ai", "Db", "Ci")
+        for end in common_endings:
+            if transforms["pascal"].endswith(end):
+                transforms["pascal"] = transforms["pascal"][: -len(end)] + end.upper()
+
     def _setup_source_structure(self, repo: Repo, transforms: NameTransforms):
         sources_root = os.path.join(repo.working_dir, "posthog", "temporal", "data_imports", "sources")
         assert os.path.exists(sources_root), f"Sources root {sources_root} not found"

posthog/management/commands/test/test_create_datastack_source.py

@@ -2,7 +2,7 @@ import pathlib
 
 import pytest
 
-from posthog.management.commands.create_datastack_source import Command
+from posthog.management.commands.create_datastack_source import Command, NameTransforms
 
 TEST_FILE_CONTENT = "a b c d e f"
 TEST_BLOCK = """\
@@ -19,6 +19,11 @@ def command():
     return Command()
 
 
+@pytest.fixture
+def transforms():
+    return NameTransforms(snake="", kebab="", pascal="", caps="", constant="")
+
+
 def test_split_file_by_regex(command: Command, tmp_path: pathlib.Path):
     path = tmp_path / "test.file"
     path.write_text(TEST_FILE_CONTENT)
@@ -43,3 +48,21 @@ def test_format_file_line(command: Command):
 
     expected = "        a"
     assert command._format_file_line("a", indent_level=2, end="")
+
+
+def test_fix_common_endings(command: Command, transforms: NameTransforms):
+    transforms["pascal"] = "CustomerIo"
+    command._fix_common_endings(transforms)
+    assert transforms["pascal"] == "CustomerIO"
+
+    transforms["pascal"] = "Something.Ai"
+    command._fix_common_endings(transforms)
+    assert transforms["pascal"] == "Something.AI"
+
+    transforms["pascal"] = "ImpressiveCi"
+    command._fix_common_endings(transforms)
+    assert transforms["pascal"] == "ImpressiveCI"
+
+    transforms["pascal"] = "DuckDb"
+    command._fix_common_endings(transforms)
+    assert transforms["pascal"] == "DuckDB"

posthog/schema.py

@@ -1370,6 +1370,7 @@ class ExperimentVariantTrendsBaseStats(BaseModel):
 
 
 class ExternalDataSourceType(StrEnum):
+    CUSTOMER_IO = "CustomerIO"
     GITHUB = "Github"
     STRIPE = "Stripe"
     HUBSPOT = "Hubspot"

posthog/temporal/data_imports/sources/__init__.py

@@ -2,6 +2,7 @@ from .bigquery.source import BigQuerySource
 from .braze.source import BrazeSource
 from .chargebee.source import ChargebeeSource
 from .common.registry import SourceRegistry
+from .customer_io.source import CustomerIOSource
 from .doit.source import DoItSource
 from .github.source import GithubSource
 from .google_ads.source import GoogleAdsSource
@@ -30,6 +31,7 @@ from .vitally.source import VitallySource
 from .zendesk.source import ZendeskSource
 
 __all__ = [
+    "CustomerIOSource",
     "GithubSource",
     "SourceRegistry",
     "BigQuerySource",

posthog/temporal/data_imports/sources/common/test/__snapshots__/test_source_config_generator.ambr

@@ -46,6 +46,7 @@
           ExternalDataSourceType.BIGQUERY: BigQuerySourceConfig,
           ExternalDataSourceType.BRAZE: BrazeSourceConfig,
           ExternalDataSourceType.CHARGEBEE: ChargebeeSourceConfig,
+          ExternalDataSourceType.CUSTOMERIO: CustomerIOSourceConfig,
           ExternalDataSourceType.DOIT: DoItSourceConfig,
           ExternalDataSourceType.GITHUB: GithubSourceConfig,
           ExternalDataSourceType.GOOGLEADS: GoogleAdsSourceConfig,

posthog/temporal/data_imports/sources/customer_io/source.py

@@ -0,0 +1,49 @@
+from typing import cast
+
+from posthog.schema import (
+    ExternalDataSourceType as SchemaExternalDataSourceType,
+    SourceConfig,
+)
+
+from posthog.temporal.data_imports.pipelines.pipeline.typings import SourceInputs, SourceResponse
+from posthog.temporal.data_imports.sources.common.base import BaseSource, FieldType
+from posthog.temporal.data_imports.sources.common.registry import SourceRegistry
+from posthog.temporal.data_imports.sources.common.schema import SourceSchema
+from posthog.temporal.data_imports.sources.generated_configs import CustomerIOSourceConfig
+
+from products.data_warehouse.backend.types import ExternalDataSourceType
+
+# TODO(Andrew J. McGehee): implement the source logic for CustomerIOSource
+
+
+@SourceRegistry.register
+class CustomerIOSource(BaseSource[CustomerIOSourceConfig]):
+    @property
+    def source_type(self) -> ExternalDataSourceType:
+        return ExternalDataSourceType.CUSTOMERIO
+
+    @property
+    def get_source_config(self) -> SourceConfig:
+        return SourceConfig(
+            name=SchemaExternalDataSourceType.CUSTOMER_IO,
+            iconPath="/static/services/customer-io.png",
+            label="Customer.io",
+            caption=None,  # only needed if you want to inline docs
+            docsUrl=None,  # TODO(Andrew J. McGehee): link to the docs in the website, full path including https://
+            fields=cast(list[FieldType], []),  # TODO(Andrew J. McGehee): add source config fields here
+            unreleasedSource=True,
+        )
+
+    def validate_credentials(self, config: CustomerIOSourceConfig, team_id: int) -> tuple[bool, str | None]:
+        # TODO(Andrew J. McGehee): implement the logic to validate the credentials of your source,
+        # e.g. check the validity of API keys. returns a tuple of whether the credentials are valid,
+        # and if not, returns an error message to return to the user
+        raise NotImplementedError()
+
+    def get_schemas(
+        self, config: CustomerIOSourceConfig, team_id: int, with_counts: bool = False
+    ) -> list[SourceSchema]:
+        raise NotImplementedError()
+
+    def source_for_pipeline(self, config: CustomerIOSourceConfig, inputs: SourceInputs) -> SourceResponse:
+        raise NotImplementedError()

posthog/temporal/data_imports/sources/generated_configs.py

@@ -72,6 +72,11 @@ class ChargebeeSourceConfig(config.Config):
     site_name: str
 
 
+@config.config
+class CustomerIOSourceConfig(config.Config):
+    pass
+
+
 @config.config
 class DoItSourceConfig(config.Config):
     api_key: str
@@ -249,6 +254,7 @@ def get_config_for_source(source: ExternalDataSourceType):
         ExternalDataSourceType.BIGQUERY: BigQuerySourceConfig,
         ExternalDataSourceType.BRAZE: BrazeSourceConfig,
         ExternalDataSourceType.CHARGEBEE: ChargebeeSourceConfig,
+        ExternalDataSourceType.CUSTOMERIO: CustomerIOSourceConfig,
         ExternalDataSourceType.DOIT: DoItSourceConfig,
         ExternalDataSourceType.GITHUB: GithubSourceConfig,
         ExternalDataSourceType.GOOGLEADS: GoogleAdsSourceConfig,

products/data_warehouse/backend/migrations/0006_alter_externaldatasource_source_type.py

@@ -0,0 +1,51 @@
+# Generated by create_datastack_source command on 2025-10-29 17:07
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+    dependencies = [
+        ("data_warehouse", "0005_dwh_anchor_time_reset"),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name="externaldatasource",
+            name="source_type",
+            field=models.CharField(
+                choices=[
+                    ("CustomerIO", "CustomerIO"),
+                    ("Github", "Github"),
+                    ("Stripe", "Stripe"),
+                    ("Hubspot", "Hubspot"),
+                    ("Postgres", "Postgres"),
+                    ("Zendesk", "Zendesk"),
+                    ("Snowflake", "Snowflake"),
+                    ("Salesforce", "Salesforce"),
+                    ("MySQL", "MySQL"),
+                    ("MongoDB", "MongoDB"),
+                    ("MSSQL", "MSSQL"),
+                    ("Vitally", "Vitally"),
+                    ("BigQuery", "BigQuery"),
+                    ("Chargebee", "Chargebee"),
+                    ("GoogleAds", "GoogleAds"),
+                    ("TemporalIO", "TemporalIO"),
+                    ("DoIt", "DoIt"),
+                    ("GoogleSheets", "GoogleSheets"),
+                    ("MetaAds", "MetaAds"),
+                    ("Klaviyo", "Klaviyo"),
+                    ("Mailchimp", "Mailchimp"),
+                    ("Braze", "Braze"),
+                    ("Mailjet", "Mailjet"),
+                    ("Redshift", "Redshift"),
+                    ("Polar", "Polar"),
+                    ("RevenueCat", "RevenueCat"),
+                    ("LinkedinAds", "LinkedinAds"),
+                    ("RedditAds", "RedditAds"),
+                    ("TikTokAds", "TikTokAds"),
+                    ("Shopify", "Shopify"),
+                ],
+                max_length=128,
+            ),
+        ),
+    ]

products/data_warehouse/backend/migrations/max_migration.txt

@@ -1 +1 @@
-0005_dwh_anchor_time_reset
+0006_alter_externaldatasource_source_type

products/data_warehouse/backend/types.py

@@ -34,6 +34,7 @@ class PartitionSettings(typing.NamedTuple):
 
 
 class ExternalDataSourceType(models.TextChoices):
+    CUSTOMERIO = "CustomerIO", "CustomerIO"
     GITHUB = "Github", "Github"
     STRIPE = "Stripe", "Stripe"
     HUBSPOT = "Hubspot", "Hubspot"