chore(devex): Clean up product model structures and remove shims (#38583)

.github/workflows/ci-backend.yml

@@ -94,7 +94,8 @@ jobs:
                         - bin/check_kafka_clickhouse_up
                       migrations:
                         - 'posthog/migrations/*.py'
-                        - 'products/**/migrations/*.py'
+                        - 'products/*/backend/migrations/*.py'
+                        - 'products/*/migrations/*.py'  # Legacy structure
 
     check-migrations:
         needs: [changes]
@@ -178,9 +179,14 @@ jobs:
                     if [[ $file =~ migrations/([0-9]+)_ ]]; then
                       migration_number="${BASH_REMATCH[1]}"
                       # Get app name by looking at the directory structure
-                      # For nested apps like products/user_interviews, we want user_interviews
-                      app_name=$(echo $file | sed -E 's|^([^/]+/)*([^/]+)/migrations/.*|\2|')
-                      echo "Checking migration $migration_number for app $app_name"
+                      # For new structure products/user_interviews/backend/migrations, we want user_interviews
+                      # For old structure products/user_interviews/migrations, we want user_interviews
+                      if [[ $file =~ products/([^/]+)/backend/migrations/ ]]; then
+                        app_name="${BASH_REMATCH[1]}"
+                      else
+                        app_name=$(echo $file | sed -E 's|^([^/]+/)*([^/]+)/migrations/.*|\2|')
+                      fi
+                        echo "Checking migration $migration_number for app $app_name"
 
                       # Get SQL output
                       SQL_OUTPUT=$(python manage.py sqlmigrate $app_name $migration_number)
@@ -440,6 +446,19 @@ jobs:
                   # Use PostHog Bot token when not on forks to enable proper snapshot updating
                   token: ${{ github.event.pull_request.head.repo.full_name == github.repository && secrets.POSTHOG_BOT_GITHUB_TOKEN || github.token }}
 
+            - name: 'Safeguard: ensure no stray Python modules at product root'
+              if: ${{ needs.changes.outputs.backend == 'true' }}
+              run: |
+                  echo "Checking that products/* only contain backend/, frontend/, or shared/ as Python code roots..."
+                  BAD_FILES=$(find products -maxdepth 2 -type f -name "*.py" ! -path "*/backend/*" ! -name "__init__.py" ! -name "conftest.py" -o -maxdepth 2 -type d -name "migrations" ! -path "*/backend/*")
+                  if [ -n "$BAD_FILES" ]; then
+                    echo "❌ Found Python code or migrations outside backend/:"
+                    echo "$BAD_FILES"
+                    echo "Please move these into the appropriate backend/ folder."
+                    exit 1
+                  fi
+                  echo "✅ No stray Python files or migrations found at product roots."
+
             # Pre-tests
 
             # Copies the fully versioned UDF xml file for use in CI testing

dags/max_ai/run_evaluation.py

@@ -12,7 +12,7 @@ from dagster_slack import SlackResource
 from pydantic import BaseModel, Field, ValidationError
 from tenacity import retry, stop_after_attempt, wait_exponential
 
-from posthog.models import Dataset, DatasetItem
+from products.llm_analytics.backend.models import Dataset, DatasetItem
 
 from dags.common import JobOwners
 from dags.max_ai.snapshot_team_data import (

dags/tests/max_ai/test_run_evaluation.py

@@ -6,7 +6,7 @@ from unittest.mock import patch
 
 import dagster
 
-from posthog.models import Dataset, DatasetItem
+from products.llm_analytics.backend.models import Dataset, DatasetItem
 
 from dags.max_ai.run_evaluation import prepare_dataset
 

mypy-baseline.txt

@@ -1137,8 +1137,8 @@ products/cdp/backend/max_tools.py:0: error: Exception must be derived from BaseE
 products/data_warehouse/backend/max_tools.py:0: error: Argument "toolkit_class" to "__init__" of "TaxonomyAgentNode" has incompatible type "HogQLGeneratorToolkit"; expected "type[TaxonomyAgentToolkit]"  [arg-type]
 products/data_warehouse/backend/max_tools.py:0: error: Argument "toolkit_class" to "__init__" of "TaxonomyAgentToolsNode" has incompatible type "HogQLGeneratorToolkit"; expected "type[TaxonomyAgentToolkit]"  [arg-type]
 products/data_warehouse/backend/max_tools.py:0: error: Value of type variable "Q" of "SchemaGeneratorOutput" cannot be "str"  [type-var]
-products/early_access_features/migrations/0001_initial_migration.py:0: error: Need type annotation for "database_operations" (hint: "database_operations: list[<type>] = ...")  [var-annotated]
-products/early_access_features/migrations/0002_alter_earlyaccessfeature_options_and_more.py:0: error: Need type annotation for "database_operations" (hint: "database_operations: list[<type>] = ...")  [var-annotated]
+products/early_access_features/backend/migrations/0001_initial_migration.py:0: error: Need type annotation for "database_operations" (hint: "database_operations: list[<type>] = ...")  [var-annotated]
+products/early_access_features/backend/migrations/0002_alter_earlyaccessfeature_options_and_more.py:0: error: Need type annotation for "database_operations" (hint: "database_operations: list[<type>] = ...")  [var-annotated]
 products/error_tracking/backend/max_tools.py:0: error: Exception must be derived from BaseException  [misc]
 products/experiments/stats/tests/test_bayesian.py:0: error: Value of type "object" is not indexable  [index]
 products/experiments/stats/tests/test_bayesian.py:0: error: Value of type "object" is not indexable  [index]

posthog/api/test/test_team.py

@@ -16,7 +16,7 @@ from temporalio.service import RPCError
 
 from posthog.api.test.batch_exports.conftest import start_test_worker
 from posthog.constants import AvailableFeature
-from posthog.models import ActivityLog, EarlyAccessFeature
+from posthog.models import ActivityLog
 from posthog.models.async_deletion.async_deletion import AsyncDeletion, DeletionType
 from posthog.models.dashboard import Dashboard
 from posthog.models.instance_setting import get_instance_setting
@@ -31,6 +31,8 @@ from posthog.temporal.common.schedule import describe_schedule
 from posthog.test.test_utils import create_group_type_mapping_without_created_at
 from posthog.utils import get_instance_realm
 
+from products.early_access_features.backend.models import EarlyAccessFeature
+
 from ee.models.rbac.access_control import AccessControl
 
 

posthog/models/__init__.py

@@ -92,12 +92,6 @@ from .user_group import UserGroup, UserGroupMembership
 from .user_scene_personalisation import UserScenePersonalisation
 from .web_experiment import WebExperiment
 
-# Keeping products imports at the bottom to avoid circular imports errors
-# Products Imports
-from products.tasks.backend.models import Task
-from products.early_access_features.backend.models import EarlyAccessFeature
-from products.llm_analytics.backend.models import Dataset, DatasetItem
-
 from .oauth import OAuthAccessToken, OAuthApplication, OAuthGrant, OAuthIDToken, OAuthRefreshToken
 
 __all__ = [
@@ -120,10 +114,7 @@ __all__ = [
     "DashboardTile",
     "DashboardTemplate",
     "DataColorTheme",
-    "Dataset",
-    "DatasetItem",
     "DeletionType",
-    "EarlyAccessFeature",
     "Element",
     "ElementGroup",
     "Entity",
@@ -205,7 +196,6 @@ __all__ = [
     "Survey",
     "Tag",
     "TaggedItem",
-    "Task",
     "Team",
     "TeamRevenueAnalyticsConfig",
     "TeamMarketingAnalyticsConfig",

posthog/settings/web.py

@@ -29,12 +29,12 @@ AXES_META_PRECEDENCE_ORDER = ["HTTP_X_FORWARDED_FOR", "REMOTE_ADDR"]
 # TODO: Automatically generate these like we do for the frontend
 # NOTE: Add these definitions here and on `tach.toml`
 PRODUCTS_APPS = [
-    "products.early_access_features",
-    "products.tasks",
-    "products.links",
-    "products.revenue_analytics",
-    "products.user_interviews",
-    "products.llm_analytics",
+    "products.early_access_features.backend.apps.EarlyAccessFeaturesConfig",
+    "products.tasks.backend.apps.TasksConfig",
+    "products.links.backend.apps.LinksConfig",
+    "products.revenue_analytics.backend.apps.RevenueAnalyticsConfig",
+    "products.user_interviews.backend.apps.UserInterviewsConfig",
+    "products.llm_analytics.backend.apps.LlmAnalyticsConfig",
 ]
 
 INSTALLED_APPS = [

posthog/tasks/early_access_feature.py

@@ -6,9 +6,10 @@ from posthog.hogql.constants import MAX_SELECT_RETURNED_ROWS, LimitContext
 from posthog.hogql.query import execute_hogql_query
 
 from posthog.cloud_utils import is_cloud
-from posthog.models import EarlyAccessFeature
 from posthog.ph_client import get_client
 
+from products.early_access_features.backend.models import EarlyAccessFeature
+
 logger = structlog.get_logger(__name__)
 
 POSTHOG_TEAM_ID = 2

posthog/tasks/environments_rollback.py

@@ -12,7 +12,6 @@ from posthog.models import (
     Annotation,
     Cohort,
     Dashboard,
-    EarlyAccessFeature,
     EventDefinition,
     Experiment,
     FeatureFlag,
@@ -29,6 +28,8 @@ from posthog.models import (
 from posthog.models.organization import OrganizationMembership
 from posthog.ph_client import get_client
 
+from products.early_access_features.backend.models import EarlyAccessFeature
+
 
 @dataclass
 class RollbackEventContext:

posthog/tasks/test/test_environments_rollback.py

@@ -5,7 +5,6 @@ from django.test import TransactionTestCase
 from posthog.models import (
     Annotation,
     Dashboard,
-    EarlyAccessFeature,
     EventDefinition,
     FeatureFlag,
     Insight,
@@ -18,6 +17,8 @@ from posthog.models.organization import Organization, OrganizationMembership
 from posthog.tasks.environments_rollback import environments_rollback_migration
 from posthog.test.test_utils import create_group_type_mapping_without_created_at
 
+from products.early_access_features.backend.models import EarlyAccessFeature
+
 
 class TestEnvironmentsRollbackTask(TransactionTestCase):
     def setUp(self) -> None:

products/README.md

@@ -1,15 +1,93 @@
-# Product Folders
+# Products
+
+Each product in PostHog is a **vertical slice**: it contains its backend (Django app), frontend (React/TypeScript), and optionally shared code.
+This structure ensures product features are self-contained and can evolve independently.
+
+The **entire product folder** (`products/<product_name>/`) is treated as a **Turborepo package**.
+Backend and frontend are sub-parts of that package.
 
 This is the (future) home for all PostHog products ([RFC](https://github.com/PostHog/product-internal/pull/703)).
 
-## Dev guidelines
+## Folder structure
+
+```txt
+products/
+  __init__.py
+  <product_name>/           # Turborepo package boundary
+    __init__.py             # allows imports like products.<product>.backend.*
+    backend/                # Django app
+      __init__.py           # marks backend as Python package/Django app
+      models.py
+      migrations/
+      api.py
+      serializers.py
+      tests/                # backend tests live here
+    frontend/               # frontend app
+      components/
+      pages/
+      tests/                # frontend tests live here
+    shared/                 # optional: cross-cutting code for both backend & frontend
+    package.json            # defines the product package in Turborepo
+    manifest.tsx            # describes the product's features
+```
+
+## Backend conventions
+
+- Each `backend/` folder is a **real Django app**.
+
+- Register it in `INSTALLED_APPS` via `AppConfig`:
+
+    ```python
+    # products/feature_flags/backend/apps.py
+    from django.apps import AppConfig
+
+    class FeatureFlagsConfig(AppConfig):
+        name = "products.feature_flags.backend"
+        label = "feature_flags"
+        verbose_name = "Feature Flags"
+    ```
+
+- ✅ Always use the **real Python path** for imports:
+
+    ```python
+    from products.feature_flags.backend.models import FeatureFlag
+    ```
+
+- ✅ For relations, use **string app labels**:
+
+    ```python
+    class Experiment(models.Model):
+        feature_flag = models.ForeignKey(
+            "feature_flags.FeatureFlag",
+            on_delete=models.CASCADE,
+        )
+    ```
+
+- ❌ Do **not** import models from `posthog.models` or create re-exports like `products.feature_flags.models`.
+
+This avoids circular imports and keeps migrations/app labels stable.
+
+## Frontend conventions
+
+- Each `frontend/` directory contains the frontend app for the product.
+- It lives under the same package as the backend.
+- Backend and frontend tooling can be independent (`requirements.txt` vs. `package.json`) but remain in the same Turborepo package.
+- Tests for frontend code live inside `frontend/tests/`.
+
+## Shared code
+
+If backend and frontend need shared schemas, validators, or constants, put them in a `shared/` directory under the product.
+Keep shared code minimal to avoid tight coupling.
+
+## Product requirements
 
 - Each high level product should have its own folder.
-    - Please keep the top level folders `under_score` cased, as dashes make it hard to import files in some languages (e.g. Python).
+  - Please keep the top level folders `under_score` cased, as dashes make it hard to import files in some languages (e.g. Python).
 - Each product has a few required files / folders:
-    - `manifest.tsx` - describes the product's features. All manifest files are combined into `frontend/src/products.tsx` on build.
-    - `package.json` - describes the frontend dependencies. Ideally they should all be `peerDependencies` of whatever is in `frontend/package.json`
-    - `__init__.py` - marks the directory as a python package, needed if you include the backend.
+  - `manifest.tsx` - describes the product's features. All manifest files are combined into `frontend/src/products.tsx` on build.
+  - `package.json` - describes the frontend dependencies. Ideally they should all be `peerDependencies` of whatever is in `frontend/package.json`
+  - `__init__.py` - allows imports like `products.<product>.backend.*` (only if backend exists)
+    - `backend/__init__.py` - marks the backend directory as a Python package/Django app (only if backend exists).
     - `frontend/` - React frontend code. We run prettier/eslint only on files in the `frontend` folder on commit.
     - `backend/` - Python backend code. It's treated as a separate django app.
 
@@ -17,38 +95,41 @@ This is the (future) home for all PostHog products ([RFC](https://github.com/Pos
 
 - Create a new folder `products/your_product_name`, keep it underscore-cased.
 - Create a `manifest.tsx` file
-    - Describe the product's frontend `scenes`, `routes`, `urls`, file system types, and project tree (navbar) items.
-    - All manifest files are combined into a single `frontend/src/products.tsx` file on build.
-    - NOTE: we don't copy imports into `products.tsx`. If you add new icons, update the imports manually in `frontend/src/products.tsx`. It only needs to be done once.
-    - NOTE: if you want to add a link to the old pre-project-tree navbar, do so manually in `frontend/src/layout/navigation-3000/navigationLogic.tsx`
+  - Describe the product's frontend `scenes`, `routes`, `urls`, file system types, and project tree (navbar) items.
+  - All manifest files are combined into a single `frontend/src/products.tsx` file on build.
+  - NOTE: we don't copy imports into `products.tsx`. If you add new icons, update the imports manually in `frontend/src/products.tsx`. It only needs to be done once.
+  - NOTE: if you want to add a link to the old pre-project-tree navbar, do so manually in `frontend/src/layout/navigation-3000/navigationLogic.tsx`
 - Create a `package.json` file:
-    - Keep the package name as `@posthog/products-your-product-name`. Include `@posthog/products-` in the name.
-    - Update the global `frontend/package.json`: add your new npm package under `dependencies`.
-    - If your scenes are linked up with the right paths, things should just work.
-    - Each scene can either export a React component as its default export, or define a `export const scene: SceneExport = { logic, component }` object to export both a logic and a component. This way the logic stays mounted when you move away from the page. This is useful if you don't want to reload everything each time the scene is loaded.
-- Create a `__init__.py` file if your product has python backend code.
-    - This is needed to mark the directory as a Python package / Django app.
-    - Modify `posthog/settings/web.py` and add your new product under `PRODUCTS_APPS`.
-    - Modify `tach.toml` and add a new block for your product. We use `tach` to track cross-dependencies between python apps.
-    - Modify `posthog/api/__init__.py` and add your API routes as you normally would (e.g. `import products.early_access_features.backend.api as early_access_feature`)
-    - NOTE: we will automate some of these steps in the future, but for now, please do them manually.
+  - Keep the package name as `@posthog/products-your-product-name`. Include `@posthog/products-` in the name.
+  - Update the global `frontend/package.json`: add your new npm package under `dependencies`.
+  - If your scenes are linked up with the right paths, things should just work.
+  - Each scene can either export a React component as its default export, or define a `export const scene: SceneExport = { logic, component }` object to export both a logic and a component. This way the logic stays mounted when you move away from the page. This is useful if you don't want to reload everything each time the scene is loaded.
+- Create `__init__.py` and `backend/__init__.py` files if your product has python backend code.
+  - `__init__.py` allows imports like `products.<name>.backend.*`
+  - `backend/__init__.py` marks the backend directory as a Python package / Django app.
+  - Register the backend as a Django app with an `AppConfig` that sets `label = "<name>"` (not `products.<name>`).
+  - Modify `posthog/settings/web.py` and add your new product under `PRODUCTS_APPS`.
+  - Modify `tach.toml` and add a new block for your product. We use `tach` to track cross-dependencies between python apps.
+  - Modify `posthog/api/__init__.py` and add your API routes as you normally would (e.g. `import products.early_access_features.backend.api as early_access_feature`)
+  - NOTE: we will automate some of these steps in the future, but for now, please do them manually.
 
 ## Adding or moving backend models and migrations
 
 - Create or move your backend models under the product's `backend/` folder.
-- Import and export them under `posthog/models/__init__.py` (see `EarlyAccessFeature` for an example)
+- Use direct imports from the product location (e.g., `from products.experiments.backend.models import Experiment`)
+- Use string-based foreign key references to avoid circular imports (e.g., `models.ForeignKey("posthog.Team", on_delete=models.CASCADE)`)
 - Create a `products/your_product_name/backend/migrations` folder.
 - Run `python manage.py makemigrations your_product_name -n initial_migration`
 - If this is a brand-new model, you're done.
 - If you're moving a model from the old `posthog/models/` folder, there are more things to do:
-    - Make sure the model's `Meta` class has `db_table = 'old_table_name'` set along with `managed = True`.
-    - Run `python manage.py makemigrations posthog -n remove_old_product_name`
-    - The generated migrations will want to `DROP TABLE` your old model, and `CREATE TABLE` the new one. This is not what we want.
-    - Instead, we want to run `migrations.SeparateDatabaseAndState` in both migrations.
-    - Follow the example in `posthog/migrations/0548_migrate_early_access_features.py` and `products/early_access_features/migrations/0001_initial_migration.py`.
-    - Move all operations into `state_operations = []` and keep the `database_operations = []` empty in both migrations.
-    - Run and test this a few times before merging. Data loss is irreversible.
+  - Make sure the model's `Meta` class has `db_table = 'old_table_name'` set along with `managed = True`.
+  - Run `python manage.py makemigrations posthog -n remove_old_product_name`
+  - The generated migrations will want to `DROP TABLE` your old model, and `CREATE TABLE` the new one. This is not what we want.
+  - Instead, we want to run `migrations.SeparateDatabaseAndState` in both migrations.
+  - Follow the example in `posthog/migrations/0548_migrate_early_access_features.py` and `products/early_access_features/migrations/0001_initial_migration.py`.
+  - Move all operations into `state_operations = []` and keep the `database_operations = []` empty in both migrations.
+  - Run and test this a few times before merging. Data loss is irreversible.
 
-## TODO:
+## TODO
 
 - [ ] A story for Python testing - run tests automatically, only test apps that changed, etc

products/cdp/backend/apps.py

@@ -0,0 +1,7 @@
+from django.apps import AppConfig
+
+
+class CdpConfig(AppConfig):
+    default_auto_field = "django.db.models.BigAutoField"
+    name = "products.cdp.backend"
+    label = "cdp"

products/customer_analytics/backend/apps.py

@@ -0,0 +1,7 @@
+from django.apps import AppConfig
+
+
+class CustomerAnalyticsConfig(AppConfig):
+    default_auto_field = "django.db.models.BigAutoField"
+    name = "products.customer_analytics.backend"
+    label = "customer_analytics"

products/data_warehouse/backend/apps.py

@@ -0,0 +1,7 @@
+from django.apps import AppConfig
+
+
+class DataWarehouseConfig(AppConfig):
+    default_auto_field = "django.db.models.BigAutoField"
+    name = "products.data_warehouse.backend"
+    label = "data_warehouse"

products/early_access_features/backend/apps.py

@@ -0,0 +1,7 @@
+from django.apps import AppConfig
+
+
+class EarlyAccessFeaturesConfig(AppConfig):
+    default_auto_field = "django.db.models.BigAutoField"
+    name = "products.early_access_features.backend"
+    label = "early_access_features"

products/error_tracking/backend/apps.py

@@ -0,0 +1,7 @@
+from django.apps import AppConfig
+
+
+class ErrorTrackingConfig(AppConfig):
+    default_auto_field = "django.db.models.BigAutoField"
+    name = "products.error_tracking.backend"
+    label = "error_tracking"

products/experiments/backend/apps.py

@@ -0,0 +1,7 @@
+from django.apps import AppConfig
+
+
+class ExperimentsConfig(AppConfig):
+    default_auto_field = "django.db.models.BigAutoField"
+    name = "products.experiments.backend"
+    label = "experiments"

products/links/backend/apps.py

@@ -0,0 +1,7 @@
+from django.apps import AppConfig
+
+
+class LinksConfig(AppConfig):
+    default_auto_field = "django.db.models.BigAutoField"
+    name = "products.links.backend"
+    label = "links"

products/llm_analytics/backend/apps.py

@@ -0,0 +1,7 @@
+from django.apps import AppConfig
+
+
+class LlmAnalyticsConfig(AppConfig):
+    default_auto_field = "django.db.models.BigAutoField"
+    name = "products.llm_analytics.backend"
+    label = "llm_analytics"

products/logs/backend/apps.py

@@ -0,0 +1,7 @@
+from django.apps import AppConfig
+
+
+class LogsConfig(AppConfig):
+    default_auto_field = "django.db.models.BigAutoField"
+    name = "products.logs.backend"
+    label = "logs"

products/managed_migrations/backend/apps.py

@@ -0,0 +1,7 @@
+from django.apps import AppConfig
+
+
+class ManagedMigrationsConfig(AppConfig):
+    default_auto_field = "django.db.models.BigAutoField"
+    name = "products.managed_migrations.backend"
+    label = "managed_migrations"

products/marketing_analytics/backend/apps.py

@@ -0,0 +1,7 @@
+from django.apps import AppConfig
+
+
+class MarketingAnalyticsConfig(AppConfig):
+    default_auto_field = "django.db.models.BigAutoField"
+    name = "products.marketing_analytics.backend"
+    label = "marketing_analytics"

products/messaging/backend/apps.py

@@ -0,0 +1,7 @@
+from django.apps import AppConfig
+
+
+class MessagingConfig(AppConfig):
+    default_auto_field = "django.db.models.BigAutoField"
+    name = "products.messaging.backend"
+    label = "messaging"

products/product_analytics/backend/apps.py

@@ -0,0 +1,7 @@
+from django.apps import AppConfig
+
+
+class ProductAnalyticsConfig(AppConfig):
+    default_auto_field = "django.db.models.BigAutoField"
+    name = "products.product_analytics.backend"
+    label = "product_analytics"

products/replay/backend/apps.py

@@ -0,0 +1,7 @@
+from django.apps import AppConfig
+
+
+class ReplayConfig(AppConfig):
+    default_auto_field = "django.db.models.BigAutoField"
+    name = "products.replay.backend"
+    label = "replay"

products/revenue_analytics/backend/apps.py

@@ -0,0 +1,7 @@
+from django.apps import AppConfig
+
+
+class RevenueAnalyticsConfig(AppConfig):
+    default_auto_field = "django.db.models.BigAutoField"
+    name = "products.revenue_analytics.backend"
+    label = "revenue_analytics"

products/surveys/backend/apps.py

@@ -0,0 +1,7 @@
+from django.apps import AppConfig
+
+
+class SurveysConfig(AppConfig):
+    default_auto_field = "django.db.models.BigAutoField"
+    name = "products.surveys.backend"
+    label = "surveys"

products/tasks/backend/apps.py

@@ -0,0 +1,7 @@
+from django.apps import AppConfig
+
+
+class TasksConfig(AppConfig):
+    default_auto_field = "django.db.models.BigAutoField"
+    name = "products.tasks.backend"
+    label = "tasks"

products/user_interviews/backend/apps.py

@@ -0,0 +1,7 @@
+from django.apps import AppConfig
+
+
+class UserInterviewsConfig(AppConfig):
+    default_auto_field = "django.db.models.BigAutoField"
+    name = "products.user_interviews.backend"
+    label = "user_interviews"