feat: export to video migration part (#37213)

2026-02-04 03:01:23 +01:00 · 2025-08-28 13:37:59 +02:00
parent 9c32ee5e4e
commit 1ea7b9b5ef
11 changed files with 131 additions and 3 deletions
--- a/.github/workflows/container-images-cd.yml
+++ b/.github/workflows/container-images-cd.yml
@@ -228,6 +228,11 @@ jobs:
              run: |
                  echo "changed=$((git diff --name-only HEAD^ HEAD | grep -qE '^ee/billing/|^posthog/temporal/quota_limiting|^posthog/temporal/common|^posthog/management/commands/start_temporal_worker.py$|^pyproject.toml$|^bin/temporal-django-worker$' && echo true) || echo false)" >> $GITHUB_OUTPUT

+            - name: Check for changes that affect video export temporal worker
+              id: check_changes_video_export_temporal_worker
+              run: |
+                  echo "changed=$((git diff --name-only HEAD^ HEAD | grep -qE '^posthog/temporal/exports_video|^posthog/tasks/exports/video_exporter.py|^posthog/temporal/common|^posthog/management/commands/start_temporal_worker.py$|^pyproject.toml$|^bin/temporal-django-worker$' && echo true) || echo false)" >> $GITHUB_OUTPUT
+
            - name: Trigger Billing Temporal Worker Cloud deployment
              if: steps.check_changes_billing_temporal_worker.outputs.changed == 'true'
              uses: peter-evans/repository-dispatch@ff45666b9427631e3450c54a1bcbee4d9ff4d7c0 # v3
@@ -249,6 +254,27 @@ jobs:
                        "timestamp": "${{ github.event.head_commit.timestamp }}"
                      }

+            - name: Trigger Video Export Temporal Worker Cloud deployment
+              if: steps.check_changes_video_export_temporal_worker.outputs.changed == 'true'
+              uses: peter-evans/repository-dispatch@ff45666b9427631e3450c54a1bcbee4d9ff4d7c0 # v3
+              with:
+                  token: ${{ steps.deployer.outputs.token }}
+                  repository: PostHog/charts
+                  event-type: commit_state_update
+                  client-payload: |
+                      {
+                        "values": {
+                          "image": {
+                            "sha": "${{ steps.build.outputs.digest }}"
+                          }
+                        },
+                        "release": "temporal-worker-video-export",
+                        "commit": ${{ toJson(github.event.head_commit) }},
+                        "repository": ${{ toJson(github.repository) }},
+                        "labels": ${{ steps.labels.outputs.labels }},
+                        "timestamp": "${{ github.event.head_commit.timestamp }}"
+                      }
+
            - name: Trigger General Purpose Temporal Worker Cloud deployment
              if: steps.check_changes_general_purpose_temporal_worker.outputs.changed == 'true'
              uses: peter-evans/repository-dispatch@ff45666b9427631e3450c54a1bcbee4d9ff4d7c0 # v3
--- a/14
+++ b/14
@@ -141,6 +141,7 @@ COPY --from=frontend-build /code/frontend/dist /code/frontend/dist
 RUN SKIP_SERVICE_VERSION_REQUIREMENTS=1 STATIC_COLLECTION=1 DATABASE_URL='postgres:///' REDIS_URL='redis:///' python manage.py collectstatic --noinput


+
 #
 # ---------------------------------------------------------
 #
@@ -180,7 +181,8 @@ RUN apt-get update && \
    "libxmlsec1" \
    "libxmlsec1-dev" \
    "libxml2" \
-    "gettext-base"
+    "gettext-base" \
+    "ffmpeg=7:5.1.6-0+deb12u1"

 # Install MS SQL dependencies
 RUN curl https://packages.microsoft.com/keys/microsoft.asc | tee /etc/apt/trusted.gpg.d/microsoft.asc
@@ -228,6 +230,16 @@ COPY --from=posthog-build --chown=posthog:posthog /python-runtime /python-runtim
 ENV PATH=/python-runtime/bin:$PATH \
    PYTHONPATH=/python-runtime

+# Install Playwright Chromium browser for video export (as root for system deps)
+USER root
+RUN /python-runtime/bin/python -m playwright install --with-deps chromium
+USER posthog
+
+# Validate video export dependencies
+RUN ffmpeg -version
+RUN /python-runtime/bin/python -c "import playwright; print('Playwright package imported successfully')"
+RUN /python-runtime/bin/python -c "from playwright.sync_api import sync_playwright; print('Playwright sync API available')"
+
 # Copy the frontend assets from the frontend-build stage.
 # TODO: this copy should not be necessary, we should remove it once we verify everything still works.
 COPY --from=frontend-build --chown=posthog:posthog /code/frontend/dist /code/frontend/dist
--- a/bin/mprocs-vite.yaml
+++ b/bin/mprocs-vite.yaml
@@ -38,6 +38,9 @@ procs:
        # added a sleep to give the docker stuff time to start
        shell: 'bin/check_kafka_clickhouse_up && bin/check_temporal_up && python manage.py start_temporal_worker --task-queue max-ai-task-queue --metrics-port 8006'

+    temporal-worker-video-export:
+        shell: 'bin/check_kafka_clickhouse_up && bin/check_temporal_up && python manage.py start_temporal_worker --task-queue video-export-task-queue --metrics-port 8009'
+
    dagster:
        shell: |-
            bin/check_postgres_up && \
--- a/bin/mprocs.yaml
+++ b/bin/mprocs.yaml
@@ -40,6 +40,9 @@ procs:
    temporal-worker-billing:
        shell: 'bin/check_kafka_clickhouse_up && bin/check_temporal_up && python manage.py start_temporal_worker --task-queue billing-task-queue --metrics-port 8008'

+    temporal-worker-video-export:
+        shell: 'bin/check_kafka_clickhouse_up && bin/check_temporal_up && python manage.py start_temporal_worker --task-queue video-export-task-queue --metrics-port 8009'
+
    dagster:
        shell: |-
            bin/check_postgres_up && \
--- a/posthog/constants.py
+++ b/posthog/constants.py
@@ -322,6 +322,7 @@ GENERAL_PURPOSE_TASK_QUEUE = "general-purpose-task-queue"
 TASKS_TASK_QUEUE = "tasks-task-queue"
 TEST_TASK_QUEUE = "test-task-queue"
 BILLING_TASK_QUEUE = "billing-task-queue"
+VIDEO_EXPORT_TASK_QUEUE = "video-export-task-queue"

 PERMITTED_FORUM_DOMAINS = ["localhost", "posthog.com"]

--- a/posthog/management/commands/start_temporal_worker.py
+++ b/posthog/management/commands/start_temporal_worker.py
@@ -24,6 +24,7 @@ from posthog.constants import (
    SYNC_BATCH_EXPORTS_TASK_QUEUE,
    TASKS_TASK_QUEUE,
    TEST_TASK_QUEUE,
+    VIDEO_EXPORT_TASK_QUEUE,
 )
 from posthog.temporal.ai import (
    ACTIVITIES as AI_ACTIVITIES,
@@ -89,6 +90,11 @@ from products.tasks.backend.temporal import (
 BILLING_WORKFLOWS: list = []
 BILLING_ACTIVITIES: list = []

+# TODO: Add video export workflows and activities once ready
+VIDEO_EXPORT_WORKFLOWS: list = []
+VIDEO_EXPORT_ACTIVITIES: list = []
+
+
 # Workflow and activity index
 WORKFLOWS_DICT = {
    SYNC_BATCH_EXPORTS_TASK_QUEUE: BATCH_EXPORTS_WORKFLOWS,
@@ -108,6 +114,7 @@ WORKFLOWS_DICT = {
    MAX_AI_TASK_QUEUE: AI_WORKFLOWS,
    TEST_TASK_QUEUE: TEST_WORKFLOWS,
    BILLING_TASK_QUEUE: BILLING_WORKFLOWS,
+    VIDEO_EXPORT_TASK_QUEUE: VIDEO_EXPORT_WORKFLOWS,
 }
 ACTIVITIES_DICT = {
    SYNC_BATCH_EXPORTS_TASK_QUEUE: BATCH_EXPORTS_ACTIVITIES,
@@ -127,6 +134,7 @@ ACTIVITIES_DICT = {
    MAX_AI_TASK_QUEUE: AI_ACTIVITIES,
    TEST_TASK_QUEUE: TEST_ACTIVITIES,
    BILLING_TASK_QUEUE: BILLING_ACTIVITIES,
+    VIDEO_EXPORT_TASK_QUEUE: VIDEO_EXPORT_ACTIVITIES,
 }

 TASK_QUEUE_METRIC_PREFIXES = {
--- a/posthog/migrations/0832_alter_exportedasset_export_format.py
+++ b/posthog/migrations/0832_alter_exportedasset_export_format.py
@@ -0,0 +1,31 @@
+# Generated by Django 4.2.22 on 2025-08-27 09:48
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+    dependencies = [
+        ("posthog", "0831_alter_groupusagemetric_bytecode_and_more"),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name="exportedasset",
+            name="export_format",
+            field=models.CharField(
+                choices=[
+                    ("image/png", "image/png"),
+                    ("application/pdf", "application/pdf"),
+                    ("text/csv", "text/csv"),
+                    (
+                        "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet",
+                        "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet",
+                    ),
+                    ("video/webm", "video/webm"),
+                    ("video/mp4", "video/mp4"),
+                    ("image/gif", "image/gif"),
+                ],
+                max_length=100,
+            ),
+        ),
+    ]
--- a/posthog/migrations/max_migration.txt
+++ b/posthog/migrations/max_migration.txt
@@ -1 +1 @@
-0831_alter_groupusagemetric_bytecode_and_more
+0832_alter_exportedasset_export_format
--- a/posthog/models/exported_asset.py
+++ b/posthog/models/exported_asset.py
@@ -45,8 +45,18 @@ class ExportedAsset(models.Model):
            "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet",
            "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet",
        )
+        WEBM = "video/webm", "video/webm"
+        MP4 = "video/mp4", "video/mp4"
+        GIF = "image/gif", "image/gif"

-    SUPPORTED_FORMATS = [ExportFormat.PNG, ExportFormat.CSV, ExportFormat.XLSX]
+    SUPPORTED_FORMATS = [
+        ExportFormat.PNG,
+        ExportFormat.CSV,
+        ExportFormat.XLSX,
+        ExportFormat.WEBM,
+        ExportFormat.MP4,
+        ExportFormat.GIF,
+    ]

    # Relations
    team = models.ForeignKey("Team", on_delete=models.CASCADE)
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -162,6 +162,7 @@ dependencies = [
    "django-admin-inline-paginator===0.4.0",
    "fastavro>=1.12.0",
    "pydantic-avro>=0.9.0",
+    "playwright>=1.54.0",
 ]

 [dependency-groups]
--- a/uv.lock
+++ b/uv.lock
@@ -3968,6 +3968,25 @@ wheels = [
    { url = "https://files.pythonhosted.org/packages/fe/39/979e8e21520d4e47a0bbe349e2713c0aac6f3d853d0e5b34d76206c439aa/platformdirs-4.3.8-py3-none-any.whl", hash = "sha256:ff7059bb7eb1179e2685604f4aaf157cfd9535242bd23742eadc3c13542139b4", size = 18567, upload-time = "2025-05-07T22:47:40.376Z" },
 ]

+[[package]]
+name = "playwright"
+version = "1.54.0"
+source = { registry = "https://pypi.org/simple" }
+dependencies = [
+    { name = "greenlet" },
+    { name = "pyee" },
+]
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/f3/09/33d5bfe393a582d8dac72165a9e88b274143c9df411b65ece1cc13f42988/playwright-1.54.0-py3-none-macosx_10_13_x86_64.whl", hash = "sha256:bf3b845af744370f1bd2286c2a9536f474cc8a88dc995b72ea9a5be714c9a77d", size = 40439034, upload-time = "2025-07-22T13:58:04.816Z" },
+    { url = "https://files.pythonhosted.org/packages/e1/7b/51882dc584f7aa59f446f2bb34e33c0e5f015de4e31949e5b7c2c10e54f0/playwright-1.54.0-py3-none-macosx_11_0_arm64.whl", hash = "sha256:780928b3ca2077aea90414b37e54edd0c4bbb57d1aafc42f7aa0b3fd2c2fac02", size = 38702308, upload-time = "2025-07-22T13:58:08.211Z" },
+    { url = "https://files.pythonhosted.org/packages/73/a1/7aa8ae175b240c0ec8849fcf000e078f3c693f9aa2ffd992da6550ea0dff/playwright-1.54.0-py3-none-macosx_11_0_universal2.whl", hash = "sha256:81d0b6f28843b27f288cfe438af0a12a4851de57998009a519ea84cee6fbbfb9", size = 40439037, upload-time = "2025-07-22T13:58:11.37Z" },
+    { url = "https://files.pythonhosted.org/packages/34/a9/45084fd23b6206f954198296ce39b0acf50debfdf3ec83a593e4d73c9c8a/playwright-1.54.0-py3-none-manylinux1_x86_64.whl", hash = "sha256:09919f45cc74c64afb5432646d7fef0d19fff50990c862cb8d9b0577093f40cc", size = 45920135, upload-time = "2025-07-22T13:58:14.494Z" },
+    { url = "https://files.pythonhosted.org/packages/02/d4/6a692f4c6db223adc50a6e53af405b45308db39270957a6afebddaa80ea2/playwright-1.54.0-py3-none-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:13ae206c55737e8e3eae51fb385d61c0312eeef31535643bb6232741b41b6fdc", size = 45302695, upload-time = "2025-07-22T13:58:18.901Z" },
+    { url = "https://files.pythonhosted.org/packages/72/7a/4ee60a1c3714321db187bebbc40d52cea5b41a856925156325058b5fca5a/playwright-1.54.0-py3-none-win32.whl", hash = "sha256:0b108622ffb6906e28566f3f31721cd57dda637d7e41c430287804ac01911f56", size = 35469309, upload-time = "2025-07-22T13:58:21.917Z" },
+    { url = "https://files.pythonhosted.org/packages/aa/77/8f8fae05a242ef639de963d7ae70a69d0da61d6d72f1207b8bbf74ffd3e7/playwright-1.54.0-py3-none-win_amd64.whl", hash = "sha256:9e5aee9ae5ab1fdd44cd64153313a2045b136fcbcfb2541cc0a3d909132671a2", size = 35469311, upload-time = "2025-07-22T13:58:24.707Z" },
+    { url = "https://files.pythonhosted.org/packages/33/ff/99a6f4292a90504f2927d34032a4baf6adb498dc3f7cf0f3e0e22899e310/playwright-1.54.0-py3-none-win_arm64.whl", hash = "sha256:a975815971f7b8dca505c441a4c56de1aeb56a211290f8cc214eeef5524e8d75", size = 31239119, upload-time = "2025-07-22T13:58:27.56Z" },
+]
+
 [[package]]
 name = "pluggy"
 version = "1.6.0"
@@ -4107,6 +4126,7 @@ dependencies = [
    { name = "pdpyras" },
    { name = "phonenumberslite" },
    { name = "pillow" },
+    { name = "playwright" },
    { name = "posthoganalytics" },
    { name = "psutil" },
    { name = "psycopg", extra = ["binary"] },
@@ -4334,6 +4354,7 @@ requires-dist = [
    { name = "pdpyras", specifier = "==5.2.0" },
    { name = "phonenumberslite", specifier = "==8.13.6" },
    { name = "pillow", specifier = "==10.2.0" },
+    { name = "playwright", specifier = ">=1.54.0" },
    { name = "posthoganalytics", specifier = ">=6.3.1" },
    { name = "psutil", specifier = "==6.0.0" },
    { name = "psycopg", extras = ["binary"], specifier = "==3.2.4" },
@@ -4804,6 +4825,18 @@ wheels = [
    { url = "https://files.pythonhosted.org/packages/b0/5f/1ebfd430df05c4f9e438dd3313c4456eab937d976f6ab8ce81a98f9fb381/pydot-3.0.4-py3-none-any.whl", hash = "sha256:bfa9c3fc0c44ba1d132adce131802d7df00429d1a79cc0346b0a5cd374dbe9c6", size = 35776, upload-time = "2025-01-05T16:18:42.836Z" },
 ]

+[[package]]
+name = "pyee"
+version = "13.0.0"
+source = { registry = "https://pypi.org/simple" }
+dependencies = [
+    { name = "typing-extensions" },
+]
+sdist = { url = "https://files.pythonhosted.org/packages/95/03/1fd98d5841cd7964a27d729ccf2199602fe05eb7a405c1462eb7277945ed/pyee-13.0.0.tar.gz", hash = "sha256:b391e3c5a434d1f5118a25615001dbc8f669cf410ab67d04c4d4e07c55481c37", size = 31250, upload-time = "2025-03-17T18:53:15.955Z" }
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/9b/4d/b9add7c84060d4c1906abe9a7e5359f2a60f7a9a4f67268b2766673427d8/pyee-13.0.0-py3-none-any.whl", hash = "sha256:48195a3cddb3b1515ce0695ed76036b5ccc2ef3a9f963ff9f77aec0139845498", size = 15730, upload-time = "2025-03-17T18:53:14.532Z" },
+]
+
 [[package]]
 name = "pygments"
 version = "2.19.1"