diff --git a/.github/workflows/container-images-cd.yml b/.github/workflows/container-images-cd.yml index 7080af4824..04f9edaaad 100644 --- a/.github/workflows/container-images-cd.yml +++ b/.github/workflows/container-images-cd.yml @@ -228,6 +228,11 @@ jobs: run: | echo "changed=$((git diff --name-only HEAD^ HEAD | grep -qE '^ee/billing/|^posthog/temporal/quota_limiting|^posthog/temporal/common|^posthog/management/commands/start_temporal_worker.py$|^pyproject.toml$|^bin/temporal-django-worker$' && echo true) || echo false)" >> $GITHUB_OUTPUT + - name: Check for changes that affect video export temporal worker + id: check_changes_video_export_temporal_worker + run: | + echo "changed=$((git diff --name-only HEAD^ HEAD | grep -qE '^posthog/temporal/exports_video|^posthog/tasks/exports/video_exporter.py|^posthog/temporal/common|^posthog/management/commands/start_temporal_worker.py$|^pyproject.toml$|^bin/temporal-django-worker$' && echo true) || echo false)" >> $GITHUB_OUTPUT + - name: Trigger Billing Temporal Worker Cloud deployment if: steps.check_changes_billing_temporal_worker.outputs.changed == 'true' uses: peter-evans/repository-dispatch@ff45666b9427631e3450c54a1bcbee4d9ff4d7c0 # v3 @@ -249,6 +254,27 @@ jobs: "timestamp": "${{ github.event.head_commit.timestamp }}" } + - name: Trigger Video Export Temporal Worker Cloud deployment + if: steps.check_changes_video_export_temporal_worker.outputs.changed == 'true' + uses: peter-evans/repository-dispatch@ff45666b9427631e3450c54a1bcbee4d9ff4d7c0 # v3 + with: + token: ${{ steps.deployer.outputs.token }} + repository: PostHog/charts + event-type: commit_state_update + client-payload: | + { + "values": { + "image": { + "sha": "${{ steps.build.outputs.digest }}" + } + }, + "release": "temporal-worker-video-export", + "commit": ${{ toJson(github.event.head_commit) }}, + "repository": ${{ toJson(github.repository) }}, + "labels": ${{ steps.labels.outputs.labels }}, + "timestamp": "${{ github.event.head_commit.timestamp }}" + } + - name: Trigger General Purpose Temporal Worker Cloud deployment if: steps.check_changes_general_purpose_temporal_worker.outputs.changed == 'true' uses: peter-evans/repository-dispatch@ff45666b9427631e3450c54a1bcbee4d9ff4d7c0 # v3 diff --git a/Dockerfile b/Dockerfile index 853f129f36..62d7303198 100644 --- a/Dockerfile +++ b/Dockerfile @@ -141,6 +141,7 @@ COPY --from=frontend-build /code/frontend/dist /code/frontend/dist RUN SKIP_SERVICE_VERSION_REQUIREMENTS=1 STATIC_COLLECTION=1 DATABASE_URL='postgres:///' REDIS_URL='redis:///' python manage.py collectstatic --noinput + # # --------------------------------------------------------- # @@ -180,7 +181,8 @@ RUN apt-get update && \ "libxmlsec1" \ "libxmlsec1-dev" \ "libxml2" \ - "gettext-base" + "gettext-base" \ + "ffmpeg=7:5.1.6-0+deb12u1" # Install MS SQL dependencies RUN curl https://packages.microsoft.com/keys/microsoft.asc | tee /etc/apt/trusted.gpg.d/microsoft.asc @@ -228,6 +230,16 @@ COPY --from=posthog-build --chown=posthog:posthog /python-runtime /python-runtim ENV PATH=/python-runtime/bin:$PATH \ PYTHONPATH=/python-runtime +# Install Playwright Chromium browser for video export (as root for system deps) +USER root +RUN /python-runtime/bin/python -m playwright install --with-deps chromium +USER posthog + +# Validate video export dependencies +RUN ffmpeg -version +RUN /python-runtime/bin/python -c "import playwright; print('Playwright package imported successfully')" +RUN /python-runtime/bin/python -c "from playwright.sync_api import sync_playwright; print('Playwright sync API available')" + # Copy the frontend assets from the frontend-build stage. # TODO: this copy should not be necessary, we should remove it once we verify everything still works. COPY --from=frontend-build --chown=posthog:posthog /code/frontend/dist /code/frontend/dist diff --git a/bin/mprocs-vite.yaml b/bin/mprocs-vite.yaml index e8fe894d23..538910585b 100755 --- a/bin/mprocs-vite.yaml +++ b/bin/mprocs-vite.yaml @@ -38,6 +38,9 @@ procs: # added a sleep to give the docker stuff time to start shell: 'bin/check_kafka_clickhouse_up && bin/check_temporal_up && python manage.py start_temporal_worker --task-queue max-ai-task-queue --metrics-port 8006' + temporal-worker-video-export: + shell: 'bin/check_kafka_clickhouse_up && bin/check_temporal_up && python manage.py start_temporal_worker --task-queue video-export-task-queue --metrics-port 8009' + dagster: shell: |- bin/check_postgres_up && \ diff --git a/bin/mprocs.yaml b/bin/mprocs.yaml index bdf34436e0..2fd31178ce 100755 --- a/bin/mprocs.yaml +++ b/bin/mprocs.yaml @@ -40,6 +40,9 @@ procs: temporal-worker-billing: shell: 'bin/check_kafka_clickhouse_up && bin/check_temporal_up && python manage.py start_temporal_worker --task-queue billing-task-queue --metrics-port 8008' + temporal-worker-video-export: + shell: 'bin/check_kafka_clickhouse_up && bin/check_temporal_up && python manage.py start_temporal_worker --task-queue video-export-task-queue --metrics-port 8009' + dagster: shell: |- bin/check_postgres_up && \ diff --git a/posthog/constants.py b/posthog/constants.py index d602b55eb8..5ddc0b3ba7 100644 --- a/posthog/constants.py +++ b/posthog/constants.py @@ -322,6 +322,7 @@ GENERAL_PURPOSE_TASK_QUEUE = "general-purpose-task-queue" TASKS_TASK_QUEUE = "tasks-task-queue" TEST_TASK_QUEUE = "test-task-queue" BILLING_TASK_QUEUE = "billing-task-queue" +VIDEO_EXPORT_TASK_QUEUE = "video-export-task-queue" PERMITTED_FORUM_DOMAINS = ["localhost", "posthog.com"] diff --git a/posthog/management/commands/start_temporal_worker.py b/posthog/management/commands/start_temporal_worker.py index 0a1bf098f3..a635c7db19 100644 --- a/posthog/management/commands/start_temporal_worker.py +++ b/posthog/management/commands/start_temporal_worker.py @@ -24,6 +24,7 @@ from posthog.constants import ( SYNC_BATCH_EXPORTS_TASK_QUEUE, TASKS_TASK_QUEUE, TEST_TASK_QUEUE, + VIDEO_EXPORT_TASK_QUEUE, ) from posthog.temporal.ai import ( ACTIVITIES as AI_ACTIVITIES, @@ -89,6 +90,11 @@ from products.tasks.backend.temporal import ( BILLING_WORKFLOWS: list = [] BILLING_ACTIVITIES: list = [] +# TODO: Add video export workflows and activities once ready +VIDEO_EXPORT_WORKFLOWS: list = [] +VIDEO_EXPORT_ACTIVITIES: list = [] + + # Workflow and activity index WORKFLOWS_DICT = { SYNC_BATCH_EXPORTS_TASK_QUEUE: BATCH_EXPORTS_WORKFLOWS, @@ -108,6 +114,7 @@ WORKFLOWS_DICT = { MAX_AI_TASK_QUEUE: AI_WORKFLOWS, TEST_TASK_QUEUE: TEST_WORKFLOWS, BILLING_TASK_QUEUE: BILLING_WORKFLOWS, + VIDEO_EXPORT_TASK_QUEUE: VIDEO_EXPORT_WORKFLOWS, } ACTIVITIES_DICT = { SYNC_BATCH_EXPORTS_TASK_QUEUE: BATCH_EXPORTS_ACTIVITIES, @@ -127,6 +134,7 @@ ACTIVITIES_DICT = { MAX_AI_TASK_QUEUE: AI_ACTIVITIES, TEST_TASK_QUEUE: TEST_ACTIVITIES, BILLING_TASK_QUEUE: BILLING_ACTIVITIES, + VIDEO_EXPORT_TASK_QUEUE: VIDEO_EXPORT_ACTIVITIES, } TASK_QUEUE_METRIC_PREFIXES = { diff --git a/posthog/migrations/0832_alter_exportedasset_export_format.py b/posthog/migrations/0832_alter_exportedasset_export_format.py new file mode 100644 index 0000000000..69c06441d4 --- /dev/null +++ b/posthog/migrations/0832_alter_exportedasset_export_format.py @@ -0,0 +1,31 @@ +# Generated by Django 4.2.22 on 2025-08-27 09:48 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + dependencies = [ + ("posthog", "0831_alter_groupusagemetric_bytecode_and_more"), + ] + + operations = [ + migrations.AlterField( + model_name="exportedasset", + name="export_format", + field=models.CharField( + choices=[ + ("image/png", "image/png"), + ("application/pdf", "application/pdf"), + ("text/csv", "text/csv"), + ( + "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet", + "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet", + ), + ("video/webm", "video/webm"), + ("video/mp4", "video/mp4"), + ("image/gif", "image/gif"), + ], + max_length=100, + ), + ), + ] diff --git a/posthog/migrations/max_migration.txt b/posthog/migrations/max_migration.txt index 6c9c1f7d61..c7f54133ee 100644 --- a/posthog/migrations/max_migration.txt +++ b/posthog/migrations/max_migration.txt @@ -1 +1 @@ -0831_alter_groupusagemetric_bytecode_and_more +0832_alter_exportedasset_export_format diff --git a/posthog/models/exported_asset.py b/posthog/models/exported_asset.py index 6748d1d0bf..f18f2d4195 100644 --- a/posthog/models/exported_asset.py +++ b/posthog/models/exported_asset.py @@ -45,8 +45,18 @@ class ExportedAsset(models.Model): "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet", "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet", ) + WEBM = "video/webm", "video/webm" + MP4 = "video/mp4", "video/mp4" + GIF = "image/gif", "image/gif" - SUPPORTED_FORMATS = [ExportFormat.PNG, ExportFormat.CSV, ExportFormat.XLSX] + SUPPORTED_FORMATS = [ + ExportFormat.PNG, + ExportFormat.CSV, + ExportFormat.XLSX, + ExportFormat.WEBM, + ExportFormat.MP4, + ExportFormat.GIF, + ] # Relations team = models.ForeignKey("Team", on_delete=models.CASCADE) diff --git a/pyproject.toml b/pyproject.toml index a3d926a455..1d3a4d041f 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -162,6 +162,7 @@ dependencies = [ "django-admin-inline-paginator===0.4.0", "fastavro>=1.12.0", "pydantic-avro>=0.9.0", + "playwright>=1.54.0", ] [dependency-groups] diff --git a/uv.lock b/uv.lock index 920bac7772..a5dd578fe1 100644 --- a/uv.lock +++ b/uv.lock @@ -3968,6 +3968,25 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/fe/39/979e8e21520d4e47a0bbe349e2713c0aac6f3d853d0e5b34d76206c439aa/platformdirs-4.3.8-py3-none-any.whl", hash = "sha256:ff7059bb7eb1179e2685604f4aaf157cfd9535242bd23742eadc3c13542139b4", size = 18567, upload-time = "2025-05-07T22:47:40.376Z" }, ] +[[package]] +name = "playwright" +version = "1.54.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "greenlet" }, + { name = "pyee" }, +] +wheels = [ + { url = "https://files.pythonhosted.org/packages/f3/09/33d5bfe393a582d8dac72165a9e88b274143c9df411b65ece1cc13f42988/playwright-1.54.0-py3-none-macosx_10_13_x86_64.whl", hash = "sha256:bf3b845af744370f1bd2286c2a9536f474cc8a88dc995b72ea9a5be714c9a77d", size = 40439034, upload-time = "2025-07-22T13:58:04.816Z" }, + { url = "https://files.pythonhosted.org/packages/e1/7b/51882dc584f7aa59f446f2bb34e33c0e5f015de4e31949e5b7c2c10e54f0/playwright-1.54.0-py3-none-macosx_11_0_arm64.whl", hash = "sha256:780928b3ca2077aea90414b37e54edd0c4bbb57d1aafc42f7aa0b3fd2c2fac02", size = 38702308, upload-time = "2025-07-22T13:58:08.211Z" }, + { url = "https://files.pythonhosted.org/packages/73/a1/7aa8ae175b240c0ec8849fcf000e078f3c693f9aa2ffd992da6550ea0dff/playwright-1.54.0-py3-none-macosx_11_0_universal2.whl", hash = "sha256:81d0b6f28843b27f288cfe438af0a12a4851de57998009a519ea84cee6fbbfb9", size = 40439037, upload-time = "2025-07-22T13:58:11.37Z" }, + { url = "https://files.pythonhosted.org/packages/34/a9/45084fd23b6206f954198296ce39b0acf50debfdf3ec83a593e4d73c9c8a/playwright-1.54.0-py3-none-manylinux1_x86_64.whl", hash = "sha256:09919f45cc74c64afb5432646d7fef0d19fff50990c862cb8d9b0577093f40cc", size = 45920135, upload-time = "2025-07-22T13:58:14.494Z" }, + { url = "https://files.pythonhosted.org/packages/02/d4/6a692f4c6db223adc50a6e53af405b45308db39270957a6afebddaa80ea2/playwright-1.54.0-py3-none-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:13ae206c55737e8e3eae51fb385d61c0312eeef31535643bb6232741b41b6fdc", size = 45302695, upload-time = "2025-07-22T13:58:18.901Z" }, + { url = "https://files.pythonhosted.org/packages/72/7a/4ee60a1c3714321db187bebbc40d52cea5b41a856925156325058b5fca5a/playwright-1.54.0-py3-none-win32.whl", hash = "sha256:0b108622ffb6906e28566f3f31721cd57dda637d7e41c430287804ac01911f56", size = 35469309, upload-time = "2025-07-22T13:58:21.917Z" }, + { url = "https://files.pythonhosted.org/packages/aa/77/8f8fae05a242ef639de963d7ae70a69d0da61d6d72f1207b8bbf74ffd3e7/playwright-1.54.0-py3-none-win_amd64.whl", hash = "sha256:9e5aee9ae5ab1fdd44cd64153313a2045b136fcbcfb2541cc0a3d909132671a2", size = 35469311, upload-time = "2025-07-22T13:58:24.707Z" }, + { url = "https://files.pythonhosted.org/packages/33/ff/99a6f4292a90504f2927d34032a4baf6adb498dc3f7cf0f3e0e22899e310/playwright-1.54.0-py3-none-win_arm64.whl", hash = "sha256:a975815971f7b8dca505c441a4c56de1aeb56a211290f8cc214eeef5524e8d75", size = 31239119, upload-time = "2025-07-22T13:58:27.56Z" }, +] + [[package]] name = "pluggy" version = "1.6.0" @@ -4107,6 +4126,7 @@ dependencies = [ { name = "pdpyras" }, { name = "phonenumberslite" }, { name = "pillow" }, + { name = "playwright" }, { name = "posthoganalytics" }, { name = "psutil" }, { name = "psycopg", extra = ["binary"] }, @@ -4334,6 +4354,7 @@ requires-dist = [ { name = "pdpyras", specifier = "==5.2.0" }, { name = "phonenumberslite", specifier = "==8.13.6" }, { name = "pillow", specifier = "==10.2.0" }, + { name = "playwright", specifier = ">=1.54.0" }, { name = "posthoganalytics", specifier = ">=6.3.1" }, { name = "psutil", specifier = "==6.0.0" }, { name = "psycopg", extras = ["binary"], specifier = "==3.2.4" }, @@ -4804,6 +4825,18 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/b0/5f/1ebfd430df05c4f9e438dd3313c4456eab937d976f6ab8ce81a98f9fb381/pydot-3.0.4-py3-none-any.whl", hash = "sha256:bfa9c3fc0c44ba1d132adce131802d7df00429d1a79cc0346b0a5cd374dbe9c6", size = 35776, upload-time = "2025-01-05T16:18:42.836Z" }, ] +[[package]] +name = "pyee" +version = "13.0.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/95/03/1fd98d5841cd7964a27d729ccf2199602fe05eb7a405c1462eb7277945ed/pyee-13.0.0.tar.gz", hash = "sha256:b391e3c5a434d1f5118a25615001dbc8f669cf410ab67d04c4d4e07c55481c37", size = 31250, upload-time = "2025-03-17T18:53:15.955Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/9b/4d/b9add7c84060d4c1906abe9a7e5359f2a60f7a9a4f67268b2766673427d8/pyee-13.0.0-py3-none-any.whl", hash = "sha256:48195a3cddb3b1515ce0695ed76036b5ccc2ef3a9f963ff9f77aec0139845498", size = 15730, upload-time = "2025-03-17T18:53:14.532Z" }, +] + [[package]] name = "pygments" version = "2.19.1"