From 98cb47d4de1c43b5f21dc22667f553c7b23188c8 Mon Sep 17 00:00:00 2001 From: Nick Best Date: Thu, 30 Oct 2025 16:50:26 -0700 Subject: [PATCH] chore(migrations): register a sqlx migrator image (#40602) --- .github/workflows/ci-e2e-playwright.yml | 8 ++++---- .github/workflows/rust-docker-build.yml | 8 ++++++++ plugin-server/package.json | 4 ++-- ....persons-migrate => Dockerfile.sqlx-migrate} | 8 +++----- rust/bin/migrate-behavioral-cohorts | 2 +- rust/bin/{migrate-all => migrate-entry} | 17 ++++++++++++++--- rust/bin/migrate-persons | 2 +- 7 files changed, 33 insertions(+), 16 deletions(-) rename rust/{Dockerfile.persons-migrate => Dockerfile.sqlx-migrate} (88%) rename rust/bin/{migrate-all => migrate-entry} (84%) diff --git a/.github/workflows/ci-e2e-playwright.yml b/.github/workflows/ci-e2e-playwright.yml index c83412b7f9..43d0cbce9a 100644 --- a/.github/workflows/ci-e2e-playwright.yml +++ b/.github/workflows/ci-e2e-playwright.yml @@ -268,18 +268,18 @@ jobs: python manage.py migrate --noinput # Build and run Rust migrations using dedicated container - docker build -f rust/Dockerfile.persons-migrate -t posthog-migrations:latest ./rust + docker build -f rust/Dockerfile.sqlx-migrate -t posthog-migrations:latest ./rust docker run --rm --network host \ -e PERSONS_DATABASE_URL="postgres://posthog:posthog@localhost:5432/posthog_persons" \ -e CYCLOTRON_DATABASE_URL="postgres://posthog:posthog@localhost:5432/cyclotron" \ - posthog-migrations:latest + posthog-migrations:latest all # Build and run Rust migrations using dedicated container - docker build -f rust/Dockerfile.persons-migrate -t posthog-migrations:latest ./rust + docker build -f rust/Dockerfile.sqlx-migrate -t posthog-migrations:latest ./rust docker run --rm --network host \ -e PERSONS_DATABASE_URL="postgres://posthog:posthog@localhost:5432/posthog_e2e_test" \ -e CYCLOTRON_DATABASE_URL="postgres://posthog:posthog@localhost:5432/cyclotron" \ - posthog-migrations:latest + posthog-migrations:latest all python manage.py setup_dev wait diff --git a/.github/workflows/rust-docker-build.yml b/.github/workflows/rust-docker-build.yml index fee9ba6ae2..e42a92ac87 100644 --- a/.github/workflows/rust-docker-build.yml +++ b/.github/workflows/rust-docker-build.yml @@ -30,6 +30,9 @@ jobs: - image: hook-migrator dockerfile: ./rust/Dockerfile.migrate-hooks project: c1bwj4j4qg + - image: sqlx-migrate + dockerfile: ./rust/Dockerfile.sqlx-migrate + project: jmsz6gms8g - image: cyclotron-janitor dockerfile: ./rust/Dockerfile project: r4zm8vtlbw @@ -69,6 +72,7 @@ jobs: hook-janitor_digest: ${{ steps.digest.outputs.hook-janitor_digest }} hook-worker_digest: ${{ steps.digest.outputs.hook-worker_digest }} hook-migrator_digest: ${{ steps.digest.outputs.hook-migrator_digest }} + sqlx-migrate_digest: ${{ steps.digest.outputs.sqlx-migrate_digest }} cymbal_digest: ${{ steps.digest.outputs.cymbal_digest }} embedding-worker_digest: ${{ steps.digest.outputs.embedding-worker_digest }} feature-flags_digest: ${{ steps.digest.outputs.feature-flags_digest }} @@ -209,6 +213,10 @@ jobs: values: image: sha: '${{ needs.build.outputs.kafka-deduplicator_digest }}' + - release: sqlx-migrate + values: + image: + sha: '${{ needs.build.outputs.sqlx-migrate_digest }}' steps: - name: get deployer token id: deployer diff --git a/plugin-server/package.json b/plugin-server/package.json index 9ce7a9439f..de1c015d3f 100644 --- a/plugin-server/package.json +++ b/plugin-server/package.json @@ -27,11 +27,11 @@ "prepublishOnly": "pnpm build", "setup:dev:clickhouse": "cd .. && DEBUG=1 python manage.py migrate_clickhouse", "setup:test": "cd .. && TEST=1 python manage.py setup_test_environment && cd plugin-server && pnpm run setup:test:rust", - "setup:test:rust": "CYCLOTRON_DATABASE_NAME=test_cyclotron PERSONS_DATABASE_NAME=test_persons BEHAVIORAL_COHORTS_DATABASE_NAME=test_behavioral_cohorts ../rust/bin/migrate-all all && PERSONS_DATABASE_NAME=test_persons_migration ../rust/bin/migrate-persons", + "setup:test:rust": "CYCLOTRON_DATABASE_NAME=test_cyclotron PERSONS_DATABASE_NAME=test_persons BEHAVIORAL_COHORTS_DATABASE_NAME=test_behavioral_cohorts ../rust/bin/migrate-entry all && PERSONS_DATABASE_NAME=test_persons_migration ../rust/bin/migrate-persons", "migrate:persons": "../rust/bin/migrate-persons", "migrate:cyclotron": "../rust/bin/migrate-cyclotron", "migrate:behavioral-cohorts": "../rust/bin/migrate-behavioral-cohorts", - "migrate:rust": "../rust/bin/migrate-all all", + "migrate:rust": "../rust/bin/migrate-entry all", "services:start": "cd .. && docker compose -f docker-compose.dev.yml up", "services:stop": "cd .. && docker compose -f docker-compose.dev.yml down", "services:clean": "cd .. && docker compose -f docker-compose.dev.yml rm -v", diff --git a/rust/Dockerfile.persons-migrate b/rust/Dockerfile.sqlx-migrate similarity index 88% rename from rust/Dockerfile.persons-migrate rename to rust/Dockerfile.sqlx-migrate index c34d9c45f8..0dd804a2fd 100644 --- a/rust/Dockerfile.persons-migrate +++ b/rust/Dockerfile.sqlx-migrate @@ -24,13 +24,11 @@ COPY ./behavioral_cohorts_migrations ./behavioral_cohorts_migrations COPY ./bin/migrate-persons ./bin/migrate-persons COPY ./bin/migrate-cyclotron ./bin/migrate-cyclotron COPY ./bin/migrate-behavioral-cohorts ./bin/migrate-behavioral-cohorts -COPY ./bin/migrate-all ./bin/migrate-all +COPY ./bin/migrate-entry ./bin/migrate-entry -RUN chmod +x /migrations/bin/migrate-all && \ +RUN chmod +x /migrations/bin/migrate-entry && \ chmod +x /migrations/bin/migrate-persons && \ chmod +x /migrations/bin/migrate-cyclotron && \ chmod +x /migrations/bin/migrate-behavioral-cohorts -ENTRYPOINT ["/migrations/bin/migrate-all"] - -CMD ["all"] +ENTRYPOINT ["/migrations/bin/migrate-entry"] diff --git a/rust/bin/migrate-behavioral-cohorts b/rust/bin/migrate-behavioral-cohorts index 869b0d2f61..835862a441 100755 --- a/rust/bin/migrate-behavioral-cohorts +++ b/rust/bin/migrate-behavioral-cohorts @@ -1,4 +1,4 @@ #!/bin/sh SCRIPT_DIR=$(dirname "$(readlink -f "$0")") -exec "$SCRIPT_DIR/migrate-all" behavioral-cohorts \ No newline at end of file +exec "$SCRIPT_DIR/migrate-entry" behavioral-cohorts \ No newline at end of file diff --git a/rust/bin/migrate-all b/rust/bin/migrate-entry similarity index 84% rename from rust/bin/migrate-all rename to rust/bin/migrate-entry index a271f52ecb..3db4320d9e 100755 --- a/rust/bin/migrate-all +++ b/rust/bin/migrate-entry @@ -1,7 +1,18 @@ #!/bin/bash set -e -MIGRATION_TYPE="${1:-all}" +MIGRATION_TYPE="$1" + +if [ -z "$MIGRATION_TYPE" ]; then + echo "Error: No migration type specified" + echo "" + echo "Usage: $0 " + echo " all - Run all migrations" + echo " persons - Run only persons migrations" + echo " cyclotron - Run only cyclotron migrations" + echo " behavioral-cohorts - Run only behavioral cohorts migrations" + exit 1 +fi # Determine the base directory for migrations # In Docker, we use /migrations; locally, we use the parent directory @@ -80,8 +91,8 @@ case "$MIGRATION_TYPE" in run_behavioral_cohorts_migrations ;; *) - echo "Usage: $0 [all|persons|cyclotron|behavioral-cohorts]" - echo " all - Run all migrations (default)" + echo "Usage: $0 " + echo " all - Run all migrations" echo " persons - Run only persons migrations" echo " cyclotron - Run only cyclotron migrations" echo " behavioral-cohorts - Run only behavioral cohorts migrations" diff --git a/rust/bin/migrate-persons b/rust/bin/migrate-persons index 0be7afe83e..cb02edabac 100755 --- a/rust/bin/migrate-persons +++ b/rust/bin/migrate-persons @@ -1,4 +1,4 @@ #!/bin/sh SCRIPT_DIR=$(dirname "$(readlink -f "$0")") -exec "$SCRIPT_DIR/migrate-all" persons +exec "$SCRIPT_DIR/migrate-entry" persons