posthog/.github/workflows/ci-backend.yml

# This workflow runs all of our backend django tests.
#
# If these tests get too slow, look at increasing concurrency and re-timing the tests by manually dispatching
# .github/workflows/ci-backend-update-test-timing.yml action
name: Backend CI
on:
    push:
        branches:
            - master
    workflow_dispatch:
        inputs:
            clickhouseServerVersion:
                description: ClickHouse server version. Leave blank for default
                type: string
    pull_request:

concurrency:
    group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }}
    cancel-in-progress: true
env:
    SECRET_KEY: '6b01eee4f945ca25045b5aab440b953461faf08693a9abbf1166dc7c6b9772da' # unsafe - for testing only
    DATABASE_URL: 'postgres://posthog:posthog@localhost:5432/posthog'
    REDIS_URL: 'redis://localhost'
    CLICKHOUSE_HOST: 'localhost'
    CLICKHOUSE_SECURE: 'False'
    CLICKHOUSE_VERIFY: 'False'
    TEST: 1
    CLICKHOUSE_SERVER_IMAGE_VERSION: ${{ github.event.inputs.clickhouseServerVersion || '' }}
    OBJECT_STORAGE_ENABLED: 'True'
    OBJECT_STORAGE_ENDPOINT: 'http://localhost:19000'
    OBJECT_STORAGE_ACCESS_KEY_ID: 'object_storage_root_user'
    OBJECT_STORAGE_SECRET_ACCESS_KEY: 'object_storage_root_password'
    # tests would intermittently fail in GH actions
    # with exit code 134 _after passing_ all tests
    # this appears to fix it
    # absolute wild tbh https://stackoverflow.com/a/75503402
    DISPLAY: ':99.0'
    OIDC_RSA_PRIVATE_KEY: 'test'
jobs:
    # Job to decide if we should run backend ci
    # See https://github.com/dorny/paths-filter#conditional-execution for more details
    changes:
        runs-on: ubuntu-latest
        timeout-minutes: 5
        name: Determine need to run backend and migration checks
        # Set job outputs to values from filter step
        outputs:
            backend: ${{ steps.filter.outputs.backend }}
            backend_files: ${{ steps.filter.outputs.backend_files }}
            migrations: ${{ steps.filter.outputs.migrations }}
            migrations_files: ${{ steps.filter.outputs.migrations_files }}
            tasks_temporal: ${{ steps.filter.outputs.tasks_temporal }}
        steps:
            # For pull requests it's not necessary to checkout the code, but we
            # also want this to run on master so we need to checkout
            - uses: actions/checkout@v4
              with:
                  clean: false
            - name: Clean up data directories with container permissions
              run: |
                  # Use docker to clean up files created by containers
                  [ -d "data" ] && docker run --rm -v "$(pwd)/data:/data" alpine sh -c "rm -rf /data/seaweedfs /data/minio" || true
              continue-on-error: true

            - uses: dorny/paths-filter@4512585405083f25c027a35db413c2b3b9006d50 # v2
              id: filter
              with:
                  list-files: 'escape'
                  filters: |
                      backend:
                        # Avoid running backend tests for irrelevant changes
                        # NOTE: we are at risk of missing a dependency here. We could make
                        # the dependencies more clear if we separated the backend/frontend
                        # code completely
                        # really we should ignore ee/frontend/** but dorny doesn't support that
                        # - '!ee/frontend/**'
                        # including the negated rule appears to work
                        # but makes it always match because the checked file always isn't `ee/frontend/**` 🙈
                        - 'ee/**/*'
                        - 'common/hogvm/**/*'
                        - 'posthog/**/*'
                        - 'products/**/backend/**/*'
                        - 'bin/*.py'
                        - pyproject.toml
                        - uv.lock
                        - requirements.txt
                        - requirements-dev.txt
                        - mypy.ini
                        - pytest.ini
                        - frontend/src/queries/schema.json # Used for generating schema.py
                        - common/plugin_transpiler/src # Used for transpiling plugins
                        # Make sure we run if someone is explicitly changing the workflow
                        - .github/workflows/ci-backend.yml
                        # We use docker compose for tests, make sure we rerun on
                        # changes to docker-compose.dev.yml e.g. dependency
                        # version changes
                        - docker-compose.dev.yml
                        - docker-compose.base.yml
                        - frontend/public/email/*
                        - docker/clickhouse
                        # These scripts are used in the CI
                        - bin/check_temporal_up
                        - bin/check_kafka_clickhouse_up
                      migrations:
                        - docker/clickhouse
                        - 'posthog/migrations/*.py'
                        - 'products/*/backend/migrations/*.py'
                        - 'products/*/migrations/*.py'  # Legacy structure
                        - 'rust/persons_migrations/*.sql'
                        - 'rust/bin/migrate-persons'
                      tasks_temporal:
                        - 'products/tasks/backend/temporal/**/*'

    check-migrations:
        needs: [changes]
        if: needs.changes.outputs.backend == 'true'
        timeout-minutes: 20

        name: Validate Django and CH migrations
        runs-on: ubuntu-latest

        steps:
            - uses: actions/checkout@v4
              with:
                  clean: false
            - name: Clean up data directories with container permissions
              run: |
                  # Use docker to clean up files created by containers
                  [ -d "data" ] && docker run --rm -v "$(pwd)/data:/data" alpine sh -c "rm -rf /data/seaweedfs /data/minio" || true
              continue-on-error: true

            - name: Stop/Start stack with Docker Compose
              run: |
                  docker compose -f docker-compose.dev.yml down
                  docker compose -f docker-compose.dev.yml up -d &

            - name: Set up Python
              uses: actions/setup-python@v5
              with:
                  python-version: 3.12.11
                  token: ${{ secrets.POSTHOG_BOT_PAT }}

            - name: Install uv
              id: setup-uv
              uses: astral-sh/setup-uv@3259c6206f993105e3a61b142c2d97bf4b9ef83d # v7.1.0
              with:
                  enable-cache: true
                  version: 0.8.19

            - name: Install SAML (python3-saml) dependencies
              if: steps.setup-uv.outputs.cache-hit != 'true'
              run: |
                  sudo apt-get update
                  sudo apt-get install libxml2-dev libxmlsec1-dev libxmlsec1-openssl
            - name: Install Rust
              uses: dtolnay/rust-toolchain@6691ebadcb18182cc1391d07c9f295f657c593cd # 1.88
              with:
                  toolchain: 1.88.0
                  components: cargo

            - name: Cache Rust dependencies
              uses: Swatinem/rust-cache@f13886b937689c021905a6b90929199931d60db1 # v2.8.1

            - name: Install sqlx-cli
              run: |
                  cargo install sqlx-cli --version 0.8.0 --features postgres --no-default-features --locked

            # First running migrations from master, to simulate the real-world scenario
            - name: Checkout master
              uses: actions/checkout@v4
              with:
                  ref: master
                  clean: false

            - name: Install python dependencies for master
              run: |
                  UV_PROJECT_ENVIRONMENT=.venv-master uv sync --frozen --dev

            - name: Wait for services to be available
              run: |
                  bin/check_postgres_up
                  bin/check_kafka_clickhouse_up

            - name: Run migrations up to master
              run: |
                  # Run Django migrations first (excluding managed=False models)
                  .venv-master/bin/python manage.py migrate
                  # Then run persons migrations using sqlx; comment out until we've merged
                  # DATABASE_URL="postgres://posthog:posthog@localhost:5432/posthog_persons" \
                  #   sqlx database create
                  # DATABASE_URL="postgres://posthog:posthog@localhost:5432/posthog_persons" \
                  #   sqlx migrate run --source rust/persons_migrations/

            # Now we can consider this PR's migrations
            - name: Checkout this PR
              uses: actions/checkout@v4
              with:
                  clean: false

            - name: Install python dependencies for this PR
              run: |
                  UV_PROJECT_ENVIRONMENT=$pythonLocation uv sync --frozen --dev

            - name: Check migrations and post SQL comment
              if: github.event_name == 'pull_request' && needs.changes.outputs.migrations == 'true'
              env:
                  GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
              run: |
                  # Read the changed files from the output
                  CHANGED_FILES="${{ needs.changes.outputs.migrations_files }}"

                  # If no migration files changed, exit
                  if [ -z "$CHANGED_FILES" ]; then
                    echo "No migration files changed"
                    exit 0
                  fi

                  # Initialize comment body for SQL changes
                  COMMENT_BODY="## Migration SQL Changes\n\nHey 👋, we've detected some migrations on this PR. Here's the SQL output for each migration, make sure they make sense:\n\n"

                  # Process each changed migration file (excluding Rust migrations)
                  for file in $CHANGED_FILES; do
                    # Skip Rust migrations as they're handled separately by sqlx
                    if [[ $file =~ rust/persons_migrations ]]; then
                      continue
                    fi
                    if [[ $file =~ migrations/([0-9]+)_ ]]; then
                      migration_number="${BASH_REMATCH[1]}"
                      # Get app name by looking at the directory structure
                      # For new structure products/user_interviews/backend/migrations, we want user_interviews
                      # For old structure products/user_interviews/migrations, we want user_interviews
                      if [[ $file =~ products/([^/]+)/backend/migrations/ ]]; then
                        app_name="${BASH_REMATCH[1]}"
                      else
                        app_name=$(echo $file | sed -E 's|^([^/]+/)*([^/]+)/migrations/.*|\2|')
                      fi
                        echo "Checking migration $migration_number for app $app_name"

                      # Get SQL output
                      SQL_OUTPUT=$(python manage.py sqlmigrate $app_name $migration_number)

                      # Add to comment body
                      COMMENT_BODY+="#### [\`$file\`](https:\/\/github.com\/${{ github.repository }}\/blob\/${{ github.sha }}\/$file)\n\`\`\`sql\n$SQL_OUTPUT\n\`\`\`\n\n"
                    fi
                  done

                  # Get existing comments
                  COMMENTS=$(curl -s -H "Authorization: token $GITHUB_TOKEN" \
                    -H "Accept: application/vnd.github.v3+json" \
                    "https://api.github.com/repos/${{ github.repository }}/issues/${{ github.event.pull_request.number }}/comments")

                  # Extract comment ID if exists
                  SQL_COMMENT_ID=$(echo "$COMMENTS" | jq -r '.[] | select(.body | startswith("## Migration SQL Changes")) | .id' | head -1)

                  # Add timestamp and commit SHA to SQL changes
                  TIMESTAMP=$(date -u '+%Y-%m-%d %H:%M UTC')
                  COMMIT_SHA="${{ github.event.pull_request.head.sha }}"
                  COMMIT_SHORT="${COMMIT_SHA:0:7}"
                  COMMENT_BODY+="\n*Last updated: $TIMESTAMP ([${COMMIT_SHORT}](https://github.com/${{ github.repository }}/commit/${COMMIT_SHA}))*"

                  # Convert \n into actual newlines
                  COMMENT_BODY=$(printf '%b' "$COMMENT_BODY")
                  COMMENT_BODY_JSON=$(jq -n --arg body "$COMMENT_BODY" '{body: $body}')

                  if [ -n "$SQL_COMMENT_ID" ]; then
                    # Update existing comment
                    echo "Updating existing SQL comment $SQL_COMMENT_ID"
                    curl -X PATCH \
                      -H "Authorization: token $GITHUB_TOKEN" \
                      -H "Accept: application/vnd.github.v3+json" \
                      "https://api.github.com/repos/${{ github.repository }}/issues/comments/$SQL_COMMENT_ID" \
                      -d "$COMMENT_BODY_JSON"
                  else
                    # Post new SQL comment to PR
                    echo "Posting new SQL comment to PR"
                    curl -X POST \
                      -H "Authorization: token $GITHUB_TOKEN" \
                      -H "Accept: application/vnd.github.v3+json" \
                      "https://api.github.com/repos/${{ github.repository }}/issues/${{ github.event.pull_request.number }}/comments" \
                      -d "$COMMENT_BODY_JSON"
                  fi

            - name: Run migration risk analysis and post comment
              if: github.event_name == 'pull_request'
              env:
                  GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
              run: |
                  # Get risk analysis for all unapplied migrations (including third-party)
                  set +e  # Don't exit immediately on error
                  RISK_ANALYSIS=$(python manage.py analyze_migration_risk --fail-on-blocked 2>/dev/null)
                  EXIT_CODE=$?
                  set -e  # Re-enable exit on error

                  # Save analysis to file for artifact upload
                  if [ -n "$RISK_ANALYSIS" ]; then
                    echo "$RISK_ANALYSIS" > migration_analysis.md
                  fi

                  # Get existing comments
                  COMMENTS=$(curl -s -H "Authorization: token $GITHUB_TOKEN" \
                    -H "Accept: application/vnd.github.v3+json" \
                    "https://api.github.com/repos/${{ github.repository }}/issues/${{ github.event.pull_request.number }}/comments")

                  # Extract comment ID if exists
                  COMMENT_ID=$(echo "$COMMENTS" | jq -r '.[] | select(.body | startswith("## 🔍 Migration Risk Analysis")) | .id' | head -1)

                  if [ -n "$RISK_ANALYSIS" ] && echo "$RISK_ANALYSIS" | grep -q "Summary:"; then
                    # Add timestamp and commit SHA to analysis
                    TIMESTAMP=$(date -u '+%Y-%m-%d %H:%M UTC')
                    COMMIT_SHA="${{ github.event.pull_request.head.sha }}"
                    COMMIT_SHORT="${COMMIT_SHA:0:7}"
                    RISK_COMMENT="## 🔍 Migration Risk Analysis\n\nWe've analyzed your migrations for potential risks.\n\n$RISK_ANALYSIS\n\n*Last updated: $TIMESTAMP ([${COMMIT_SHORT}](https://github.com/${{ github.repository }}/commit/${COMMIT_SHA}))*"
                    RISK_COMMENT=$(printf '%b' "$RISK_COMMENT")
                    RISK_COMMENT_JSON=$(jq -n --arg body "$RISK_COMMENT" '{body: $body}')

                    if [ -n "$COMMENT_ID" ]; then
                      # Update existing comment
                      echo "Updating existing risk analysis comment $COMMENT_ID"
                      curl -X PATCH \
                        -H "Authorization: token $GITHUB_TOKEN" \
                        -H "Accept: application/vnd.github.v3+json" \
                        "https://api.github.com/repos/${{ github.repository }}/issues/comments/$COMMENT_ID" \
                        -d "$RISK_COMMENT_JSON"
                    else
                      # Create new comment if none exists
                      echo "Posting new risk analysis comment to PR"
                      curl -X POST \
                        -H "Authorization: token $GITHUB_TOKEN" \
                        -H "Accept: application/vnd.github.v3+json" \
                        "https://api.github.com/repos/${{ github.repository }}/issues/${{ github.event.pull_request.number }}/comments" \
                        -d "$RISK_COMMENT_JSON"
                    fi
                  elif [ -n "$COMMENT_ID" ]; then
                    # No migrations to analyze but comment exists - delete it
                    echo "Deleting risk analysis comment (no migrations to analyze)"
                    curl -X DELETE \
                      -H "Authorization: token $GITHUB_TOKEN" \
                      -H "Accept: application/vnd.github.v3+json" \
                      "https://api.github.com/repos/${{ github.repository }}/issues/comments/$COMMENT_ID"
                  else
                    echo "No migrations to analyze and no existing comment"
                  fi

                  # Fail the job if there were blocked migrations
                  if [ $EXIT_CODE -ne 0 ]; then
                    exit $EXIT_CODE
                  fi

            - name: Upload migration analysis artifact
              if: always() && github.event_name == 'pull_request'
              uses: actions/upload-artifact@v4
              with:
                  name: migration-analysis
                  path: migration_analysis.md
                  if-no-files-found: ignore

            - name: Run migrations for this PR
              run: |
                  # Run Django migrations first (excluding managed=False models)
                  python manage.py migrate
                  # Then run persons migrations using sqlx
                  DATABASE_URL="postgres://posthog:posthog@localhost:5432/posthog_persons" \
                    sqlx migrate run --source rust/persons_migrations/

            - name: Check migrations
              run: |
                  DATABASE_URL="postgres://posthog:posthog@localhost:5432/posthog_persons" \
                    sqlx migrate info --source rust/persons_migrations/
                  python manage.py makemigrations --check --dry-run
                  git fetch origin master
                  # Check migration safety using old SQL-based checker (still uses stdin from git diff)
                  echo "${{ needs.changes.outputs.migrations_files }}" | grep -v migrations/0001_ | grep -v 'rust/persons_migrations' | python manage.py test_migrations_are_safe

            - name: Check CH migrations
              run: |
                  # Same as above, except now for CH looking at files that were added in posthog/clickhouse/migrations/
                  git diff --name-status origin/master..HEAD | grep "A\sposthog/clickhouse/migrations/" | grep -v README | awk '{print $2}' | python manage.py test_ch_migrations_are_safe

    django:
        needs: [changes]
        if: needs.changes.outputs.backend == 'true'
        # increase for tmate testing
        timeout-minutes: 30

        name: Django tests – ${{ matrix.segment }} (persons-on-events ${{ matrix.person-on-events && 'on' || 'off' }}), Py ${{ matrix.python-version }}, ${{ matrix.clickhouse-server-image }} (${{matrix.group}}/${{ matrix.concurrency }})
        runs-on: depot-ubuntu-latest

        strategy:
            fail-fast: false
            matrix:
                python-version: ['3.12.11']
                clickhouse-server-image: ['clickhouse/clickhouse-server:25.8.11.66']
                segment: ['Core']
                person-on-events: [false]
                # :NOTE: Keep concurrency and groups in sync
                concurrency: [40]
                group:
                    [
                        1,
                        2,
                        3,
                        4,
                        5,
                        6,
                        7,
                        8,
                        9,
                        10,
                        11,
                        12,
                        13,
                        14,
                        15,
                        16,
                        17,
                        18,
                        19,
                        20,
                        21,
                        22,
                        23,
                        24,
                        25,
                        26,
                        27,
                        28,
                        29,
                        30,
                        31,
                        32,
                        33,
                        34,
                        35,
                        36,
                        37,
                        38,
                        39,
                        40,
                    ]
                include:
                    - segment: 'Core'
                      person-on-events: true
                      clickhouse-server-image: 'clickhouse/clickhouse-server:25.8.11.66'
                      python-version: '3.12.11'
                      concurrency: 10
                      group: 1
                    - segment: 'Core'
                      person-on-events: true
                      clickhouse-server-image: 'clickhouse/clickhouse-server:25.8.11.66'
                      python-version: '3.12.11'
                      concurrency: 10
                      group: 2
                    - segment: 'Core'
                      person-on-events: true
                      clickhouse-server-image: 'clickhouse/clickhouse-server:25.8.11.66'
                      python-version: '3.12.11'
                      concurrency: 10
                      group: 3
                    - segment: 'Core'
                      person-on-events: true
                      clickhouse-server-image: 'clickhouse/clickhouse-server:25.8.11.66'
                      python-version: '3.12.11'
                      concurrency: 10
                      group: 4
                    - segment: 'Core'
                      person-on-events: true
                      clickhouse-server-image: 'clickhouse/clickhouse-server:25.8.11.66'
                      python-version: '3.12.11'
                      concurrency: 10
                      group: 5
                    - segment: 'Core'
                      person-on-events: true
                      clickhouse-server-image: 'clickhouse/clickhouse-server:25.8.11.66'
                      python-version: '3.12.11'
                      concurrency: 10
                      group: 6
                    - segment: 'Core'
                      person-on-events: true
                      clickhouse-server-image: 'clickhouse/clickhouse-server:25.8.11.66'
                      python-version: '3.12.11'
                      concurrency: 10
                      group: 7
                    - segment: 'Core'
                      person-on-events: true
                      clickhouse-server-image: 'clickhouse/clickhouse-server:25.8.11.66'
                      python-version: '3.12.11'
                      concurrency: 10
                      group: 8
                    - segment: 'Core'
                      person-on-events: true
                      clickhouse-server-image: 'clickhouse/clickhouse-server:25.8.11.66'
                      python-version: '3.12.11'
                      concurrency: 10
                      group: 9
                    - segment: 'Core'
                      person-on-events: true
                      clickhouse-server-image: 'clickhouse/clickhouse-server:25.8.11.66'
                      python-version: '3.12.11'
                      concurrency: 10
                      group: 10
                    - segment: 'Temporal'
                      person-on-events: false
                      clickhouse-server-image: 'clickhouse/clickhouse-server:25.8.11.66'
                      python-version: '3.12.11'
                      concurrency: 10
                      group: 1
                    - segment: 'Temporal'
                      person-on-events: false
                      clickhouse-server-image: 'clickhouse/clickhouse-server:25.8.11.66'
                      python-version: '3.12.11'
                      concurrency: 10
                      group: 2
                    - segment: 'Temporal'
                      person-on-events: false
                      clickhouse-server-image: 'clickhouse/clickhouse-server:25.8.11.66'
                      python-version: '3.12.11'
                      concurrency: 10
                      group: 3
                    - segment: 'Temporal'
                      person-on-events: false
                      clickhouse-server-image: 'clickhouse/clickhouse-server:25.8.11.66'
                      python-version: '3.12.11'
                      concurrency: 10
                      group: 4
                    - segment: 'Temporal'
                      person-on-events: false
                      clickhouse-server-image: 'clickhouse/clickhouse-server:25.8.11.66'
                      python-version: '3.12.11'
                      concurrency: 10
                      group: 5
                    - segment: 'Temporal'
                      person-on-events: false
                      clickhouse-server-image: 'clickhouse/clickhouse-server:25.8.11.66'
                      python-version: '3.12.11'
                      concurrency: 10
                      group: 6
                    - segment: 'Temporal'
                      person-on-events: false
                      clickhouse-server-image: 'clickhouse/clickhouse-server:25.8.11.66'
                      python-version: '3.12.11'
                      concurrency: 10
                      group: 7
                    - segment: 'Temporal'
                      person-on-events: false
                      clickhouse-server-image: 'clickhouse/clickhouse-server:25.8.11.66'
                      python-version: '3.12.11'
                      concurrency: 10
                      group: 8
                    - segment: 'Temporal'
                      person-on-events: false
                      clickhouse-server-image: 'clickhouse/clickhouse-server:25.8.11.66'
                      python-version: '3.12.11'
                      concurrency: 10
                      group: 9
                    - segment: 'Temporal'
                      person-on-events: false
                      clickhouse-server-image: 'clickhouse/clickhouse-server:25.8.11.66'
                      python-version: '3.12.11'
                      concurrency: 10
                      group: 10

        steps:
            - uses: actions/checkout@v4
              with:
                  fetch-depth: 1
                  repository: ${{ github.event.pull_request.head.repo.full_name }}
                  ref: ${{ github.event.pull_request.head.ref }}
                  # Use PostHog Bot token when not on forks to enable proper snapshot updating
                  token: ${{ github.event.pull_request.head.repo.full_name == github.repository && secrets.POSTHOG_BOT_PAT || github.token }}
                  clean: false
            - name: Clean up data directories with container permissions
              run: |
                  # Use docker to clean up files created by containers
                  [ -d "data" ] && docker run --rm -v "$(pwd)/data:/data" alpine sh -c "rm -rf /data/seaweedfs /data/minio" || true
              continue-on-error: true

            - name: 'Safeguard: ensure no stray Python modules at product root'
              run: |
                  echo "Checking that products/* only contain backend/, frontend/, or shared/ as Python code roots..."
                  BAD_FILES=$(find products -maxdepth 2 -type f -name "*.py" ! -path "*/backend/*" ! -name "__init__.py" ! -name "conftest.py" -o -maxdepth 2 -type d -name "migrations" ! -path "*/backend/*")
                  if [ -n "$BAD_FILES" ]; then
                    echo "❌ Found Python code or migrations outside backend/:"
                    echo "$BAD_FILES"
                    echo "Please move these into the appropriate backend/ folder."
                    exit 1
                  fi
                  echo "✅ No stray Python files or migrations found at product roots."

            # Pre-tests

            # Copies the fully versioned UDF xml file for use in CI testing
            - name: Stop/Start stack with Docker Compose
              shell: bash
              run: |
                  export CLICKHOUSE_SERVER_IMAGE=${{ matrix.clickhouse-server-image }}
                  export DOCKER_REGISTRY_PREFIX="us-east1-docker.pkg.dev/posthog-301601/mirror/"
                  cp posthog/user_scripts/latest_user_defined_function.xml docker/clickhouse/user_defined_function.xml

                  # Start docker compose in background
                  (
                      max_attempts=3
                      attempt=1
                      delay=5

                      while [ $attempt -le $max_attempts ]; do
                          echo "Attempt $attempt of $max_attempts to start stack..."

                          if docker compose -f docker-compose.dev.yml down && \
                            docker compose -f docker-compose.dev.yml up -d; then
                              echo "Stack started successfully"
                              exit 0
                          fi

                          echo "Failed to start stack on attempt $attempt"

                          if [ $attempt -lt $max_attempts ]; then
                              sleep_time=$((delay * 2 ** (attempt - 1)))
                              echo "Waiting ${sleep_time} seconds before retry..."
                              sleep $sleep_time
                          fi

                          attempt=$((attempt + 1))
                      done

                      echo "Failed to start stack after $max_attempts attempts"
                      exit 1
                  ) &

            - name: Add Kafka and ClickHouse to /etc/hosts
              shell: bash
              run: echo "127.0.0.1 kafka clickhouse" | sudo tee -a /etc/hosts

            - name: Set up Python
              uses: actions/setup-python@v5
              with:
                  python-version: ${{ matrix.python-version }}
                  token: ${{ secrets.POSTHOG_BOT_PAT }}

            - name: Install uv
              id: setup-uv-tests
              uses: astral-sh/setup-uv@3259c6206f993105e3a61b142c2d97bf4b9ef83d # v7.1.0
              with:
                  enable-cache: true
                  version: 0.8.19

            - name: Install SAML (python3-saml) dependencies
              if: ${{ needs.changes.outputs.backend == 'true' && steps.setup-uv-tests.outputs.cache-hit != 'true' }}
              shell: bash
              run: |
                  sudo apt-get update && sudo apt-get install libxml2-dev libxmlsec1-dev libxmlsec1-openssl

            - name: Install Rust
              if: needs.changes.outputs.backend == 'true'
              uses: dtolnay/rust-toolchain@6691ebadcb18182cc1391d07c9f295f657c593cd # 1.88
              with:
                  toolchain: 1.88.0
                  components: cargo

            - name: Cache Rust dependencies
              if: needs.changes.outputs.backend == 'true'
              uses: Swatinem/rust-cache@f13886b937689c021905a6b90929199931d60db1 # v2.8.1

            - name: Install sqlx-cli
              if: needs.changes.outputs.backend == 'true'
              run: |
                  cargo install sqlx-cli --version 0.8.0 --features postgres --no-default-features --locked

            - name: Determine if hogql-parser has changed compared to master
              shell: bash
              id: hogql-parser-diff
              run: |
                  git fetch --no-tags --prune --depth=1 origin master
                  changed=$(git diff --quiet HEAD origin/master -- common/hogql_parser/ && echo "false" || echo "true")
                  echo "changed=$changed" >> $GITHUB_OUTPUT

            - name: Install pnpm
              uses: pnpm/action-setup@a7487c7e89a18df4991f7f222e4898a00d66ddda # v4

            - name: Set up Node.js
              uses: actions/setup-node@v4
              with:
                  node-version: 22.17.1
                  cache: pnpm

            # tests would intermittently fail in GH actions
            # with exit code 134 _after passing_ all tests
            # this appears to fix it
            # absolute wild tbh https://stackoverflow.com/a/75503402
            - uses: tlambert03/setup-qt-libs@19e4ef2d781d81f5f067182e228b54ec90d23b76 # v1

            - name: Install plugin_transpiler
              shell: bash
              run: |
                  pnpm --filter=@posthog/plugin-transpiler... install --frozen-lockfile
                  bin/turbo --filter=@posthog/plugin-transpiler build

            - name: Install Python dependencies
              shell: bash
              run: |
                  UV_PROJECT_ENVIRONMENT=$pythonLocation uv sync --frozen --dev

            - name: Install the working version of hogql-parser
              if: ${{ needs.changes.outputs.backend == 'true' && steps.hogql-parser-diff.outputs.changed == 'true' }}
              shell: bash
              # This is not cached currently, as it's important to build the current HEAD version of hogql-parser if it has
              # changed (requirements.txt has the already-published version)
              run: |
                  sudo apt-get install libboost-all-dev unzip cmake curl uuid pkg-config
                  curl https://www.antlr.org/download/antlr4-cpp-runtime-4.13.1-source.zip --output antlr4-source.zip
                  # Check that the downloaded archive is the expected runtime - a security measure
                  anltr_known_md5sum="c875c148991aacd043f733827644a76f"
                  antlr_found_ms5sum="$(md5sum antlr4-source.zip | cut -d' ' -f1)"
                  if [[ "$anltr_known_md5sum" != "$antlr_found_ms5sum" ]]; then
                      echo "Unexpected MD5 sum of antlr4-source.zip!"
                      echo "Known: $anltr_known_md5sum"
                      echo "Found: $antlr_found_ms5sum"
                      exit 64
                  fi
                  unzip antlr4-source.zip -d antlr4-source && cd antlr4-source
                  cmake .
                  DESTDIR=out make install
                  sudo cp -r out/usr/local/include/antlr4-runtime /usr/include/
                  sudo cp out/usr/local/lib/libantlr4-runtime.so* /usr/lib/
                  sudo ldconfig
                  cd ..
                  pip install ./common/hogql_parser

            - name: Set up needed files
              shell: bash
              run: |
                  mkdir -p frontend/dist
                  touch frontend/dist/index.html
                  touch frontend/dist/layout.html
                  touch frontend/dist/exporter.html
                  ./bin/download-mmdb

            - name: Wait for services to be available
              shell: bash
              run: |
                  bin/check_kafka_clickhouse_up
                  bin/check_postgres_up

            - name: Wait for Temporal
              if: ${{ needs.changes.outputs.backend == 'true' && matrix.segment == 'Temporal' }}
              shell: bash
              run: |
                  bin/check_temporal_up

            - name: Determine if --snapshot-update should be on
              # Skip on forks (due to GITHUB_TOKEN being read-only in PRs coming from them) except for persons-on-events
              # runs, as we want to ignore snapshots diverging there
              if: ${{ needs.changes.outputs.backend == 'true' && (github.event.pull_request.head.repo.full_name == github.repository || matrix.person-on-events) }}
              shell: bash
              run: echo "PYTEST_ARGS=--snapshot-update" >> $GITHUB_ENV # We can only update snapshots within the PostHog org

            # Tests
            - name: Run Core tests
              id: run-core-tests
              if: ${{ needs.changes.outputs.backend == 'true' && matrix.segment == 'Core' }}
              env:
                  PERSON_ON_EVENTS_V2_ENABLED: ${{ matrix.person-on-events && 'true' || 'false' }}
              shell: bash
              run: | # async_migrations covered in ci-async-migrations.yml
                  set +e
                  pytest ${{
                      matrix.person-on-events
                      && './posthog/clickhouse/ ./posthog/queries/ ./posthog/api/test/test_insight* ./posthog/api/test/dashboards/test_dashboard.py'
                      || 'posthog products'
                  }} ${{ matrix.person-on-events && 'ee/clickhouse/' || 'ee/' }} -m "not async_migrations" \
                      --ignore=posthog/temporal \
                      --ignore=products/batch_exports/backend/tests/temporal \
                      --ignore=common/hogvm/python/test \
                      ${{ matrix.person-on-events && '--ignore=posthog/hogql_queries' || '' }} \
                      ${{ matrix.person-on-events && '--ignore=posthog/hogql' || '' }} \
                      --splits ${{ matrix.concurrency }} --group ${{ matrix.group }} \
                      --durations=1000 --durations-min=1.0 --store-durations \
                      --splitting-algorithm=duration_based_chunks \
                      $PYTEST_ARGS
                  exit_code=$?
                  set -e
                  if [ $exit_code -eq 5 ]; then
                      echo "No tests collected for this shard, this is expected when splitting tests"
                      exit 0
                  else
                      exit $exit_code
                  fi

            # Uncomment this code to create an ssh-able console so you can debug issues with github actions
            # (Consider changing the timeout in ci-backend.yml to have more time)
            # - name: Setup tmate session
            #   if: failure()
            #   uses: mxschmitt/action-tmate@v3

            - name: Run /decide read replica tests
              id: run-decide-read-replica-tests
              if: ${{ needs.changes.outputs.backend == 'true' && matrix.segment == 'Core' && matrix.group == 1 && !matrix.person-on-events }}
              env:
                  POSTHOG_DB_NAME: posthog
                  READ_REPLICA_OPT_IN: 'decide,PersonalAPIKey, local_evaluation'
                  POSTHOG_POSTGRES_READ_HOST: localhost
                  POSTHOG_DB_PASSWORD: posthog
                  POSTHOG_DB_USER: posthog
              shell: bash
              run: |
                  pytest posthog/api/test/test_decide.py::TestDecideUsesReadReplica \
                      --durations=1000 --durations-min=1.0 \
                      $PYTEST_ARGS

            - name: Run Temporal tests
              id: run-temporal-tests
              if: ${{ needs.changes.outputs.backend == 'true' && matrix.segment == 'Temporal' }}
              shell: bash
              env:
                  AWS_S3_ALLOW_UNSAFE_RENAME: 'true'
                  MODAL_TOKEN_ID: ${{ needs.changes.outputs.tasks_temporal == 'true' && secrets.MODAL_TOKEN_ID || '' }}
                  MODAL_TOKEN_SECRET: ${{ needs.changes.outputs.tasks_temporal == 'true' && secrets.MODAL_TOKEN_SECRET || '' }}
              run: |
                  set +e
                  pytest posthog/temporal products/batch_exports/backend/tests/temporal products/tasks/backend/temporal -m "not async_migrations" \
                      --splits ${{ matrix.concurrency }} --group ${{ matrix.group }} \
                      --durations=100 --durations-min=1.0 --store-durations \
                      --splitting-algorithm=duration_based_chunks \
                      $PYTEST_ARGS
                  exit_code=$?
                  set -e
                  if [ $exit_code -eq 5 ]; then
                      echo "No tests collected for this shard, this is expected when splitting tests"
                      exit 0
                  else
                      exit $exit_code
                  fi

            # Post tests
            - name: Show docker compose logs on failure
              if: failure() && (needs.changes.outputs.backend == 'true' && steps.run-core-tests.outcome != 'failure' && steps.run-decide-read-replica-tests.outcome != 'failure' && steps.run-temporal-tests.outcome != 'failure')
              shell: bash
              run: docker compose -f docker-compose.dev.yml logs

            - name: Upload updated timing data as artifacts
              uses: actions/upload-artifact@v4
              if: ${{ needs.changes.outputs.backend == 'true' && !matrix.person-on-events && matrix.clickhouse-server-image == 'clickhouse/clickhouse-server:25.8.11.66' }}
              with:
                  name: timing_data-${{ matrix.segment }}-${{ matrix.group }}
                  path: .test_durations
                  include-hidden-files: true
                  retention-days: 2

            - name: Verify new snapshots for flakiness
              if: ${{ github.event.pull_request.head.repo.full_name == 'PostHog/posthog' && needs.changes.outputs.backend == 'true' && !matrix.person-on-events }}
              shell: bash
              run: |
                  .github/scripts/verify-new-snapshots.sh

            - uses: EndBug/add-and-commit@a94899bca583c204427a224a7af87c02f9b325d5 # v9
              # Also skip for persons-on-events runs, as we want to ignore snapshots diverging there
              if: ${{ github.event.pull_request.head.repo.full_name == 'PostHog/posthog' && needs.changes.outputs.backend == 'true' && !matrix.person-on-events }}
              with:
                  add: '["ee", "./**/*.ambr", "posthog/queries/", "posthog/migrations", "posthog/tasks", "posthog/hogql/"]'
                  message: 'Update query snapshots'
                  pull: --rebase --autostash # Make sure we're up-to-date with other segments' updates
                  default_author: github_actions
                  github_token: ${{ secrets.POSTHOG_BOT_PAT }}

            - name: Check if any snapshot changes were left uncomitted
              id: changed-files
              if: ${{ github.event.pull_request.head.repo.full_name == 'PostHog/posthog' && needs.changes.outputs.backend == 'true' && !matrix.person-on-events }}
              run: |
                  if [[ -z $(git status -s | grep -v ".test_durations" | tr -d "\n") ]]
                  then
                    echo 'files_found=false' >> $GITHUB_OUTPUT
                  else
                    echo 'diff=$(git status --porcelain)' >> $GITHUB_OUTPUT
                    echo 'files_found=true' >> $GITHUB_OUTPUT
                  fi

            - name: Fail CI if some snapshots have been updated but not committed
              if: steps.changed-files.outputs.files_found == 'true' && steps.add-and-commit.outcome == 'success'
              run: |
                  echo "${{ steps.changed-files.outputs.diff }}"
                  exit 1

            - name: Archive email renders
              uses: actions/upload-artifact@v4
              if: needs.changes.outputs.backend == 'true' && matrix.segment == 'Core' && !matrix.person-on-events
              with:
                  name: email_renders-${{ github.sha }}-${{ github.run_attempt }}-${{ matrix.segment }}-${{ matrix.person-on-events }}-${{ matrix.group }}
                  path: posthog/tasks/test/__emails__
                  retention-days: 1

    # Job just to collate the status of the matrix jobs for requiring passing status
    django_tests:
        needs: [django, check-migrations, async-migrations]
        name: Django Tests Pass
        runs-on: ubuntu-latest
        if: always()
        steps:
            - name: Check matrix outcome
              run: |
                  # The `needs.django.result` will be 'success' only if all jobs in the matrix succeeded.
                  # Otherwise, it will be 'failure'.
                  if [[ "${{ needs.django.result }}" != "success" && "${{ needs.django.result }}" != "skipped" ]]; then
                    echo "One or more jobs in the Django test matrix failed."
                    exit 1
                  fi

                  # Check migration validation - must pass for Django Tests to be green
                  if [[ "${{ needs.check-migrations.result }}" != "success" && "${{ needs.check-migrations.result }}" != "skipped" ]]; then
                    echo "Migration checks failed."
                    exit 1
                  fi

                  # Check async migrations - must pass for Django Tests to be green
                  if [[ "${{ needs.async-migrations.result }}" != "success" && "${{ needs.async-migrations.result }}" != "skipped" ]]; then
                    echo "Async migrations tests failed."
                    exit 1
                  fi

                  echo "All checks passed."

    async-migrations:
        name: Async migrations tests -  ${{ matrix.clickhouse-server-image }}
        needs: [changes]
        strategy:
            fail-fast: false
            matrix:
                clickhouse-server-image: ['clickhouse/clickhouse-server:25.8.11.66']
        if: needs.changes.outputs.backend == 'true'
        runs-on: ubuntu-latest
        steps:
            - name: 'Checkout repo'
              uses: actions/checkout@v4
              with:
                  fetch-depth: 1
                  clean: false
            - name: Clean up data directories with container permissions
              run: |
                  # Use docker to clean up files created by containers
                  [ -d "data" ] && docker run --rm -v "$(pwd)/data:/data" alpine sh -c "rm -rf /data/seaweedfs /data/minio" || true
              continue-on-error: true

            - name: Start stack with Docker Compose
              run: |
                  export CLICKHOUSE_SERVER_IMAGE_VERSION=${{ matrix.clickhouse-server-image }}
                  docker compose -f docker-compose.dev.yml down
                  docker compose -f docker-compose.dev.yml up -d &

            - name: Set up Python
              uses: actions/setup-python@v5
              with:
                  python-version-file: 'pyproject.toml'

            - name: Install uv
              id: setup-uv-async
              uses: astral-sh/setup-uv@3259c6206f993105e3a61b142c2d97bf4b9ef83d # v7.1.0
              with:
                  enable-cache: true
                  version: 0.8.19

            - name: Install SAML (python3-saml) dependencies
              if: steps.setup-uv-async.outputs.cache-hit != 'true'
              run: |
                  sudo apt-get update
                  sudo apt-get install libxml2-dev libxmlsec1-dev libxmlsec1-openssl

            - name: Install Rust
              uses: dtolnay/rust-toolchain@6691ebadcb18182cc1391d07c9f295f657c593cd # 1.88
              with:
                  toolchain: 1.88.0
                  components: cargo

            - name: Cache Rust dependencies
              uses: Swatinem/rust-cache@f13886b937689c021905a6b90929199931d60db1 # v2.8.1

            - name: Install sqlx-cli
              run: |
                  cargo install sqlx-cli --version 0.8.0 --features postgres --no-default-features --locked

            - name: Install python dependencies
              shell: bash
              run: |
                  UV_PROJECT_ENVIRONMENT=$pythonLocation uv sync --frozen --dev

            - name: Add Kafka and ClickHouse to /etc/hosts
              run: sudo echo "127.0.0.1 kafka clickhouse" | sudo tee -a /etc/hosts

            - name: Set up needed files
              run: |
                  mkdir -p frontend/dist
                  touch frontend/dist/index.html
                  touch frontend/dist/layout.html
                  touch frontend/dist/exporter.html

            - name: Wait for services to be available
              shell: bash
              run: |
                  bin/check_kafka_clickhouse_up
                  bin/check_postgres_up

            - name: Run async migrations tests
              run: |
                  pytest -m "async_migrations"

    calculate-running-time:
        name: Calculate running time
        needs: [django, async-migrations]
        runs-on: ubuntu-latest
        if: # Run on pull requests to PostHog/posthog + on PostHog/posthog outside of PRs - but never on forks
            needs.changes.outputs.backend == 'true' && (
            (github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name == 'PostHog/posthog') ||
            (github.event_name != 'pull_request' && github.repository == 'PostHog/posthog'))
        steps:
            - name: Calculate running time
              run: |
                  gh auth login --with-token < <(echo ${{ secrets.GITHUB_TOKEN }})
                  run_id=${GITHUB_RUN_ID}
                  repo=${GITHUB_REPOSITORY}
                  run_info=$(gh api repos/${repo}/actions/runs/${run_id})
                  echo run_info: ${run_info}
                  # name is the name of the workflow file
                  # run_started_at is the start time of the workflow
                  # we want to get the number of seconds between the start time and now
                  name=$(echo ${run_info} | jq -r '.name')
                  run_url=$(echo ${run_info} | jq -r '.url')
                  run_started_at=$(echo ${run_info} | jq -r '.run_started_at')
                  run_attempt=$(echo ${run_info} | jq -r '.run_attempt')
                  start_seconds=$(date -d "${run_started_at}" +%s)
                  now_seconds=$(date +%s)
                  duration=$((now_seconds-start_seconds))
                  echo running_time_duration_seconds=${duration} >> $GITHUB_ENV
                  echo running_time_run_url=${run_url} >> $GITHUB_ENV
                  echo running_time_run_attempt=${run_attempt} >> $GITHUB_ENV
                  echo running_time_run_id=${run_id} >> $GITHUB_ENV
                  echo running_time_run_started_at=${run_started_at} >> $GITHUB_ENV
            - name: Capture running time to PostHog
              uses: PostHog/posthog-github-action@v0.1
              with:
                  posthog-token: ${{secrets.POSTHOG_API_TOKEN}}
                  event: 'posthog-ci-running-time'
                  properties: '{"runner": "depot", "duration_seconds": ${{ env.running_time_duration_seconds }}, "run_url": "${{ env.running_time_run_url }}", "run_attempt": "${{ env.running_time_run_attempt }}", "run_id": "${{ env.running_time_run_id }}", "run_started_at": "${{ env.running_time_run_started_at }}"}'