posthog/.github/workflows/ci-dagster.yml

# This workflow runs all of our dagster tests.
name: Dagster CI
on:
    push:
        branches:
            - master
    pull_request:

env:
    SECRET_KEY: '6b01eee4f945ca25045b5aab440b953461faf08693a9abbf1166dc7c6b9772da' # unsafe - for testing only
    DATABASE_URL: 'postgres://posthog:posthog@localhost:5432/posthog'
    REDIS_URL: 'redis://localhost'
    CLICKHOUSE_HOST: 'localhost'
    CLICKHOUSE_SECURE: 'False'
    CLICKHOUSE_VERIFY: 'False'
    TEST: 1
    OBJECT_STORAGE_ENABLED: 'True'
    OBJECT_STORAGE_ENDPOINT: 'http://localhost:19000'
    OBJECT_STORAGE_ACCESS_KEY_ID: 'object_storage_root_user'
    OBJECT_STORAGE_SECRET_ACCESS_KEY: 'object_storage_root_password'
    # tests would intermittently fail in GH actions
    # with exit code 134 _after passing_ all tests
    # this appears to fix it
    # absolute wild tbh https://stackoverflow.com/a/75503402
    DISPLAY: ':99.0'
    OIDC_RSA_PRIVATE_KEY: 'test'
jobs:
    # Job to decide if we should run dagster ci
    # See https://github.com/dorny/paths-filter#conditional-execution for more details
    changes:
        runs-on: ubuntu-latest
        timeout-minutes: 5
        name: Determine need to run dagster checks
        # Set job outputs to values from filter step
        outputs:
            dagster: ${{ steps.filter.outputs.dagster }}
        steps:
            # For pull requests it's not necessary to checkout the code, but we
            # also want this to run on master so we need to checkout
            - uses: actions/checkout@v4
              with:
                  clean: false

            - uses: dorny/paths-filter@4512585405083f25c027a35db413c2b3b9006d50 # v2
              id: filter
              with:
                  filters: |
                      dagster:
                        - 'dags/**'
                        # Dagster depends on code from the posthog package, so ensure we run on changes to it
                        - 'posthog/**/*'
                        # Make sure we run if someone is explicitly change the workflow
                        - .github/workflows/ci-dagster.yml
                        # We use docker compose for tests, make sure we rerun on
                        # changes to docker-compose.dev.yml e.g. dependency
                        # version changes
                        - docker-compose.dev.yml
                        - frontend/public/email/*
                        # These scripts are used in the CI
                        - bin/check_kafka_clickhouse_up

    dagster:
        name: Dagster tests
        needs: changes
        strategy:
            fail-fast: false
            matrix:
                clickhouse-server-image: ['clickhouse/clickhouse-server:25.6.9.98']
        if: needs.changes.outputs.dagster == 'true'
        runs-on: depot-ubuntu-latest
        steps:
            - name: 'Checkout repo'
              uses: actions/checkout@v4
              with:
                  fetch-depth: 1
                  clean: false
            - name: Clean up data directories with container permissions
              run: |
                  # Use docker to clean up files created by containers
                  [ -d "data" ] && docker run --rm -v "$(pwd)/data:/data" alpine sh -c "rm -rf /data/seaweedfs /data/minio" || true
              continue-on-error: true

            - name: Start stack with Docker Compose
              run: |
                  export CLICKHOUSE_SERVER_IMAGE_VERSION=${{ matrix.clickhouse-server-image }}
                  docker compose -f docker-compose.dev.yml down
                  docker compose -f docker-compose.dev.yml up -d

            - name: Set up Python
              uses: actions/setup-python@v5
              with:
                  python-version-file: 'pyproject.toml'

            - name: Install uv
              id: setup-uv
              uses: astral-sh/setup-uv@3259c6206f993105e3a61b142c2d97bf4b9ef83d # v7.1.0
              with:
                  enable-cache: true
                  version: 0.8.19

            - name: Install SAML (python3-saml) dependencies
              if: steps.setup-uv.outputs.cache-hit != 'true'
              run: |
                  sudo apt-get update
                  sudo apt-get install libxml2-dev libxmlsec1-dev libxmlsec1-openssl

            - name: Install python dependencies
              shell: bash
              run: |
                  UV_PROJECT_ENVIRONMENT=$pythonLocation uv sync --frozen --dev

            - name: Install Rust
              uses: dtolnay/rust-toolchain@6691ebadcb18182cc1391d07c9f295f657c593cd # 1.88
              with:
                  toolchain: stable
                  components: cargo

            - name: Cache Rust dependencies
              uses: Swatinem/rust-cache@f13886b937689c021905a6b90929199931d60db1 # v2.8.1

            - name: Install sqlx-cli
              run: |
                  cargo install sqlx-cli --version 0.8.0 --features postgres --no-default-features --locked

            - name: Add Kafka and ClickHouse to /etc/hosts
              run: sudo echo "127.0.0.1 kafka clickhouse" | sudo tee -a /etc/hosts

            - name: Wait for Clickhouse & Kafka
              run: bin/check_kafka_clickhouse_up

            - name: Run migrations
              run: |
                  # Run Django migrations first
                  python manage.py migrate
                  # Then run persons migrations using sqlx
                  DATABASE_URL="postgres://posthog:posthog@localhost:5432/posthog_persons" \
                    sqlx database create
                  DATABASE_URL="postgres://posthog:posthog@localhost:5432/posthog_persons" \
                    sqlx migrate run --source rust/persons_migrations/

            - name: Run clickhouse migrations
              run: |
                  python manage.py migrate_clickhouse

            - name: Run Dagster tests
              run: |
                  pytest dags