mirror of
https://github.com/BillyOutlast/posthog.git
synced 2026-02-04 03:01:23 +01:00
a227a16675
Co-authored-by: github-actions <41898282+github-actions[bot]@users.noreply.github.com> Co-authored-by: github-actions[bot] <github-actions[bot]@users.noreply.github.com>
1010 lines
48 KiB
YAML
1010 lines
48 KiB
YAML
# This workflow runs all of our backend django tests.
|
||
#
|
||
# If these tests get too slow, look at increasing concurrency and re-timing the tests by manually dispatching
|
||
# .github/workflows/ci-backend-update-test-timing.yml action
|
||
name: Backend CI
|
||
on:
|
||
push:
|
||
branches:
|
||
- master
|
||
workflow_dispatch:
|
||
inputs:
|
||
clickhouseServerVersion:
|
||
description: ClickHouse server version. Leave blank for default
|
||
type: string
|
||
pull_request:
|
||
|
||
concurrency:
|
||
group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }}
|
||
cancel-in-progress: true
|
||
env:
|
||
SECRET_KEY: '6b01eee4f945ca25045b5aab440b953461faf08693a9abbf1166dc7c6b9772da' # unsafe - for testing only
|
||
DATABASE_URL: 'postgres://posthog:posthog@localhost:5432/posthog'
|
||
REDIS_URL: 'redis://localhost'
|
||
CLICKHOUSE_HOST: 'localhost'
|
||
CLICKHOUSE_SECURE: 'False'
|
||
CLICKHOUSE_VERIFY: 'False'
|
||
TEST: 1
|
||
CLICKHOUSE_SERVER_IMAGE_VERSION: ${{ github.event.inputs.clickhouseServerVersion || '' }}
|
||
OBJECT_STORAGE_ENABLED: 'True'
|
||
OBJECT_STORAGE_ENDPOINT: 'http://localhost:19000'
|
||
OBJECT_STORAGE_ACCESS_KEY_ID: 'object_storage_root_user'
|
||
OBJECT_STORAGE_SECRET_ACCESS_KEY: 'object_storage_root_password'
|
||
# tests would intermittently fail in GH actions
|
||
# with exit code 134 _after passing_ all tests
|
||
# this appears to fix it
|
||
# absolute wild tbh https://stackoverflow.com/a/75503402
|
||
DISPLAY: ':99.0'
|
||
OIDC_RSA_PRIVATE_KEY: 'test'
|
||
jobs:
|
||
# Job to decide if we should run backend ci
|
||
# See https://github.com/dorny/paths-filter#conditional-execution for more details
|
||
changes:
|
||
runs-on: ubuntu-latest
|
||
timeout-minutes: 5
|
||
name: Determine need to run backend and migration checks
|
||
# Set job outputs to values from filter step
|
||
outputs:
|
||
backend: ${{ steps.filter.outputs.backend }}
|
||
backend_files: ${{ steps.filter.outputs.backend_files }}
|
||
migrations: ${{ steps.filter.outputs.migrations }}
|
||
migrations_files: ${{ steps.filter.outputs.migrations_files }}
|
||
tasks_temporal: ${{ steps.filter.outputs.tasks_temporal }}
|
||
steps:
|
||
# For pull requests it's not necessary to checkout the code, but we
|
||
# also want this to run on master so we need to checkout
|
||
- uses: actions/checkout@v4
|
||
with:
|
||
clean: false
|
||
- name: Clean up data directories with container permissions
|
||
run: |
|
||
# Use docker to clean up files created by containers
|
||
[ -d "data" ] && docker run --rm -v "$(pwd)/data:/data" alpine sh -c "rm -rf /data/seaweedfs /data/minio" || true
|
||
continue-on-error: true
|
||
|
||
- uses: dorny/paths-filter@4512585405083f25c027a35db413c2b3b9006d50 # v2
|
||
id: filter
|
||
with:
|
||
list-files: 'escape'
|
||
filters: |
|
||
backend:
|
||
# Avoid running backend tests for irrelevant changes
|
||
# NOTE: we are at risk of missing a dependency here. We could make
|
||
# the dependencies more clear if we separated the backend/frontend
|
||
# code completely
|
||
# really we should ignore ee/frontend/** but dorny doesn't support that
|
||
# - '!ee/frontend/**'
|
||
# including the negated rule appears to work
|
||
# but makes it always match because the checked file always isn't `ee/frontend/**` 🙈
|
||
- 'ee/**/*'
|
||
- 'common/hogvm/**/*'
|
||
- 'posthog/**/*'
|
||
- 'products/**/backend/**/*'
|
||
- 'bin/*.py'
|
||
- pyproject.toml
|
||
- uv.lock
|
||
- requirements.txt
|
||
- requirements-dev.txt
|
||
- mypy.ini
|
||
- pytest.ini
|
||
- frontend/src/queries/schema.json # Used for generating schema.py
|
||
- common/plugin_transpiler/src # Used for transpiling plugins
|
||
# Make sure we run if someone is explicitly changing the workflow
|
||
- .github/workflows/ci-backend.yml
|
||
# We use docker compose for tests, make sure we rerun on
|
||
# changes to docker-compose.dev.yml e.g. dependency
|
||
# version changes
|
||
- docker-compose.dev.yml
|
||
- docker-compose.base.yml
|
||
- frontend/public/email/*
|
||
- docker/clickhouse
|
||
# These scripts are used in the CI
|
||
- bin/check_temporal_up
|
||
- bin/check_kafka_clickhouse_up
|
||
migrations:
|
||
- docker/clickhouse
|
||
- 'posthog/migrations/*.py'
|
||
- 'products/*/backend/migrations/*.py'
|
||
- 'products/*/migrations/*.py' # Legacy structure
|
||
- 'rust/persons_migrations/*.sql'
|
||
- 'rust/bin/migrate-persons'
|
||
tasks_temporal:
|
||
- 'products/tasks/backend/temporal/**/*'
|
||
|
||
check-migrations:
|
||
needs: [changes]
|
||
if: needs.changes.outputs.backend == 'true'
|
||
timeout-minutes: 20
|
||
|
||
name: Validate Django and CH migrations
|
||
runs-on: ubuntu-latest
|
||
|
||
steps:
|
||
- uses: actions/checkout@v4
|
||
with:
|
||
clean: false
|
||
- name: Clean up data directories with container permissions
|
||
run: |
|
||
# Use docker to clean up files created by containers
|
||
[ -d "data" ] && docker run --rm -v "$(pwd)/data:/data" alpine sh -c "rm -rf /data/seaweedfs /data/minio" || true
|
||
continue-on-error: true
|
||
|
||
- name: Stop/Start stack with Docker Compose
|
||
run: |
|
||
docker compose -f docker-compose.dev.yml down
|
||
docker compose -f docker-compose.dev.yml up -d &
|
||
|
||
- name: Set up Python
|
||
uses: actions/setup-python@v5
|
||
with:
|
||
python-version: 3.12.11
|
||
token: ${{ secrets.POSTHOG_BOT_PAT }}
|
||
|
||
- name: Install uv
|
||
id: setup-uv
|
||
uses: astral-sh/setup-uv@3259c6206f993105e3a61b142c2d97bf4b9ef83d # v7.1.0
|
||
with:
|
||
enable-cache: true
|
||
version: 0.8.19
|
||
|
||
- name: Install SAML (python3-saml) dependencies
|
||
if: steps.setup-uv.outputs.cache-hit != 'true'
|
||
run: |
|
||
sudo apt-get update
|
||
sudo apt-get install libxml2-dev libxmlsec1-dev libxmlsec1-openssl
|
||
- name: Install Rust
|
||
uses: dtolnay/rust-toolchain@6691ebadcb18182cc1391d07c9f295f657c593cd # 1.88
|
||
with:
|
||
toolchain: 1.88.0
|
||
components: cargo
|
||
|
||
- name: Cache Rust dependencies
|
||
uses: Swatinem/rust-cache@f13886b937689c021905a6b90929199931d60db1 # v2.8.1
|
||
|
||
- name: Install sqlx-cli
|
||
run: |
|
||
cargo install sqlx-cli --version 0.8.0 --features postgres --no-default-features --locked
|
||
|
||
# First running migrations from master, to simulate the real-world scenario
|
||
- name: Checkout master
|
||
uses: actions/checkout@v4
|
||
with:
|
||
ref: master
|
||
clean: false
|
||
|
||
- name: Install python dependencies for master
|
||
run: |
|
||
UV_PROJECT_ENVIRONMENT=.venv-master uv sync --frozen --dev
|
||
|
||
- name: Wait for services to be available
|
||
run: |
|
||
bin/check_postgres_up
|
||
bin/check_kafka_clickhouse_up
|
||
|
||
- name: Run migrations up to master
|
||
run: |
|
||
# Run Django migrations first (excluding managed=False models)
|
||
.venv-master/bin/python manage.py migrate
|
||
# Then run persons migrations using sqlx; comment out until we've merged
|
||
# DATABASE_URL="postgres://posthog:posthog@localhost:5432/posthog_persons" \
|
||
# sqlx database create
|
||
# DATABASE_URL="postgres://posthog:posthog@localhost:5432/posthog_persons" \
|
||
# sqlx migrate run --source rust/persons_migrations/
|
||
|
||
# Now we can consider this PR's migrations
|
||
- name: Checkout this PR
|
||
uses: actions/checkout@v4
|
||
with:
|
||
clean: false
|
||
|
||
- name: Install python dependencies for this PR
|
||
run: |
|
||
UV_PROJECT_ENVIRONMENT=$pythonLocation uv sync --frozen --dev
|
||
|
||
- name: Check migrations and post SQL comment
|
||
if: github.event_name == 'pull_request' && needs.changes.outputs.migrations == 'true'
|
||
env:
|
||
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||
run: |
|
||
# Read the changed files from the output
|
||
CHANGED_FILES="${{ needs.changes.outputs.migrations_files }}"
|
||
|
||
# If no migration files changed, exit
|
||
if [ -z "$CHANGED_FILES" ]; then
|
||
echo "No migration files changed"
|
||
exit 0
|
||
fi
|
||
|
||
# Initialize comment body for SQL changes
|
||
COMMENT_BODY="## Migration SQL Changes\n\nHey 👋, we've detected some migrations on this PR. Here's the SQL output for each migration, make sure they make sense:\n\n"
|
||
|
||
# Process each changed migration file (excluding Rust migrations)
|
||
for file in $CHANGED_FILES; do
|
||
# Skip Rust migrations as they're handled separately by sqlx
|
||
if [[ $file =~ rust/persons_migrations ]]; then
|
||
continue
|
||
fi
|
||
if [[ $file =~ migrations/([0-9]+)_ ]]; then
|
||
migration_number="${BASH_REMATCH[1]}"
|
||
# Get app name by looking at the directory structure
|
||
# For new structure products/user_interviews/backend/migrations, we want user_interviews
|
||
# For old structure products/user_interviews/migrations, we want user_interviews
|
||
if [[ $file =~ products/([^/]+)/backend/migrations/ ]]; then
|
||
app_name="${BASH_REMATCH[1]}"
|
||
else
|
||
app_name=$(echo $file | sed -E 's|^([^/]+/)*([^/]+)/migrations/.*|\2|')
|
||
fi
|
||
echo "Checking migration $migration_number for app $app_name"
|
||
|
||
# Get SQL output
|
||
SQL_OUTPUT=$(python manage.py sqlmigrate $app_name $migration_number)
|
||
|
||
# Add to comment body
|
||
COMMENT_BODY+="#### [\`$file\`](https:\/\/github.com\/${{ github.repository }}\/blob\/${{ github.sha }}\/$file)\n\`\`\`sql\n$SQL_OUTPUT\n\`\`\`\n\n"
|
||
fi
|
||
done
|
||
|
||
# Get existing comments
|
||
COMMENTS=$(curl -s -H "Authorization: token $GITHUB_TOKEN" \
|
||
-H "Accept: application/vnd.github.v3+json" \
|
||
"https://api.github.com/repos/${{ github.repository }}/issues/${{ github.event.pull_request.number }}/comments")
|
||
|
||
# Extract comment ID if exists
|
||
SQL_COMMENT_ID=$(echo "$COMMENTS" | jq -r '.[] | select(.body | startswith("## Migration SQL Changes")) | .id' | head -1)
|
||
|
||
# Add timestamp and commit SHA to SQL changes
|
||
TIMESTAMP=$(date -u '+%Y-%m-%d %H:%M UTC')
|
||
COMMIT_SHA="${{ github.event.pull_request.head.sha }}"
|
||
COMMIT_SHORT="${COMMIT_SHA:0:7}"
|
||
COMMENT_BODY+="\n*Last updated: $TIMESTAMP ([${COMMIT_SHORT}](https://github.com/${{ github.repository }}/commit/${COMMIT_SHA}))*"
|
||
|
||
# Convert \n into actual newlines
|
||
COMMENT_BODY=$(printf '%b' "$COMMENT_BODY")
|
||
COMMENT_BODY_JSON=$(jq -n --arg body "$COMMENT_BODY" '{body: $body}')
|
||
|
||
if [ -n "$SQL_COMMENT_ID" ]; then
|
||
# Update existing comment
|
||
echo "Updating existing SQL comment $SQL_COMMENT_ID"
|
||
curl -X PATCH \
|
||
-H "Authorization: token $GITHUB_TOKEN" \
|
||
-H "Accept: application/vnd.github.v3+json" \
|
||
"https://api.github.com/repos/${{ github.repository }}/issues/comments/$SQL_COMMENT_ID" \
|
||
-d "$COMMENT_BODY_JSON"
|
||
else
|
||
# Post new SQL comment to PR
|
||
echo "Posting new SQL comment to PR"
|
||
curl -X POST \
|
||
-H "Authorization: token $GITHUB_TOKEN" \
|
||
-H "Accept: application/vnd.github.v3+json" \
|
||
"https://api.github.com/repos/${{ github.repository }}/issues/${{ github.event.pull_request.number }}/comments" \
|
||
-d "$COMMENT_BODY_JSON"
|
||
fi
|
||
|
||
- name: Run migration risk analysis and post comment
|
||
if: github.event_name == 'pull_request'
|
||
env:
|
||
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||
run: |
|
||
# Get risk analysis for all unapplied migrations (including third-party)
|
||
set +e # Don't exit immediately on error
|
||
RISK_ANALYSIS=$(python manage.py analyze_migration_risk --fail-on-blocked 2>/dev/null)
|
||
EXIT_CODE=$?
|
||
set -e # Re-enable exit on error
|
||
|
||
# Save analysis to file for artifact upload
|
||
if [ -n "$RISK_ANALYSIS" ]; then
|
||
echo "$RISK_ANALYSIS" > migration_analysis.md
|
||
fi
|
||
|
||
# Get existing comments
|
||
COMMENTS=$(curl -s -H "Authorization: token $GITHUB_TOKEN" \
|
||
-H "Accept: application/vnd.github.v3+json" \
|
||
"https://api.github.com/repos/${{ github.repository }}/issues/${{ github.event.pull_request.number }}/comments")
|
||
|
||
# Extract comment ID if exists
|
||
COMMENT_ID=$(echo "$COMMENTS" | jq -r '.[] | select(.body | startswith("## 🔍 Migration Risk Analysis")) | .id' | head -1)
|
||
|
||
if [ -n "$RISK_ANALYSIS" ] && echo "$RISK_ANALYSIS" | grep -q "Summary:"; then
|
||
# Add timestamp and commit SHA to analysis
|
||
TIMESTAMP=$(date -u '+%Y-%m-%d %H:%M UTC')
|
||
COMMIT_SHA="${{ github.event.pull_request.head.sha }}"
|
||
COMMIT_SHORT="${COMMIT_SHA:0:7}"
|
||
RISK_COMMENT="## 🔍 Migration Risk Analysis\n\nWe've analyzed your migrations for potential risks.\n\n$RISK_ANALYSIS\n\n*Last updated: $TIMESTAMP ([${COMMIT_SHORT}](https://github.com/${{ github.repository }}/commit/${COMMIT_SHA}))*"
|
||
RISK_COMMENT=$(printf '%b' "$RISK_COMMENT")
|
||
RISK_COMMENT_JSON=$(jq -n --arg body "$RISK_COMMENT" '{body: $body}')
|
||
|
||
if [ -n "$COMMENT_ID" ]; then
|
||
# Update existing comment
|
||
echo "Updating existing risk analysis comment $COMMENT_ID"
|
||
curl -X PATCH \
|
||
-H "Authorization: token $GITHUB_TOKEN" \
|
||
-H "Accept: application/vnd.github.v3+json" \
|
||
"https://api.github.com/repos/${{ github.repository }}/issues/comments/$COMMENT_ID" \
|
||
-d "$RISK_COMMENT_JSON"
|
||
else
|
||
# Create new comment if none exists
|
||
echo "Posting new risk analysis comment to PR"
|
||
curl -X POST \
|
||
-H "Authorization: token $GITHUB_TOKEN" \
|
||
-H "Accept: application/vnd.github.v3+json" \
|
||
"https://api.github.com/repos/${{ github.repository }}/issues/${{ github.event.pull_request.number }}/comments" \
|
||
-d "$RISK_COMMENT_JSON"
|
||
fi
|
||
elif [ -n "$COMMENT_ID" ]; then
|
||
# No migrations to analyze but comment exists - delete it
|
||
echo "Deleting risk analysis comment (no migrations to analyze)"
|
||
curl -X DELETE \
|
||
-H "Authorization: token $GITHUB_TOKEN" \
|
||
-H "Accept: application/vnd.github.v3+json" \
|
||
"https://api.github.com/repos/${{ github.repository }}/issues/comments/$COMMENT_ID"
|
||
else
|
||
echo "No migrations to analyze and no existing comment"
|
||
fi
|
||
|
||
# Fail the job if there were blocked migrations
|
||
if [ $EXIT_CODE -ne 0 ]; then
|
||
exit $EXIT_CODE
|
||
fi
|
||
|
||
- name: Upload migration analysis artifact
|
||
if: always() && github.event_name == 'pull_request'
|
||
uses: actions/upload-artifact@v4
|
||
with:
|
||
name: migration-analysis
|
||
path: migration_analysis.md
|
||
if-no-files-found: ignore
|
||
|
||
- name: Run migrations for this PR
|
||
run: |
|
||
# Run Django migrations first (excluding managed=False models)
|
||
python manage.py migrate
|
||
# Then run persons migrations using sqlx
|
||
DATABASE_URL="postgres://posthog:posthog@localhost:5432/posthog_persons" \
|
||
sqlx migrate run --source rust/persons_migrations/
|
||
|
||
- name: Check migrations
|
||
run: |
|
||
DATABASE_URL="postgres://posthog:posthog@localhost:5432/posthog_persons" \
|
||
sqlx migrate info --source rust/persons_migrations/
|
||
python manage.py makemigrations --check --dry-run
|
||
git fetch origin master
|
||
# Check migration safety using old SQL-based checker (still uses stdin from git diff)
|
||
echo "${{ needs.changes.outputs.migrations_files }}" | grep -v migrations/0001_ | grep -v 'rust/persons_migrations' | python manage.py test_migrations_are_safe
|
||
|
||
- name: Check CH migrations
|
||
run: |
|
||
# Same as above, except now for CH looking at files that were added in posthog/clickhouse/migrations/
|
||
git diff --name-status origin/master..HEAD | grep "A\sposthog/clickhouse/migrations/" | grep -v README | awk '{print $2}' | python manage.py test_ch_migrations_are_safe
|
||
|
||
django:
|
||
needs: [changes]
|
||
if: needs.changes.outputs.backend == 'true'
|
||
# increase for tmate testing
|
||
timeout-minutes: 30
|
||
|
||
name: Django tests – ${{ matrix.segment }} (persons-on-events ${{ matrix.person-on-events && 'on' || 'off' }}), Py ${{ matrix.python-version }}, ${{ matrix.clickhouse-server-image }} (${{matrix.group}}/${{ matrix.concurrency }})
|
||
runs-on: depot-ubuntu-latest
|
||
|
||
strategy:
|
||
fail-fast: false
|
||
matrix:
|
||
python-version: ['3.12.11']
|
||
clickhouse-server-image: ['clickhouse/clickhouse-server:25.6.9.98']
|
||
segment: ['Core']
|
||
person-on-events: [false]
|
||
# :NOTE: Keep concurrency and groups in sync
|
||
concurrency: [40]
|
||
group:
|
||
[
|
||
1,
|
||
2,
|
||
3,
|
||
4,
|
||
5,
|
||
6,
|
||
7,
|
||
8,
|
||
9,
|
||
10,
|
||
11,
|
||
12,
|
||
13,
|
||
14,
|
||
15,
|
||
16,
|
||
17,
|
||
18,
|
||
19,
|
||
20,
|
||
21,
|
||
22,
|
||
23,
|
||
24,
|
||
25,
|
||
26,
|
||
27,
|
||
28,
|
||
29,
|
||
30,
|
||
31,
|
||
32,
|
||
33,
|
||
34,
|
||
35,
|
||
36,
|
||
37,
|
||
38,
|
||
39,
|
||
40,
|
||
]
|
||
include:
|
||
- segment: 'Core'
|
||
person-on-events: true
|
||
clickhouse-server-image: 'clickhouse/clickhouse-server:25.6.9.98'
|
||
python-version: '3.12.11'
|
||
concurrency: 10
|
||
group: 1
|
||
- segment: 'Core'
|
||
person-on-events: true
|
||
clickhouse-server-image: 'clickhouse/clickhouse-server:25.6.9.98'
|
||
python-version: '3.12.11'
|
||
concurrency: 10
|
||
group: 2
|
||
- segment: 'Core'
|
||
person-on-events: true
|
||
clickhouse-server-image: 'clickhouse/clickhouse-server:25.6.9.98'
|
||
python-version: '3.12.11'
|
||
concurrency: 10
|
||
group: 3
|
||
- segment: 'Core'
|
||
person-on-events: true
|
||
clickhouse-server-image: 'clickhouse/clickhouse-server:25.6.9.98'
|
||
python-version: '3.12.11'
|
||
concurrency: 10
|
||
group: 4
|
||
- segment: 'Core'
|
||
person-on-events: true
|
||
clickhouse-server-image: 'clickhouse/clickhouse-server:25.6.9.98'
|
||
python-version: '3.12.11'
|
||
concurrency: 10
|
||
group: 5
|
||
- segment: 'Core'
|
||
person-on-events: true
|
||
clickhouse-server-image: 'clickhouse/clickhouse-server:25.6.9.98'
|
||
python-version: '3.12.11'
|
||
concurrency: 10
|
||
group: 6
|
||
- segment: 'Core'
|
||
person-on-events: true
|
||
clickhouse-server-image: 'clickhouse/clickhouse-server:25.6.9.98'
|
||
python-version: '3.12.11'
|
||
concurrency: 10
|
||
group: 7
|
||
- segment: 'Core'
|
||
person-on-events: true
|
||
clickhouse-server-image: 'clickhouse/clickhouse-server:25.6.9.98'
|
||
python-version: '3.12.11'
|
||
concurrency: 10
|
||
group: 8
|
||
- segment: 'Core'
|
||
person-on-events: true
|
||
clickhouse-server-image: 'clickhouse/clickhouse-server:25.6.9.98'
|
||
python-version: '3.12.11'
|
||
concurrency: 10
|
||
group: 9
|
||
- segment: 'Core'
|
||
person-on-events: true
|
||
clickhouse-server-image: 'clickhouse/clickhouse-server:25.6.9.98'
|
||
python-version: '3.12.11'
|
||
concurrency: 10
|
||
group: 10
|
||
- segment: 'Temporal'
|
||
person-on-events: false
|
||
clickhouse-server-image: 'clickhouse/clickhouse-server:25.6.9.98'
|
||
python-version: '3.12.11'
|
||
concurrency: 10
|
||
group: 1
|
||
- segment: 'Temporal'
|
||
person-on-events: false
|
||
clickhouse-server-image: 'clickhouse/clickhouse-server:25.6.9.98'
|
||
python-version: '3.12.11'
|
||
concurrency: 10
|
||
group: 2
|
||
- segment: 'Temporal'
|
||
person-on-events: false
|
||
clickhouse-server-image: 'clickhouse/clickhouse-server:25.6.9.98'
|
||
python-version: '3.12.11'
|
||
concurrency: 10
|
||
group: 3
|
||
- segment: 'Temporal'
|
||
person-on-events: false
|
||
clickhouse-server-image: 'clickhouse/clickhouse-server:25.6.9.98'
|
||
python-version: '3.12.11'
|
||
concurrency: 10
|
||
group: 4
|
||
- segment: 'Temporal'
|
||
person-on-events: false
|
||
clickhouse-server-image: 'clickhouse/clickhouse-server:25.6.9.98'
|
||
python-version: '3.12.11'
|
||
concurrency: 10
|
||
group: 5
|
||
- segment: 'Temporal'
|
||
person-on-events: false
|
||
clickhouse-server-image: 'clickhouse/clickhouse-server:25.6.9.98'
|
||
python-version: '3.12.11'
|
||
concurrency: 10
|
||
group: 6
|
||
- segment: 'Temporal'
|
||
person-on-events: false
|
||
clickhouse-server-image: 'clickhouse/clickhouse-server:25.6.9.98'
|
||
python-version: '3.12.11'
|
||
concurrency: 10
|
||
group: 7
|
||
- segment: 'Temporal'
|
||
person-on-events: false
|
||
clickhouse-server-image: 'clickhouse/clickhouse-server:25.6.9.98'
|
||
python-version: '3.12.11'
|
||
concurrency: 10
|
||
group: 8
|
||
- segment: 'Temporal'
|
||
person-on-events: false
|
||
clickhouse-server-image: 'clickhouse/clickhouse-server:25.6.9.98'
|
||
python-version: '3.12.11'
|
||
concurrency: 10
|
||
group: 9
|
||
- segment: 'Temporal'
|
||
person-on-events: false
|
||
clickhouse-server-image: 'clickhouse/clickhouse-server:25.6.9.98'
|
||
python-version: '3.12.11'
|
||
concurrency: 10
|
||
group: 10
|
||
|
||
steps:
|
||
- uses: actions/checkout@v4
|
||
with:
|
||
fetch-depth: 1
|
||
repository: ${{ github.event.pull_request.head.repo.full_name }}
|
||
ref: ${{ github.event.pull_request.head.ref }}
|
||
# Use PostHog Bot token when not on forks to enable proper snapshot updating
|
||
token: ${{ github.event.pull_request.head.repo.full_name == github.repository && secrets.POSTHOG_BOT_PAT || github.token }}
|
||
clean: false
|
||
- name: Clean up data directories with container permissions
|
||
run: |
|
||
# Use docker to clean up files created by containers
|
||
[ -d "data" ] && docker run --rm -v "$(pwd)/data:/data" alpine sh -c "rm -rf /data/seaweedfs /data/minio" || true
|
||
continue-on-error: true
|
||
|
||
- name: 'Safeguard: ensure no stray Python modules at product root'
|
||
run: |
|
||
echo "Checking that products/* only contain backend/, frontend/, or shared/ as Python code roots..."
|
||
BAD_FILES=$(find products -maxdepth 2 -type f -name "*.py" ! -path "*/backend/*" ! -name "__init__.py" ! -name "conftest.py" -o -maxdepth 2 -type d -name "migrations" ! -path "*/backend/*")
|
||
if [ -n "$BAD_FILES" ]; then
|
||
echo "❌ Found Python code or migrations outside backend/:"
|
||
echo "$BAD_FILES"
|
||
echo "Please move these into the appropriate backend/ folder."
|
||
exit 1
|
||
fi
|
||
echo "✅ No stray Python files or migrations found at product roots."
|
||
|
||
# Pre-tests
|
||
|
||
# Copies the fully versioned UDF xml file for use in CI testing
|
||
- name: Stop/Start stack with Docker Compose
|
||
shell: bash
|
||
run: |
|
||
export CLICKHOUSE_SERVER_IMAGE=${{ matrix.clickhouse-server-image }}
|
||
export DOCKER_REGISTRY_PREFIX="us-east1-docker.pkg.dev/posthog-301601/mirror/"
|
||
cp posthog/user_scripts/latest_user_defined_function.xml docker/clickhouse/user_defined_function.xml
|
||
|
||
# Start docker compose in background
|
||
(
|
||
max_attempts=3
|
||
attempt=1
|
||
delay=5
|
||
|
||
while [ $attempt -le $max_attempts ]; do
|
||
echo "Attempt $attempt of $max_attempts to start stack..."
|
||
|
||
if docker compose -f docker-compose.dev.yml down && \
|
||
docker compose -f docker-compose.dev.yml up -d; then
|
||
echo "Stack started successfully"
|
||
exit 0
|
||
fi
|
||
|
||
echo "Failed to start stack on attempt $attempt"
|
||
|
||
if [ $attempt -lt $max_attempts ]; then
|
||
sleep_time=$((delay * 2 ** (attempt - 1)))
|
||
echo "Waiting ${sleep_time} seconds before retry..."
|
||
sleep $sleep_time
|
||
fi
|
||
|
||
attempt=$((attempt + 1))
|
||
done
|
||
|
||
echo "Failed to start stack after $max_attempts attempts"
|
||
exit 1
|
||
) &
|
||
|
||
- name: Add Kafka and ClickHouse to /etc/hosts
|
||
shell: bash
|
||
run: echo "127.0.0.1 kafka clickhouse" | sudo tee -a /etc/hosts
|
||
|
||
- name: Set up Python
|
||
uses: actions/setup-python@v5
|
||
with:
|
||
python-version: ${{ matrix.python-version }}
|
||
token: ${{ secrets.POSTHOG_BOT_PAT }}
|
||
|
||
- name: Install uv
|
||
id: setup-uv-tests
|
||
uses: astral-sh/setup-uv@3259c6206f993105e3a61b142c2d97bf4b9ef83d # v7.1.0
|
||
with:
|
||
enable-cache: true
|
||
version: 0.8.19
|
||
|
||
- name: Install SAML (python3-saml) dependencies
|
||
if: ${{ needs.changes.outputs.backend == 'true' && steps.setup-uv-tests.outputs.cache-hit != 'true' }}
|
||
shell: bash
|
||
run: |
|
||
sudo apt-get update && sudo apt-get install libxml2-dev libxmlsec1-dev libxmlsec1-openssl
|
||
|
||
- name: Determine if hogql-parser has changed compared to master
|
||
shell: bash
|
||
id: hogql-parser-diff
|
||
run: |
|
||
git fetch --no-tags --prune --depth=1 origin master
|
||
changed=$(git diff --quiet HEAD origin/master -- common/hogql_parser/ && echo "false" || echo "true")
|
||
echo "changed=$changed" >> $GITHUB_OUTPUT
|
||
|
||
- name: Install pnpm
|
||
uses: pnpm/action-setup@a7487c7e89a18df4991f7f222e4898a00d66ddda # v4
|
||
|
||
- name: Set up Node.js
|
||
uses: actions/setup-node@v4
|
||
with:
|
||
node-version: 22.17.1
|
||
cache: pnpm
|
||
|
||
# tests would intermittently fail in GH actions
|
||
# with exit code 134 _after passing_ all tests
|
||
# this appears to fix it
|
||
# absolute wild tbh https://stackoverflow.com/a/75503402
|
||
- uses: tlambert03/setup-qt-libs@19e4ef2d781d81f5f067182e228b54ec90d23b76 # v1
|
||
|
||
- name: Install plugin_transpiler
|
||
shell: bash
|
||
run: |
|
||
pnpm --filter=@posthog/plugin-transpiler... install --frozen-lockfile
|
||
bin/turbo --filter=@posthog/plugin-transpiler build
|
||
|
||
- name: Install Python dependencies
|
||
shell: bash
|
||
run: |
|
||
UV_PROJECT_ENVIRONMENT=$pythonLocation uv sync --frozen --dev
|
||
|
||
- name: Install the working version of hogql-parser
|
||
if: ${{ needs.changes.outputs.backend == 'true' && steps.hogql-parser-diff.outputs.changed == 'true' }}
|
||
shell: bash
|
||
# This is not cached currently, as it's important to build the current HEAD version of hogql-parser if it has
|
||
# changed (requirements.txt has the already-published version)
|
||
run: |
|
||
sudo apt-get install libboost-all-dev unzip cmake curl uuid pkg-config
|
||
curl https://www.antlr.org/download/antlr4-cpp-runtime-4.13.1-source.zip --output antlr4-source.zip
|
||
# Check that the downloaded archive is the expected runtime - a security measure
|
||
anltr_known_md5sum="c875c148991aacd043f733827644a76f"
|
||
antlr_found_ms5sum="$(md5sum antlr4-source.zip | cut -d' ' -f1)"
|
||
if [[ "$anltr_known_md5sum" != "$antlr_found_ms5sum" ]]; then
|
||
echo "Unexpected MD5 sum of antlr4-source.zip!"
|
||
echo "Known: $anltr_known_md5sum"
|
||
echo "Found: $antlr_found_ms5sum"
|
||
exit 64
|
||
fi
|
||
unzip antlr4-source.zip -d antlr4-source && cd antlr4-source
|
||
cmake .
|
||
DESTDIR=out make install
|
||
sudo cp -r out/usr/local/include/antlr4-runtime /usr/include/
|
||
sudo cp out/usr/local/lib/libantlr4-runtime.so* /usr/lib/
|
||
sudo ldconfig
|
||
cd ..
|
||
pip install ./common/hogql_parser
|
||
|
||
- name: Set up needed files
|
||
shell: bash
|
||
run: |
|
||
mkdir -p frontend/dist
|
||
touch frontend/dist/index.html
|
||
touch frontend/dist/layout.html
|
||
touch frontend/dist/exporter.html
|
||
./bin/download-mmdb
|
||
|
||
- name: Wait for services to be available
|
||
shell: bash
|
||
run: |
|
||
bin/check_kafka_clickhouse_up
|
||
bin/check_postgres_up
|
||
|
||
- name: Wait for Temporal
|
||
if: ${{ needs.changes.outputs.backend == 'true' && matrix.segment == 'Temporal' }}
|
||
shell: bash
|
||
run: |
|
||
bin/check_temporal_up
|
||
|
||
- name: Determine if --snapshot-update should be on
|
||
# Skip on forks (due to GITHUB_TOKEN being read-only in PRs coming from them) except for persons-on-events
|
||
# runs, as we want to ignore snapshots diverging there
|
||
if: ${{ needs.changes.outputs.backend == 'true' && (github.event.pull_request.head.repo.full_name == github.repository || matrix.person-on-events) }}
|
||
shell: bash
|
||
run: echo "PYTEST_ARGS=--snapshot-update" >> $GITHUB_ENV # We can only update snapshots within the PostHog org
|
||
|
||
# Tests
|
||
- name: Run Core tests
|
||
id: run-core-tests
|
||
if: ${{ needs.changes.outputs.backend == 'true' && matrix.segment == 'Core' }}
|
||
env:
|
||
PERSON_ON_EVENTS_V2_ENABLED: ${{ matrix.person-on-events && 'true' || 'false' }}
|
||
shell: bash
|
||
run: | # async_migrations covered in ci-async-migrations.yml
|
||
set +e
|
||
pytest ${{
|
||
matrix.person-on-events
|
||
&& './posthog/clickhouse/ ./posthog/queries/ ./posthog/api/test/test_insight* ./posthog/api/test/dashboards/test_dashboard.py'
|
||
|| 'posthog products'
|
||
}} ${{ matrix.person-on-events && 'ee/clickhouse/' || 'ee/' }} -m "not async_migrations" \
|
||
--ignore=posthog/temporal \
|
||
--ignore=products/batch_exports/backend/tests/temporal \
|
||
--ignore=common/hogvm/python/test \
|
||
${{ matrix.person-on-events && '--ignore=posthog/hogql_queries' || '' }} \
|
||
${{ matrix.person-on-events && '--ignore=posthog/hogql' || '' }} \
|
||
--splits ${{ matrix.concurrency }} --group ${{ matrix.group }} \
|
||
--durations=1000 --durations-min=1.0 --store-durations \
|
||
--splitting-algorithm=duration_based_chunks \
|
||
$PYTEST_ARGS
|
||
exit_code=$?
|
||
set -e
|
||
if [ $exit_code -eq 5 ]; then
|
||
echo "No tests collected for this shard, this is expected when splitting tests"
|
||
exit 0
|
||
else
|
||
exit $exit_code
|
||
fi
|
||
|
||
# Uncomment this code to create an ssh-able console so you can debug issues with github actions
|
||
# (Consider changing the timeout in ci-backend.yml to have more time)
|
||
# - name: Setup tmate session
|
||
# if: failure()
|
||
# uses: mxschmitt/action-tmate@v3
|
||
|
||
- name: Run /decide read replica tests
|
||
id: run-decide-read-replica-tests
|
||
if: ${{ needs.changes.outputs.backend == 'true' && matrix.segment == 'Core' && matrix.group == 1 && !matrix.person-on-events }}
|
||
env:
|
||
POSTHOG_DB_NAME: posthog
|
||
READ_REPLICA_OPT_IN: 'decide,PersonalAPIKey, local_evaluation'
|
||
POSTHOG_POSTGRES_READ_HOST: localhost
|
||
POSTHOG_DB_PASSWORD: posthog
|
||
POSTHOG_DB_USER: posthog
|
||
shell: bash
|
||
run: |
|
||
pytest posthog/api/test/test_decide.py::TestDecideUsesReadReplica \
|
||
--durations=1000 --durations-min=1.0 \
|
||
$PYTEST_ARGS
|
||
|
||
- name: Run Temporal tests
|
||
id: run-temporal-tests
|
||
if: ${{ needs.changes.outputs.backend == 'true' && matrix.segment == 'Temporal' }}
|
||
shell: bash
|
||
env:
|
||
AWS_S3_ALLOW_UNSAFE_RENAME: 'true'
|
||
RUNLOOP_API_KEY: ${{ needs.changes.outputs.tasks_temporal == 'true' && secrets.RUNLOOP_API_KEY || '' }}
|
||
run: |
|
||
set +e
|
||
pytest posthog/temporal products/batch_exports/backend/tests/temporal products/tasks/backend/temporal -m "not async_migrations" \
|
||
--splits ${{ matrix.concurrency }} --group ${{ matrix.group }} \
|
||
--durations=100 --durations-min=1.0 --store-durations \
|
||
--splitting-algorithm=duration_based_chunks \
|
||
$PYTEST_ARGS
|
||
exit_code=$?
|
||
set -e
|
||
if [ $exit_code -eq 5 ]; then
|
||
echo "No tests collected for this shard, this is expected when splitting tests"
|
||
exit 0
|
||
else
|
||
exit $exit_code
|
||
fi
|
||
|
||
# Post tests
|
||
- name: Show docker compose logs on failure
|
||
if: failure() && (needs.changes.outputs.backend == 'true' && steps.run-core-tests.outcome != 'failure' && steps.run-decide-read-replica-tests.outcome != 'failure' && steps.run-temporal-tests.outcome != 'failure')
|
||
shell: bash
|
||
run: docker compose -f docker-compose.dev.yml logs
|
||
|
||
- name: Upload updated timing data as artifacts
|
||
uses: actions/upload-artifact@v4
|
||
if: ${{ needs.changes.outputs.backend == 'true' && !matrix.person-on-events && matrix.clickhouse-server-image == 'clickhouse/clickhouse-server:25.6.9.98' }}
|
||
with:
|
||
name: timing_data-${{ matrix.segment }}-${{ matrix.group }}
|
||
path: .test_durations
|
||
include-hidden-files: true
|
||
retention-days: 2
|
||
|
||
- name: Verify new snapshots for flakiness
|
||
if: ${{ github.event.pull_request.head.repo.full_name == 'PostHog/posthog' && needs.changes.outputs.backend == 'true' && !matrix.person-on-events }}
|
||
shell: bash
|
||
run: |
|
||
.github/scripts/verify-new-snapshots.sh
|
||
|
||
- uses: EndBug/add-and-commit@a94899bca583c204427a224a7af87c02f9b325d5 # v9
|
||
# Also skip for persons-on-events runs, as we want to ignore snapshots diverging there
|
||
if: ${{ github.event.pull_request.head.repo.full_name == 'PostHog/posthog' && needs.changes.outputs.backend == 'true' && !matrix.person-on-events }}
|
||
with:
|
||
add: '["ee", "./**/*.ambr", "posthog/queries/", "posthog/migrations", "posthog/tasks", "posthog/hogql/"]'
|
||
message: 'Update query snapshots'
|
||
pull: --rebase --autostash # Make sure we're up-to-date with other segments' updates
|
||
default_author: github_actions
|
||
github_token: ${{ secrets.POSTHOG_BOT_PAT }}
|
||
|
||
- name: Check if any snapshot changes were left uncomitted
|
||
id: changed-files
|
||
if: ${{ github.event.pull_request.head.repo.full_name == 'PostHog/posthog' && needs.changes.outputs.backend == 'true' && !matrix.person-on-events }}
|
||
run: |
|
||
if [[ -z $(git status -s | grep -v ".test_durations" | tr -d "\n") ]]
|
||
then
|
||
echo 'files_found=false' >> $GITHUB_OUTPUT
|
||
else
|
||
echo 'diff=$(git status --porcelain)' >> $GITHUB_OUTPUT
|
||
echo 'files_found=true' >> $GITHUB_OUTPUT
|
||
fi
|
||
|
||
- name: Fail CI if some snapshots have been updated but not committed
|
||
if: steps.changed-files.outputs.files_found == 'true' && steps.add-and-commit.outcome == 'success'
|
||
run: |
|
||
echo "${{ steps.changed-files.outputs.diff }}"
|
||
exit 1
|
||
|
||
- name: Archive email renders
|
||
uses: actions/upload-artifact@v4
|
||
if: needs.changes.outputs.backend == 'true' && matrix.segment == 'Core' && !matrix.person-on-events
|
||
with:
|
||
name: email_renders-${{ github.sha }}-${{ github.run_attempt }}-${{ matrix.segment }}-${{ matrix.person-on-events }}-${{ matrix.group }}
|
||
path: posthog/tasks/test/__emails__
|
||
retention-days: 1
|
||
|
||
# Job just to collate the status of the matrix jobs for requiring passing status
|
||
django_tests:
|
||
needs: [django, check-migrations, async-migrations]
|
||
name: Django Tests Pass
|
||
runs-on: ubuntu-latest
|
||
if: always()
|
||
steps:
|
||
- name: Check matrix outcome
|
||
run: |
|
||
# The `needs.django.result` will be 'success' only if all jobs in the matrix succeeded.
|
||
# Otherwise, it will be 'failure'.
|
||
if [[ "${{ needs.django.result }}" != "success" && "${{ needs.django.result }}" != "skipped" ]]; then
|
||
echo "One or more jobs in the Django test matrix failed."
|
||
exit 1
|
||
fi
|
||
|
||
# Check migration validation - must pass for Django Tests to be green
|
||
if [[ "${{ needs.check-migrations.result }}" != "success" && "${{ needs.check-migrations.result }}" != "skipped" ]]; then
|
||
echo "Migration checks failed."
|
||
exit 1
|
||
fi
|
||
|
||
# Check async migrations - must pass for Django Tests to be green
|
||
if [[ "${{ needs.async-migrations.result }}" != "success" && "${{ needs.async-migrations.result }}" != "skipped" ]]; then
|
||
echo "Async migrations tests failed."
|
||
exit 1
|
||
fi
|
||
|
||
echo "All checks passed."
|
||
|
||
async-migrations:
|
||
name: Async migrations tests - ${{ matrix.clickhouse-server-image }}
|
||
needs: [changes]
|
||
strategy:
|
||
fail-fast: false
|
||
matrix:
|
||
clickhouse-server-image: ['clickhouse/clickhouse-server:25.6.9.98']
|
||
if: needs.changes.outputs.backend == 'true'
|
||
runs-on: ubuntu-latest
|
||
steps:
|
||
- name: 'Checkout repo'
|
||
uses: actions/checkout@v4
|
||
with:
|
||
fetch-depth: 1
|
||
clean: false
|
||
- name: Clean up data directories with container permissions
|
||
run: |
|
||
# Use docker to clean up files created by containers
|
||
[ -d "data" ] && docker run --rm -v "$(pwd)/data:/data" alpine sh -c "rm -rf /data/seaweedfs /data/minio" || true
|
||
continue-on-error: true
|
||
|
||
- name: Start stack with Docker Compose
|
||
run: |
|
||
export CLICKHOUSE_SERVER_IMAGE_VERSION=${{ matrix.clickhouse-server-image }}
|
||
docker compose -f docker-compose.dev.yml down
|
||
docker compose -f docker-compose.dev.yml up -d &
|
||
|
||
- name: Set up Python
|
||
uses: actions/setup-python@v5
|
||
with:
|
||
python-version-file: 'pyproject.toml'
|
||
|
||
- name: Install uv
|
||
id: setup-uv-async
|
||
uses: astral-sh/setup-uv@3259c6206f993105e3a61b142c2d97bf4b9ef83d # v7.1.0
|
||
with:
|
||
enable-cache: true
|
||
version: 0.8.19
|
||
|
||
- name: Install SAML (python3-saml) dependencies
|
||
if: steps.setup-uv-async.outputs.cache-hit != 'true'
|
||
run: |
|
||
sudo apt-get update
|
||
sudo apt-get install libxml2-dev libxmlsec1-dev libxmlsec1-openssl
|
||
|
||
- name: Install python dependencies
|
||
shell: bash
|
||
run: |
|
||
UV_PROJECT_ENVIRONMENT=$pythonLocation uv sync --frozen --dev
|
||
|
||
- name: Add Kafka and ClickHouse to /etc/hosts
|
||
run: sudo echo "127.0.0.1 kafka clickhouse" | sudo tee -a /etc/hosts
|
||
|
||
- name: Set up needed files
|
||
run: |
|
||
mkdir -p frontend/dist
|
||
touch frontend/dist/index.html
|
||
touch frontend/dist/layout.html
|
||
touch frontend/dist/exporter.html
|
||
|
||
- name: Wait for services to be available
|
||
shell: bash
|
||
run: |
|
||
bin/check_kafka_clickhouse_up
|
||
bin/check_postgres_up
|
||
|
||
- name: Run async migrations tests
|
||
run: |
|
||
pytest -m "async_migrations"
|
||
|
||
calculate-running-time:
|
||
name: Calculate running time
|
||
needs: [django, async-migrations]
|
||
runs-on: ubuntu-latest
|
||
if: # Run on pull requests to PostHog/posthog + on PostHog/posthog outside of PRs - but never on forks
|
||
needs.changes.outputs.backend == 'true' && (
|
||
(github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name == 'PostHog/posthog') ||
|
||
(github.event_name != 'pull_request' && github.repository == 'PostHog/posthog'))
|
||
steps:
|
||
- name: Calculate running time
|
||
run: |
|
||
gh auth login --with-token < <(echo ${{ secrets.GITHUB_TOKEN }})
|
||
run_id=${GITHUB_RUN_ID}
|
||
repo=${GITHUB_REPOSITORY}
|
||
run_info=$(gh api repos/${repo}/actions/runs/${run_id})
|
||
echo run_info: ${run_info}
|
||
# name is the name of the workflow file
|
||
# run_started_at is the start time of the workflow
|
||
# we want to get the number of seconds between the start time and now
|
||
name=$(echo ${run_info} | jq -r '.name')
|
||
run_url=$(echo ${run_info} | jq -r '.url')
|
||
run_started_at=$(echo ${run_info} | jq -r '.run_started_at')
|
||
run_attempt=$(echo ${run_info} | jq -r '.run_attempt')
|
||
start_seconds=$(date -d "${run_started_at}" +%s)
|
||
now_seconds=$(date +%s)
|
||
duration=$((now_seconds-start_seconds))
|
||
echo running_time_duration_seconds=${duration} >> $GITHUB_ENV
|
||
echo running_time_run_url=${run_url} >> $GITHUB_ENV
|
||
echo running_time_run_attempt=${run_attempt} >> $GITHUB_ENV
|
||
echo running_time_run_id=${run_id} >> $GITHUB_ENV
|
||
echo running_time_run_started_at=${run_started_at} >> $GITHUB_ENV
|
||
- name: Capture running time to PostHog
|
||
uses: PostHog/posthog-github-action@v0.1
|
||
with:
|
||
posthog-token: ${{secrets.POSTHOG_API_TOKEN}}
|
||
event: 'posthog-ci-running-time'
|
||
properties: '{"runner": "depot", "duration_seconds": ${{ env.running_time_duration_seconds }}, "run_url": "${{ env.running_time_run_url }}", "run_attempt": "${{ env.running_time_run_attempt }}", "run_id": "${{ env.running_time_run_id }}", "run_started_at": "${{ env.running_time_run_started_at }}"}'
|