chore(dev): Add Flox for local environment setup (#26805)

2026-02-04 03:01:23 +01:00 · 2024-12-19 00:36:53 +01:00
parent 8f49a0df41
commit a99faee2e1
11 changed files with 2071 additions and 39 deletions
--- a/.envrc
+++ b/.envrc
@@ -0,0 +1,3 @@
+if [ -z "${FLOX_VERSION}" ]; then # Don't activate if already activated
+    flox activate
+fi
--- a/.flox/.gitignore
+++ b/.flox/.gitignore
@@ -0,0 +1,4 @@
+run/
+cache/
+lib/
+log/
--- a/.flox/env.json
+++ b/.flox/env.json
@@ -0,0 +1,4 @@
+{
+    "name": "posthog",
+    "version": 1
+}
--- a/.flox/env/direnv-setup.sh
+++ b/.flox/env/direnv-setup.sh
@@ -0,0 +1,70 @@
+#!/bin/bash
+
+if ! command -v direnv &> /dev/null; then
+    # Install direnv based on platform
+    if command -v brew &> /dev/null; then
+        echo "🔄 Installing direnv using 'brew install direnv'..."
+        HOMEBREW_NO_ENV_HINTS=1 brew install -q direnv
+    elif command -v apt &> /dev/null; then
+        echo "🔄 Installing direnv using 'apt install direnv'..."
+        sudo apt update && sudo apt install -yq direnv
+    elif command -v dnf &> /dev/null; then
+        echo "🔄 Installing direnv using 'dnf install direnv'..."
+        sudo dnf install -yq direnv
+    else
+        echo "🔄 Installing direnv using 'curl -sfL https://direnv.net/install.sh | bash'"
+        curl -sfL https://direnv.net/install.sh | bash
+    fi
+    echo "✅ Installed direnv"
+else
+    echo "⏩ direnv already installed"
+fi
+
+# Determine shell and config file
+shell_name=$(basename "$SHELL")
+case "$shell_name" in
+    "bash")
+    config_file="$HOME/.bashrc"
+    hook_command='eval "$(direnv hook bash)"'
+    ;;
+    "zsh")
+    config_file="$HOME/.zshrc"
+    hook_command='eval "$(direnv hook zsh)"'
+    ;;
+    "fish")
+    config_file="$HOME/.config/fish/config.fish"
+    hook_command='direnv hook fish | source'
+    mkdir -p "$(dirname "$config_file")"
+    ;;
+    "tcsh")
+    config_file="$HOME/.cshrc"
+    hook_command='eval `direnv hook tcsh`'
+    ;;
+    *)
+    echo "Unsupported shell: $shell_name"
+    return 1
+    ;;
+esac
+
+echo "🐚 Configuring your default shell, $SHELL, for direnv"
+
+# Add hook to shell config if not already present
+if ! grep -q "direnv hook" "$config_file" 2>/dev/null; then
+    echo -e "\n# Initialize direnv - added by PostHog's Flox activation hook (../posthog/.flox/env/manifest.toml)\n$hook_command" >> "$config_file"
+    echo "✅ Injected direnv hook into $config_file"
+else
+    echo "⏩ direnv hook already present in $config_file"
+fi
+
+# Add hook to shell config if not already present
+if ! grep -q "warn_timeout" "$HOME/.config/direnv/direnv.toml" 2>/dev/null; then
+    echo "[global]\nwarn_timeout = 0 # Ignore timeout from this issue: https://github.com/direnv/direnv/issues/1065 - added by PostHog's Flox activation hook (../posthog/.flox/env/manifest.toml)" >> "$HOME/.config/direnv/direnv.toml"
+    echo "✅ Configured ~/.config/direnv/direnv.toml"
+else
+    echo "⏩ ~/.config/direnv/direnv.toml already configured"
+fi
+
+echo "💫 direnv is now active"
+
+# Allow this directory's .envrc to be loaded
+direnv allow
--- a/.flox/env/manifest.lock
+++ b/.flox/env/manifest.lock
--- a/.flox/env/manifest.toml
+++ b/.flox/env/manifest.toml
@@ -0,0 +1,138 @@
+#
+# This is a Flox environment manifest.
+# Visit flox.dev/docs/concepts/manifest/
+# or see flox-edit(1), manifest.toml(5) for more information.
+#
+# Flox manifest version managed by Flox CLI
+version = 1
+
+# List packages you wish to install in your environment inside
+# the `[install]` section.
+[install]
+# Python
+python3 = { pkg-path = "python3", version = "3.11", pkg-group = "python" }
+uv = { pkg-path = "uv", pkg-group = "python" }
+xmlsec = { pkg-path = "xmlsec", pkg-group = "python", version = "1.2.34" }
+libtool = { pkg-path = "libtool", pkg-group = "python" }
+# Node
+nodejs = { pkg-path = "nodejs_18", pkg-group = "nodejs" }
+corepack = { pkg-path = "corepack", pkg-group = "nodejs" }
+brotli = { pkg-path = "brotli", pkg-group = "nodejs" }
+# Rust toolchain (based on https://flox.dev/docs/cookbook/languages/rust/)
+cargo.pkg-path = "cargo"
+cargo.pkg-group = "rust-toolchain"
+cargo.version = "1.80.1"
+rustc.pkg-path = "rustc"
+rustc.pkg-group = "rust-toolchain"
+clippy.pkg-path = "clippy"
+clippy.pkg-group = "rust-toolchain"
+rustfmt.pkg-path = "rustfmt"
+rustfmt.pkg-group = "rust-toolchain"
+rust-lib-src.pkg-path = "rustPlatform.rustLibSrc"
+rust-lib-src.pkg-group = "rust-toolchain"
+libiconv.pkg-path = "libiconv"
+libiconv.systems = ["aarch64-darwin"]
+libiconv.pkg-group = "rust-toolchain"
+# Go
+go = { pkg-path = "go", version = "1.22", pkg-group = "go" }
+# General CLI tools
+mprocs.pkg-path = "mprocs"
+
+# Set environment variables in the `[vars]` section. These variables may not
+# reference one another, and are added to the environment without first
+# expanding them. They are available for use in the `[profile]` and `[hook]`
+# scripts.
+[vars]
+DEBUG = "1"
+POSTHOG_SKIP_MIGRATION_CHECKS = "1"
+DIRENV_LOG_FORMAT = "" # Disable direnv activation logging (in case direnv is present)
+
+# The `hook.on-activate` script is run by the *bash* shell immediately upon
+# activating an environment, and will not be invoked if Flox detects that the
+# environment has previously been activated. Variables set by the script will
+# be inherited by `[profile]` scripts defined below. Note that any stdout
+# generated by the script will be redirected to stderr.
+[hook]
+on-activate = '''
+# Guide through installing and configuring direnv if it's not present (optionally)
+if [[ -t 0 ]] && [ ! -d "$DIRENV_DIR" ] && [ ! -f "$FLOX_ENV_CACHE/.hush-direnv" ]; then
+  read -p "
+👉 Use direnv (https://direnv.net) for automatic activation of this environment by your shell.
+❓ Would you like direnv to be set up now? (y/n, default: y)" -n 1 -r
+  if [[ $REPLY =~ ^[Yy]$ || -z $REPLY ]]; then
+    $FLOX_ENV_CACHE/../env/direnv-setup.sh
+  else
+    echo "⏭️ Skipping direnv setup. This message will not be shown again, but if you change your mind, just run '.flox/bin/direnv-setup.sh'"
+    touch $FLOX_ENV_CACHE/.hush-direnv
+  fi
+  echo
+fi
+
+# Set up a Python virtual environment
+export PYTHON_DIR="$FLOX_ENV_CACHE/venv"
+if [ ! -d "$PYTHON_DIR" ]; then
+  uv venv "$PYTHON_DIR" --allow-existing
+fi
+
+echo -e "Python virtual environment path: \033[33m.flox/cache/venv\033[0m"
+echo -e "Python interpreter path, for your code editor: \033[33m.flox/cache/venv/bin/python\033[0m"
+
+source "$PYTHON_DIR/bin/activate"
+
+# Install Python dependencies (this is practically instant thanks to uv)
+uv pip install -q -r requirements.txt -r requirements-dev.txt
+
+# Install top-level Node dependencies (only if not present all yet, because this takes almost a second even with pnpm)
+# This also sets up pre-commit hooks via Husky
+if [ ! -d "node_modules" ]; then
+  pnpm install -s
+fi
+
+if [[ -t 0 ]]; then # The block below only runs when in an interactive shell
+  # Add required entries to /etc/hosts if not present
+  if ! grep -q "127.0.0.1 kafka clickhouse" /etc/hosts; then
+    echo
+    echo "🚨 Amending /etc/hosts to map hostnames 'kafka' and 'clickhouse' to 127.0.0.1..."
+    echo "127.0.0.1 kafka clickhouse" | sudo tee -a /etc/hosts 1> /dev/null
+    echo "✅ /etc/hosts amended"
+  fi
+
+  # Print intro message
+  echo -e "
+IT'S DANGEROUS TO GO ALONE! RUN THIS:
+1. \033[31mbin/migrate\033[0m - to run all migrations
+2. \033[32mbin/start\033[0m - to start the entire stack
+3. \033[34m./manage.py generate_demo_data\033[0m - to create a user with demo data
+"
+fi
+'''
+
+# Scripts defined in the `[profile]` section are *sourced* by *your shell* and
+# inherit environment variables set in the `[vars]` section and by `[hook]` scripts.
+# The `profile.common` script is sourced by all shells and special care should be
+# taken to ensure compatibility with all shells, after which exactly one of
+# `profile.{bash,fish,tcsh,zsh}` is sourced by the corresponding shell.
+[profile]
+bash = '''
+  source "$PYTHON_DIR/bin/activate"
+'''
+zsh = '''
+  source "$PYTHON_DIR/bin/activate"
+'''
+fish = '''
+  source "$PYTHON_DIR/bin/activate.fish"
+'''
+tcsh = '''
+  source "$PYTHON_DIR/bin/activate.csh"
+'''
+
+# The `[services]` section of the manifest allows you to define services.
+# Services defined here use the packages provided by the `[install]` section
+# and any variables you've defined in the `[vars]` section or `hook.on-activate` script.
+[services]
+# db.command = "postgres -D $FLOX_ENV_CACHE/postgres"
+
+# Additional options can be set in the `[options]` section. Refer to
+# manifest.toml(5) for a list of available options.
+[options]
+systems = ["aarch64-darwin", "aarch64-linux", "x86_64-darwin", "x86_64-linux"]
--- a/.gitignore
+++ b/.gitignore
@@ -1,5 +1,5 @@
+/env
 venv
-env
 .venv
 *.pyc
 __pycache__/
@@ -12,7 +12,6 @@ debug.log
 *.swo
 node_modules/
 *.code-workspace
-node_modules
 .mypy_cache
 frontend/.cache/
 frontend/dist/
@@ -76,4 +75,4 @@ pyrightconfig.json
 .temporal-worker-settings
 temp_test_run_data.json
 .temp-deepeval-cache.json
-.eslintcache
+.eslintcache
--- a/bin/migrate
+++ b/bin/migrate
@@ -1,5 +1,6 @@
 #!/bin/bash
 set -e
+
 SCRIPT_DIR=$(dirname "$(readlink -f "$0")")

 # NOTE when running in docker, rust might not exist so we need to check for it
@@ -7,8 +8,12 @@ if [ -d "$SCRIPT_DIR/../rust" ]; then
    bash $SCRIPT_DIR/../rust/bin/migrate-cyclotron
 fi

+(
+    python manage.py migrate_clickhouse 
+    python manage.py sync_replicated_schema
+) & # ClickHouse migrations can run in parallel to Postgres ones
+
 python manage.py migrate
-python manage.py migrate_clickhouse

 # NOTE: we do not apply any non-noop migrations here. Rather these are run
 # manually within the UI. See https://posthog.com/docs/runbook/async-migrations
@@ -20,4 +25,4 @@ python manage.py run_async_migrations --complete-noop-migrations
 # k8s pod deployments.
 python manage.py run_async_migrations --check

-python manage.py sync_replicated_schema
+wait $(jobs -p) # Make sure CH migrations are done before we exit
--- a/bin/mprocs.yaml
+++ b/bin/mprocs.yaml
@@ -1,24 +1,26 @@
 procs:
    celery-worker:
-        shell: 'source ./bin/celery-queues.env && python manage.py run_autoreload_celery --type=worker'
+        shell: 'bin/check_kafka_clickhouse_up && source ./bin/celery-queues.env && python manage.py run_autoreload_celery --type=worker'

    celery-beat:
-        shell: 'source ./bin/celery-queues.env && python manage.py run_autoreload_celery --type=beat'
+        shell: 'bin/check_kafka_clickhouse_up && source ./bin/celery-queues.env && python manage.py run_autoreload_celery --type=beat'

    plugin-server:
-        shell: './bin/plugin-server'
+        shell: 'bin/check_kafka_clickhouse_up && ./bin/plugin-server'

    backend:
-        shell: './bin/start-backend'
+        shell: 'bin/check_kafka_clickhouse_up && ./bin/start-backend'

    frontend:
-        shell: './bin/start-frontend'
+        shell: 'bin/check_kafka_clickhouse_up && ./bin/start-frontend'

    temporal-worker:
        # added a sleep to give the docker stuff time to start
-        shell: 'sleep 10 && python3 manage.py start_temporal_worker'
+        shell: 'bin/check_kafka_clickhouse_up && bin/check_temporal_up && python manage.py start_temporal_worker'

    docker-compose:
        shell: 'docker compose -f docker-compose.dev.yml up'
        stop:
            send-keys: ['<C-c>']
+
+mouse_scroll_speed: 1
--- a/bin/start
+++ b/bin/start
@@ -2,25 +2,11 @@

 set -e

-trap "trap - SIGTERM && kill -- -$$" SIGINT SIGTERM EXIT
-
 export DEBUG=${DEBUG:-1}
-export SKIP_SERVICE_VERSION_REQUIREMENTS=1
+export SKIP_SERVICE_VERSION_REQUIREMENTS=${SKIP_SERVICE_VERSION_REQUIREMENTS:-1}
 export BILLING_SERVICE_URL=${BILLING_SERVICE_URL:-https://billing.dev.posthog.dev}
 export HOG_HOOK_URL=${HOG_HOOK_URL:-http://localhost:3300/hoghook}

-service_warning() {
-  echo -e "\033[0;31m$1 isn't ready. You can run the stack with:\ndocker compose -f docker-compose.dev.yml up\nIf you have already ran that, just make sure that services are starting properly, and sit back.\nWaiting for $1 to start...\033[0m"
-}
-
-nc -z localhost 9092 || ( service_warning 'Kafka'; bin/check_kafka_clickhouse_up )
-curl -s 'http://localhost:8123/ping' || ( service_warning 'ClickHouse'; bin/check_kafka_clickhouse_up )
-
 [ ! -f ./share/GeoLite2-City.mmdb ] && ( curl -L "https://mmdbcdn.posthog.net/" --http1.1 | brotli --decompress --output=./share/GeoLite2-City.mmdb )

-./bin/start-worker &
-./bin/start-backend &
-./bin/start-frontend &
-./bin/temporal-django-worker &
-
-wait
+exec mprocs --config bin/mprocs.yaml
--- a/bin/start-mprocs
+++ b/bin/start-mprocs
@@ -1,12 +0,0 @@
-#!/bin/bash
-
-set -e
-
-export DEBUG=${DEBUG:-1}
-export SKIP_SERVICE_VERSION_REQUIREMENTS=${SKIP_SERVICE_VERSION_REQUIREMENTS:-1}
-export BILLING_SERVICE_URL=${BILLING_SERVICE_URL:-https://billing.dev.posthog.dev}
-export HOG_HOOK_URL=${HOG_HOOK_URL:-http://localhost:3300/hoghook}
-
-[ ! -f ./share/GeoLite2-City.mmdb ] && ( curl -L "https://mmdbcdn.posthog.net/" --http1.1 | brotli --decompress --output=./share/GeoLite2-City.mmdb )
-
-exec mprocs --config bin/mprocs.yaml