From ea7a8e0d91eba569cf3b46982082155c327e99c0 Mon Sep 17 00:00:00 2001 From: Federico Izzo Date: Thu, 27 Aug 2026 17:00:12 +0200 Subject: [PATCH] feat(toolboxes): implement reproducible gufo toolboxes for Strix Halo --- .github/workflows/build_and_publish.yml | 92 +++++++ .github/workflows/prune-old-images.yml | 24 ++ .gitignore | 4 + README.md | 102 ++++++++ docs/host-configuration.md | 61 +++++ docs/quickstart.md | 95 +++++++ docs/systemd.md | 56 ++++ flake.lock | 62 +++++ flake.nix | 246 ++++++++++++++++++ refresh-toolboxes.sh | 196 ++++++++++++++ systemd/gpu-workload-watch/gpu-workload-watch | 92 +++++++ .../gpu-workload-watch.service | 14 + systemd/gufo-serve.service | 29 +++ 13 files changed, 1073 insertions(+) create mode 100644 .github/workflows/build_and_publish.yml create mode 100644 .github/workflows/prune-old-images.yml create mode 100644 .gitignore create mode 100644 README.md create mode 100644 docs/host-configuration.md create mode 100644 docs/quickstart.md create mode 100644 docs/systemd.md create mode 100644 flake.lock create mode 100644 flake.nix create mode 100755 refresh-toolboxes.sh create mode 100755 systemd/gpu-workload-watch/gpu-workload-watch create mode 100644 systemd/gpu-workload-watch/gpu-workload-watch.service create mode 100644 systemd/gufo-serve.service diff --git a/.github/workflows/build_and_publish.yml b/.github/workflows/build_and_publish.yml new file mode 100644 index 0000000..0049f0c --- /dev/null +++ b/.github/workflows/build_and_publish.yml @@ -0,0 +1,92 @@ +name: Build & Publish Gufo Toolboxes + +on: + push: + branches: + - main + workflow_dispatch: + inputs: + images: + description: 'Toolboxes to build ("gufo-runtime", "gufo-dev", or "all")' + required: false + default: 'all' + +permissions: + contents: read + packages: write + +env: + REGISTRY: ghcr.io + IMAGE_PREFIX: ghcr.io/${{ github.repository }} + +jobs: + prepare: + runs-on: ubuntu-latest + outputs: + matrix_json: ${{ steps.matrix.outputs.matrix_json }} + steps: + - id: matrix + shell: bash + run: | + IN='${{ github.event.inputs.images || "all" }}' + if [[ "$IN" == "all" || -z "$IN" ]]; then + echo 'matrix_json=["gufo-runtime","gufo-dev"]' >> "$GITHUB_OUTPUT" + else + IN_CLEAN=$(echo "$IN" | tr -d '[:space:]') + echo "matrix_json=[\"${IN_CLEAN//,/\",\"}\"]" >> "$GITHUB_OUTPUT" + fi + + build-and-push: + needs: prepare + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + target: ${{ fromJson(needs.prepare.outputs.matrix_json) }} + + steps: + - name: Free disk space + run: | + sudo rm -rf /usr/share/dotnet /usr/local/lib/android /opt/ghc /opt/hostedtoolcache + sudo docker system prune --all --force + df -h + + - name: Checkout repository + uses: actions/checkout@v4 + + - name: Install Nix + uses: cachix/install-nix-action@v27 + with: + extra_nix_config: | + experimental-features = nix-command flakes + accept-flake-config = true + + - name: Log in to GitHub Container Registry + uses: docker/login-action@v3 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Build OCI image with Nix + run: | + echo "🔨 Building ${{ matrix.target }}-image with Nix..." + nix build .#packages.x86_64-linux.${{ matrix.target }}-image --out-link result-${{ matrix.target }} + + - name: Load and Push image to GHCR + run: | + set -euo pipefail + BUILD_TS=$(date +%Y%m%dT%H%M%S) + IMAGE_NAME="${{ env.IMAGE_PREFIX }}/${{ matrix.target }}" + + echo "đŸ“Ĩ Loading image into Docker daemon..." + docker load < result-${{ matrix.target }} + + echo "🏷ī¸ Tagging image..." + docker tag "${IMAGE_NAME}:latest" "${IMAGE_NAME}:${BUILD_TS}" + + echo "🚀 Pushing latest..." + docker push "${IMAGE_NAME}:latest" + + echo "🚀 Pushing immutable timestamp tag ${BUILD_TS}..." + docker push "${IMAGE_NAME}:${BUILD_TS}" diff --git a/.github/workflows/prune-old-images.yml b/.github/workflows/prune-old-images.yml new file mode 100644 index 0000000..624f63a --- /dev/null +++ b/.github/workflows/prune-old-images.yml @@ -0,0 +1,24 @@ +name: Prune Old Container Images + +on: + schedule: + # Run once a week at midnight Sunday + - cron: '0 0 * * 0' + workflow_dispatch: + +permissions: + packages: write + +jobs: + prune: + runs-on: ubuntu-latest + steps: + - name: Prune old container versions in GHCR + uses: snok/container-retention-policy@v3 + with: + image-names: "gufo-runtime,gufo-dev" + cut-off: 30d + account-type: org + org-name: gufo-org + keep-at-least: 3 + token: ${{ secrets.GITHUB_TOKEN }} diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..6e93aa3 --- /dev/null +++ b/.gitignore @@ -0,0 +1,4 @@ +result +result-* +.direnv/ +amd-strix-halo-toolboxes/ diff --git a/README.md b/README.md new file mode 100644 index 0000000..309b132 --- /dev/null +++ b/README.md @@ -0,0 +1,102 @@ +# Gufo Toolboxes for AMD Strix Halo + +[![CI](https://github.com/gufo-org/toolboxes/actions/workflows/build_and_publish.yml/badge.svg)](https://github.com/gufo-org/toolboxes/actions/workflows/build_and_publish.yml) + +Reproducible container toolboxes for local LLM inference, serving, development, and profiling on **AMD Ryzen AI Max ("Strix Halo")** systems (`gfx1151` RDNA 3.5 GPU + XDNA2 NPU). + +Built directly from **Nix flake derivations** to guarantee bit-for-bit reproducibility and optimal layered caching across Fedora (Toolbx), Ubuntu (Distrobox), and Arch Linux hosts. + +--- + +## đŸ“Ļ Supported Toolboxes + +| Container Name | Image (GHCR) | Description | +| :--- | :--- | :--- | +| **`gufo-runtime`** | `ghcr.io/gufo-org/toolboxes/gufo-runtime:latest` | Lightweight production inference runtime for `gufo serve`, `gufo prompt`, `gufo bench`, and `gufo diagnose`. Includes ROCm `gfx1151` and XRT/XDNA2 runtimes. | +| **`gufo-dev`** | `ghcr.io/gufo-org/toolboxes/gufo-dev:latest` | Interactive development & profiling container. Includes full ROCm SDK, HIP compiler, `rocprofv3`, Clang tools, CMake, and Ninja for kernel development. | + +--- + +## 🚀 Quick Start + +### 1. Requirements +- Linux kernel `>= 6.18.4` (see [Host Configuration](docs/host-configuration.md)) +- User in `video` and `render` groups (`sudo usermod -aG video,render $USER`) +- `podman` + `toolbox` (Fedora) or `distrobox` (Ubuntu/Debian/Arch) + +### 2. Create and Enter Toolbox + +Use the provided `refresh-toolboxes.sh` script, which automatically configures Strix Halo hardware passthroughs (`/dev/kfd`, `/dev/dri`, `/dev/accel*`): + +```bash +# Clone the repository +git clone https://github.com/gufo-org/toolboxes.git +cd toolboxes + +# Create the runtime toolbox +./refresh-toolboxes.sh gufo-runtime + +# Enter the toolbox +toolbox enter gufo-runtime # on Fedora +# OR +distrobox enter gufo-runtime # on Ubuntu / Arch +``` + +### 3. Verify Hardware & Run Inference + +Inside the toolbox: + +```bash +# Check GPU and NPU hardware status +gufo diagnose + +# Run an interactive prompt +gufo prompt --model /path/to/model.gguf -p "What is AMD Strix Halo unified memory architecture?" + +# Benchmark token generation +gufo bench --model /path/to/model.gguf -p 512 -n 128 +``` + +--- + +## 🛠ī¸ Maintainers: Building Images from Source (Nix) + +For maintainers or developers who want to rebuild the container images from source using Nix: + +```bash +# Stream the OCI image directly into local Podman/Docker: +nix run .#stream-gufo-runtime | podman load + +# Or build the image tarball: +nix build .#packages.x86_64-linux.gufo-runtime-image +``` + +--- + +## ⚙ī¸ Background Serving (Systemd) + +Run `gufo serve` as a persistent background user service: + +```bash +mkdir -p ~/.config/systemd/user +cp systemd/gufo-serve.service ~/.config/systemd/user/ +systemctl --user daemon-reload +systemctl --user start gufo-serve +systemctl --user enable gufo-serve +``` + +See [docs/systemd.md](docs/systemd.md) for full configuration options. + +--- + +## 📚 Documentation + +- [Quick Start Guide](docs/quickstart.md) +- [Host Configuration (Kernel, Firmware, Groups)](docs/host-configuration.md) +- [Systemd Integration](docs/systemd.md) + +--- + +## 📜 License + +Apache-2.0. diff --git a/docs/host-configuration.md b/docs/host-configuration.md new file mode 100644 index 0000000..09e5e68 --- /dev/null +++ b/docs/host-configuration.md @@ -0,0 +1,61 @@ +# Host Configuration Guide for AMD Strix Halo + +This document details host-level configuration for running Gufo Toolboxes reliably on AMD Ryzen AI Max ("Strix Halo", `gfx1151` GPU + XDNA2 NPU). + +--- + +## 1. Kernel and Firmware Requirements + +### Recommended Setup +- **Linux Kernel**: `>= 6.18.4` (e.g. `6.18.9+`). Kernels older than 6.18.4 exhibit stability bugs and crashes under sustained ROCm compute on gfx1151. +- **Linux Firmware**: `20260110` or later. +- **Avoid**: `linux-firmware-20251125` (breaks ROCm initialization on gfx1151). + +### Kernel Boot Parameters +Add the following parameters to your bootloader (e.g. GRUB or systemd-boot): + +```text +amdgpu.gttsize=65536 +``` +*(On 128 GiB unified memory systems, this ensures sufficient GTT allocation for large model weights and KV caches).* + +--- + +## 2. Device Permissions and User Groups + +Ensure your host user account belongs to the required hardware access groups: + +```bash +sudo usermod -aG video,render $USER +``` + +Verify device existence: +```bash +# GPU ROCm / KFD interface +ls -l /dev/kfd /dev/dri/renderD* + +# XDNA2 NPU interface +ls -l /dev/accel/accel* /dev/amdxdna 2>/dev/null || echo "NPU driver not loaded" +``` + +--- + +## 3. Ulimits (Locked Memory) + +Large unified memory buffers require unlimited memlock limits. In `/etc/security/limits.d/99-gufo.conf`: + +```text +* soft memlock unlimited +* hard memlock unlimited +``` + +--- + +## 4. Power Profiles + +For sustained LLM inference on Strix Halo laptops or mini PCs, switch power profiles to `throughput-performance`: + +```bash +sudo tuned-adm profile throughput-performance +``` +*(Or install the provided `systemd/gpu-workload-watch` service to automate this on GPU activity).* diff --git a/docs/quickstart.md b/docs/quickstart.md new file mode 100644 index 0000000..e289ed1 --- /dev/null +++ b/docs/quickstart.md @@ -0,0 +1,95 @@ +# Gufo Toolboxes Quick Start + +This guide gets you up and running with **Gufo Toolboxes** on AMD Ryzen AI Max ("Strix Halo") systems featuring the `gfx1151` RDNA 3.5 GPU and XDNA2 NPU. + +--- + +## 1. Prerequisites + +- **Host OS**: Fedora 42/43/44, Ubuntu 24.04+, Arch Linux, or openSUSE Tumbleweed. +- **Kernel**: Linux kernel >= `6.18.4` (older kernels have known gfx1151 stability issues). +- **Firmware**: Avoid `linux-firmware-20251125` due to known ROCm regressions. +- **Container Tooling**: + - Fedora: `toolbox` and `podman` + - Ubuntu / Debian / Arch: `distrobox` and `podman` + +--- + +## 2. Choosing a Toolbox + +| Toolbox | Image | Use Case | +| :--- | :--- | :--- | +| **`gufo-runtime`** | `ghcr.io/gufo-org/toolboxes/gufo-runtime:latest` | Lightweight production inference & model serving (`gufo serve`, `gufo prompt`, `gufo bench`) | +| **`gufo-dev`** | `ghcr.io/gufo-org/toolboxes/gufo-dev:latest` | Development, C++/HIP kernel tuning, and profiling (`rocprofv3`, Clang, CMake, Ninja) | + +--- + +## 3. Creating and Entering the Toolbox + +### Using `refresh-toolboxes.sh` (Recommended) + +The included `refresh-toolboxes.sh` script automatically detects your OS (`toolbox` or `distrobox`) and configures all Strix Halo GPU (`/dev/kfd`, `/dev/dri`) and NPU (`/dev/accel*`) passthroughs: + +```bash +# Pull & create gufo-runtime +./refresh-toolboxes.sh gufo-runtime + +# Or create the development toolbox +./refresh-toolboxes.sh gufo-dev + +# Or build the image locally from Nix without pulling from GHCR +./refresh-toolboxes.sh --local gufo-runtime +``` + +### Manual Creation + +#### Fedora (`toolbox`) +```bash +toolbox create gufo-runtime \ + --image ghcr.io/gufo-org/toolboxes/gufo-runtime:latest \ + -- --device /dev/dri --device /dev/kfd --device /dev/accel/accel0 \ + --group-add video --group-add render --group-add sudo \ + --security-opt seccomp=unconfined --ulimit memlock=-1 + +toolbox enter gufo-runtime +``` + +#### Ubuntu / Debian / Arch (`distrobox`) +```bash +distrobox create \ + --name gufo-runtime \ + --image ghcr.io/gufo-org/toolboxes/gufo-runtime:latest \ + --additional-flags "--device /dev/dri --device /dev/kfd --device /dev/accel/accel0 --group-add video --group-add render --group-add sudo --security-opt seccomp=unconfined --ulimit memlock=-1" + +distrobox enter gufo-runtime +``` + +--- + +## 4. Verifying Hardware Acceleration + +Inside the toolbox, run: + +```bash +# Verify GPU / NPU hardware diagnostics +gufo diagnose +``` + +--- + +## 5. Running Inference & Server + +### Interactive Prompt +```bash +gufo prompt --model /path/to/model.gguf -p "Explain quantum computing in simple terms." +``` + +### Benchmark +```bash +gufo bench --model /path/to/model.gguf -p 512 -n 128 +``` + +### Server +```bash +gufo serve --model /path/to/model.gguf --host 0.0.0.0 --port 8080 +``` diff --git a/docs/systemd.md b/docs/systemd.md new file mode 100644 index 0000000..61bc5d7 --- /dev/null +++ b/docs/systemd.md @@ -0,0 +1,56 @@ +# Systemd Integration for Gufo Server + +You can run `gufo serve` as a background systemd user service via Podman. + +--- + +## 1. Directory Setup + +Create the local models folder on your host: + +```bash +mkdir -p ~/.local/share/gufo/models +``` + +Place your model GGUF file inside: +```bash +cp /path/to/my-model.gguf ~/.local/share/gufo/models/model.gguf +``` + +--- + +## 2. Installing the Service + +Copy the systemd service file into your user services directory: + +```bash +mkdir -p ~/.config/systemd/user/ +cp systemd/gufo-serve.service ~/.config/systemd/user/ +``` + +Reload the systemd user daemon: +```bash +systemctl --user daemon-reload +``` + +--- + +## 3. Managing the Service + +```bash +# Start the server +systemctl --user start gufo-serve + +# Enable on boot (persistent login session) +systemctl --user enable gufo-serve +loginctl enable-linger $USER + +# Check status and logs +systemctl --user status gufo-serve +journalctl --user -u gufo-serve -f + +# Stop the server +systemctl --user stop gufo-serve +``` + +The server will be reachable on `http://127.0.0.1:8080`. diff --git a/flake.lock b/flake.lock new file mode 100644 index 0000000..2707da3 --- /dev/null +++ b/flake.lock @@ -0,0 +1,62 @@ +{ + "nodes": { + "gufo-engine": { + "inputs": { + "nixpkgs": "nixpkgs" + }, + "locked": { + "lastModified": 1787839541, + "narHash": "sha256-yHvZodZEC0v2hL9AQEaw4XFXzexMeMT7uT/yu69JK6w=", + "ref": "refs/heads/main", + "rev": "81ba5df82e2d57dd899f53c0f1aaf5e98fd8cbc9", + "revCount": 200, + "type": "git", + "url": "ssh://git@github.com/francescobozzo/strix-halo.cpp.git" + }, + "original": { + "type": "git", + "url": "ssh://git@github.com/francescobozzo/strix-halo.cpp.git" + } + }, + "nixpkgs": { + "locked": { + "lastModified": 1786384358, + "narHash": "sha256-RzPPiWeUtuvymnpuEWsdtzli5w4kjZs49FqEs3/1u+I=", + "owner": "NixOS", + "repo": "nixpkgs", + "rev": "2fcb964de67fcf60b43471c55d5d99e61a9ccb5a", + "type": "github" + }, + "original": { + "owner": "NixOS", + "ref": "nixos-unstable", + "repo": "nixpkgs", + "type": "github" + } + }, + "nixpkgs_2": { + "locked": { + "lastModified": 1787498568, + "narHash": "sha256-9i/VTdusq/+NM/tz+J1Re+ojkMB8MBf0QshnYfzHz30=", + "owner": "NixOS", + "repo": "nixpkgs", + "rev": "56c02bc00adcf003215cc4bd996d6efaf4cff188", + "type": "github" + }, + "original": { + "owner": "NixOS", + "ref": "nixos-unstable", + "repo": "nixpkgs", + "type": "github" + } + }, + "root": { + "inputs": { + "gufo-engine": "gufo-engine", + "nixpkgs": "nixpkgs_2" + } + } + }, + "root": "root", + "version": 7 +} diff --git a/flake.nix b/flake.nix new file mode 100644 index 0000000..fa40129 --- /dev/null +++ b/flake.nix @@ -0,0 +1,246 @@ +{ + description = "Gufo Toolboxes - Reproducible OCI toolboxes for AMD Strix Halo (gfx1151 GPU + XDNA2 NPU)"; + + inputs = { + nixpkgs.url = "github:NixOS/nixpkgs/nixos-unstable"; + gufo-engine = { + url = "git+ssh://git@github.com/francescobozzo/strix-halo.cpp.git"; + }; + }; + + outputs = + { + self, + nixpkgs, + gufo-engine, + }: + let + supportedSystems = [ "x86_64-linux" ]; + forAllSystems = nixpkgs.lib.genAttrs supportedSystems; + pkgs = forAllSystems (system: import nixpkgs { inherit system; }); + + # Common base packages needed for Toolbx and Distrobox compatibility + commonToolboxPkgs = + system: + let + p = pkgs.${system}; + in + [ + p.bashInteractive + p.coreutils + p.findutils + p.gnugrep + p.gnused + p.gawk + p.procps + p.shadow + p.sudo + p.util-linux + p.glibcLocales + p.cacert + p.radeontop + p.which + p.curl + p.jq + p.less + p.ncurses + ]; + + # Common /etc and /bin initialization setup for toolbx/distrobox + extraSetupCommands = + system: + let + p = pkgs.${system}; + in + '' + mkdir -p bin usr/bin usr/sbin usr/lib usr/lib64 tmp etc etc/pam.d var/tmp var/log + chmod 1777 tmp var/tmp + + # Standard symlinks + ln -sf /bin/bash bin/sh + ln -sf /bin/bash usr/bin/sh + ln -sf /bin/env usr/bin/env + + # Create gufo alias / compatibility wrapper if strix binary is present + if [ -f bin/strix ]; then + ln -sf strix bin/gufo + fi + + # Ensure etc is writable and clean any read-only links from packages + chmod -R u+w etc 2>/dev/null || true + rm -f etc/passwd etc/group etc/shadow etc/sudoers etc/os-release + + # Basic /etc files for toolbox user injection & pam + cat > etc/passwd << 'EOF' + root:x:0:0:root:/root:/bin/bash + nobody:x:65534:65534:Nobody:/:/bin/false + EOF + + cat > etc/group << 'EOF' + root:x:0: + video:x:44: + render:x:107: + wheel:x:10: + sudo:x:27: + users:x:100: + nogroup:x:65534: + EOF + + cat > etc/shadow << 'EOF' + root:!::0::::: + nobody:!::0::::: + EOF + chmod 0640 etc/shadow + + cat > etc/sudoers << 'EOF' + root ALL=(ALL:ALL) ALL + %wheel ALL=(ALL:ALL) NOPASSWD: ALL + %sudo ALL=(ALL:ALL) NOPASSWD: ALL + EOF + chmod 0440 etc/sudoers + + # pam.d config for user elevation + cat > etc/pam.d/other << 'EOF' + auth sufficient pam_permit.so + account sufficient pam_permit.so + password sufficient pam_permit.so + session sufficient pam_permit.so + EOF + + # os-release to identify as Gufo Toolbox (Toolbx requires ID/NAME) + cat > etc/os-release << 'EOF' + NAME="Gufo Toolbox" + ID=gufo + ID_LIKE="fedora" + VERSION_ID="1.0" + PRETTY_NAME="Gufo Strix Halo Toolbox (gfx1151 + XDNA2)" + HOME_URL="https://github.com/gufo-org/toolboxes" + SUPPORT_URL="https://github.com/gufo-org/toolboxes/issues" + BUG_REPORT_URL="https://github.com/gufo-org/toolboxes/issues" + EOF + ln -sf ../etc/os-release usr/lib/os-release + ''; + + commonEnv = system: [ + "PATH=/bin:/usr/bin:/usr/local/bin" + "LANG=en_US.UTF-8" + "LC_ALL=en_US.UTF-8" + "ROCM_PATH=${pkgs.${system}.rocmPackages.clr}" + "HIP_PLATFORM=amd" + ]; + in + { + packages = forAllSystems ( + system: + let + p = pkgs.${system}; + engine = gufo-engine.packages.${system}.default; + in + { + # 1. gufo-runtime: Lightweight serving & inference OCI image + gufo-runtime-image = p.dockerTools.buildLayeredImage { + name = "ghcr.io/gufo-org/toolboxes/gufo-runtime"; + tag = "latest"; + contents = [ engine ] ++ (commonToolboxPkgs system); + extraCommands = extraSetupCommands system; + config = { + Env = commonEnv system; + Cmd = [ "/bin/bash" ]; + }; + }; + + # Streamed variant for fast local loading via `nix run .#stream-gufo-runtime | podman load` + stream-gufo-runtime = p.dockerTools.streamLayeredImage { + name = "ghcr.io/gufo-org/toolboxes/gufo-runtime"; + tag = "latest"; + contents = [ engine ] ++ (commonToolboxPkgs system); + extraCommands = extraSetupCommands system; + config = { + Env = commonEnv system; + Cmd = [ "/bin/bash" ]; + }; + }; + + # 2. gufo-dev: Development, kernel tuning & profiling OCI image + gufo-dev-image = p.dockerTools.buildLayeredImage { + name = "ghcr.io/gufo-org/toolboxes/gufo-dev"; + tag = "latest"; + contents = [ + engine + p.git + p.cmake + p.ninja + p.gdb + p.clang-tools + p.rocmPackages.rocprofiler-sdk + p.rocmPackages.clr + p.rocmPackages.hipblas + p.rocmPackages.hipblaslt + p.rocmPackages.hipcub + p.rocmPackages.rocprim + p.rocmPackages.rocwmma + ] ++ (commonToolboxPkgs system); + extraCommands = extraSetupCommands system; + config = { + Env = commonEnv system ++ [ + "ROCM_PATH=${p.rocmPackages.clr}" + "STRIX_HIPCUB_ROOT=${p.rocmPackages.hipcub}" + "STRIX_ROCPRIM_ROOT=${p.rocmPackages.rocprim}" + "STRIX_ROCWMMA_ROOT=${p.rocmPackages.rocwmma}" + ]; + Cmd = [ "/bin/bash" ]; + }; + }; + + stream-gufo-dev = p.dockerTools.streamLayeredImage { + name = "ghcr.io/gufo-org/toolboxes/gufo-dev"; + tag = "latest"; + contents = [ + engine + p.git + p.cmake + p.ninja + p.gdb + p.clang-tools + p.rocmPackages.rocprofiler-sdk + p.rocmPackages.clr + p.rocmPackages.hipblas + p.rocmPackages.hipblaslt + p.rocmPackages.hipcub + p.rocmPackages.rocprim + p.rocmPackages.rocwmma + ] ++ (commonToolboxPkgs system); + extraCommands = extraSetupCommands system; + config = { + Env = commonEnv system ++ [ + "ROCM_PATH=${p.rocmPackages.clr}" + "STRIX_HIPCUB_ROOT=${p.rocmPackages.hipcub}" + "STRIX_ROCPRIM_ROOT=${p.rocmPackages.rocprim}" + "STRIX_ROCWMMA_ROOT=${p.rocmPackages.rocwmma}" + ]; + Cmd = [ "/bin/bash" ]; + }; + }; + + default = self.packages.${system}.gufo-runtime-image; + } + ); + + apps = forAllSystems ( + system: + let + p = pkgs.${system}; + in + { + stream-gufo-runtime = { + type = "app"; + program = "${self.packages.${system}.stream-gufo-runtime}"; + }; + stream-gufo-dev = { + type = "app"; + program = "${self.packages.${system}.stream-gufo-dev}"; + }; + } + ); + }; +} diff --git a/refresh-toolboxes.sh b/refresh-toolboxes.sh new file mode 100755 index 0000000..840a714 --- /dev/null +++ b/refresh-toolboxes.sh @@ -0,0 +1,196 @@ +#!/usr/bin/env bash +set -euo pipefail + +# Gufo Toolboxes Refresh & Provisioning Script +# Target: AMD Ryzen AI Max "Strix Halo" (gfx1151 GPU + XDNA2 NPU) + +declare -A IMAGES +IMAGES["gufo-runtime"]="ghcr.io/gufo-org/toolboxes/gufo-runtime:latest" +IMAGES["gufo-dev"]="ghcr.io/gufo-org/toolboxes/gufo-dev:latest" + +FORCE_RECREATE=false + +usage() { + cat << EOF +Usage: $0 [options] [all|gufo-runtime|gufo-dev] + +A helper script to create and manage Gufo Toolboxes on AMD Strix Halo. +Requires only Podman/Docker and Toolbox/Distrobox (no Nix required). + +Options: + -f, --force Remove existing toolbox before recreating + -h, --help Show this help message + +Available Toolboxes: + - gufo-runtime Lightweight inference runtime (gufo serve, prompt, bench, diagnose) + - gufo-dev Full development, profiling (rocprofv3), and kernel tuning environment + - all Create/refresh both toolboxes + +Examples: + $0 gufo-runtime + $0 gufo-dev + $0 -f all +EOF + exit 1 +} + +# Parse options +POSITIONAL=() +while [[ $# -gt 0 ]]; do + case "$1" in + -f|--force) + FORCE_RECREATE=true + shift + ;; + -h|--help) + usage + ;; + *) + POSITIONAL+=("$1") + shift + ;; + esac +done +set -- "${POSITIONAL[@]}" + +if [[ $# -lt 1 ]]; then + usage +fi + +TARGET="$1" +SELECTED_TOOLBOXES=() + +if [[ "$TARGET" == "all" ]]; then + SELECTED_TOOLBOXES=("gufo-runtime" "gufo-dev") +elif [[ -v IMAGES["$TARGET"] ]]; then + SELECTED_TOOLBOXES=("$TARGET") +else + echo "Error: Unknown target '$TARGET'" >&2 + usage +fi + +# Detect container runtime (podman vs docker) +CONTAINER_ENGINE="" +if command -v podman >/dev/null 2>&1; then + CONTAINER_ENGINE="podman" +elif command -v docker >/dev/null 2>&1; then + CONTAINER_ENGINE="docker" +else + echo "Error: Neither 'podman' nor 'docker' is installed." >&2 + exit 1 +fi + +# Detect toolbox CLI (toolbox vs distrobox vs standalone container engine) +TOOLBOX_CMD="" +if [[ -f /etc/os-release ]]; then + . /etc/os-release + if [[ "${ID:-}" =~ ^(ubuntu|debian)$ ]] && command -v distrobox >/dev/null 2>&1; then + TOOLBOX_CMD="distrobox" + fi +fi + +if [[ -z "$TOOLBOX_CMD" ]]; then + if command -v toolbox >/dev/null 2>&1; then + TOOLBOX_CMD="toolbox" + elif command -v distrobox >/dev/null 2>&1; then + TOOLBOX_CMD="distrobox" + else + TOOLBOX_CMD="$CONTAINER_ENGINE" + echo "ℹī¸ Neither 'toolbox' nor 'distrobox' found; using '$CONTAINER_ENGINE' directly." + fi +fi + +# Detect Strix Halo Devices +PASSTHROUGH_ARGS=() + +# 1. GPU (gfx1151) +if [[ -e /dev/kfd ]]; then + PASSTHROUGH_ARGS+=("--device" "/dev/kfd") +else + echo "⚠ī¸ Warning: /dev/kfd not found. ROCm compute might fail without proper kernel driver." >&2 +fi + +if [[ -d /dev/dri ]]; then + PASSTHROUGH_ARGS+=("--device" "/dev/dri") +fi + +# 2. XDNA2 NPU (/dev/accel* or /dev/amdxdna) +NPU_FOUND=false +for accel in /dev/accel/accel* /dev/amdxdna; do + if [[ -e "$accel" ]]; then + PASSTHROUGH_ARGS+=("--device" "$accel") + NPU_FOUND=true + fi +done +if [[ "$NPU_FOUND" = true ]]; then + echo "⚡ XDNA2 NPU device detected and mapped." +else + echo "ℹī¸ Note: No XDNA2 NPU device (/dev/accel/accel* or /dev/amdxdna) detected." +fi + +# 3. Optional RDMA / InfiniBand +if [[ -d /dev/infiniband ]]; then + echo "⚡ InfiniBand/RoCE devices detected." + PASSTHROUGH_ARGS+=("--device" "/dev/infiniband" "--group-add" "rdma") +fi + +# Permissions and capabilities +PASSTHROUGH_ARGS+=( + "--group-add" "video" + "--group-add" "render" + "--security-opt" "seccomp=unconfined" + "--ulimit" "memlock=-1" +) + +# Process each selected toolbox +for TB in "${SELECTED_TOOLBOXES[@]}"; do + IMG="${IMAGES[$TB]}" + + echo "==================================================" + echo "đŸ“Ļ Processing Toolbox: $TB" + echo "==================================================" + + echo "âŦ‡ī¸ Pulling image $IMG..." + "$CONTAINER_ENGINE" pull "$IMG" || { + echo "⚠ī¸ Failed to pull $IMG. Please check your network connection or image tag." >&2 + exit 1 + } + + # Check if container already exists + CONTAINER_EXISTS=false + if "$CONTAINER_ENGINE" container exists "$TB" 2>/dev/null || "$CONTAINER_ENGINE" inspect "$TB" >/dev/null 2>&1; then + CONTAINER_EXISTS=true + fi + + if [[ "$CONTAINER_EXISTS" = true ]]; then + if [[ "$FORCE_RECREATE" = true ]]; then + echo "🗑ī¸ Removing existing container '$TB'..." + if [[ "$TOOLBOX_CMD" == "distrobox" ]]; then + distrobox rm -f "$TB" + else + toolbox rm -f "$TB" + fi + else + echo "ℹī¸ Toolbox '$TB' already exists. Use -f or --force to recreate." + continue + fi + fi + + echo "🚀 Creating toolbox '$TB' using $TOOLBOX_CMD..." + if [[ "$TOOLBOX_CMD" == "distrobox" ]]; then + distrobox create \ + --name "$TB" \ + --image "$IMG" \ + --additional-flags "${PASSTHROUGH_ARGS[*]}" + else + toolbox create "$TB" \ + --image "$IMG" \ + -- "${PASSTHROUGH_ARGS[@]}" + fi + + echo "✅ Successfully created '$TB'!" + echo "👉 Enter with: $TOOLBOX_CMD enter $TB" +done + +echo "" +echo "🎉 All requested toolboxes are ready." diff --git a/systemd/gpu-workload-watch/gpu-workload-watch b/systemd/gpu-workload-watch/gpu-workload-watch new file mode 100755 index 0000000..101037a --- /dev/null +++ b/systemd/gpu-workload-watch/gpu-workload-watch @@ -0,0 +1,92 @@ +#!/usr/bin/env bash + +set -u + +export PATH="/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin" + +load_threshold=20 +idle_threshold=10 +load_seconds=2 +idle_seconds=60 +tuned_command_timeout_seconds=20 +tuned_reconcile_seconds=15 +tuned_restart_after_failures=2 +ectool_command_timeout_seconds=10 +fan_reconcile_seconds=15 + +ectool_bin="$(command -v ectool || true)" +tuned_adm_bin="$(command -v tuned-adm || true)" +systemctl_bin="$(command -v systemctl || true)" +timeout_bin="$(command -v timeout || true)" +systemd_notify_bin="$(command -v systemd-notify || true)" + +fan_mode="unknown" +fan_reconcile_count=0 +tuned_mode="unknown" +tuned_failures=0 +tuned_reconcile_count=0 +load_count=0 +idle_count=0 +no_workload_count=0 +gpu_busy_file="" +tuned_worker_pid="" + +for candidate in /sys/class/drm/card*/device/gpu_busy_percent; do + if [[ -r "${candidate}" ]] && [[ "$(< "${candidate%/gpu_busy_percent}/vendor")" == "0x1002" ]]; then + gpu_busy_file="${candidate}" + break + fi +done + +if [[ -z "${gpu_busy_file}" ]]; then + echo "No AMDGPU gpu_busy_percent counter found" >&2 + exit 1 +fi + +gpu_workload_is_running() { + local proc comm lower + for proc in /proc/[0-9]*; do + [[ -r "${proc}/comm" ]] || continue + read -r comm < "${proc}/comm" 2>/dev/null || continue + lower="${comm,,}" + case "${lower}" in + gufo*|strix*|llama-*|ds4-*|*hipfire*|*vllm*) + return 0 + ;; + esac + done + return 1 +} + +apply_tuned_profile() { + local profile="$1" + if [[ -n "${tuned_adm_bin}" ]]; then + "${timeout_bin}" "${tuned_command_timeout_seconds}" "${tuned_adm_bin}" profile "${profile}" >/dev/null 2>&1 || true + fi +} + +echo "Starting Gufo GPU Workload Watcher on ${gpu_busy_file}..." + +while true; do + read -r busy < "${gpu_busy_file}" 2>/dev/null || busy=0 + + if gpu_workload_is_running && (( busy >= load_threshold )); then + (( load_count++ )) + idle_count=0 + if (( load_count >= load_seconds )) && [[ "${tuned_mode}" != "throughput-performance" ]]; then + echo "GPU active (${busy}%). Switching to throughput-performance profile." + apply_tuned_profile "throughput-performance" + tuned_mode="throughput-performance" + fi + else + (( idle_count++ )) + load_count=0 + if (( idle_count >= idle_seconds )) && [[ "${tuned_mode}" != "balanced" ]]; then + echo "GPU idle (${busy}%). Switching to balanced profile." + apply_tuned_profile "balanced" + tuned_mode="balanced" + fi + fi + + sleep 1 +done diff --git a/systemd/gpu-workload-watch/gpu-workload-watch.service b/systemd/gpu-workload-watch/gpu-workload-watch.service new file mode 100644 index 0000000..14a995e --- /dev/null +++ b/systemd/gpu-workload-watch/gpu-workload-watch.service @@ -0,0 +1,14 @@ +[Unit] +Description=Manage power profile for Gufo GPU inference workloads +Wants=tuned.service +After=multi-user.target tuned.service + +[Service] +Type=simple +ExecStart=/usr/local/sbin/gpu-workload-watch +Restart=always +RestartSec=2s +LimitCORE=0 + +[Install] +WantedBy=multi-user.target diff --git a/systemd/gufo-serve.service b/systemd/gufo-serve.service new file mode 100644 index 0000000..8caff15 --- /dev/null +++ b/systemd/gufo-serve.service @@ -0,0 +1,29 @@ +[Unit] +Description=Gufo Inference Server (AMD Strix Halo gfx1151 + XDNA2) +After=network.target local-fs.target +Documentation=https://github.com/gufo-org/toolboxes + +[Service] +Type=simple +Environment=HOST=127.0.0.1 +Environment=PORT=8080 +Environment=MODEL_PATH=%h/.local/share/gufo/models/model.gguf +Environment=THREADS=8 + +# Run inside the gufo-runtime container using podman +ExecStart=/usr/bin/podman run --rm --name gufo-serve \ + --device /dev/kfd --device /dev/dri \ + --device /dev/accel/accel0 \ + --group-add video --group-add render \ + --security-opt seccomp=unconfined --ulimit memlock=-1 \ + -v %h/.local/share/gufo/models:/models:ro,z \ + -p ${PORT}:${PORT} \ + ghcr.io/gufo-org/toolboxes/gufo-runtime:latest \ + gufo serve --model /models/model.gguf --host ${HOST} --port ${PORT} --threads ${THREADS} + +ExecStop=/usr/bin/podman stop -t 10 gufo-serve +Restart=on-failure +RestartSec=5s + +[Install] +WantedBy=default.target