Add scripts for managing and testing LLM backends in Proxmox CT

2026-10-11 21:29:58 +00:00 · 2026-03-01 22:40:54 -05:00
parent 3337a49370
commit beda020cf3
3 changed files with 395 additions and 0 deletions
@@ -12,6 +12,10 @@ Toolkit for building an **unprivileged Ubuntu 24.04 LXC** on Proxmox and install
 	- Registers ROCm 7.2 `noble` apt repos and installs a chosen ROCm meta package.
 - `scripts/create_pytorch_venv_in_ct.sh`
 	- Creates a Python 3.12 venv in the CT and installs AMD ROCm 7.2 PyTorch wheels.
+- `scripts/manage_llm_backends_in_ct.sh`
+	- Installs/updates vLLM, Ollama, and llama.cpp and configures systemd services.
+- `scripts/test_llm_backends_in_ct.sh`
+	- Runs health checks for vLLM, Ollama, and llama.cpp services/endpoints.

 ## Requirements

@@ -65,6 +69,48 @@ pct exec 120 -- bash -lc '/opt/rocm/bin/rocm-smi || true'
 sudo bash ./scripts/create_pytorch_venv_in_ct.sh --ctid 120 --venv-path /opt/rocm-pytorch-venv
 ```

+6) Install LLM backends + services in CT:
+
+```bash
+sudo bash ./scripts/manage_llm_backends_in_ct.sh --ctid 120 --action install --backend all --venv-path /opt/rocm-pytorch-venv
+```
+
+Update later:
+
+```bash
+sudo bash ./scripts/manage_llm_backends_in_ct.sh --ctid 120 --action update --backend all --venv-path /opt/rocm-pytorch-venv
+```
+
+Backend-specific examples:
+
+```bash
+sudo bash ./scripts/manage_llm_backends_in_ct.sh --ctid 120 --action install --backend vllm --venv-path /opt/rocm-pytorch-venv
+sudo bash ./scripts/manage_llm_backends_in_ct.sh --ctid 120 --action install --backend ollama
+sudo bash ./scripts/manage_llm_backends_in_ct.sh --ctid 120 --action install --backend llama-cpp
+```
+
+7) Test backend services/endpoints:
+
+```bash
+sudo bash ./scripts/test_llm_backends_in_ct.sh --ctid 120 --backend all
+```
+
+Verbose diagnostics on failures:
+
+```bash
+sudo bash ./scripts/test_llm_backends_in_ct.sh --ctid 120 --backend all --verbose
+```
+
+Test one backend only:
+
+```bash
+sudo bash ./scripts/test_llm_backends_in_ct.sh --ctid 120 --backend vllm
+sudo bash ./scripts/test_llm_backends_in_ct.sh --ctid 120 --backend ollama
+sudo bash ./scripts/test_llm_backends_in_ct.sh --ctid 120 --backend llama-cpp
+```
+
+
+
 ## ROCm package options

 `install_rocm_in_ct.sh` defaults to `rocm`, but you can pass alternatives, for example:
@@ -114,3 +160,6 @@ Reference:

 - https://rocm.docs.amd.com/projects/install-on-linux/en/latest/install/install-methods/package-manager/package-manager-ubuntu.html
 - https://rocm.docs.amd.com/projects/radeon-ryzen/en/latest/docs/install/installryz/native_linux/install-pytorch.html
+- https://docs.vllm.ai/en/latest/getting_started/installation/gpu/
+- https://docs.ollama.com/linux
+- https://github.com/ggml-org/llama.cpp
@@ -0,0 +1,211 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+usage() {
+  cat <<'EOF'
+Install or update LLM backends (vLLM, Ollama, llama.cpp) and manage systemd services in a Proxmox CT.
+
+Usage:
+  sudo ./scripts/manage_llm_backends_in_ct.sh \
+    --ctid 120 \
+    --action install \
+    --backend all
+
+Options:
+  --ctid <id>              Proxmox CTID (required)
+  --action <install|update>
+                           Action to perform (default: install)
+  --backend <vllm|ollama|llama-cpp|all>
+                           Backend target (default: all)
+  --venv-path <path>       Python venv path for vLLM (default: /opt/rocm-pytorch-venv)
+  --llama-dir <path>       llama.cpp source dir (default: /opt/llama.cpp)
+
+Notes:
+- vLLM is installed/updated into the specified ROCm PyTorch venv.
+- Ollama follows official Linux installer flow.
+- llama.cpp is built with HIP support (`-DGGML_HIP=ON`) for ROCm.
+EOF
+}
+
+require_root() {
+  if [[ "${EUID}" -ne 0 ]]; then
+    echo "This script must be run as root." >&2
+    exit 1
+  fi
+}
+
+CTID=""
+ACTION="install"
+BACKEND="all"
+VENV_PATH="/opt/rocm-pytorch-venv"
+LLAMA_DIR="/opt/llama.cpp"
+
+while [[ $# -gt 0 ]]; do
+  case "$1" in
+    --ctid) CTID="$2"; shift 2 ;;
+    --action) ACTION="$2"; shift 2 ;;
+    --backend) BACKEND="$2"; shift 2 ;;
+    --venv-path) VENV_PATH="$2"; shift 2 ;;
+    --llama-dir) LLAMA_DIR="$2"; shift 2 ;;
+    -h|--help) usage; exit 0 ;;
+    *)
+      echo "Unknown argument: $1" >&2
+      usage
+      exit 1
+      ;;
+  esac
+done
+
+if [[ -z "${CTID}" ]]; then
+  echo "--ctid is required." >&2
+  usage
+  exit 1
+fi
+
+if [[ "${ACTION}" != "install" && "${ACTION}" != "update" ]]; then
+  echo "--action must be install or update" >&2
+  exit 1
+fi
+
+case "${BACKEND}" in
+  vllm|ollama|llama-cpp|all) ;;
+  *)
+    echo "--backend must be one of: vllm, ollama, llama-cpp, all" >&2
+    exit 1
+    ;;
+esac
+
+require_root
+
+if ! pct status "${CTID}" >/dev/null 2>&1; then
+  echo "Container ${CTID} does not exist." >&2
+  exit 1
+fi
+
+if ! pct status "${CTID}" | grep -q "status: running"; then
+  echo "Starting CT ${CTID}..."
+  pct start "${CTID}"
+fi
+
+echo "Running ${ACTION} for ${BACKEND} in CT ${CTID}..."
+pct exec "${CTID}" -- env ACTION="${ACTION}" BACKEND="${BACKEND}" VENV_PATH="${VENV_PATH}" LLAMA_DIR="${LLAMA_DIR}" bash -s <<'IN_CT'
+set -euo pipefail
+export DEBIAN_FRONTEND=noninteractive
+
+apt update
+apt install -y \
+  ca-certificates \
+  curl \
+  git \
+  build-essential \
+  cmake \
+  ninja-build \
+  pkg-config \
+  python3 \
+  python3-pip
+
+install_or_update_vllm() {
+  if [[ ! -x "$VENV_PATH/bin/python" ]]; then
+    echo "vLLM requires existing venv at $VENV_PATH. Create it first (create_pytorch_venv_in_ct.sh)." >&2
+    exit 1
+  fi
+
+  "$VENV_PATH/bin/python" -m pip install --upgrade pip setuptools wheel
+  "$VENV_PATH/bin/python" -m pip install --upgrade vllm
+
+  cat >/etc/default/vllm <<'EOF'
+VLLM_MODEL=/opt/models/your-model
+VLLM_PORT=8000
+EOF
+
+  cat >/etc/systemd/system/vllm.service <<EOF
+[Unit]
+Description=vLLM OpenAI-Compatible API Server
+After=network-online.target
+Wants=network-online.target
+
+[Service]
+Type=simple
+User=root
+EnvironmentFile=-/etc/default/vllm
+ExecStart=/bin/bash -lc '$VENV_PATH/bin/python -m vllm.entrypoints.openai.api_server --host 0.0.0.0 --port \${VLLM_PORT:-8000} --model "\${VLLM_MODEL}"'
+Restart=always
+RestartSec=5
+
+[Install]
+WantedBy=multi-user.target
+EOF
+
+  systemctl daemon-reload
+  systemctl enable --now vllm
+}
+
+install_or_update_ollama() {
+  curl -fsSL https://ollama.com/install.sh | bash
+  systemctl enable --now ollama || true
+}
+
+install_or_update_llama_cpp() {
+  if [[ -d "$LLAMA_DIR/.git" ]]; then
+    git -C "$LLAMA_DIR" pull --ff-only
+  else
+    rm -rf "$LLAMA_DIR"
+    git clone https://github.com/ggml-org/llama.cpp "$LLAMA_DIR"
+  fi
+
+  cmake -S "$LLAMA_DIR" -B "$LLAMA_DIR/build" -G Ninja -DCMAKE_BUILD_TYPE=Release -DGGML_HIP=ON
+  cmake --build "$LLAMA_DIR/build" -j"$(nproc)"
+
+  cat >/etc/default/llama-cpp <<'EOF'
+LLAMA_MODEL=/opt/models/your-model.gguf
+LLAMA_PORT=8080
+LLAMA_CTX=4096
+EOF
+
+  cat >/etc/systemd/system/llama-cpp.service <<EOF
+[Unit]
+Description=llama.cpp HTTP Server
+After=network-online.target
+Wants=network-online.target
+
+[Service]
+Type=simple
+User=root
+EnvironmentFile=-/etc/default/llama-cpp
+ExecStart=/bin/bash -lc '$LLAMA_DIR/build/bin/llama-server -m "\${LLAMA_MODEL}" -c \${LLAMA_CTX:-4096} --host 0.0.0.0 --port \${LLAMA_PORT:-8080}'
+Restart=always
+RestartSec=5
+
+[Install]
+WantedBy=multi-user.target
+EOF
+
+  systemctl daemon-reload
+  systemctl enable --now llama-cpp
+}
+
+case "$BACKEND" in
+  vllm)
+    install_or_update_vllm
+    ;;
+  ollama)
+    install_or_update_ollama
+    ;;
+  llama-cpp)
+    install_or_update_llama_cpp
+    ;;
+  all)
+    install_or_update_vllm
+    install_or_update_ollama
+    install_or_update_llama_cpp
+    ;;
+esac
+
+systemctl daemon-reload
+IN_CT
+
+echo "Done."
+echo "Service status commands:"
+echo "  pct exec ${CTID} -- systemctl status vllm --no-pager"
+echo "  pct exec ${CTID} -- systemctl status ollama --no-pager"
+echo "  pct exec ${CTID} -- systemctl status llama-cpp --no-pager"
@@ -0,0 +1,135 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+usage() {
+  cat <<'EOF'
+Run health checks for vLLM, Ollama, and llama.cpp services inside a Proxmox CT.
+
+Usage:
+  sudo ./scripts/test_llm_backends_in_ct.sh --ctid 120 [--backend all] [--verbose]
+
+Options:
+  --ctid <id>                Proxmox CTID (required)
+  --backend <vllm|ollama|llama-cpp|all>
+                             Backend to test (default: all)
+  --verbose                  Print service diagnostics on failures
+EOF
+}
+
+require_root() {
+  if [[ "${EUID}" -ne 0 ]]; then
+    echo "This script must be run as root." >&2
+    exit 1
+  fi
+}
+
+CTID=""
+BACKEND="all"
+VERBOSE="no"
+
+while [[ $# -gt 0 ]]; do
+  case "$1" in
+    --ctid) CTID="$2"; shift 2 ;;
+    --backend) BACKEND="$2"; shift 2 ;;
+    --verbose) VERBOSE="yes"; shift 1 ;;
+    -h|--help) usage; exit 0 ;;
+    *)
+      echo "Unknown argument: $1" >&2
+      usage
+      exit 1
+      ;;
+  esac
+done
+
+if [[ -z "${CTID}" ]]; then
+  echo "--ctid is required." >&2
+  usage
+  exit 1
+fi
+
+case "${BACKEND}" in
+  vllm|ollama|llama-cpp|all) ;;
+  *)
+    echo "--backend must be one of: vllm, ollama, llama-cpp, all" >&2
+    exit 1
+    ;;
+esac
+
+require_root
+
+if ! pct status "${CTID}" >/dev/null 2>&1; then
+  echo "Container ${CTID} does not exist." >&2
+  exit 1
+fi
+
+if ! pct status "${CTID}" | grep -q "status: running"; then
+  echo "Starting CT ${CTID}..."
+  pct start "${CTID}"
+fi
+
+echo "Testing ${BACKEND} backend(s) in CT ${CTID} (verbose=${VERBOSE})..."
+
+pct exec "${CTID}" -- env BACKEND="${BACKEND}" VERBOSE="${VERBOSE}" bash -s <<'IN_CT'
+set -euo pipefail
+
+if ! command -v curl >/dev/null 2>&1; then
+  apt update
+  apt install -y curl
+fi
+
+PASS=0
+FAIL=0
+
+run_check() {
+  local name="$1"
+  local cmd="$2"
+  local service="${3:-}"
+  if eval "$cmd"; then
+    echo "[PASS] $name"
+    PASS=$((PASS+1))
+  else
+    echo "[FAIL] $name"
+    if [[ "$VERBOSE" == "yes" && -n "$service" ]]; then
+      echo "--- ${service}: systemctl status ---"
+      systemctl status "$service" --no-pager || true
+      echo "--- ${service}: journalctl (last 80 lines) ---"
+      journalctl -u "$service" -n 80 --no-pager || true
+    fi
+    FAIL=$((FAIL+1))
+  fi
+}
+
+check_vllm() {
+  run_check "vLLM systemd active" "systemctl is-active --quiet vllm" "vllm"
+  run_check "vLLM HTTP health" "curl -fsS --max-time 5 http://127.0.0.1:8000/health >/dev/null" "vllm"
+}
+
+check_ollama() {
+  run_check "Ollama systemd active" "systemctl is-active --quiet ollama" "ollama"
+  run_check "Ollama version" "ollama --version >/dev/null" "ollama"
+  run_check "Ollama tags endpoint" "curl -fsS --max-time 5 http://127.0.0.1:11434/api/tags >/dev/null" "ollama"
+}
+
+check_llama_cpp() {
+  run_check "llama.cpp systemd active" "systemctl is-active --quiet llama-cpp" "llama-cpp"
+  run_check "llama.cpp health" "curl -fsS --max-time 5 http://127.0.0.1:8080/health >/dev/null" "llama-cpp"
+}
+
+case "$BACKEND" in
+  vllm) check_vllm ;;
+  ollama) check_ollama ;;
+  llama-cpp) check_llama_cpp ;;
+  all)
+    check_vllm
+    check_ollama
+    check_llama_cpp
+    ;;
+esac
+
+echo "Summary: pass=$PASS fail=$FAIL"
+if [[ "$FAIL" -gt 0 ]]; then
+  exit 1
+fi
+IN_CT
+
+echo "Done."