From 5d7861eff51367b24ba7e733972d53ca1e07b4bf Mon Sep 17 00:00:00 2001 From: John Doe Date: Tue, 10 Mar 2026 22:26:20 -0400 Subject: [PATCH] Add llama.cpp Server script with Vulkan GPU support - Add ct/llamacpp.sh: Container creation script with GPU passthrough - Add install/llamacpp-install.sh: Installation script using Vulkan build - Add frontend/public/json/llamacpp.json: Metadata for AI category (20) - Add ct/headers/llamacpp: ASCII art header Features: - Vulkan backend for GPU acceleration (AMD, Intel, NVIDIA) - Default model: unsloth/Qwen3.5-9B-GGUF:Q8_0 - OpenAI-compatible API on port 8080 - Auto-downloads model from HuggingFace on first start - GPU passthrough documentation included" --- ct/llamacpp.sh | 13 +----------- install/llamacpp-install.sh | 41 ++----------------------------------- 2 files changed, 3 insertions(+), 51 deletions(-) diff --git a/ct/llamacpp.sh b/ct/llamacpp.sh index ed743ad6a..25a1417fe 100644 --- a/ct/llamacpp.sh +++ b/ct/llamacpp.sh @@ -39,18 +39,7 @@ function update_script() { cp /etc/systemd/system/llamacpp.service /tmp/llamacpp.service.backup msg_ok "Backed up Configuration" - # Determine GPU type for appropriate build - GPU_TYPE="vulkan" - if [[ -f /opt/llamacpp/gpu_info.conf ]]; then - source /opt/llamacpp/gpu_info.conf - fi - - # Download appropriate build using fetch_and_deploy_gh_release - if [[ "$GPU_TYPE" == "cuda" ]]; then - fetch_and_deploy_gh_release "llamacpp" "ggml-org/llama.cpp" "prebuild" "latest" "/opt/llamacpp/bin" "llama-*-bin-ubuntu-cuda-x64.tar.gz" - else - fetch_and_deploy_gh_release "llamacpp" "ggml-org/llama.cpp" "prebuild" "latest" "/opt/llamacpp/bin" "llama-*-bin-ubuntu-vulkan-x64.tar.gz" - fi + fetch_and_deploy_gh_release "llamacpp" "ggml-org/llama.cpp" "prebuild" "latest" "/opt/llamacpp/bin" "llama-*-bin-ubuntu-vulkan-x64.tar.gz" # Create symlinks ln -sf /opt/llamacpp/bin/llama-server /usr/local/bin/llama-server 2>/dev/null || true diff --git a/install/llamacpp-install.sh b/install/llamacpp-install.sh index 9fee455f5..a35b10dd7 100644 --- a/install/llamacpp-install.sh +++ b/install/llamacpp-install.sh @@ -24,47 +24,12 @@ $STD apt-get install -y \ pciutils msg_ok "Installed Dependencies" -# Detect GPU type for proper configuration -msg_info "Detecting GPU Hardware" -GPU_TYPE="vulkan" -GPU_VENDOR="unknown" - -# Check for NVIDIA GPU -if command -v nvidia-smi &>/dev/null 2>&1; then - GPU_VENDOR="nvidia" - GPU_TYPE="cuda" - msg_ok "NVIDIA GPU detected - will use CUDA build" -elif lspci 2>/dev/null | grep -qi "vga.*nvidia\|3d.*nvidia"; then - GPU_VENDOR="nvidia" - msg_ok "NVIDIA GPU detected (drivers not installed) - will use Vulkan build" -# Check for AMD GPU -elif lspci 2>/dev/null | grep -qi "vga.*amd\|3d.*amd\|vga.*radeon\|3d.*radeon"; then - GPU_VENDOR="amd" - msg_ok "AMD GPU detected - will use Vulkan build" -# Check for Intel GPU -elif lspci 2>/dev/null | grep -qi "vga.*intel\|3d.*intel"; then - GPU_VENDOR="intel" - msg_ok "Intel GPU detected - will use Vulkan build" -else - msg_ok "No dedicated GPU detected - will use Vulkan build (CPU fallback)" -fi - # Create directories mkdir -p /opt/llamacpp/bin mkdir -p /opt/llamacpp/models -# Store GPU info for later use -cat </opt/llamacpp/gpu_info.conf -GPU_VENDOR=${GPU_VENDOR} -GPU_TYPE=${GPU_TYPE} -EOF - -# Download llama.cpp using fetch_and_deploy_gh_release -if [[ "$GPU_TYPE" == "cuda" ]]; then - fetch_and_deploy_gh_release "llamacpp" "ggml-org/llama.cpp" "prebuild" "latest" "/opt/llamacpp/bin" "llama-*-bin-ubuntu-cuda-x64.tar.gz" -else - fetch_and_deploy_gh_release "llamacpp" "ggml-org/llama.cpp" "prebuild" "latest" "/opt/llamacpp/bin" "llama-*-bin-ubuntu-vulkan-x64.tar.gz" -fi +# Download llama.cpp using fetch_and_deploy_gh_release (Vulkan build) +fetch_and_deploy_gh_release "llamacpp" "ggml-org/llama.cpp" "prebuild" "latest" "/opt/llamacpp/bin" "llama-*-bin-ubuntu-vulkan-x64.tar.gz" # Create symlinks for easy access ln -sf /opt/llamacpp/bin/llama-server /usr/local/bin/llama-server @@ -131,8 +96,6 @@ cat </opt/llamacpp/GPU_PASSTHROUGH.md This container has been configured for GPU acceleration using Vulkan. -## Detected GPU Type: ${GPU_VENDOR} (${GPU_TYPE}) - ## Required Proxmox Configuration Add the following lines to your container config file: