From 96ce1215df8ec23fea0fb20be11d6c8d3dbae7bd Mon Sep 17 00:00:00 2001 From: John Doe Date: Sun, 1 Mar 2026 23:28:13 -0500 Subject: [PATCH] Add preset options and selection guide for Ollama memory profile script --- README.md | 18 ++++++ docs/OLLAMA_ROCM_TROUBLESHOOTING.md | 18 ++++++ scripts/set_ollama_memory_profile_in_ct.sh | 74 ++++++++++++++++++++-- 3 files changed, 105 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index 77afd81..ddefae5 100644 --- a/README.md +++ b/README.md @@ -133,6 +133,24 @@ sudo bash ./scripts/test_llm_backends_in_ct.sh --ctid 120 --backend llama-cpp sudo bash ./scripts/set_ollama_memory_profile_in_ct.sh --ctid 120 ``` +Use built-in presets: + +```bash +sudo bash ./scripts/set_ollama_memory_profile_in_ct.sh --ctid 120 --preset safe +sudo bash ./scripts/set_ollama_memory_profile_in_ct.sh --ctid 120 --preset balanced +sudo bash ./scripts/set_ollama_memory_profile_in_ct.sh --ctid 120 --preset max +``` + +Preset selection quick guide: + +| Model size (typical) | Suggested preset | Notes | +|---|---|---| +| 7B to 14B | `max` | Highest throughput, more aggressive memory/concurrency. | +| 20B to 32B | `balanced` | Best first choice for stability on large models. | +| 30B+ with load failures/OOM | `safe` | Use when `ROCm error: out of memory` appears. | + +If a model fails to load, step down from `max` → `balanced` → `safe` before changing manual values. + Custom values example: ```bash diff --git a/docs/OLLAMA_ROCM_TROUBLESHOOTING.md b/docs/OLLAMA_ROCM_TROUBLESHOOTING.md index d96b5cd..ac3986f 100644 --- a/docs/OLLAMA_ROCM_TROUBLESHOOTING.md +++ b/docs/OLLAMA_ROCM_TROUBLESHOOTING.md @@ -43,6 +43,24 @@ One-command helper from the Proxmox host: sudo bash ./scripts/set_ollama_memory_profile_in_ct.sh --ctid ``` +Preset examples: + +```bash +sudo bash ./scripts/set_ollama_memory_profile_in_ct.sh --ctid --preset safe +sudo bash ./scripts/set_ollama_memory_profile_in_ct.sh --ctid --preset balanced +sudo bash ./scripts/set_ollama_memory_profile_in_ct.sh --ctid --preset max +``` + +Preset selection quick guide: + +| Model size (typical) | Suggested preset | Notes | +|---|---|---| +| 7B to 14B | `max` | Highest throughput, more aggressive memory/concurrency. | +| 20B to 32B | `balanced` | Best first choice for stability on large models. | +| 30B+ with load failures/OOM | `safe` | Use when `ROCm error: out of memory` appears. | + +If a model fails to load, step down from `max` → `balanced` → `safe` before manual tuning. + Manual method: Create a memory-tuning drop-in: diff --git a/scripts/set_ollama_memory_profile_in_ct.sh b/scripts/set_ollama_memory_profile_in_ct.sh index e1d0264..7b09b36 100644 --- a/scripts/set_ollama_memory_profile_in_ct.sh +++ b/scripts/set_ollama_memory_profile_in_ct.sh @@ -6,14 +6,20 @@ usage() { Apply a safe Ollama memory profile inside a Proxmox CT (ROCm/LXC friendly). Usage: - sudo ./scripts/set_ollama_memory_profile_in_ct.sh --ctid 120 + sudo ./scripts/set_ollama_memory_profile_in_ct.sh --ctid 120 [--preset balanced] Options: --ctid Proxmox CTID (required) + --preset + Preset baseline (default: balanced) --context-length OLLAMA_CONTEXT_LENGTH (default: 8192) --num-parallel OLLAMA_NUM_PARALLEL (default: 1) --max-loaded-models OLLAMA_MAX_LOADED_MODELS (default: 1) --flash-attention OLLAMA_FLASH_ATTENTION true|false (default: false) + +Notes: + - Preset sets baseline values. + - Explicit flags override preset values. EOF } @@ -25,18 +31,25 @@ require_root() { } CTID="" +PRESET="balanced" CONTEXT_LENGTH="8192" NUM_PARALLEL="1" MAX_LOADED_MODELS="1" FLASH_ATTENTION="false" +EXPLICIT_CONTEXT_LENGTH="no" +EXPLICIT_NUM_PARALLEL="no" +EXPLICIT_MAX_LOADED_MODELS="no" +EXPLICIT_FLASH_ATTENTION="no" + while [[ $# -gt 0 ]]; do case "$1" in --ctid) CTID="$2"; shift 2 ;; - --context-length) CONTEXT_LENGTH="$2"; shift 2 ;; - --num-parallel) NUM_PARALLEL="$2"; shift 2 ;; - --max-loaded-models) MAX_LOADED_MODELS="$2"; shift 2 ;; - --flash-attention) FLASH_ATTENTION="$2"; shift 2 ;; + --preset) PRESET="$2"; shift 2 ;; + --context-length) CONTEXT_LENGTH="$2"; EXPLICIT_CONTEXT_LENGTH="yes"; shift 2 ;; + --num-parallel) NUM_PARALLEL="$2"; EXPLICIT_NUM_PARALLEL="yes"; shift 2 ;; + --max-loaded-models) MAX_LOADED_MODELS="$2"; EXPLICIT_MAX_LOADED_MODELS="yes"; shift 2 ;; + --flash-attention) FLASH_ATTENTION="$2"; EXPLICIT_FLASH_ATTENTION="yes"; shift 2 ;; -h|--help) usage; exit 0 ;; *) echo "Unknown argument: $1" >&2 @@ -46,6 +59,57 @@ while [[ $# -gt 0 ]]; do esac done +case "$PRESET" in + safe) + PRESET_CONTEXT_LENGTH="4096" + PRESET_NUM_PARALLEL="1" + PRESET_MAX_LOADED_MODELS="1" + PRESET_FLASH_ATTENTION="false" + ;; + balanced) + PRESET_CONTEXT_LENGTH="8192" + PRESET_NUM_PARALLEL="1" + PRESET_MAX_LOADED_MODELS="1" + PRESET_FLASH_ATTENTION="false" + ;; + max) + PRESET_CONTEXT_LENGTH="16384" + PRESET_NUM_PARALLEL="2" + PRESET_MAX_LOADED_MODELS="2" + PRESET_FLASH_ATTENTION="true" + ;; + *) + echo "--preset must be one of: safe, balanced, max" >&2 + exit 1 + ;; +esac + +CUSTOM_CONTEXT_LENGTH="$CONTEXT_LENGTH" +CUSTOM_NUM_PARALLEL="$NUM_PARALLEL" +CUSTOM_MAX_LOADED_MODELS="$MAX_LOADED_MODELS" +CUSTOM_FLASH_ATTENTION="$FLASH_ATTENTION" + +CONTEXT_LENGTH="$PRESET_CONTEXT_LENGTH" +NUM_PARALLEL="$PRESET_NUM_PARALLEL" +MAX_LOADED_MODELS="$PRESET_MAX_LOADED_MODELS" +FLASH_ATTENTION="$PRESET_FLASH_ATTENTION" + +if [[ "$EXPLICIT_CONTEXT_LENGTH" == "yes" ]]; then + CONTEXT_LENGTH="$CUSTOM_CONTEXT_LENGTH" +fi + +if [[ "$EXPLICIT_NUM_PARALLEL" == "yes" ]]; then + NUM_PARALLEL="$CUSTOM_NUM_PARALLEL" +fi + +if [[ "$EXPLICIT_MAX_LOADED_MODELS" == "yes" ]]; then + MAX_LOADED_MODELS="$CUSTOM_MAX_LOADED_MODELS" +fi + +if [[ "$EXPLICIT_FLASH_ATTENTION" == "yes" ]]; then + FLASH_ATTENTION="$CUSTOM_FLASH_ATTENTION" +fi + if [[ -z "$CTID" ]]; then echo "--ctid is required." >&2 usage