mirror of
https://github.com/Heretek-AI/interface.git
synced 2026-11-09 10:49:48 -10:00
John SmithandClaude Code
7e2bc00b3c
Versions and defaults refreshed against upstream: - LiteLLM v1.40.13-stable -> v1.100.0 (latest stable, 2026-09-06). Empirically verified the v1.100.0 base image ships redisvl 0.4.1 already, so the redisvl pin stays at 0.4.1 (no bump needed). - CachyLLama commit pinned to 609f40e (2026-09-08), which includes the ssd_cold_start_used KV-cache-corruption fix and tracks llama.cpp v0.4.0. CACHYLLAMA_COMMIT exposed as a compose build-arg default in .env. - Default LLM switched to Ornith-1.5-35B-A3B-CRACK-Q4_K_M.gguf (Qwen-3.5 hybrid GatedDeltaNet+attention MoE, 35B/3B active, fits 24GB GPU at Q4_K_M). Context dropped to 8192 to fit KV headroom; added --jinja and publisher sampling defaults (temp=1.0, top_p=0.95, top_k=20). Reasoning mode gated by LLM_CHAT_TEMPLATE_KWARGS (off by default). - Default embedder switched to Harrier Q5_K_M (424 MiB) per model card recommendation; embedding dim stays 1024 so the RediSearch index needs no migration. README updated with VRAM budget, sampling, and upgrade guidance. Co-Authored-By: Claude Code <[email protected]>
97 lines
3.1 KiB
Docker
97 lines
3.1 KiB
Docker
# syntax=docker/dockerfile:1.4
|
|
#
|
|
# CachyLLama Vulkan inference engine image.
|
|
# Build arg CACHYLLAMA_COMMIT pins the upstream fewtarius/CachyLLama commit
|
|
# so builds are byte-reproducible. Override via `docker build
|
|
# --build-arg CACHYLLAMA_COMMIT=<sha>` or compose `build.args`.
|
|
ARG UBUNTU_VERSION=24.04
|
|
# Pinned to CachyLLama master HEAD on 2026-09-08. Includes the
|
|
# "ssd_cold_start_used flag misuse causing KV cache corruption" fix and
|
|
# tracks upstream llama.cpp v0.4.0.
|
|
ARG CACHYLLAMA_COMMIT=609f40e360ab47d00530985f608f0d37a34ae43f
|
|
|
|
FROM ubuntu:${UBUNTU_VERSION} AS builder
|
|
|
|
ENV DEBIAN_FRONTEND=noninteractive
|
|
|
|
RUN apt-get update && apt-get install -y --no-install-recommends \
|
|
build-essential \
|
|
cmake \
|
|
git \
|
|
pkg-config \
|
|
ca-certificates \
|
|
curl \
|
|
libvulkan-dev \
|
|
glslc \
|
|
spirv-headers \
|
|
libgomp1 \
|
|
&& rm -rf /var/lib/apt/lists/*
|
|
|
|
WORKDIR /app
|
|
|
|
# Clone CachyLLama inference engine at a pinned commit.
|
|
RUN git clone https://github.com/fewtarius/CachyLLama.git . && \
|
|
git checkout "${CACHYLLAMA_COMMIT}"
|
|
|
|
# Build with Vulkan compute acceleration and CPU-backend dynamic loading.
|
|
RUN cmake -B build \
|
|
-DGGML_VULKAN=ON \
|
|
-DGGML_NATIVE=OFF \
|
|
-DLLAMA_BUILD_TESTS=OFF \
|
|
-DGGML_BACKEND_DL=ON \
|
|
-DGGML_CPU_ALL_VARIANTS=ON \
|
|
-DCMAKE_BUILD_TYPE=Release && \
|
|
cmake --build build --config Release -j$(nproc) --target llama-server llama-cli
|
|
|
|
# Aggregate production binaries and libraries
|
|
RUN mkdir -p /app/dist/bin /app/dist/lib && \
|
|
cp build/bin/llama-server /app/dist/bin/ && \
|
|
cp build/bin/llama-cli /app/dist/bin/ && \
|
|
find build -name "*.so*" -exec cp -P {} /app/dist/lib/ \;
|
|
|
|
# ---------------------------------------------------------
|
|
# Production Runtime Environment
|
|
# ---------------------------------------------------------
|
|
FROM ubuntu:${UBUNTU_VERSION} AS runner
|
|
|
|
ENV DEBIAN_FRONTEND=noninteractive
|
|
ENV LD_LIBRARY_PATH=/usr/local/lib
|
|
ENV LLAMA_ARG_HOST=0.0.0.0
|
|
ENV LLAMA_ARG_PORT=8080
|
|
|
|
RUN apt-get update && apt-get install -y --no-install-recommends \
|
|
libvulkan1 \
|
|
mesa-vulkan-drivers \
|
|
libglvnd0 \
|
|
libgl1 \
|
|
libgomp1 \
|
|
curl \
|
|
ca-certificates \
|
|
&& apt-get clean \
|
|
&& rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/*
|
|
|
|
# Create non-root user with stable UID/GID so compose-level `group_add: video,
|
|
# render` and host bind-mount permissions line up predictably.
|
|
RUN groupadd -g 1000 llama && \
|
|
useradd -u 1000 -g llama -m -s /usr/sbin/nologin llama && \
|
|
mkdir -p /data/models /data/ssd_cache && \
|
|
chown -R llama:llama /data /home/llama
|
|
|
|
WORKDIR /app
|
|
|
|
COPY --from=builder --chown=root:root /app/dist/lib/ /usr/local/lib/
|
|
COPY --from=builder --chown=root:root /app/dist/bin/llama-server /usr/local/bin/llama-server
|
|
COPY --from=builder --chown=root:root /app/dist/bin/llama-cli /usr/local/bin/llama-cli
|
|
|
|
USER llama
|
|
|
|
EXPOSE 8080
|
|
|
|
# Vulkan ICD initialization can be slow on first boot; give it 30s before
|
|
# the first healthcheck attempt.
|
|
HEALTHCHECK --interval=30s --timeout=5s --start-period=30s --retries=3 \
|
|
CMD curl -fsS http://localhost:8080/health || exit 1
|
|
|
|
ENTRYPOINT ["llama-server"]
|
|
CMD ["--help"]
|