diff --git a/.github/workflows/build-llamacpp-binaries.yml b/.github/workflows/build-llamacpp-binaries.yml index 18834dd83..342001d0e 100644 --- a/.github/workflows/build-llamacpp-binaries.yml +++ b/.github/workflows/build-llamacpp-binaries.yml @@ -48,6 +48,24 @@ jobs: -DGGML_FMA=ON -DGGML_F16C=ON -DGGML_BMI2=ON + - os: ubuntu-latest + name: linux-x64-vulkan + vulkan: true + build_target: llama-funasr-sensevoice + timeout_minutes: 90 + cmake_flags: >- + -DGGML_NATIVE=OFF + -DGGML_VULKAN=ON + -DGGML_AVX=OFF + -DGGML_AVX2=OFF + -DGGML_AVX_VNNI=OFF + -DGGML_AVX512=OFF + -DGGML_AVX512_VBMI=OFF + -DGGML_AVX512_VNNI=OFF + -DGGML_AVX512_BF16=OFF + -DGGML_FMA=OFF + -DGGML_F16C=OFF + -DGGML_BMI2=OFF - os: ubuntu-24.04-arm name: linux-arm64 cmake_flags: -DGGML_NATIVE=OFF @@ -124,6 +142,12 @@ jobs: - name: Show CUDA compiler if: matrix.cuda run: nvcc --version + - name: Install Vulkan SDK packages + if: matrix.vulkan + run: | + sudo apt-get update + sudo apt-get install -y libvulkan-dev glslc spirv-headers vulkan-tools + vulkaninfo --summary || true - name: Configure and build run: | # Release binaries must be portable across user machines, not tuned to @@ -131,9 +155,9 @@ jobs: # AVX-VNNI instructions and then crash with SIGILL on ordinary CPUs. # Keep the default x64 package conservative, and publish explicit # x64-avx2 assets for machines that support AVX2/FMA/F16C/BMI2. - # The CUDA release asset is SenseVoiceSmall-specific for now; building - # only that target keeps the Windows CUDA job bounded and matches the - # backend currently exposed by `--backend cuda`. + # The GPU release assets are SenseVoiceSmall-specific for now; building + # only that target keeps the bounded jobs aligned with the backend + # currently exposed by `--backend cuda` / `--backend vulkan`. cmake_args=( -B build -DCMAKE_BUILD_TYPE=Release diff --git a/runtime/llama.cpp/README.md b/runtime/llama.cpp/README.md index 6f7cd62e2..d751ee089 100644 --- a/runtime/llama.cpp/README.md +++ b/runtime/llama.cpp/README.md @@ -94,6 +94,38 @@ with the CUDA Toolkit version configured by the release workflow. A binary built without `-DGGML_CUDA=ON` exits with a clear message if `--backend cuda` is requested. +### Optional Linux Vulkan backend for SenseVoiceSmall + +Tagged releases also publish `funasr-llamacpp-linux-x64-vulkan.tar.gz` for +SenseVoiceSmall graph execution through ggml's Vulkan backend. This is useful on +Linux systems with AMD, Intel, NVIDIA, or integrated GPUs that expose a working +Vulkan driver/ICD. Download the `linux-x64-vulkan` asset, install your vendor GPU +driver, then select the backend at runtime: + +```bash +# From the extracted linux-x64-vulkan package: +./llama-funasr-sensevoice \ + -m sensevoice-small-q8.gguf --vad fsmn-vad.gguf -a sample.wav --backend vulkan +``` + +Build from source when you need a local Vulkan SDK, distro-specific driver +stack, or to validate a device before release packaging: + +```bash +sudo apt-get install libvulkan-dev glslc spirv-headers vulkan-tools +vulkaninfo --summary +cmake -B build-vulkan -DCMAKE_BUILD_TYPE=Release -DGGML_VULKAN=ON +cmake --build build-vulkan -j --target llama-funasr-sensevoice +./build-vulkan/bin/llama-funasr-sensevoice \ + -m sensevoice-small-f16.gguf -a sample.wav --backend vulkan +``` + +`--backend cpu` remains the default and is what the portable cross-platform +prebuilt binaries use. A binary built without `-DGGML_VULKAN=ON` exits with a +clear message if `--backend vulkan` is requested. Vulkan performance and device +availability depend on the installed GPU driver/ICD rather than on CUDA compute +capability. + ## Build (shared) ```bash git clone https://github.com/ggml-org/llama.cpp && cd llama.cpp @@ -137,12 +169,13 @@ Response: {"text": "transcribed text"} ``` -CUDA-enabled SenseVoice builds can be selected with `--backend cuda`. Extra -binary flags can be forwarded with repeated `--extra-arg`, for example -`--extra-arg --keep-tags`. This wrapper starts one subprocess per request, so it -is best for local tools, demos, and integration tests. For sustained production -traffic, use the Python `funasr-server` OpenAI-compatible service or build a -dedicated native server around the C++ runtime. +CUDA- and Vulkan-enabled SenseVoice builds can be selected with `--backend cuda` +or `--backend vulkan`. Extra binary flags can be forwarded with repeated +`--extra-arg`, for example `--extra-arg --keep-tags`. This wrapper starts one subprocess per request, +so it is best for local tools, demos, and integration +tests. For sustained production traffic, use the Python `funasr-server` +OpenAI-compatible service or build a dedicated native server around the C++ +runtime. ## Validation diff --git a/runtime/llama.cpp/sensevoice/funasr-sensevoice/funasr-sensevoice.cpp b/runtime/llama.cpp/sensevoice/funasr-sensevoice/funasr-sensevoice.cpp index 4d915ea40..99e8577d6 100644 --- a/runtime/llama.cpp/sensevoice/funasr-sensevoice/funasr-sensevoice.cpp +++ b/runtime/llama.cpp/sensevoice/funasr-sensevoice/funasr-sensevoice.cpp @@ -10,6 +10,7 @@ #include "ggml-backend.h" #include "gguf.h" +#include #include #include #include @@ -66,6 +67,29 @@ struct graph_backend { bool is_cpu=true; }; +static std::string lower_copy(const char*s){ + std::string out=s?s:""; + for(char&c:out)c=(char)std::tolower((unsigned char)c); + return out; +} + +static ggml_backend_dev_t find_gpu_backend_device(const std::string&backend_name){ + ggml_backend_load_all(); + for(size_t i=0;i