llama.cpp: add srt output (#3480)

Add optional SRT output to Fun-ASR-Nano, SenseVoiceSmall, and Paraformer llama.cpp runtimes. Preserve existing non-SRT output behavior and document VAD-aligned subtitle generation.

runtime/llama.cpp/README.md

@@ -59,6 +59,25 @@ cmake -B build -DCMAKE_BUILD_TYPE=Release      # fetches pinned llama.cpp; stati
 cmake --build build -j                          # -> build/bin/llama-funasr-* (all tools)
 ```
 
+## SRT subtitle output
+
+Fun-ASR-Nano, SenseVoiceSmall, and Paraformer accept `--srt` and write standard
+SRT entries to stdout. Use FSMN-VAD segmentation for speech-aligned timestamps;
+Fun-ASR-Nano can also timestamp fixed windows selected with `--chunk`. Without
+segmentation, SenseVoiceSmall and Paraformer emit one entry spanning the input.
+
+```bash
+./build/bin/llama-funasr-cli --enc encoder.gguf -m llm.gguf \
+  --vad fsmn-vad.gguf -a audio.wav --srt > audio.srt
+./build/bin/llama-funasr-sensevoice -m sensevoice-small.gguf \
+  --vad fsmn-vad.gguf -a audio.wav --srt > audio.srt
+./build/bin/llama-funasr-paraformer -m paraformer.gguf \
+  --vad fsmn-vad.gguf -a audio.wav --srt > audio.srt
+```
+
+Progress and timing diagnostics remain on stderr, so redirecting stdout produces
+a clean subtitle file. Normal text output is unchanged when `--srt` is omitted.
+
 ### Optional Windows CUDA backend for SenseVoiceSmall
 
 The CPU release ZIPs are portable packages. Tagged releases also publish

runtime/llama.cpp/fun-asr-nano/funasr-cli/funasr-cli.cpp

@@ -28,6 +28,7 @@
 #include "funasr_audio.h"
 // built-in FSMN-VAD front end (single-binary --vad segmentation)
 #include "funasr_vad.h"
+#include "funasr_srt.h"
 #include <utility>
 
 // ======================= kaldi fbank + LFR =======================
@@ -166,9 +167,18 @@ static int decode_batch(llama_context*ctx,int n,llama_token*tok,float*embd,int n
     int r=llama_decode(ctx,b); n_past+=n; return r;
 }
 
+static void print_usage(const char*argv0){
+    fprintf(stderr,
+        "usage: %s --enc enc.gguf -m llm.gguf -a audio.wav"
+        " [-n npred] [--chunk sec]"
+        " [--vad fsmn-vad.gguf [--vad-maxseg ms]]"
+        " [--srt] [--rep R]\n", argv0);
+}
+
 int main(int argc,char**argv){
     std::string enc_path,llm_path,wav_path,vad_path; int npred=512; double chunk_sec=0; float rep=1.0f;
     int vad_maxseg=30000;
+    bool srt_mode=false;
     for(int i=1;i<argc;i++){
         if(!strcmp(argv[i],"--enc")&&i+1<argc)enc_path=argv[++i];
         else if(!strcmp(argv[i],"-m")&&i+1<argc)llm_path=argv[++i];
@@ -177,10 +187,11 @@ int main(int argc,char**argv){
         else if(!strcmp(argv[i],"--chunk")&&i+1<argc)chunk_sec=atof(argv[++i]);
         else if(!strcmp(argv[i],"--vad")&&i+1<argc)vad_path=argv[++i];
         else if(!strcmp(argv[i],"--vad-maxseg")&&i+1<argc)vad_maxseg=atoi(argv[++i]);
+        else if(!strcmp(argv[i],"--srt"))srt_mode=true;
         else if(!strcmp(argv[i],"--rep")&&i+1<argc)rep=atof(argv[++i]);
-        else {fprintf(stderr,"usage: %s --enc enc.gguf -m llm.gguf -a audio.wav [-n npred] [--chunk sec] [--vad fsmn-vad.gguf [--vad-maxseg ms]]\n",argv[0]);return 1;}
+        else { print_usage(argv[0]); return 1; }
     }
-    if(enc_path.empty()||llm_path.empty()||wav_path.empty()){fprintf(stderr,"missing args\n");return 1;}
+    if(enc_path.empty()||llm_path.empty()||wav_path.empty()){fprintf(stderr,"missing args\n");print_usage(argv[0]);return 1;}
 
     std::vector<float> wav;
     if(!funasr_load_audio_16k_mono(wav_path.c_str(),wav)){fprintf(stderr,"failed to read audio\n");return 1;}
@@ -220,10 +231,13 @@ int main(int argc,char**argv){
         int chunk_n = chunk_sec > 0 ? std::max(1, (int)(chunk_sec*16000)) : (int)wav.size();
         for(size_t off=0; off<wav.size(); off+=chunk_n) wins.push_back({(int)off,(int)std::min((size_t)chunk_n,wav.size()-off)});
     }
-    std::string full;
+
+    int srt_idx = 0;
     for (auto& w : wins) {
         int off = w.first, len = w.second;
         if (len < WINLEN) continue;                    // too short for one frame
+        int start_ms = (int)((int64_t)off * 1000 / 16000);
+        int end_ms   = (int)((int64_t)(off + len) * 1000 / 16000);
         std::vector<float> seg(wav.begin()+off, wav.begin()+off+len);
         int T=0; auto fbank=compute_fbank(seg,T);
         int D=0; auto adp=run_encoder(em,fbank,T,560,D);
@@ -234,16 +248,32 @@ int main(int argc,char**argv){
         decode_batch(ctx,pre.size(),pre.data(),nullptr,0,n_past,false);
         decode_batch(ctx,n_aud,nullptr,adp.data(),D,n_past,false);
         decode_batch(ctx,suf.size(),suf.data(),nullptr,0,n_past,true);
+
+        // In SRT mode, buffer tokens and print the whole entry via format_srt_line.
+        // In normal mode, stream tokens directly (original behavior, no per-segment newline).
+        std::string seg_text;
         llama_token tk=llama_sampler_sample(smpl,ctx,-1);
         for(int i=0;i<npred;i++){
             if(llama_vocab_is_eog(vocab,tk))break;
             char buf[256]; int k=llama_token_to_piece(vocab,tk,buf,sizeof(buf),0,true);
-            if(k>0) full.append(buf,k);
+            if(k>0){
+                seg_text.append(buf, k);
+            }
             decode_batch(ctx,1,&tk,nullptr,0,n_past,true);
             tk=llama_sampler_sample(smpl,ctx,-1);
         }
+
+        if(srt_mode){
+            if (seg_text != "/sil") {
+                srt_idx++;
+                format_srt_line(srt_idx, start_ms, end_ms, seg_text);
+            }
+        } else {
+            printf("%s", seg_text.c_str());
+        }
+        fflush(stdout);
     }
-    printf("%s\n", full.c_str());
+    if(!srt_mode) printf("\n");
     int64_t t2=ggml_time_us();
     fprintf(stderr,"[done] %.2fs ; chunk=%.0fs\n",(t2-t0)/1e6, chunk_sec);
     llama_sampler_free(smpl); llama_free(ctx); llama_model_free(model);

runtime/llama.cpp/funasr-common/funasr_srt.h

@@ -0,0 +1,23 @@
+// Shared SRT subtitle formatting for FunASR ggml runtimes.
+#pragma once
+#include <cstdio>
+#include <string>
+
+static void format_srt_timestamp(int ms, char * buf, size_t buf_size) {
+    if (ms < 0) ms = 0;
+    int total_sec = ms / 1000;
+    int rem_ms    = ms % 1000;
+    int sec       = total_sec % 60;
+    int total_min = total_sec / 60;
+    int min       = total_min % 60;
+    int hours     = total_min / 60;
+    snprintf(buf, buf_size, "%02d:%02d:%02d,%03d", hours, min, sec, rem_ms);
+}
+
+// Print a complete SRT entry: index, timestamp range, text, blank separator.
+static void format_srt_line(int srt_idx, int start_ms, int end_ms, const std::string & text) {
+    char ts0[32], ts1[32];
+    format_srt_timestamp(start_ms, ts0, sizeof(ts0));
+    format_srt_timestamp(end_ms,   ts1, sizeof(ts1));
+    printf("%d\n%s --> %s\n%s\n\n", srt_idx, ts0, ts1, text.c_str());
+}

runtime/llama.cpp/paraformer/funasr-paraformer/funasr-paraformer.cpp

@@ -23,6 +23,7 @@ static const float LN_EPS = 1e-5f;
 #define FUNASR_AUDIO_IMPLEMENTATION
 #include "funasr_audio.h"
 #include "funasr_vad.h"     // built-in FSMN-VAD front end (--vad segmentation)
+#include "funasr_srt.h"
 #include <utility>
 static const int FS=16000,WINLEN=400,SHIFT=160,NFFT=512,NMEL=80,LFR_M=7,LFR_N=6;
 static const float PREEMPH=0.97f,LOWF=20.0f,HIGHF=8000.0f;
@@ -129,12 +130,13 @@ static std::string detok_pf(const std::vector<int>&ids,const std::vector<std::st
 }
 
 int main(int argc,char**argv){
-  std::string gguf_path,wav_path,vad_path; int vad_maxseg=30000; bool ids_mode=false;
+  std::string gguf_path,wav_path,vad_path; int vad_maxseg=30000; bool ids_mode=false,srt_mode=false;
   for(int i=1;i<argc;i++){if(!strcmp(argv[i],"-m")&&i+1<argc)gguf_path=argv[++i];else if(!strcmp(argv[i],"-a")&&i+1<argc)wav_path=argv[++i];
     else if(!strcmp(argv[i],"--vad")&&i+1<argc)vad_path=argv[++i];
     else if(!strcmp(argv[i],"--vad-maxseg")&&i+1<argc)vad_maxseg=atoi(argv[++i]);
     else if(!strcmp(argv[i],"--ids"))ids_mode=true;
-    else{fprintf(stderr,"usage: %s -m paraformer.gguf -a audio.wav [--vad fsmn-vad.gguf [--vad-maxseg ms]] [--ids]\n",argv[0]);return 1;}}
+    else if(!strcmp(argv[i],"--srt"))srt_mode=true;
+    else{fprintf(stderr,"usage: %s -m paraformer.gguf -a audio.wav [--vad fsmn-vad.gguf [--vad-maxseg ms]] [--srt] [--ids]\n",argv[0]);return 1;}}
   if(gguf_path.empty()||wav_path.empty()){fprintf(stderr,"missing args\n");return 1;}
   model m;gguf_init_params gp={false,&m.ctx_w};gguf_context*gg=gguf_init_from_file(gguf_path.c_str(),gp);if(!gg){fprintf(stderr,"gguf load failed\n");return 1;}
   auto rdi=[&](const char*k,int d){int i=gguf_find_key(gg,k);return i<0?d:(int)gguf_get_val_u32(gg,i);};
@@ -152,9 +154,9 @@ int main(int argc,char**argv){
   float*ow=(float*)m.g("predictor.cif_output.weight")->data; // [1,512]
   float ob=((float*)m.g("predictor.cif_output.bias")->data)[0];
 
-  // Full pipeline (fbank -> CMVN -> encoder -> CIF -> decoder) on one wav window; prints token IDs.
-  auto run_seg=[&](const std::vector<float>& wav){
-    int T=0;auto fb=compute_fbank(wav,T); if(T<1)return;
+  // Full pipeline (fbank -> CMVN -> encoder -> CIF -> decoder) on one wav window; returns text.
+  auto run_seg=[&](const std::vector<float>& wav) -> std::string {
+    int T=0;auto fb=compute_fbank(wav,T); if(T<1)return "";
     for(int t=0;t<T;t++)for(int d=0;d<F;d++)fb[(size_t)t*F+d]=(fb[(size_t)t*F+d]+shift[d])*scale[d];  // CMVN
     {float sc=sqrtf((float)D);for(auto&v:fb)v*=sc;}add_posenc(fb,T,F);
     std::vector<float>enc;
@@ -183,7 +185,7 @@ int main(int argc,char**argv){
       for(int d=0;d<D;d++) frame[d]+=cur*hid[(size_t)t*D+d];
       if(fire){ acoustic.insert(acoustic.end(),frame.begin(),frame.end()); integrate-=1.0f;
         for(int d=0;d<D;d++) frame[d]=rem*hid[(size_t)t*D+d]; } }
-    int N=acoustic.size()/D; if(N<1)return;
+    int N=acoustic.size()/D; if(N<1)return "";
     std::vector<float> logits;
     {ggml_init_params cp={(size_t)2048*1024*1024,nullptr,true};ggml_context*c=ggml_init(cp);
      ggml_tensor*tgt=ggml_new_tensor_2d(c,GGML_TYPE_F32,D,N);ggml_set_input(tgt);
@@ -197,21 +199,34 @@ int main(int argc,char**argv){
      logits=run_graph(c,x,tgt,acoustic.data(),mem,enc.data());ggml_free(c);}
     std::vector<int> seg_ids; seg_ids.reserve(N);
     for(int n=0;n<N;n++){const float*col=&logits[(size_t)n*V];int am=0;float best=col[0];for(int v=1;v<V;v++)if(col[v]>best){best=col[v];am=v;}seg_ids.push_back(am);}
-    if(emit_ids){ for(int id:seg_ids) printf("%d ",id); }
-    else { std::string t=detok_pf(seg_ids,vocab); printf("%s",t.c_str()); }
+    if(emit_ids){ std::string s; for(int id:seg_ids){ s+=std::to_string(id); s+=" "; } return s; }
+    else { return detok_pf(seg_ids,vocab); }
   };
 
   int64_t t0=ggml_time_us();
   std::vector<float>wav;if(!funasr_load_audio_16k_mono(wav_path.c_str(),wav)){fprintf(stderr,"read audio failed\n");return 1;}
+  int srt_idx=0;
   if(!vad_path.empty()){
     std::vector<std::pair<int,int>> segs;
     if(!funasr_vad_segments(vad_path,wav,vad_maxseg,segs)){fprintf(stderr,"vad failed\n");return 1;}
     for(auto&s:segs){ int off=(int)((int64_t)s.first*16000/1000), end=(int)((int64_t)s.second*16000/1000);
       if(end>(int)wav.size())end=wav.size(); if(end-off<WINLEN)continue;
-      std::vector<float> seg(wav.begin()+off,wav.begin()+end); run_seg(seg); }
+      std::vector<float> seg(wav.begin()+off,wav.begin()+end);
+      std::string text=run_seg(seg);
+      if(text.empty())continue;
+      if(srt_mode){ srt_idx++; format_srt_line(srt_idx,s.first,s.second,text); }
+      else { printf("%s",text.c_str()); }
+      fflush(stdout);
+    }
     fprintf(stderr,"[paraformer] %zu vad segments\n",segs.size());
-  } else { run_seg(wav); }
-  printf("\n");
+  } else {
+    int end_ms=(int)((int64_t)wav.size()*1000/16000);
+    std::string text=run_seg(wav);
+    if(srt_mode){ if(!text.empty()) format_srt_line(1,0,end_ms,text); }
+    else { printf("%s",text.c_str()); }
+    fflush(stdout);
+  }
+  if(!srt_mode) printf("\n");
   fprintf(stderr,"[paraformer] done %.2fs\n",(ggml_time_us()-t0)/1e6);
   if(m.ctx_w) ggml_free(m.ctx_w);
   return 0;

runtime/llama.cpp/sensevoice/funasr-sensevoice/funasr-sensevoice.cpp

@@ -25,6 +25,7 @@ static const float LN_EPS = 1e-5f;
 #define FUNASR_AUDIO_IMPLEMENTATION
 #include "funasr_audio.h"
 #include "funasr_vad.h"     // built-in FSMN-VAD front end (--vad segmentation)
+#include "funasr_srt.h"
 #include <utility>
 static const int FS=16000,WINLEN=400,SHIFT=160,NFFT=512,NMEL=80,LFR_M=7,LFR_N=6;
 static const float PREEMPH=0.97f,LOWF=20.0f,HIGHF=8000.0f;
@@ -168,7 +169,7 @@ static std::string detok_sv(const std::vector<int>&ids,const std::vector<std::st
 }
 
 int main(int argc,char**argv){
-  std::string gguf_path,fbank_path,wav_path,vad_path; int vad_maxseg=30000; bool ids_mode=false,keep_tags=false;
+  std::string gguf_path,fbank_path,wav_path,vad_path; int vad_maxseg=30000; bool ids_mode=false,keep_tags=false,srt_mode=false;
   std::string backend_name="cpu";
   for(int i=1;i<argc;i++){ if(!strcmp(argv[i],"-m")&&i+1<argc)gguf_path=argv[++i];
     else if(!strcmp(argv[i],"-f")&&i+1<argc)fbank_path=argv[++i];
@@ -178,7 +179,8 @@ int main(int argc,char**argv){
     else if(!strcmp(argv[i],"--backend")&&i+1<argc)backend_name=argv[++i];
     else if(!strcmp(argv[i],"--ids"))ids_mode=true;
     else if(!strcmp(argv[i],"--keep-tags"))keep_tags=true;
-    else {fprintf(stderr,"usage: %s -m sensevoice.gguf (-a audio.wav | -f fbank.bin) [--vad fsmn-vad.gguf [--vad-maxseg ms]] [--backend cpu|cuda|vulkan] [--ids] [--keep-tags]\n",argv[0]);return 1;} }
+    else if(!strcmp(argv[i],"--srt"))srt_mode=true;
+    else {fprintf(stderr,"usage: %s -m sensevoice.gguf (-a audio.wav | -f fbank.bin) [--vad fsmn-vad.gguf [--vad-maxseg ms]] [--backend cpu|cuda|vulkan] [--srt] [--ids] [--keep-tags]\n",argv[0]);return 1;} }
   if(gguf_path.empty()||(fbank_path.empty()&&wav_path.empty())){fprintf(stderr,"missing args\n");return 1;}
   graph_backend graph_be=make_graph_backend(backend_name);
 
@@ -201,8 +203,8 @@ int main(int argc,char**argv){
   // it does NOT apply am.mvn CMVN (that path is unused at inference). Applying it
   // makes the encoder predict <|nospeech|>. So no CMVN here.
   float*emb=(float*)m.g("embed.weight")->data;   // [16, 560] row-major
-  // Run encoder+CTC on one fbank window [T,F]; prints greedy-CTC token IDs (no newline).
-  auto run_seg=[&](const std::vector<float>& fb,int T){
+  // Run encoder+CTC on one fbank window [T,F]; returns decoded text string.
+  auto run_seg=[&](const std::vector<float>& fb,int T) -> std::string {
     int N=nq+T; std::vector<float> inp((size_t)N*F);
     for(int i=0;i<nq;i++) memcpy(&inp[(size_t)i*F], &emb[(size_t)qtok[i]*F], F*sizeof(float));
     memcpy(&inp[(size_t)nq*F], fb.data(), (size_t)T*F*sizeof(float));
@@ -225,33 +227,47 @@ int main(int argc,char**argv){
     for(int n=0;n<N;n++){ const float*col=&lg[(size_t)n*V]; int am=0; float best=col[0];
       for(int v=1;v<V;v++) if(col[v]>best){best=col[v];am=v;}
       if(am!=prev && am!=m.c.blank) seg_ids.push_back(am); prev=am; }
-    if(emit_ids){ for(int id:seg_ids) printf("%d ",id); }
-    else { std::string t=detok_sv(seg_ids,vocab,keep_tags); printf("%s",t.c_str()); }
+    std::string result;
+    if(emit_ids){ for(int id:seg_ids){ result+=std::to_string(id); result+=" "; } }
+    else { result=detok_sv(seg_ids,vocab,keep_tags); }
     ggml_gallocr_free(ga); ggml_free(c);
+    return result;
   };
 
   int64_t t0=ggml_time_us();
+  int srt_idx=0;
   if(!vad_path.empty()){
     std::vector<float> wav; if(!funasr_load_audio_16k_mono(wav_path.c_str(),wav)){fprintf(stderr,"read audio failed\n");return 1;}
     std::vector<std::pair<int,int>> segs;
     if(!funasr_vad_segments(vad_path,wav,vad_maxseg,segs)){fprintf(stderr,"vad failed\n");return 1;}
     for(auto&s:segs){ int off=(int)((int64_t)s.first*16000/1000), end=(int)((int64_t)s.second*16000/1000);
       if(end>(int)wav.size())end=wav.size(); if(end-off<WINLEN)continue;
-      std::vector<float> seg(wav.begin()+off,wav.begin()+end); int t=0; auto fb=compute_fbank(seg,t); run_seg(fb,t); }
+      std::vector<float> seg(wav.begin()+off,wav.begin()+end); int t=0; auto fb=compute_fbank(seg,t);
+      std::string text=run_seg(fb,t);
+      if(text.empty())continue;
+      if(srt_mode){ srt_idx++; format_srt_line(srt_idx,s.first,s.second,text); }
+      else { printf("%s",text.c_str()); }
+      fflush(stdout);
+    }
     fprintf(stderr,"[sensevoice] %zu vad segments\n",segs.size());
   } else {
-    int32_t T=0,Fc=F; std::vector<float> fb;
+    int32_t T=0,Fc=F; std::vector<float> fb; int end_ms=0;
     if(!wav_path.empty()){
       std::vector<float> wav; if(!funasr_load_audio_16k_mono(wav_path.c_str(),wav)){fprintf(stderr,"read audio failed\n");return 1;}
+      end_ms=(int)((int64_t)wav.size()*1000/16000);
       int t=0; fb=compute_fbank(wav,t); T=t;
     } else {
       FILE*f=fopen(fbank_path.c_str(),"rb"); if(!f){fprintf(stderr,"open fbank\n");return 1;}
       if(fread(&T,4,1,f)!=1||fread(&Fc,4,1,f)!=1){fclose(f);return 1;}
       fb.resize((size_t)T*Fc); if((int)fread(fb.data(),4,fb.size(),f)!=(int)fb.size()){fclose(f);return 1;} fclose(f);
+      end_ms=(int)((int64_t)T*LFR_N*SHIFT*1000/FS);
     }
-    run_seg(fb,T);
+    std::string text=run_seg(fb,T);
+    if(srt_mode){ if(!text.empty()) format_srt_line(1,0,end_ms,text); }
+    else { printf("%s",text.c_str()); }
+    fflush(stdout);
   }
-  printf("\n");
+  if(!srt_mode) printf("\n");
   fprintf(stderr,"[sensevoice] done %.2fs\n",(ggml_time_us()-t0)/1e6);
   ggml_backend_free(graph_be.backend);
   if(m.ctx_w) ggml_free(m.ctx_w);