diff --git a/runtime/llama.cpp/README.md b/runtime/llama.cpp/README.md index 48f097944..93194553b 100644 --- a/runtime/llama.cpp/README.md +++ b/runtime/llama.cpp/README.md @@ -59,6 +59,25 @@ cmake -B build -DCMAKE_BUILD_TYPE=Release # fetches pinned llama.cpp; stati cmake --build build -j # -> build/bin/llama-funasr-* (all tools) ``` +## SRT subtitle output + +Fun-ASR-Nano, SenseVoiceSmall, and Paraformer accept `--srt` and write standard +SRT entries to stdout. Use FSMN-VAD segmentation for speech-aligned timestamps; +Fun-ASR-Nano can also timestamp fixed windows selected with `--chunk`. Without +segmentation, SenseVoiceSmall and Paraformer emit one entry spanning the input. + +```bash +./build/bin/llama-funasr-cli --enc encoder.gguf -m llm.gguf \ + --vad fsmn-vad.gguf -a audio.wav --srt > audio.srt +./build/bin/llama-funasr-sensevoice -m sensevoice-small.gguf \ + --vad fsmn-vad.gguf -a audio.wav --srt > audio.srt +./build/bin/llama-funasr-paraformer -m paraformer.gguf \ + --vad fsmn-vad.gguf -a audio.wav --srt > audio.srt +``` + +Progress and timing diagnostics remain on stderr, so redirecting stdout produces +a clean subtitle file. Normal text output is unchanged when `--srt` is omitted. + ### Optional Windows CUDA backend for SenseVoiceSmall The CPU release ZIPs are portable packages. Tagged releases also publish diff --git a/runtime/llama.cpp/fun-asr-nano/funasr-cli/funasr-cli.cpp b/runtime/llama.cpp/fun-asr-nano/funasr-cli/funasr-cli.cpp index 26f60d52b..eb3b6f0b2 100644 --- a/runtime/llama.cpp/fun-asr-nano/funasr-cli/funasr-cli.cpp +++ b/runtime/llama.cpp/fun-asr-nano/funasr-cli/funasr-cli.cpp @@ -28,6 +28,7 @@ #include "funasr_audio.h" // built-in FSMN-VAD front end (single-binary --vad segmentation) #include "funasr_vad.h" +#include "funasr_srt.h" #include // ======================= kaldi fbank + LFR ======================= @@ -166,9 +167,18 @@ static int decode_batch(llama_context*ctx,int n,llama_token*tok,float*embd,int n int r=llama_decode(ctx,b); n_past+=n; return r; } +static void print_usage(const char*argv0){ + fprintf(stderr, + "usage: %s --enc enc.gguf -m llm.gguf -a audio.wav" + " [-n npred] [--chunk sec]" + " [--vad fsmn-vad.gguf [--vad-maxseg ms]]" + " [--srt] [--rep R]\n", argv0); +} + int main(int argc,char**argv){ std::string enc_path,llm_path,wav_path,vad_path; int npred=512; double chunk_sec=0; float rep=1.0f; int vad_maxseg=30000; + bool srt_mode=false; for(int i=1;i wav; if(!funasr_load_audio_16k_mono(wav_path.c_str(),wav)){fprintf(stderr,"failed to read audio\n");return 1;} @@ -220,10 +231,13 @@ int main(int argc,char**argv){ int chunk_n = chunk_sec > 0 ? std::max(1, (int)(chunk_sec*16000)) : (int)wav.size(); for(size_t off=0; off seg(wav.begin()+off, wav.begin()+off+len); int T=0; auto fbank=compute_fbank(seg,T); int D=0; auto adp=run_encoder(em,fbank,T,560,D); @@ -234,16 +248,32 @@ int main(int argc,char**argv){ decode_batch(ctx,pre.size(),pre.data(),nullptr,0,n_past,false); decode_batch(ctx,n_aud,nullptr,adp.data(),D,n_past,false); decode_batch(ctx,suf.size(),suf.data(),nullptr,0,n_past,true); + + // In SRT mode, buffer tokens and print the whole entry via format_srt_line. + // In normal mode, stream tokens directly (original behavior, no per-segment newline). + std::string seg_text; llama_token tk=llama_sampler_sample(smpl,ctx,-1); for(int i=0;i0) full.append(buf,k); + if(k>0){ + seg_text.append(buf, k); + } decode_batch(ctx,1,&tk,nullptr,0,n_past,true); tk=llama_sampler_sample(smpl,ctx,-1); } + + if(srt_mode){ + if (seg_text != "/sil") { + srt_idx++; + format_srt_line(srt_idx, start_ms, end_ms, seg_text); + } + } else { + printf("%s", seg_text.c_str()); + } + fflush(stdout); } - printf("%s\n", full.c_str()); + if(!srt_mode) printf("\n"); int64_t t2=ggml_time_us(); fprintf(stderr,"[done] %.2fs ; chunk=%.0fs\n",(t2-t0)/1e6, chunk_sec); llama_sampler_free(smpl); llama_free(ctx); llama_model_free(model); diff --git a/runtime/llama.cpp/funasr-common/funasr_srt.h b/runtime/llama.cpp/funasr-common/funasr_srt.h new file mode 100644 index 000000000..2975b4477 --- /dev/null +++ b/runtime/llama.cpp/funasr-common/funasr_srt.h @@ -0,0 +1,23 @@ +// Shared SRT subtitle formatting for FunASR ggml runtimes. +#pragma once +#include +#include + +static void format_srt_timestamp(int ms, char * buf, size_t buf_size) { + if (ms < 0) ms = 0; + int total_sec = ms / 1000; + int rem_ms = ms % 1000; + int sec = total_sec % 60; + int total_min = total_sec / 60; + int min = total_min % 60; + int hours = total_min / 60; + snprintf(buf, buf_size, "%02d:%02d:%02d,%03d", hours, min, sec, rem_ms); +} + +// Print a complete SRT entry: index, timestamp range, text, blank separator. +static void format_srt_line(int srt_idx, int start_ms, int end_ms, const std::string & text) { + char ts0[32], ts1[32]; + format_srt_timestamp(start_ms, ts0, sizeof(ts0)); + format_srt_timestamp(end_ms, ts1, sizeof(ts1)); + printf("%d\n%s --> %s\n%s\n\n", srt_idx, ts0, ts1, text.c_str()); +} diff --git a/runtime/llama.cpp/paraformer/funasr-paraformer/funasr-paraformer.cpp b/runtime/llama.cpp/paraformer/funasr-paraformer/funasr-paraformer.cpp index 10c0c4d03..4e220fe03 100644 --- a/runtime/llama.cpp/paraformer/funasr-paraformer/funasr-paraformer.cpp +++ b/runtime/llama.cpp/paraformer/funasr-paraformer/funasr-paraformer.cpp @@ -23,6 +23,7 @@ static const float LN_EPS = 1e-5f; #define FUNASR_AUDIO_IMPLEMENTATION #include "funasr_audio.h" #include "funasr_vad.h" // built-in FSMN-VAD front end (--vad segmentation) +#include "funasr_srt.h" #include static const int FS=16000,WINLEN=400,SHIFT=160,NFFT=512,NMEL=80,LFR_M=7,LFR_N=6; static const float PREEMPH=0.97f,LOWF=20.0f,HIGHF=8000.0f; @@ -129,12 +130,13 @@ static std::string detok_pf(const std::vector&ids,const std::vectordata; // [1,512] float ob=((float*)m.g("predictor.cif_output.bias")->data)[0]; - // Full pipeline (fbank -> CMVN -> encoder -> CIF -> decoder) on one wav window; prints token IDs. - auto run_seg=[&](const std::vector& wav){ - int T=0;auto fb=compute_fbank(wav,T); if(T<1)return; + // Full pipeline (fbank -> CMVN -> encoder -> CIF -> decoder) on one wav window; returns text. + auto run_seg=[&](const std::vector& wav) -> std::string { + int T=0;auto fb=compute_fbank(wav,T); if(T<1)return ""; for(int t=0;tenc; @@ -183,7 +185,7 @@ int main(int argc,char**argv){ for(int d=0;d logits; {ggml_init_params cp={(size_t)2048*1024*1024,nullptr,true};ggml_context*c=ggml_init(cp); ggml_tensor*tgt=ggml_new_tensor_2d(c,GGML_TYPE_F32,D,N);ggml_set_input(tgt); @@ -197,21 +199,34 @@ int main(int argc,char**argv){ logits=run_graph(c,x,tgt,acoustic.data(),mem,enc.data());ggml_free(c);} std::vector seg_ids; seg_ids.reserve(N); for(int n=0;nbest){best=col[v];am=v;}seg_ids.push_back(am);} - if(emit_ids){ for(int id:seg_ids) printf("%d ",id); } - else { std::string t=detok_pf(seg_ids,vocab); printf("%s",t.c_str()); } + if(emit_ids){ std::string s; for(int id:seg_ids){ s+=std::to_string(id); s+=" "; } return s; } + else { return detok_pf(seg_ids,vocab); } }; int64_t t0=ggml_time_us(); std::vectorwav;if(!funasr_load_audio_16k_mono(wav_path.c_str(),wav)){fprintf(stderr,"read audio failed\n");return 1;} + int srt_idx=0; if(!vad_path.empty()){ std::vector> segs; if(!funasr_vad_segments(vad_path,wav,vad_maxseg,segs)){fprintf(stderr,"vad failed\n");return 1;} for(auto&s:segs){ int off=(int)((int64_t)s.first*16000/1000), end=(int)((int64_t)s.second*16000/1000); if(end>(int)wav.size())end=wav.size(); if(end-off seg(wav.begin()+off,wav.begin()+end); run_seg(seg); } + std::vector seg(wav.begin()+off,wav.begin()+end); + std::string text=run_seg(seg); + if(text.empty())continue; + if(srt_mode){ srt_idx++; format_srt_line(srt_idx,s.first,s.second,text); } + else { printf("%s",text.c_str()); } + fflush(stdout); + } fprintf(stderr,"[paraformer] %zu vad segments\n",segs.size()); - } else { run_seg(wav); } - printf("\n"); + } else { + int end_ms=(int)((int64_t)wav.size()*1000/16000); + std::string text=run_seg(wav); + if(srt_mode){ if(!text.empty()) format_srt_line(1,0,end_ms,text); } + else { printf("%s",text.c_str()); } + fflush(stdout); + } + if(!srt_mode) printf("\n"); fprintf(stderr,"[paraformer] done %.2fs\n",(ggml_time_us()-t0)/1e6); if(m.ctx_w) ggml_free(m.ctx_w); return 0; diff --git a/runtime/llama.cpp/sensevoice/funasr-sensevoice/funasr-sensevoice.cpp b/runtime/llama.cpp/sensevoice/funasr-sensevoice/funasr-sensevoice.cpp index 99e8577d6..42e6f0474 100644 --- a/runtime/llama.cpp/sensevoice/funasr-sensevoice/funasr-sensevoice.cpp +++ b/runtime/llama.cpp/sensevoice/funasr-sensevoice/funasr-sensevoice.cpp @@ -25,6 +25,7 @@ static const float LN_EPS = 1e-5f; #define FUNASR_AUDIO_IMPLEMENTATION #include "funasr_audio.h" #include "funasr_vad.h" // built-in FSMN-VAD front end (--vad segmentation) +#include "funasr_srt.h" #include static const int FS=16000,WINLEN=400,SHIFT=160,NFFT=512,NMEL=80,LFR_M=7,LFR_N=6; static const float PREEMPH=0.97f,LOWF=20.0f,HIGHF=8000.0f; @@ -168,7 +169,7 @@ static std::string detok_sv(const std::vector&ids,const std::vector. So no CMVN here. float*emb=(float*)m.g("embed.weight")->data; // [16, 560] row-major - // Run encoder+CTC on one fbank window [T,F]; prints greedy-CTC token IDs (no newline). - auto run_seg=[&](const std::vector& fb,int T){ + // Run encoder+CTC on one fbank window [T,F]; returns decoded text string. + auto run_seg=[&](const std::vector& fb,int T) -> std::string { int N=nq+T; std::vector inp((size_t)N*F); for(int i=0;ibest){best=col[v];am=v;} if(am!=prev && am!=m.c.blank) seg_ids.push_back(am); prev=am; } - if(emit_ids){ for(int id:seg_ids) printf("%d ",id); } - else { std::string t=detok_sv(seg_ids,vocab,keep_tags); printf("%s",t.c_str()); } + std::string result; + if(emit_ids){ for(int id:seg_ids){ result+=std::to_string(id); result+=" "; } } + else { result=detok_sv(seg_ids,vocab,keep_tags); } ggml_gallocr_free(ga); ggml_free(c); + return result; }; int64_t t0=ggml_time_us(); + int srt_idx=0; if(!vad_path.empty()){ std::vector wav; if(!funasr_load_audio_16k_mono(wav_path.c_str(),wav)){fprintf(stderr,"read audio failed\n");return 1;} std::vector> segs; if(!funasr_vad_segments(vad_path,wav,vad_maxseg,segs)){fprintf(stderr,"vad failed\n");return 1;} for(auto&s:segs){ int off=(int)((int64_t)s.first*16000/1000), end=(int)((int64_t)s.second*16000/1000); if(end>(int)wav.size())end=wav.size(); if(end-off seg(wav.begin()+off,wav.begin()+end); int t=0; auto fb=compute_fbank(seg,t); run_seg(fb,t); } + std::vector seg(wav.begin()+off,wav.begin()+end); int t=0; auto fb=compute_fbank(seg,t); + std::string text=run_seg(fb,t); + if(text.empty())continue; + if(srt_mode){ srt_idx++; format_srt_line(srt_idx,s.first,s.second,text); } + else { printf("%s",text.c_str()); } + fflush(stdout); + } fprintf(stderr,"[sensevoice] %zu vad segments\n",segs.size()); } else { - int32_t T=0,Fc=F; std::vector fb; + int32_t T=0,Fc=F; std::vector fb; int end_ms=0; if(!wav_path.empty()){ std::vector wav; if(!funasr_load_audio_16k_mono(wav_path.c_str(),wav)){fprintf(stderr,"read audio failed\n");return 1;} + end_ms=(int)((int64_t)wav.size()*1000/16000); int t=0; fb=compute_fbank(wav,t); T=t; } else { FILE*f=fopen(fbank_path.c_str(),"rb"); if(!f){fprintf(stderr,"open fbank\n");return 1;} if(fread(&T,4,1,f)!=1||fread(&Fc,4,1,f)!=1){fclose(f);return 1;} fb.resize((size_t)T*Fc); if((int)fread(fb.data(),4,fb.size(),f)!=(int)fb.size()){fclose(f);return 1;} fclose(f); + end_ms=(int)((int64_t)T*LFR_N*SHIFT*1000/FS); } - run_seg(fb,T); + std::string text=run_seg(fb,T); + if(srt_mode){ if(!text.empty()) format_srt_line(1,0,end_ms,text); } + else { printf("%s",text.c_str()); } + fflush(stdout); } - printf("\n"); + if(!srt_mode) printf("\n"); fprintf(stderr,"[sensevoice] done %.2fs\n",(ggml_time_us()-t0)/1e6); ggml_backend_free(graph_be.backend); if(m.ctx_w) ggml_free(m.ctx_w);