|
| 1 | +// funasr_vad.h — single-header FSMN-VAD for the funasr ggml runtime. |
| 2 | +// Exposes funasr_vad_segments(): 16k mono wav -> speech segments [start_ms,end_ms]. |
| 3 | +// Front end (80-mel fbank + LFR m5n1 + CMVN) and FSMN encoder validated bit-exact vs |
| 4 | +// PyTorch fsmn-vad; the host state machine reproduces E2EVadModel (DEFAULT_SILENCE_SCHEDULE, |
| 5 | +// chunk-stepped) to within 1 frame (10ms) of fsmn-vad.generate on the 184-clip set. |
| 6 | +#pragma once |
| 7 | +#include "ggml.h" |
| 8 | +#include "ggml-cpu.h" |
| 9 | +#include "ggml-alloc.h" |
| 10 | +#include "ggml-backend.h" |
| 11 | +#include "gguf.h" |
| 12 | +#include <cmath> |
| 13 | +#include <cstdint> |
| 14 | +#include <cstdio> |
| 15 | +#include <cstring> |
| 16 | +#include <map> |
| 17 | +#include <string> |
| 18 | +#include <utility> |
| 19 | +#include <vector> |
| 20 | + |
| 21 | +namespace funasr_vad_impl { |
| 22 | +static const int FS=16000,WINLEN=400,SHIFT=160,NFFT=512,NMEL=80; |
| 23 | +static const float PREEMPH=0.97f,LOWF=20.0f,HIGHF=8000.0f; |
| 24 | +static inline float melf(float f){return 1127.0f*logf(1.0f+f/700.0f);} |
| 25 | +static void fftc(std::vector<float>&re,std::vector<float>&im,int n){for(int i=1,j=0;i<n;i++){int b=n>>1;for(;j&b;b>>=1)j^=b;j^=b;if(i<j){std::swap(re[i],re[j]);std::swap(im[i],im[j]);}} |
| 26 | + for(int len=2;len<=n;len<<=1){double a=-2.0*M_PI/len;float wr=cosf(a),wi=sinf(a);for(int i=0;i<n;i+=len){float cr=1,ci=0;for(int k=0;k<len/2;k++){float ur=re[i+k],ui=im[i+k]; |
| 27 | + float vr=re[i+k+len/2]*cr-im[i+k+len/2]*ci,vi=re[i+k+len/2]*ci+im[i+k+len/2]*cr;re[i+k]=ur+vr;im[i+k]=ui+vi;re[i+k+len/2]=ur-vr;im[i+k+len/2]=ui-vi;float nc=cr*wr-ci*wi;ci=cr*wi+ci*wr;cr=nc;}}}} |
| 28 | +static std::vector<std::vector<float>> fbank80(std::vector<float> wav){ |
| 29 | + for(auto&v:wav)v*=32768.0f; std::vector<float>win(WINLEN); |
| 30 | + for(int i=0;i<WINLEN;i++)win[i]=0.54f-0.46f*cosf(2.0f*M_PI*i/(WINLEN-1)); |
| 31 | + const int NB=NFFT/2+1; float bw=(float)FS/NFFT,ml=melf(LOWF),mh=melf(HIGHF),dm=(mh-ml)/(NMEL+1); |
| 32 | + std::vector<std::vector<float>>fb(NMEL,std::vector<float>(NB,0.0f)); |
| 33 | + for(int m=0;m<NMEL;m++){float L=ml+m*dm,C=ml+(m+1)*dm,R=ml+(m+2)*dm;for(int k=0;k<NB;k++){float mf=melf(bw*k);if(mf>L&&mf<R)fb[m][k]=mf<=C?(mf-L)/(C-L):(R-mf)/(R-C);}} |
| 34 | + int N=wav.size(),T=(N-WINLEN)/SHIFT+1; if(T<1)T=0; std::vector<std::vector<float>>feat(T,std::vector<float>(NMEL)); |
| 35 | + std::vector<float>re(NFFT),im(NFFT),fr(WINLEN);const float fl=1.1920929e-07f; |
| 36 | + for(int t=0;t<T;t++){const float*s=wav.data()+t*SHIFT;double mn=0;for(int i=0;i<WINLEN;i++)mn+=s[i];mn/=WINLEN; |
| 37 | + for(int i=0;i<WINLEN;i++)fr[i]=s[i]-(float)mn;for(int i=WINLEN-1;i>0;i--)fr[i]-=PREEMPH*fr[i-1];fr[0]-=PREEMPH*fr[0]; |
| 38 | + for(int i=0;i<NFFT;i++){re[i]=i<WINLEN?fr[i]*win[i]:0.0f;im[i]=0.0f;}fftc(re,im,NFFT); |
| 39 | + for(int m=0;m<NMEL;m++){float e=0;for(int k=0;k<NB;k++)if(fb[m][k]>0)e+=fb[m][k]*(re[k]*re[k]+im[k]*im[k]);feat[t][m]=logf(e>fl?e:fl);}} |
| 40 | + return feat; |
| 41 | +} |
| 42 | +static std::vector<float> lfr(const std::vector<std::vector<float>>&feat,int m,int n,int&T_out){ |
| 43 | + int T=feat.size(),D=NMEL,pad=(m-1)/2; int Tl=T>0?(T+n-1)/n:0; |
| 44 | + std::vector<std::vector<float>> pf; pf.reserve(T+pad+m); |
| 45 | + for( int i= 0;i<pad;i++)pf. push_back(feat[ 0]); |
| 46 | + for(int t=0;t<T;t++)pf.push_back(feat[t]); |
| 47 | + while((int)pf.size()<(Tl-1)*n+m)pf.push_back(feat[T-1]); |
| 48 | + std::vector<float> out((size_t)Tl*m*D); |
| 49 | + for(int i=0;i<Tl;i++)for(int j=0;j<m;j++)memcpy(&out[((size_t)i*m+j)*D],pf[i*n+j].data(),D*sizeof(float)); |
| 50 | + T_out=Tl; return out; |
| 51 | +} |
| 52 | +struct vad{ggml_context*ctx=nullptr;std::map<std::string,ggml_tensor*>t; |
| 53 | + ggml_tensor*g(const std::string&n){auto it=t.find(n);if(it==t.end()){fprintf(stderr,"vad: missing %s\n",n.c_str());return nullptr;}return it->second;}}; |
| 54 | +static ggml_tensor* lin(ggml_context*c,ggml_tensor*w,ggml_tensor*b,ggml_tensor*x){auto y=ggml_mul_mat(c,w,x);return b?ggml_add(c,y,b):y;} |
| 55 | +} // namespace funasr_vad_impl |
| 56 | + |
| 57 | +// Run FSMN-VAD on a 16k mono float waveform; fills segs with [start_ms,end_ms] speech spans. |
| 58 | +// max_seg_ms caps a single segment (e.g. 30000); pass <=0 to use 60000 (model default). |
| 59 | +inline bool funasr_vad_segments(const std::string& gguf_path, const std::vector<float>& wav, |
| 60 | + int max_seg_ms, std::vector<std::pair<int,int>>& segs, int nthreads=8){ |
| 61 | + using namespace funasr_vad_impl; |
| 62 | + segs.clear(); |
| 63 | + vad m; gguf_init_params ip={false,&m.ctx}; gguf_context*gg=gguf_init_from_file(gguf_path.c_str(),ip); |
| 64 | + if(!gg){fprintf(stderr,"vad: cannot load %s\n",gguf_path.c_str());return false;} |
| 65 | + auto rd=[&](const char*k,int d){int i=gguf_find_key(gg,k);return i<0?d:(int)gguf_get_val_u32(gg,i);}; |
| 66 | + int idim=rd("vad.input_dim",400),pd=rd("vad.proj_dim",128),nl=rd("vad.fsmn_layers",4),lorder=rd("vad.lorder",20), |
| 67 | + od=rd("vad.output_dim",248),lm=rd("vad.lfr_m",5),ln=rd("vad.lfr_n",1); |
| 68 | + for(int i=0;i<gguf_get_n_tensors(gg);i++){const char*nm=gguf_get_tensor_name(gg,i);m.t[nm]=ggml_get_tensor(m.ctx,nm);} |
| 69 | + gguf_free(gg); |
| 70 | + |
| 71 | + auto feat=fbank80(wav); int T=0; auto feats=lfr(feat,lm,ln,T); // [T,400] |
| 72 | + if(T<1){if(m.ctx)ggml_free(m.ctx);return true;} // too short -> no speech |
| 73 | + float*shift=(float*)m.g("cmvn.shift")->data,*scale=(float*)m.g("cmvn.scale")->data; |
| 74 | + for(int t=0;t<T;t++)for(int d=0;d<idim;d++)feats[(size_t)t*idim+d]=(feats[(size_t)t*idim+d]+shift[d])*scale[d]; |
| 75 | + |
| 76 | + ggml_backend_t be=ggml_backend_cpu_init(); |
| 77 | + ggml_init_params cp={(size_t)512*1024*1024,nullptr,true}; ggml_context*c=ggml_init(cp); |
| 78 | + ggml_tensor*x=ggml_new_tensor_2d(c,GGML_TYPE_F32,idim,T); ggml_set_input(x); |
| 79 | + ggml_tensor*h=lin(c,m.g("encoder.in_linear1.linear.weight"),m.g("encoder.in_linear1.linear.bias"),x); |
| 80 | + h=lin(c,m.g("encoder.in_linear2.linear.weight"),m.g("encoder.in_linear2.linear.bias"),h); h=ggml_relu(c,h); |
| 81 | + for(int i=0;i<nl;i++){std::string p="encoder.fsmn."+std::to_string(i)+"."; |
| 82 | + ggml_tensor*z=ggml_mul_mat(c,m.g(p+"linear.linear.weight"),h); |
| 83 | + ggml_tensor*fk=m.g(p+"fsmn_block.conv_left.weight"); ggml_tensor*zp=ggml_pad_ext(c,z,0,0,lorder-1,0,0,0,0,0); ggml_tensor*acc=z; |
| 84 | + for(int j=0;j<lorder;j++){auto sl=ggml_view_2d(c,zp,pd,T,zp->nb[1],(size_t)j*zp->nb[1]);auto wj=ggml_view_1d(c,fk,pd,(size_t)j*fk->nb[1]);acc=ggml_add(c,acc,ggml_mul(c,ggml_cont(c,sl),wj));} |
| 85 | + ggml_tensor*a=lin(c,m.g(p+"affine.linear.weight"),m.g(p+"affine.linear.bias"),acc); h=ggml_relu(c,a);} |
| 86 | + h=lin(c,m.g("encoder.out_linear1.linear.weight"),m.g("encoder.out_linear1.linear.bias"),h); |
| 87 | + h=lin(c,m.g("encoder.out_linear2.linear.weight"),m.g("encoder.out_linear2.linear.bias"),h); |
| 88 | + h=ggml_soft_max(c,h); ggml_set_output(h); |
| 89 | + ggml_cgraph*gf=ggml_new_graph(c); ggml_build_forward_expand(gf,h); |
| 90 | + ggml_gallocr_t ga=ggml_gallocr_new(ggml_backend_cpu_buffer_type()); ggml_gallocr_alloc_graph(ga,gf); |
| 91 | + ggml_backend_tensor_set(x,feats.data(),0,ggml_nbytes(x)); ggml_backend_cpu_set_n_threads(be,nthreads); |
| 92 | + bool ok=ggml_backend_graph_compute(be,gf)==GGML_STATUS_SUCCESS; |
| 93 | + std::vector<float> sc((size_t)od*T); if(ok)ggml_backend_tensor_get(h,sc.data(),0,ggml_nbytes(h)); |
| 94 | + ggml_gallocr_free(ga);ggml_free(c);ggml_backend_free(be);if(m.ctx)ggml_free(m.ctx); |
| 95 | + if(!ok)return false; |
| 96 | + |
| 97 | + // ===== E2EVadModel state machine (host) -> speech segments [start_ms,end_ms] ===== |
| 98 | + const int FR=10; // ms per frame (frame_in_ms) |
| 99 | + const int win=20; // window_size_ms 200 / 10 |
| 100 | + const int s2s=15, sp2s=15; // sil_to_speech / speech_to_sil thres (150/10) |
| 101 | + const int lookahead_end=100/FR; // lookahead_time_end_point 100/10 = 10 (do_extend) |
| 102 | + int max_seg = (max_seg_ms>0 ? max_seg_ms : 60000)/FR; // max_single_segment frames |
| 103 | + const int start_lookback = win + 200/FR; // LatencyFrmNumAtStartPoint = 40 |
| 104 | + // End-silence threshold from DEFAULT_SILENCE_SCHEDULE, stepped at chunk boundaries (chunk_size |
| 105 | + // 60000ms = 6000 frames). At each boundary, if mid-segment (InSpeech) or in_speech latched, |
| 106 | + // accumulated_ms += 60000; threshold = schedule(accumulated). Reset to 0 on each segment emit. |
| 107 | + const int CHUNK=60000/FR; |
| 108 | + int acc=0, insp=0; int max_end_sil, end_lookback; |
| 109 | + auto recompute=[&](){ |
| 110 | + int s; if(acc<=10000)s=2000; else if(acc<=20000)s=1000; else if(acc<=30000)s=800; |
| 111 | + else if(acc<=40000)s=600; else if(acc<=50000)s=400; else if(acc<=60000)s=200; else s=100; |
| 112 | + int ms=s-150; if(ms<0)ms=0; max_end_sil=ms/FR; |
| 113 | + end_lookback=max_end_sil-lookahead_end-1; if(end_lookback<0)end_lookback=0; |
| 114 | + }; |
| 115 | + recompute(); |
| 116 | + std::vector<int> wbuf(win,0); int wpos=0,wsum=0,pre=0; |
| 117 | + int st=0, cstart=-1, csil=0, prev_end=0; |
| 118 | + auto reset=[&](){ std::fill(wbuf.begin(),wbuf.end(),0); wpos=0; wsum=0; pre=0; csil=0; st=0; cstart=-1; acc=0; insp=0; }; |
| 119 | + auto emit=[&]( int s, int e){ if(s<prev_end)s=prev_end; if(s< 0)s= 0; if(e>T)e=T; if(e>s){segs. push_back({s,e}); prev_end=e;} }; |
| 120 | + for(int t=0;t<T;t++){ |
| 121 | + if(t>0 && t%CHUNK==0){ if(st==1||insp){acc+=60000; insp=1;} recompute(); } |
| 122 | + float sil=sc[(size_t)t*od+0]; |
| 123 | + int fs = ((1.0f-sil) >= sil + 0.5f) ? 1 : 0; // speech_noise_thres=0.5 |
| 124 | + wsum -= wbuf[wpos]; wsum += fs; wbuf[wpos]=fs; wpos=(wpos+1)%win; |
| 125 | + int ch; |
| 126 | + if(pre==0 && wsum>=s2s){pre=1; ch=3;} |
| 127 | + else if(pre==1 && wsum<=sp2s){pre=0; ch=1;} |
| 128 | + else ch = pre==0?0:2; |
| 129 | + if(ch==3){ csil=0; |
| 130 | + if(st== 0){ cstart=t-start_lookback; if(cstart<prev_end)cstart=prev_end; if(cstart< 0)cstart= 0; st= 1; } |
| 131 | + else if(st==1 && t-cstart+1>max_seg){ emit(cstart,t); reset(); } |
| 132 | + } else if(ch==1||ch==2){ csil=0; |
| 133 | + if(st==1 && t-cstart+1>max_seg){ emit(cstart,t); reset(); } |
| 134 | + } else { csil++; |
| 135 | + if(st==1){ |
| 136 | + if(csil>=max_end_sil){ emit(cstart, t-end_lookback); reset(); } |
| 137 | + else if(t-cstart+1>max_seg){ emit(cstart,t); reset(); } |
| 138 | + } |
| 139 | + } |
| 140 | + } |
| 141 | + if(st==1) emit(cstart,T); |
| 142 | + // convert frame indices -> ms |
| 143 | + for(auto&s:segs){ s.first*=FR; s.second*=FR; } |
| 144 | + return true; |
| 145 | +} |
0 commit comments