From 177845d8be403715b418a92c675e3647bba4a1d4 Mon Sep 17 00:00:00 2001 From: CJ Pais Date: Tue, 7 Jul 2026 03:48:11 -0700 Subject: [PATCH 01/11] stage 1 --- docs/porting/families/parakeet.md | 8 + .../intake.json | 232 ++++++++++++++++++ ...r-parakeet-streaming-0.6b-v1.manifest.json | 12 + 3 files changed, 252 insertions(+) create mode 100644 reports/porting/parakeet/multitalker-parakeet-streaming-0.6b-v1/intake.json create mode 100644 tests/golden/parakeet/multitalker-parakeet-streaming-0.6b-v1.manifest.json diff --git a/docs/porting/families/parakeet.md b/docs/porting/families/parakeet.md index 1a70c958..17fabe71 100644 --- a/docs/porting/families/parakeet.md +++ b/docs/porting/families/parakeet.md @@ -146,6 +146,14 @@ Allowed statuses: `PASS` | `SKIP — not exposed by runtime` | | nemotron-3.5-asr-streaming-0.6b | Batch (offline) | run_batch fast path | `uv run scripts/batch_parity.py --model --list --batch-sizes 2,4,8 --backend cpu --language en-US` + `uv run scripts/batch_tensor_parity.py --model --wav samples/jfk.wav --batch 4 --backend cpu` | text byte-equal vs serial at sizes 2/4/8 (golden frozen at `tests/golden/batch/nemotron-3.5-asr-streaming-0.6b.cpu.json`); CPU tensor parity bit-exact (max_abs=0.0) at batch=4 on jfk.wav | PASS | | ctc-1.1b | Transcribe (CTC head) | explicit en | `build/bin/transcribe-cli -m models/parakeet-ctc-1.1b/parakeet-ctc-1.1b-F32.gguf --language en samples/jfk.wav` | English transcript | PASS | | ctc-0.6b | Transcribe (CTC head) | explicit en | `build/bin/transcribe-cli -m models/parakeet-ctc-0.6b/parakeet-ctc-0.6b-F32.gguf --language en samples/jfk.wav` | English transcript | PASS | +| multitalker-parakeet-streaming-0.6b-v1 | Transcribe (single_speaker_mode, offline / cache-aware att_context_size=[70,13], 1.12s chunk) | explicit en | `build/bin/transcribe-cli -m models/multitalker-parakeet-streaming-0.6b-v1/multitalker-parakeet-streaming-0.6b-v1-F32.gguf --language en samples/jfk.wav` | English transcript with PnC | TODO | +| multitalker-parakeet-streaming-0.6b-v1 | Transcribe (single_speaker_mode) — no-hint | auto/default | `build/bin/transcribe-cli -m samples/jfk.wav` (no `--language`; English-only checkpoint) | English transcript with PnC | TODO | +| multitalker-parakeet-streaming-0.6b-v1 | Punctuation/casing | output | same as the explicit-en row | output contains capital letters and `,.?!` | TODO | +| multitalker-parakeet-streaming-0.6b-v1 | Streaming (single_speaker_mode, cache reuse across chunks) | streaming | `build/bin/transcribe-cli -m --language en --backend cpu --threads 1 --stream-chunk-ms 1120 --stream-att-right 13 samples/jfk.wav` | byte-equal transcript vs single-speaker one-shot at the default `att_context_size=[70,13]` (1.12s chunk) | TODO | +| multitalker-parakeet-streaming-0.6b-v1 | Other latency settings ([70,0]/[70,1]/[70,6]/[70,13]) | runtime-selectable att_context_size | `… --stream-chunk-ms 1120 --stream-att-right {0,1,6,13} …` | all four R settings stream a valid single-speaker transcript; R=6/13 byte-equal to one-shot, R=0/1 differ only in trailing punctuation (lower lookahead) | TODO | +| multitalker-parakeet-streaming-0.6b-v1 | Batch (offline, single_speaker_mode) | run_batch fast path | `uv run scripts/batch_parity.py --model --list --batch-sizes 2,4,8 --backend cpu --language en` + `uv run scripts/batch_tensor_parity.py --model --wav samples/jfk.wav --batch 4 --backend cpu` | text byte-equal vs serial at sizes 2/4/8; CPU tensor parity bit-exact (max_abs=0.0) at batch=4 on jfk.wav | TODO | +| multitalker-parakeet-streaming-0.6b-v1 | Multitalker / speaker-attributed ASR (speaker-kernel injection + external Sortformer diarization + multi-instance + SegLST) | multitalker | (no runtime surface today) | per-speaker cpWER vs NeMo `speech_to_text_multitalker_streaming_infer.py` on AMI/CH109/Mixer6 | OUT OF SCOPE — requires (1) a ported streaming Sortformer diarizer, (2) speaker-kernel injection tensors + layer-0 forward, (3) N-instance-per-speaker orchestration, (4) a SegLST speaker-tagged output contract; none exist in transcribe.cpp today. Brought back in scope by a dedicated multitalker workstream (see the accompanying brief). | +| multitalker-parakeet-streaming-0.6b-v1 | Speaker diarization (produce speaker turns) | diarization | (not a capability of this checkpoint) | n/a | OUT OF SCOPE — this checkpoint CONSUMES external diarization (Sortformer); it does not produce speaker turns. Brought back in scope only if a Sortformer diarizer is ported as a separate family. | | all variants | Word timestamps | only if exposed | `transcribe-cli --timestamps word -m ` (any variant) | per-word `t0_ms`/`t1_ms` in JSON output | PASS — derived host-side from emit-frame indices (TDT/RNNT) or per-frame argmax (CTC); same code path as the existing v2/v3 word-timestamp gate, no per-variant differences | ## Open decisions before Stage 3 (convert) diff --git a/reports/porting/parakeet/multitalker-parakeet-streaming-0.6b-v1/intake.json b/reports/porting/parakeet/multitalker-parakeet-streaming-0.6b-v1/intake.json new file mode 100644 index 00000000..44dc09c6 --- /dev/null +++ b/reports/porting/parakeet/multitalker-parakeet-streaming-0.6b-v1/intake.json @@ -0,0 +1,232 @@ +{ + "schema_version": "transcribe-intake-v1", + "family": "parakeet", + "hf_repo": "nvidia/multitalker-parakeet-streaming-0.6b-v1", + "hf_revision": "8749fc71fd6e2d88ef230159bbf2aea69b524ee1", + "sources": { + "config": { + "kind": "hf_file", + "path": "config.json", + "status": "missing" + }, + "preprocessor": { + "kind": "hf_file", + "path": "preprocessor_config.json|feature_extractor_config.json", + "status": "missing" + }, + "tokenizer_config": { + "kind": "hf_file", + "path": "tokenizer_config.json", + "status": "missing" + }, + "tokenizer_json": { + "kind": "hf_file", + "path": "tokenizer.json", + "status": "missing" + }, + "generation_config": { + "kind": "hf_file", + "path": "generation_config.json", + "status": "missing" + }, + "safetensors_metadata": { + "kind": "hf_api", + "path": "HfApi.get_safetensors_metadata", + "status": "missing", + "detail": "header-only floating dtype distribution; no tensor payloads downloaded" + }, + "nemo_model_config": { + "kind": "hf_file", + "path": "multitalker-parakeet-streaming-0.6b-v1.nemo::model_config.yaml", + "status": "found", + "detail": "streamed from the .nemo tar; canonical for frontend/tokenizer/encoder/decoder/joint and the speaker-kernel additions" + } + }, + "variants": [ + { + "name": "multitalker-parakeet-streaming-0.6b-v1", + "memory_gb": 2.32, + "files": [ + "multitalker-parakeet-streaming-0.6b-v1.nemo" + ] + } + ], + "config": { + "architecture_candidates": [ + "encoder-transducer" + ], + "key_fields": { + "architectures": [ + "EncDecMultiTalkerRNNTBPEModel" + ], + "target": "nemo.collections.asr.models.multitalker_asr_models.EncDecMultiTalkerRNNTBPEModel", + "model_type": "parakeet_multitalker_rnnt_cache_aware", + "nemo_version": "2.6.0rc0", + "based_on": "nvidia/nemotron-speech-streaming-en-0.6b (fine-tuned; ASR backbone identical)", + "encoder.type": "Cache-Aware FastConformer (chunked_limited attention + chunked causal convs) + speaker-kernel injection", + "encoder.n_layers": 24, + "encoder.d_model": 1024, + "encoder.n_heads": 8, + "encoder.ff_expansion_factor": 4, + "encoder.conv_kernel_size": 9, + "encoder.conv_context_size": "causal", + "encoder.conv_norm_type": "layer_norm", + "encoder.use_bias": false, + "encoder.subsampling": "dw_striding", + "encoder.subsampling_factor": 8, + "encoder.subsampling_conv_channels": 256, + "encoder.causal_downsampling": true, + "encoder.self_attention_model": "rel_pos", + "encoder.att_context_style": "chunked_limited", + "encoder.att_context_size.options": [ + [ + 70, + 13 + ], + [ + 70, + 6 + ], + [ + 70, + 1 + ], + [ + 70, + 0 + ] + ], + "encoder.att_context_size.units": "80ms frames", + "encoder.att_context_size.default": [ + 70, + 13 + ], + "encoder.feat_in": 128, + "decoder.type": "RNNTDecoder (predictor)", + "decoder.pred_rnn_layers": 2, + "decoder.pred_hidden": 640, + "decoder.blank_as_pad": true, + "joint.type": "RNNTJoint", + "joint.joint_hidden": 640, + "joint.durations": null, + "joint.num_extra_outputs": 0, + "tokenizer.type": "bpe", + "tokenizer.vocab_size": 1024, + "tokenizer.has_punctuation_capitalization": true, + "decoding.strategy": "greedy_batch", + "decoding.greedy.max_symbols": 10, + "multitalker.spk_kernel_type": "ff", + "multitalker.spk_kernel_layers": [ + 0 + ], + "multitalker.add_bg_spk_kernel": true, + "multitalker.freeze_diar": true, + "multitalker.spk_supervision_strategy": "rttm (train) / diar (inference)", + "multitalker.num_speakers": 4, + "multitalker.num_mel_frame_per_asr_frame": 8, + "multitalker.external_diar_model": "nvidia/diar_streaming_sortformer_4spk-v2.1 (Streaming Sortformer; NOT shipped in this repo)", + "multitalker.output_format": "SegLST (speaker-tagged segments)" + }, + "varying_across_variants": [] + }, + "dtype": { + "expected": "float32", + "source": "manual", + "evidence": "NeMo .nemo archive ships the model as a float32 PyTorch state_dict (no safetensors), consistent with every parakeet-family variant in this repo including the base nemotron-speech-streaming-en-0.6b it is fine-tuned from.", + "details": { + "config_declared": null, + "header_distribution": {} + }, + "expected_f32_tensors": [] + }, + "frontend": { + "sample_rate": 16000, + "n_mels": 128, + "hop_length": 160, + "fft_size": 512, + "window": "hann_periodic", + "normalization": "none", + "preemphasis": null, + "dither": 1e-05, + "center": true, + "padding_mode": "reflect", + "mel_filterbank_norm": "slaney" + }, + "tokenizer": { + "type": "sentencepiece", + "vocab_size": 1024, + "special_tokens": { + "blank": 1024 + }, + "has_language_tokens": false, + "vocab_sha256": null + }, + "capabilities": { + "languages": [ + "en" + ], + "language_detection": false, + "translation": false, + "timestamps": [ + "token", + "word" + ], + "streaming": true, + "speaker_diarization": false + }, + "upstream_benchmarks": [ + { + "dataset": "LibriSpeech test-clean (single_speaker_mode=True, 1.12s chunk)", + "language": "en", + "metric": "wer", + "score": 2.19, + "score_unit": "percent", + "source": "https://huggingface.co/nvidia/multitalker-parakeet-streaming-0.6b-v1", + "notes": "Single-speaker-mode ASR path (multitalker disabled). This is the LibriSpeech-comparable English acceptance target for a first port. whisper-normalizer PnC-stripped scoring, same as the base model." + }, + { + "dataset": "LibriSpeech test-other (single_speaker_mode=True, 1.12s chunk)", + "language": "en", + "metric": "wer", + "score": 4.76, + "score_unit": "percent", + "source": "https://huggingface.co/nvidia/multitalker-parakeet-streaming-0.6b-v1", + "notes": "Single-speaker-mode ASR path." + }, + { + "dataset": "Open ASR Leaderboard (8-set avg, single_speaker_mode=True)", + "language": "en", + "metric": "wer", + "score": 7.44, + "score_unit": "percent", + "source": "https://huggingface.co/nvidia/multitalker-parakeet-streaming-0.6b-v1", + "notes": "Single-speaker mode. Base nemotron-speech-streaming-en-0.6b is 7.16; the multitalker fine-tune preserves single-speaker performance (+0.28 avg)." + }, + { + "dataset": "AMI-IHM (multitalker, Streaming Sortformer v2 diarization)", + "language": "en", + "metric": "other", + "score": 21.26, + "score_unit": "percent", + "source": "https://huggingface.co/nvidia/multitalker-parakeet-streaming-0.6b-v1", + "notes": "metric=cpWER (concatenated minimum-permutation WER). MULTITALKER path (context only, NOT a v1 gate). cpWER requires external Sortformer diarization + per-speaker instances + SegLST. AMI-SDM 37.44, CH109 15.81, Mixer6 23.81." + } + ], + "reference_framework": "nemo", + "reference_rationale": "NVIDIA NeMo first-party framework. Same rationale as the rest of the parakeet family and the base model it is fine-tuned from, nemotron-speech-streaming-en-0.6b: the HF repo ships only a `.nemo` tar archive (consumable by nemo.collections.asr.models.ASRModel.from_pretrained) plus helper scripts; there is no Transformers modeling class and no safetensors. The class is EncDecMultiTalkerRNNTBPEModel. The multitalker oracle is NeMo's `examples/asr/asr_cache_aware_streaming/speech_to_text_multitalker_streaming_infer.py` plus the SpeakerTaggedASR helper in `nemo.collections.asr.parts.utils.multispk_transcribe_utils` and the shipped `multitalker_transcript_config.py`. CRITICAL: the multitalker path additionally requires a SEPARATE external streaming diarization model (nvidia/diar_streaming_sortformer_4spk-v2.1, a Sortformer EncLabelModel) that is NOT part of this checkpoint; without it only single_speaker_mode is runnable. The base ASR backbone (frontend, cache-aware FastConformer encoder, RNN-T decoder/joint, tokenizer) is byte-for-byte the same shape as the already-ported nemotron-speech-streaming-en-0.6b.", + "architecture_pattern": "encoder-transducer", + "known_risks": [ + "MULTITALKER core mechanism (speaker-kernel injection). The only weight-level addition vs the base streaming model is a learnable speaker kernel of type `ff` injected at encoder layer 0 (`spk_kernel_layers=[0]`), plus a background-speaker kernel (`add_bg_spk_kernel=true`). The kernel is generated from per-frame per-speaker speech-activity supervision (diarization). At inference the supervision comes from an external Sortformer diarizer (spk_supervision='diar'). This injection modulates the pre-encode features so the encoder locks onto one target speaker. Porting it requires (a) new GGUF tensors for the speaker-kernel FF module + bg kernel, (b) a per-frame speaker-activity input to the encoder, (c) applying the kernel at layer 0. This is the central new architecture and has NO analogue in the existing parakeet code.", + "EXTERNAL diarization dependency (hard blocker for the multitalker path). Multitalker/speaker-attributed ASR is UNUSABLE without a streaming speaker diarization model (nvidia/diar_streaming_sortformer_4spk-v2.1, a NEST/Sortformer EncLabelModel) that is a completely separate checkpoint and architecture (Sortformer, not FastConformer-RNNT). transcribe.cpp has no diarization runtime today. A first port either (a) ports only single_speaker_mode (no diarization, collapses to plain streaming ASR), or (b) takes on the much larger scope of also porting Sortformer + the streaming speaker cache. This scope split is the primary decision for the family and is the subject of the accompanying brief.", + "MULTI-INSTANCE orchestration. The model runs ONE encoder instance per speaker (max_num_of_spks=4, num_speakers=4): every instance sees the same audio but a different speaker kernel, and produces that speaker's transcript. Runtime must fan out N encoder+decoder passes per chunk, each with its own cache-aware K/V + conv + decoder state, then merge per-speaker hypotheses into a single speaker-tagged transcript. This is an orchestration layer above the single-model runtime, plus per-speaker cache management (`cache_gating`, `cache_gating_buffer_size=2`) and a `parallel_speaker_strategy` flag.", + "OUTPUT contract is SegLST, not a flat transcript. Multitalker output is speaker-tagged segments (SegLST: {session_id, speaker, start_time, end_time, words}) merged across instances and across streaming chunks, with sentence-break heuristics (word_window=50, sent_break_sec=30, fix_prev_words_count=5). transcribe.cpp's current output contract is a single flat transcript; exposing multitalker means a new output shape and CLI surface (per-speaker channels / diarized JSON).", + "single_speaker_mode is the LibriSpeech-comparable path. Setting single_speaker_mode=True disables the multitalker machinery and the model behaves as a plain cache-aware streaming RNN-T (LS test-clean 2.19 / test-other 4.76). This path reuses the existing nemotron-speech-streaming-en-0.6b runtime almost entirely and is the realistic v1 acceptance target. Confirm at convert time whether single_speaker_mode requires a specific speaker-kernel input (e.g. all-active / bg kernel) or fully bypasses layer-0 injection.", + "Encoder attention is chunked_limited (carried from the base). att_context_style='chunked_limited', att_context_size menu [[70,13],[70,6],[70,1],[70,0]] in 80ms frames, default [70,13] = 1.12s latency. Frames are grouped into chunks of R+1 and each chunk sees itself + floor(L/(R+1)) prior chunks; this is NOT a per-frame sliding window. Already implemented for nemotron-speech-streaming-en-0.6b; reuse verbatim.", + "Chunked causal depthwise convolutions (carried from the base). conv_context_size='causal', causal_downsampling=true. Per-chunk causal conv with state carried across chunks; in one-shot mode the state zero-inits but the causal pattern must still be applied. Already handled by the existing cache-aware parakeet code.", + "Frontend has NO normalization and NO preemphasis (carried from the base, confirmed from model_config.yaml: normalize='NA'). Differs from every non-streaming parakeet variant (which use preemphasis=0.97 + per_feature norm). The converter must emit the 'no frontend normalization' GGUF KV and the C++ frontend must skip per-feature norm; a silent fall-through degrades WER without changing shapes. Already handled for nemotron-speech-streaming-en-0.6b.", + "RNN-T backbone is identical to nemotron-speech-streaming-en-0.6b: 24L / d=1024 / 8h / kernel=9 / dw_striding x8 / subsampling_conv_channels=256, RNNTDecoder 2x640, RNNTJoint 640 (no duration head, joint.durations=None), BPE vocab 1024 with blank at 1024, PnC in vocab. The existing convert-parakeet.py + src/arch/parakeet handle all of this; only the speaker-kernel tensors and the multitalker forward path are new.", + "WER normalization: whisper-normalizer (PnC strip + lowercase + NFC + number norm) on both hyp and ref, same as the base model, required for the single-speaker LibriSpeech numbers to be comparable.", + "License: NVIDIA Open Model License (not Apache-2.0), same as the base. Affects the Stage 8 HF card YAML." + ], + "intake_gaps": [] +} \ No newline at end of file diff --git a/tests/golden/parakeet/multitalker-parakeet-streaming-0.6b-v1.manifest.json b/tests/golden/parakeet/multitalker-parakeet-streaming-0.6b-v1.manifest.json new file mode 100644 index 00000000..d4e9520c --- /dev/null +++ b/tests/golden/parakeet/multitalker-parakeet-streaming-0.6b-v1.manifest.json @@ -0,0 +1,12 @@ +{ + "schema": "transcribe-golden-manifest-v1", + "family": "parakeet", + "variant": "multitalker-parakeet-streaming-0.6b-v1", + "source_model": { + "hf_repo": "nvidia/multitalker-parakeet-streaming-0.6b-v1", + "hf_revision": "8749fc71fd6e2d88ef230159bbf2aea69b524ee1" + }, + "expected_dtype": "float32", + "dtype_source": "manual", + "_skeleton": true +} \ No newline at end of file From a06b1aec21a290dd79c047834780c7bb3863bbd6 Mon Sep 17 00:00:00 2001 From: CJ Pais Date: Tue, 7 Jul 2026 17:02:57 -0700 Subject: [PATCH 02/11] stage2 --- docs/porting/families/parakeet.md | 18 + ...r-parakeet-streaming-0.6b-v1.manifest.json | 43 +- ...ultitalker-parakeet-streaming-0.6b-v1.json | 91 +++ ...-parakeet-streaming-0.6b-v1.streaming.json | 764 ++++++++++++++++++ 4 files changed, 915 insertions(+), 1 deletion(-) create mode 100644 tests/tolerances/multitalker-parakeet-streaming-0.6b-v1.json create mode 100644 tests/tolerances/multitalker-parakeet-streaming-0.6b-v1.streaming.json diff --git a/docs/porting/families/parakeet.md b/docs/porting/families/parakeet.md index 17fabe71..5f09006e 100644 --- a/docs/porting/families/parakeet.md +++ b/docs/porting/families/parakeet.md @@ -190,6 +190,24 @@ be resolved before the corresponding port enters porting-3-convert: convert time from `model.cfg.preprocessor.features` inside the .nemo archive. Wrong default silently degrades WER without changing tensor shapes elsewhere. +5. **Layer-0 speaker-kernel injection** (`multitalker-parakeet-streaming-0.6b-v1`): + RESOLVED at Stage 2. The single-speaker path is NOT numerically identical + to `nemotron-speech-streaming-en-0.6b`. `EncDecMultiTalkerRNNTBPEModel` + registers a `forward_pre_hook` on `encoder.layers[0]` (from + `SpeakerKernelMixin`, `spk_kernel_layers=[0]`) that fires + **unconditionally** — even with no diarization / speaker targets. In + single-speaker mode the hook computes, at the INPUT of conformer layer 0: + `x += spk_kernels.0(x)` (speaker mask defaults to all-ones) then + `x += bg_spk_kernels.0(0)` (background mask defaults to zeros → a constant + bias vector). Each kernel is an FF block: + `Linear(1024,1024) → ReLU → Dropout(id at eval) → Linear(1024,1024)` with + bias. Stage 3 MUST export `spk_kernels.0.{0,3}.{weight,bias}` and + `bg_spk_kernels.0.{0,3}.{weight,bias}` (2 FF modules) and emit a GGUF KV + marking layer-0 injection; Stage 4 MUST apply the injection at the layer-0 + input. Skipping it silently degrades single-speaker WER (a structural-cfg + distinction, not a shape change). The full multitalker path (per-frame + diarization mask instead of all-ones, N-instance orchestration, SegLST) is + OUT OF SCOPE per Stage 1 — see the multitalker integration brief. ## Tooling: NeMo-aware preflight diff --git a/tests/golden/parakeet/multitalker-parakeet-streaming-0.6b-v1.manifest.json b/tests/golden/parakeet/multitalker-parakeet-streaming-0.6b-v1.manifest.json index d4e9520c..8c0c5dee 100644 --- a/tests/golden/parakeet/multitalker-parakeet-streaming-0.6b-v1.manifest.json +++ b/tests/golden/parakeet/multitalker-parakeet-streaming-0.6b-v1.manifest.json @@ -6,7 +6,48 @@ "hf_repo": "nvidia/multitalker-parakeet-streaming-0.6b-v1", "hf_revision": "8749fc71fd6e2d88ef230159bbf2aea69b524ee1" }, + "reference": { + "kind": "nemo", + "source": "https://github.com/NVIDIA-NeMo/NeMo", + "revision": "6967f48fda2a", + "entrypoint": "scripts/dump_reference_parakeet_nemo.py", + "note": "EncDecMultiTalkerRNNTBPEModel run in single-speaker mode (no diarization/speaker targets set -> layer-0 speaker kernel applies with an all-ones mask; bg kernel with a zeros mask). Multitalker path is OUT OF SCOPE per Stage 1 sign-off." + }, "expected_dtype": "float32", "dtype_source": "manual", - "_skeleton": true + "frontend": { + "sample_rate": 16000, + "n_mels": 128, + "hop_length": 160, + "fft_size": 512, + "win_length": 400, + "window": "hann_periodic", + "normalization": "none", + "preemphasis": null, + "dither": 1e-05 + }, + "tokenizer_summary": { + "type": "bpe", + "vocab_size": 1024, + "special_tokens": { + "blank": 1024 + } + }, + "capabilities": { + "languages": [ + "en" + ], + "language_detection": false, + "translation": false, + "timestamps": [ + "token", + "word" + ], + "streaming": true, + "speaker_diarization": false + }, + "tolerance_file": "tests/tolerances/multitalker-parakeet-streaming-0.6b-v1.json", + "cases": [ + "jfk" + ] } \ No newline at end of file diff --git a/tests/tolerances/multitalker-parakeet-streaming-0.6b-v1.json b/tests/tolerances/multitalker-parakeet-streaming-0.6b-v1.json new file mode 100644 index 00000000..aa4bc27f --- /dev/null +++ b/tests/tolerances/multitalker-parakeet-streaming-0.6b-v1.json @@ -0,0 +1,91 @@ +{ + "_comment": [ + "multitalker-parakeet-streaming-0.6b-v1 PROVISIONAL per-tensor tolerances (Stage 2 oracle).", + "", + "Magnitude-aware: max_abs = max(1e-4 * p99_abs, 1e-6);", + " mean_abs = max(1e-5 * rms, 1e-6).", + "When a tensor name appears in multiple (case, stage_dir) sidecars the", + "MAX p99_abs / rms across them is used.", + "", + "PROVISIONAL: Stage 4 finalizes against observed C++ drift. Every entry", + "carries _provisional: true. DO NOT ship while _provisional entries remain.", + "", + "Model note: this is EncDecMultiTalkerRNNTBPEModel run in SINGLE-SPEAKER", + "mode. The base ASR backbone is identical to nemotron-speech-streaming-en-0.6b,", + "BUT a speaker-kernel FF residual is applied at the INPUT of conformer", + "layer 0 (forward_pre_hook, applied unconditionally): x += spk_ff(x) with an", + "all-ones mask, then x += bg_ff(0) (constant bias). Stage 3 MUST convert", + "spk_kernels.0.* and bg_spk_kernels.0.* and Stage 4 MUST apply this injection;", + "the single-speaker path is NOT numerically identical to the base model." + ], + "dec.embed.0": { + "max_abs": 1e-06, + "mean_abs": 1e-06, + "_provisional": true + }, + "dec.enc_out": { + "max_abs": 1.9078139215707727e-05, + "mean_abs": 1e-06, + "_provisional": true + }, + "dec.joint.0": { + "max_abs": 0.005523143569946289, + "mean_abs": 0.00032965458532807114, + "_provisional": true + }, + "dec.lstm.0.c.0": { + "max_abs": 7.170465892553333e-05, + "mean_abs": 2.375066490445493e-06, + "_provisional": true + }, + "dec.lstm.0.h.0": { + "max_abs": 3.142599546909333e-05, + "mean_abs": 1.0800957467118525e-06, + "_provisional": true + }, + "dec.lstm.1.c.0": { + "max_abs": 8.615481072664261e-05, + "mean_abs": 4.230338949264561e-06, + "_provisional": true + }, + "dec.lstm.1.h.0": { + "max_abs": 3.389731752872467e-05, + "mean_abs": 1e-06, + "_provisional": true + }, + "enc.block.0.out": { + "max_abs": 0.003004386568069458, + "mean_abs": 8.872089894130494e-05, + "_provisional": true + }, + "enc.block.12.out": { + "max_abs": 0.003371430587768542, + "mean_abs": 7.71841725708955e-05, + "_provisional": true + }, + "enc.block.23.out": { + "max_abs": 1.9078139215707727e-05, + "mean_abs": 1e-06, + "_provisional": true + }, + "enc.final": { + "max_abs": 1.9078139215707727e-05, + "mean_abs": 1e-06, + "_provisional": true + }, + "enc.mel.in": { + "max_abs": 0.0016611215076446535, + "mean_abs": 0.0001012863475937363, + "_provisional": true + }, + "enc.pos_emb": { + "max_abs": 9.999976158142091e-05, + "mean_abs": 7.071067828517533e-06, + "_provisional": true + }, + "enc.pre_encode.out": { + "max_abs": 0.06187373504638646, + "mean_abs": 0.0017322517756891871, + "_provisional": true + } +} \ No newline at end of file diff --git a/tests/tolerances/multitalker-parakeet-streaming-0.6b-v1.streaming.json b/tests/tolerances/multitalker-parakeet-streaming-0.6b-v1.streaming.json new file mode 100644 index 00000000..81941127 --- /dev/null +++ b/tests/tolerances/multitalker-parakeet-streaming-0.6b-v1.streaming.json @@ -0,0 +1,764 @@ +{ + "_comment": [ + "multitalker-parakeet-streaming-0.6b-v1 PROVISIONAL per-tensor tolerances (Stage 2 oracle).", + "", + "Magnitude-aware: max_abs = max(1e-4 * p99_abs, 1e-6);", + " mean_abs = max(1e-5 * rms, 1e-6).", + "When a tensor name appears in multiple (case, stage_dir) sidecars the", + "MAX p99_abs / rms across them is used.", + "", + "PROVISIONAL: Stage 4 finalizes against observed C++ drift. Every entry", + "carries _provisional: true. DO NOT ship while _provisional entries remain.", + "Streaming stage tensors (att_context_size=[70,13], 1.12s chunk)." + ], + "stream.chunk.0.cache_lc_in_0": { + "max_abs": 1e-06, + "mean_abs": 1e-06, + "_provisional": true + }, + "stream.chunk.0.cache_lc_in_12": { + "max_abs": 1e-06, + "mean_abs": 1e-06, + "_provisional": true + }, + "stream.chunk.0.cache_lc_in_23": { + "max_abs": 1e-06, + "mean_abs": 1e-06, + "_provisional": true + }, + "stream.chunk.0.cache_lc_out_0": { + "max_abs": 1.3008432164788315e-05, + "mean_abs": 1e-06, + "_provisional": true + }, + "stream.chunk.0.cache_lc_out_12": { + "max_abs": 2.8398949261754838e-06, + "mean_abs": 1e-06, + "_provisional": true + }, + "stream.chunk.0.cache_lc_out_23": { + "max_abs": 1.1010672606527852e-05, + "mean_abs": 1e-06, + "_provisional": true + }, + "stream.chunk.0.cache_lt_in_0": { + "max_abs": 1e-06, + "mean_abs": 1e-06, + "_provisional": true + }, + "stream.chunk.0.cache_lt_in_12": { + "max_abs": 1e-06, + "mean_abs": 1e-06, + "_provisional": true + }, + "stream.chunk.0.cache_lt_in_23": { + "max_abs": 1e-06, + "mean_abs": 1e-06, + "_provisional": true + }, + "stream.chunk.0.cache_lt_out_0": { + "max_abs": 0.00129465841293335, + "mean_abs": 3.407372497108534e-05, + "_provisional": true + }, + "stream.chunk.0.cache_lt_out_12": { + "max_abs": 0.0017155433769226083, + "mean_abs": 3.6446279329674886e-05, + "_provisional": true + }, + "stream.chunk.0.cache_lt_out_23": { + "max_abs": 0.0005470749974250801, + "mean_abs": 1.2106562129439416e-05, + "_provisional": true + }, + "stream.chunk.0.channel_len": { + "max_abs": 0.0014, + "mean_abs": 0.00014000000000000001, + "_provisional": true + }, + "stream.chunk.0.enc_out": { + "max_abs": 1.7253484055399895e-05, + "mean_abs": 1e-06, + "_provisional": true + }, + "stream.chunk.0.mel_in": { + "max_abs": 0.0016635532379150391, + "mean_abs": 0.00010467764062887827, + "_provisional": true + }, + "stream.chunk.1.cache_lc_in_0": { + "max_abs": 1.3008432164788315e-05, + "mean_abs": 1e-06, + "_provisional": true + }, + "stream.chunk.1.cache_lc_in_12": { + "max_abs": 2.8398949261754838e-06, + "mean_abs": 1e-06, + "_provisional": true + }, + "stream.chunk.1.cache_lc_in_23": { + "max_abs": 1.1010672606527852e-05, + "mean_abs": 1e-06, + "_provisional": true + }, + "stream.chunk.1.cache_lc_out_0": { + "max_abs": 1.5874018907547004e-05, + "mean_abs": 1e-06, + "_provisional": true + }, + "stream.chunk.1.cache_lc_out_12": { + "max_abs": 3.661312390118844e-06, + "mean_abs": 1e-06, + "_provisional": true + }, + "stream.chunk.1.cache_lc_out_23": { + "max_abs": 1.2638793796300969e-05, + "mean_abs": 1e-06, + "_provisional": true + }, + "stream.chunk.1.cache_lt_in_0": { + "max_abs": 0.00129465841293335, + "mean_abs": 3.407372497108534e-05, + "_provisional": true + }, + "stream.chunk.1.cache_lt_in_12": { + "max_abs": 0.0017155433769226083, + "mean_abs": 3.6446279329674886e-05, + "_provisional": true + }, + "stream.chunk.1.cache_lt_in_23": { + "max_abs": 0.0005470749974250801, + "mean_abs": 1.2106562129439416e-05, + "_provisional": true + }, + "stream.chunk.1.cache_lt_out_0": { + "max_abs": 0.0012747896041870128, + "mean_abs": 3.367902436206685e-05, + "_provisional": true + }, + "stream.chunk.1.cache_lt_out_12": { + "max_abs": 0.001866611431121827, + "mean_abs": 3.916475561918408e-05, + "_provisional": true + }, + "stream.chunk.1.cache_lt_out_23": { + "max_abs": 0.0005453417639732367, + "mean_abs": 1.4001124989410301e-05, + "_provisional": true + }, + "stream.chunk.1.channel_len": { + "max_abs": 0.0028, + "mean_abs": 0.00028000000000000003, + "_provisional": true + }, + "stream.chunk.1.enc_out": { + "max_abs": 1.9079485833644858e-05, + "mean_abs": 1e-06, + "_provisional": true + }, + "stream.chunk.1.mel_in": { + "max_abs": 0.0016477878952026366, + "mean_abs": 9.115775346884796e-05, + "_provisional": true + }, + "stream.chunk.2.cache_lc_in_0": { + "max_abs": 1.5874018907547004e-05, + "mean_abs": 1e-06, + "_provisional": true + }, + "stream.chunk.2.cache_lc_in_12": { + "max_abs": 3.661312390118844e-06, + "mean_abs": 1e-06, + "_provisional": true + }, + "stream.chunk.2.cache_lc_in_23": { + "max_abs": 1.2638793796300969e-05, + "mean_abs": 1e-06, + "_provisional": true + }, + "stream.chunk.2.cache_lc_out_0": { + "max_abs": 1.6420035868883166e-05, + "mean_abs": 1e-06, + "_provisional": true + }, + "stream.chunk.2.cache_lc_out_12": { + "max_abs": 4.235666163265739e-06, + "mean_abs": 1e-06, + "_provisional": true + }, + "stream.chunk.2.cache_lc_out_23": { + "max_abs": 1.3597931206226375e-05, + "mean_abs": 1e-06, + "_provisional": true + }, + "stream.chunk.2.cache_lt_in_0": { + "max_abs": 0.0012747896041870128, + "mean_abs": 3.367902436206685e-05, + "_provisional": true + }, + "stream.chunk.2.cache_lt_in_12": { + "max_abs": 0.001866611431121827, + "mean_abs": 3.916475561918408e-05, + "_provisional": true + }, + "stream.chunk.2.cache_lt_in_23": { + "max_abs": 0.0005453417639732367, + "mean_abs": 1.4001124989410301e-05, + "_provisional": true + }, + "stream.chunk.2.cache_lt_out_0": { + "max_abs": 0.001035860824584961, + "mean_abs": 2.7844846226694173e-05, + "_provisional": true + }, + "stream.chunk.2.cache_lt_out_12": { + "max_abs": 0.0020686249885559084, + "mean_abs": 4.04995574051368e-05, + "_provisional": true + }, + "stream.chunk.2.cache_lt_out_23": { + "max_abs": 0.00043792397785186773, + "mean_abs": 1.1110986077387435e-05, + "_provisional": true + }, + "stream.chunk.2.channel_len": { + "max_abs": 0.004200000000000001, + "mean_abs": 0.00042, + "_provisional": true + }, + "stream.chunk.2.enc_out": { + "max_abs": 1.8786662966012952e-05, + "mean_abs": 1e-06, + "_provisional": true + }, + "stream.chunk.2.mel_in": { + "max_abs": 0.0015503223133087158, + "mean_abs": 0.00011653641447048212, + "_provisional": true + }, + "stream.chunk.3.cache_lc_in_0": { + "max_abs": 1.6420035868883166e-05, + "mean_abs": 1e-06, + "_provisional": true + }, + "stream.chunk.3.cache_lc_in_12": { + "max_abs": 4.235666163265739e-06, + "mean_abs": 1e-06, + "_provisional": true + }, + "stream.chunk.3.cache_lc_in_23": { + "max_abs": 1.3597931206226375e-05, + "mean_abs": 1e-06, + "_provisional": true + }, + "stream.chunk.3.cache_lc_out_0": { + "max_abs": 1.762249697744852e-05, + "mean_abs": 1e-06, + "_provisional": true + }, + "stream.chunk.3.cache_lc_out_12": { + "max_abs": 4.634001653641463e-06, + "mean_abs": 1e-06, + "_provisional": true + }, + "stream.chunk.3.cache_lc_out_23": { + "max_abs": 1.4167193681001697e-05, + "mean_abs": 1e-06, + "_provisional": true + }, + "stream.chunk.3.cache_lt_in_0": { + "max_abs": 0.001035860824584961, + "mean_abs": 2.7844846226694173e-05, + "_provisional": true + }, + "stream.chunk.3.cache_lt_in_12": { + "max_abs": 0.0020686249885559084, + "mean_abs": 4.04995574051368e-05, + "_provisional": true + }, + "stream.chunk.3.cache_lt_in_23": { + "max_abs": 0.00043792397785186773, + "mean_abs": 1.1110986077387435e-05, + "_provisional": true + }, + "stream.chunk.3.cache_lt_out_0": { + "max_abs": 0.0011561623067855844, + "mean_abs": 3.139138916511903e-05, + "_provisional": true + }, + "stream.chunk.3.cache_lt_out_12": { + "max_abs": 0.0019240166225433354, + "mean_abs": 4.012968807607016e-05, + "_provisional": true + }, + "stream.chunk.3.cache_lt_out_23": { + "max_abs": 0.0006169222645759585, + "mean_abs": 1.515453647116478e-05, + "_provisional": true + }, + "stream.chunk.3.channel_len": { + "max_abs": 0.0056, + "mean_abs": 0.0005600000000000001, + "_provisional": true + }, + "stream.chunk.3.enc_out": { + "max_abs": 1.6622247621417004e-05, + "mean_abs": 1e-06, + "_provisional": true + }, + "stream.chunk.3.mel_in": { + "max_abs": 0.001660875274658203, + "mean_abs": 9.622120975163936e-05, + "_provisional": true + }, + "stream.chunk.4.cache_lc_in_0": { + "max_abs": 1.762249697744852e-05, + "mean_abs": 1e-06, + "_provisional": true + }, + "stream.chunk.4.cache_lc_in_12": { + "max_abs": 4.634001653641463e-06, + "mean_abs": 1e-06, + "_provisional": true + }, + "stream.chunk.4.cache_lc_in_23": { + "max_abs": 1.4167193681001697e-05, + "mean_abs": 1e-06, + "_provisional": true + }, + "stream.chunk.4.cache_lc_out_0": { + "max_abs": 1.7965956419706414e-05, + "mean_abs": 1e-06, + "_provisional": true + }, + "stream.chunk.4.cache_lc_out_12": { + "max_abs": 4.886499505490066e-06, + "mean_abs": 1e-06, + "_provisional": true + }, + "stream.chunk.4.cache_lc_out_23": { + "max_abs": 1.5018582731485378e-05, + "mean_abs": 1e-06, + "_provisional": true + }, + "stream.chunk.4.cache_lt_in_0": { + "max_abs": 0.0011561623067855844, + "mean_abs": 3.139138916511903e-05, + "_provisional": true + }, + "stream.chunk.4.cache_lt_in_12": { + "max_abs": 0.0019240166225433354, + "mean_abs": 4.012968807607016e-05, + "_provisional": true + }, + "stream.chunk.4.cache_lt_in_23": { + "max_abs": 0.0006169222645759585, + "mean_abs": 1.515453647116478e-05, + "_provisional": true + }, + "stream.chunk.4.cache_lt_out_0": { + "max_abs": 0.0010491846704483037, + "mean_abs": 2.8803309296311065e-05, + "_provisional": true + }, + "stream.chunk.4.cache_lt_out_12": { + "max_abs": 0.0018458886489868176, + "mean_abs": 4.0284489757683054e-05, + "_provisional": true + }, + "stream.chunk.4.cache_lt_out_23": { + "max_abs": 0.0005636508593559268, + "mean_abs": 1.3180694872271984e-05, + "_provisional": true + }, + "stream.chunk.4.channel_len": { + "max_abs": 0.007, + "mean_abs": 0.0007000000000000001, + "_provisional": true + }, + "stream.chunk.4.enc_out": { + "max_abs": 2.1179130151867858e-05, + "mean_abs": 1e-06, + "_provisional": true + }, + "stream.chunk.4.mel_in": { + "max_abs": 0.0015525638256072996, + "mean_abs": 0.00011270580396839228, + "_provisional": true + }, + "stream.chunk.5.cache_lc_in_0": { + "max_abs": 1.7965956419706414e-05, + "mean_abs": 1e-06, + "_provisional": true + }, + "stream.chunk.5.cache_lc_in_12": { + "max_abs": 4.886499505490066e-06, + "mean_abs": 1e-06, + "_provisional": true + }, + "stream.chunk.5.cache_lc_in_23": { + "max_abs": 1.5018582731485378e-05, + "mean_abs": 1e-06, + "_provisional": true + }, + "stream.chunk.5.cache_lc_out_0": { + "max_abs": 1.8002947837114417e-05, + "mean_abs": 1e-06, + "_provisional": true + }, + "stream.chunk.5.cache_lc_out_12": { + "max_abs": 4.958076093345887e-06, + "mean_abs": 1e-06, + "_provisional": true + }, + "stream.chunk.5.cache_lc_out_23": { + "max_abs": 1.4514548420906077e-05, + "mean_abs": 1e-06, + "_provisional": true + }, + "stream.chunk.5.cache_lt_in_0": { + "max_abs": 0.0010491846704483037, + "mean_abs": 2.8803309296311065e-05, + "_provisional": true + }, + "stream.chunk.5.cache_lt_in_12": { + "max_abs": 0.0018458886489868176, + "mean_abs": 4.0284489757683054e-05, + "_provisional": true + }, + "stream.chunk.5.cache_lt_in_23": { + "max_abs": 0.0005636508593559268, + "mean_abs": 1.3180694872271984e-05, + "_provisional": true + }, + "stream.chunk.5.cache_lt_out_0": { + "max_abs": 0.001214497030258179, + "mean_abs": 3.226833464901226e-05, + "_provisional": true + }, + "stream.chunk.5.cache_lt_out_12": { + "max_abs": 0.001899435621261597, + "mean_abs": 4.010647615056664e-05, + "_provisional": true + }, + "stream.chunk.5.cache_lt_out_23": { + "max_abs": 0.0005633621072769167, + "mean_abs": 1.36036978067309e-05, + "_provisional": true + }, + "stream.chunk.5.channel_len": { + "max_abs": 0.007, + "mean_abs": 0.0007000000000000001, + "_provisional": true + }, + "stream.chunk.5.enc_out": { + "max_abs": 1.8819435313343986e-05, + "mean_abs": 1e-06, + "_provisional": true + }, + "stream.chunk.5.mel_in": { + "max_abs": 0.0016615625114440918, + "mean_abs": 9.371562068416148e-05, + "_provisional": true + }, + "stream.chunk.6.cache_lc_in_0": { + "max_abs": 1.8002947837114417e-05, + "mean_abs": 1e-06, + "_provisional": true + }, + "stream.chunk.6.cache_lc_in_12": { + "max_abs": 4.958076093345887e-06, + "mean_abs": 1e-06, + "_provisional": true + }, + "stream.chunk.6.cache_lc_in_23": { + "max_abs": 1.4514548420906077e-05, + "mean_abs": 1e-06, + "_provisional": true + }, + "stream.chunk.6.cache_lc_out_0": { + "max_abs": 1.759598450362688e-05, + "mean_abs": 1e-06, + "_provisional": true + }, + "stream.chunk.6.cache_lc_out_12": { + "max_abs": 4.974784672260289e-06, + "mean_abs": 1e-06, + "_provisional": true + }, + "stream.chunk.6.cache_lc_out_23": { + "max_abs": 1.4584889575839045e-05, + "mean_abs": 1e-06, + "_provisional": true + }, + "stream.chunk.6.cache_lt_in_0": { + "max_abs": 0.001214497030258179, + "mean_abs": 3.226833464901226e-05, + "_provisional": true + }, + "stream.chunk.6.cache_lt_in_12": { + "max_abs": 0.001899435621261597, + "mean_abs": 4.010647615056664e-05, + "_provisional": true + }, + "stream.chunk.6.cache_lt_in_23": { + "max_abs": 0.0005633621072769167, + "mean_abs": 1.36036978067309e-05, + "_provisional": true + }, + "stream.chunk.6.cache_lt_out_0": { + "max_abs": 0.0011369490146636964, + "mean_abs": 3.1325558439493076e-05, + "_provisional": true + }, + "stream.chunk.6.cache_lt_out_12": { + "max_abs": 0.0020269768905639652, + "mean_abs": 4.305163344313818e-05, + "_provisional": true + }, + "stream.chunk.6.cache_lt_out_23": { + "max_abs": 0.0006229003634452821, + "mean_abs": 1.403079167721605e-05, + "_provisional": true + }, + "stream.chunk.6.channel_len": { + "max_abs": 0.007, + "mean_abs": 0.0007000000000000001, + "_provisional": true + }, + "stream.chunk.6.enc_out": { + "max_abs": 1.9951290562748903e-05, + "mean_abs": 1e-06, + "_provisional": true + }, + "stream.chunk.6.mel_in": { + "max_abs": 0.001640410678863525, + "mean_abs": 9.860686542991035e-05, + "_provisional": true + }, + "stream.chunk.7.cache_lc_in_0": { + "max_abs": 1.759598450362688e-05, + "mean_abs": 1e-06, + "_provisional": true + }, + "stream.chunk.7.cache_lc_in_12": { + "max_abs": 4.974784672260289e-06, + "mean_abs": 1e-06, + "_provisional": true + }, + "stream.chunk.7.cache_lc_in_23": { + "max_abs": 1.4584889575839045e-05, + "mean_abs": 1e-06, + "_provisional": true + }, + "stream.chunk.7.cache_lc_out_0": { + "max_abs": 1.7881013616919533e-05, + "mean_abs": 1e-06, + "_provisional": true + }, + "stream.chunk.7.cache_lc_out_12": { + "max_abs": 4.948631994426254e-06, + "mean_abs": 1e-06, + "_provisional": true + }, + "stream.chunk.7.cache_lc_out_23": { + "max_abs": 1.4707824811339391e-05, + "mean_abs": 1e-06, + "_provisional": true + }, + "stream.chunk.7.cache_lt_in_0": { + "max_abs": 0.0011369490146636964, + "mean_abs": 3.1325558439493076e-05, + "_provisional": true + }, + "stream.chunk.7.cache_lt_in_12": { + "max_abs": 0.0020269768905639652, + "mean_abs": 4.305163344313818e-05, + "_provisional": true + }, + "stream.chunk.7.cache_lt_in_23": { + "max_abs": 0.0006229003634452821, + "mean_abs": 1.403079167721605e-05, + "_provisional": true + }, + "stream.chunk.7.cache_lt_out_0": { + "max_abs": 0.0011851717004776024, + "mean_abs": 3.145116820762147e-05, + "_provisional": true + }, + "stream.chunk.7.cache_lt_out_12": { + "max_abs": 0.0019229018115997324, + "mean_abs": 4.012440119281274e-05, + "_provisional": true + }, + "stream.chunk.7.cache_lt_out_23": { + "max_abs": 0.0006446033411026006, + "mean_abs": 1.522956991252987e-05, + "_provisional": true + }, + "stream.chunk.7.channel_len": { + "max_abs": 0.007, + "mean_abs": 0.0007000000000000001, + "_provisional": true + }, + "stream.chunk.7.enc_out": { + "max_abs": 1.9170742407441134e-05, + "mean_abs": 1e-06, + "_provisional": true + }, + "stream.chunk.7.mel_in": { + "max_abs": 0.0016593084449768069, + "mean_abs": 0.00010335153344210517, + "_provisional": true + }, + "stream.chunk.8.cache_lc_in_0": { + "max_abs": 1.7881013616919533e-05, + "mean_abs": 1e-06, + "_provisional": true + }, + "stream.chunk.8.cache_lc_in_12": { + "max_abs": 4.948631994426254e-06, + "mean_abs": 1e-06, + "_provisional": true + }, + "stream.chunk.8.cache_lc_in_23": { + "max_abs": 1.4707824811339391e-05, + "mean_abs": 1e-06, + "_provisional": true + }, + "stream.chunk.8.cache_lc_out_0": { + "max_abs": 1.7756513878703127e-05, + "mean_abs": 1e-06, + "_provisional": true + }, + "stream.chunk.8.cache_lc_out_12": { + "max_abs": 4.810116160660997e-06, + "mean_abs": 1e-06, + "_provisional": true + }, + "stream.chunk.8.cache_lc_out_23": { + "max_abs": 1.4881478622555733e-05, + "mean_abs": 1e-06, + "_provisional": true + }, + "stream.chunk.8.cache_lt_in_0": { + "max_abs": 0.0011851717004776024, + "mean_abs": 3.145116820762147e-05, + "_provisional": true + }, + "stream.chunk.8.cache_lt_in_12": { + "max_abs": 0.0019229018115997324, + "mean_abs": 4.012440119281274e-05, + "_provisional": true + }, + "stream.chunk.8.cache_lt_in_23": { + "max_abs": 0.0006446033411026006, + "mean_abs": 1.522956991252987e-05, + "_provisional": true + }, + "stream.chunk.8.cache_lt_out_0": { + "max_abs": 0.0011563131313323985, + "mean_abs": 3.155277608997167e-05, + "_provisional": true + }, + "stream.chunk.8.cache_lt_out_12": { + "max_abs": 0.0018374736843109142, + "mean_abs": 3.9883290914659174e-05, + "_provisional": true + }, + "stream.chunk.8.cache_lt_out_23": { + "max_abs": 0.0006105101847648624, + "mean_abs": 1.4465373597920161e-05, + "_provisional": true + }, + "stream.chunk.8.channel_len": { + "max_abs": 0.007, + "mean_abs": 0.0007000000000000001, + "_provisional": true + }, + "stream.chunk.8.enc_out": { + "max_abs": 1.956931285560131e-05, + "mean_abs": 1e-06, + "_provisional": true + }, + "stream.chunk.8.mel_in": { + "max_abs": 0.001651513826370239, + "mean_abs": 9.543340046170219e-05, + "_provisional": true + }, + "stream.chunk.9.cache_lc_in_0": { + "max_abs": 1.7756513878703127e-05, + "mean_abs": 1e-06, + "_provisional": true + }, + "stream.chunk.9.cache_lc_in_12": { + "max_abs": 4.810116160660997e-06, + "mean_abs": 1e-06, + "_provisional": true + }, + "stream.chunk.9.cache_lc_in_23": { + "max_abs": 1.4881478622555733e-05, + "mean_abs": 1e-06, + "_provisional": true + }, + "stream.chunk.9.cache_lc_out_0": { + "max_abs": 1.801336298882962e-05, + "mean_abs": 1e-06, + "_provisional": true + }, + "stream.chunk.9.cache_lc_out_12": { + "max_abs": 4.760253824293622e-06, + "mean_abs": 1e-06, + "_provisional": true + }, + "stream.chunk.9.cache_lc_out_23": { + "max_abs": 1.4701267391443272e-05, + "mean_abs": 1e-06, + "_provisional": true + }, + "stream.chunk.9.cache_lt_in_0": { + "max_abs": 0.0011563131313323985, + "mean_abs": 3.155277608997167e-05, + "_provisional": true + }, + "stream.chunk.9.cache_lt_in_12": { + "max_abs": 0.0018374736843109142, + "mean_abs": 3.9883290914659174e-05, + "_provisional": true + }, + "stream.chunk.9.cache_lt_in_23": { + "max_abs": 0.0006105101847648624, + "mean_abs": 1.4465373597920161e-05, + "_provisional": true + }, + "stream.chunk.9.cache_lt_out_0": { + "max_abs": 0.0010901470489501955, + "mean_abs": 2.995375954716209e-05, + "_provisional": true + }, + "stream.chunk.9.cache_lt_out_12": { + "max_abs": 0.0019135723266601567, + "mean_abs": 4.109583343672264e-05, + "_provisional": true + }, + "stream.chunk.9.cache_lt_out_23": { + "max_abs": 0.0006757690634727483, + "mean_abs": 1.557291357738545e-05, + "_provisional": true + }, + "stream.chunk.9.channel_len": { + "max_abs": 0.007, + "mean_abs": 0.0007000000000000001, + "_provisional": true + }, + "stream.chunk.9.enc_out": { + "max_abs": 1.854820944368835e-05, + "mean_abs": 1e-06, + "_provisional": true + }, + "stream.chunk.9.mel_in": { + "max_abs": 0.0016445308208465578, + "mean_abs": 9.642163761926509e-05, + "_provisional": true + } +} \ No newline at end of file From 11ab2bd28652c19f8628ac816d4960671ec2f3e6 Mon Sep 17 00:00:00 2001 From: CJ Pais Date: Wed, 8 Jul 2026 01:28:49 -0700 Subject: [PATCH 03/11] stage 3 --- docs/porting/families/parakeet.md | 18 ++++++ scripts/convert-parakeet.py | 103 +++++++++++++++++++++++++++++- 2 files changed, 120 insertions(+), 1 deletion(-) diff --git a/docs/porting/families/parakeet.md b/docs/porting/families/parakeet.md index 5f09006e..914df1b6 100644 --- a/docs/porting/families/parakeet.md +++ b/docs/porting/families/parakeet.md @@ -209,6 +209,24 @@ be resolved before the corresponding port enters porting-3-convert: diarization mask instead of all-ones, N-instance orchestration, SegLST) is OUT OF SCOPE per Stage 1 — see the multitalker integration brief. + **Stage 3 resolution (DONE).** `convert-parakeet.py` gates emission on a + `spk_kernel_layers` profile key. The 8 source tensors are emitted verbatim + (fp32 passthrough) under the loader's `enc.` prefix, keeping the source + layer index and the parameter-free `.1`/`.2` Sequential slots implicit: + - `spk_kernels..{0,3}.{weight,bias}` → `enc.spk_kernel..{0,3}.{weight,bias}` + - `bg_spk_kernels..{0,3}.{weight,bias}` → `enc.bg_spk_kernel..{0,3}.{weight,bias}` + + Both Linear slots are `[1024,1024]` + `[1024]` bias. The layer-0 injection + is marked by three KVs Stage 4 reads: + `stt.parakeet.encoder.spk_kernel_type` (`"ff"`), + `stt.parakeet.encoder.spk_kernel_layers` (int array, `[0]`), + `stt.parakeet.encoder.add_bg_spk_kernel` (bool, `true`). The current loader + ignores the extra tensors + KVs and runs this GGUF on the existing + `nemotron-speech-streaming` path; the jfk smoke still decodes correctly + because clean single-speaker greedy decode is robust to the (unapplied) + injection drift — Stage 4 applies it for numerical parity against the + oracle. + ## Tooling: NeMo-aware preflight `scripts/preflight.py` knows how to read `.nemo` archives when the diff --git a/scripts/convert-parakeet.py b/scripts/convert-parakeet.py index b2f52cdd..b0562b42 100644 --- a/scripts/convert-parakeet.py +++ b/scripts/convert-parakeet.py @@ -348,6 +348,38 @@ "license_name": "openmdw-1.1", "license_link": "https://openmdw.ai/license/1-1/", }, + # 0.6B English cache-aware streaming MULTI-TALKER RNN-T. Same + # FastConformer streaming encoder as nemotron-speech-streaming-en-0.6b + # (24L / d_model=1024 / 8h / kernel=9 / subsampling=8, + # att_context_style=chunked_limited, layer_norm conv, [70,13] default) + # fine-tuned with a SpeakerKernelMixin (NeMo class + # EncDecMultiTalkerRNNTBPEModel). At each layer in spk_kernel_layers it + # injects a per-speaker FF kernel (`spk_kernels.`) plus a + # background-speaker FF kernel (`bg_spk_kernels.`, add_bg_spk_kernel). + # The injection is a forward_pre_hook that runs UNCONDITIONALLY: even + # single-speaker inference adds spk_kernels.(x) (all-ones speaker + # mask) + bg_spk_kernels.(0) (zeros mask -> constant bias), so these + # tensors are load-bearing for the in-scope single-speaker path, not + # just multi-speaker decoding. v1 port targets single-speaker mode at + # [70,13]; multi-speaker decoding (external Sortformer diarization, + # SegLST speaker-tagged output) is a deferred Stage B workstream. + # spk_kernel_layers present on the profile is the switch that turns on + # kernel-tensor emission + the stt.parakeet.encoder.spk_kernel_* KV. + "multitalker-parakeet-streaming-0.6b-v1": { + "variant": "multitalker-parakeet-streaming-0.6b-v1", + "display_name": "Multitalker Parakeet Streaming EN", + "version": "v1", + "size_label": "0.6B", + "basename": "parakeet-rnnt", + "head_kind": "rnnt", + "expected_vocab_size": 1024, + "languages": ["en"], + "lang_detect": False, + "spk_kernel_layers": [0], + "license": "other", + "license_name": "nvidia-open-model-license", + "license_link": "https://www.nvidia.com/en-us/agreements/enterprise-software/nvidia-open-model-license/", + }, } @@ -1166,6 +1198,35 @@ def _from_module_or_cfg(attr: str, cfg_key: str, default=None): ] +# Speaker-kernel FF tensors (multitalker-parakeet-streaming-0.6b-v1). +# NeMo's SpeakerKernelMixin attaches, at each layer index in +# spk_kernel_layers, a per-speaker FF kernel `spk_kernels.` and (when +# add_bg_spk_kernel) a background-speaker FF kernel `bg_spk_kernels.`. +# Both are top-level state_dict modules (NOT under `encoder.`) shaped +# nn.Sequential(Linear(d,d), ReLU, Dropout, Linear(d,d)) -- only the .0 +# and .3 Linear slots carry tensors; .1 (ReLU) / .2 (Dropout) are +# parameter-free, mirroring PRE_ENCODE_TABLE's index convention. The gguf +# names take the loader's `enc.` prefix and keep the source layer index so +# the loader can bind per-layer and apply the always-on injection at the +# right block. See src/arch/parakeet family doc open-decision #5. +def spk_kernel_table(layer: int) -> list[tuple[str, str]]: + return [ + (f"spk_kernels.{layer}.0.weight", f"enc.spk_kernel.{layer}.0.weight"), + (f"spk_kernels.{layer}.0.bias", f"enc.spk_kernel.{layer}.0.bias"), + (f"spk_kernels.{layer}.3.weight", f"enc.spk_kernel.{layer}.3.weight"), + (f"spk_kernels.{layer}.3.bias", f"enc.spk_kernel.{layer}.3.bias"), + ] + + +def bg_spk_kernel_table(layer: int) -> list[tuple[str, str]]: + return [ + (f"bg_spk_kernels.{layer}.0.weight", f"enc.bg_spk_kernel.{layer}.0.weight"), + (f"bg_spk_kernels.{layer}.0.bias", f"enc.bg_spk_kernel.{layer}.0.bias"), + (f"bg_spk_kernels.{layer}.3.weight", f"enc.bg_spk_kernel.{layer}.3.weight"), + (f"bg_spk_kernels.{layer}.3.bias", f"enc.bg_spk_kernel.{layer}.3.bias"), + ] + + # NeMo state_dict contains buffers and preprocessor tables that the # loader computes itself at runtime. Skipping them silently keeps the # unused-key check meaningful for genuine misses. @@ -1246,8 +1307,14 @@ def convert(model_spec: str, out_path: Path, repo_id: str | None = None) -> None prefer_direct = bool(profile.get("prefer_direct_load", False)) drop_aux_ctc = "tdt_ctc" in slug # hybrid checkpoints carry an aux CTC head we drop has_prompt = bool(profile.get("has_prompt", False)) + # Speaker-kernel FF injection (multitalker variants). spk_kernel_layers + # on the profile is the switch; add_bg_spk_kernel is read from the live + # cfg so it can never disagree with the checkpoint's kernel set. + spk_kernel_layers = [int(l) for l in (profile.get("spk_kernel_layers") or [])] + has_spk_kernels = bool(spk_kernel_layers) print(f"Variant: {profile['variant']} (head_kind={head_kind}" - f"{', prompt=on' if has_prompt else ''})") + f"{', prompt=on' if has_prompt else ''}" + f"{', spk_kernels=on' if has_spk_kernels else ''})") model = load_nemo_model(model_spec, prefer_direct=prefer_direct) @@ -1515,6 +1582,21 @@ def convert(model_spec: str, out_path: Path, repo_id: str | None = None) -> None int(hp["enc_stream_sampling_frames_first"]), ) + # Speaker-kernel FF injection (multitalker variants). These KVs tell + # the loader which encoder layers carry an always-on speaker-kernel + # injection and whether the background kernel is present, so Stage 4 + # can bind enc.spk_kernel.* / enc.bg_spk_kernel.* and replay the + # forward_pre_hook. spk_kernel_type / add_bg_spk_kernel are read from + # the live cfg; spk_kernel_layers comes from the profile switch. + add_bg_spk_kernel = bool(config.get("add_bg_spk_kernel", False)) + if has_spk_kernels: + writer.add_string("stt.parakeet.encoder.spk_kernel_type", + str(config.get("spk_kernel_type", "ff"))) + writer.add_array("stt.parakeet.encoder.spk_kernel_layers", + [int(l) for l in spk_kernel_layers]) + writer.add_bool("stt.parakeet.encoder.add_bg_spk_kernel", + add_bg_spk_kernel) + if head_kind != "ctc": writer.add_uint32("stt.parakeet.predictor.hidden", hp["pred_hidden"]) writer.add_uint32("stt.parakeet.predictor.n_layers", hp["pred_n_layers"]) @@ -1698,6 +1780,17 @@ def add_combined(nemo_a: str, nemo_b: str, gguf_name: str) -> None: for nemo_name, gguf_name in PROMPT_MLP_TABLE: add(nemo_name, gguf_name) + # Speaker-kernel FF tensors (multitalker variants). One (Linear.0, + # Linear.3) pair per layer, plus the background kernel when the + # checkpoint carries it. Emitted verbatim (fp32 passthrough). + if has_spk_kernels: + for layer in spk_kernel_layers: + for nemo_name, gguf_name in spk_kernel_table(layer): + add(nemo_name, gguf_name) + if add_bg_spk_kernel: + for nemo_name, gguf_name in bg_spk_kernel_table(layer): + add(nemo_name, gguf_name) + per_layer_tensors = len(ENCODER_BLOCK_TABLE) + ( len(ENCODER_BLOCK_BIAS_TABLE) if hp["enc_use_bias"] else 0 ) @@ -1714,11 +1807,19 @@ def add_combined(nemo_a: str, nemo_b: str, gguf_name: str) -> None: + len(JOINT_TABLE) ) prompt_tensors = len(PROMPT_MLP_TABLE) if has_prompt else 0 + if has_spk_kernels: + per_spk_layer = len(spk_kernel_table(0)) + ( + len(bg_spk_kernel_table(0)) if add_bg_spk_kernel else 0 + ) + spk_tensors = len(spk_kernel_layers) * per_spk_layer + else: + spk_tensors = 0 expected = ( len(PRE_ENCODE_TABLE) + hp["enc_n_layers"] * per_layer_tensors + head_tensors + prompt_tensors + + spk_tensors ) if n_added != expected: raise RuntimeError( From c55a09d5ba9b368ea85595df33fba36d88813aca Mon Sep 17 00:00:00 2001 From: CJ Pais Date: Thu, 9 Jul 2026 00:29:32 -0700 Subject: [PATCH 04/11] stage 4 --- docs/porting/families/parakeet.md | 16 +- reports/porting/parakeet/forward-map.md | 8 + src/arch/parakeet/encoder.cpp | 95 ++- src/arch/parakeet/model.cpp | 28 +- src/arch/parakeet/weights.cpp | 76 ++ src/arch/parakeet/weights.h | 56 +- src/transcribe-batch-util.cpp | 17 +- ...talker-parakeet-streaming-0.6b-v1.cpu.json | 15 + ...ultitalker-parakeet-streaming-0.6b-v1.json | 103 +-- ...-parakeet-streaming-0.6b-v1.streaming.json | 807 ++---------------- 10 files changed, 394 insertions(+), 827 deletions(-) create mode 100644 tests/golden/batch/multitalker-parakeet-streaming-0.6b-v1.cpu.json diff --git a/docs/porting/families/parakeet.md b/docs/porting/families/parakeet.md index 914df1b6..6cd42b51 100644 --- a/docs/porting/families/parakeet.md +++ b/docs/porting/families/parakeet.md @@ -146,14 +146,14 @@ Allowed statuses: `PASS` | `SKIP — not exposed by runtime` | | nemotron-3.5-asr-streaming-0.6b | Batch (offline) | run_batch fast path | `uv run scripts/batch_parity.py --model --list --batch-sizes 2,4,8 --backend cpu --language en-US` + `uv run scripts/batch_tensor_parity.py --model --wav samples/jfk.wav --batch 4 --backend cpu` | text byte-equal vs serial at sizes 2/4/8 (golden frozen at `tests/golden/batch/nemotron-3.5-asr-streaming-0.6b.cpu.json`); CPU tensor parity bit-exact (max_abs=0.0) at batch=4 on jfk.wav | PASS | | ctc-1.1b | Transcribe (CTC head) | explicit en | `build/bin/transcribe-cli -m models/parakeet-ctc-1.1b/parakeet-ctc-1.1b-F32.gguf --language en samples/jfk.wav` | English transcript | PASS | | ctc-0.6b | Transcribe (CTC head) | explicit en | `build/bin/transcribe-cli -m models/parakeet-ctc-0.6b/parakeet-ctc-0.6b-F32.gguf --language en samples/jfk.wav` | English transcript | PASS | -| multitalker-parakeet-streaming-0.6b-v1 | Transcribe (single_speaker_mode, offline / cache-aware att_context_size=[70,13], 1.12s chunk) | explicit en | `build/bin/transcribe-cli -m models/multitalker-parakeet-streaming-0.6b-v1/multitalker-parakeet-streaming-0.6b-v1-F32.gguf --language en samples/jfk.wav` | English transcript with PnC | TODO | -| multitalker-parakeet-streaming-0.6b-v1 | Transcribe (single_speaker_mode) — no-hint | auto/default | `build/bin/transcribe-cli -m samples/jfk.wav` (no `--language`; English-only checkpoint) | English transcript with PnC | TODO | -| multitalker-parakeet-streaming-0.6b-v1 | Punctuation/casing | output | same as the explicit-en row | output contains capital letters and `,.?!` | TODO | -| multitalker-parakeet-streaming-0.6b-v1 | Streaming (single_speaker_mode, cache reuse across chunks) | streaming | `build/bin/transcribe-cli -m --language en --backend cpu --threads 1 --stream-chunk-ms 1120 --stream-att-right 13 samples/jfk.wav` | byte-equal transcript vs single-speaker one-shot at the default `att_context_size=[70,13]` (1.12s chunk) | TODO | -| multitalker-parakeet-streaming-0.6b-v1 | Other latency settings ([70,0]/[70,1]/[70,6]/[70,13]) | runtime-selectable att_context_size | `… --stream-chunk-ms 1120 --stream-att-right {0,1,6,13} …` | all four R settings stream a valid single-speaker transcript; R=6/13 byte-equal to one-shot, R=0/1 differ only in trailing punctuation (lower lookahead) | TODO | -| multitalker-parakeet-streaming-0.6b-v1 | Batch (offline, single_speaker_mode) | run_batch fast path | `uv run scripts/batch_parity.py --model --list --batch-sizes 2,4,8 --backend cpu --language en` + `uv run scripts/batch_tensor_parity.py --model --wav samples/jfk.wav --batch 4 --backend cpu` | text byte-equal vs serial at sizes 2/4/8; CPU tensor parity bit-exact (max_abs=0.0) at batch=4 on jfk.wav | TODO | -| multitalker-parakeet-streaming-0.6b-v1 | Multitalker / speaker-attributed ASR (speaker-kernel injection + external Sortformer diarization + multi-instance + SegLST) | multitalker | (no runtime surface today) | per-speaker cpWER vs NeMo `speech_to_text_multitalker_streaming_infer.py` on AMI/CH109/Mixer6 | OUT OF SCOPE — requires (1) a ported streaming Sortformer diarizer, (2) speaker-kernel injection tensors + layer-0 forward, (3) N-instance-per-speaker orchestration, (4) a SegLST speaker-tagged output contract; none exist in transcribe.cpp today. Brought back in scope by a dedicated multitalker workstream (see the accompanying brief). | -| multitalker-parakeet-streaming-0.6b-v1 | Speaker diarization (produce speaker turns) | diarization | (not a capability of this checkpoint) | n/a | OUT OF SCOPE — this checkpoint CONSUMES external diarization (Sortformer); it does not produce speaker turns. Brought back in scope only if a Sortformer diarizer is ported as a separate family. | +| multitalker-parakeet-streaming-0.6b-v1 | Transcribe (single_speaker_mode, offline / cache-aware att_context_size=[70,13], 1.12s chunk) | explicit en | `build/bin/transcribe-cli -m models/multitalker-parakeet-streaming-0.6b-v1/multitalker-parakeet-streaming-0.6b-v1-F32.gguf --language en samples/jfk.wav` | English transcript with PnC | PASS | +| multitalker-parakeet-streaming-0.6b-v1 | Transcribe (single_speaker_mode) — no-hint | auto/default | `build/bin/transcribe-cli -m samples/jfk.wav` (no `--language`; English-only checkpoint) | English transcript with PnC | PASS | +| multitalker-parakeet-streaming-0.6b-v1 | Punctuation/casing | output | same as the explicit-en row | output contains capital letters and `,.?!` | PASS | +| multitalker-parakeet-streaming-0.6b-v1 | Streaming (single_speaker_mode, cache reuse across chunks) | streaming | `build/bin/transcribe-cli -m --language en --backend cpu --threads 1 --stream-chunk-ms 1120 --stream-att-right 13 samples/jfk.wav` | byte-equal transcript vs single-speaker one-shot at the default `att_context_size=[70,13]` (1.12s chunk) | PASS | +| multitalker-parakeet-streaming-0.6b-v1 | Other latency settings ([70,0]/[70,1]/[70,6]/[70,13]) | runtime-selectable att_context_size | `… --stream-chunk-ms 1120 --stream-att-right {0,1,6,13} …` | all four R settings stream a valid single-speaker transcript; R=6/13 byte-equal to one-shot, R=0/1 differ only in trailing punctuation (lower lookahead) | PASS | +| multitalker-parakeet-streaming-0.6b-v1 | Batch (offline, single_speaker_mode) | run_batch fast path | `uv run scripts/batch_parity.py --model --list --batch-sizes 2,4,8 --backend cpu --language en` + `uv run scripts/batch_tensor_parity.py --model --wav samples/jfk.wav --batch 4 --backend cpu` | text byte-equal vs serial at sizes 2/4/8 (golden frozen at `tests/golden/batch/multitalker-parakeet-streaming-0.6b-v1.cpu.json`); same-length CPU tensor parity bit-exact (max_abs=0.0) at batch=4 on jfk.wav; **diverse-length** flash tensor parity bit-exact (max_abs=0.0) across arbitrary length mixes; full test-clean batch-8 WER == batch-1 (2.19%) after the causal var-len pre-encode masking fix (see forward-map Deviations) | PASS | +| multitalker-parakeet-streaming-0.6b-v1 | Multitalker / speaker-attributed ASR (speaker-kernel injection + external Sortformer diarization + multi-instance + SegLST) | multitalker | (no runtime surface today) | per-speaker cpWER vs NeMo `speech_to_text_multitalker_streaming_infer.py` on AMI/CH109/Mixer6 | SKIP — not exposed by runtime (OUT OF SCOPE) — requires (1) a ported streaming Sortformer diarizer, (2) speaker-kernel injection tensors + layer-0 forward, (3) N-instance-per-speaker orchestration, (4) a SegLST speaker-tagged output contract; none exist in transcribe.cpp today. Brought back in scope by a dedicated multitalker workstream (see the accompanying brief). | +| multitalker-parakeet-streaming-0.6b-v1 | Speaker diarization (produce speaker turns) | diarization | (not a capability of this checkpoint) | n/a | SKIP — not exposed by runtime (OUT OF SCOPE) — this checkpoint CONSUMES external diarization (Sortformer); it does not produce speaker turns. Brought back in scope only if a Sortformer diarizer is ported as a separate family. | | all variants | Word timestamps | only if exposed | `transcribe-cli --timestamps word -m ` (any variant) | per-word `t0_ms`/`t1_ms` in JSON output | PASS — derived host-side from emit-frame indices (TDT/RNNT) or per-frame argmax (CTC); same code path as the existing v2/v3 word-timestamp gate, no per-variant differences | ## Open decisions before Stage 3 (convert) diff --git a/reports/porting/parakeet/forward-map.md b/reports/porting/parakeet/forward-map.md index 0d8907a3..d58db4e6 100644 --- a/reports/porting/parakeet/forward-map.md +++ b/reports/porting/parakeet/forward-map.md @@ -32,6 +32,7 @@ introduced by Stage 4 are `n_layers` (17/24/42), `d_model` (512/1024), |-------|--------------------|--------------|-------------|--------------------|----------------| | Pre-encode subsampling (factor 8 = 3x stride-2 conv) | `model.encoder.pre_encode` (dw_striding) | `[T_enc, d_model]` | `enc.pre_encode.out` | `conf::build_pre_encode` over `PreEncodeView`. `pre_encode_freq = subsampling_channels * (n_mels / 8)` — 256·10=2560 (n_mels=80) or 256·16=4096 (n_mels=128). Final linear projects to d_model. | existing parakeet | | Relative positional embedding | `RelPositionalEncoding` | `[2*T_enc-1, d_model]` | `enc.pos_emb` | host-built sin/cos table, uploaded as graph input | unchanged | +| Speaker-kernel injection (multitalker variants) | `SpeakerKernelMixin._get_spk_kernel_hook_pre_layer` (forward_pre_hook on `encoder.layers[0]`) | `[T_enc, d_model]` | (folded into `enc.block.0.out`) | `apply_spk_kernel_injection` in `encoder.cpp`, on the block-0 input (post pre-encode + xscale): `x += W3·relu(W0·x+b0)+b3` then `x += W3_bg·relu(b0_bg)+b3_bg` (bg input is all-zeros → constant broadcast). Gated on `hp.has_spk_kernel` (KV `stt.parakeet.encoder.spk_kernel_layers`); no-op on every other variant. | new — `multitalker-parakeet-streaming-0.6b-v1` only | | Conformer block (FF1 → rel-pos MHSA → ConvModule(BN) → FF2 → LN) | `model.encoder.layers[i]` | `[T_enc, d_model]` | `enc.block.{0, n/2, n-1}.out` | `conf::build_conformer_block` over `BlockView`. Bias slots populated only when `enc_use_bias=true` (11 biases per block: ff1.linear1/linear2, attn.q/k/v/out, conv.pw1/dw/pw2, ff2.linear1/linear2). | existing parakeet — extended for biases | | Final encoder LN | implicit in `layers[-1]` (norm_out) | `[T_enc, d_model]` | `enc.final` | `conf::named` on the final tensor | unchanged | | Prompt MLP (multilingual variants only) | `model.prompt_kernel` (NeMo `EncDecRNNTBPEModelWithPrompt`): `concat(enc[d_model], one_hot(prompt_id)[num_prompts])` → `Linear(d_model+P → prompt_hidden)` → ReLU → `Linear(prompt_hidden → d_model)` | `[T_enc, d_model]` | `enc.prompted` (encoder-side) + `dec.enc_out_prompted` (decoder-side; same tensor, separate dump for drift attribution) | host-built one-hot broadcast over T frames, then two-layer MLP via ggml_mul_mat with bias adds and `ggml_relu` between. Loader reads `stt.parakeet.prompt.*` KVs and the four `prompt.mlp.{0,2}.{weight,bias}` tensors. When the KVs are absent the path is a no-op (English-only variants). | new — `nemotron-3.5-asr-streaming-0.6b` is the first variant exercising the prompt-conditioned RNN-T joint feed | @@ -104,6 +105,12 @@ cache to mis-index. - **TDT-CTC hybrids (`tdt_ctc-110m`, `tdt_ctc-1.1b`)**: the upstream checkpoints carry both heads, but the converter emits TDT-only (the auxiliary CTC head is silently dropped — pure CTC variants cover that path). At Stage 4 these load through the standard TDT path with `head_kind="tdt"`. - **Streaming attention (`att_context_style` = "chunked_limited")**: introduced by `nemotron-speech-streaming-en-0.6b`. NeMo's cache-aware streaming model trains with chunked attention but keeps the full `RelPositionalEncoding` (pos_emb buffer at 2T-1, not the shortened `LocalAttRelPositionalEncoding` size). The "regular" style is the existing path used by every other variant. Loader reads `stt.parakeet.encoder.att_context_style` as optional with default `"regular"` so legacy GGUFs are unaffected; only `att_context_style == "chunked_limited"` selects the new mask, computed as `chunk_size = att_context_right + 1`, `left_chunks = att_context_left / chunk_size` (every k-frame whose chunk index is in `[q_chunk - left_chunks, q_chunk]` is allowed). The mask is built host-side and threaded as a graph input that broadcasts across heads; added to `matrix_bd` before `flash_attn_ext`. - **Causal depthwise conv (`conv_context_left`, `conv_context_right`)**: streaming-friendly Conformer convolution. NeMo's offline default is centered `(k-1)/2` on both sides; nemotron-streaming uses `[k-1, 0]` (left=8, right=0 for k=9) so the depthwise convolution does not look ahead. Loader reads both as optional with default `-1`; when `-1`, the conformer uses centered `(k-1)/2`. Otherwise depthwise conv padding becomes `(left, right)`. +- **Speaker-kernel FF injection (`spk_kernel_layers`, multitalker)**: introduced by `multitalker-parakeet-streaming-0.6b-v1` (NeMo `EncDecMultiTalkerRNNTBPEModel` / `SpeakerKernelMixin`). An always-on `forward_pre_hook` on `encoder.layers[0]` injects a per-speaker FF kernel and a background FF kernel into the block-0 input; it fires **unconditionally**, so it is load-bearing for the in-scope single-speaker path, not just multi-speaker decoding. In single-speaker mode it reduces to `x += W3·relu(W0·x+b0)+b3` (spk, per frame) then `x += W3_bg·relu(b0_bg)+b3_bg` (bg input masked to zeros → a per-channel constant broadcast over T). Loader reads `stt.parakeet.encoder.{spk_kernel_type,spk_kernel_layers,add_bg_spk_kernel}` (optional; absent → no-op) and binds `enc.spk_kernel.0.{0,3}.{weight,bias}` + `enc.bg_spk_kernel.0.{0,3}.{weight,bias}`. Only layer-0 injection is implemented (the checkpoint ships `spk_kernel_layers=[0]`); the loader rejects any other layer index. Applied in both the offline and the cache-aware streaming encoder graphs. +- **Variable-length batched pre-encode masking (causal variants)** — found + fixed during the multitalker Stage 4 via tensor bisection of batched-vs-single encoder output. Two shared-code bugs in the offline var-len batched path, both specific to the cache-aware streaming (causal `chunked_limited`) variants and invisible to same-length / text-only parity: + 1. **Off-by-one frame count** — `model.cpp::pre_encode_t_out` used the non-causal subsample formula `(in-1)/2+1`; causal pre-encode needs `in/2+1`. Undercounted `real_tenc` (dropped the last encoder frame → truncated hyps) and mis-sized the pad masks. Fixed by making `pre_encode_t_out` causal-aware (mirrors `weights.cpp::pre_encode_F_prime`). + 2. **Unmasked causal subsampling** — masked subsampling was gated `var_len_masks && !causal_pre_encode`, so for causal variants the zero-padded mel tail passed through `conv0`, whose bias made it relu to a NON-zero constant; the causal convs' right-context (`right = stride-1`) then pulled that constant into the last valid encoder frame (a large boundary-frame drift vs single-shot). Fixed by enabling masked subsampling for the causal path too (the per-stage valid lengths are now causal-aware, so the same masks are correct). + Result: batched var-len encoder output is **bit-exact** (0.0) to single-shot on the production flash path across arbitrary length mixes; LibriSpeech test-clean batch-8 WER matches batch-1 exactly (2.19%). Also fixes the sibling `nemotron-3.5` / `nemotron-speech-streaming` (0.098 → 0.0). + 3. **Manual-path all-masked-query NaN** — in a var-len batch a padded query row (past the real length) can have its entire chunked-attention window fall on padded keys; with a `-inf` key-pad sentinel the manual (`TRANSCRIBE_NO_FLASH`) softmax computes `0/0 = NaN`, which then poisons real frames at the next layer via `0*NaN`. Fixed in `transcribe-batch-util.cpp::fill_keypad_mask` by using a large FINITE negative sentinel (`-1e30`) instead of `-inf`: bit-exact with `-inf` for any row that has a non-masked key (`expf` underflows to exactly 0), casts to `-inf` in F16 so flash is unchanged, but makes a fully-masked row softmax to a finite value instead of NaN. Manual diverse-length batched output now matches single-shot to f32 reduction-order noise (~1e-7); flash is exactly 0.0. - **LayerNorm in conv module (`conv_norm_type` = "layer_norm")**: streaming-stable replacement for BatchNorm. Same tensor names (`conv.bn.weight` / `conv.bn.bias`) but the `running_mean` / `running_var` tensors are absent and the per-batch statistics are computed online (it really is a LayerNorm over `d_model`, not a fused affine). Loader reads `stt.parakeet.encoder.conv_norm_type` as optional with default `"batch_norm"`; when `"layer_norm"`, the BN-fusion step is skipped and `conv_module` applies an unfused LayerNorm using the same scale/bias tensors. ## Variant Notes @@ -118,4 +125,5 @@ cache to mis-index. - **`ctc-0.6b`**: 24 layers, d_model=1024, n_mels=80, `use_bias=true`, **CTC head**. **First exercise of CTC head loading and CTC greedy decode.** - **`ctc-1.1b`**: 42 layers, d_model=1024, n_mels=80, `use_bias=true`, CTC head. Sibling of ctc-0.6b on the 1.1B encoder. - **`nemotron-speech-streaming-en-0.6b`**: 24 layers, d_model=1024, **n_mels=128**, `use_bias=false`, RNNT head, pred_n_layers=2. **First variant exercising `att_context_style="chunked_limited"`, causal depthwise conv (`conv_context_left=8, conv_context_right=0` for kernel=9), `conv_norm_type="layer_norm"`, and `fe_normalize="none"` (NeMo's `normalize="NA"` no-op, canonicalised at convert time).** Frontend feature stats (per-feature mean/std) are baked into training rather than computed at inference, so the C++ mel frontend emits unnormalised log-mel and the loader accepts `normalize="none"` alongside `"per_feature"`. Streaming runtime knobs (lookahead, encoder chunk size, decoder context) are deferred to the streaming bring-up — offline-only in v1, same as `unified-en-0.6b`. +- **`multitalker-parakeet-streaming-0.6b-v1`**: 24 layers, d_model=1024, **n_mels=128**, `use_bias=false`, RNNT head, pred_n_layers=2. Same cache-aware streaming FastConformer as `nemotron-speech-streaming-en-0.6b` (chunked_limited `[70,13]`, causal depthwise conv, layer_norm conv, `fe_normalize="none"`); **first and only variant exercising the always-on layer-0 speaker-kernel FF injection** (see the Deviations bullet). Ported in SINGLE-SPEAKER mode; the full multitalker path (external Sortformer diarization, N-instance orchestration, SegLST) is OUT OF SCOPE (see the multitalker integration brief). The +8 speaker-kernel tensors are the only weight-set delta from the English predecessor. - **`nemotron-3.5-asr-streaming-0.6b`**: 24 layers, d_model=1024, **n_mels=128**, `use_bias=false`, RNNT head, pred_n_layers=2. Same FastConformer geometry as the English predecessor; **first variant exercising the multilingual prompt-conditioning MLP** (`EncDecRNNTBPEModelWithPrompt`, num_prompts=128, prompt_hidden read from the GGUF; encoder output passes through `prompt_kernel` before the RNN-T joint sees it). **Left context shrinks 70 → 56 frames** in the chunked_limited mask (training-time `att_context_size=[56,13]`); the runtime mask code reads the GGUF KV, so this is a data-only delta. Vocab grows 1024 → 13087 with 39 explicit `` SPM tag tokens used by `target_lang=auto`. Auxiliary CTC head present in the upstream checkpoint but dropped at convert time per family Open-Decision #1. diff --git a/src/arch/parakeet/encoder.cpp b/src/arch/parakeet/encoder.cpp index 0bea80ee..6f69dacd 100644 --- a/src/arch/parakeet/encoder.cpp +++ b/src/arch/parakeet/encoder.cpp @@ -247,6 +247,54 @@ std::vector parse_sub_block_env(int n_layers) { return out; } +// Speaker-kernel FF forward (multitalker variants): returns +// W3·relu(W0·x + b0) + b3. x ne = [d_model, T]; the two Linears keep +// d_model → d_model. Matches NeMo get_spk_kernel_class('ff') at eval +// (Dropout is identity). +ggml_tensor * build_spk_kernel_ff(ggml_context * ctx, ggml_tensor * x, const ParakeetSpkKernelFF & ff) { + ggml_tensor * h = ggml_mul_mat(ctx, ff.lin0_w, x); + h = ggml_add(ctx, h, ff.lin0_b); + h = ggml_relu(ctx, h); + ggml_tensor * y = ggml_mul_mat(ctx, ff.lin3_w, h); + y = ggml_add(ctx, y, ff.lin3_b); + return y; +} + +// Apply the always-on layer-0 speaker-kernel injection to the block-0 +// input `x` (post pre-encode + xscale). Mirrors +// multitalker_asr_mixins.py::_get_spk_kernel_hook_pre_layer in +// single-speaker mode (spk_targets / bg_spk_targets both None): +// x += spk_kernel(x) (all-ones speaker mask → x) +// x += bg_spk_kernel(0) (all-zeros bg mask → constant) +// The bg input is exactly zeros, so bg_spk_kernel(0) = W3_bg·relu(b0_bg) + +// b3_bg is a per-channel constant [d_model] broadcast over the time axis. +// No-op when hp.has_spk_kernel is false. See family doc open-decision #5. +ggml_tensor * apply_spk_kernel_injection(ggml_context * ctx, + ggml_tensor * x, + const ParakeetWeights & w, + const ParakeetHParams & hp) { + if (!hp.has_spk_kernel) { + return x; + } + for (const auto & kernel : w.spk_kernels) { + if (kernel.layer != 0) { + continue; // loader rejects non-zero layers; defensive + } + ggml_tensor * x_spk = build_spk_kernel_ff(ctx, x, kernel.spk); + x = ggml_add(ctx, x, x_spk); + if (kernel.has_bg) { + // bg input is all-zeros → relu(W0_bg·0 + b0_bg) = relu(b0_bg); + // W3_bg·relu(b0_bg) + b3_bg is a [d_model] constant broadcast + // over the T axis by ggml_add. + ggml_tensor * hb = ggml_relu(ctx, kernel.bg.lin0_b); + ggml_tensor * c_bg = ggml_mul_mat(ctx, kernel.bg.lin3_w, hb); + c_bg = ggml_add(ctx, c_bg, kernel.bg.lin3_b); + x = ggml_add(ctx, x, c_bg); + } + } + return x; +} + } // namespace EncoderBuild build_encoder_graph(ggml_context * ctx, @@ -314,10 +362,18 @@ EncoderBuild build_encoder_graph(ggml_context * ctx, transcribe::debug::mark_tensor_for_dump(eb.mel_in); // Build the pre_encode subgraph. Variable-length offline batching - // enables masked subsampling (zero each conv intermediate's padded - // time region). Only the non-causal pre-encode is masked. + // enables masked subsampling (zero each conv intermediate's padded time + // region after every ReLU). Required for BOTH the non-causal and the + // causal (cache-aware streaming) pre-encode: conv0 adds a bias, so the + // zero-padded mel tail relu's to a NON-zero constant, and the causal + // convs' right-context (right = stride-1) then pulls that constant into + // the last valid encoder frame — a large boundary-frame drift vs + // single-shot (which has no padding tail). Masking after each ReLU + // re-zeroes the padded region so the boundary matches single-shot. The + // per-stage valid lengths are computed causal-aware in model.cpp + // (pre_encode_t_out), so the same masks are correct for either padding. conf::PreEncodeValidMasks pe_masks; - const bool mask_pre_encode = var_len_masks && !policy.causal_pre_encode; + const bool mask_pre_encode = var_len_masks; ggml_tensor * x = conf::build_pre_encode(ctx, to_view(w.pre_encode), eb.mel_in, policy, /*name_prefix=*/"enc.pre_encode", /*error_tag=*/"parakeet", mask_pre_encode ? &pe_masks : nullptr); @@ -348,6 +404,15 @@ EncoderBuild build_encoder_graph(ggml_context * ctx, x = conf::named(x, "enc.pre_encode.xscaled"); } + // ----- speaker-kernel injection (multitalker variants) ----- + // Always-on layer-0 pre-hook: injects the speaker + background FF kernels + // into the block-0 input. NeMo applies it to the post-pos-enc (post + // xscale) audio_signal, exactly here. No-op unless hp.has_spk_kernel. + if (hp.has_spk_kernel) { + x = apply_spk_kernel_injection(ctx, x, w, hp); + x = conf::named(x, "enc.pre_encode.spk_injected"); + } + // ----- Conformer blocks ----- // // Every block goes through conf::build_conformer_block. Block 0 passes @@ -421,13 +486,15 @@ EncoderBuild build_encoder_graph(ggml_context * ctx, bparams.n_head = hp.enc_n_heads; bparams.conv_kernel = hp.enc_conv_kernel; bparams.kv_type = kv_type; - // Flash attention by default. Flash batches correctly (the batched - // encoder output is bit-identical to single-shot flash), so the batch - // axis does not change the path. TRANSCRIBE_NO_FLASH forces the manual - // F32 mul_mat + soft_max path — used by the bit-exact CPU tensor gate - // (flash casts the rel-pos mask to F16, manual stays F32); - // TRANSCRIBE_FORCE_FLASH forces it back on. Parakeet has no attention - // decoder, so the decoder flag is a throwaway. + // Flash attention by default. Same-length batches are bit-identical to + // single-shot; variable-length batches rely on the attn_pad_mask + + // conv_pad_mask to isolate utterances (a small boundary-frame drift + // remains vs single-shot on the cache-aware streaming chunked path — + // see the parakeet family doc). TRANSCRIBE_NO_FLASH forces the manual + // F32 mul_mat + soft_max path — + // used by the bit-exact CPU tensor gate (flash casts the rel-pos mask + // to F16, manual stays F32); TRANSCRIBE_FORCE_FLASH forces it back on. + // Parakeet has no attention decoder, so the decoder flag is a throwaway. bool enc_use_flash = true, dec_use_flash = false; transcribe::flash::apply_env_overrides(enc_use_flash, dec_use_flash); bparams.use_flash = enc_use_flash; @@ -681,6 +748,14 @@ EncoderBuild build_encoder_graph_streaming(ggml_context * ctx, x = ggml_cont(ctx, x); } + // ----- speaker-kernel injection (multitalker variants) ----- + // Same always-on layer-0 pre-hook as the offline path, applied to the + // per-chunk block-0 input (post pre-encode + xscale + drop-extra slice). + // No-op unless hp.has_spk_kernel. + if (hp.has_spk_kernel) { + x = apply_spk_kernel_injection(ctx, x, w, hp); + } + const int64_t T_q_new = x->ne[1]; const int64_t T_cache = cache_io.channel_in[0]->ne[1]; const int64_t T_virt = T_cache + T_q_new; diff --git a/src/arch/parakeet/model.cpp b/src/arch/parakeet/model.cpp index 3b49742b..bad08ccd 100644 --- a/src/arch/parakeet/model.cpp +++ b/src/arch/parakeet/model.cpp @@ -1210,9 +1210,17 @@ transcribe_status run(transcribe_session * session, // batches build attn key-padding + conv valid-frame masks so a padded // tail can't corrupt real frames and decode each at its own T_enc; // same-length batches skip the masks and are bit-identical to single-shot. -static int pre_encode_t_out(int in) { - // One stride-2, kernel-3, pad-1 conv: floor((in + 2 - 3)/2) + 1. - return (in - 1) / 2 + 1; +static int pre_encode_t_out(int in, bool causal) { + // One stride-2, kernel-3 conv on the time axis. The padding — and hence + // the output length — depends on whether the pre-encode is causal: + // non-causal (offline): symmetric pad-1 -> floor((in + 2 - 3)/2) + 1 = floor((in-1)/2)+1 + // causal (cache-aware streaming): pad (k-1, s-1) = (2, 1) -> floor((in + 3 - 3)/2) + 1 = floor(in/2)+1 + // The single-shot graph applies the correct causal conv, so the batched + // valid-length bookkeeping MUST use the matching formula or it undercounts + // causal variants by one frame per stage (dropping the last encoder frame + // and mis-sizing the pad masks). Mirrors ConvPolicy::causal_pre_encode / + // pre_encode_F_prime in weights.cpp. + return causal ? (in / 2 + 1) : ((in - 1) / 2 + 1); } static transcribe_status run_batch_encode(ParakeetSession * pc, @@ -1289,14 +1297,18 @@ static transcribe_status run_batch_encode(ParakeetSession * } // Per-utterance valid encoder-frame count (the same subsample the conv - // stack applies: 3 stride-2 convs on the time axis). + // stack applies: 3 stride-2 convs on the time axis). Causal pre-encode + // (cache-aware streaming variants) uses a different output-length formula + // than the offline non-causal path; see pre_encode_t_out. + const bool causal_pre_encode = + (pm->hparams.enc_att_context_style == ParakeetHParams::AttContextStyle::ChunkedLimited); std::vector real_tenc(static_cast(n), T_enc); if (var_len) { for (int b = 0; b < n; ++b) { int t = nf[b]; - t = pre_encode_t_out(t); - t = pre_encode_t_out(t); - t = pre_encode_t_out(t); + t = pre_encode_t_out(t, causal_pre_encode); + t = pre_encode_t_out(t, causal_pre_encode); + t = pre_encode_t_out(t, causal_pre_encode); real_tenc[b] = std::min(t, T_enc); } // Attention key-padding mask [T_enc, 1, 1, n] (0 real / -INF padded) @@ -1317,7 +1329,7 @@ static transcribe_status run_batch_encode(ParakeetSession * for (int b = 0; b < n; ++b) { int v = nf[b]; for (int d = 0; d < n_down; ++d) { - v = pre_encode_t_out(v); + v = pre_encode_t_out(v, causal_pre_encode); } if (v > H) { v = H; diff --git a/src/arch/parakeet/weights.cpp b/src/arch/parakeet/weights.cpp index 1126fedf..4d3e6aa3 100644 --- a/src/arch/parakeet/weights.cpp +++ b/src/arch/parakeet/weights.cpp @@ -501,6 +501,58 @@ transcribe_status read_parakeet_hparams(const gguf_context * gguf, ParakeetHPara } } + // Speaker-kernel FF injection. Optional (multitalker variants only). + // Presence of stt.parakeet.encoder.spk_kernel_layers is the gate; absent + // leaves has_spk_kernel false and the encoder skips the injection. + if (gguf_find_key(gguf, "stt.parakeet.encoder.spk_kernel_layers") >= 0) { + switch (read_int32_array_kv(gguf, "stt.parakeet.encoder.spk_kernel_layers", hp.spk_kernel_layers)) { + case KvResult::Ok: + break; + case KvResult::Absent: + break; // find_key said present; unreachable + case KvResult::BadType: + log_msg(TRANSCRIBE_LOG_LEVEL_ERROR, + "parakeet: stt.parakeet.encoder.spk_kernel_layers " + "has wrong type (expected int32 array)"); + return TRANSCRIBE_ERR_GGUF; + } + if (!hp.spk_kernel_layers.empty()) { + hp.has_spk_kernel = true; + if (auto st = read_required_string_kv(gguf, "stt.parakeet.encoder.spk_kernel_type", kFamilyTag, + hp.spk_kernel_type); + st != TRANSCRIBE_OK) { + return st; + } + // Only the FF kernel (nn.Sequential Linear-ReLU-Dropout-Linear) is + // implemented; NeMo leaves conv2d/mha kernels as TODO. + if (hp.spk_kernel_type != "ff") { + log_msg(TRANSCRIBE_LOG_LEVEL_ERROR, + "parakeet: unsupported spk_kernel_type \"%s\" " + "(only \"ff\" is implemented)", + hp.spk_kernel_type.c_str()); + return TRANSCRIBE_ERR_GGUF; + } + if (auto st = read_optional_bool_kv(gguf, "stt.parakeet.encoder.add_bg_spk_kernel", kFamilyTag, + /*default_value=*/false, hp.add_bg_spk_kernel); + st != TRANSCRIBE_OK) { + return st; + } + // Only layer-0 pre-hook injection is implemented. NeMo registers a + // forward_pre_hook for idx==0 and a post-hook (after layer idx-1) + // for idx>0; the post path is untested here and this variant ships + // spk_kernel_layers=[0]. + for (int32_t l : hp.spk_kernel_layers) { + if (l != 0) { + log_msg(TRANSCRIBE_LOG_LEVEL_ERROR, + "parakeet: spk_kernel_layers entry %d unsupported " + "(only layer-0 injection implemented)", + l); + return TRANSCRIBE_ERR_GGUF; + } + } + } + } + // Cross-field invariants — caught here rather than surfacing as // confusing shape mismatches downstream. if (hp.enc_n_layers <= 0 || hp.enc_d_model <= 0 || hp.enc_n_heads <= 0 || hp.enc_d_ff <= 0 || @@ -869,6 +921,30 @@ transcribe_status build_parakeet_weights(ggml_context * ctx_meta, GET_F32(weights.prompt.mlp2_b, "prompt.mlp.2.bias", d_model); } + // ----- speaker-kernel FF (multitalker variants, has_spk_kernel only) ----- + // Two Linears per kernel (nn.Sequential .0 / .3), both [d_model, d_model] + // + [d_model] bias; the optional bg kernel mirrors the spk kernel. + if (hp.has_spk_kernel) { + weights.spk_kernels.clear(); + weights.spk_kernels.reserve(hp.spk_kernel_layers.size()); + for (int32_t layer : hp.spk_kernel_layers) { + ParakeetSpkKernel kernel{}; + kernel.layer = layer; + kernel.has_bg = hp.add_bg_spk_kernel; + GET_LIN(kernel.spk.lin0_w, lname("enc.spk_kernel.%d.0.weight", layer), d_model, d_model); + GET_F32(kernel.spk.lin0_b, lname("enc.spk_kernel.%d.0.bias", layer), d_model); + GET_LIN(kernel.spk.lin3_w, lname("enc.spk_kernel.%d.3.weight", layer), d_model, d_model); + GET_F32(kernel.spk.lin3_b, lname("enc.spk_kernel.%d.3.bias", layer), d_model); + if (hp.add_bg_spk_kernel) { + GET_LIN(kernel.bg.lin0_w, lname("enc.bg_spk_kernel.%d.0.weight", layer), d_model, d_model); + GET_F32(kernel.bg.lin0_b, lname("enc.bg_spk_kernel.%d.0.bias", layer), d_model); + GET_LIN(kernel.bg.lin3_w, lname("enc.bg_spk_kernel.%d.3.weight", layer), d_model, d_model); + GET_F32(kernel.bg.lin3_b, lname("enc.bg_spk_kernel.%d.3.bias", layer), d_model); + } + weights.spk_kernels.push_back(kernel); + } + } + return TRANSCRIBE_OK; } diff --git a/src/arch/parakeet/weights.h b/src/arch/parakeet/weights.h index b6e291ee..a1da539f 100644 --- a/src/arch/parakeet/weights.h +++ b/src/arch/parakeet/weights.h @@ -209,6 +209,29 @@ struct ParakeetHParams { std::vector prompt_dictionary_indices; int32_t prompt_auto_id = -1; + // Speaker-kernel FF injection (multitalker variants, NeMo + // EncDecMultiTalkerRNNTBPEModel / SpeakerKernelMixin). At each layer in + // spk_kernel_layers, an always-on forward_pre_hook injects a per-speaker + // FF kernel plus (when add_bg_spk_kernel) a background FF kernel into the + // block input. The hook fires UNCONDITIONALLY: even single-speaker + // inference (no diarization targets) applies it, so it is load-bearing for + // the single-speaker path, not just multi-speaker decoding. In + // single-speaker mode the layer-0 injection reduces to a deterministic + // transform of the block-0 input (post pre-encode + xscale): + // x += W3·relu(W0·x + b0) + b3 (spk kernel, per frame) + // x += W3_bg·relu(b0_bg) + b3_bg (bg kernel; NeMo masks the + // bg input to all-zeros, so + // its output is a constant + // vector broadcast over time) + // Gated on has_spk_kernel (stt.parakeet.encoder.spk_kernel_layers + // present). Only layer-0 injection is implemented (this variant ships + // spk_kernel_layers=[0]); NeMo's idx>0 post-hook path is rejected at load. + // Today: multitalker-parakeet-streaming-0.6b-v1. + bool has_spk_kernel = false; + std::string spk_kernel_type; // "ff" + std::vector spk_kernel_layers; // e.g. [0] + bool add_bg_spk_kernel = false; + // Derived. Convenience accessors keyed off the above. int32_t enc_head_dim() const { return enc_n_heads > 0 ? enc_d_model / enc_n_heads : 0; } @@ -348,13 +371,34 @@ struct ParakeetPromptMlp { ggml_tensor * mlp2_b = nullptr; // [d_model] }; +// Speaker-kernel FF module (multitalker variants). NeMo +// nn.Sequential(Linear.0, ReLU, Dropout, Linear.3); the .1 (ReLU) / .2 +// (Dropout, identity at eval) slots are parameter-free, so only the two +// Linears carry tensors. Both are [d_model, d_model] + [d_model] bias. +struct ParakeetSpkKernelFF { + ggml_tensor * lin0_w = nullptr; // [d_model, d_model] + ggml_tensor * lin0_b = nullptr; // [d_model] + ggml_tensor * lin3_w = nullptr; // [d_model, d_model] + ggml_tensor * lin3_b = nullptr; // [d_model] +}; + +// One (spk, optional bg) kernel pair bound to an encoder layer index. +// Loaded only when hp.has_spk_kernel. +struct ParakeetSpkKernel { + int32_t layer = -1; + ParakeetSpkKernelFF spk; // enc.spk_kernel..* + bool has_bg = false; + ParakeetSpkKernelFF bg; // enc.bg_spk_kernel..* (add_bg_spk_kernel) +}; + struct ParakeetWeights { - ParakeetPreEncode pre_encode; - std::vector blocks; // hp.enc_n_layers entries - ParakeetPredictor predictor; // empty when head_kind=CTC - ParakeetJoint joint; // empty when head_kind=CTC - ParakeetCtcHead ctc_head; // populated when head_kind=CTC - ParakeetPromptMlp prompt; // populated when hp.has_prompt + ParakeetPreEncode pre_encode; + std::vector blocks; // hp.enc_n_layers entries + ParakeetPredictor predictor; // empty when head_kind=CTC + ParakeetJoint joint; // empty when head_kind=CTC + ParakeetCtcHead ctc_head; // populated when head_kind=CTC + ParakeetPromptMlp prompt; // populated when hp.has_prompt + std::vector spk_kernels; // populated when hp.has_spk_kernel }; // Walk the canonical tensor list, look up each tensor by name, validate diff --git a/src/transcribe-batch-util.cpp b/src/transcribe-batch-util.cpp index b556e805..947139df 100644 --- a/src/transcribe-batch-util.cpp +++ b/src/transcribe-batch-util.cpp @@ -145,12 +145,25 @@ void fill_keypad_mask(ggml_tensor * mask, const std::vector & real_lens, in if (mask == nullptr) { return; } - const float ninf = -std::numeric_limits::infinity(); + // Padded keys get a large FINITE negative, not -inf. A finite sentinel is + // bit-exact with -inf for any query that has at least one non-masked key + // (expf(-1e30 - finite_row_max) underflows to exactly 0.0f, same as + // exp(-inf)), and it casts to -inf in F16 so the flash path is unchanged. + // The reason it must be finite: in a variable-length batch a padded QUERY + // row (a frame past the real length) can have its entire chunked-attention + // window fall on padded keys. With -inf that row is all -inf and the + // manual (non-flash) softmax computes 0/0 = NaN, which then poisons real + // frames at the next layer via 0*NaN (a masked padded key still multiplies + // its NaN value). A finite sentinel makes such a row softmax to a finite + // (near-uniform) value instead, so no NaN is ever created or propagated. + // Flash attention skips masked keys and never hit this; this keeps the + // manual/bit-exact path robust for diverse-length batches too. + const float kMaskNeg = -1e30f; std::vector buf(static_cast(T) * n); for (int b = 0; b < n; ++b) { const int real = real_lens[static_cast(b)]; for (int k = 0; k < T; ++k) { - buf[static_cast(b) * T + k] = (k < real) ? 0.0f : ninf; + buf[static_cast(b) * T + k] = (k < real) ? 0.0f : kMaskNeg; } } ggml_backend_tensor_set(mask, buf.data(), 0, buf.size() * sizeof(float)); diff --git a/tests/golden/batch/multitalker-parakeet-streaming-0.6b-v1.cpu.json b/tests/golden/batch/multitalker-parakeet-streaming-0.6b-v1.cpu.json new file mode 100644 index 00000000..90fa823f --- /dev/null +++ b/tests/golden/batch/multitalker-parakeet-streaming-0.6b-v1.cpu.json @@ -0,0 +1,15 @@ +{ + "model": "models/multitalker-parakeet-streaming-0.6b-v1/multitalker-parakeet-streaming-0.6b-v1-F32.gguf", + "backend": "cpu", + "language": "en", + "texts": { + "samples/wer/fleurs-en/fleurs-en-1003119935936341070.wav": "However, due to the slow communication channels, styles in the West could lag behind by 25 to 30 years.", + "samples/wer/fleurs-en/fleurs-en-10052240106321793346.wav": "All nouns alongside the world say for you always begin with a capital letter even in the middle of a sentence", + "samples/wer/fleurs-en/fleurs-en-10167324587744183095.wav": "to the north and within easy reach is a romantic and fascinating town of Sintra in which was made famous to foreigners after a glowing account of its splendor as recorded by Lord Byron.", + "samples/wer/fleurs-en/fleurs-en-10197164397713068203.wav": "the cabbage juice changes color depending on how acidic basic alkaline the chemical is.", + "samples/wer/fleurs-en/fleurs-en-10233995782544396174.wav": "Many people don't think about them as dinosaurs because they have feathers and can fly.", + "samples/wer/fleurs-en/fleurs-en-10235738076911595987.wav": "The hospital has followed protocol for infection control, including separating the patient from others to prevent possible infection of others.", + "samples/wer/fleurs-en/fleurs-en-10323357556845456822.wav": "The Northern Marianas Emergency Management Office said that there were no damages reported in the nation.", + "samples/wer/fleurs-en/fleurs-en-10335587906469246566.wav": "Twentieth Century research has shown that there are two pools of genetic variation hidden in expressed." + } +} diff --git a/tests/tolerances/multitalker-parakeet-streaming-0.6b-v1.json b/tests/tolerances/multitalker-parakeet-streaming-0.6b-v1.json index aa4bc27f..27356219 100644 --- a/tests/tolerances/multitalker-parakeet-streaming-0.6b-v1.json +++ b/tests/tolerances/multitalker-parakeet-streaming-0.6b-v1.json @@ -1,91 +1,96 @@ { "_comment": [ - "multitalker-parakeet-streaming-0.6b-v1 PROVISIONAL per-tensor tolerances (Stage 2 oracle).", + "multitalker-parakeet-streaming-0.6b-v1 FINALIZED per-tensor tolerances for compare_tensors.py.", "", - "Magnitude-aware: max_abs = max(1e-4 * p99_abs, 1e-6);", - " mean_abs = max(1e-5 * rms, 1e-6).", - "When a tensor name appears in multiple (case, stage_dir) sidecars the", - "MAX p99_abs / rms across them is used.", + "Correctness regime (Stage 4 contract):", + " - reference-dtype GGUF (F32) at models/multitalker-parakeet-streaming-0.6b-v1/*-F32.gguf", + " - C++ build: CPU backend, --threads 1 (default validate.py invocation)", + " - KV cache type: AUTO (resolves to F32 on the F32 weights)", + " - mel frontend: C++ transcribe-mel (no env-var ref injection); enc.mel.in", + " carries a real production-mel budget, not a pinned 0.0.", + " - reference: NeMo 2.8.0rc0 fp32 inference via", + " scripts/dump_reference_parakeet_nemo.py (offline mode, native", + " att_context_style='chunked_limited', single-speaker).", "", - "PROVISIONAL: Stage 4 finalizes against observed C++ drift. Every entry", - "carries _provisional: true. DO NOT ship while _provisional entries remain.", + "Model note: this is EncDecMultiTalkerRNNTBPEModel run in SINGLE-SPEAKER mode.", + "The base ASR backbone is identical to nemotron-speech-streaming-en-0.6b, but a", + "speaker-kernel FF residual is injected at the INPUT of conformer layer 0", + "(NeMo forward_pre_hook, applied unconditionally): x += spk_ff(x) (all-ones", + "mask) then x += bg_ff(0) (a constant bias). The C++ path implements this in", + "src/arch/parakeet/encoder.cpp::apply_spk_kernel_injection.", "", - "Model note: this is EncDecMultiTalkerRNNTBPEModel run in SINGLE-SPEAKER", - "mode. The base ASR backbone is identical to nemotron-speech-streaming-en-0.6b,", - "BUT a speaker-kernel FF residual is applied at the INPUT of conformer", - "layer 0 (forward_pre_hook, applied unconditionally): x += spk_ff(x) with an", - "all-ones mask, then x += bg_ff(0) (constant bias). Stage 3 MUST convert", - "spk_kernels.0.* and bg_spk_kernels.0.* and Stage 4 MUST apply this injection;", - "the single-speaker path is NOT numerically identical to the base model." + "Dominant drift source: CPU F32 conformer accumulation order across 24 blocks,", + "plus the added layer-0 FF injection. All drift is well-understood accumulation,", + "NOT a localized mask/position bug (transcript is exact; drift peaks mid-stack", + "and shrinks at enc.final via the per-block LayerNorm). Cross-check: every", + "shared encoder tensor drifts LESS than the finalized budget of the", + "nemotron-speech-streaming-en-0.6b sibling (same encoder, no spk-kernel):", + "enc.final 1.9e-3 vs sibling 7.0e-3; block.0 8.4e-3 vs 6.5e-1; block.12", + "3.9e-2 vs 5.7e-1. Since a subtly wrong injection would drift ABOVE the", + "sibling, not below, this bounds the injection as correct.", + "", + "Widened vs Stage 2 provisional (recipe finalized = max(1.5*observed, provisional)):", + " - enc.block.0.out : 3.0e-3 -> 1.3e-2 (~4.3x; injection + block-0 accumulation)", + " - enc.block.12.out: 3.4e-3 -> 6.0e-2 (~18x; mid-stack peak, ~0.11% relative)", + " - enc.final / enc.block.23.out / dec.enc_out: 1.9e-5 -> 3.0e-3 (final encoder", + " output, one underlying tensor; ~1.0% relative, vs 3.7% absorbed by the sibling)", + "All other tensors: observed drift below the magnitude budget -> provisional kept.", + "dec.embed.0 is a pure GGUF embedding read -> pinned to exact 0.0." ], "dec.embed.0": { - "max_abs": 1e-06, - "mean_abs": 1e-06, - "_provisional": true + "max_abs": 0.0, + "mean_abs": 0.0 }, "dec.enc_out": { - "max_abs": 1.9078139215707727e-05, - "mean_abs": 1e-06, - "_provisional": true + "max_abs": 0.003, + "mean_abs": 7e-05 }, "dec.joint.0": { "max_abs": 0.005523143569946289, - "mean_abs": 0.00032965458532807114, - "_provisional": true + "mean_abs": 0.00032965458532807114 }, "dec.lstm.0.c.0": { "max_abs": 7.170465892553333e-05, - "mean_abs": 2.375066490445493e-06, - "_provisional": true + "mean_abs": 2.375066490445493e-06 }, "dec.lstm.0.h.0": { "max_abs": 3.142599546909333e-05, - "mean_abs": 1.0800957467118525e-06, - "_provisional": true + "mean_abs": 1.0800957467118525e-06 }, "dec.lstm.1.c.0": { "max_abs": 8.615481072664261e-05, - "mean_abs": 4.230338949264561e-06, - "_provisional": true + "mean_abs": 4.230338949264561e-06 }, "dec.lstm.1.h.0": { "max_abs": 3.389731752872467e-05, - "mean_abs": 1e-06, - "_provisional": true + "mean_abs": 1e-06 }, "enc.block.0.out": { - "max_abs": 0.003004386568069458, - "mean_abs": 8.872089894130494e-05, - "_provisional": true + "max_abs": 0.013, + "mean_abs": 0.0005 }, "enc.block.12.out": { - "max_abs": 0.003371430587768542, - "mean_abs": 7.71841725708955e-05, - "_provisional": true + "max_abs": 0.06, + "mean_abs": 0.002 }, "enc.block.23.out": { - "max_abs": 1.9078139215707727e-05, - "mean_abs": 1e-06, - "_provisional": true + "max_abs": 0.003, + "mean_abs": 7e-05 }, "enc.final": { - "max_abs": 1.9078139215707727e-05, - "mean_abs": 1e-06, - "_provisional": true + "max_abs": 0.003, + "mean_abs": 7e-05 }, "enc.mel.in": { "max_abs": 0.0016611215076446535, - "mean_abs": 0.0001012863475937363, - "_provisional": true + "mean_abs": 0.0001012863475937363 }, "enc.pos_emb": { "max_abs": 9.999976158142091e-05, - "mean_abs": 7.071067828517533e-06, - "_provisional": true + "mean_abs": 7.071067828517533e-06 }, "enc.pre_encode.out": { "max_abs": 0.06187373504638646, - "mean_abs": 0.0017322517756891871, - "_provisional": true + "mean_abs": 0.0017322517756891871 } -} \ No newline at end of file +} diff --git a/tests/tolerances/multitalker-parakeet-streaming-0.6b-v1.streaming.json b/tests/tolerances/multitalker-parakeet-streaming-0.6b-v1.streaming.json index 81941127..bdfe7687 100644 --- a/tests/tolerances/multitalker-parakeet-streaming-0.6b-v1.streaming.json +++ b/tests/tolerances/multitalker-parakeet-streaming-0.6b-v1.streaming.json @@ -1,764 +1,83 @@ { "_comment": [ - "multitalker-parakeet-streaming-0.6b-v1 PROVISIONAL per-tensor tolerances (Stage 2 oracle).", + "multitalker-parakeet-streaming-0.6b-v1 streaming per-tensor tolerances for validate_streaming.py.", "", - "Magnitude-aware: max_abs = max(1e-4 * p99_abs, 1e-6);", - " mean_abs = max(1e-5 * rms, 1e-6).", - "When a tensor name appears in multiple (case, stage_dir) sidecars the", - "MAX p99_abs / rms across them is used.", + "Calibration regime (matches /porting-2-oracle Stage 4 recipe):", + " - reference: NeMo 2.8.0rc0 conformer_stream_step on jfk.wav, R=13 (att_context [70,13]),", + " single-speaker EncDecMultiTalkerRNNTBPEModel (layer-0 spk-kernel injection applied).", + " - observed: C++ stream path, --backend cpu --threads 1, F32 GGUF, chunk cadence 500 ms.", + " - per kind = max-of-max across all chunks and dumped layers (0/12/23).", + " - per-entry: max_abs = max(1.5 x observed_max_abs, max(1e-4 x p99_abs, 1e-6))", + " mean_abs = max(1.5 x observed_mean_abs, max(1e-5 x rms, 1e-6))", "", - "PROVISIONAL: Stage 4 finalizes against observed C++ drift. Every entry", - "carries _provisional: true. DO NOT ship while _provisional entries remain.", - "Streaming stage tensors (att_context_size=[70,13], 1.12s chunk)." + "Model note: the streaming encoder applies the same always-on layer-0 speaker-kernel", + "FF injection as the offline path (encoder.cpp::apply_spk_kernel_injection), per chunk", + "on the post-drop block-0 input. Result: 150/150 tensors within tolerance, edit_dist 0,", + "transcript byte-exact vs NeMo at R=13.", + "", + "Cross-check: streaming enc_out drift (obs 5.5e-6) is TIGHTER than the offline C++", + "enc.final drift (1.9e-3) on the same audio -- clean per-chunk caches accumulate less", + "than a single 24-block offline pass. Every kind's budget is magnitude-dominated except", + "cache_lt / mel_in (1.5x-observed dominated). Same convention as the", + "nemotron-speech-streaming-en-0.6b sibling.", + "", + "Pattern-keyed: stream.chunk..[_] all share the same '' entry;", + "validate_streaming.py expands these at harness time." ], - "stream.chunk.0.cache_lc_in_0": { - "max_abs": 1e-06, - "mean_abs": 1e-06, - "_provisional": true - }, - "stream.chunk.0.cache_lc_in_12": { - "max_abs": 1e-06, - "mean_abs": 1e-06, - "_provisional": true - }, - "stream.chunk.0.cache_lc_in_23": { - "max_abs": 1e-06, - "mean_abs": 1e-06, - "_provisional": true - }, - "stream.chunk.0.cache_lc_out_0": { - "max_abs": 1.3008432164788315e-05, - "mean_abs": 1e-06, - "_provisional": true - }, - "stream.chunk.0.cache_lc_out_12": { - "max_abs": 2.8398949261754838e-06, - "mean_abs": 1e-06, - "_provisional": true - }, - "stream.chunk.0.cache_lc_out_23": { - "max_abs": 1.1010672606527852e-05, - "mean_abs": 1e-06, - "_provisional": true - }, - "stream.chunk.0.cache_lt_in_0": { - "max_abs": 1e-06, - "mean_abs": 1e-06, - "_provisional": true - }, - "stream.chunk.0.cache_lt_in_12": { - "max_abs": 1e-06, - "mean_abs": 1e-06, - "_provisional": true - }, - "stream.chunk.0.cache_lt_in_23": { - "max_abs": 1e-06, - "mean_abs": 1e-06, - "_provisional": true - }, - "stream.chunk.0.cache_lt_out_0": { - "max_abs": 0.00129465841293335, - "mean_abs": 3.407372497108534e-05, - "_provisional": true - }, - "stream.chunk.0.cache_lt_out_12": { - "max_abs": 0.0017155433769226083, - "mean_abs": 3.6446279329674886e-05, - "_provisional": true - }, - "stream.chunk.0.cache_lt_out_23": { - "max_abs": 0.0005470749974250801, - "mean_abs": 1.2106562129439416e-05, - "_provisional": true - }, - "stream.chunk.0.channel_len": { - "max_abs": 0.0014, - "mean_abs": 0.00014000000000000001, - "_provisional": true - }, - "stream.chunk.0.enc_out": { - "max_abs": 1.7253484055399895e-05, - "mean_abs": 1e-06, - "_provisional": true - }, - "stream.chunk.0.mel_in": { - "max_abs": 0.0016635532379150391, - "mean_abs": 0.00010467764062887827, - "_provisional": true - }, - "stream.chunk.1.cache_lc_in_0": { - "max_abs": 1.3008432164788315e-05, - "mean_abs": 1e-06, - "_provisional": true - }, - "stream.chunk.1.cache_lc_in_12": { - "max_abs": 2.8398949261754838e-06, - "mean_abs": 1e-06, - "_provisional": true - }, - "stream.chunk.1.cache_lc_in_23": { - "max_abs": 1.1010672606527852e-05, - "mean_abs": 1e-06, - "_provisional": true - }, - "stream.chunk.1.cache_lc_out_0": { - "max_abs": 1.5874018907547004e-05, - "mean_abs": 1e-06, - "_provisional": true - }, - "stream.chunk.1.cache_lc_out_12": { - "max_abs": 3.661312390118844e-06, - "mean_abs": 1e-06, - "_provisional": true - }, - "stream.chunk.1.cache_lc_out_23": { - "max_abs": 1.2638793796300969e-05, - "mean_abs": 1e-06, - "_provisional": true - }, - "stream.chunk.1.cache_lt_in_0": { - "max_abs": 0.00129465841293335, - "mean_abs": 3.407372497108534e-05, - "_provisional": true - }, - "stream.chunk.1.cache_lt_in_12": { - "max_abs": 0.0017155433769226083, - "mean_abs": 3.6446279329674886e-05, - "_provisional": true - }, - "stream.chunk.1.cache_lt_in_23": { - "max_abs": 0.0005470749974250801, - "mean_abs": 1.2106562129439416e-05, - "_provisional": true - }, - "stream.chunk.1.cache_lt_out_0": { - "max_abs": 0.0012747896041870128, - "mean_abs": 3.367902436206685e-05, - "_provisional": true - }, - "stream.chunk.1.cache_lt_out_12": { - "max_abs": 0.001866611431121827, - "mean_abs": 3.916475561918408e-05, - "_provisional": true - }, - "stream.chunk.1.cache_lt_out_23": { - "max_abs": 0.0005453417639732367, - "mean_abs": 1.4001124989410301e-05, - "_provisional": true - }, - "stream.chunk.1.channel_len": { - "max_abs": 0.0028, - "mean_abs": 0.00028000000000000003, - "_provisional": true - }, - "stream.chunk.1.enc_out": { - "max_abs": 1.9079485833644858e-05, - "mean_abs": 1e-06, - "_provisional": true - }, - "stream.chunk.1.mel_in": { - "max_abs": 0.0016477878952026366, - "mean_abs": 9.115775346884796e-05, - "_provisional": true - }, - "stream.chunk.2.cache_lc_in_0": { - "max_abs": 1.5874018907547004e-05, - "mean_abs": 1e-06, - "_provisional": true - }, - "stream.chunk.2.cache_lc_in_12": { - "max_abs": 3.661312390118844e-06, - "mean_abs": 1e-06, - "_provisional": true - }, - "stream.chunk.2.cache_lc_in_23": { - "max_abs": 1.2638793796300969e-05, - "mean_abs": 1e-06, - "_provisional": true - }, - "stream.chunk.2.cache_lc_out_0": { - "max_abs": 1.6420035868883166e-05, - "mean_abs": 1e-06, - "_provisional": true - }, - "stream.chunk.2.cache_lc_out_12": { - "max_abs": 4.235666163265739e-06, - "mean_abs": 1e-06, - "_provisional": true - }, - "stream.chunk.2.cache_lc_out_23": { - "max_abs": 1.3597931206226375e-05, - "mean_abs": 1e-06, - "_provisional": true - }, - "stream.chunk.2.cache_lt_in_0": { - "max_abs": 0.0012747896041870128, - "mean_abs": 3.367902436206685e-05, - "_provisional": true - }, - "stream.chunk.2.cache_lt_in_12": { - "max_abs": 0.001866611431121827, - "mean_abs": 3.916475561918408e-05, - "_provisional": true - }, - "stream.chunk.2.cache_lt_in_23": { - "max_abs": 0.0005453417639732367, - "mean_abs": 1.4001124989410301e-05, - "_provisional": true - }, - "stream.chunk.2.cache_lt_out_0": { - "max_abs": 0.001035860824584961, - "mean_abs": 2.7844846226694173e-05, - "_provisional": true - }, - "stream.chunk.2.cache_lt_out_12": { - "max_abs": 0.0020686249885559084, - "mean_abs": 4.04995574051368e-05, - "_provisional": true - }, - "stream.chunk.2.cache_lt_out_23": { - "max_abs": 0.00043792397785186773, - "mean_abs": 1.1110986077387435e-05, - "_provisional": true - }, - "stream.chunk.2.channel_len": { - "max_abs": 0.004200000000000001, - "mean_abs": 0.00042, - "_provisional": true - }, - "stream.chunk.2.enc_out": { - "max_abs": 1.8786662966012952e-05, - "mean_abs": 1e-06, - "_provisional": true - }, - "stream.chunk.2.mel_in": { - "max_abs": 0.0015503223133087158, - "mean_abs": 0.00011653641447048212, - "_provisional": true - }, - "stream.chunk.3.cache_lc_in_0": { - "max_abs": 1.6420035868883166e-05, - "mean_abs": 1e-06, - "_provisional": true - }, - "stream.chunk.3.cache_lc_in_12": { - "max_abs": 4.235666163265739e-06, - "mean_abs": 1e-06, - "_provisional": true - }, - "stream.chunk.3.cache_lc_in_23": { - "max_abs": 1.3597931206226375e-05, - "mean_abs": 1e-06, - "_provisional": true - }, - "stream.chunk.3.cache_lc_out_0": { - "max_abs": 1.762249697744852e-05, - "mean_abs": 1e-06, - "_provisional": true - }, - "stream.chunk.3.cache_lc_out_12": { - "max_abs": 4.634001653641463e-06, - "mean_abs": 1e-06, - "_provisional": true - }, - "stream.chunk.3.cache_lc_out_23": { - "max_abs": 1.4167193681001697e-05, - "mean_abs": 1e-06, - "_provisional": true - }, - "stream.chunk.3.cache_lt_in_0": { - "max_abs": 0.001035860824584961, - "mean_abs": 2.7844846226694173e-05, - "_provisional": true - }, - "stream.chunk.3.cache_lt_in_12": { - "max_abs": 0.0020686249885559084, - "mean_abs": 4.04995574051368e-05, - "_provisional": true - }, - "stream.chunk.3.cache_lt_in_23": { - "max_abs": 0.00043792397785186773, - "mean_abs": 1.1110986077387435e-05, - "_provisional": true - }, - "stream.chunk.3.cache_lt_out_0": { - "max_abs": 0.0011561623067855844, - "mean_abs": 3.139138916511903e-05, - "_provisional": true - }, - "stream.chunk.3.cache_lt_out_12": { - "max_abs": 0.0019240166225433354, - "mean_abs": 4.012968807607016e-05, - "_provisional": true - }, - "stream.chunk.3.cache_lt_out_23": { - "max_abs": 0.0006169222645759585, - "mean_abs": 1.515453647116478e-05, - "_provisional": true - }, - "stream.chunk.3.channel_len": { - "max_abs": 0.0056, - "mean_abs": 0.0005600000000000001, - "_provisional": true - }, - "stream.chunk.3.enc_out": { - "max_abs": 1.6622247621417004e-05, - "mean_abs": 1e-06, - "_provisional": true - }, - "stream.chunk.3.mel_in": { - "max_abs": 0.001660875274658203, - "mean_abs": 9.622120975163936e-05, - "_provisional": true - }, - "stream.chunk.4.cache_lc_in_0": { - "max_abs": 1.762249697744852e-05, - "mean_abs": 1e-06, - "_provisional": true - }, - "stream.chunk.4.cache_lc_in_12": { - "max_abs": 4.634001653641463e-06, - "mean_abs": 1e-06, - "_provisional": true - }, - "stream.chunk.4.cache_lc_in_23": { - "max_abs": 1.4167193681001697e-05, - "mean_abs": 1e-06, - "_provisional": true - }, - "stream.chunk.4.cache_lc_out_0": { - "max_abs": 1.7965956419706414e-05, - "mean_abs": 1e-06, - "_provisional": true - }, - "stream.chunk.4.cache_lc_out_12": { - "max_abs": 4.886499505490066e-06, - "mean_abs": 1e-06, - "_provisional": true - }, - "stream.chunk.4.cache_lc_out_23": { - "max_abs": 1.5018582731485378e-05, - "mean_abs": 1e-06, - "_provisional": true - }, - "stream.chunk.4.cache_lt_in_0": { - "max_abs": 0.0011561623067855844, - "mean_abs": 3.139138916511903e-05, - "_provisional": true - }, - "stream.chunk.4.cache_lt_in_12": { - "max_abs": 0.0019240166225433354, - "mean_abs": 4.012968807607016e-05, - "_provisional": true - }, - "stream.chunk.4.cache_lt_in_23": { - "max_abs": 0.0006169222645759585, - "mean_abs": 1.515453647116478e-05, - "_provisional": true - }, - "stream.chunk.4.cache_lt_out_0": { - "max_abs": 0.0010491846704483037, - "mean_abs": 2.8803309296311065e-05, - "_provisional": true - }, - "stream.chunk.4.cache_lt_out_12": { - "max_abs": 0.0018458886489868176, - "mean_abs": 4.0284489757683054e-05, - "_provisional": true - }, - "stream.chunk.4.cache_lt_out_23": { - "max_abs": 0.0005636508593559268, - "mean_abs": 1.3180694872271984e-05, - "_provisional": true - }, - "stream.chunk.4.channel_len": { - "max_abs": 0.007, - "mean_abs": 0.0007000000000000001, - "_provisional": true - }, - "stream.chunk.4.enc_out": { - "max_abs": 2.1179130151867858e-05, - "mean_abs": 1e-06, - "_provisional": true - }, - "stream.chunk.4.mel_in": { - "max_abs": 0.0015525638256072996, - "mean_abs": 0.00011270580396839228, - "_provisional": true - }, - "stream.chunk.5.cache_lc_in_0": { - "max_abs": 1.7965956419706414e-05, - "mean_abs": 1e-06, - "_provisional": true - }, - "stream.chunk.5.cache_lc_in_12": { - "max_abs": 4.886499505490066e-06, - "mean_abs": 1e-06, - "_provisional": true - }, - "stream.chunk.5.cache_lc_in_23": { - "max_abs": 1.5018582731485378e-05, - "mean_abs": 1e-06, - "_provisional": true - }, - "stream.chunk.5.cache_lc_out_0": { - "max_abs": 1.8002947837114417e-05, - "mean_abs": 1e-06, - "_provisional": true - }, - "stream.chunk.5.cache_lc_out_12": { - "max_abs": 4.958076093345887e-06, - "mean_abs": 1e-06, - "_provisional": true - }, - "stream.chunk.5.cache_lc_out_23": { - "max_abs": 1.4514548420906077e-05, - "mean_abs": 1e-06, - "_provisional": true - }, - "stream.chunk.5.cache_lt_in_0": { - "max_abs": 0.0010491846704483037, - "mean_abs": 2.8803309296311065e-05, - "_provisional": true - }, - "stream.chunk.5.cache_lt_in_12": { - "max_abs": 0.0018458886489868176, - "mean_abs": 4.0284489757683054e-05, - "_provisional": true - }, - "stream.chunk.5.cache_lt_in_23": { - "max_abs": 0.0005636508593559268, - "mean_abs": 1.3180694872271984e-05, - "_provisional": true - }, - "stream.chunk.5.cache_lt_out_0": { - "max_abs": 0.001214497030258179, - "mean_abs": 3.226833464901226e-05, - "_provisional": true - }, - "stream.chunk.5.cache_lt_out_12": { - "max_abs": 0.001899435621261597, - "mean_abs": 4.010647615056664e-05, - "_provisional": true - }, - "stream.chunk.5.cache_lt_out_23": { - "max_abs": 0.0005633621072769167, - "mean_abs": 1.36036978067309e-05, - "_provisional": true - }, - "stream.chunk.5.channel_len": { - "max_abs": 0.007, - "mean_abs": 0.0007000000000000001, - "_provisional": true - }, - "stream.chunk.5.enc_out": { - "max_abs": 1.8819435313343986e-05, - "mean_abs": 1e-06, - "_provisional": true - }, - "stream.chunk.5.mel_in": { - "max_abs": 0.0016615625114440918, - "mean_abs": 9.371562068416148e-05, - "_provisional": true - }, - "stream.chunk.6.cache_lc_in_0": { + "cache_lc_in": { "max_abs": 1.8002947837114417e-05, "mean_abs": 1e-06, - "_provisional": true - }, - "stream.chunk.6.cache_lc_in_12": { - "max_abs": 4.958076093345887e-06, - "mean_abs": 1e-06, - "_provisional": true + "_observed_max": 5.904585123062134e-06, + "_observed_mean": 1.3644430385539863e-07, + "_ref_p99": 0.18002947837114416, + "_ref_rms": 0.06312107247075341 }, - "stream.chunk.6.cache_lc_in_23": { - "max_abs": 1.4514548420906077e-05, - "mean_abs": 1e-06, - "_provisional": true - }, - "stream.chunk.6.cache_lc_out_0": { - "max_abs": 1.759598450362688e-05, - "mean_abs": 1e-06, - "_provisional": true - }, - "stream.chunk.6.cache_lc_out_12": { - "max_abs": 4.974784672260289e-06, - "mean_abs": 1e-06, - "_provisional": true - }, - "stream.chunk.6.cache_lc_out_23": { - "max_abs": 1.4584889575839045e-05, + "cache_lc_out": { + "max_abs": 1.801336298882962e-05, "mean_abs": 1e-06, - "_provisional": true - }, - "stream.chunk.6.cache_lt_in_0": { - "max_abs": 0.001214497030258179, - "mean_abs": 3.226833464901226e-05, - "_provisional": true + "_observed_max": 5.904585123062134e-06, + "_observed_mean": 1.3644430385539863e-07, + "_ref_p99": 0.18013362988829618, + "_ref_rms": 0.06470617190344634 }, - "stream.chunk.6.cache_lt_in_12": { - "max_abs": 0.001899435621261597, - "mean_abs": 4.010647615056664e-05, - "_provisional": true - }, - "stream.chunk.6.cache_lt_in_23": { - "max_abs": 0.0005633621072769167, - "mean_abs": 1.36036978067309e-05, - "_provisional": true - }, - "stream.chunk.6.cache_lt_out_0": { - "max_abs": 0.0011369490146636964, - "mean_abs": 3.1325558439493076e-05, - "_provisional": true - }, - "stream.chunk.6.cache_lt_out_12": { - "max_abs": 0.0020269768905639652, + "cache_lt_in": { + "max_abs": 0.0020686249885559084, "mean_abs": 4.305163344313818e-05, - "_provisional": true - }, - "stream.chunk.6.cache_lt_out_23": { - "max_abs": 0.0006229003634452821, - "mean_abs": 1.403079167721605e-05, - "_provisional": true - }, - "stream.chunk.6.channel_len": { - "max_abs": 0.007, - "mean_abs": 0.0007000000000000001, - "_provisional": true - }, - "stream.chunk.6.enc_out": { - "max_abs": 1.9951290562748903e-05, - "mean_abs": 1e-06, - "_provisional": true - }, - "stream.chunk.6.mel_in": { - "max_abs": 0.001640410678863525, - "mean_abs": 9.860686542991035e-05, - "_provisional": true - }, - "stream.chunk.7.cache_lc_in_0": { - "max_abs": 1.759598450362688e-05, - "mean_abs": 1e-06, - "_provisional": true - }, - "stream.chunk.7.cache_lc_in_12": { - "max_abs": 4.974784672260289e-06, - "mean_abs": 1e-06, - "_provisional": true + "_observed_max": 0.00021599233150482178, + "_observed_mean": 5.8020809245371685e-06, + "_ref_p99": 20.686249885559082, + "_ref_rms": 4.305163344313818 }, - "stream.chunk.7.cache_lc_in_23": { - "max_abs": 1.4584889575839045e-05, - "mean_abs": 1e-06, - "_provisional": true - }, - "stream.chunk.7.cache_lc_out_0": { - "max_abs": 1.7881013616919533e-05, - "mean_abs": 1e-06, - "_provisional": true - }, - "stream.chunk.7.cache_lc_out_12": { - "max_abs": 4.948631994426254e-06, - "mean_abs": 1e-06, - "_provisional": true - }, - "stream.chunk.7.cache_lc_out_23": { - "max_abs": 1.4707824811339391e-05, - "mean_abs": 1e-06, - "_provisional": true - }, - "stream.chunk.7.cache_lt_in_0": { - "max_abs": 0.0011369490146636964, - "mean_abs": 3.1325558439493076e-05, - "_provisional": true - }, - "stream.chunk.7.cache_lt_in_12": { - "max_abs": 0.0020269768905639652, + "cache_lt_out": { + "max_abs": 0.0020686249885559084, "mean_abs": 4.305163344313818e-05, - "_provisional": true - }, - "stream.chunk.7.cache_lt_in_23": { - "max_abs": 0.0006229003634452821, - "mean_abs": 1.403079167721605e-05, - "_provisional": true + "_observed_max": 0.00021599233150482178, + "_observed_mean": 5.8020809245371685e-06, + "_ref_p99": 20.686249885559082, + "_ref_rms": 4.305163344313818 }, - "stream.chunk.7.cache_lt_out_0": { - "max_abs": 0.0011851717004776024, - "mean_abs": 3.145116820762147e-05, - "_provisional": true - }, - "stream.chunk.7.cache_lt_out_12": { - "max_abs": 0.0019229018115997324, - "mean_abs": 4.012440119281274e-05, - "_provisional": true - }, - "stream.chunk.7.cache_lt_out_23": { - "max_abs": 0.0006446033411026006, - "mean_abs": 1.522956991252987e-05, - "_provisional": true - }, - "stream.chunk.7.channel_len": { + "channel_len": { "max_abs": 0.007, "mean_abs": 0.0007000000000000001, - "_provisional": true - }, - "stream.chunk.7.enc_out": { - "max_abs": 1.9170742407441134e-05, - "mean_abs": 1e-06, - "_provisional": true - }, - "stream.chunk.7.mel_in": { - "max_abs": 0.0016593084449768069, - "mean_abs": 0.00010335153344210517, - "_provisional": true - }, - "stream.chunk.8.cache_lc_in_0": { - "max_abs": 1.7881013616919533e-05, - "mean_abs": 1e-06, - "_provisional": true - }, - "stream.chunk.8.cache_lc_in_12": { - "max_abs": 4.948631994426254e-06, - "mean_abs": 1e-06, - "_provisional": true - }, - "stream.chunk.8.cache_lc_in_23": { - "max_abs": 1.4707824811339391e-05, - "mean_abs": 1e-06, - "_provisional": true - }, - "stream.chunk.8.cache_lc_out_0": { - "max_abs": 1.7756513878703127e-05, - "mean_abs": 1e-06, - "_provisional": true - }, - "stream.chunk.8.cache_lc_out_12": { - "max_abs": 4.810116160660997e-06, - "mean_abs": 1e-06, - "_provisional": true - }, - "stream.chunk.8.cache_lc_out_23": { - "max_abs": 1.4881478622555733e-05, - "mean_abs": 1e-06, - "_provisional": true - }, - "stream.chunk.8.cache_lt_in_0": { - "max_abs": 0.0011851717004776024, - "mean_abs": 3.145116820762147e-05, - "_provisional": true + "_observed_max": 0.0, + "_observed_mean": 0.0, + "_ref_p99": 70.0, + "_ref_rms": 70.0 }, - "stream.chunk.8.cache_lt_in_12": { - "max_abs": 0.0019229018115997324, - "mean_abs": 4.012440119281274e-05, - "_provisional": true - }, - "stream.chunk.8.cache_lt_in_23": { - "max_abs": 0.0006446033411026006, - "mean_abs": 1.522956991252987e-05, - "_provisional": true - }, - "stream.chunk.8.cache_lt_out_0": { - "max_abs": 0.0011563131313323985, - "mean_abs": 3.155277608997167e-05, - "_provisional": true - }, - "stream.chunk.8.cache_lt_out_12": { - "max_abs": 0.0018374736843109142, - "mean_abs": 3.9883290914659174e-05, - "_provisional": true - }, - "stream.chunk.8.cache_lt_out_23": { - "max_abs": 0.0006105101847648624, - "mean_abs": 1.4465373597920161e-05, - "_provisional": true - }, - "stream.chunk.8.channel_len": { - "max_abs": 0.007, - "mean_abs": 0.0007000000000000001, - "_provisional": true - }, - "stream.chunk.8.enc_out": { - "max_abs": 1.956931285560131e-05, - "mean_abs": 1e-06, - "_provisional": true - }, - "stream.chunk.8.mel_in": { - "max_abs": 0.001651513826370239, - "mean_abs": 9.543340046170219e-05, - "_provisional": true - }, - "stream.chunk.9.cache_lc_in_0": { - "max_abs": 1.7756513878703127e-05, - "mean_abs": 1e-06, - "_provisional": true - }, - "stream.chunk.9.cache_lc_in_12": { - "max_abs": 4.810116160660997e-06, - "mean_abs": 1e-06, - "_provisional": true - }, - "stream.chunk.9.cache_lc_in_23": { - "max_abs": 1.4881478622555733e-05, - "mean_abs": 1e-06, - "_provisional": true - }, - "stream.chunk.9.cache_lc_out_0": { - "max_abs": 1.801336298882962e-05, - "mean_abs": 1e-06, - "_provisional": true - }, - "stream.chunk.9.cache_lc_out_12": { - "max_abs": 4.760253824293622e-06, - "mean_abs": 1e-06, - "_provisional": true - }, - "stream.chunk.9.cache_lc_out_23": { - "max_abs": 1.4701267391443272e-05, - "mean_abs": 1e-06, - "_provisional": true - }, - "stream.chunk.9.cache_lt_in_0": { - "max_abs": 0.0011563131313323985, - "mean_abs": 3.155277608997167e-05, - "_provisional": true - }, - "stream.chunk.9.cache_lt_in_12": { - "max_abs": 0.0018374736843109142, - "mean_abs": 3.9883290914659174e-05, - "_provisional": true - }, - "stream.chunk.9.cache_lt_in_23": { - "max_abs": 0.0006105101847648624, - "mean_abs": 1.4465373597920161e-05, - "_provisional": true - }, - "stream.chunk.9.cache_lt_out_0": { - "max_abs": 0.0010901470489501955, - "mean_abs": 2.995375954716209e-05, - "_provisional": true - }, - "stream.chunk.9.cache_lt_out_12": { - "max_abs": 0.0019135723266601567, - "mean_abs": 4.109583343672264e-05, - "_provisional": true - }, - "stream.chunk.9.cache_lt_out_23": { - "max_abs": 0.0006757690634727483, - "mean_abs": 1.557291357738545e-05, - "_provisional": true - }, - "stream.chunk.9.channel_len": { - "max_abs": 0.007, - "mean_abs": 0.0007000000000000001, - "_provisional": true - }, - "stream.chunk.9.enc_out": { - "max_abs": 1.854820944368835e-05, + "enc_out": { + "max_abs": 2.1179130151867858e-05, "mean_abs": 1e-06, - "_provisional": true + "_observed_max": 5.4854899644851685e-06, + "_observed_mean": 2.2248031176072152e-07, + "_ref_p99": 0.21179130151867856, + "_ref_rms": 0.07786172341954185 }, - "stream.chunk.9.mel_in": { - "max_abs": 0.0016445308208465578, - "mean_abs": 9.642163761926509e-05, - "_provisional": true + "mel_in": { + "max_abs": 0.0016635532379150391, + "mean_abs": 0.00011653641447048212, + "_observed_max": 0.00025653839111328125, + "_observed_mean": 2.6702478742453055e-06, + "_ref_p99": 16.63553237915039, + "_ref_rms": 11.65364144704821 } -} \ No newline at end of file +} From e6f7bb149d1c95b83f1ef6e83b9f88fce8ca3cb3 Mon Sep 17 00:00:00 2001 From: CJ Pais Date: Sun, 12 Jul 2026 20:46:40 -0700 Subject: [PATCH 05/11] docs --- README.md | 1 + .../multitalker-parakeet-streaming-0.6b-v1.md | 200 ++++++++++++++++++ docs/porting/families/parakeet.md | 7 +- ...ultitalker-parakeet-streaming-0.6b-v1.yaml | 89 ++++++++ 4 files changed, 296 insertions(+), 1 deletion(-) create mode 100644 docs/models/multitalker-parakeet-streaming-0.6b-v1.md create mode 100644 scripts/hf_cards/multitalker-parakeet-streaming-0.6b-v1.yaml diff --git a/README.md b/README.md index 8c8f575d..a20bb5db 100644 --- a/README.md +++ b/README.md @@ -21,6 +21,7 @@ C/C++ speech-to-text inference library. Runs diverse STT model families via [GGU | FunASR Nano | `fun-asr-nano-2512`, `fun-asr-mlt-nano-2512` | [docs/models/fun-asr-nano.md](docs/models/fun-asr-nano.md) | | Nemotron Speech Streaming | `nemotron-speech-streaming-en-0.6b` | [docs/models/nemotron-speech-streaming-en-0.6b.md](docs/models/nemotron-speech-streaming-en-0.6b.md) | | Nemotron 3.5 ASR Streaming | `nemotron-3.5-asr-streaming-0.6b` (multilingual, 40 locales) | [docs/models/nemotron-3.5-asr-streaming-0.6b.md](docs/models/nemotron-3.5-asr-streaming-0.6b.md) | +| Multitalker Parakeet Streaming | `multitalker-parakeet-streaming-0.6b-v1` (single-speaker ASR path only) | [docs/models/multitalker-parakeet-streaming-0.6b-v1.md](docs/models/multitalker-parakeet-streaming-0.6b-v1.md) | | Granite Speech 4 / 4.1 | `granite-4.0-1b-speech`, `granite-speech-4.1-2b{,-plus,-nar}` | [docs/models/granite-speech.md](docs/models/granite-speech.md) | | Voxtral | `voxtral-mini-3b-2507`, `voxtral-small-24b-2507` (audio-LLM; transcription + translation) | [docs/models/voxtral.md](docs/models/voxtral.md) | | Voxtral Realtime | `voxtral-mini-4b-realtime-2602` (streaming audio-LLM) | [docs/models/voxtral-realtime.md](docs/models/voxtral-realtime.md) | diff --git a/docs/models/multitalker-parakeet-streaming-0.6b-v1.md b/docs/models/multitalker-parakeet-streaming-0.6b-v1.md new file mode 100644 index 00000000..7bb38f66 --- /dev/null +++ b/docs/models/multitalker-parakeet-streaming-0.6b-v1.md @@ -0,0 +1,200 @@ +# Multitalker Parakeet Streaming 0.6B v1 + +NVIDIA's [`nvidia/multitalker-parakeet-streaming-0.6b-v1`](https://huggingface.co/nvidia/multitalker-parakeet-streaming-0.6b-v1) +ported to transcribe.cpp. A 0.6B-parameter cache-aware streaming +FastConformer encoder with an RNN-T transducer decoder, fine-tuned from +[`nvidia/nemotron-speech-streaming-en-0.6b`](https://huggingface.co/nvidia/nemotron-speech-streaming-en-0.6b). + +## What it's for + +Offline and cache-aware **streaming** English speech-to-text with greedy +RNN-T decoding. Outputs cased, punctuated transcripts. Token- and +word-level timestamps are available. + +Upstream this is a **multitalker (speaker-attributed)** checkpoint: it can +transcribe several overlapping speakers into per-speaker channels. That +path depends on machinery this port does **not** ship (see +[Known limitations](#known-limitations)). transcribe.cpp exposes only the +model's `single_speaker_mode` ASR path, which disables the multitalker +machinery and runs the checkpoint as a plain cache-aware streaming RNN-T: +identical frontend, encoder, decoder, and tokenizer as its +`nemotron-speech-streaming-en-0.6b` base. This is the LibriSpeech-comparable +path (upstream reports test-clean 2.19% in this mode). + +This port runs the model in both **offline** and **cache-aware streaming** +modes: + +- Offline (`transcribe_run`): the full audio is consumed in one pass. + The encoder preserves the upstream `att_context_size=[70, 13]` (1.12s) + cache-aware attention mask end-to-end. The always-on layer-0 + speaker-kernel FF injection is applied with the single-speaker (all-active) + supervision, matching NeMo's `single_speaker_mode`. +- Streaming (`transcribe_stream_{begin,feed,finalize}` / + `transcribe-cli --stream-chunk-ms N`): incremental PCM feeds drive + per-chunk encoder forward passes with `cache_last_channel` / + `cache_last_time` carried across calls, and an LSTM-state-carrying + RNN-T greedy decoder. All four latency settings from the upstream + multi-lookahead training menu are selectable via + `--stream-att-right R ∈ {0, 1, 6, 13}` (lookahead 0 / 80 / 480 / + 1040 ms respectively). Default = `13` (max accuracy). + +See NVIDIA's [model card](https://huggingface.co/nvidia/multitalker-parakeet-streaming-0.6b-v1) +for training data, intended use, the multitalker methodology, and the full +latency-vs-accuracy table. + +Licensed under the [NVIDIA Open Model License](https://www.nvidia.com/en-us/agreements/enterprise-software/nvidia-open-model-license/). +Ported from upstream commit +[`8749fc7`](https://huggingface.co/nvidia/multitalker-parakeet-streaming-0.6b-v1/commit/8749fc71fd6e2d88ef230159bbf2aea69b524ee1), +pinned 2026-07-12. + +## Download + +| Quantization | Download | Size | WER (LibriSpeech test-clean, offline) | +| --- | --- | ---: | ---: | +| F32 | [multitalker-parakeet-streaming-0.6b-v1-F32.gguf](https://huggingface.co/handy-computer/multitalker-parakeet-streaming-0.6b-v1-gguf/resolve/main/multitalker-parakeet-streaming-0.6b-v1-F32.gguf) | 2.49 GB | 2.19% | +| F16 | [multitalker-parakeet-streaming-0.6b-v1-F16.gguf](https://huggingface.co/handy-computer/multitalker-parakeet-streaming-0.6b-v1-gguf/resolve/main/multitalker-parakeet-streaming-0.6b-v1-F16.gguf) | 1.25 GB | 2.19% | +| Q8_0 | [multitalker-parakeet-streaming-0.6b-v1-Q8_0.gguf](https://huggingface.co/handy-computer/multitalker-parakeet-streaming-0.6b-v1-gguf/resolve/main/multitalker-parakeet-streaming-0.6b-v1-Q8_0.gguf) | 734 MB | 2.18% | +| Q6_K | [multitalker-parakeet-streaming-0.6b-v1-Q6_K.gguf](https://huggingface.co/handy-computer/multitalker-parakeet-streaming-0.6b-v1-gguf/resolve/main/multitalker-parakeet-streaming-0.6b-v1-Q6_K.gguf) | 604 MB | 2.20% | +| Q5_K_M | [multitalker-parakeet-streaming-0.6b-v1-Q5_K_M.gguf](https://huggingface.co/handy-computer/multitalker-parakeet-streaming-0.6b-v1-gguf/resolve/main/multitalker-parakeet-streaming-0.6b-v1-Q5_K_M.gguf) | 542 MB | 2.18% | +| Q4_K_M | [multitalker-parakeet-streaming-0.6b-v1-Q4_K_M.gguf](https://huggingface.co/handy-computer/multitalker-parakeet-streaming-0.6b-v1-gguf/resolve/main/multitalker-parakeet-streaming-0.6b-v1-Q4_K_M.gguf) | 478 MB | 2.18% | + +WER is measured on the full LibriSpeech test-clean split (2620 utterances) +in `single_speaker_mode` with greedy RNN-T decoding, whisper-normalizer +scoring (PnC-stripped), and no external LM. F32 reference baseline: 2.19%. +The measured NeMo `single_speaker_mode` reference on the same split is +2.19%, and NVIDIA's self-reported number is 2.19% (from the +[HF model card](https://huggingface.co/nvidia/multitalker-parakeet-streaming-0.6b-v1)), +so every quant matches the upstream reference within rounding. + +## Streaming parity + +Cache-aware streaming was validated tensor-by-tensor against NeMo's +`conformer_stream_step` reference via `scripts/validate_streaming.py`. On +`samples/jfk.wav` at `--backend cpu --threads 1`, the always-on layer-0 +speaker-kernel injection is applied per chunk on the post-drop block-0 +input (matching the offline path), and the harness reports **150/150 +streaming-tensor pairs within tolerance and the transcript byte-exact vs +NeMo at R=13** (`att_context_size=[70, 13]`, 1.12s chunk). Streaming +`enc_out` drift (observed 5.5e-6) is tighter than the offline C++ +`enc.final` drift (1.9e-3) on the same audio, because clean per-chunk +caches accumulate less error than a single 24-block offline pass. + +Reproduce: + +```bash +uv run --project scripts/envs/parakeet scripts/validate_streaming.py \ + --hf-model nvidia/multitalker-parakeet-streaming-0.6b-v1 \ + --gguf models/multitalker-parakeet-streaming-0.6b-v1/multitalker-parakeet-streaming-0.6b-v1-F32.gguf \ + --audio samples/jfk.wav \ + --out build/validate_streaming/multitalker/jfk \ + --right 13 6 1 0 \ + --backend cpu --threads 1 \ + --tolerances tests/tolerances/multitalker-parakeet-streaming-0.6b-v1.streaming.json +``` + +## Quick Start + +```bash +cmake -B build +cmake --build build + +build/bin/transcribe-cli \ + -m models/multitalker-parakeet-streaming-0.6b-v1/multitalker-parakeet-streaming-0.6b-v1-Q8_0.gguf \ + samples/jfk.wav +``` + +If your audio is not already 16 kHz mono WAV, convert it first: + +```bash +ffmpeg -i input.mp3 -ar 16000 -ac 1 output.wav +``` + +## Performance + +_Publication benchmarks are pending; they are collected on reference +machines and will be filled here once available._ Reproduce with: + +```bash +uv run scripts/bench/run.py \ + --models multitalker-parakeet-streaming-0.6b-v1 \ + --quants q8_0,q4_k_m \ + --samples jfk,dots \ + --backends metal,cpu,vulkan \ + --iters 3 --warmup 1 \ + --name multitalker-parakeet-streaming-0.6b-v1-publication +``` + +## Numerical Validation + +transcribe.cpp is validated tensor-by-tensor against NeMo on +`samples/jfk.wav` via `scripts/validate.py`. Per-tensor tolerances live +in a per-variant file +([`tests/tolerances/multitalker-parakeet-streaming-0.6b-v1.json`](../../tests/tolerances/multitalker-parakeet-streaming-0.6b-v1.json)) +rather than the family-shared one because the unnormalised log-mel +(NeMo's `normalize="NA"` no-op) lands on a different magnitude scale than +the per-feature-normalised siblings, and because the layer-0 +speaker-kernel injection is unique to this variant. The family-level +forward map at +[`reports/porting/parakeet/forward-map.md`](../../reports/porting/parakeet/forward-map.md) +documents the per-stage divergence sources. + +| Field | Value | +| --- | --- | +| Reference | NeMo, `nvidia/multitalker-parakeet-streaming-0.6b-v1` (`single_speaker_mode`) | +| Dump script | `scripts/dump_reference_parakeet_nemo.py` | +| Manifest | `tests/golden/parakeet/multitalker-parakeet-streaming-0.6b-v1.manifest.json` | +| Command | `uv run scripts/validate.py all --family parakeet --variant multitalker-parakeet-streaming-0.6b-v1` | + +## Batch + +The model ships an explicit `run_batch()` parallel fast path. Batch output +is WER-neutral: byte-equal to the serial single-stream path at batch sizes +2 / 4 / 8 (golden frozen at +`tests/golden/batch/multitalker-parakeet-streaming-0.6b-v1.cpu.json`), +CPU same-length tensor parity is bit-exact (max_abs 0.0) at batch=4 on +`jfk.wav`, diverse-length flash tensor parity is bit-exact across arbitrary +length mixes, and full test-clean batch-8 WER equals batch-1 (2.19%). + +## Known limitations + +- **Single-speaker only. The multitalker / speaker-attributed ASR path is + not ported.** Upstream, this checkpoint transcribes several overlapping + speakers into per-speaker channels. That path requires (1) an external + streaming speaker-diarization model + ([`nvidia/diar_streaming_sortformer_4spk-v2.1`](https://huggingface.co/nvidia/diar_streaming_sortformer_4spk-v2.1), + a separate Sortformer checkpoint that this repo does not ship), (2) + per-frame speaker-kernel injection driven by that diarizer's supervision, + (3) one encoder+decoder instance per speaker (up to 4), and (4) a + speaker-tagged SegLST output contract. None of these exist in + transcribe.cpp today, so the port runs `single_speaker_mode` and produces + a single flat transcript. It does **not** separate speakers, diarize, or + emit speaker turns. +- **English only.** The model is English-only by training. +- **No translation and no language detection.** + +## Reproduction + +### Convert + +```bash +uv run --project scripts/envs/parakeet \ + scripts/convert-parakeet.py nvidia/multitalker-parakeet-streaming-0.6b-v1 +``` + +### Quantize + +Run `transcribe-quantize` once per target quant. Example for Q8_0; repeat +with `F16`, `Q6_K`, `Q5_K_M`, `Q4_K_M`: + +```bash +build/bin/transcribe-quantize \ + models/multitalker-parakeet-streaming-0.6b-v1/multitalker-parakeet-streaming-0.6b-v1-F32.gguf \ + models/multitalker-parakeet-streaming-0.6b-v1/multitalker-parakeet-streaming-0.6b-v1-Q8_0.gguf \ + --quant Q8_0 +``` + +### Validate + +```bash +uv run scripts/validate.py all --family parakeet --variant multitalker-parakeet-streaming-0.6b-v1 +``` diff --git a/docs/porting/families/parakeet.md b/docs/porting/families/parakeet.md index 6cd42b51..ed2ba96f 100644 --- a/docs/porting/families/parakeet.md +++ b/docs/porting/families/parakeet.md @@ -31,7 +31,12 @@ CI-overlap grounds; see the family WER summary for rationale. - **Cache-aware streaming RNN-T (encoder-transducer)**: `nemotron-speech-streaming-en-0.6b` (English-only), `nemotron-3.5-asr-streaming-0.6b` (multilingual, 40 locales, - language one-hot conditioning) — intake only, Stages 2-8 TODO + language one-hot conditioning) — intake only, Stages 2-8 TODO, + `multitalker-parakeet-streaming-0.6b-v1` (English-only, fine-tuned + from `nemotron-speech-streaming-en-0.6b`; ships the + `single_speaker_mode` ASR path only — the multitalker / + speaker-attribution machinery is out of scope, see Capability + Validation rows) Per-variant intake JSON: `reports/porting/parakeet//intake.json`. diff --git a/scripts/hf_cards/multitalker-parakeet-streaming-0.6b-v1.yaml b/scripts/hf_cards/multitalker-parakeet-streaming-0.6b-v1.yaml new file mode 100644 index 00000000..5bdc26b1 --- /dev/null +++ b/scripts/hf_cards/multitalker-parakeet-streaming-0.6b-v1.yaml @@ -0,0 +1,89 @@ +# Spec for the HF README of handy-computer/multitalker-parakeet-streaming-0.6b-v1-gguf. +# Consumed by scripts/hf_cards/generate.py. + +hf_repo: nvidia/multitalker-parakeet-streaming-0.6b-v1 +target_repo: handy-computer/multitalker-parakeet-streaming-0.6b-v1-gguf +transcribe_docs_url: https://github.com/handy-computer/transcribe.cpp/blob/main/docs/models/multitalker-parakeet-streaming-0.6b-v1.md + +upstream_commit: 8749fc7 +pin_date: 2026-07-12 + +# Validation pin for the most recent upload. Updated on each release; +# older HF revisions carry whatever value was current at their upload time. +validation: + reference: NeMo + commit: c55a09d + date: 2026-07-12 + +# Upstream is the NVIDIA Open Model License (not a standard SPDX id), so +# the HF `license` field gets `other` and `license_name` / `license_link` +# carry the actual identifier + URL into the YAML frontmatter (HF prefers +# this when license=other). `license_display` is the human-facing form +# rendered in the README body. +license: other +license_name: nvidia-open-model-license +license_link: https://www.nvidia.com/en-us/agreements/enterprise-software/nvidia-open-model-license/ +license_display: NVIDIA Open Model License +pipeline_tag: automatic-speech-recognition +languages: + - en +tags: + - gguf + - transcribe.cpp + - asr + - speech-to-text + - parakeet + - conformer + - rnnt + - streaming + - cache-aware + +summary: | + Offline and cache-aware streaming English speech-to-text with punctuation and capitalization. A 0.6B-parameter cache-aware streaming FastConformer encoder with an RNN-T transducer decoder, fine-tuned from nvidia/nemotron-speech-streaming-en-0.6b. Upstream this is a multitalker (speaker-attributed) checkpoint, but transcribe.cpp ships only its single_speaker_mode ASR path: the multitalker machinery (external Sortformer diarization, per-speaker instances, speaker-tagged SegLST output) is not ported, so this port produces a single flat transcript and does not separate speakers. The encoder preserves the upstream att_context_size=[70, 13] (1.12s) cache-aware attention mask; all four latency lookahead settings are selectable. + +default_quant_index: 2 # Q8_0 + +# Capability flags for the transcribe_cpp metadata block. +capabilities: + streaming: true + translate: false + lang_detect: false + timestamps: token # none | segment | word | token + +# NOTE: `perf` is intentionally omitted until publication benchmarks are +# collected on the reference machines. Without `perf`, generate.py skips the +# machine-readable `transcribe_cpp:` metadata block (RTF + WER + capability +# flags); the human-readable Download/WER table below still renders. Add a +# `perf:` block (see nemotron-speech-streaming-en-0.6b.yaml) and re-render +# once the bench numbers land. + +wer: + source: LibriSpeech test-clean, offline + notes: | + WER measured on the full LibriSpeech test-clean split (2620 utterances) in single_speaker_mode with greedy RNN-T decoding and whisper-normalizer (PnC-stripped) scoring. F32 reference baseline: 2.19%. The measured NeMo single_speaker_mode reference and NVIDIA's self-reported number on the same split are both 2.19%. The multitalker / speaker-attribution path is not ported; see the transcribe.cpp model page for details. + +quants: + - name: F32 + filename: multitalker-parakeet-streaming-0.6b-v1-F32.gguf + size: 2.49 GB + wer: 2.19% + - name: F16 + filename: multitalker-parakeet-streaming-0.6b-v1-F16.gguf + size: 1.25 GB + wer: 2.19% + - name: Q8_0 + filename: multitalker-parakeet-streaming-0.6b-v1-Q8_0.gguf + size: 734 MB + wer: 2.18% + - name: Q6_K + filename: multitalker-parakeet-streaming-0.6b-v1-Q6_K.gguf + size: 604 MB + wer: 2.20% + - name: Q5_K_M + filename: multitalker-parakeet-streaming-0.6b-v1-Q5_K_M.gguf + size: 542 MB + wer: 2.18% + - name: Q4_K_M + filename: multitalker-parakeet-streaming-0.6b-v1-Q4_K_M.gguf + size: 478 MB + wer: 2.18% From dff49ec3b08bd6a34b06fa992ac66b553f1ba699 Mon Sep 17 00:00:00 2001 From: CJ Pais Date: Mon, 13 Jul 2026 11:58:45 +0800 Subject: [PATCH 06/11] 4750u --- .../multitalker-parakeet-streaming-0.6b-v1.md | 19 +++++++++++++++++-- ...ultitalker-parakeet-streaming-0.6b-v1.yaml | 14 ++++++++------ 2 files changed, 25 insertions(+), 8 deletions(-) diff --git a/docs/models/multitalker-parakeet-streaming-0.6b-v1.md b/docs/models/multitalker-parakeet-streaming-0.6b-v1.md index 7bb38f66..a59f728f 100644 --- a/docs/models/multitalker-parakeet-streaming-0.6b-v1.md +++ b/docs/models/multitalker-parakeet-streaming-0.6b-v1.md @@ -111,8 +111,23 @@ ffmpeg -i input.mp3 -ar 16000 -ac 1 output.wav ## Performance -_Publication benchmarks are pending; they are collected on reference -machines and will be filled here once available._ Reproduce with: +Cells are wall-clock latency (mean over 3 iterations after 1 warmup), +with speedup over realtime in parentheses. Units: `ms` below 1 s, `s` +above (2 decimal places). + +### AMD Ryzen 7 4750U Pro + +| Backend | Sample | Q8_0 | Q4_K_M | +| ------- | ------------ | ------------: | ------------: | +| Vulkan | jfk (11.0s) | 466 ms (24×) | 475 ms (23×) | +| Vulkan | dots (35.3s) | 1.36 s (26×) | 1.39 s (26×) | +| CPU | jfk (11.0s) | 751 ms (15×) | 816 ms (13×) | +| CPU | dots (35.3s) | 2.99 s (12×) | 3.12 s (11×) | + +Fedora 43, transcribe.cpp `c55a09d`. Vulkan device: `AMD Radeon +Graphics (RADV RENOIR)`. + +Benchmark reproduction: ```bash uv run scripts/bench/run.py \ diff --git a/scripts/hf_cards/multitalker-parakeet-streaming-0.6b-v1.yaml b/scripts/hf_cards/multitalker-parakeet-streaming-0.6b-v1.yaml index 5bdc26b1..c8d4d82a 100644 --- a/scripts/hf_cards/multitalker-parakeet-streaming-0.6b-v1.yaml +++ b/scripts/hf_cards/multitalker-parakeet-streaming-0.6b-v1.yaml @@ -50,12 +50,14 @@ capabilities: lang_detect: false timestamps: token # none | segment | word | token -# NOTE: `perf` is intentionally omitted until publication benchmarks are -# collected on the reference machines. Without `perf`, generate.py skips the -# machine-readable `transcribe_cpp:` metadata block (RTF + WER + capability -# flags); the human-readable Download/WER table below still renders. Add a -# `perf:` block (see nemotron-speech-streaming-en-0.6b.yaml) and re-render -# once the bench numbers land. +# Speedup-over-realtime (xRT) per rig/backend, from docs/models; published +# raw as rtf_ in the metadata block. Values average the Q8_0 jfk and +# dots publication cells. Only the 4750U reference machine has been benched +# so far (no Apple/Metal run yet). +perf: + ryzen-4750u: + cpu: 13 + vulkan: 25 wer: source: LibriSpeech test-clean, offline From bbd245194964d05b753ba570f6e64b3ceef9c00f Mon Sep 17 00:00:00 2001 From: CJ Pais Date: Sun, 12 Jul 2026 21:21:03 -0700 Subject: [PATCH 07/11] format --- src/arch/parakeet/weights.h | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/src/arch/parakeet/weights.h b/src/arch/parakeet/weights.h index a1da539f..5c12c632 100644 --- a/src/arch/parakeet/weights.h +++ b/src/arch/parakeet/weights.h @@ -228,8 +228,8 @@ struct ParakeetHParams { // spk_kernel_layers=[0]); NeMo's idx>0 post-hook path is rejected at load. // Today: multitalker-parakeet-streaming-0.6b-v1. bool has_spk_kernel = false; - std::string spk_kernel_type; // "ff" - std::vector spk_kernel_layers; // e.g. [0] + std::string spk_kernel_type; // "ff" + std::vector spk_kernel_layers; // e.g. [0] bool add_bg_spk_kernel = false; // Derived. Convenience accessors keyed off the above. @@ -385,20 +385,20 @@ struct ParakeetSpkKernelFF { // One (spk, optional bg) kernel pair bound to an encoder layer index. // Loaded only when hp.has_spk_kernel. struct ParakeetSpkKernel { - int32_t layer = -1; - ParakeetSpkKernelFF spk; // enc.spk_kernel..* + int32_t layer = -1; + ParakeetSpkKernelFF spk; // enc.spk_kernel..* bool has_bg = false; - ParakeetSpkKernelFF bg; // enc.bg_spk_kernel..* (add_bg_spk_kernel) + ParakeetSpkKernelFF bg; // enc.bg_spk_kernel..* (add_bg_spk_kernel) }; struct ParakeetWeights { ParakeetPreEncode pre_encode; - std::vector blocks; // hp.enc_n_layers entries - ParakeetPredictor predictor; // empty when head_kind=CTC - ParakeetJoint joint; // empty when head_kind=CTC - ParakeetCtcHead ctc_head; // populated when head_kind=CTC - ParakeetPromptMlp prompt; // populated when hp.has_prompt - std::vector spk_kernels; // populated when hp.has_spk_kernel + std::vector blocks; // hp.enc_n_layers entries + ParakeetPredictor predictor; // empty when head_kind=CTC + ParakeetJoint joint; // empty when head_kind=CTC + ParakeetCtcHead ctc_head; // populated when head_kind=CTC + ParakeetPromptMlp prompt; // populated when hp.has_prompt + std::vector spk_kernels; // populated when hp.has_spk_kernel }; // Walk the canonical tensor list, look up each tensor by name, validate From 26273f4f30f5b2b86bfb96b93a32c1e9dc5d6637 Mon Sep 17 00:00:00 2001 From: CJ Pais Date: Mon, 13 Jul 2026 12:31:27 +0800 Subject: [PATCH 08/11] merge m4 max --- docs/models/multitalker-parakeet-streaming-0.6b-v1.md | 11 +++++++++++ .../multitalker-parakeet-streaming-0.6b-v1.yaml | 6 ++++-- 2 files changed, 15 insertions(+), 2 deletions(-) diff --git a/docs/models/multitalker-parakeet-streaming-0.6b-v1.md b/docs/models/multitalker-parakeet-streaming-0.6b-v1.md index a59f728f..13db657b 100644 --- a/docs/models/multitalker-parakeet-streaming-0.6b-v1.md +++ b/docs/models/multitalker-parakeet-streaming-0.6b-v1.md @@ -115,6 +115,17 @@ Cells are wall-clock latency (mean over 3 iterations after 1 warmup), with speedup over realtime in parentheses. Units: `ms` below 1 s, `s` above (2 decimal places). +### Apple M4 Max + +| Backend | Sample | Q8_0 | Q4_K_M | +| ------- | ------------ | ------------: | ------------: | +| Metal | jfk (11.0s) | 67 ms (164×) | 69 ms (159×) | +| Metal | dots (35.3s) | 184 ms (192×) | 185 ms (191×) | +| CPU | jfk (11.0s) | 310 ms (36×) | 307 ms (36×) | +| CPU | dots (35.3s) | 1.05 s (34×) | 1.03 s (34×) | + +macOS 26.5.1, transcribe.cpp `c55a09d`. + ### AMD Ryzen 7 4750U Pro | Backend | Sample | Q8_0 | Q4_K_M | diff --git a/scripts/hf_cards/multitalker-parakeet-streaming-0.6b-v1.yaml b/scripts/hf_cards/multitalker-parakeet-streaming-0.6b-v1.yaml index c8d4d82a..fffaed84 100644 --- a/scripts/hf_cards/multitalker-parakeet-streaming-0.6b-v1.yaml +++ b/scripts/hf_cards/multitalker-parakeet-streaming-0.6b-v1.yaml @@ -52,9 +52,11 @@ capabilities: # Speedup-over-realtime (xRT) per rig/backend, from docs/models; published # raw as rtf_ in the metadata block. Values average the Q8_0 jfk and -# dots publication cells. Only the 4750U reference machine has been benched -# so far (no Apple/Metal run yet). +# dots publication cells. perf: + m4-max: + metal: 178.3 + cpu: 34.6 ryzen-4750u: cpu: 13 vulkan: 25 From 8ae1a75baa4b839779b5048bba1f0c32a3a2a025 Mon Sep 17 00:00:00 2001 From: CJ Pais Date: Mon, 13 Jul 2026 14:47:49 +0800 Subject: [PATCH 09/11] clean up comments --- .../multitalker-parakeet-streaming-0.6b-v1.md | 26 ++------- docs/porting/families/parakeet.md | 13 ++--- scripts/convert-parakeet.py | 49 +++------------- src/arch/parakeet/encoder.cpp | 58 +++++-------------- src/arch/parakeet/model.cpp | 22 ++----- src/arch/parakeet/weights.cpp | 17 ++---- src/arch/parakeet/weights.h | 30 ++-------- src/transcribe-batch-util.cpp | 20 ++----- 8 files changed, 54 insertions(+), 181 deletions(-) diff --git a/docs/models/multitalker-parakeet-streaming-0.6b-v1.md b/docs/models/multitalker-parakeet-streaming-0.6b-v1.md index 13db657b..44d1fc0c 100644 --- a/docs/models/multitalker-parakeet-streaming-0.6b-v1.md +++ b/docs/models/multitalker-parakeet-streaming-0.6b-v1.md @@ -16,27 +16,12 @@ transcribe several overlapping speakers into per-speaker channels. That path depends on machinery this port does **not** ship (see [Known limitations](#known-limitations)). transcribe.cpp exposes only the model's `single_speaker_mode` ASR path, which disables the multitalker -machinery and runs the checkpoint as a plain cache-aware streaming RNN-T: -identical frontend, encoder, decoder, and tokenizer as its -`nemotron-speech-streaming-en-0.6b` base. This is the LibriSpeech-comparable -path (upstream reports test-clean 2.19% in this mode). +machinery and runs the checkpoint as a cache-aware streaming RNN-T with +the base model's frontend, encoder backbone, decoder, and tokenizer plus +the checkpoint's always-on layer-0 speaker-kernel injection. This port runs the model in both **offline** and **cache-aware streaming** -modes: - -- Offline (`transcribe_run`): the full audio is consumed in one pass. - The encoder preserves the upstream `att_context_size=[70, 13]` (1.12s) - cache-aware attention mask end-to-end. The always-on layer-0 - speaker-kernel FF injection is applied with the single-speaker (all-active) - supervision, matching NeMo's `single_speaker_mode`. -- Streaming (`transcribe_stream_{begin,feed,finalize}` / - `transcribe-cli --stream-chunk-ms N`): incremental PCM feeds drive - per-chunk encoder forward passes with `cache_last_channel` / - `cache_last_time` carried across calls, and an LSTM-state-carrying - RNN-T greedy decoder. All four latency settings from the upstream - multi-lookahead training menu are selectable via - `--stream-att-right R ∈ {0, 1, 6, 13}` (lookahead 0 / 80 / 480 / - 1040 ms respectively). Default = `13` (max accuracy). +modes. See NVIDIA's [model card](https://huggingface.co/nvidia/multitalker-parakeet-streaming-0.6b-v1) for training data, intended use, the multitalker methodology, and the full @@ -63,8 +48,7 @@ in `single_speaker_mode` with greedy RNN-T decoding, whisper-normalizer scoring (PnC-stripped), and no external LM. F32 reference baseline: 2.19%. The measured NeMo `single_speaker_mode` reference on the same split is 2.19%, and NVIDIA's self-reported number is 2.19% (from the -[HF model card](https://huggingface.co/nvidia/multitalker-parakeet-streaming-0.6b-v1)), -so every quant matches the upstream reference within rounding. +[HF model card](https://huggingface.co/nvidia/multitalker-parakeet-streaming-0.6b-v1)). ## Streaming parity diff --git a/docs/porting/families/parakeet.md b/docs/porting/families/parakeet.md index ed2ba96f..e51497c7 100644 --- a/docs/porting/families/parakeet.md +++ b/docs/porting/families/parakeet.md @@ -157,7 +157,7 @@ Allowed statuses: `PASS` | `SKIP — not exposed by runtime` | | multitalker-parakeet-streaming-0.6b-v1 | Streaming (single_speaker_mode, cache reuse across chunks) | streaming | `build/bin/transcribe-cli -m --language en --backend cpu --threads 1 --stream-chunk-ms 1120 --stream-att-right 13 samples/jfk.wav` | byte-equal transcript vs single-speaker one-shot at the default `att_context_size=[70,13]` (1.12s chunk) | PASS | | multitalker-parakeet-streaming-0.6b-v1 | Other latency settings ([70,0]/[70,1]/[70,6]/[70,13]) | runtime-selectable att_context_size | `… --stream-chunk-ms 1120 --stream-att-right {0,1,6,13} …` | all four R settings stream a valid single-speaker transcript; R=6/13 byte-equal to one-shot, R=0/1 differ only in trailing punctuation (lower lookahead) | PASS | | multitalker-parakeet-streaming-0.6b-v1 | Batch (offline, single_speaker_mode) | run_batch fast path | `uv run scripts/batch_parity.py --model --list --batch-sizes 2,4,8 --backend cpu --language en` + `uv run scripts/batch_tensor_parity.py --model --wav samples/jfk.wav --batch 4 --backend cpu` | text byte-equal vs serial at sizes 2/4/8 (golden frozen at `tests/golden/batch/multitalker-parakeet-streaming-0.6b-v1.cpu.json`); same-length CPU tensor parity bit-exact (max_abs=0.0) at batch=4 on jfk.wav; **diverse-length** flash tensor parity bit-exact (max_abs=0.0) across arbitrary length mixes; full test-clean batch-8 WER == batch-1 (2.19%) after the causal var-len pre-encode masking fix (see forward-map Deviations) | PASS | -| multitalker-parakeet-streaming-0.6b-v1 | Multitalker / speaker-attributed ASR (speaker-kernel injection + external Sortformer diarization + multi-instance + SegLST) | multitalker | (no runtime surface today) | per-speaker cpWER vs NeMo `speech_to_text_multitalker_streaming_infer.py` on AMI/CH109/Mixer6 | SKIP — not exposed by runtime (OUT OF SCOPE) — requires (1) a ported streaming Sortformer diarizer, (2) speaker-kernel injection tensors + layer-0 forward, (3) N-instance-per-speaker orchestration, (4) a SegLST speaker-tagged output contract; none exist in transcribe.cpp today. Brought back in scope by a dedicated multitalker workstream (see the accompanying brief). | +| multitalker-parakeet-streaming-0.6b-v1 | Multitalker / speaker-attributed ASR (speaker-kernel injection + external Sortformer diarization + multi-instance + SegLST) | multitalker | (no runtime surface today) | per-speaker cpWER vs NeMo `speech_to_text_multitalker_streaming_infer.py` on AMI/CH109/Mixer6 | SKIP — not exposed by runtime (OUT OF SCOPE) — the single-speaker kernel path is implemented, but full speaker attribution still requires (1) a ported streaming Sortformer diarizer, (2) target-driven per-frame kernel masks, (3) N-instance-per-speaker orchestration, and (4) a SegLST speaker-tagged output contract. | | multitalker-parakeet-streaming-0.6b-v1 | Speaker diarization (produce speaker turns) | diarization | (not a capability of this checkpoint) | n/a | SKIP — not exposed by runtime (OUT OF SCOPE) — this checkpoint CONSUMES external diarization (Sortformer); it does not produce speaker turns. Brought back in scope only if a Sortformer diarizer is ported as a separate family. | | all variants | Word timestamps | only if exposed | `transcribe-cli --timestamps word -m ` (any variant) | per-word `t0_ms`/`t1_ms` in JSON output | PASS — derived host-side from emit-frame indices (TDT/RNNT) or per-frame argmax (CTC); same code path as the existing v2/v3 word-timestamp gate, no per-variant differences | @@ -224,13 +224,10 @@ be resolved before the corresponding port enters porting-3-convert: Both Linear slots are `[1024,1024]` + `[1024]` bias. The layer-0 injection is marked by three KVs Stage 4 reads: `stt.parakeet.encoder.spk_kernel_type` (`"ff"`), - `stt.parakeet.encoder.spk_kernel_layers` (int array, `[0]`), - `stt.parakeet.encoder.add_bg_spk_kernel` (bool, `true`). The current loader - ignores the extra tensors + KVs and runs this GGUF on the existing - `nemotron-speech-streaming` path; the jfk smoke still decodes correctly - because clean single-speaker greedy decode is robust to the (unapplied) - injection drift — Stage 4 applies it for numerical parity against the - oracle. + `stt.parakeet.encoder.spk_kernel_layers` (int array, `[0]`), and + `stt.parakeet.encoder.add_bg_spk_kernel` (bool, `true`). The loader binds + these tensors and applies the injection in both offline and streaming + encoder graphs. ## Tooling: NeMo-aware preflight diff --git a/scripts/convert-parakeet.py b/scripts/convert-parakeet.py index b0562b42..650a7131 100644 --- a/scripts/convert-parakeet.py +++ b/scripts/convert-parakeet.py @@ -348,23 +348,9 @@ "license_name": "openmdw-1.1", "license_link": "https://openmdw.ai/license/1-1/", }, - # 0.6B English cache-aware streaming MULTI-TALKER RNN-T. Same - # FastConformer streaming encoder as nemotron-speech-streaming-en-0.6b - # (24L / d_model=1024 / 8h / kernel=9 / subsampling=8, - # att_context_style=chunked_limited, layer_norm conv, [70,13] default) - # fine-tuned with a SpeakerKernelMixin (NeMo class - # EncDecMultiTalkerRNNTBPEModel). At each layer in spk_kernel_layers it - # injects a per-speaker FF kernel (`spk_kernels.`) plus a - # background-speaker FF kernel (`bg_spk_kernels.`, add_bg_spk_kernel). - # The injection is a forward_pre_hook that runs UNCONDITIONALLY: even - # single-speaker inference adds spk_kernels.(x) (all-ones speaker - # mask) + bg_spk_kernels.(0) (zeros mask -> constant bias), so these - # tensors are load-bearing for the in-scope single-speaker path, not - # just multi-speaker decoding. v1 port targets single-speaker mode at - # [70,13]; multi-speaker decoding (external Sortformer diarization, - # SegLST speaker-tagged output) is a deferred Stage B workstream. - # spk_kernel_layers present on the profile is the switch that turns on - # kernel-tensor emission + the stt.parakeet.encoder.spk_kernel_* KV. + # Cache-aware streaming RNN-T with always-on speaker-kernel FF + # injection. Full speaker-attributed decoding requires an external + # Sortformer diarizer and is outside this variant's runtime scope. "multitalker-parakeet-streaming-0.6b-v1": { "variant": "multitalker-parakeet-streaming-0.6b-v1", "display_name": "Multitalker Parakeet Streaming EN", @@ -1198,17 +1184,8 @@ def _from_module_or_cfg(attr: str, cfg_key: str, default=None): ] -# Speaker-kernel FF tensors (multitalker-parakeet-streaming-0.6b-v1). -# NeMo's SpeakerKernelMixin attaches, at each layer index in -# spk_kernel_layers, a per-speaker FF kernel `spk_kernels.` and (when -# add_bg_spk_kernel) a background-speaker FF kernel `bg_spk_kernels.`. -# Both are top-level state_dict modules (NOT under `encoder.`) shaped -# nn.Sequential(Linear(d,d), ReLU, Dropout, Linear(d,d)) -- only the .0 -# and .3 Linear slots carry tensors; .1 (ReLU) / .2 (Dropout) are -# parameter-free, mirroring PRE_ENCODE_TABLE's index convention. The gguf -# names take the loader's `enc.` prefix and keep the source layer index so -# the loader can bind per-layer and apply the always-on injection at the -# right block. See src/arch/parakeet family doc open-decision #5. +# Speaker kernels are top-level NeMo Sequential modules. Only the two +# Linear slots (.0 and .3) carry tensors. def spk_kernel_table(layer: int) -> list[tuple[str, str]]: return [ (f"spk_kernels.{layer}.0.weight", f"enc.spk_kernel.{layer}.0.weight"), @@ -1307,9 +1284,7 @@ def convert(model_spec: str, out_path: Path, repo_id: str | None = None) -> None prefer_direct = bool(profile.get("prefer_direct_load", False)) drop_aux_ctc = "tdt_ctc" in slug # hybrid checkpoints carry an aux CTC head we drop has_prompt = bool(profile.get("has_prompt", False)) - # Speaker-kernel FF injection (multitalker variants). spk_kernel_layers - # on the profile is the switch; add_bg_spk_kernel is read from the live - # cfg so it can never disagree with the checkpoint's kernel set. + # The profile opts the variant into speaker-kernel conversion. spk_kernel_layers = [int(l) for l in (profile.get("spk_kernel_layers") or [])] has_spk_kernels = bool(spk_kernel_layers) print(f"Variant: {profile['variant']} (head_kind={head_kind}" @@ -1582,12 +1557,8 @@ def convert(model_spec: str, out_path: Path, repo_id: str | None = None) -> None int(hp["enc_stream_sampling_frames_first"]), ) - # Speaker-kernel FF injection (multitalker variants). These KVs tell - # the loader which encoder layers carry an always-on speaker-kernel - # injection and whether the background kernel is present, so Stage 4 - # can bind enc.spk_kernel.* / enc.bg_spk_kernel.* and replay the - # forward_pre_hook. spk_kernel_type / add_bg_spk_kernel are read from - # the live cfg; spk_kernel_layers comes from the profile switch. + # Kernel type and background presence come from the checkpoint config; + # the profile selects the supported layer set. add_bg_spk_kernel = bool(config.get("add_bg_spk_kernel", False)) if has_spk_kernels: writer.add_string("stt.parakeet.encoder.spk_kernel_type", @@ -1780,9 +1751,7 @@ def add_combined(nemo_a: str, nemo_b: str, gguf_name: str) -> None: for nemo_name, gguf_name in PROMPT_MLP_TABLE: add(nemo_name, gguf_name) - # Speaker-kernel FF tensors (multitalker variants). One (Linear.0, - # Linear.3) pair per layer, plus the background kernel when the - # checkpoint carries it. Emitted verbatim (fp32 passthrough). + # Emit the two Linear slots for each configured speaker kernel. if has_spk_kernels: for layer in spk_kernel_layers: for nemo_name, gguf_name in spk_kernel_table(layer): diff --git a/src/arch/parakeet/encoder.cpp b/src/arch/parakeet/encoder.cpp index 6f69dacd..16db9d62 100644 --- a/src/arch/parakeet/encoder.cpp +++ b/src/arch/parakeet/encoder.cpp @@ -247,10 +247,8 @@ std::vector parse_sub_block_env(int n_layers) { return out; } -// Speaker-kernel FF forward (multitalker variants): returns -// W3·relu(W0·x + b0) + b3. x ne = [d_model, T]; the two Linears keep -// d_model → d_model. Matches NeMo get_spk_kernel_class('ff') at eval -// (Dropout is identity). +// Evaluate W3 * relu(W0 * x + b0) + b3. NeMo's intervening dropout is +// inactive at inference. ggml_tensor * build_spk_kernel_ff(ggml_context * ctx, ggml_tensor * x, const ParakeetSpkKernelFF & ff) { ggml_tensor * h = ggml_mul_mat(ctx, ff.lin0_w, x); h = ggml_add(ctx, h, ff.lin0_b); @@ -260,15 +258,9 @@ ggml_tensor * build_spk_kernel_ff(ggml_context * ctx, ggml_tensor * x, const Par return y; } -// Apply the always-on layer-0 speaker-kernel injection to the block-0 -// input `x` (post pre-encode + xscale). Mirrors -// multitalker_asr_mixins.py::_get_spk_kernel_hook_pre_layer in -// single-speaker mode (spk_targets / bg_spk_targets both None): -// x += spk_kernel(x) (all-ones speaker mask → x) -// x += bg_spk_kernel(0) (all-zeros bg mask → constant) -// The bg input is exactly zeros, so bg_spk_kernel(0) = W3_bg·relu(b0_bg) + -// b3_bg is a per-channel constant [d_model] broadcast over the time axis. -// No-op when hp.has_spk_kernel is false. See family doc open-decision #5. +// Single-speaker mode applies the speaker kernel to x and the background +// kernel to zeros. The latter reduces to a per-channel constant broadcast +// over time. ggml_tensor * apply_spk_kernel_injection(ggml_context * ctx, ggml_tensor * x, const ParakeetWeights & w, @@ -283,9 +275,7 @@ ggml_tensor * apply_spk_kernel_injection(ggml_context * ctx, ggml_tensor * x_spk = build_spk_kernel_ff(ctx, x, kernel.spk); x = ggml_add(ctx, x, x_spk); if (kernel.has_bg) { - // bg input is all-zeros → relu(W0_bg·0 + b0_bg) = relu(b0_bg); - // W3_bg·relu(b0_bg) + b3_bg is a [d_model] constant broadcast - // over the T axis by ggml_add. + // W3_bg * relu(b0_bg) + b3_bg is broadcast over time. ggml_tensor * hb = ggml_relu(ctx, kernel.bg.lin0_b); ggml_tensor * c_bg = ggml_mul_mat(ctx, kernel.bg.lin3_w, hb); c_bg = ggml_add(ctx, c_bg, kernel.bg.lin3_b); @@ -361,17 +351,9 @@ EncoderBuild build_encoder_graph(ggml_context * ctx, // pass reads it (no-op in normal builds). transcribe::debug::mark_tensor_for_dump(eb.mel_in); - // Build the pre_encode subgraph. Variable-length offline batching - // enables masked subsampling (zero each conv intermediate's padded time - // region after every ReLU). Required for BOTH the non-causal and the - // causal (cache-aware streaming) pre-encode: conv0 adds a bias, so the - // zero-padded mel tail relu's to a NON-zero constant, and the causal - // convs' right-context (right = stride-1) then pulls that constant into - // the last valid encoder frame — a large boundary-frame drift vs - // single-shot (which has no padding tail). Masking after each ReLU - // re-zeroes the padded region so the boundary matches single-shot. The - // per-stage valid lengths are computed causal-aware in model.cpp - // (pre_encode_t_out), so the same masks are correct for either padding. + // Mask every subsampling stage in variable-length batches. Conv bias + // otherwise turns the padded mel tail nonzero, which can affect the last + // valid frame of a causal convolution. conf::PreEncodeValidMasks pe_masks; const bool mask_pre_encode = var_len_masks; ggml_tensor * x = conf::build_pre_encode(ctx, to_view(w.pre_encode), eb.mel_in, policy, @@ -404,10 +386,7 @@ EncoderBuild build_encoder_graph(ggml_context * ctx, x = conf::named(x, "enc.pre_encode.xscaled"); } - // ----- speaker-kernel injection (multitalker variants) ----- - // Always-on layer-0 pre-hook: injects the speaker + background FF kernels - // into the block-0 input. NeMo applies it to the post-pos-enc (post - // xscale) audio_signal, exactly here. No-op unless hp.has_spk_kernel. + // NeMo applies the optional layer-0 injection after xscaling. if (hp.has_spk_kernel) { x = apply_spk_kernel_injection(ctx, x, w, hp); x = conf::named(x, "enc.pre_encode.spk_injected"); @@ -486,15 +465,9 @@ EncoderBuild build_encoder_graph(ggml_context * ctx, bparams.n_head = hp.enc_n_heads; bparams.conv_kernel = hp.enc_conv_kernel; bparams.kv_type = kv_type; - // Flash attention by default. Same-length batches are bit-identical to - // single-shot; variable-length batches rely on the attn_pad_mask + - // conv_pad_mask to isolate utterances (a small boundary-frame drift - // remains vs single-shot on the cache-aware streaming chunked path — - // see the parakeet family doc). TRANSCRIBE_NO_FLASH forces the manual - // F32 mul_mat + soft_max path — - // used by the bit-exact CPU tensor gate (flash casts the rel-pos mask - // to F16, manual stays F32); TRANSCRIBE_FORCE_FLASH forces it back on. - // Parakeet has no attention decoder, so the decoder flag is a throwaway. + // Flash attention is the default. TRANSCRIBE_NO_FLASH selects the F32 + // mul_mat + soft_max path used by CPU tensor validation; flash casts + // the relative-position mask to F16. Parakeet has no attention decoder. bool enc_use_flash = true, dec_use_flash = false; transcribe::flash::apply_env_overrides(enc_use_flash, dec_use_flash); bparams.use_flash = enc_use_flash; @@ -748,10 +721,7 @@ EncoderBuild build_encoder_graph_streaming(ggml_context * ctx, x = ggml_cont(ctx, x); } - // ----- speaker-kernel injection (multitalker variants) ----- - // Same always-on layer-0 pre-hook as the offline path, applied to the - // per-chunk block-0 input (post pre-encode + xscale + drop-extra slice). - // No-op unless hp.has_spk_kernel. + // Apply the same optional layer-0 injection to each streaming chunk. if (hp.has_spk_kernel) { x = apply_spk_kernel_injection(ctx, x, w, hp); } diff --git a/src/arch/parakeet/model.cpp b/src/arch/parakeet/model.cpp index bad08ccd..921d23c2 100644 --- a/src/arch/parakeet/model.cpp +++ b/src/arch/parakeet/model.cpp @@ -1211,15 +1211,8 @@ transcribe_status run(transcribe_session * session, // tail can't corrupt real frames and decode each at its own T_enc; // same-length batches skip the masks and are bit-identical to single-shot. static int pre_encode_t_out(int in, bool causal) { - // One stride-2, kernel-3 conv on the time axis. The padding — and hence - // the output length — depends on whether the pre-encode is causal: - // non-causal (offline): symmetric pad-1 -> floor((in + 2 - 3)/2) + 1 = floor((in-1)/2)+1 - // causal (cache-aware streaming): pad (k-1, s-1) = (2, 1) -> floor((in + 3 - 3)/2) + 1 = floor(in/2)+1 - // The single-shot graph applies the correct causal conv, so the batched - // valid-length bookkeeping MUST use the matching formula or it undercounts - // causal variants by one frame per stage (dropping the last encoder frame - // and mis-sizing the pad masks). Mirrors ConvPolicy::causal_pre_encode / - // pre_encode_F_prime in weights.cpp. + // Stride-2, kernel-3 output length. Causal padding (2, 1) produces one + // more frame than symmetric pad-1 for even inputs. return causal ? (in / 2 + 1) : ((in - 1) / 2 + 1); } @@ -1296,10 +1289,7 @@ static transcribe_status run_batch_encode(ParakeetSession * return TRANSCRIBE_ERR_GGUF; } - // Per-utterance valid encoder-frame count (the same subsample the conv - // stack applies: 3 stride-2 convs on the time axis). Causal pre-encode - // (cache-aware streaming variants) uses a different output-length formula - // than the offline non-causal path; see pre_encode_t_out. + // Match each utterance's valid length to the three subsampling convs. const bool causal_pre_encode = (pm->hparams.enc_att_context_style == ParakeetHParams::AttContextStyle::ChunkedLimited); std::vector real_tenc(static_cast(n), T_enc); @@ -1316,10 +1306,8 @@ static transcribe_status run_batch_encode(ParakeetSession * transcribe::fill_keypad_mask(eb.attn_pad_mask_in, real_tenc, T_enc, n); transcribe::fill_valid_frame_mask(eb.conv_pad_mask_in, real_tenc, T_enc, n); - // Pre-encode valid-frame masks (masked subsampling). One per ReLU - // stage; the valid time length at stage k is the per-utterance mel - // length downsampled k times by the (in-1)/2+1 conv formula. Each - // mask is ne=[1, H_stage, 1, n] -> host index b*H_stage + h. + // One valid-frame mask per subsampling ReLU. Each mask is + // ne=[1, H_stage, 1, n], with host index b*H_stage + h. auto fill_pe_mask = [&](ggml_tensor * mask, int n_down) { if (mask == nullptr) { return; diff --git a/src/arch/parakeet/weights.cpp b/src/arch/parakeet/weights.cpp index 4d3e6aa3..9a6e083e 100644 --- a/src/arch/parakeet/weights.cpp +++ b/src/arch/parakeet/weights.cpp @@ -501,9 +501,7 @@ transcribe_status read_parakeet_hparams(const gguf_context * gguf, ParakeetHPara } } - // Speaker-kernel FF injection. Optional (multitalker variants only). - // Presence of stt.parakeet.encoder.spk_kernel_layers is the gate; absent - // leaves has_spk_kernel false and the encoder skips the injection. + // Speaker-kernel metadata is optional; absent means no injection. if (gguf_find_key(gguf, "stt.parakeet.encoder.spk_kernel_layers") >= 0) { switch (read_int32_array_kv(gguf, "stt.parakeet.encoder.spk_kernel_layers", hp.spk_kernel_layers)) { case KvResult::Ok: @@ -523,8 +521,7 @@ transcribe_status read_parakeet_hparams(const gguf_context * gguf, ParakeetHPara st != TRANSCRIBE_OK) { return st; } - // Only the FF kernel (nn.Sequential Linear-ReLU-Dropout-Linear) is - // implemented; NeMo leaves conv2d/mha kernels as TODO. + // NeMo's other advertised kernel types are not implemented. if (hp.spk_kernel_type != "ff") { log_msg(TRANSCRIBE_LOG_LEVEL_ERROR, "parakeet: unsupported spk_kernel_type \"%s\" " @@ -537,10 +534,8 @@ transcribe_status read_parakeet_hparams(const gguf_context * gguf, ParakeetHPara st != TRANSCRIBE_OK) { return st; } - // Only layer-0 pre-hook injection is implemented. NeMo registers a - // forward_pre_hook for idx==0 and a post-hook (after layer idx-1) - // for idx>0; the post path is untested here and this variant ships - // spk_kernel_layers=[0]. + // Layer 0 uses a pre-hook; later layers require a different + // post-hook placement that is not implemented. for (int32_t l : hp.spk_kernel_layers) { if (l != 0) { log_msg(TRANSCRIBE_LOG_LEVEL_ERROR, @@ -921,9 +916,7 @@ transcribe_status build_parakeet_weights(ggml_context * ctx_meta, GET_F32(weights.prompt.mlp2_b, "prompt.mlp.2.bias", d_model); } - // ----- speaker-kernel FF (multitalker variants, has_spk_kernel only) ----- - // Two Linears per kernel (nn.Sequential .0 / .3), both [d_model, d_model] - // + [d_model] bias; the optional bg kernel mirrors the spk kernel. + // ----- optional speaker-kernel FF ----- if (hp.has_spk_kernel) { weights.spk_kernels.clear(); weights.spk_kernels.reserve(hp.spk_kernel_layers.size()); diff --git a/src/arch/parakeet/weights.h b/src/arch/parakeet/weights.h index 5c12c632..6ab4f4cb 100644 --- a/src/arch/parakeet/weights.h +++ b/src/arch/parakeet/weights.h @@ -209,24 +209,9 @@ struct ParakeetHParams { std::vector prompt_dictionary_indices; int32_t prompt_auto_id = -1; - // Speaker-kernel FF injection (multitalker variants, NeMo - // EncDecMultiTalkerRNNTBPEModel / SpeakerKernelMixin). At each layer in - // spk_kernel_layers, an always-on forward_pre_hook injects a per-speaker - // FF kernel plus (when add_bg_spk_kernel) a background FF kernel into the - // block input. The hook fires UNCONDITIONALLY: even single-speaker - // inference (no diarization targets) applies it, so it is load-bearing for - // the single-speaker path, not just multi-speaker decoding. In - // single-speaker mode the layer-0 injection reduces to a deterministic - // transform of the block-0 input (post pre-encode + xscale): - // x += W3·relu(W0·x + b0) + b3 (spk kernel, per frame) - // x += W3_bg·relu(b0_bg) + b3_bg (bg kernel; NeMo masks the - // bg input to all-zeros, so - // its output is a constant - // vector broadcast over time) - // Gated on has_spk_kernel (stt.parakeet.encoder.spk_kernel_layers - // present). Only layer-0 injection is implemented (this variant ships - // spk_kernel_layers=[0]); NeMo's idx>0 post-hook path is rejected at load. - // Today: multitalker-parakeet-streaming-0.6b-v1. + // Optional single-speaker FF injection at the block-0 input. The speaker + // kernel sees x; the background kernel sees zeros and adds a constant. + // Nonzero layer indices are rejected during loading. bool has_spk_kernel = false; std::string spk_kernel_type; // "ff" std::vector spk_kernel_layers; // e.g. [0] @@ -371,10 +356,8 @@ struct ParakeetPromptMlp { ggml_tensor * mlp2_b = nullptr; // [d_model] }; -// Speaker-kernel FF module (multitalker variants). NeMo -// nn.Sequential(Linear.0, ReLU, Dropout, Linear.3); the .1 (ReLU) / .2 -// (Dropout, identity at eval) slots are parameter-free, so only the two -// Linears carry tensors. Both are [d_model, d_model] + [d_model] bias. +// NeMo Sequential(Linear, ReLU, Dropout, Linear); dropout is inactive at +// inference and only the two Linear slots carry tensors. struct ParakeetSpkKernelFF { ggml_tensor * lin0_w = nullptr; // [d_model, d_model] ggml_tensor * lin0_b = nullptr; // [d_model] @@ -382,8 +365,7 @@ struct ParakeetSpkKernelFF { ggml_tensor * lin3_b = nullptr; // [d_model] }; -// One (spk, optional bg) kernel pair bound to an encoder layer index. -// Loaded only when hp.has_spk_kernel. +// Speaker and optional background kernels for one encoder layer. struct ParakeetSpkKernel { int32_t layer = -1; ParakeetSpkKernelFF spk; // enc.spk_kernel..* diff --git a/src/transcribe-batch-util.cpp b/src/transcribe-batch-util.cpp index 947139df..11372a9a 100644 --- a/src/transcribe-batch-util.cpp +++ b/src/transcribe-batch-util.cpp @@ -145,25 +145,15 @@ void fill_keypad_mask(ggml_tensor * mask, const std::vector & real_lens, in if (mask == nullptr) { return; } - // Padded keys get a large FINITE negative, not -inf. A finite sentinel is - // bit-exact with -inf for any query that has at least one non-masked key - // (expf(-1e30 - finite_row_max) underflows to exactly 0.0f, same as - // exp(-inf)), and it casts to -inf in F16 so the flash path is unchanged. - // The reason it must be finite: in a variable-length batch a padded QUERY - // row (a frame past the real length) can have its entire chunked-attention - // window fall on padded keys. With -inf that row is all -inf and the - // manual (non-flash) softmax computes 0/0 = NaN, which then poisons real - // frames at the next layer via 0*NaN (a masked padded key still multiplies - // its NaN value). A finite sentinel makes such a row softmax to a finite - // (near-uniform) value instead, so no NaN is ever created or propagated. - // Flash attention skips masked keys and never hit this; this keeps the - // manual/bit-exact path robust for diverse-length batches too. - const float kMaskNeg = -1e30f; + // Keep fully masked rows finite on the manual softmax path; -inf would + // produce NaNs. For rows with a valid key, exp(-1e30) still underflows to + // zero, and F16 conversion preserves the flash path's -inf sentinel. + const float mask_neg = -1e30f; std::vector buf(static_cast(T) * n); for (int b = 0; b < n; ++b) { const int real = real_lens[static_cast(b)]; for (int k = 0; k < T; ++k) { - buf[static_cast(b) * T + k] = (k < real) ? 0.0f : kMaskNeg; + buf[static_cast(b) * T + k] = (k < real) ? 0.0f : mask_neg; } } ggml_backend_tensor_set(mask, buf.data(), 0, buf.size() * sizeof(float)); From e18e8745a4692e6025f8f97c3f71c6dc24ab9915 Mon Sep 17 00:00:00 2001 From: CJ Pais Date: Mon, 13 Jul 2026 17:08:51 +0800 Subject: [PATCH 10/11] testing code --- scripts/batch_tensor_parity.py | 115 ++++++++++++++++++------------ scripts/wer/remote/modal_sweep.py | 5 +- src/arch/parakeet/model.cpp | 14 ++-- src/arch/parakeet/parakeet.h | 5 ++ src/transcribe-batch-util.h | 8 +-- tests/CMakeLists.txt | 16 +++++ tests/batch_mask_unit.cpp | 109 ++++++++++++++++++++++++++++ 7 files changed, 213 insertions(+), 59 deletions(-) create mode 100644 tests/batch_mask_unit.cpp diff --git a/scripts/batch_tensor_parity.py b/scripts/batch_tensor_parity.py index f6bb22fa..a0abb8bf 100644 --- a/scripts/batch_tensor_parity.py +++ b/scripts/batch_tensor_parity.py @@ -6,21 +6,18 @@ """ batch_tensor_parity.py - numerical gate for the parakeet batched encoder. -Proves that the per-utterance encoder output of transcribe_run_batch is -identical, tensor-for-tensor, to the single-shot encoder output for the same -wav. This is the gate the variable-length padding mask must keep green: a -wrong mask perturbs real frames and shows up here as drift, where WER alone -would miss it. - -Apples-to-apples requires both paths on the SAME attention kernel. The -single-shot path defaults to flash attention (which casts the rel-pos mask to -F16); the batched path always runs the manual F32 attention. We force the -single-shot path to manual with TRANSCRIBE_NO_FLASH=1 so the comparison is -exact on CPU. - -Mechanism (TRANSCRIBE_DUMP_DIR): single-shot dumps `dec.enc_out`; the batched -run dumps `dec.enc_out.b{i}` per utterance. We feed B copies of one clip and -assert every batched slice equals the single-shot dump. +Proves that each per-utterance encoder output from transcribe_run_batch +matches its single-shot output. A list of varied-length WAVs exercises padding +masks and causal subsampling lengths; repeating one WAV covers same-length +batching. + +Apples-to-apples requires both paths on the same attention kernel. Manual F32 +attention remains the default for this tool; --flash validates the production +flash-attention policy separately. + +Mechanism (TRANSCRIBE_DUMP_DIR): each single-shot run dumps `dec.enc_out`; +the batched run dumps `dec.enc_out.b{i}`. The valid slices must have identical +sizes and values. Usage: uv run scripts/batch_tensor_parity.py \\ @@ -53,7 +50,11 @@ def run(cli, model, args, dump_dir, backend, extra_env=None): env = dict(os.environ) env["TRANSCRIBE_DUMP_DIR"] = str(dump_dir) if extra_env: - env.update(extra_env) + for key, value in extra_env.items(): + if value is None: + env.pop(key, None) + else: + env[key] = value cmd = [str(cli), "-m", str(model), "--backend", backend, "-q"] + args r = subprocess.run(cmd, env=env, capture_output=True, text=True) if r.returncode != 0: @@ -69,7 +70,11 @@ def main() -> int: repo = find_repo_root(Path(__file__)) ap = argparse.ArgumentParser() ap.add_argument("--model", type=Path, required=True) - ap.add_argument("--wav", type=Path, default=repo / "samples/jfk.wav") + src = ap.add_mutually_exclusive_group() + src.add_argument("--wav", type=Path, + help="single WAV repeated --batch times (default: samples/jfk.wav)") + src.add_argument("--list", type=Path, + help="varied-length WAV list; its length must equal --batch") ap.add_argument("--batch", type=int, default=4) ap.add_argument("--cli", type=Path, default=repo / "build/bin/transcribe-cli") ap.add_argument("--backend", default="cpu") @@ -80,47 +85,69 @@ def main() -> int: "run writes .b{i} per utterance. parakeet uses " "dec.enc_out; sensevoice (no host decoder) uses " "ctc.log_probs.") - ap.add_argument("--no-flash", dest="no_flash", action="store_true", - default=None, - help="force TRANSCRIBE_NO_FLASH=1 on both paths (parakeet " - "needs this for a bit-exact gate; sensevoice runs flash " - "on both same-length paths so it is not required)") + attn = ap.add_mutually_exclusive_group() + attn.add_argument("--no-flash", action="store_true", + help="force the manual F32 attention path (the default)") + attn.add_argument("--flash", action="store_true", + help="use the default production flash-attention policy") args = ap.parse_args() - for p in (args.model, args.wav, args.cli): + for p in (args.model, args.cli): if not Path(p).exists(): raise SystemExit(f"missing: {p}") - - # parakeet's bit-exact gate needs manual attention on both paths (flash - # casts the rel-pos mask to F16); sensevoice has no rel-pos mask and runs - # flash on both same-length paths, so the env is a harmless no-op there. - no_flash = True if args.no_flash is None else args.no_flash - env = {"TRANSCRIBE_NO_FLASH": "1"} if no_flash else None - - with tempfile.TemporaryDirectory() as td: + if args.batch < 1: + raise SystemExit("--batch must be positive") + + if args.list: + if not args.list.exists(): + raise SystemExit(f"missing: {args.list}") + wavs = [Path(line.strip()) for line in args.list.read_text().splitlines() + if line.strip() and not line.lstrip().startswith("#")] + if len(wavs) != args.batch: + raise SystemExit( + f"--list has {len(wavs)} entries; expected --batch={args.batch}") + else: + wav = args.wav or (repo / "samples/jfk.wav") + wavs = [wav] * args.batch + for wav in wavs: + if not wav.exists(): + raise SystemExit(f"missing: {wav}") + + # Manual attention remains the default for backward compatibility. The + # --flash mode separately validates the production policy. + env = {"TRANSCRIBE_NO_FLASH": None} if args.flash else {"TRANSCRIBE_NO_FLASH": "1"} + + tmp_root = repo / "tmp" + tmp_root.mkdir(parents=True, exist_ok=True) + with tempfile.TemporaryDirectory(dir=tmp_root) as td: td = Path(td) - single_dir = td / "single" batch_dir = td / "batch" - single_dir.mkdir(); batch_dir.mkdir() + batch_dir.mkdir() + + singles = [] + for i, wav in enumerate(wavs): + single_dir = td / f"single-{i}" + single_dir.mkdir() + run(args.cli, args.model, [str(wav)], single_dir, args.backend, + extra_env=env) + single = load(single_dir, args.dump_name) + singles.append(single) + print(f"single {i}: {wav} {single.size} elems " + f"rms={np.sqrt((single**2).mean()):.4e}") - # Single-shot. - run(args.cli, args.model, [str(args.wav)], single_dir, args.backend, - extra_env=env) - - # Batched run of B copies (same length -> the real fused encoder). list_file = td / "list.txt" - list_file.write_text("\n".join([str(args.wav)] * args.batch) + "\n") + list_file.write_text("\n".join(str(wav) for wav in wavs) + "\n") run(args.cli, args.model, ["--batch", str(list_file), "--batch-size", str(args.batch)], batch_dir, args.backend, extra_env=env) - single = load(single_dir, args.dump_name) - print(f"single-shot {args.dump_name}: {single.size} elems " - f"rms={np.sqrt((single**2).mean()):.4e}") - ok = True - for i in range(args.batch): + for i, single in enumerate(singles): b = load(batch_dir, f"{args.dump_name}.b{i}") + if not np.isfinite(b).all(): + print(f" [FAIL] utt {i}: batched dump contains non-finite values") + ok = False + continue if b.size != single.size: print(f" [FAIL] utt {i}: size {b.size} != {single.size}") ok = False diff --git a/scripts/wer/remote/modal_sweep.py b/scripts/wer/remote/modal_sweep.py index 868d549c..503f62ba 100644 --- a/scripts/wer/remote/modal_sweep.py +++ b/scripts/wer/remote/modal_sweep.py @@ -972,6 +972,7 @@ def batch_sweep( n_utts: int = 128, batch_sizes: str = "1,2,4,8,16", sort_by_length: bool = True, + language: str = "", clean: bool = False, ) -> None: """Throughput sweep of transcribe_run_batch across batch sizes for ONE @@ -992,6 +993,7 @@ def batch_sweep( Pass --no-sort-by-length to measure the naive case where the manifest order is arbitrary and each batch pads to its longest clip (shows the cost of NOT bucketing). + --language optional language/locale override for every utterance. --clean force a clean build. modal run scripts/wer/remote/modal_sweep.py::batch_sweep \\ @@ -1022,6 +1024,7 @@ def batch_sweep( print(f" n_utts = {n_utts}") print(f" batches = {sizes}") print(f" sort = {'length-sorted' if sort_by_length else 'NAIVE (unsorted)'}") + print(f" language = {language or 'manifest/default'}") print("============================================================") print(f">>> build (arch sm_{arch} -> {build_dir})") @@ -1036,7 +1039,7 @@ def batch_sweep( # Launch all batch sizes in parallel (each its own container). Pass the # locally-computed build_dir so the runner reads exactly what build() wrote. futs = [(bs, runner.spawn(repo, model_file, dataset, n, bs, sort_by_length, - build_dir, "none", 0, dataset_status)) + build_dir, "none", language, 0, -1, dataset_status)) for bs in sizes] rows: list[tuple] = [] diff --git a/src/arch/parakeet/model.cpp b/src/arch/parakeet/model.cpp index 921d23c2..735f84ae 100644 --- a/src/arch/parakeet/model.cpp +++ b/src/arch/parakeet/model.cpp @@ -1210,12 +1210,6 @@ transcribe_status run(transcribe_session * session, // batches build attn key-padding + conv valid-frame masks so a padded // tail can't corrupt real frames and decode each at its own T_enc; // same-length batches skip the masks and are bit-identical to single-shot. -static int pre_encode_t_out(int in, bool causal) { - // Stride-2, kernel-3 output length. Causal padding (2, 1) produces one - // more frame than symmetric pad-1 for even inputs. - return causal ? (in / 2 + 1) : ((in - 1) / 2 + 1); -} - static transcribe_status run_batch_encode(ParakeetSession * pc, ParakeetModel * pm, const std::vector> & mels, @@ -1296,9 +1290,9 @@ static transcribe_status run_batch_encode(ParakeetSession * if (var_len) { for (int b = 0; b < n; ++b) { int t = nf[b]; - t = pre_encode_t_out(t, causal_pre_encode); - t = pre_encode_t_out(t, causal_pre_encode); - t = pre_encode_t_out(t, causal_pre_encode); + t = pre_encode_time_out(t, causal_pre_encode); + t = pre_encode_time_out(t, causal_pre_encode); + t = pre_encode_time_out(t, causal_pre_encode); real_tenc[b] = std::min(t, T_enc); } // Attention key-padding mask [T_enc, 1, 1, n] (0 real / -INF padded) @@ -1317,7 +1311,7 @@ static transcribe_status run_batch_encode(ParakeetSession * for (int b = 0; b < n; ++b) { int v = nf[b]; for (int d = 0; d < n_down; ++d) { - v = pre_encode_t_out(v, causal_pre_encode); + v = pre_encode_time_out(v, causal_pre_encode); } if (v > H) { v = H; diff --git a/src/arch/parakeet/parakeet.h b/src/arch/parakeet/parakeet.h index 9c4b1fff..0574e544 100644 --- a/src/arch/parakeet/parakeet.h +++ b/src/arch/parakeet/parakeet.h @@ -31,6 +31,11 @@ typedef struct ggml_backend_sched * ggml_backend_sched_t; namespace transcribe::parakeet { +// Output length of one stride-2, kernel-3 pre-encode convolution. +inline int pre_encode_time_out(int input_frames, bool causal) { + return causal ? (input_frames / 2 + 1) : ((input_frames - 1) / 2 + 1); +} + // Host-side attention-mask helpers for the streaming paths. Declared // here so the per-mask unit tests can call them without a ggml backend // or a GGUF on disk. diff --git a/src/transcribe-batch-util.h b/src/transcribe-batch-util.h index c5379965..9be59eed 100644 --- a/src/transcribe-batch-util.h +++ b/src/transcribe-batch-util.h @@ -75,10 +75,10 @@ void pack_pad_channel_major(std::vector & dst, int n_ch, int T_max); -// Fill an attention key-padding mask (f32) of logical shape [T, .., n] with the -// host ordering index b*T + k: 0.0 on real keys (k < real_lens[b]), -inf on -// padded keys. No-op when `mask` is null. The element order is identical for -// the conformer [T,1,1,n] and SAN-M [T,1,1,n] key-padding tensors. +// Fill an attention key-padding mask (f32) of logical shape [T, .., n] with +// host ordering index b*T + k: 0.0 on real keys and a large finite negative on +// padded keys. The finite sentinel prevents NaNs on fully masked manual-softmax +// rows. No-op when `mask` is null. void fill_keypad_mask(ggml_tensor * mask, const std::vector & real_lens, int T, int n); // Fill a conv valid-frame mask (f32) with host ordering index b*T + t: 1.0 on diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index b35bb291..b36c8e15 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -149,6 +149,22 @@ transcribe_apply_warnings(transcribe_parakeet_chunked_limited_with_rc_mask_unit) add_test(NAME transcribe_parakeet_chunked_limited_with_rc_mask_unit COMMAND transcribe_parakeet_chunked_limited_with_rc_mask_unit) +# ----------------------------------------------------------------------------- +# Variable-length batch mask and causal subsampling unit test +# ----------------------------------------------------------------------------- + +add_executable(transcribe_batch_mask_unit + batch_mask_unit.cpp) + +target_link_libraries(transcribe_batch_mask_unit PRIVATE transcribe ggml) + +target_include_directories(transcribe_batch_mask_unit PRIVATE + ${CMAKE_SOURCE_DIR}/src) + +transcribe_apply_warnings(transcribe_batch_mask_unit) + +add_test(NAME transcribe_batch_mask_unit COMMAND transcribe_batch_mask_unit) + # ----------------------------------------------------------------------------- # F16 → F32 conv pointwise promotion unit test # ----------------------------------------------------------------------------- diff --git a/tests/batch_mask_unit.cpp b/tests/batch_mask_unit.cpp new file mode 100644 index 00000000..ae97d7bd --- /dev/null +++ b/tests/batch_mask_unit.cpp @@ -0,0 +1,109 @@ +#include "arch/parakeet/parakeet.h" +#include "ggml-backend.h" +#include "ggml.h" +#include "transcribe-batch-util.h" + +#include +#include +#include +#include + +namespace { + +int g_failures = 0; + +#define CHECK(cond) \ + do { \ + if (!(cond)) { \ + std::fprintf(stderr, "FAIL %s:%d: %s\n", __FILE__, __LINE__, #cond); \ + ++g_failures; \ + } \ + } while (0) + +} // namespace + +int main() { + using transcribe::parakeet::pre_encode_time_out; + + // Symmetric and causal padding agree for odd inputs. Causal padding keeps + // one additional frame for even inputs, including at every x8 stage. + CHECK(pre_encode_time_out(5, false) == 3); + CHECK(pre_encode_time_out(5, true) == 3); + CHECK(pre_encode_time_out(6, false) == 3); + CHECK(pre_encode_time_out(6, true) == 4); + int regular = 100; + int causal = 100; + for (int i = 0; i < 3; ++i) { + regular = pre_encode_time_out(regular, false); + causal = pre_encode_time_out(causal, true); + } + CHECK(regular == 13); + CHECK(causal == 14); + + ggml_backend_t backend = ggml_backend_init_by_type(GGML_BACKEND_DEVICE_TYPE_CPU, nullptr); + if (backend == nullptr) { + std::fprintf(stderr, "SKIP: could not initialize CPU backend\n"); + return 77; + } + + ggml_init_params params{}; + params.mem_size = 1024 * 1024; + params.no_alloc = true; + ggml_context * ctx = ggml_init(params); + if (ctx == nullptr) { + ggml_backend_free(backend); + return EXIT_FAILURE; + } + + constexpr int T = 4; + constexpr int B = 2; + ggml_tensor * mask = ggml_new_tensor_2d(ctx, GGML_TYPE_F32, T, B); + ggml_tensor * softmax = ggml_soft_max(ctx, mask); + ggml_cgraph * graph = ggml_new_graph(ctx); + ggml_build_forward_expand(graph, softmax); + + ggml_backend_buffer_t buffer = ggml_backend_alloc_ctx_tensors(ctx, backend); + if (buffer == nullptr) { + ggml_free(ctx); + ggml_backend_free(backend); + return EXIT_FAILURE; + } + + // Row 0 has two valid keys. Row 1 models a padded query whose complete + // local-attention window falls on padded keys. + transcribe::fill_keypad_mask(mask, { 2, 0 }, T, B); + + std::vector mask_values(T * B); + ggml_backend_tensor_get(mask, mask_values.data(), 0, mask_values.size() * sizeof(float)); + CHECK(mask_values[0] == 0.0f); + CHECK(mask_values[1] == 0.0f); + for (int i = 2; i < T * B; ++i) { + CHECK(std::isfinite(mask_values[static_cast(i)])); + CHECK(mask_values[static_cast(i)] < -1e20f); + } + + CHECK(ggml_backend_graph_compute(backend, graph) == GGML_STATUS_SUCCESS); + std::vector probabilities(T * B); + ggml_backend_tensor_get(softmax, probabilities.data(), 0, probabilities.size() * sizeof(float)); + for (float value : probabilities) { + CHECK(std::isfinite(value)); + } + CHECK(std::fabs(probabilities[0] - 0.5f) < 1e-6f); + CHECK(std::fabs(probabilities[1] - 0.5f) < 1e-6f); + CHECK(probabilities[2] == 0.0f); + CHECK(probabilities[3] == 0.0f); + for (int i = T; i < T * B; ++i) { + CHECK(std::fabs(probabilities[static_cast(i)] - 0.25f) < 1e-6f); + } + + ggml_backend_buffer_free(buffer); + ggml_free(ctx); + ggml_backend_free(backend); + + if (g_failures != 0) { + std::fprintf(stderr, "batch_mask_unit: %d failures\n", g_failures); + return EXIT_FAILURE; + } + std::fprintf(stdout, "batch_mask_unit: ok\n"); + return EXIT_SUCCESS; +} From 9721b9b8640b8b47e4d876a999a51ff5a653f0b2 Mon Sep 17 00:00:00 2001 From: CJ Pais Date: Mon, 13 Jul 2026 17:13:40 +0800 Subject: [PATCH 11/11] format --- tests/batch_mask_unit.cpp | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/tests/batch_mask_unit.cpp b/tests/batch_mask_unit.cpp index ae97d7bd..80a92a86 100644 --- a/tests/batch_mask_unit.cpp +++ b/tests/batch_mask_unit.cpp @@ -47,19 +47,19 @@ int main() { } ggml_init_params params{}; - params.mem_size = 1024 * 1024; - params.no_alloc = true; + params.mem_size = 1024 * 1024; + params.no_alloc = true; ggml_context * ctx = ggml_init(params); if (ctx == nullptr) { ggml_backend_free(backend); return EXIT_FAILURE; } - constexpr int T = 4; - constexpr int B = 2; + constexpr int T = 4; + constexpr int B = 2; ggml_tensor * mask = ggml_new_tensor_2d(ctx, GGML_TYPE_F32, T, B); ggml_tensor * softmax = ggml_soft_max(ctx, mask); - ggml_cgraph * graph = ggml_new_graph(ctx); + ggml_cgraph * graph = ggml_new_graph(ctx); ggml_build_forward_expand(graph, softmax); ggml_backend_buffer_t buffer = ggml_backend_alloc_ctx_tensors(ctx, backend);