fix(helper): widen NVDEC pool to 32 + retry truncated video via software

NVENC's NVDEC reference-pool starvation surfaces a second failure mode
beyond the cold-start one the seg-0 software-decode path already
handles: at certain content offsets inside HEVC sources (Tubi catalog
at the 16-24 s mark in observed sessions), the decoder runs out of
reference frames mid-segment and the encoder drops most video frames.
The TS file passes the sync-byte gate and the PES-presence gate
because PAT/PMT and audio land normally, but ffprobe reports the
video stream as 0.7-1.0 s of duration against a 4 s window. The
player either freezes when the video stream ends or desyncs audio.

Two changes:

1. Bump -extra_hw_frames from 16 to 32 on the NVENC backend. 16 was a
   first guess and works for shallow reference chains but Tubi content
   at certain time offsets needs more. 32 covers the deepest chains
   observed in the wild without meaningfully growing GPU memory
   (each surface is ~1 MB at 480p; the total pool is bounded).

2. Post-encode video-duration check in HelperLeaseWorker.RunAsync.
   After ValidateMpegTs passes, on the hardware path only, run
   ProbeVideoDurationAsync against the output and compare to the
   lease's expected duration. If the video stream came back at < 80 %
   of expected, retry the encode via BuildSegmentCommandSoftwareFallback
   (the same path seg 0 uses) before uploading. Catches the truncated
   output locally so a known-bad segment never reaches the server.

Author: RealWhyKnot

src/WKVRCProxy/Helper/HelperLeaseWorker.cs

@@ -183,6 +183,68 @@ public static async Task<HelperLeaseRunResult> RunAsync(
                 return Result(false, "local_validation_failed", validationError);
             }
 
+            // Post-encode video-duration check. NVDEC reference-pool starvation
+            // on HEVC sources produces a TS that's structurally valid -- sync
+            // bytes plus PES packets plus full-length audio -- but where the
+            // encoder dropped most of the video frames mid-segment, leaving
+            // ~0.7-1.0 s of video out of the 4 s window. Players that try to
+            // decode it freeze when the video stream ends mid-segment. The
+            // local validator can't catch this without probing duration, and
+            // the server has no way to retry on the same helper. If we're on
+            // the hardware path AND the video came back short, fall back to
+            // software decode (same path seg 0 uses) before uploading.
+            if (!softwareDecodeFirst && leaseFrame.Duration > 0)
+            {
+                double? vidDuration = await ProbeVideoDurationAsync(ffmpegLocation.Path, outputPath, deadlineCts.Token).ConfigureAwait(false);
+                if (vidDuration.HasValue && vidDuration.Value < leaseFrame.Duration * 0.8)
+                {
+                    WarnLease(leaseFrame, "truncated_video", "video_dur="
+                        + vidDuration.Value.ToString("0.###", System.Globalization.CultureInfo.InvariantCulture)
+                        + " expected=" + leaseFrame.Duration.ToString("0.###", System.Globalization.CultureInfo.InvariantCulture)
+                        + " bytes=" + info.Length + " retry=software");
+                    TryDeleteOutput(outputPath);
+
+                    TranscodeFfmpegCommand swCommand = TranscodeWorkerProcess.BuildSegmentCommandSoftwareFallback(
+                        ffmpegLocation.Path,
+                        lease,
+                        encoder,
+                        outputPath,
+                        leaseFrame.TargetWidth,
+                        leaseFrame.TargetHeight,
+                        leaseFrame.TargetBitrateKbps,
+                        hasAudio: leaseFrame.HasAudio,
+                        quality: quality);
+
+                    LogLease(leaseFrame, "ffmpeg sw-retry start", "reason=truncated_video deadline_ms=" + deadlineMs);
+                    ffmpegResult = await RunFfmpegAsync(swCommand, deadlineCts.Token, ct).ConfigureAwait(false);
+                    if (ffmpegResult.TimedOut)
+                    {
+                        WarnLease(leaseFrame, "deadline", "sw-retry ffmpeg exceeded "
+                            + deadlineMs + "ms stderr=" + Snip(ffmpegResult.Stderr, 180));
+                        return Result(false, "deadline", "helper sw-retry exceeded the server deadline");
+                    }
+                    if (ffmpegResult.ExitCode != 0)
+                    {
+                        WarnLease(leaseFrame, "ffmpeg sw-retry failed", "exit=" + ffmpegResult.ExitCode
+                            + " stderr=" + Snip(ffmpegResult.Stderr, 180));
+                        return Result(false, "ffmpeg_failed", Snip(ffmpegResult.Stderr, 240));
+                    }
+                    info = new FileInfo(outputPath);
+                    if (!info.Exists || info.Length <= 0)
+                    {
+                        WarnLease(leaseFrame, "empty sw-retry output", "ffmpeg sw-retry completed without a segment");
+                        return Result(false, "empty_output", "ffmpeg sw-retry completed without a segment");
+                    }
+                    validationError = ValidateMpegTs(outputPath, info.Length);
+                    if (validationError != null)
+                    {
+                        WarnLease(leaseFrame, "sw-retry local_validation_failed", validationError + " bytes=" + info.Length);
+                        return Result(false, "local_validation_failed", validationError);
+                    }
+                    LogLease(leaseFrame, "ffmpeg sw-retry ok", "bytes=" + info.Length);
+                }
+            }
+
             string uploadUrlHost = ExtractUrlHost(leaseFrame.UploadUrl);
             LogLease(leaseFrame, "upload start", "upload_url_host=" + uploadUrlHost
                 + " bytes=" + info.Length);
@@ -279,6 +341,77 @@ private static HelperRuntimeSignals DefaultSignals()
 
     private sealed record FfmpegProcessResult(bool TimedOut, int ExitCode, string Stderr);
 
+    // Probe the video stream's duration via ffprobe. Returns null when the
+    // probe can't be run, the binary is missing, or the file has no video
+    // stream. Best-effort: any failure short-circuits to null so the caller
+    // can decide what to do (the normal case is "skip the truncated-video
+    // retry and proceed to upload"; the validator on the server will catch
+    // the file if it's structurally bad).
+    private static async Task<double?> ProbeVideoDurationAsync(
+        string ffmpegPath,
+        string outputPath,
+        CancellationToken ct)
+    {
+        string? ffprobePath = TryResolveFfprobePath(ffmpegPath);
+        if (string.IsNullOrEmpty(ffprobePath)) return null;
+
+        var psi = new ProcessStartInfo
+        {
+            FileName = ffprobePath,
+            UseShellExecute = false,
+            RedirectStandardOutput = true,
+            RedirectStandardError = true,
+            CreateNoWindow = true,
+        };
+        psi.ArgumentList.Add("-v");
+        psi.ArgumentList.Add("error");
+        psi.ArgumentList.Add("-show_entries");
+        psi.ArgumentList.Add("stream=duration");
+        psi.ArgumentList.Add("-select_streams");
+        psi.ArgumentList.Add("v:0");
+        psi.ArgumentList.Add("-of");
+        psi.ArgumentList.Add("csv=p=0");
+        psi.ArgumentList.Add(outputPath);
+
+        try
+        {
+            using var p = new Process { StartInfo = psi };
+            p.Start();
+            var stdoutTask = p.StandardOutput.ReadToEndAsync(ct);
+            _ = p.StandardError.ReadToEndAsync(ct);
+            await p.WaitForExitAsync(ct).ConfigureAwait(false);
+            if (p.ExitCode != 0) return null;
+            string stdout = (await stdoutTask.ConfigureAwait(false)).Trim();
+            if (string.IsNullOrEmpty(stdout)) return null;
+            if (double.TryParse(stdout, System.Globalization.NumberStyles.Float,
+                System.Globalization.CultureInfo.InvariantCulture, out double secs)
+                && secs > 0)
+            {
+                return secs;
+            }
+            return null;
+        }
+        catch
+        {
+            return null;
+        }
+    }
+
+    private static string? TryResolveFfprobePath(string ffmpegPath)
+    {
+        try
+        {
+            string? dir = Path.GetDirectoryName(ffmpegPath);
+            if (string.IsNullOrEmpty(dir)) return null;
+            string probe = Path.Combine(dir, OperatingSystem.IsWindows() ? "ffprobe.exe" : "ffprobe");
+            return File.Exists(probe) ? probe : null;
+        }
+        catch
+        {
+            return null;
+        }
+    }
+
     private static async Task<FfmpegProcessResult> RunFfmpegAsync(
         TranscodeFfmpegCommand command,
         CancellationToken deadlineToken,

src/WKVRCProxy/Helper/TranscodeWorkerProcess.cs

@@ -249,8 +249,12 @@ private static IReadOnlyList<string> HardwareDecodeOptionsFor(HardwareEncoderCap
             // (Tubi's catalog among them) use longer reference chains than the cuvid
             // default 25 covers and surface as "Could not find ref with POC N" decoder
             // warnings followed by a black/short segment. Bumping the pool stops the
-            // decoder from dropping reference frames it still needs.
-            HardwareEncoderBackend.Nvenc => new[] { "-hwaccel", "cuda", "-hwaccel_output_format", "cuda", "-extra_hw_frames", "16" },
+            // decoder from dropping reference frames it still needs. 16 was the first
+            // bump; observed Tubi content at certain time offsets still overruns it
+            // and the encoder writes only ~0.7-1 s of video out of the 4 s window
+            // before NVDEC drops the chain. 32 covers the deepest reference chains
+            // seen in the wild without meaningfully growing GPU memory.
+            HardwareEncoderBackend.Nvenc => new[] { "-hwaccel", "cuda", "-hwaccel_output_format", "cuda", "-extra_hw_frames", "32" },
             HardwareEncoderBackend.Qsv => new[] { "-hwaccel", "qsv", "-hwaccel_output_format", "qsv" },
             HardwareEncoderBackend.Amf or HardwareEncoderBackend.MediaFoundation =>
                 new[] { "-hwaccel", "d3d11va", "-hwaccel_output_format", "d3d11" },