feat(macos): wire 4:4:4 BiPlanar capture for ProRes end-to-end

ProRes profiles are intrinsically 10-bit (proxy / lt / standard / hq)
or 12-bit (4444 / 4444 XQ); FFmpeg's prores_videotoolbox supported
input pix_fmt list does not include the 4:2:0 BiPlanar formats (NV12 /
P010) Sunshine has historically delivered for H.264 / HEVC / AV1. The
422 family wants 4:2:2 or higher chroma and the 4444 family wants
4:4:4 natively. Until now, the macOS capture path could only produce
4:2:0 buffers, so the ProRes encoder probe failed at avcodec_open2
with "Couldn't open" regardless of profile selection or colorspace
configuration.

Wire the 4:4:4 BiPlanar capture path end-to-end:

  * Add nv24 / p410 to platf::pix_fmt_e and the from_pix_fmt switch.
  * Map AV_PIX_FMT_NV24 / AV_PIX_FMT_P410 in video::map_pix_fmt.
  * Populate the previously-NONE 4:4:4 pix_fmt slots on the
    videotoolbox encoder declaration and add YUV444_SUPPORT to its
    flags. H.264 and HEVC VideoToolbox don't gain anything new
    functionally — they remain 4:2:0 only on Apple Silicon hardware —
    but the 4:4:4 probe runs against them harmlessly and falls
    through with their YUV444 capability bit set false, which is
    correct.
  * Route nv24 / p410 through nv12_zero_device in
    display.mm::make_avcodec_encode_device, alongside the existing
    nv12 / p010 paths.
  * Set the matching CVPixelBufferType
    (kCVPixelFormatType_444YpCbCr*BiPlanarVideoRange) per pix_fmt in
    nv12_zero_device::init.

Update the ProRes encoder probe in test_prores to use a ProRes-shaped
config:

  * dynamicRange = 1 (10-bit pix_fmt slot instead of 8-bit).
  * encoderCscMode = 3 (full range, BT.709 instead of BT.601).
  * chromaSamplingType = 1 (4:4:4 P410 instead of 4:2:0 P010).

Any ProRes probe that succeeds inherently uses 10-bit input, so
DYNAMIC_RANGE is promoted eagerly here rather than relying on
test_hdr_and_yuv444 below (which gates on PASSED and only sets
DYNAMIC_RANGE itself).

Verified end-to-end on M4 Max: the startup probe now produces
"Found ProRes encoder: prores_videotoolbox [videotoolbox]", where
previously the encoder failed to open at every config permutation
attempted.

This is the runtime-side companion to the build-deps change that
adds prores_videotoolbox to the FFmpeg configure's --enable-encoder
flag (LizardByte/build-deps#693). Without that, the encoder symbol
is missing from libavcodec.a's static codec list and
avcodec_find_encoder_by_name returns null regardless of the wiring
in this commit.

Author: Nottlespike

src/platform/common.h

@@ -241,6 +241,8 @@ namespace platf {
     yuv420p10,  ///< YUV 4:2:0 10-bit
     nv12,  ///< NV12
     p010,  ///< P010
+    nv24,  ///< NV24 (YUV 4:4:4 8-bit BiPlanar)
+    p410,  ///< P410 (YUV 4:4:4 10-bit BiPlanar)
     ayuv,  ///< AYUV
     yuv444p16,  ///< Planar 10-bit (shifted to 16-bit) YUV 4:4:4
     yuv444p,  ///< Planar 8-bit YUV 4:4:4
@@ -258,6 +260,8 @@ namespace platf {
       _CONVERT(yuv420p10);
       _CONVERT(nv12);
       _CONVERT(p010);
+      _CONVERT(nv24);
+      _CONVERT(p410);
       _CONVERT(ayuv);
       _CONVERT(yuv444p16);
       _CONVERT(yuv444p);

src/platform/macos/display.mm

@@ -301,7 +301,15 @@ capture_e capture(const push_captured_image_cb_t &push_captured_image_cb, const
         av_capture.pixelFormat = kCVPixelFormatType_32BGRA;
 
         return std::make_unique<avcodec_encode_device_t>();
-      } else if (pix_fmt == pix_fmt_e::nv12 || pix_fmt == pix_fmt_e::p010) {
+      } else if (pix_fmt == pix_fmt_e::nv12 || pix_fmt == pix_fmt_e::p010 ||
+                 pix_fmt == pix_fmt_e::nv24 || pix_fmt == pix_fmt_e::p410) {
+        // nv12 / p010 are 4:2:0 BiPlanar (8 / 10 bit); nv24 / p410 are the
+        // 4:4:4 BiPlanar equivalents required by prores_videotoolbox for
+        // ProRes 422 profiles (encoder downsamples internally) and ProRes
+        // 4444 (native). nv12_zero_device is format-agnostic at the wrap
+        // layer — it sets the capture-side CVPixelBufferType and then wraps
+        // frames for AV_PIX_FMT_VIDEOTOOLBOX, so the same device handles all
+        // four.
         auto device = std::make_unique<nv12_zero_device>();
 
         device->init((void *) av_capture, pix_fmt, setResolution, setPixelFormat);

src/platform/macos/nv12_zero_device.cpp

@@ -56,7 +56,30 @@ namespace platf {
   }
 
   int nv12_zero_device::init(void *display, pix_fmt_e pix_fmt, resolution_fn_t resolution_fn, const pixel_format_fn_t &pixel_format_fn) {
-    pixel_format_fn(display, pix_fmt == pix_fmt_e::nv12 ? kCVPixelFormatType_420YpCbCr8BiPlanarVideoRange : kCVPixelFormatType_420YpCbCr10BiPlanarVideoRange);
+    // Map the abstract pix_fmt_e to the matching CVPixelBufferType. The
+    // 4:2:0 BiPlanar formats (NV12 / P010) cover H.264 / HEVC / AV1; the
+    // 4:4:4 BiPlanar formats (NV24 / P410) cover ProRes (422 profiles via
+    // encoder-internal downsample, 4444 profiles natively).
+    OSType cv_format;
+    switch (pix_fmt) {
+      case pix_fmt_e::nv12:
+        cv_format = kCVPixelFormatType_420YpCbCr8BiPlanarVideoRange;
+        break;
+      case pix_fmt_e::nv24:
+        cv_format = kCVPixelFormatType_444YpCbCr8BiPlanarVideoRange;
+        break;
+      case pix_fmt_e::p410:
+        cv_format = kCVPixelFormatType_444YpCbCr10BiPlanarVideoRange;
+        break;
+      case pix_fmt_e::p010:
+      default:
+        // p010 is the historical 10-bit 4:2:0 path; the default fall-through
+        // matches it because display.mm::make_avcodec_encode_device is the
+        // source of truth for which pix_fmt values reach this method.
+        cv_format = kCVPixelFormatType_420YpCbCr10BiPlanarVideoRange;
+        break;
+    }
+    pixel_format_fn(display, cv_format);
 
     this->display = display;
     this->resolution_fn = std::move(resolution_fn);

src/video.cpp

@@ -1124,8 +1124,15 @@ namespace video {
       AV_PIX_FMT_VIDEOTOOLBOX,
       AV_PIX_FMT_NV12,
       AV_PIX_FMT_P010,
-      AV_PIX_FMT_NONE,
-      AV_PIX_FMT_NONE,
+      // YUV 4:4:4 BiPlanar formats are required by prores_videotoolbox: the
+      // 422 family (proxy / lt / standard / hq) wants 4:2:2 or higher chroma
+      // and the 4444 family wants 4:4:4 natively. Feeding 4:4:4 P410 lets
+      // the encoder downsample to the chosen 422 profile internally. H.264
+      // and HEVC VideoToolbox will simply fail the 4:4:4 probe on Apple
+      // Silicon (hardware encoder is 4:2:0 only for those codecs) and the
+      // capability bit stays false for them, which is correct.
+      AV_PIX_FMT_NV24,
+      AV_PIX_FMT_P410,
       vt_init_avcodec_hardware_input_buffer
     ),
     {
@@ -1197,7 +1204,13 @@ namespace video {
       {},  // Fallback options
       "prores_videotoolbox"s,
     },
-    PARALLEL_ENCODING
+    // YUV444_SUPPORT enables the 4:4:4 probe path; only ProRes 4444 /
+    // 4444 XQ profiles consume it natively on macOS but the flag is
+    // per-encoder family rather than per-codec, so H.264 / HEVC will
+    // probe at 4:4:4 too and fall through with their YUV444 capability
+    // bit set false (Apple Silicon's hardware H.264/HEVC encoder is
+    // 4:2:0 only).
+    PARALLEL_ENCODING | YUV444_SUPPORT
   };
 #endif
 
@@ -2783,17 +2796,55 @@ namespace video {
     }
 
     if (test_prores) {
-      config_max_ref_frames.videoFormat = SUNSHINE_FORMAT_PRORES;
-      config_autoselect.videoFormat = SUNSHINE_FORMAT_PRORES;
-
-      if (disp->is_codec_supported(encoder.prores.name, config_autoselect)) {
-        auto max_ref_frames_prores = validate_config(disp, encoder, config_max_ref_frames);
+      // ProRes profiles are intrinsically 10-bit (proxy / lt / standard / hq)
+      // or 12-bit (4444 / 4444 XQ) — there is no 8-bit ProRes input path in
+      // the FFmpeg encoder. Probe with dynamicRange = 1 so validate_config
+      // feeds the 10-bit pix_fmt (P010) to prores_videotoolbox rather than
+      // the 8-bit NV12 the H.264/HEVC probes use; otherwise the encoder
+      // legitimately refuses to open and PASSED stays false even though the
+      // downstream HDR probe would have succeeded.
+      config_t prores_max_ref_frames = config_max_ref_frames;
+      config_t prores_autoselect = config_autoselect;
+      prores_max_ref_frames.videoFormat = SUNSHINE_FORMAT_PRORES;
+      prores_autoselect.videoFormat = SUNSHINE_FORMAT_PRORES;
+      prores_max_ref_frames.dynamicRange = 1;
+      prores_autoselect.dynamicRange = 1;
+      // encoderCscMode = 3 (full range, BT.709) — prores_videotoolbox rejects
+      // the BT.601 colorspace the default SDR config carries (encoderCscMode
+      // = 1) even at 10-bit, because ProRes was never intended for SD content.
+      // BT.709 matches what test_hdr_and_yuv444 already uses for HDR probes
+      // and what the encoder actually expects. dynamicRange = 1 above promotes
+      // the color_trc to PQ where supported by the VT compression session;
+      // otherwise the encoder keeps BT.709 SDR tags, which it also accepts.
+      prores_max_ref_frames.encoderCscMode = 3;
+      prores_autoselect.encoderCscMode = 3;
+      // chromaSamplingType = 1 (4:4:4) selects the P410 pix_fmt slot.
+      // prores_videotoolbox's supported input pix_fmt list does not include
+      // 4:2:0 BiPlanar formats (P010) for any profile — the 422 family
+      // (proxy / lt / standard / hq) wants 4:2:2 or higher chroma, and the
+      // 4444 family wants 4:4:4 natively. Feeding 4:4:4 P410 lets the
+      // encoder downsample to the selected profile internally; feeding
+      // P010 makes it refuse to open with "Couldn't open".
+      prores_max_ref_frames.chromaSamplingType = 1;
+      prores_autoselect.chromaSamplingType = 1;
+
+      if (disp->is_codec_supported(encoder.prores.name, prores_autoselect)) {
+        auto max_ref_frames_prores = validate_config(disp, encoder, prores_max_ref_frames);
         auto autoselect_prores = max_ref_frames_prores >= 0 ?
                                    max_ref_frames_prores :
-                                   validate_config(disp, encoder, config_autoselect);
+                                   validate_config(disp, encoder, prores_autoselect);
 
         encoder.prores[encoder_t::REF_FRAMES_RESTRICT] = max_ref_frames_prores >= 0;
         encoder.prores[encoder_t::PASSED] = max_ref_frames_prores >= 0 || autoselect_prores >= 0;
+
+        // Any ProRes probe that succeeds inherently uses 10-bit input, so
+        // promote DYNAMIC_RANGE here. test_hdr_and_yuv444 below gates on
+        // PASSED and only sets DYNAMIC_RANGE itself; setting it eagerly here
+        // makes the encoder's capability advertisement consistent for clients
+        // that opt into ProRes via prores_mode > 0.
+        if (encoder.prores[encoder_t::PASSED]) {
+          encoder.prores[encoder_t::DYNAMIC_RANGE] = true;
+        }
       } else {
         BOOST_LOG(info) << "Encoder ["sv << encoder.prores.name << "] is not supported on this GPU"sv;
         encoder.prores.capabilities.reset();
@@ -3365,6 +3416,10 @@ namespace video {
         return platf::pix_fmt_e::yuv444p;
       case AV_PIX_FMT_YUV444P16:
         return platf::pix_fmt_e::yuv444p16;
+      case AV_PIX_FMT_NV24:
+        return platf::pix_fmt_e::nv24;
+      case AV_PIX_FMT_P410:
+        return platf::pix_fmt_e::p410;
       default:
         return platf::pix_fmt_e::unknown;
     }