harden vulkan encoder fallback and KMS monitor failure handling

Author: codeafridi

src/platform/linux/kmsgrab.cpp

@@ -611,6 +611,13 @@ namespace platf {
       int init(const std::string &display_name, const ::video::config_t &config) {
         delay = std::chrono::nanoseconds {1s} / config.framerate;
 
+        if (kms::card_descriptors.empty()) {
+          // GCOVR_EXCL_START - requires KMS monitor enumeration to fail on the host.
+          BOOST_LOG(error) << "No KMS monitor descriptors are available; aborting monitor lookup for ["sv << display_name << ']';
+          return -1;
+          // GCOVR_EXCL_STOP
+        }
+
         int monitor_index = util::from_view(display_name);
         int monitor = 0;
 
@@ -777,7 +784,7 @@ namespace platf {
           }
         }
 
-        BOOST_LOG(error) << "Couldn't find monitor ["sv << monitor_index << ']';
+        BOOST_LOG(error) << "Couldn't find monitor ["sv << monitor_index << "] in "sv << monitor << " enumerated KMS monitor(s)"sv;
         return -1;
 
       // Neatly break from nested for loop
@@ -1616,6 +1623,12 @@ namespace platf {
   std::vector<std::string> kms_display_names(mem_type_e hwdevice_type) {
     int count = 0;
 
+    kms::env_width = 0;
+    kms::env_height = 0;
+    kms::env_logical_width = 0;
+    kms::env_logical_height = 0;
+    kms::card_descriptors.clear();
+
     if (!fs::exists("/dev/dri")) {
       BOOST_LOG(warning) << "Couldn't find /dev/dri, kmsgrab won't be enabled"sv;
       return {};
@@ -1728,13 +1741,16 @@ namespace platf {
       correlate_to_wayland(cds);
     }
 
-    // Deduce the full virtual desktop size
-    kms::env_width = 0;
-    kms::env_height = 0;
+    BOOST_LOG(debug) << "Enumerated "sv << display_names.size() << " KMS monitor(s)"sv;
 
-    kms::env_logical_width = 0;
-    kms::env_logical_height = 0;
+    if (display_names.empty()) {
+      // GCOVR_EXCL_START - requires KMS monitor enumeration to fail on the host.
+      BOOST_LOG(error) << "No KMS monitors were found during enumeration"sv;
+      return {};
+      // GCOVR_EXCL_STOP
+    }
 
+    // Deduce the full virtual desktop size
     for (auto &card_descriptor : cds) {
       for (auto &[_, monitor_descriptor] : card_descriptor.crtc_to_monitor) {
         BOOST_LOG(debug) << "Monitor description"sv;

src/platform/linux/vulkan_encode.cpp

@@ -106,6 +106,62 @@ namespace vk {
     return -1;
   }
 
+  // GCOVR_EXCL_START - depends on the host Vulkan driver and physical GPU capabilities.
+  static bool physical_device_supports_h264_encode(VkPhysicalDevice physical_device) {
+    uint32_t count = 0;
+    if (vkEnumerateDeviceExtensionProperties(physical_device, nullptr, &count, nullptr) != VK_SUCCESS) {
+      return false;
+    }
+
+    std::vector<VkExtensionProperties> extensions(count);
+    if (vkEnumerateDeviceExtensionProperties(physical_device, nullptr, &count, extensions.data()) != VK_SUCCESS) {
+      return false;
+    }
+
+    bool video_queue = false;
+    bool encode_queue = false;
+    bool encode_h264 = false;
+
+    for (const auto &extension : extensions) {
+      const std::string_view name {extension.extensionName};
+      video_queue = video_queue || name == "VK_KHR_video_queue"sv;
+      encode_queue = encode_queue || name == "VK_KHR_video_encode_queue"sv;
+      encode_h264 = encode_h264 || name == "VK_KHR_video_encode_h264"sv;
+    }
+
+    return video_queue && encode_queue && encode_h264;
+  }
+
+  static bool has_h264_encode_physical_device() {
+    VkApplicationInfo app = {VK_STRUCTURE_TYPE_APPLICATION_INFO};
+    app.apiVersion = VK_API_VERSION_1_1;
+
+    VkInstanceCreateInfo ci = {VK_STRUCTURE_TYPE_INSTANCE_CREATE_INFO};
+    ci.pApplicationInfo = &app;
+
+    VkInstance instance = VK_NULL_HANDLE;
+    if (vkCreateInstance(&ci, nullptr, &instance) != VK_SUCCESS) {
+      return false;
+    }
+
+    uint32_t count = 0;
+    if (vkEnumeratePhysicalDevices(instance, &count, nullptr) != VK_SUCCESS || count == 0) {
+      vkDestroyInstance(instance, nullptr);
+      return false;
+    }
+
+    std::vector<VkPhysicalDevice> physical_devices(count);
+    if (vkEnumeratePhysicalDevices(instance, &count, physical_devices.data()) != VK_SUCCESS) {
+      vkDestroyInstance(instance, nullptr);
+      return false;
+    }
+
+    const bool supported = std::any_of(std::begin(physical_devices), std::end(physical_devices), physical_device_supports_h264_encode);
+    vkDestroyInstance(instance, nullptr);
+    return supported;
+  }
+  // GCOVR_EXCL_STOP
+
   struct PushConstants {
     std::array<float, 4> color_vec_y;
     std::array<float, 4> color_vec_u;
@@ -1025,11 +1081,19 @@ namespace vk {
   }
 
   bool validate() {
-    if (!avcodec_find_encoder_by_name("h264_vulkan") && !avcodec_find_encoder_by_name("hevc_vulkan")) {
+    if (!avcodec_find_encoder_by_name("h264_vulkan")) {
+      BOOST_LOG(info) << "FFmpeg h264_vulkan encoder is not available"sv;
       return false;
     }
+
+    if (!has_h264_encode_physical_device()) {
+      BOOST_LOG(info) << "Vulkan H.264 video encode is not supported by this device"sv;
+      return false;
+    }
+
     AVBufferRef *dev = nullptr;
     if (create_vulkan_hwdevice(&dev) < 0) {
+      BOOST_LOG(info) << "Failed to create Vulkan hardware device for encoder validation"sv;
       return false;
     }
     av_buffer_unref(&dev);

src/video.cpp

@@ -31,6 +31,12 @@ extern "C" {
 #include "sync.h"
 #include "video.h"
 
+#ifdef SUNSHINE_BUILD_VULKAN
+  #if defined(__linux__) || defined(linux) || defined(__linux) || defined(__FreeBSD__)
+    #include "platform/linux/vulkan_encode.h"
+  #endif
+#endif
+
 #ifdef _WIN32
 extern "C" {
   #include <libavutil/hwcontext_d3d11va.h>
@@ -1181,6 +1187,8 @@ namespace video {
   int active_av1_mode;
   bool last_encoder_probe_supported_ref_frames_invalidation = false;
   std::array<bool, 3> last_encoder_probe_supported_yuv444_for_codec = {};
+  constexpr auto no_display_available_msg = "Unable to start capture because no display is available"sv;
+  constexpr auto no_encoder_selected_msg = "No encoder selected; aborting capture instead of using invalid encoder state"sv;
 
   void reset_display(std::shared_ptr<platf::display_t> &disp, const platf::mem_type_e &type, const std::string &display_name, const config_t &config) {
     // We try this twice, in case we still get an error on reinitialization
@@ -1214,16 +1222,20 @@ namespace video {
     }
 
     // Refresh the display names
-    auto old_display_names = std::move(display_names);
+    auto had_display_names = !display_names.empty();
     display_names = platf::display_names(dev_type);
 
-    // If we now have no displays, let's put the old display array back and fail
-    if (display_names.empty() && !old_display_names.empty()) {
-      BOOST_LOG(error) << "No displays were found after reenumeration!"sv;
-      display_names = std::move(old_display_names);
-      return;
-    } else if (display_names.empty()) {
-      display_names.emplace_back(output_name);
+    // If we now have no displays, fail instead of reusing stale display names.
+    if (display_names.empty()) {
+      if (!had_display_names && !output_name.empty()) {
+        display_names.emplace_back(output_name);
+      } else {
+        // GCOVR_EXCL_START - requires display enumeration to fail during capture.
+        BOOST_LOG(error) << (had_display_names ? "No displays were found after reenumeration!"sv : "No displays were found during enumeration!"sv);
+        current_display_index = -1;
+        return;
+        // GCOVR_EXCL_STOP
+      }
     }
 
     // We now have a new display name list, so reset the index back to 0
@@ -1284,6 +1296,12 @@ namespace video {
     std::vector<std::string> display_names;
     int display_p = -1;
     refresh_displays(encoder.platform_formats->dev_type, display_names, display_p);
+    if (display_p < 0 || display_names.empty()) {
+      // GCOVR_EXCL_START - requires display enumeration to fail during capture.
+      BOOST_LOG(error) << no_display_available_msg;
+      return;
+      // GCOVR_EXCL_STOP
+    }
     auto disp = platf::display(encoder.platform_formats->dev_type, display_names[display_p], capture_ctxs.front().config);
     if (!disp) {
       return;
@@ -1897,27 +1915,34 @@ namespace video {
       // Allow the encoding device a final opportunity to set/unset or override any options
       encode_device->init_codec_options(ctx.get(), &options);
 
-      if (auto status = avcodec_open2(ctx.get(), codec, &options)) {
-        char err_str[AV_ERROR_MAX_STRING_SIZE] {0};
+      auto status = avcodec_open2(ctx.get(), codec, &options);
+      if (!status) {
+        // Successfully opened the codec
+        break;
+      }
 
-        if (!video_format.fallback_options.empty() && retries == 0) {
-          BOOST_LOG(info)
-            << "Retrying with fallback configuration options for ["sv << video_format.name << "] after error: "sv
-            << av_make_error_string(err_str, AV_ERROR_MAX_STRING_SIZE, status);
+      char err_str[AV_ERROR_MAX_STRING_SIZE] {0};
+      if (!video_format.fallback_options.empty() && retries == 0) {
+        BOOST_LOG(info)
+          << "Retrying with fallback configuration options for ["sv << video_format.name << "] after error: "sv
+          << av_make_error_string(err_str, AV_ERROR_MAX_STRING_SIZE, status);
 
-          continue;
-        } else {
-          BOOST_LOG(error)
-            << "Could not open codec ["sv
-            << video_format.name << "]: "sv
-            << av_make_error_string(err_str, AV_ERROR_MAX_STRING_SIZE, status);
+        continue;
+      }
 
-          return nullptr;
-        }
+      BOOST_LOG(error)
+        << "Could not open codec ["sv
+        << video_format.name << "]: "sv
+        << av_make_error_string(err_str, AV_ERROR_MAX_STRING_SIZE, status);
+
+      if (encoder.name == "vulkan"sv) {
+        BOOST_LOG(error)
+          << "Vulkan encoder setup failed for ["sv << video_format.name
+          << "]; this GPU or driver does not expose the required Vulkan video encode capability. "
+          << "Sunshine will discard this encoder and continue probing fallback encoders."sv;
       }
 
-      // Successfully opened the codec
-      break;
+      return nullptr;
     }
 
     avcodec_frame_t frame {av_frame_alloc()};
@@ -2286,6 +2311,12 @@ namespace video {
     while (encode_session_ctx_queue.running()) {
       // Refresh display names since a display removal might have caused the reinitialization
       refresh_displays(encoder.platform_formats->dev_type, display_names, display_p);
+      if (display_p < 0 || display_names.empty()) {
+        // GCOVR_EXCL_START - requires display enumeration to fail during capture.
+        BOOST_LOG(error) << no_display_available_msg;
+        return encode_e::error;
+        // GCOVR_EXCL_STOP
+      }
 
       // Process any pending display switch with the new list of displays
       if (switch_display_event->peek()) {
@@ -2491,6 +2522,13 @@ namespace video {
         display = ref->display_wp->lock();
       }
 
+      if (!chosen_encoder) {
+        // GCOVR_EXCL_START - defensive guard for invalid global encoder state.
+        BOOST_LOG(error) << no_encoder_selected_msg;
+        return;
+        // GCOVR_EXCL_STOP
+      }
+
       auto &encoder = *chosen_encoder;
 
       auto encode_device = make_encode_device(*display, encoder, config);
@@ -2533,6 +2571,13 @@ namespace video {
   ) {
     auto idr_events = mail->event<bool>(mail::idr);
 
+    if (!chosen_encoder) {
+      // GCOVR_EXCL_START - defensive guard for invalid global encoder state.
+      BOOST_LOG(error) << no_encoder_selected_msg;
+      return;
+      // GCOVR_EXCL_STOP
+    }
+
     idr_events->raise(true);
     if (chosen_encoder->flags & PARALLEL_ENCODING) {
       capture_async(std::move(mail), config, channel_data);
@@ -2628,6 +2673,14 @@ namespace video {
     encoder.hevc.capabilities.set();
     encoder.av1.capabilities.set();
 
+#ifdef SUNSHINE_BUILD_VULKAN
+    if (encoder.name == "vulkan"sv && !vk::validate()) {
+      fg.disable();
+      BOOST_LOG(info) << "Encoder ["sv << encoder.name << "] is not supported on this GPU"sv;
+      return false;
+    }
+#endif
+
     // First, test encoder viability
     config_t config_max_ref_frames {1920, 1080, 60, 6000, 1000, 1, 1, 1, 0, 0, 0};
     config_t config_autoselect {1920, 1080, 60, 6000, 1000, 1, 0, 1, 0, 0, 0};
@@ -3013,6 +3066,7 @@ namespace video {
 
     if (encode_device && encode_device->data) {
       if (((vulkan_init_avcodec_hardware_input_buffer_fn) encode_device->data)(encode_device, &hw_device_buf)) {
+        BOOST_LOG(error) << "Failed to create a Vulkan device from the capture backend; aborting Vulkan encoder setup"sv;
         return -1;
       }
       return hw_device_buf;