harden vulkan encoder fallback and KMS monitor failure handling

Author: codeafridi

src/platform/linux/kmsgrab.cpp

@@ -611,6 +611,11 @@ namespace platf {
       int init(const std::string &display_name, const ::video::config_t &config) {
         delay = std::chrono::nanoseconds {1s} / config.framerate;
 
+        if (kms::card_descriptors.empty()) {
+          BOOST_LOG(error) << "No KMS monitor descriptors are available; aborting monitor lookup for ["sv << display_name << ']';
+          return -1;
+        }
+
         int monitor_index = util::from_view(display_name);
         int monitor = 0;
 
@@ -777,7 +782,7 @@ namespace platf {
           }
         }
 
-        BOOST_LOG(error) << "Couldn't find monitor ["sv << monitor_index << ']';
+        BOOST_LOG(error) << "Couldn't find monitor ["sv << monitor_index << "] in "sv << monitor << " enumerated KMS monitor(s)"sv;
         return -1;
 
       // Neatly break from nested for loop
@@ -1616,6 +1621,12 @@ namespace platf {
   std::vector<std::string> kms_display_names(mem_type_e hwdevice_type) {
     int count = 0;
 
+    kms::env_width = 0;
+    kms::env_height = 0;
+    kms::env_logical_width = 0;
+    kms::env_logical_height = 0;
+    kms::card_descriptors.clear();
+
     if (!fs::exists("/dev/dri")) {
       BOOST_LOG(warning) << "Couldn't find /dev/dri, kmsgrab won't be enabled"sv;
       return {};
@@ -1728,13 +1739,14 @@ namespace platf {
       correlate_to_wayland(cds);
     }
 
-    // Deduce the full virtual desktop size
-    kms::env_width = 0;
-    kms::env_height = 0;
+    BOOST_LOG(debug) << "Enumerated "sv << display_names.size() << " KMS monitor(s)"sv;
 
-    kms::env_logical_width = 0;
-    kms::env_logical_height = 0;
+    if (display_names.empty()) {
+      BOOST_LOG(error) << "No KMS monitors were found during enumeration"sv;
+      return {};
+    }
 
+    // Deduce the full virtual desktop size
     for (auto &card_descriptor : cds) {
       for (auto &[_, monitor_descriptor] : card_descriptor.crtc_to_monitor) {
         BOOST_LOG(debug) << "Monitor description"sv;

src/video.cpp

@@ -1181,6 +1181,8 @@ namespace video {
   int active_av1_mode;
   bool last_encoder_probe_supported_ref_frames_invalidation = false;
   std::array<bool, 3> last_encoder_probe_supported_yuv444_for_codec = {};
+  constexpr auto no_display_available_msg = "Unable to start capture because no display is available"sv;
+  constexpr auto no_encoder_selected_msg = "No encoder selected; aborting capture instead of using invalid encoder state"sv;
 
   void reset_display(std::shared_ptr<platf::display_t> &disp, const platf::mem_type_e &type, const std::string &display_name, const config_t &config) {
     // We try this twice, in case we still get an error on reinitialization
@@ -1214,16 +1216,18 @@ namespace video {
     }
 
     // Refresh the display names
-    auto old_display_names = std::move(display_names);
+    auto had_display_names = !display_names.empty();
     display_names = platf::display_names(dev_type);
 
-    // If we now have no displays, let's put the old display array back and fail
-    if (display_names.empty() && !old_display_names.empty()) {
-      BOOST_LOG(error) << "No displays were found after reenumeration!"sv;
-      display_names = std::move(old_display_names);
-      return;
-    } else if (display_names.empty()) {
-      display_names.emplace_back(output_name);
+    // If we now have no displays, fail instead of reusing stale display names.
+    if (display_names.empty()) {
+      if (!had_display_names && !output_name.empty()) {
+        display_names.emplace_back(output_name);
+      } else {
+        BOOST_LOG(error) << (had_display_names ? "No displays were found after reenumeration!"sv : "No displays were found during enumeration!"sv);
+        current_display_index = -1;
+        return;
+      }
     }
 
     // We now have a new display name list, so reset the index back to 0
@@ -1284,6 +1288,10 @@ namespace video {
     std::vector<std::string> display_names;
     int display_p = -1;
     refresh_displays(encoder.platform_formats->dev_type, display_names, display_p);
+    if (display_p < 0 || display_names.empty()) {
+      BOOST_LOG(error) << no_display_available_msg;
+      return;
+    }
     auto disp = platf::display(encoder.platform_formats->dev_type, display_names[display_p], capture_ctxs.front().config);
     if (!disp) {
       return;
@@ -1897,27 +1905,34 @@ namespace video {
       // Allow the encoding device a final opportunity to set/unset or override any options
       encode_device->init_codec_options(ctx.get(), &options);
 
-      if (auto status = avcodec_open2(ctx.get(), codec, &options)) {
-        char err_str[AV_ERROR_MAX_STRING_SIZE] {0};
+      auto status = avcodec_open2(ctx.get(), codec, &options);
+      if (!status) {
+        // Successfully opened the codec
+        break;
+      }
 
-        if (!video_format.fallback_options.empty() && retries == 0) {
-          BOOST_LOG(info)
-            << "Retrying with fallback configuration options for ["sv << video_format.name << "] after error: "sv
-            << av_make_error_string(err_str, AV_ERROR_MAX_STRING_SIZE, status);
+      char err_str[AV_ERROR_MAX_STRING_SIZE] {0};
+      if (!video_format.fallback_options.empty() && retries == 0) {
+        BOOST_LOG(info)
+          << "Retrying with fallback configuration options for ["sv << video_format.name << "] after error: "sv
+          << av_make_error_string(err_str, AV_ERROR_MAX_STRING_SIZE, status);
 
-          continue;
-        } else {
-          BOOST_LOG(error)
-            << "Could not open codec ["sv
-            << video_format.name << "]: "sv
-            << av_make_error_string(err_str, AV_ERROR_MAX_STRING_SIZE, status);
+        continue;
+      }
 
-          return nullptr;
-        }
+      BOOST_LOG(error)
+        << "Could not open codec ["sv
+        << video_format.name << "]: "sv
+        << av_make_error_string(err_str, AV_ERROR_MAX_STRING_SIZE, status);
+
+      if (encoder.name == "vulkan"sv) {
+        BOOST_LOG(error)
+          << "Vulkan encoder setup failed for ["sv << video_format.name
+          << "]; this GPU or driver does not expose the required Vulkan video encode capability. "
+          << "Sunshine will discard this encoder and continue probing fallback encoders."sv;
       }
 
-      // Successfully opened the codec
-      break;
+      return nullptr;
     }
 
     avcodec_frame_t frame {av_frame_alloc()};
@@ -2286,6 +2301,10 @@ namespace video {
     while (encode_session_ctx_queue.running()) {
       // Refresh display names since a display removal might have caused the reinitialization
       refresh_displays(encoder.platform_formats->dev_type, display_names, display_p);
+      if (display_p < 0 || display_names.empty()) {
+        BOOST_LOG(error) << no_display_available_msg;
+        return encode_e::error;
+      }
 
       // Process any pending display switch with the new list of displays
       if (switch_display_event->peek()) {
@@ -2491,6 +2510,11 @@ namespace video {
         display = ref->display_wp->lock();
       }
 
+      if (!chosen_encoder) {
+        BOOST_LOG(error) << no_encoder_selected_msg;
+        return;
+      }
+
       auto &encoder = *chosen_encoder;
 
       auto encode_device = make_encode_device(*display, encoder, config);
@@ -2533,6 +2557,11 @@ namespace video {
   ) {
     auto idr_events = mail->event<bool>(mail::idr);
 
+    if (!chosen_encoder) {
+      BOOST_LOG(error) << no_encoder_selected_msg;
+      return;
+    }
+
     idr_events->raise(true);
     if (chosen_encoder->flags & PARALLEL_ENCODING) {
       capture_async(std::move(mail), config, channel_data);
@@ -3013,6 +3042,7 @@ namespace video {
 
     if (encode_device && encode_device->data) {
       if (((vulkan_init_avcodec_hardware_input_buffer_fn) encode_device->data)(encode_device, &hw_device_buf)) {
+        BOOST_LOG(error) << "Failed to create a Vulkan device from the capture backend; aborting Vulkan encoder setup"sv;
         return -1;
       }
       return hw_device_buf;