[GPU] Keep NMS boundary-sensitive subgraph in FP32 to fix Detectron2 FasterRCNN GPU extra detection

src/common/transformations/tests/common_optimizations/nop_elimination.cpp

@@ -1447,6 +1447,21 @@ TEST(nop_elimination, gather_to_squeeze) {
     run_and_check(func_axis_3);
 }
 
+TEST(nop_elimination, keep_gather_with_dynamic_dimension) {
+    auto arg = std::make_shared<op::v0::Parameter>(element::f32, PartialShape{Dimension::dynamic(), 1});
+    auto indices = op::v0::Constant::create(element::i64, Shape{}, vector<int64_t>{0});
+    auto axis = op::v0::Constant::create(element::i64, Shape{}, vector<int64_t>{1});
+    auto gather = std::make_shared<op::v8::Gather>(arg, indices, axis);
+    auto model = std::make_shared<ov::Model>(OutputVector{gather}, ParameterVector{arg});
+
+    pass::Manager pass_manager;
+    pass_manager.register_pass<ov::pass::NopElimination>();
+    pass_manager.run_passes(model);
+
+    EXPECT_EQ(count_ops_of_type<op::v8::Gather>(model), 1);
+    EXPECT_EQ(count_ops_of_type<op::v0::Squeeze>(model), 0);
+}
+
 TEST(nop_elimination, not_gather_to_squeeze_with_vector_indices) {
     auto generate_func = [](int64_t gather_axis) {
         ov::Shape shape{3, 3, 4, 4};

src/core/shape_inference/include/multiclass_nms_shape_inference.hpp

@@ -93,16 +93,21 @@ std::vector<TRShape> shape_infer(const util::MulticlassNmsBase* op,
             const auto& num_images = has_rois_num ? rois_num_shape[0] : scores_shape[0];
 
             auto& selected_boxes = output_shapes[0][0];
-            selected_boxes =
-                (nms_top_k > -1) ? TDim(std::min<V>(boxes_shape[1].get_max_length(), nms_top_k)) : boxes_shape[1];
+            if (nms_top_k > -1) {
+                const auto boxes_upper_bound = boxes_shape[1].get_max_length();
+                selected_boxes = TDim(boxes_upper_bound < 0 ? nms_top_k : std::min<V>(boxes_upper_bound, nms_top_k));
+            } else {
+                selected_boxes = boxes_shape[1];
+            }
 
             if (ignore_bg_class && (background_class > -1) && (background_class < num_classes.get_max_length())) {
                 selected_boxes *= std::max<V>(1, num_classes.get_max_length() - 1);
             } else {
                 selected_boxes *= num_classes;
             }
 
-            if (keep_top_k > -1 && (keep_top_k < selected_boxes.get_max_length())) {
+            if (keep_top_k > -1 &&
+                (selected_boxes.get_max_length() < 0 || keep_top_k < selected_boxes.get_max_length())) {
                 selected_boxes = TDim(keep_top_k);
             }
 

src/core/tests/type_prop/multiclass_nms.cpp

@@ -275,6 +275,19 @@ TYPED_TEST(type_prop, multiclass_nms_output_shape_1dim_keep_topk) {
     EXPECT_EQ(nms->get_output_shape(2), (Shape{2}));
 }
 
+TYPED_TEST(type_prop, multiclass_nms_dynamic_boxes_with_finite_topk) {
+    const auto boxes = make_shared<op::v0::Parameter>(element::f32, PartialShape{1, Dimension::dynamic(), 4});
+    const auto scores = make_shared<op::v0::Parameter>(element::f32, PartialShape{1, 80, Dimension::dynamic()});
+    this->attrs.nms_top_k = -1;
+    this->attrs.keep_top_k = 100;
+
+    const auto nms = make_shared<TypeParam>(boxes, scores, this->attrs);
+
+    EXPECT_EQ(nms->get_output_partial_shape(0), (PartialShape{{0, 100}, 6}));
+    EXPECT_EQ(nms->get_output_partial_shape(1), (PartialShape{{0, 100}, 1}));
+    EXPECT_EQ(nms->get_output_partial_shape(2), (PartialShape{1}));
+}
+
 TYPED_TEST(type_prop, multiclass_nms_input_f16) {
     const auto boxes = make_shared<op::v0::Parameter>(element::f16, Shape{2, 7, 4});
     const auto scores = make_shared<op::v0::Parameter>(element::f16, Shape{2, 5, 7});