From 114fc7634b51ac99b7709bcf75314eb6b99ad11e Mon Sep 17 00:00:00 2001 From: Mr-Neutr0n <64578610+Mr-Neutr0n@users.noreply.github.com> Date: Fri, 6 Feb 2026 01:49:20 +0530 Subject: [PATCH 1/2] Fix GradCAM ReLU placement per original paper Move ReLU (clamp) from intermediate gradients to the final gradcam computation. Per the original GradCAM paper, ReLU should be applied to the weighted combination of feature maps (cams * grads), not to the gradients alone. Fixes #789 Co-Authored-By: Claude Opus 4.5 --- lavis/models/blip_models/blip_image_text_matching.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/lavis/models/blip_models/blip_image_text_matching.py b/lavis/models/blip_models/blip_image_text_matching.py index a691984f8..f5c84b64d 100644 --- a/lavis/models/blip_models/blip_image_text_matching.py +++ b/lavis/models/blip_models/blip_image_text_matching.py @@ -175,11 +175,13 @@ def compute_gradcam(model, visual_input, text_input, tokenized_text, block_num=6 # assume using vit with 576 num image patch cams = cams[:, :, :, 1:].reshape(visual_input.size(0), 12, -1, 24, 24) * mask grads = ( - grads[:, :, :, 1:].clamp(0).reshape(visual_input.size(0), 12, -1, 24, 24) + grads[:, :, :, 1:].reshape(visual_input.size(0), 12, -1, 24, 24) * mask ) - gradcams = cams * grads + # Apply ReLU to the final gradcam, not to intermediate gradients + # as per the original GradCAM paper + gradcams = (cams * grads).clamp(min=0) gradcam_list = [] for ind in range(visual_input.size(0)): From cc95d8791be1d675c844599ff4c01bacf37a452e Mon Sep 17 00:00:00 2001 From: Mr-Neutr0n <64578610+Mr-Neutr0n@users.noreply.github.com> Date: Fri, 6 Feb 2026 02:05:22 +0530 Subject: [PATCH 2/2] Trigger CLA recheck