From 2709edb8bec889819fddd77012ae6415d9aaecf7 Mon Sep 17 00:00:00 2001
From: Sheetal Sattiraju <sheetalsattiraju@Mac.attlocal.net>
Date: Sun, 28 Jun 2026 14:19:58 -0700
Subject: [PATCH 1/2] Add MeanReciprocalRank metric for recommender systems
 (#2631)

---
 ignite/metrics/rec_sys/__init__.py |   3 +-
 ignite/metrics/rec_sys/mrr.py      | 198 +++++++++++++++++++++++++++++
 2 files changed, 200 insertions(+), 1 deletion(-)
 create mode 100644 ignite/metrics/rec_sys/mrr.py

diff --git a/ignite/metrics/rec_sys/__init__.py b/ignite/metrics/rec_sys/__init__.py
index 6fa17fdca928..cd5379f466b2 100644
--- a/ignite/metrics/rec_sys/__init__.py
+++ b/ignite/metrics/rec_sys/__init__.py
@@ -1,4 +1,5 @@
 from ignite.metrics.rec_sys.hitrate import HitRate
 from ignite.metrics.rec_sys.ndcg import NDCG
+from ignite.metrics.rec_sys.mrr import MeanReciprocalRank
 
-__all__ = ["HitRate", "NDCG"]
+__all__ = ["HitRate", "NDCG", "MeanReciprocalRank"]
\ No newline at end of file
diff --git a/ignite/metrics/rec_sys/mrr.py b/ignite/metrics/rec_sys/mrr.py
new file mode 100644
index 000000000000..ebd345b12622
--- /dev/null
+++ b/ignite/metrics/rec_sys/mrr.py
@@ -0,0 +1,198 @@
+from collections.abc import Callable
+
+import torch
+
+from ignite.exceptions import NotComputableError
+from ignite.metrics.metric import Metric, reinit__is_reduced, sync_all_reduce
+
+__all__ = ["MeanReciprocalRank"]
+
+
+class MeanReciprocalRank(Metric):
+    r"""Calculates Mean Reciprocal Rank (MRR) at k for Recommendation Systems.
+
+    The Mean Reciprocal Rank measures the average of the reciprocal ranks of
+    the first relevant item in the predicted ranking for each user.
+    If no relevant item is found in the top-k predictions, the reciprocal rank for that user is 0.
+
+    Math: MRR@K = (1 / N) * sum(1 / rank_i) for i = 1 to N
+
+    - ``update`` must receive output of the form ``(y_pred, y)``.
+    - ``y_pred`` is expected to be raw logits or probability score for each item in the catalog.
+    - ``y`` is expected to be binary (only 0s and 1s) values where `1` indicates relevant item.
+    - ``y_pred`` and ``y`` are only allowed shape :math:`(batch, num_items)`.
+    - returns a list of MRR ordered by the sorted values of ``top_k``.
+
+    Args:
+        top_k: a single positive integer or a list of positive integers that specifies `k` for
+            calculating MRR@top-k. If a single int is provided, it will be wrapped in a list.
+            Default is 1.
+        ignore_zero_hits: if True, users with no relevant items (ground truth tensor being all zeros)
+            are ignored in computation of MRR. if set False, such users are counted with a reciprocal
+            rank of 0. By default, True.
+        output_transform: a callable that is used to transform the
+            :class:`~ignite.engine.engine.Engine`'s ``process_function``'s output into the
+            form expected by the metric.
+            The output is expected to be a tuple `(prediction, target)`
+            where `prediction` and `target` are tensors
+            of shape ``(batch, num_items)``.
+        device: specifies which device updates are accumulated on. Setting the
+            metric's device to be the same as your ``update`` arguments ensures the ``update`` method is
+            non-blocking. By default, CPU.
+        skip_unrolling: specifies whether input should be unrolled or not before being
+            processed. Should be true for multi-output models.
+
+    Examples:
+        .. testcode:: 1
+
+            metric = MeanReciprocalRank(top_k=[1, 2, 3, 4])
+            metric.attach(default_evaluator, "mrr")
+            y_pred = torch.Tensor([
+                [4.0, 2.0, 3.0, 1.0],
+                [1.0, 2.0, 3.0, 4.0]
+            ])
+            y_true = torch.Tensor([
+                [0.0, 0.0, 1.0, 1.0],
+                [0.0, 0.0, 0.0, 0.0]
+            ])
+            state = default_evaluator.run([(y_pred, y_true)])
+            print(state.metrics["mrr"])
+
+        .. testoutput:: 1
+
+            [0.0, 0.5, 0.5, 0.5]
+
+        ignore_zero_hits=False case
+        
+        .. testcode:: 2
+
+            metric = MeanReciprocalRank(top_k=[1, 2, 3, 4], ignore_zero_hits=False)
+            metric.attach(default_evaluator, "mrr")
+            y_pred = torch.Tensor([
+                [4.0, 2.0, 3.0, 1.0],
+                [1.0, 2.0, 3.0, 4.0]
+            ])
+            y_true = torch.Tensor([
+                [0.0, 0.0, 1.0, 1.0],
+                [0.0, 0.0, 0.0, 0.0]
+            ])
+            state = default_evaluator.run([(y_pred, y_true)])
+            print(state.metrics["mrr"])
+
+        .. testoutput:: 2
+
+            [0.0, 0.25, 0.25, 0.25]
+            
+        int top_k case
+
+        .. testcode:: 3
+
+            metric = MeanReciprocalRank(top_k=2)
+            metric.attach(default_evaluator, "mrr")
+            y_pred = torch.Tensor([
+                [4.0, 2.0, 3.0, 1.0],
+            ])
+            y_true = torch.Tensor([
+                [0.0, 0.0, 1.0, 0.0],
+            ])
+            state = default_evaluator.run([(y_pred, y_true)])
+            print(state.metrics["mrr"])
+
+        .. testoutput:: 3
+
+            [0.0]
+    """
+
+    required_output_keys = ("y_pred", "y")
+    _state_dict_all_req_keys = ("_sum_mrr_per_k", "_num_examples")
+
+
+    def __init__(
+        self,
+        top_k: list[int] | int = 1,
+        ignore_zero_hits: bool = True,
+        output_transform: Callable = lambda x: x,
+        device: str | torch.device = torch.device("cpu"),
+        skip_unrolling: bool = False,
+    ):
+
+        if not isinstance(top_k, (int, list)):
+            raise ValueError("top_k must be either int or a list[int]")
+
+        top_k = [top_k] if isinstance(top_k, int) else top_k
+
+        if len(top_k) == 0:
+            raise ValueError("top_k must have at least one positive value")
+
+        if any(k <= 0 for k in top_k):
+            raise ValueError("top_k must be list of positive integers only.")
+
+        self.top_k = sorted(top_k)
+        self.ignore_zero_hits = ignore_zero_hits
+
+        super().__init__(output_transform, device=device, skip_unrolling=skip_unrolling)
+
+    @reinit__is_reduced
+    def reset(self) -> None:
+        self._sum_mrr_per_k = torch.zeros(len(self.top_k), device=self._device)
+        self._num_examples = 0
+
+    @reinit__is_reduced
+    def update(self,output: tuple[torch.Tensor, torch.Tensor]) -> None:
+        if len(output) != 2:
+            raise ValueError(f"output should be in format `(y_pred,y)` but got tuple of {len(output)} tensors.")
+
+        y_pred, y = output
+
+        if y_pred.shape != y.shape:
+            raise ValueError(f"y_pred and y must be in the same shape, got {y_pred.shape} != {y.shape}.")
+
+        if self.ignore_zero_hits:
+            # Remove users with no relevant items, like [0,0,0,0]
+            valid_mask = torch.any(y > 0, dim=-1)
+            y_pred = y_pred[valid_mask]
+            y = y[valid_mask]
+
+        if y.shape[0] == 0:
+            return
+
+        max_k = self.top_k[-1]
+        # indices of top-ranked preds
+        _, indices = torch.topk(y_pred, k=max_k, dim=-1)
+
+
+        # Rearrange labels according to predicted ranking; example: [0,1,0,1]
+        # ranked_relevance = [0,1,1,0]
+        ranked_relevance = torch.gather(y, dim=-1, index=indices)
+
+        for i, k in enumerate(self.top_k):
+
+            # Only consider top-k recommendations
+            relevance_k = ranked_relevance[:, :k]
+
+            # Determine whether user has
+            # at least one relevant item in top-k.
+            has_hit = torch.any( relevance_k > 0, dim=-1)
+
+            # Find position of first relevant item. Example: [0,0,1]
+            # argmax -> 2; rank -> 3
+            # edge case: argmax([0,0,0]) returns 0, but corrected using has_hit.
+            first_pos = (torch.argmax((relevance_k > 0).int(),dim=-1) + 1)
+
+            # Reciprocal rank; users with no hit receive 0
+            rr = torch.where(has_hit, 1.0 / first_pos.float(), torch.zeros_like(first_pos, dtype=torch.float32))
+
+            # Add batch contribution
+            self._sum_mrr_per_k[i] += rr.sum().to(self._device)
+
+        self._num_examples += y.shape[0]
+
+    @sync_all_reduce("_sum_mrr_per_k", "_num_examples")
+    def compute(self) -> list[float]:
+        if self._num_examples == 0:
+            raise NotComputableError("MeanReciprocalRank must have at least one example.")
+
+        # avg reciprocal rank over all users
+        rates = (self._sum_mrr_per_k/ self._num_examples).tolist()
+
+        return rates

From 115de01fd573658b511334c13ca64b349a349c7e Mon Sep 17 00:00:00 2001
From: Sheetal Sattiraju <69981970+sheetalsattiraju@users.noreply.github.com>
Date: Mon, 6 Jul 2026 20:26:21 -0700
Subject: [PATCH 2/2] Add files via upload

---
 tests/ignite/metrics/rec_sys/test_mrr.py | 217 +++++++++++++++++++++++
 1 file changed, 217 insertions(+)
 create mode 100644 tests/ignite/metrics/rec_sys/test_mrr.py

diff --git a/tests/ignite/metrics/rec_sys/test_mrr.py b/tests/ignite/metrics/rec_sys/test_mrr.py
new file mode 100644
index 000000000000..0dfa2e870663
--- /dev/null
+++ b/tests/ignite/metrics/rec_sys/test_mrr.py
@@ -0,0 +1,217 @@
+import pytest
+import torch
+import numpy as np
+
+import ignite.distributed as idist
+from ignite.engine import Engine
+from ignite.exceptions import NotComputableError
+from ignite.metrics.rec_sys.mrr import MeanReciprocalRank
+
+
+def manual_mrr(
+    y_pred: np.ndarray,
+    y: np.ndarray,
+    top_k: list[int],
+    ignore_zero_hits: bool = True,
+) -> list[float]:
+    """Manual implementation of MRR using numpy for verification."""
+    sorted_top_k = sorted(top_k)
+
+    if ignore_zero_hits:
+        valid_mask = np.any(y > 0, axis=-1)
+        y_pred = y_pred[valid_mask]
+        y = y[valid_mask]
+
+    n_samples = y.shape[0]
+    if n_samples == 0:
+        raise ValueError("No valid samples for manual MRR computation.")
+
+    sorted_indices = np.argsort(-y_pred, axis=-1)
+  
+    results = []
+    for k in sorted_top_k:
+        k_indices = sorted_indices[:, :k]
+        rr_sum = 0.0
+        for i in range(n_samples):
+            ranked_labels = y[i, k_indices[i]]
+            hit_positions = np.where(ranked_labels > 0)[0]
+            if len(hit_positions) > 0:
+                rr_sum += 1.0 / (hit_positions[0] + 1)
+        results.append(rr_sum / n_samples)
+
+    return results
+
+
+def test_zero_sample():
+    metric = MeanReciprocalRank(top_k=[1, 5])
+    with pytest.raises(NotComputableError, match=r"MeanReciprocalRank must have at least one example"):
+        metric.compute()
+
+
+def test_shape_mismatch():
+    metric = MeanReciprocalRank(top_k=[1])
+    y_pred = torch.randn(4, 10)
+    y = torch.ones(4, 5)  # Mismatched items count
+    with pytest.raises(ValueError, match="y_pred and y must be in the same shape"):
+        metric.update((y_pred, y))
+
+
+def test_empty_top_k():
+    with pytest.raises(ValueError, match="top_k must have at least one positive value"):
+        MeanReciprocalRank(top_k=[])
+
+
+def test_invalid_top_k_type():
+    with pytest.raises(ValueError, match="top_k must be either int or a list"):
+        MeanReciprocalRank(top_k="invalid")
+
+
+def test_int_top_k(available_device):
+    metric = MeanReciprocalRank(top_k=2, device=available_device)
+    y_pred = torch.tensor([[4.0, 2.0, 3.0, 1.0]])
+    y_true = torch.tensor([[0.0, 0.0, 1.0, 0.0]])
+    metric.update((y_pred, y_true))
+    res = metric.compute()
+    expected = manual_mrr(y_pred.numpy(), y_true.numpy(), [2])
+    np.testing.assert_allclose(res, expected)
+
+
+@pytest.mark.parametrize("top_k", [[1], [1, 2, 4]])
+@pytest.mark.parametrize("ignore_zero_hits", [True, False])
+def test_compute(top_k, ignore_zero_hits, available_device):
+    metric = MeanReciprocalRank(
+        top_k=top_k,
+        ignore_zero_hits=ignore_zero_hits,
+        device=available_device,
+    )
+  
+    y_pred = torch.tensor([[4.0, 2.0, 3.0, 1.0], [1.0, 2.0, 3.0, 4.0]])
+    y_true = torch.tensor([[0, 0, 1.0, 1.0], [0, 0, 0.0, 0.0]])
+  
+    metric.update((y_pred, y_true))
+    res = metric.compute()
+  
+    expected = manual_mrr(
+        y_pred.numpy(),
+        y_true.numpy(),
+        top_k,
+        ignore_zero_hits=ignore_zero_hits,
+    )
+  
+    assert isinstance(res, list)
+    assert len(res) == len(top_k)
+    np.testing.assert_allclose(res, expected)
+
+
+def test_accumulator_detached(available_device):
+    metric = MeanReciprocalRank(top_k=[1], device=available_device)
+    y_pred = torch.randn(4, 5, requires_grad=True)
+    y = torch.randint(0, 2, (4, 5)).float()
+    metric.update((y_pred, y))
+  
+    assert metric._sum_mrr_per_k.requires_grad is False
+    assert metric._sum_mrr_per_k.is_leaf is True
+
+
+def test_all_zero_targets_ignore():
+    metric = MeanReciprocalRank(top_k=[1, 3], ignore_zero_hits=True)
+  
+    y_pred = torch.randn(4, 5)
+    y = torch.zeros(4, 5)
+  
+    metric.update((y_pred, y))
+  
+    with pytest.raises(NotComputableError):
+        metric.compute()
+
+
+def test_first_hit_position():
+    """MRR specifically rewards rank position; verify reciprocal rank values directly."""
+    metric = MeanReciprocalRank(top_k=[3])
+    # relevant item is at rank 3 (index 2 after sorting by score desc)
+    # scores: [3.0, 2.0, 1.0] -> rank order: index 0, index 1, index 2
+    # relevant: index 2 only -> first hit at rank 3 -> RR = 1/3
+    y_pred = torch.tensor([[3.0, 2.0, 1.0]])
+    y_true = torch.tensor([[0.0, 0.0, 1.0]])
+    metric.update((y_pred, y_true))
+    res = metric.compute()
+    np.testing.assert_allclose(res, [1 / 3], rtol=1e-5)
+
+
+def test_first_hit_beats_later_hits():
+    """MRR only uses the first relevant item, not subsequent ones."""
+    metric = MeanReciprocalRank(top_k=[4])
+    # scores desc: index 0 (rank 1, relevant), index 1 (rank 2, relevant)
+    # MRR should only use rank 1 -> RR = 1.0
+    y_pred = torch.tensor([[4.0, 3.0, 2.0, 1.0]])
+    y_true = torch.tensor([[1.0, 1.0, 0.0, 0.0]])
+    metric.update((y_pred, y_true))
+    res = metric.compute()
+    np.testing.assert_allclose(res, [1.0], rtol=1e-5)
+
+
+@pytest.mark.usefixtures("distributed")
+class TestDistributed:
+    def test_integration(self):
+        n_iters = 10
+        batch_size = 4
+        num_items = 20
+        top_k = [1, 5]
+
+        rank = idist.get_rank()
+        torch.manual_seed(42 + rank)
+        device = idist.device()
+
+        metric_devices = [torch.device("cpu")]
+        if device.type != "xla":
+            metric_devices.append(device)
+
+        for metric_device in metric_devices:
+            all_y_true = torch.randint(0, 2, (n_iters * batch_size, num_items)).float().to(device)
+            all_y_pred = torch.randn((n_iters * batch_size, num_items)).to(device)
+
+            for ignore_zero_hits in [True, False]:
+                engine = Engine(
+                    lambda e, i: (
+                        all_y_pred[i * batch_size : (i + 1) * batch_size],
+                        all_y_true[i * batch_size : (i + 1) * batch_size],
+                    )
+                )
+
+                m = MeanReciprocalRank(
+                    top_k=top_k,
+                    ignore_zero_hits=ignore_zero_hits,
+                    device=metric_device,
+                )
+                m.attach(engine, "mrr")
+                engine.run(range(n_iters), max_epochs=1)
+
+                global_y_true = idist.all_gather(all_y_true).cpu().numpy()
+                global_y_pred = idist.all_gather(all_y_pred).cpu().numpy()
+
+                res = engine.state.metrics["mrr"]
+              
+                true_res = manual_mrr(
+                    global_y_pred,
+                    global_y_true,
+                    top_k,
+                    ignore_zero_hits=ignore_zero_hits,
+                )
+
+                assert isinstance(res, list)
+                assert res == pytest.approx(true_res)
+
+                engine.state.metrics.clear()
+
+    def test_accumulator_device(self):
+        device = idist.device()
+        metric = MeanReciprocalRank(top_k=[1, 5], device=device)
+
+        assert metric._device == device
+        assert metric._sum_mrr_per_k.device == device
+
+        y_pred = torch.randn(2, 10)
+        y = torch.zeros(2, 10)
+        metric.update((y_pred, y))
+
+        assert metric._sum_mrr_per_k.device == device
\ No newline at end of file