diff --git a/docs/source/metrics.rst b/docs/source/metrics.rst
index 8b0560081a3e..b6f45488f008 100644
--- a/docs/source/metrics.rst
+++ b/docs/source/metrics.rst
@@ -360,6 +360,8 @@ Complete list of metrics
     FID
     CosineSimilarity
     Entropy
+    ExpectedCalibrationError
+    MaximumCalibrationError
     KLDivergence
     JSDivergence
     MatthewsCorrCoef
diff --git a/ignite/metrics/__init__.py b/ignite/metrics/__init__.py
index 0d1c968a24d1..e94c9cc211e7 100644
--- a/ignite/metrics/__init__.py
+++ b/ignite/metrics/__init__.py
@@ -5,6 +5,7 @@
 from ignite.metrics.accumulation import Average, GeometricAverage, VariableAccumulation
 from ignite.metrics.accuracy import Accuracy
 from ignite.metrics.average_precision import AveragePrecision
+from ignite.metrics.calibration_error import ExpectedCalibrationError, MaximumCalibrationError
 from ignite.metrics.classification_report import ClassificationReport
 from ignite.metrics.cohen_kappa import CohenKappa
 from ignite.metrics.confusion_matrix import ConfusionMatrix, DiceCoefficient, IoU, JaccardIndex, mIoU
@@ -69,6 +70,8 @@
     "Average",
     "DiceCoefficient",
     "Entropy",
+    "ExpectedCalibrationError",
+    "MaximumCalibrationError",
     "EpochMetric",
     "Fbeta",
     "FID",
diff --git a/ignite/metrics/calibration_error.py b/ignite/metrics/calibration_error.py
new file mode 100644
index 000000000000..dbffca81bf29
--- /dev/null
+++ b/ignite/metrics/calibration_error.py
@@ -0,0 +1,247 @@
+from collections.abc import Callable, Sequence
+
+import torch
+
+from ignite.exceptions import NotComputableError
+from ignite.metrics.metric import Metric, reinit__is_reduced, sync_all_reduce
+
+__all__ = ["ExpectedCalibrationError", "MaximumCalibrationError"]
+
+
+class _BaseCalibrationError(Metric):
+    """Base class accumulating the per-bin statistics shared by calibration metrics.
+
+    Predictions are grouped into ``num_bins`` equal-width confidence bins. For each bin only three
+    fixed-size aggregates are kept: the number of correct predictions, the sum of confidences and the
+    number of samples. This makes the memory cost ``O(num_bins)`` (independent of the number of
+    samples) and lets the final result be computed without any Python loop over bins.
+
+    - ``update`` must receive output of the form ``(y_pred, y)``.
+    - ``y_pred`` is expected to be **probabilities** (already normalized with softmax/sigmoid). For
+      multiclass inputs :math:`(B, C)` or :math:`(B, C, ...)` are allowed and the confidence of a
+      sample is its maximum class probability. For binary inputs :math:`(B,)` or :math:`(B, 1)`,
+      ``y_pred`` is the probability of the positive class and the confidence is
+      :math:`\\max(p, 1 - p)`. If your model outputs logits, apply an activation through
+      ``output_transform``.
+    - ``y`` holds the ground truth class indices (or 0/1 labels in the binary case).
+    """
+
+    _state_dict_all_req_keys = ("_bin_correct", "_bin_conf", "_bin_count")
+
+    def __init__(
+        self,
+        num_bins: int = 10,
+        output_transform: Callable = lambda x: x,
+        device: str | torch.device = torch.device("cpu"),
+        skip_unrolling: bool = False,
+    ):
+        if not isinstance(num_bins, int) or num_bins < 1:
+            raise ValueError(f"Argument num_bins must be a positive integer, got {num_bins}.")
+        self.num_bins = num_bins
+        super().__init__(output_transform=output_transform, device=device, skip_unrolling=skip_unrolling)
+
+    @reinit__is_reduced
+    def reset(self) -> None:
+        self._bin_correct = torch.zeros(self.num_bins, device=self._device)
+        self._bin_conf = torch.zeros(self.num_bins, device=self._device)
+        self._bin_count = torch.zeros(self.num_bins, device=self._device)
+
+    def _confidence_and_correct(self, y_pred: torch.Tensor, y: torch.Tensor) -> tuple[torch.Tensor, torch.Tensor]:
+        if y_pred.ndim == y.ndim + 1:
+            num_classes = y_pred.shape[1]
+            if num_classes >= 2:
+                # (B, C, ...) -> (B, ..., C) -> (B*..., C), regarding as B*... predictions
+                y_pred = y_pred.movedim(1, -1).reshape(-1, num_classes)
+                conf, pred = y_pred.max(dim=1)
+                y = y.reshape(-1)
+            else:
+                # (B, 1, ...) single positive-class probability -> binary
+                y_pred = y_pred.reshape(-1)
+                y = y.reshape(-1)
+                conf = torch.maximum(y_pred, 1.0 - y_pred)
+                pred = (y_pred >= 0.5).long()
+        elif y_pred.ndim == y.ndim:
+            # binary: y_pred is the probability of the positive class
+            y_pred = y_pred.reshape(-1)
+            y = y.reshape(-1)
+            conf = torch.maximum(y_pred, 1.0 - y_pred)
+            pred = (y_pred >= 0.5).long()
+        else:
+            raise ValueError(
+                "y_pred must have shape (B, C) or (B, C, ...) for multiclass inputs or (B,) or (B, 1) "
+                "for binary inputs, and be compatible with y; "
+                f"got y_pred shape {tuple(y_pred.shape)} and y shape {tuple(y.shape)}."
+            )
+
+        correct = pred.eq(y.view_as(pred)).to(self._device, dtype=self._bin_count.dtype)
+        conf = conf.to(self._device, dtype=self._bin_count.dtype)
+        return conf, correct
+
+    @reinit__is_reduced
+    def update(self, output: Sequence[torch.Tensor]) -> None:
+        y_pred, y = output[0].detach(), output[1].detach()
+        conf, correct = self._confidence_and_correct(y_pred, y)
+
+        bin_idx = torch.clamp((conf * self.num_bins).long(), 0, self.num_bins - 1)
+        self._bin_count.index_add_(0, bin_idx, torch.ones_like(conf))
+        self._bin_conf.index_add_(0, bin_idx, conf)
+        self._bin_correct.index_add_(0, bin_idx, correct)
+
+    def _accuracy_confidence_gap(self) -> torch.Tensor:
+        nonempty = self._bin_count > 0
+        acc = torch.zeros_like(self._bin_count)
+        conf = torch.zeros_like(self._bin_count)
+        acc[nonempty] = self._bin_correct[nonempty] / self._bin_count[nonempty]
+        conf[nonempty] = self._bin_conf[nonempty] / self._bin_count[nonempty]
+        return (acc - conf).abs()
+
+
+class ExpectedCalibrationError(_BaseCalibrationError):
+    r"""Calculates the `Expected Calibration Error (ECE)
+    <https://arxiv.org/abs/1706.04599>`_.
+
+    Predictions are grouped into :math:`M` equal-width confidence bins and the ECE is the weighted
+    average of the gap between accuracy and confidence within each bin:
+
+    .. math:: \text{ECE} = \sum_{m=1}^{M} \frac{|B_m|}{n} \left| \text{acc}(B_m) - \text{conf}(B_m) \right|
+
+    where :math:`B_m` is the set of samples whose confidence falls into the :math:`m`-th bin,
+    :math:`n` is the total number of samples, :math:`\text{acc}(B_m)` is the accuracy within the bin
+    and :math:`\text{conf}(B_m)` is the mean confidence within the bin. A perfectly calibrated model
+    has :math:`\text{ECE} = 0`.
+
+    - ``update`` must receive output of the form ``(y_pred, y)``.
+    - ``y_pred`` is expected to be **probabilities** (already normalized). Multiclass shapes
+      :math:`(B, C)` / :math:`(B, C, ...)` and binary shapes :math:`(B,)` / :math:`(B, 1)` are
+      supported. If your model outputs logits, apply an activation through ``output_transform``.
+
+    Args:
+        num_bins: number of equal-width confidence bins used to group the predictions. Default 10.
+        output_transform: a callable that is used to transform the
+            :class:`~ignite.engine.engine.Engine`'s ``process_function``'s output into the
+            form expected by the metric. This can be useful if, for example, you have a multi-output model and
+            you want to compute the metric with respect to one of the outputs.
+            By default, metrics require the output as ``(y_pred, y)`` or ``{'y_pred': y_pred, 'y': y}``.
+        device: specifies which device updates are accumulated on. Setting the
+            metric's device to be the same as your ``update`` arguments ensures the ``update`` method is
+            non-blocking. By default, CPU.
+        skip_unrolling: specifies whether output should be unrolled before being fed to update method. Should be
+            true for multi-output model, for example, if ``y_pred`` contains multi-output as ``(y_pred_a, y_pred_b)``
+            Alternatively, ``output_transform`` can be used to handle this.
+
+    Examples:
+        To use with ``Engine`` and ``process_function``, simply attach the metric instance to the engine.
+        The output of the engine's ``process_function`` needs to be in the format of
+        ``(y_pred, y)`` or ``{'y_pred': y_pred, 'y': y, ...}``. If not, ``output_transform`` can be added
+        to the metric to transform the output into the form expected by the metric.
+
+        For more information on how metric works with :class:`~ignite.engine.engine.Engine`, visit :ref:`attach-engine`.
+
+        .. include:: defaults.rst
+            :start-after: :orphan:
+
+        .. testcode::
+
+            metric = ExpectedCalibrationError(num_bins=5)
+            metric.attach(default_evaluator, 'ece')
+            y_true = torch.tensor([0, 0, 1, 0])
+            y_pred = torch.tensor([
+                [0.9, 0.1],
+                [0.9, 0.1],
+                [0.1, 0.9],
+                [0.3, 0.7],
+            ])
+            state = default_evaluator.run([[y_pred, y_true]])
+            print(state.metrics['ece'])
+
+        .. testoutput::
+
+            0.2500000...
+
+    .. versionadded:: 0.6.0
+    """
+
+    @sync_all_reduce("_bin_correct", "_bin_conf", "_bin_count")
+    def compute(self) -> float:
+        n = self._bin_count.sum()
+        if n == 0:
+            raise NotComputableError(
+                "ExpectedCalibrationError must have at least one example before it can be computed."
+            )
+        gap = self._accuracy_confidence_gap()
+        return ((self._bin_count / n) * gap).sum().item()
+
+
+class MaximumCalibrationError(_BaseCalibrationError):
+    r"""Calculates the `Maximum Calibration Error (MCE)
+    <https://arxiv.org/abs/1706.04599>`_.
+
+    Predictions are grouped into :math:`M` equal-width confidence bins and the MCE is the largest gap
+    between accuracy and confidence across the (non-empty) bins:
+
+    .. math:: \text{MCE} = \max_{m \in \{1, ..., M\}} \left| \text{acc}(B_m) - \text{conf}(B_m) \right|
+
+    where :math:`B_m` is the set of samples whose confidence falls into the :math:`m`-th bin,
+    :math:`\text{acc}(B_m)` is the accuracy within the bin and :math:`\text{conf}(B_m)` is the mean
+    confidence within the bin. Unlike :class:`~ignite.metrics.ExpectedCalibrationError`, the MCE is not
+    weighted by bin size and so reflects the worst-case miscalibration.
+
+    - ``update`` must receive output of the form ``(y_pred, y)``.
+    - ``y_pred`` is expected to be **probabilities** (already normalized). Multiclass shapes
+      :math:`(B, C)` / :math:`(B, C, ...)` and binary shapes :math:`(B,)` / :math:`(B, 1)` are
+      supported. If your model outputs logits, apply an activation through ``output_transform``.
+
+    Args:
+        num_bins: number of equal-width confidence bins used to group the predictions. Default 10.
+        output_transform: a callable that is used to transform the
+            :class:`~ignite.engine.engine.Engine`'s ``process_function``'s output into the
+            form expected by the metric. This can be useful if, for example, you have a multi-output model and
+            you want to compute the metric with respect to one of the outputs.
+            By default, metrics require the output as ``(y_pred, y)`` or ``{'y_pred': y_pred, 'y': y}``.
+        device: specifies which device updates are accumulated on. Setting the
+            metric's device to be the same as your ``update`` arguments ensures the ``update`` method is
+            non-blocking. By default, CPU.
+        skip_unrolling: specifies whether output should be unrolled before being fed to update method. Should be
+            true for multi-output model, for example, if ``y_pred`` contains multi-output as ``(y_pred_a, y_pred_b)``
+            Alternatively, ``output_transform`` can be used to handle this.
+
+    Examples:
+        To use with ``Engine`` and ``process_function``, simply attach the metric instance to the engine.
+        The output of the engine's ``process_function`` needs to be in the format of
+        ``(y_pred, y)`` or ``{'y_pred': y_pred, 'y': y, ...}``. If not, ``output_tranform`` can be added
+        to the metric to transform the output into the form expected by the metric.
+
+        For more information on how metric works with :class:`~ignite.engine.engine.Engine`, visit :ref:`attach-engine`.
+
+        .. include:: defaults.rst
+            :start-after: :orphan:
+
+        .. testcode::
+
+            metric = MaximumCalibrationError(num_bins=5)
+            metric.attach(default_evaluator, 'mce')
+            y_true = torch.tensor([0, 0, 1, 0])
+            y_pred = torch.tensor([
+                [0.9, 0.1],
+                [0.9, 0.1],
+                [0.1, 0.9],
+                [0.3, 0.7],
+            ])
+            state = default_evaluator.run([[y_pred, y_true]])
+            print(state.metrics['mce'])
+
+        .. testoutput::
+
+            0.6999999...
+
+    .. versionadded:: 0.6.0
+    """
+
+    @sync_all_reduce("_bin_correct", "_bin_conf", "_bin_count")
+    def compute(self) -> float:
+        if self._bin_count.sum() == 0:
+            raise NotComputableError(
+                "MaximumCalibrationError must have at least one example before it can be computed."
+            )
+        nonempty = self._bin_count > 0
+        return self._accuracy_confidence_gap()[nonempty].max().item()
diff --git a/tests/ignite/metrics/test_calibration_error.py b/tests/ignite/metrics/test_calibration_error.py
new file mode 100644
index 000000000000..6345ae7e1cbf
--- /dev/null
+++ b/tests/ignite/metrics/test_calibration_error.py
@@ -0,0 +1,228 @@
+import numpy as np
+import pytest
+import torch
+from scipy.special import softmax
+
+import ignite.distributed as idist
+
+from ignite.engine import Engine
+from ignite.exceptions import NotComputableError
+from ignite.metrics import ExpectedCalibrationError, MaximumCalibrationError
+
+
+def _bin_stats(conf: np.ndarray, correct: np.ndarray, num_bins: int):
+    """Reference per-bin aggregation mirroring the Baal binning (floor(conf * num_bins))."""
+    bin_idx = np.clip((conf * num_bins).astype(np.int64), 0, num_bins - 1)
+    count = np.zeros(num_bins)
+    conf_sum = np.zeros(num_bins)
+    correct_sum = np.zeros(num_bins)
+    np.add.at(count, bin_idx, 1.0)
+    np.add.at(conf_sum, bin_idx, conf)
+    np.add.at(correct_sum, bin_idx, correct.astype(np.float64))
+    nonempty = count > 0
+    acc = np.zeros(num_bins)
+    avg_conf = np.zeros(num_bins)
+    acc[nonempty] = correct_sum[nonempty] / count[nonempty]
+    avg_conf[nonempty] = conf_sum[nonempty] / count[nonempty]
+    return count, np.abs(acc - avg_conf), nonempty
+
+
+def np_ece(conf: np.ndarray, correct: np.ndarray, num_bins: int) -> float:
+    count, gap, _ = _bin_stats(conf, correct, num_bins)
+    return float((count / count.sum() * gap).sum())
+
+
+def np_mce(conf: np.ndarray, correct: np.ndarray, num_bins: int) -> float:
+    count, gap, nonempty = _bin_stats(conf, correct, num_bins)
+    return float(gap[nonempty].max())
+
+
+def _conf_correct(np_y_pred: np.ndarray, np_y: np.ndarray):
+    """Extract per-sample confidence and correctness from probabilities, matching the metric."""
+    if np_y_pred.ndim == np_y.ndim + 1 and np_y_pred.shape[1] >= 2:
+        probs = np_y_pred.transpose(0, *range(2, np_y_pred.ndim), 1).reshape(-1, np_y_pred.shape[1])
+        conf = probs.max(axis=1)
+        pred = probs.argmax(axis=1)
+        y = np_y.reshape(-1)
+    else:
+        p = np_y_pred.reshape(-1)
+        conf = np.maximum(p, 1.0 - p)
+        pred = (p >= 0.5).astype(np.int64)
+        y = np_y.reshape(-1)
+    return conf, (pred == y)
+
+
+def test_zero_sample():
+    for cls, name in [
+        (ExpectedCalibrationError, "ExpectedCalibrationError"),
+        (MaximumCalibrationError, "MaximumCalibrationError"),
+    ]:
+        metric = cls()
+        with pytest.raises(
+            NotComputableError, match=rf"{name} must have at least one example before it can be computed"
+        ):
+            metric.compute()
+
+
+@pytest.mark.parametrize("num_bins", [0, -3, 2.5, "10"])
+def test_invalid_num_bins(num_bins):
+    with pytest.raises(ValueError, match=r"Argument num_bins must be a positive integer"):
+        ExpectedCalibrationError(num_bins=num_bins)
+
+
+def test_invalid_shape():
+    metric = ExpectedCalibrationError()
+    y_pred = torch.rand(4, 3, 2)
+    y = torch.randint(0, 3, size=(4,))
+    with pytest.raises(ValueError, match=r"y_pred must have shape \(B, C\) or \(B, C, ...\)"):
+        metric.update((y_pred, y))
+
+
+@pytest.fixture(params=range(6))
+def test_case(request):
+    return [
+        # multiclass (N, C), single update
+        (softmax(torch.randn(100, 10).numpy(), axis=1), torch.randint(0, 10, size=[100]).numpy(), 1),
+        (softmax(torch.randn(100, 4).numpy(), axis=1), torch.randint(0, 4, size=[100]).numpy(), 1),
+        # multiclass, batched updates
+        (softmax(torch.randn(100, 10).numpy(), axis=1), torch.randint(0, 10, size=[100]).numpy(), 16),
+        # binary (N,)
+        (torch.rand(100).numpy(), torch.randint(0, 2, size=[100]).numpy(), 1),
+        (torch.rand(100).numpy(), torch.randint(0, 2, size=[100]).numpy(), 16),
+        # multiclass image segmentation (N, C, H, W)
+        (softmax(torch.randn(50, 5, 8, 8).numpy(), axis=1), torch.randint(0, 5, size=(50, 8, 8)).numpy(), 16),
+    ][request.param]
+
+
+@pytest.mark.parametrize("metric_cls, np_ref", [(ExpectedCalibrationError, np_ece), (MaximumCalibrationError, np_mce)])
+@pytest.mark.parametrize("num_bins", [5, 10, 15])
+def test_compute(metric_cls, np_ref, num_bins, test_case, available_device):
+    np_y_pred, np_y, batch_size = test_case
+    y_pred = torch.tensor(np_y_pred)
+    y = torch.tensor(np_y)
+
+    metric = metric_cls(num_bins=num_bins, device=available_device)
+    assert metric._device == torch.device(available_device)
+
+    metric.reset()
+    if batch_size > 1:
+        n_iters = y.shape[0] // batch_size + 1
+        for i in range(n_iters):
+            idx = i * batch_size
+            metric.update((y_pred[idx : idx + batch_size], y[idx : idx + batch_size]))
+    else:
+        metric.update((y_pred, y))
+
+    conf, correct = _conf_correct(np_y_pred, np_y)
+    expected = np_ref(conf, correct, num_bins)
+
+    res = metric.compute()
+    assert isinstance(res, float)
+    assert res == pytest.approx(expected, abs=1e-5)
+
+
+def test_calibration_extremes(available_device):
+    # Confidently correct -> perfect calibration -> ECE = MCE = 0
+    y_pred = torch.tensor([[0.0, 1.0], [0.0, 1.0]])
+    y = torch.tensor([1, 1])
+    ece = ExpectedCalibrationError(device=available_device)
+    ece.update((y_pred, y))
+    assert ece.compute() == pytest.approx(0.0)
+
+    # Confidently wrong -> worst calibration -> ECE = MCE = 1
+    y_wrong = torch.tensor([0, 0])
+    mce = MaximumCalibrationError(device=available_device)
+    mce.update((y_pred, y_wrong))
+    assert mce.compute() == pytest.approx(1.0)
+
+
+def test_batched_vs_single_shot(available_device):
+    torch.manual_seed(0)
+    y_pred = softmax(torch.randn(120, 6).numpy(), axis=1)
+    y = torch.randint(0, 6, size=[120])
+
+    single = ExpectedCalibrationError(num_bins=10, device=available_device)
+    single.update((torch.tensor(y_pred), y))
+
+    batched = ExpectedCalibrationError(num_bins=10, device=available_device)
+    for i in range(0, 120, 16):
+        batched.update((torch.tensor(y_pred[i : i + 16]), y[i : i + 16]))
+
+    assert single.compute() == pytest.approx(batched.compute(), abs=1e-6)
+
+
+def test_accumulator_detached(available_device):
+    metric = ExpectedCalibrationError(device=available_device)
+    y_pred = torch.tensor([[0.9, 0.1], [0.3, 0.7]], requires_grad=True)
+    y = torch.tensor([0, 1])
+    metric.update((y_pred, y))
+
+    assert not metric._bin_conf.requires_grad
+    assert not metric._bin_correct.requires_grad
+    assert not metric._bin_count.requires_grad
+
+
+@pytest.mark.usefixtures("distributed")
+class TestDistributed:
+    @pytest.mark.parametrize(
+        "metric_cls, np_ref", [(ExpectedCalibrationError, np_ece), (MaximumCalibrationError, np_mce)]
+    )
+    def test_integration(self, metric_cls, np_ref):
+        tol = 1e-5
+        num_bins = 10
+        device = idist.device()
+        rank = idist.get_rank()
+        torch.manual_seed(12 + rank)
+
+        n_iters = 80
+        batch_size = 16
+        n_cls = 10
+
+        metric_devices = [torch.device("cpu")]
+        if device.type != "xla":
+            metric_devices.append(idist.device())
+
+        for metric_device in metric_devices:
+            y_true = torch.randint(0, n_cls, size=[n_iters * batch_size], dtype=torch.long).to(device)
+            y_preds = torch.softmax(torch.randn(n_iters * batch_size, n_cls), dim=1).to(device)
+
+            def update(engine, i):
+                return (
+                    y_preds[i * batch_size : (i + 1) * batch_size],
+                    y_true[i * batch_size : (i + 1) * batch_size],
+                )
+
+            engine = Engine(update)
+            m = metric_cls(num_bins=num_bins, device=metric_device)
+            m.attach(engine, "cal_err")
+
+            engine.run(data=list(range(n_iters)), max_epochs=1)
+
+            y_preds = idist.all_gather(y_preds)
+            y_true = idist.all_gather(y_true)
+
+            assert "cal_err" in engine.state.metrics
+            res = engine.state.metrics["cal_err"]
+
+            conf, correct = _conf_correct(y_preds.cpu().numpy(), y_true.cpu().numpy())
+            true_res = np_ref(conf, correct, num_bins)
+
+            assert res == pytest.approx(true_res, abs=tol)
+
+    def test_accumulator_device(self):
+        device = idist.device()
+        metric_devices = [torch.device("cpu")]
+        if device.type != "xla":
+            metric_devices.append(idist.device())
+
+        for metric_device in metric_devices:
+            metric = ExpectedCalibrationError(device=metric_device)
+            for dev in [metric._device, metric._bin_count.device, metric._bin_conf.device, metric._bin_correct.device]:
+                assert dev == metric_device, f"{type(dev)}:{dev} vs {type(metric_device)}:{metric_device}"
+
+            y_pred = torch.tensor([[0.9, 0.1], [0.2, 0.8]]).to(device)
+            y = torch.tensor([0, 1]).to(device)
+            metric.update((y_pred, y))
+
+            for dev in [metric._device, metric._bin_count.device, metric._bin_conf.device, metric._bin_correct.device]:
+                assert dev == metric_device, f"{type(dev)}:{dev} vs {type(metric_device)}:{metric_device}"