Source code for caliber.multiclass_classification.metrics.ece

import numpy as np
from sklearn.metrics import accuracy_score


[docs] def expected_calibration_error( targets: np.ndarray, probs: np.ndarray, n_bins: int = 10, min_prob_bin: float = 0.0 ) -> float: preds = np.argmax(probs, axis=1) confs = np.max(probs, axis=1) bin_edges = np.linspace(0, 1, n_bins + 1) bin_indices = np.digitize(confs, bin_edges) ece = 0 for i in range(1, n_bins + 1): mask = bin_indices == i prob_bin = np.mean(mask) if prob_bin > min_prob_bin: _preds = preds[mask] _confs = confs[mask] _targets = targets[mask] acc = accuracy_score(_targets, _preds) avg_conf = np.mean(_confs) ece += prob_bin * np.abs(acc - avg_conf) return ece