Summary  
This chapter covers methods for evaluating the performance of probabilistic classifiers both before and after applying calibration techniques, demonstrating how calibration (e.g., Platt scaling, isotonic regression) affects the reliability of predicted probabilities.

General domain of usage  
Machine learning classification

import unittest
import numpy as np
import importlib
import re
from sklearn.metrics import brier_score_loss

def _dynamic_test(test_case, condition, success, failure):
    if condition:
        test_case._testMethodName = success
        test_case.assertTrue(True)
    else:
        test_case._testMethodName = failure
        test_case.fail(failure)


class TestUserCode(unittest.TestCase):

    @classmethod
    def setUpClass(cls):
        cls.m = importlib.import_module("user_code")

    def test_probabilities_exist(self):
        yb = getattr(self.m, "y_prob_before", None)
        ya = getattr(self.m, "y_prob_after", None)
        cond = yb is not None and ya is not None and len(yb) == len(ya)
        _dynamic_test(
            self,
            cond,
            "Predicted probabilities before and after calibration exist.",
            "Expected y_prob_before and y_prob_after to be defined."
        )

    def test_brier_before_after(self):
        # Brier score must be computed as float values
        bb = getattr(self.m, "brier_before", None)
        ba = getattr(self.m, "brier_after", None)
        cond = isinstance(bb, float) and isinstance(ba, float)
        _dynamic_test(
            self,
            cond,
            "Brier scores are computed.",
            "Expected brier_before and brier_after to be float values."
        )

    def test_ece_before_after(self):
        eb = getattr(self.m, "ece_before", None)
        ea = getattr(self.m, "ece_after", None)
        cond = isinstance(eb, float) and isinstance(ea, float)
        _dynamic_test(
            self,
            cond,
            "ECE values are computed.",
            "Expected ece_before and ece_after to be float values."
        )

    def test_predict_proba_called(self):
        # Detect at least one usage of predict_proba
        with open("user_code.py") as f:
            code = f.read()
        cond = "predict_proba" in code
        _dynamic_test(
            self,
            cond,
            "predict_proba is used in the solution.",
            "Expected usage of predict_proba()."
        )


if __name__ == "__main__":
    unittest.main()

test_main.py

Master the principles and practical techniques for measuring, interpreting, and improving the probabilistic calibration of machine learning models. Learn to use reliability diagrams, calibration metrics, and modern calibration methods to ensure your models produce trustworthy probability estimates.

Explore the core concepts of model calibration, including the meaning of calibration, visual tools for assessment, and key quantitative metrics.

Apply and compare practical calibration techniques, and evaluate their effects on different models.

Explore real-world calibration scenarios, method selection, stability assessment, and end-to-end workflows.

Evaluation Before and After Calibration

Solution

Evaluation Before and After Calibration

Solution