From ec6127676f48eff9e3b104be0752cffa55b1b407 Mon Sep 17 00:00:00 2001 From: knutdrand Date: Wed, 9 Sep 2026 14:56:20 +0200 Subject: [PATCH] feat: add binary classification and proper-scoring alert metrics Seven metrics scoring the alert decision and the probability behind it: precision, false alarm rate, F1 and Matthews correlation on the decision; Brier, log score and Brier skill on the probability. Brier skill against the climatological base rate is the number worth leading with, since a raw score flatters any model that rarely alerts when outbreaks are rare. Three of them are ratios of aggregates with no per-cell value, so they needed a base class the metric system did not have. GlobalOnlyMetric reports a single value and raises when asked for a breakdown, rather than returning a number that looks per-cell and is not. The plan said this set would fit the existing contract; that was wrong for F1, Matthews and skill, and deferring all three would have left only the metrics nobody should lead with. AUROC now needs no new machinery when we want it. optimization_direction is set only where both error types are priced. Precision and false alarm rate join sensitivity and specificity in leaving it unset: precision is maximised by alerting once and only when certain, false alarm rate by never alerting at all. The plot-metric picker now lists only metrics with a per-cell value. Every metric plot breaks a score down by horizon, location or period, so offering a global-only metric rendered an error instead of a chart. --- chap_core/assessment/metrics/__init__.py | 25 +++ chap_core/assessment/metrics/base.py | 38 +++++ .../metrics/outbreak_classification.py | 143 ++++++++++++++++++ .../assessment/metrics/outbreak_detection.py | 36 +++-- .../metrics/outbreak_probability.py | 116 ++++++++++++++ .../rest_api/v1/routers/visualization.py | 6 + tests/evaluation/conftest.py | 30 ++++ .../evaluation/test_outbreak_alert_metrics.py | 104 +++++++++++++ 8 files changed, 484 insertions(+), 14 deletions(-) create mode 100644 chap_core/assessment/metrics/outbreak_classification.py create mode 100644 chap_core/assessment/metrics/outbreak_probability.py create mode 100644 tests/evaluation/test_outbreak_alert_metrics.py diff --git a/chap_core/assessment/metrics/__init__.py b/chap_core/assessment/metrics/__init__.py index e0b947c00..4c8318ecd 100644 --- a/chap_core/assessment/metrics/__init__.py +++ b/chap_core/assessment/metrics/__init__.py @@ -19,6 +19,7 @@ DEFAULT_OUTPUT_DIMENSIONS, AggregationOp, DeterministicMetric, + GlobalOnlyMetric, Metric, MetricSpec, OptimizationDirection, @@ -86,7 +87,9 @@ def _discover_metrics(): example_metric, mae, mape, + outbreak_classification, outbreak_detection, + outbreak_probability, percentile_coverage, rmse, test_metrics, @@ -104,11 +107,22 @@ def _discover_metrics(): from chap_core.assessment.metrics.example_metric import ExampleMetric from chap_core.assessment.metrics.mae import MAEMetric from chap_core.assessment.metrics.mape import MAPEMetric +from chap_core.assessment.metrics.outbreak_classification import ( + FalseAlarmRateMetric, + MatthewsCorrelationMetric, + OutbreakF1Metric, + OutbreakPrecisionMetric, +) from chap_core.assessment.metrics.outbreak_detection import ( OutbreakAccuracyMetric, SensitivityMetric, SpecificityMetric, ) +from chap_core.assessment.metrics.outbreak_probability import ( + BrierScoreMetric, + BrierSkillScoreMetric, + OutbreakLogScoreMetric, +) from chap_core.assessment.metrics.peak_diff import PeakPeriodLagMetric, PeakValueDiffMetric from chap_core.assessment.metrics.percentile_coverage import ( Coverage10_90Metric, @@ -132,6 +146,8 @@ def _discover_metrics(): __all__ = [ "DEFAULT_OUTPUT_DIMENSIONS", "AggregationOp", + "BrierScoreMetric", + "BrierSkillScoreMetric", "CRPSLog1pMetric", "CRPSMetric", "CRPSNormMetric", @@ -140,11 +156,17 @@ def _discover_metrics(): "DataDimension", "DeterministicMetric", "ExampleMetric", + "FalseAlarmRateMetric", + "GlobalOnlyMetric", "MAEMetric", "MAPEMetric", + "MatthewsCorrelationMetric", "Metric", "MetricSpec", "OutbreakAccuracyMetric", + "OutbreakF1Metric", + "OutbreakLogScoreMetric", + "OutbreakPrecisionMetric", "PeakPeriodLagMetric", "PeakValueDiffMetric", "PercentileCoverageMetric", @@ -226,6 +248,9 @@ def compute_all_detailed_metrics(evaluation: Evaluation) -> pd.DataFrame: metric = metric_factory(historical_observations=historical_df) if not metric.is_applicable(flat_data.observations): continue + if not metric.spec.output_dimensions: + # Global-only metric (F1, skill scores): no per-cell value to export. + continue try: detailed = metric.get_detailed_metric(flat_data.observations, flat_data.forecasts) except Exception: diff --git a/chap_core/assessment/metrics/base.py b/chap_core/assessment/metrics/base.py index e54d76a87..37d233de6 100644 --- a/chap_core/assessment/metrics/base.py +++ b/chap_core/assessment/metrics/base.py @@ -235,6 +235,44 @@ def get_description(self) -> str: return self.spec.description +class GlobalOnlyMetric(Metric): + """Base for metrics that cannot be computed per cell and then averaged. + + F1, Matthews correlation and skill scores are ratios of aggregates: there is + no per-cell value whose mean recovers the metric. Such a metric is defined + only over a whole set of scored cells, so it reports one global value and + refuses to be broken down by dimension rather than returning a number that + looks per-cell but is not. + + Subclasses implement :meth:`compute_global` and declare + ``output_dimensions=()`` in their spec. + """ + + def get_metric( + self, + observations: pa.typing.DataFrame[FlatObserved], + forecasts: pa.typing.DataFrame[FlatForecasts], + dimensions: tuple[DataDimension, ...] = (), + ) -> pd.DataFrame: + if dimensions: + named = ", ".join(d.value for d in dimensions) + raise ValueError( + f"{type(self).__name__} is defined over a whole set of scored cells, so it cannot be broken down by {named}." + ) + null_mask = observations.disease_cases.isnull() # type: ignore[attr-defined] + observations = observations[~null_mask] # type: ignore[index] + value = self.compute_global(observations, forecasts) # type: ignore[arg-type] + return self._validate_output(pd.DataFrame({"metric": [value]}), ()) + + @abstractmethod + def compute_global(self, observations: pd.DataFrame, forecasts: pd.DataFrame) -> float: + """Compute the metric over every scored cell at once.""" + raise NotImplementedError + + def compute_detailed(self, observations: pd.DataFrame, forecasts: pd.DataFrame) -> pd.DataFrame: + raise NotImplementedError(f"{type(self).__name__} has no per-cell value.") + + class DeterministicMetric(Metric): """ Base class for deterministic metrics that operate on the median of samples. diff --git a/chap_core/assessment/metrics/outbreak_classification.py b/chap_core/assessment/metrics/outbreak_classification.py new file mode 100644 index 000000000..60a73b750 --- /dev/null +++ b/chap_core/assessment/metrics/outbreak_classification.py @@ -0,0 +1,143 @@ +"""Binary classification metrics for outbreak alerts. + +These score the alert decision, after the exceedance probability has been cut at +:data:`~chap_core.assessment.metrics.outbreak_detection.ALERT_SAMPLE_FRACTION`. + +Only F1 and Matthews correlation carry an ``optimization_direction``. Precision +and false-alarm rate, like sensitivity and specificity before them, are each +maximised by a degenerate model -- precision by alerting once and only when +certain, false-alarm rate by never alerting at all -- so neither is valid as a +standalone objective even though both are informative alongside the others. +""" + +from __future__ import annotations + +from typing import TYPE_CHECKING + +import numpy as np + +from chap_core.assessment.metrics import metric +from chap_core.assessment.metrics.base import ( + AggregationOp, + GlobalOnlyMetric, + Metric, + MetricSpec, + OptimizationDirection, +) +from chap_core.assessment.metrics.outbreak_detection import OutbreakScoredMixin, _as_metric + +if TYPE_CHECKING: + import pandas as pd + + +def _confusion(frame: pd.DataFrame) -> tuple[int, int, int, int]: + """Counts of (true positive, false positive, true negative, false negative).""" + alert = frame["alert"] == 1.0 + outbreak = frame["outbreak"] == 1.0 + return ( + int((alert & outbreak).sum()), + int((alert & ~outbreak).sum()), + int((~alert & ~outbreak).sum()), + int((~alert & outbreak).sum()), + ) + + +@metric() +class OutbreakPrecisionMetric(OutbreakScoredMixin, Metric): + """Precision: the share of raised alerts that turned out to be real outbreaks. + + The counterpart to sensitivity. A pipeline that alerts constantly scores well + on sensitivity and badly here; one that alerts almost never does the reverse. + """ + + spec = MetricSpec( + metric_id="outbreak_precision", + metric_name="Outbreak Precision", + aggregation_op=AggregationOp.MEAN, + description="Share of raised alerts that were real outbreaks", + ) + + def compute_detailed(self, observations: pd.DataFrame, forecasts: pd.DataFrame) -> pd.DataFrame: + frame = self.outbreak_frame(observations, forecasts) + alerted = frame[frame["alert"] == 1.0] + return _as_metric(alerted, alerted["outbreak"]) + + +@metric() +class FalseAlarmRateMetric(OutbreakScoredMixin, Metric): + """False-alarm rate: the share of quiet periods that were alerted anyway. + + One minus specificity, reported directly because it is the number an + operational team feels -- how often a warning turns out to be nothing. + """ + + spec = MetricSpec( + metric_id="false_alarm_rate", + metric_name="False Alarm Rate", + aggregation_op=AggregationOp.MEAN, + description="Share of non-outbreak periods that raised an alert", + ) + + def compute_detailed(self, observations: pd.DataFrame, forecasts: pd.DataFrame) -> pd.DataFrame: + frame = self.outbreak_frame(observations, forecasts) + quiet = frame[frame["outbreak"] == 0.0] + return _as_metric(quiet, quiet["alert"]) + + +@metric() +class OutbreakF1Metric(OutbreakScoredMixin, GlobalOnlyMetric): + """F1: the harmonic mean of precision and sensitivity. + + A ratio of aggregates, so it has no per-cell value. Unlike its two parts it + prices both error types, which makes it usable as an objective. + """ + + spec = MetricSpec( + metric_id="outbreak_f1", + metric_name="Outbreak F1", + output_dimensions=(), + aggregation_op=AggregationOp.MEAN, + description="Harmonic mean of outbreak precision and sensitivity", + optimization_direction=OptimizationDirection.MAXIMIZE, + ) + + def compute_global(self, observations: pd.DataFrame, forecasts: pd.DataFrame) -> float: + true_positive, false_positive, _, false_negative = _confusion(self.outbreak_frame(observations, forecasts)) + denominator = 2 * true_positive + false_positive + false_negative + if denominator == 0: + return float("nan") + return 2 * true_positive / denominator + + +@metric() +class MatthewsCorrelationMetric(OutbreakScoredMixin, GlobalOnlyMetric): + """Matthews correlation between alerts and outbreaks, in [-1, 1]. + + Uses all four cells of the confusion matrix, so unlike F1 it does not ignore + true negatives -- worth having when outbreaks are rare and quiet periods + dominate. Zero means no better than chance. + """ + + spec = MetricSpec( + metric_id="outbreak_mcc", + metric_name="Outbreak Matthews Correlation", + output_dimensions=(), + aggregation_op=AggregationOp.MEAN, + description="Correlation between alert and outbreak status across the full confusion matrix", + optimization_direction=OptimizationDirection.MAXIMIZE, + ) + + def compute_global(self, observations: pd.DataFrame, forecasts: pd.DataFrame) -> float: + true_positive, false_positive, true_negative, false_negative = _confusion( + self.outbreak_frame(observations, forecasts) + ) + denominator = np.sqrt( + float(true_positive + false_positive) + * float(true_positive + false_negative) + * float(true_negative + false_positive) + * float(true_negative + false_negative) + ) + if denominator == 0: + # A row or column of the matrix is empty, so correlation is undefined. + return float("nan") + return float((true_positive * true_negative - false_positive * false_negative) / denominator) diff --git a/chap_core/assessment/metrics/outbreak_detection.py b/chap_core/assessment/metrics/outbreak_detection.py index 17b970f64..63b331da1 100644 --- a/chap_core/assessment/metrics/outbreak_detection.py +++ b/chap_core/assessment/metrics/outbreak_detection.py @@ -35,7 +35,7 @@ #: Fraction of forecast samples that must exceed the threshold for an alert to be raised. ALERT_SAMPLE_FRACTION = 0.5 -_OUTBREAK_COLUMNS = ["location", "time_period", "horizon_distance", "outbreak", "alert"] +_OUTBREAK_COLUMNS = ["location", "time_period", "horizon_distance", "outbreak", "alert", "probability"] _METRIC_DIMENSIONS = ["location", "time_period", "horizon_distance"] @@ -75,9 +75,10 @@ def outbreak_and_alert( Returns: One row per ``(location, time_period, horizon_distance)`` that has both a computable threshold and a forecast, with columns - ``[location, time_period, horizon_distance, outbreak, alert]``. ``outbreak`` - is 1.0 where observed cases exceed the channel; ``alert`` is 1.0 where more - than :data:`ALERT_SAMPLE_FRACTION` of the samples exceed it. Cells whose + ``[location, time_period, horizon_distance, outbreak, alert, probability]``. + ``outbreak`` is 1.0 where observed cases exceed the channel; ``probability`` + is the share of samples above it and ``alert`` is 1.0 where that share + exceeds :data:`ALERT_SAMPLE_FRACTION`. Cells whose threshold cannot be computed (a single historical value gives an undefined standard deviation) are dropped. """ @@ -108,7 +109,7 @@ def outbreak_and_alert( labelled["outbreak"] = (labelled["disease_cases"] > labelled["threshold"]).astype(float) merged = labelled[["location", "time_period", "outbreak"]].merge( - probabilities[[*_METRIC_DIMENSIONS, "alert"]], + probabilities[[*_METRIC_DIMENSIONS, "alert", "probability"]], on=["location", "time_period"], how="inner", ) @@ -122,18 +123,25 @@ def _as_metric(rows: pd.DataFrame, values: pd.Series) -> pd.DataFrame: return result -class _OutbreakMetric(Metric): - """Shared applicability rule for metrics scored against a seasonal threshold.""" +class OutbreakScoredMixin: + """Applicability rule and frame access shared by every metric scored against the channel. + + Mixed in ahead of :class:`Metric` or :class:`GlobalOnlyMetric` depending on + whether the metric has a per-cell value. + """ + + historical_observations: pd.DataFrame | None def is_applicable(self, observations: pa.typing.DataFrame[FlatObserved]) -> bool: return self.historical_observations is not None and has_season_buckets(observations) - def _frame(self, observations: pd.DataFrame, forecasts: pd.DataFrame) -> pd.DataFrame: + def outbreak_frame(self, observations: pd.DataFrame, forecasts: pd.DataFrame) -> pd.DataFrame: + """Labelled and scored cells: see :func:`outbreak_and_alert`.""" return outbreak_and_alert(self.historical_observations, observations, forecasts) @metric() -class SensitivityMetric(_OutbreakMetric): +class SensitivityMetric(OutbreakScoredMixin, Metric): """Sensitivity (true positive rate) for outbreak detection. Measures the proportion of actual outbreaks that were correctly @@ -151,13 +159,13 @@ class SensitivityMetric(_OutbreakMetric): ) def compute_detailed(self, observations: pd.DataFrame, forecasts: pd.DataFrame) -> pd.DataFrame: - frame = self._frame(observations, forecasts) + frame = self.outbreak_frame(observations, forecasts) outbreaks = frame[frame["outbreak"] == 1.0] return _as_metric(outbreaks, outbreaks["alert"]) @metric() -class SpecificityMetric(_OutbreakMetric): +class SpecificityMetric(OutbreakScoredMixin, Metric): """Specificity (true negative rate) for outbreak detection. Measures the proportion of non-outbreak periods that were correctly @@ -175,13 +183,13 @@ class SpecificityMetric(_OutbreakMetric): ) def compute_detailed(self, observations: pd.DataFrame, forecasts: pd.DataFrame) -> pd.DataFrame: - frame = self._frame(observations, forecasts) + frame = self.outbreak_frame(observations, forecasts) quiet = frame[frame["outbreak"] == 0.0] return _as_metric(quiet, 1.0 - quiet["alert"]) @metric() -class OutbreakAccuracyMetric(_OutbreakMetric): +class OutbreakAccuracyMetric(OutbreakScoredMixin, Metric): """Accuracy for outbreak detection. Measures the proportion of all periods where the alert status @@ -200,5 +208,5 @@ class OutbreakAccuracyMetric(_OutbreakMetric): ) def compute_detailed(self, observations: pd.DataFrame, forecasts: pd.DataFrame) -> pd.DataFrame: - frame = self._frame(observations, forecasts) + frame = self.outbreak_frame(observations, forecasts) return _as_metric(frame, (frame["alert"] == frame["outbreak"]).astype(float)) diff --git a/chap_core/assessment/metrics/outbreak_probability.py b/chap_core/assessment/metrics/outbreak_probability.py new file mode 100644 index 000000000..5517f807d --- /dev/null +++ b/chap_core/assessment/metrics/outbreak_probability.py @@ -0,0 +1,116 @@ +"""Proper scoring rules for outbreak alert probabilities. + +These score the exceedance probability itself rather than the alert decision, so +a model that says 0.6 is treated differently from one that says 0.95. Because the +channel that labels an outbreak is the same one the forecast is scored against, +the probability is a probability of exactly the labelled event -- which is what +makes these scores proper rather than merely numeric. + +Brier skill is the number worth leading with. Outbreaks are rare, so a raw score +looks excellent for a model that never alerts; skill prices it against the +climatological base rate instead of against nothing. +""" + +from __future__ import annotations + +from typing import TYPE_CHECKING + +import numpy as np + +from chap_core.assessment.metrics import metric +from chap_core.assessment.metrics.base import ( + AggregationOp, + GlobalOnlyMetric, + Metric, + MetricSpec, + OptimizationDirection, +) +from chap_core.assessment.metrics.outbreak_detection import OutbreakScoredMixin, _as_metric + +if TYPE_CHECKING: + import pandas as pd + +#: Probabilities are clipped this far from 0 and 1 before taking a logarithm, so a +#: confident miss costs a large finite amount rather than infinity. +LOG_SCORE_CLIP = 1e-6 + + +@metric() +class BrierScoreMetric(OutbreakScoredMixin, Metric): + """Brier score: the mean squared error of the alert probability. + + Zero is perfect, one is a confidently wrong forecast every time. Decomposes + per cell, so it can be reported per location, period or horizon. + """ + + spec = MetricSpec( + metric_id="outbreak_brier", + metric_name="Outbreak Brier Score", + aggregation_op=AggregationOp.MEAN, + description="Mean squared error of the outbreak alert probability", + optimization_direction=OptimizationDirection.MINIMIZE, + ) + + def compute_detailed(self, observations: pd.DataFrame, forecasts: pd.DataFrame) -> pd.DataFrame: + frame = self.outbreak_frame(observations, forecasts) + return _as_metric(frame, (frame["probability"] - frame["outbreak"]) ** 2) + + +@metric() +class OutbreakLogScoreMetric(OutbreakScoredMixin, Metric): + """Logarithmic score of the alert probability, clipped at :data:`LOG_SCORE_CLIP`. + + Punishes confident misses far harder than Brier does, which is the point: it + is the score that notices a model asserting certainty it has not earned. The + clip is what keeps a single confident miss from being infinite, so read this + alongside Brier rather than instead of it. + """ + + spec = MetricSpec( + metric_id="outbreak_log_score", + metric_name="Outbreak Log Score", + aggregation_op=AggregationOp.MEAN, + description="Negative log likelihood of the observed outbreak status under the alert probability", + optimization_direction=OptimizationDirection.MINIMIZE, + ) + + def compute_detailed(self, observations: pd.DataFrame, forecasts: pd.DataFrame) -> pd.DataFrame: + frame = self.outbreak_frame(observations, forecasts) + probability = frame["probability"].clip(LOG_SCORE_CLIP, 1 - LOG_SCORE_CLIP) + outbreak = frame["outbreak"] + score = -(outbreak * np.log(probability) + (1 - outbreak) * np.log(1 - probability)) + return _as_metric(frame, score) + + +@metric() +class BrierSkillScoreMetric(OutbreakScoredMixin, GlobalOnlyMetric): + """Brier skill against climatology: how much the forecast beats the base rate. + + One is perfect, zero means the forecast is no better than always predicting + the historical outbreak frequency, and negative means it is worse than that. + A ratio of aggregates, so there is no per-cell value. + + Undefined when every scored cell is an outbreak or none is: the climatological + reference is then perfect and there is no skill to measure against it. + """ + + spec = MetricSpec( + metric_id="outbreak_brier_skill", + metric_name="Outbreak Brier Skill Score", + output_dimensions=(), + aggregation_op=AggregationOp.MEAN, + description="Brier score improvement over a climatological base-rate forecast", + optimization_direction=OptimizationDirection.MAXIMIZE, + ) + + def compute_global(self, observations: pd.DataFrame, forecasts: pd.DataFrame) -> float: + frame = self.outbreak_frame(observations, forecasts) + if frame.empty: + return float("nan") + outbreak = frame["outbreak"] + base_rate = float(outbreak.mean()) + reference = base_rate * (1 - base_rate) + if reference == 0: + return float("nan") + brier = float(((frame["probability"] - outbreak) ** 2).mean()) + return 1 - brier / reference diff --git a/chap_core/rest_api/v1/routers/visualization.py b/chap_core/rest_api/v1/routers/visualization.py index 7c201adc1..3637e25b1 100644 --- a/chap_core/rest_api/v1/routers/visualization.py +++ b/chap_core/rest_api/v1/routers/visualization.py @@ -75,6 +75,11 @@ def get_available_metrics(backtest_id: int): Use this to populate a metric picker in a UI before requesting a specific plot. The result is the same regardless of ``backtest_id`` — the path takes it for symmetry with the render endpoint. + + Only metrics that have a per-cell value are listed. Every metric plot breaks a score + down by horizon, location or period, so a metric defined only over a whole set of + cells (F1, Matthews correlation, skill scores) has nothing to plot and would render + an error rather than a chart. Read those from the backtest's aggregate metrics instead. """ logger.info(f"Getting available metrics for backtest {backtest_id}") logger.info(f"Available metrics: {available_metrics.keys()}") @@ -85,6 +90,7 @@ def get_available_metrics(backtest_id: int): description=metric_factory().get_description(), ) for metric_id, metric_factory in available_metrics.items() + if metric_factory.spec.output_dimensions ] diff --git a/tests/evaluation/conftest.py b/tests/evaluation/conftest.py index 9ebcddb6a..5710a1bd6 100644 --- a/tests/evaluation/conftest.py +++ b/tests/evaluation/conftest.py @@ -482,3 +482,33 @@ def _make(location, time_period, horizon, samples): ) return _make + + +@pytest.fixture +def balanced_alert_scenario(two_season_history, make_flat_forecasts): + """One outbreak and one quiet period, each scored at two horizons. + + June's channel sits near 111 and November's near 61, so the observations + below are clearly on either side of them. Scoring each period at two horizons + -- one alerting, one not -- yields exactly one true positive, false negative, + false positive and true negative, so every confusion-matrix metric has a + hand-checkable value. + + Returns ``(historical_observations, observations, forecasts)``. + """ + observations = pd.DataFrame( + [ + {"location": "A", "time_period": "2023-06", "disease_cases": 200.0}, + {"location": "A", "time_period": "2023-11", "disease_cases": 20.0}, + ] + ) + forecasts = pd.concat( + [ + make_flat_forecasts("A", "2023-06", 1, [300.0] * 10), # outbreak, alerted -> TP + make_flat_forecasts("A", "2023-06", 2, [10.0] * 10), # outbreak, missed -> FN + make_flat_forecasts("A", "2023-11", 1, [300.0] * 10), # quiet, alerted -> FP + make_flat_forecasts("A", "2023-11", 2, [10.0] * 10), # quiet, silent -> TN + ], + ignore_index=True, + ) + return two_season_history, observations, forecasts diff --git a/tests/evaluation/test_outbreak_alert_metrics.py b/tests/evaluation/test_outbreak_alert_metrics.py new file mode 100644 index 000000000..6683a85de --- /dev/null +++ b/tests/evaluation/test_outbreak_alert_metrics.py @@ -0,0 +1,104 @@ +import numpy as np +import pytest + +from chap_core.assessment.flat_representations import DataDimension +from chap_core.assessment.metrics.outbreak_classification import ( + FalseAlarmRateMetric, + MatthewsCorrelationMetric, + OutbreakF1Metric, + OutbreakPrecisionMetric, +) +from chap_core.assessment.metrics.outbreak_probability import ( + LOG_SCORE_CLIP, + BrierScoreMetric, + BrierSkillScoreMetric, + OutbreakLogScoreMetric, +) + +_GLOBAL_ONLY = (OutbreakF1Metric, MatthewsCorrelationMetric, BrierSkillScoreMetric) + + +def _global(metric_cls, scenario): + historical, observations, forecasts = scenario + metric = metric_cls(historical_observations=historical) + return metric.get_global_metric(observations, forecasts).iloc[0]["metric"] + + +def test_precision_counts_only_the_alerts(balanced_alert_scenario): + """One of the two raised alerts was a real outbreak.""" + assert _global(OutbreakPrecisionMetric, balanced_alert_scenario) == pytest.approx(0.5) + + +def test_false_alarm_rate_counts_only_the_quiet_periods(balanced_alert_scenario): + """One of the two quiet periods was alerted anyway.""" + assert _global(FalseAlarmRateMetric, balanced_alert_scenario) == pytest.approx(0.5) + + +def test_f1_is_the_harmonic_mean_of_precision_and_sensitivity(balanced_alert_scenario): + """Precision and sensitivity are both 0.5 here, so F1 is too.""" + assert _global(OutbreakF1Metric, balanced_alert_scenario) == pytest.approx(0.5) + + +def test_matthews_correlation_is_zero_for_chance_agreement(balanced_alert_scenario): + """A balanced confusion matrix means alerts carry no information at all.""" + assert _global(MatthewsCorrelationMetric, balanced_alert_scenario) == pytest.approx(0.0) + + +def test_brier_score_is_the_mean_squared_error(balanced_alert_scenario): + """Probabilities are 1, 0, 1, 0 against outcomes 1, 1, 0, 0 -- two cells wrong outright.""" + assert _global(BrierScoreMetric, balanced_alert_scenario) == pytest.approx(0.5) + + +def test_brier_skill_is_negative_when_worse_than_the_base_rate(balanced_alert_scenario): + """Half the cells are outbreaks, so climatology scores 0.25 and the model 0.5.""" + assert _global(BrierSkillScoreMetric, balanced_alert_scenario) == pytest.approx(-1.0) + + +def test_log_score_clips_confident_misses(balanced_alert_scenario): + """Two cells are confidently wrong; the clip makes each cost -log(eps) rather than infinity.""" + expected = (2 * -np.log(LOG_SCORE_CLIP) + 2 * -np.log(1 - LOG_SCORE_CLIP)) / 4 + assert _global(OutbreakLogScoreMetric, balanced_alert_scenario) == pytest.approx(expected) + assert np.isfinite(expected) + + +@pytest.mark.parametrize("metric_cls", _GLOBAL_ONLY, ids=lambda c: c.spec.metric_id) +def test_global_only_metrics_refuse_a_breakdown(metric_cls, balanced_alert_scenario): + """A ratio of aggregates has no per-cell value, so asking for one is an error, not a wrong number.""" + historical, observations, forecasts = balanced_alert_scenario + metric = metric_cls(historical_observations=historical) + with pytest.raises(ValueError, match="cannot be broken down"): + metric.get_metric(observations, forecasts, dimensions=(DataDimension.location,)) + + +@pytest.mark.parametrize("metric_cls", _GLOBAL_ONLY, ids=lambda c: c.spec.metric_id) +def test_global_only_metrics_declare_no_output_dimensions(metric_cls): + """This is the contract compute_all_detailed_metrics keys on to skip them.""" + assert metric_cls.spec.output_dimensions == () + + +def test_brier_skill_is_undefined_without_both_outcomes(two_season_history, make_flat_forecasts): + """With no quiet period the climatological reference is perfect and skill has no meaning.""" + import pandas as pd + + observations = pd.DataFrame([{"location": "A", "time_period": "2023-06", "disease_cases": 200.0}]) + forecasts = make_flat_forecasts("A", "2023-06", 1, [300.0] * 10) + metric = BrierSkillScoreMetric(historical_observations=two_season_history) + assert np.isnan(metric.get_global_metric(observations, forecasts).iloc[0]["metric"]) + + +def test_only_two_sided_metrics_are_optimization_objectives(): + """Precision and false-alarm rate are each maximised by a degenerate model, like sensitivity before them.""" + assert OutbreakPrecisionMetric.spec.optimization_direction is None + assert FalseAlarmRateMetric.spec.optimization_direction is None + for metric_cls in _GLOBAL_ONLY + (BrierScoreMetric, OutbreakLogScoreMetric): + assert metric_cls.spec.optimization_direction is not None + + +def test_plot_metric_picker_omits_unplottable_metrics(): + """Every metric plot breaks a score down by a dimension a global-only metric has no values for.""" + from chap_core.rest_api.v1.routers.visualization import get_available_metrics + + offered = {metric.id for metric in get_available_metrics(backtest_id=1)} + assert "outbreak_brier" in offered + for metric_cls in _GLOBAL_ONLY: + assert metric_cls.spec.metric_id not in offered