feat: expose metric unit and target in the metrics API (CLIM-1103)

chap_core/assessment/metrics/__init__.py

@@ -23,6 +23,7 @@
     MetricSpec,
     OptimizationDirection,
     ProbabilisticMetric,
+    TargetBehavior,
 )
 from chap_core.database.tables import Backtest
 
@@ -58,7 +59,7 @@ def get_metric(metric_id: str) -> type[Metric] | None:
 
 
 def list_metrics() -> list[dict]:
-    """List all registered metrics with metadata (id, name, description, aggregation_op)."""
+    """List all registered metrics with scoring and presentation metadata."""
     result = []
     for metric_cls in _metrics_registry.values():
         spec = metric_cls.spec
@@ -67,6 +68,9 @@ def list_metrics() -> list[dict]:
                 "id": spec.metric_id,
                 "name": spec.metric_name,
                 "description": spec.description,
+                "unit": spec.unit,
+                "target": spec.target,
+                "target_behavior": spec.target_behavior.value,
                 "aggregation_op": spec.aggregation_op.value,
                 "optimization_direction": (
                     spec.optimization_direction.value if spec.optimization_direction is not None else None
@@ -282,11 +286,11 @@ def get_optimization_direction(metric_id: str) -> OptimizationDirection:
     if metric_cls is None:
         raise ValueError(f"Unknown metric {metric_id!r}")
 
-    direction = metric_cls.spec.optimization_direction
-    if direction is None:
+    spec = metric_cls.spec
+    if spec.optimization_direction is None or not spec.valid_hpo_objective:
         raise ValueError(
             f"Metric {metric_id!r} is not defined as a direct HPO objective. "
             "Choose a metric with an explicit optimization direction."
         )
 
-    return direction
+    return spec.optimization_direction

chap_core/assessment/metrics/above_truth.py

@@ -34,6 +34,7 @@ class RatioAboveTruthMetric(ProbabilisticMetric):
         aggregation_op=AggregationOp.MEAN,
         description="Proportion of forecast samples exceeding the observed value (0.5 = unbiased)",
         optimization_direction=None,
+        target=0.5,
     )
 
     def compute_sample_metric(self, samples: np.ndarray, observed: float) -> float:

chap_core/assessment/metrics/base.py

@@ -41,6 +41,15 @@ class OptimizationDirection(StrEnum):
     MAXIMIZE = "maximize"
 
 
+class TargetBehavior(StrEnum):
+    """How a score should be judged against the target."""
+
+    # Deviating from the target in either direction is worse.
+    CLOSEST = "closest"
+    # Scores below the target are worse, scores above it are no worse than the target.
+    AT_LEAST = "at_least"
+
+
 @dataclass(frozen=True)
 class MetricSpec:
     """
@@ -52,8 +61,17 @@ class MetricSpec:
     output_dimensions: tuple[DataDimension, ...] = DEFAULT_OUTPUT_DIMENSIONS
     aggregation_op: AggregationOp = AggregationOp.MEAN
     description: str = "No description provided"
-    # None means the metic is not directly usable as a scalar optimization objective.
+    # Which way is better when reading a score. None means neither direction is better.
     optimization_direction: OptimizationDirection | None = None
+    # Whether the score is sound to optimize on its own. False for metrics a model can
+    # game, so they keep a direction for presentation without becoming HPO objectives.
+    valid_hpo_objective: bool = True
+    # Display suffix for the raw score; None when no fixed unit applies.
+    unit: str | None = None
+    # Ideal value in raw score units for metrics where neither direction is better.
+    target: float | None = None
+    # How to judge a score against the target. Only meaningful when target is set.
+    target_behavior: TargetBehavior = TargetBehavior.CLOSEST
 
 
 class Metric(ABC):

chap_core/assessment/metrics/mape.py

@@ -34,6 +34,7 @@ class MAPEMetric(DeterministicMetric):
         aggregation_op=AggregationOp.MEAN,
         description="Mean Absolute Percentage Error - average absolute error as percent of observed",
         optimization_direction=OptimizationDirection.MINIMIZE,
+        unit="%",
     )
 
     def compute_point_metric(self, forecast: float, observed: float) -> float:

chap_core/assessment/metrics/outbreak_detection.py

@@ -22,6 +22,7 @@
     AggregationOp,
     Metric,
     MetricSpec,
+    OptimizationDirection,
 )
 
 # The outbreak metrics score forecasts against seasonal thresholds, computed by the
@@ -142,15 +143,17 @@ class SensitivityMetric(_OutbreakMetric):
     Measures the proportion of actual outbreaks that were correctly
     predicted (alerted) by the forecast.
 
-    Not a valid standalone optimization objective: it is maximised by alerting
-    every period, so ``optimization_direction`` is deliberately unset.
+    Higher is better to read, but it is not a valid standalone optimization
+    objective: it is maximised by alerting every period.
     """
 
     spec = MetricSpec(
         metric_id="sensitivity",
         metric_name="Sensitivity",
         aggregation_op=AggregationOp.MEAN,
         description="True positive rate for outbreak detection alerts",
+        optimization_direction=OptimizationDirection.MAXIMIZE,
+        valid_hpo_objective=False,
     )
 
     def compute_detailed(self, observations: pd.DataFrame, forecasts: pd.DataFrame) -> pd.DataFrame:
@@ -166,15 +169,17 @@ class SpecificityMetric(_OutbreakMetric):
     Measures the proportion of non-outbreak periods that were correctly
     not alerted by the forecast.
 
-    Not a valid standalone optimization objective: it is maximised by never
-    alerting, so ``optimization_direction`` is deliberately unset.
+    Higher is better to read, but it is not a valid standalone optimization
+    objective: it is maximised by never alerting.
     """
 
     spec = MetricSpec(
         metric_id="specificity",
         metric_name="Specificity",
         aggregation_op=AggregationOp.MEAN,
         description="True negative rate for outbreak detection alerts",
+        optimization_direction=OptimizationDirection.MAXIMIZE,
+        valid_hpo_objective=False,
     )
 
     def compute_detailed(self, observations: pd.DataFrame, forecasts: pd.DataFrame) -> pd.DataFrame:
@@ -190,16 +195,17 @@ class OutbreakAccuracyMetric(_OutbreakMetric):
     Measures the proportion of all periods where the alert status
     correctly matches the outbreak status: (TP + TN) / (TP + TN + FP + FN).
 
-    Not a valid standalone optimization objective: outbreaks are rare, so it is
-    close to maximised by never alerting. ``optimization_direction`` is
-    deliberately unset.
+    Higher is better to read, but it is not a valid standalone optimization
+    objective: outbreaks are rare, so it is close to maximised by never alerting.
     """
 
     spec = MetricSpec(
         metric_id="outbreak_accuracy",
         metric_name="Outbreak Accuracy",
         aggregation_op=AggregationOp.MEAN,
         description="Proportion of correctly classified outbreak/non-outbreak periods",
+        optimization_direction=OptimizationDirection.MAXIMIZE,
+        valid_hpo_objective=False,
     )
 
     def compute_detailed(self, observations: pd.DataFrame, forecasts: pd.DataFrame) -> pd.DataFrame:

chap_core/assessment/metrics/peak_diff.py

@@ -70,6 +70,7 @@ class PeakValueDiffMetric(Metric):
         aggregation_op=AggregationOp.MEAN,
         description="Truth peak value minus predicted peak value, per horizon",
         optimization_direction=None,
+        target=0.0,
     )
 
     def compute_detailed(self, observations: pd.DataFrame, forecasts: pd.DataFrame) -> pd.DataFrame:
@@ -130,6 +131,8 @@ class PeakPeriodLagMetric(Metric):
         aggregation_op=AggregationOp.MEAN,
         description="Lag in time periods between true and predicted peak (pred - truth), per horizon",
         optimization_direction=None,
+        unit="periods",
+        target=0.0,
     )
 
     def compute_detailed(self, observations: pd.DataFrame, forecasts: pd.DataFrame) -> pd.DataFrame:

chap_core/assessment/metrics/percentile_coverage.py

@@ -9,6 +9,7 @@
     AggregationOp,
     MetricSpec,
     ProbabilisticMetric,
+    TargetBehavior,
 )
 
 
@@ -42,8 +43,10 @@ class Coverage10_90Metric(PercentileCoverageMetric):
         metric_id="coverage_10_90",
         metric_name="Coverage 10-90",
         aggregation_op=AggregationOp.MEAN,
-        description="Proportion of observations within 10th-90th percentile",
+        description="Proportion of observations within 10th-90th percentile (higher is better, up to the 0.8 target)",
         optimization_direction=None,
+        target=0.8,
+        target_behavior=TargetBehavior.AT_LEAST,
     )
     low_percentile = 10
     high_percentile = 90
@@ -57,8 +60,10 @@ class Coverage25_75Metric(PercentileCoverageMetric):
         metric_id="coverage_25_75",
         metric_name="Coverage 25-75",
         aggregation_op=AggregationOp.MEAN,
-        description="Proportion of observations within 25th-75th percentile",
+        description="Proportion of observations within 25th-75th percentile (higher is better, up to the 0.5 target)",
         optimization_direction=None,
+        target=0.5,
+        target_behavior=TargetBehavior.AT_LEAST,
     )
     low_percentile = 25
     high_percentile = 75

chap_core/rest_api/v1/routers/visualization.py

@@ -18,7 +18,7 @@
 from chap_core.assessment.evaluation import Evaluation
 from chap_core.assessment.metric_plots import get_metric_plots_registry, list_metric_plots
 from chap_core.assessment.metrics import available_metrics
-from chap_core.assessment.metrics.base import OptimizationDirection
+from chap_core.assessment.metrics.base import OptimizationDirection, TargetBehavior
 from chap_core.database.base_tables import DBModel
 from chap_core.database.dataset_tables import DataSet
 from chap_core.database.tables import Backtest
@@ -63,11 +63,26 @@ class MetricInfo(DBModel):
     id: str = Field(description="Canonical metric identifier used in URLs and request bodies.")
     display_name: str = Field(description="Human-friendly metric name shown in pickers.")
     description: str = Field(default="", description="Short paragraph explaining what the metric measures.")
+    unit: str | None = Field(default=None, description="Display suffix for the raw score, e.g. '%' for MAPE.")
+    target: float | None = Field(
+        default=None,
+        description="Ideal value in raw score units, e.g. 0.8 for 80% coverage. Null when no fixed target applies.",
+    )
+    target_behavior: TargetBehavior = Field(
+        default=TargetBehavior.CLOSEST,
+        description=(
+            "How to judge a score against ``target``, only meaningful when ``target`` is set. "
+            "'closest' means deviating in either direction is worse. 'at_least' means higher is "
+            "better up to the target and flat above it, so only scores below the target should be "
+            "flagged as bad."
+        ),
+    )
     optimization_direction: OptimizationDirection | None = Field(
         default=None,
         description=(
             "Whether a lower ('minimize') or higher ('maximize') score is better. "
-            "Null for metrics where neither direction is better, such as coverage ratios."
+            "Null for metrics where neither direction is better; those set ``target`` instead, "
+            "and ``target_behavior`` says how to judge a score against it."
         ),
     )
 
@@ -78,7 +93,7 @@ class MetricInfo(DBModel):
     summary="Discover which scoring metrics are available",
 )
 def get_available_metrics(backtest_id: int):
-    """List the metrics you can score a backtest with (CRPS, MAE, ...), with a human-friendly name, description and optimization direction for each.
+    """List the metrics you can score a backtest with (CRPS, MAE, ...), with a human-friendly name, description, optimization direction, unit and target for each.
 
     Use this to populate a metric picker in a UI before requesting a specific plot. The
     result is the same regardless of ``backtest_id`` — the path takes it for symmetry
@@ -92,6 +107,9 @@ def get_available_metrics(backtest_id: int):
             display_name=metric_factory().get_name(),
             description=metric_factory().get_description(),
             optimization_direction=metric_factory.spec.optimization_direction,
+            target_behavior=metric_factory.spec.target_behavior,
+            unit=metric_factory.spec.unit,
+            target=metric_factory.spec.target,
         )
         for metric_id, metric_factory in available_metrics.items()
     ]

docs/contributor/creating_custom_metrics.md

@@ -122,17 +122,36 @@ from chap_core.assessment.metrics.base import ProbabilisticMetric
 ## MetricSpec Configuration
 
 ```python
-from chap_core.assessment.metrics.base import AggregationOp, MetricSpec
+from chap_core.assessment.metrics.base import AggregationOp, MetricSpec, TargetBehavior
 
 spec = MetricSpec(
     metric_id="unique_id",              # Used in APIs and registry
     metric_name="Display Name",          # Human-readable name
     aggregation_op=AggregationOp.MEAN,   # MEAN, SUM, or ROOT_MEAN_SQUARE
     description="What this metric measures",
     optimization_direction=None,        # MINIMIZE, MAXIMIZE or None
+    valid_hpo_objective=True,           # False when the score is not sound to optimize alone
+    unit=None,                          # Display suffix for the raw score, e.g. "%"
+    target=None,                        # Ideal raw value when neither direction is better, e.g. 0.8
+    target_behavior=TargetBehavior.CLOSEST,  # CLOSEST or AT_LEAST, only used with a target
 )
 ```
 
+The metric catalogue API returns `unit`, `target` and `target_behavior` alongside
+the optimization direction. A metric with `optimization_direction=None` should set
+a `target`, and `target_behavior` tells clients how to judge a score against it:
+`CLOSEST` means deviating in either direction is worse (ratio above truth, peak
+difference), while `AT_LEAST` means higher is better up to the target and flat
+above it, so only scores below the target should be flagged as bad (coverage
+metrics). Units do not rescale scores: MAPE is already a percentage, while
+coverage targets use fractions such as `0.8`.
+
+`optimization_direction` says which way is better when reading a score, not that the
+metric is a sound thing to optimize. Set `valid_hpo_objective=False` when a model can
+game the score: the outbreak metrics keep `MAXIMIZE` so clients colour high scores as
+good, but HPO rejects them, since sensitivity is maximised by always alerting and
+specificity by never alerting.
+
 ## Complete Examples
 
 ### Example: RMSE-style Metric

tests/evaluation/test_metrics.py

@@ -497,6 +497,13 @@ def test_peak_value_diff_metric_weekly(flat_observations_week, flat_forecasts_we
     pd.testing.assert_frame_equal(result_sorted, expected_sorted)
 
 
+def test_peak_metric_specs_expose_unit_and_target():
+    assert PeakValueDiffMetric.spec.target == 0.0
+    assert PeakValueDiffMetric.spec.unit is None
+    assert PeakPeriodLagMetric.spec.target == 0.0
+    assert PeakPeriodLagMetric.spec.unit == "periods"
+
+
 def test_peak_period_lag_metric_weekly(flat_observations_week, flat_forecasts_week):
     """
     PeakPeriodLagMetric should return:

tests/evaluation/test_outbreak_detection_metrics.py

@@ -2,6 +2,8 @@
 import pandas as pd
 import pytest
 
+from chap_core.assessment.metrics import get_optimization_direction
+from chap_core.assessment.metrics.base import OptimizationDirection
 from chap_core.assessment.metrics.outbreak_detection import (
     OutbreakAccuracyMetric,
     SensitivityMetric,
@@ -303,7 +305,15 @@ def test_outbreak_and_alert_drops_uncomputable_thresholds(threshold_value):
     assert outbreak_and_alert(historical, observations, forecasts).empty
 
 
+def test_outbreak_metrics_read_as_higher_is_better():
+    """Higher scores are better to a reader, so clients get a direction to colour them by."""
+    for metric_cls in (SensitivityMetric, SpecificityMetric, OutbreakAccuracyMetric):
+        assert metric_cls.spec.optimization_direction == OptimizationDirection.MAXIMIZE
+
+
 def test_outbreak_metrics_are_not_optimization_objectives():
     """None of the three is valid alone: two are maximised by degenerate models, accuracy by silence."""
     for metric_cls in (SensitivityMetric, SpecificityMetric, OutbreakAccuracyMetric):
-        assert metric_cls.spec.optimization_direction is None
+        assert metric_cls.spec.valid_hpo_objective is False
+        with pytest.raises(ValueError):
+            get_optimization_direction(metric_cls.spec.metric_id)

tests/integration/rest_api/test_python_endpoints.py

@@ -5,6 +5,7 @@
 import pytest
 from sqlmodel import select, Session
 
+from chap_core.assessment.metrics import list_metrics
 from chap_core.database.tables import Backtest
 from chap_core.rest_api.data_models import BacktestFull
 from chap_core.rest_api.v1.routers.visualization import (
@@ -29,6 +30,32 @@ def test_available_metrics_expose_optimization_direction():
     assert metrics["coverage_10_90"].optimization_direction is None
 
 
+def test_available_metrics_expose_unit_and_target():
+    metrics = {metric.id: metric for metric in get_available_metrics(backtest_id=1)}
+    assert metrics["mape"].unit == "%"
+    assert metrics["mape"].target is None
+    assert metrics["coverage_10_90"].target == 0.8
+    assert metrics["coverage_25_75"].target == 0.5
+    assert metrics["ratio_above_truth"].target == 0.5
+    assert metrics["mae"].unit is None
+    assert metrics["mae"].target is None
+    for entry in list_metrics():
+        assert metrics[entry["id"]].unit == entry["unit"]
+        assert metrics[entry["id"]].target == entry["target"]
+    for metric in metrics.values():
+        if metric.optimization_direction is None:
+            assert metric.target is not None, metric.id
+
+
+def test_available_metrics_expose_target_behavior():
+    metrics = {metric.id: metric for metric in get_available_metrics(backtest_id=1)}
+    assert metrics["coverage_10_90"].target_behavior == "at_least"
+    assert metrics["coverage_25_75"].target_behavior == "at_least"
+    assert metrics["ratio_above_truth"].target_behavior == "closest"
+    for entry in list_metrics():
+        assert metrics[entry["id"]].target_behavior == entry["target_behavior"]
+
+
 def all_metric_ids():
     metrics = get_available_metrics(backtest_id=1)
     return [metric.id for metric in metrics]