fix: point MLflow at a writable tracking store in the worker

chap_core/runners/mlflow_runner.py

@@ -1,4 +1,5 @@
 import logging
+import os
 
 import mlflow.exceptions
 import mlflow.projects
@@ -10,8 +11,27 @@
 logger = logging.getLogger(__name__)
 
 
+def _set_default_tracking_uri():
+    """Point MLflow at a writable tracking store when the deployment has not configured one.
+
+    MLflow defaults its tracking store to the relative URI ``sqlite:///mlflow.db``, which it
+    resolves against the process working directory. In the worker container that directory is
+    ``/app`` on a read-only filesystem, so creating a run fails with "unable to open database
+    file". CHAP_RUNS_DIR is writable in every deployment, so use it as the fallback location.
+    """
+    if os.environ.get("MLFLOW_TRACKING_URI"):
+        return
+
+    from chap_core.models.utils import CHAP_RUNS_DIR
+
+    db_path = (CHAP_RUNS_DIR / "mlflow.db").absolute()
+    logger.debug(f"MLFLOW_TRACKING_URI not set, defaulting MLflow tracking store to {db_path}")
+    mlflow.set_tracking_uri(f"sqlite:///{db_path}")
+
+
 class MlFlowTrainPredictRunner(TrainPredictRunner):
     def __init__(self, model_path, model_configuration_filename=None, train_params=None):
+        _set_default_tracking_uri()
         self.model_path = model_path
         self.model_configuration_filename = model_configuration_filename
 

tests/runners/test_runners.py

@@ -1,6 +1,8 @@
+import os
 from pathlib import Path
 from unittest.mock import patch, MagicMock, ANY
 
+import mlflow
 import yaml
 
 from chap_core.exceptions import CommandLineException, ModelFailedException
@@ -21,6 +23,18 @@
 from chap_core.util import docker_available
 
 
+@pytest.fixture
+def mlflow_tracking_uri_reset():
+    """Undo the global tracking URI, which set_tracking_uri also mirrors into the environment."""
+    original_env = os.environ.get("MLFLOW_TRACKING_URI")
+    yield
+    mlflow.set_tracking_uri(None)
+    if original_env is None:
+        os.environ.pop("MLFLOW_TRACKING_URI", None)
+    else:
+        os.environ["MLFLOW_TRACKING_URI"] = original_env
+
+
 def test_command_line_runner():
     command = "echo 'test'"
     runner = CommandLineRunner(Path("."))
@@ -326,7 +340,29 @@ def test_command_line_runner_report_raises_when_no_command():
         runner.report("model.pkl", "historic.csv", "report.pdf")
 
 
-def test_mlflow_runner_report_invokes_report_entry_point(tmp_path):
+def test_mlflow_runner_defaults_tracking_uri_to_runs_dir(tmp_path, monkeypatch, mlflow_tracking_uri_reset):
+    """MLflow defaults its tracking store to a sqlite file in the working directory, which is
+    read-only in the worker container. Fall back to the runs directory, which is writable."""
+    mlflow.set_tracking_uri(None)
+    os.environ.pop("MLFLOW_TRACKING_URI", None)
+    monkeypatch.setattr("chap_core.models.utils.CHAP_RUNS_DIR", tmp_path)
+
+    MlFlowTrainPredictRunner(model_path=tmp_path)
+
+    assert mlflow.get_tracking_uri() == f"sqlite:///{tmp_path / 'mlflow.db'}"
+
+
+def test_mlflow_runner_keeps_configured_tracking_uri(tmp_path, mlflow_tracking_uri_reset):
+    """A tracking URI configured by the deployment must win over the fallback."""
+    mlflow.set_tracking_uri(None)
+    os.environ["MLFLOW_TRACKING_URI"] = "http://mlflow.example:5000"
+
+    MlFlowTrainPredictRunner(model_path=tmp_path)
+
+    assert mlflow.get_tracking_uri() == "http://mlflow.example:5000"
+
+
+def test_mlflow_runner_report_invokes_report_entry_point(tmp_path, mlflow_tracking_uri_reset):
     runner = MlFlowTrainPredictRunner(model_path=tmp_path)
     with patch("mlflow.projects.run") as mock_run:
         mock_run.return_value = MagicMock()
@@ -345,7 +381,7 @@ def test_mlflow_runner_report_invokes_report_entry_point(tmp_path):
         }
 
 
-def test_mlflow_runner_report_wraps_execution_errors(tmp_path):
+def test_mlflow_runner_report_wraps_execution_errors(tmp_path, mlflow_tracking_uri_reset):
     import mlflow.exceptions
 
     runner = MlFlowTrainPredictRunner(model_path=tmp_path)