feat(eval): add --no-interleave-scoring flag

README.md

@@ -179,6 +179,12 @@ uv run vf-eval vf-environment-name --sampling-args '{"reasoning_effort": "low"}'
 
 Use `vf-eval -s` to save outputs as dataset-formatted JSON, and view all locally-saved eval results with `vf-tui`.
 
+Note on scoring behavior: by default, `vf-eval` uses interleaved scoring, where each rollout is scored individually as it completes (`score_rollout`). If your environment's rubric relies on seeing the entire batch of completions at once (e.g., in a custom `score_rollouts` method), you can force the old batch-scoring behavior with the `--no-interleave-scoring` flag:
+
+```bash
+uv run vf-eval vf-environment-name --no-interleave-scoring
+```
+
 ### ToolEnv
 
 For many applications involving tool use, you can use `ToolEnv` to leverage models' native tool/function-calling capabilities in an agentic loop. Tools must be stateless and idempotent—each call should be fully determined by the provided arguments—because the environment will automatically terminate once the assistant responds without tool calls. Tools can be specified as generic Python functions (with type hints and docstrings), which will then be passed in JSON schema form to each inference request.

tests/test_eval_cli.py

@@ -70,6 +70,7 @@ def __init__(self, api_key=None, base_url=None):
         save_dataset=False,
         save_to_hf_hub=False,
         hf_hub_dataset_name="",
+        interleave_scoring=True,
     )
 
     sa = captured["sampling_args"]
@@ -118,6 +119,7 @@ def __init__(self, api_key=None, base_url=None):
         save_dataset=False,
         save_to_hf_hub=False,
         hf_hub_dataset_name="",
+        interleave_scoring=True,
     )
 
     sa = captured["sampling_args"]

verifiers/scripts/eval.py

@@ -40,6 +40,7 @@ def eval_environment(
     save_dataset: bool,
     save_to_hf_hub: bool,
     hf_hub_dataset_name: str,
+    interleave_scoring: bool,
     extra_headers: Dict[str, str],
 ):
     setup_logging("DEBUG" if verbose else "INFO")
@@ -119,6 +120,7 @@ def eval_environment(
         num_examples=num_examples,
         rollouts_per_example=rollouts_per_example,
         max_concurrent=max_concurrent,
+        interleave_scoring=interleave_scoring,
     )
     end_time = time.time()
     logger.info(f"Evaluation completed in {end_time - start_time:.2f} seconds")
@@ -337,6 +339,13 @@ def main():
         default="",
         help="Name of dataset to save to Hugging Face Hub",
     )
+    parser.add_argument(
+        "--no-interleave-scoring",
+        dest="interleave_scoring",
+        default=True,
+        action="store_false",
+        help="Disable interleaved scoring and generation. This will use the score_rollouts method instead.",
+    )
     args = parser.parse_args()
 
     # Build headers from repeated --header flags
@@ -368,6 +377,7 @@ def main():
         save_dataset=args.save_dataset,
         save_to_hf_hub=args.save_to_hf_hub,
         hf_hub_dataset_name=args.hf_hub_dataset_name,
+        interleave_scoring=args.interleave_scoring,
         extra_headers=merged_headers,
     )