Skip to content

Commit bbfc2fe

Browse files
committed
Fix loss weighting for short streaming evaluation batches
1 parent 41f9d39 commit bbfc2fe

2 files changed

Lines changed: 33 additions & 1 deletion

File tree

‎src/transformers/trainer.py‎

100755100644
Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2996,7 +2996,8 @@ def evaluation_loop(
29962996

29972997
# Update containers
29982998
if losses is not None:
2999-
losses = self.gather_function(losses.repeat(batch_size))
2999+
loss_batch_size = observed_batch_size if observed_batch_size is not None else batch_size
3000+
losses = self.gather_function(losses.repeat(loss_batch_size))
30003001
all_losses.add(losses)
30013002
if inputs_decode is not None:
30023003
inputs_decode = self.accelerator.pad_across_processes(inputs_decode, dim=1, pad_index=-100)

‎tests/trainer/test_trainer_data.py‎

Lines changed: 31 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1159,6 +1159,37 @@ def test_evaluation_iterable_dataset(self):
11591159
expected_acc = AlmostAccuracy()((pred, y))["accuracy"]
11601160
self.assertAlmostEqual(results["eval_accuracy"], expected_acc)
11611161

1162+
def test_iterable_evaluation_loss_uses_actual_batch_size(self):
1163+
for dispatch in (False, True):
1164+
for accumulation in (None, 1):
1165+
for batch_size in (1, 3, 8):
1166+
with self.subTest(dispatch=dispatch, accumulation=accumulation, batch_size=batch_size):
1167+
dataset = SampleIterableDataset(length=7)
1168+
trainer = get_regression_trainer(
1169+
a=1.5,
1170+
b=2.5,
1171+
output_dir=self.get_auto_remove_tmp_dir(),
1172+
use_cpu=True,
1173+
per_device_eval_batch_size=batch_size,
1174+
eval_accumulation_steps=accumulation,
1175+
accelerator_config={"dispatch_batches": dispatch},
1176+
report_to=[],
1177+
)
1178+
x, y = dataset.dataset.x, dataset.dataset.ys[0]
1179+
expected_predictions = 1.5 * x + 2.5
1180+
expected_loss = ((expected_predictions - y) ** 2).mean()
1181+
map_result = trainer.evaluate(dataset.dataset)
1182+
iterable_result = trainer.evaluate(dataset)
1183+
prediction = trainer.predict(dataset)
1184+
for result, prefix in (
1185+
(map_result, "eval"),
1186+
(iterable_result, "eval"),
1187+
(prediction.metrics, "test"),
1188+
):
1189+
self.assertAlmostEqual(result[f"{prefix}_loss"], expected_loss, places=6)
1190+
np.testing.assert_allclose(prediction.predictions, expected_predictions)
1191+
np.testing.assert_array_equal(prediction.label_ids, y)
1192+
11621193
def test_predict_iterable_dataset(self):
11631194
config = RegressionModelConfig(a=1.5, b=2.5)
11641195
model = RegressionPreTrainedModel(config)

0 commit comments

Comments
 (0)