@@ -1159,6 +1159,37 @@ def test_evaluation_iterable_dataset(self):
11591159 expected_acc = AlmostAccuracy ()((pred , y ))["accuracy" ]
11601160 self .assertAlmostEqual (results ["eval_accuracy" ], expected_acc )
11611161
1162+ def test_iterable_evaluation_loss_uses_actual_batch_size (self ):
1163+ for dispatch in (False , True ):
1164+ for accumulation in (None , 1 ):
1165+ for batch_size in (1 , 3 , 8 ):
1166+ with self .subTest (dispatch = dispatch , accumulation = accumulation , batch_size = batch_size ):
1167+ dataset = SampleIterableDataset (length = 7 )
1168+ trainer = get_regression_trainer (
1169+ a = 1.5 ,
1170+ b = 2.5 ,
1171+ output_dir = self .get_auto_remove_tmp_dir (),
1172+ use_cpu = True ,
1173+ per_device_eval_batch_size = batch_size ,
1174+ eval_accumulation_steps = accumulation ,
1175+ accelerator_config = {"dispatch_batches" : dispatch },
1176+ report_to = [],
1177+ )
1178+ x , y = dataset .dataset .x , dataset .dataset .ys [0 ]
1179+ expected_predictions = 1.5 * x + 2.5
1180+ expected_loss = ((expected_predictions - y ) ** 2 ).mean ()
1181+ map_result = trainer .evaluate (dataset .dataset )
1182+ iterable_result = trainer .evaluate (dataset )
1183+ prediction = trainer .predict (dataset )
1184+ for result , prefix in (
1185+ (map_result , "eval" ),
1186+ (iterable_result , "eval" ),
1187+ (prediction .metrics , "test" ),
1188+ ):
1189+ self .assertAlmostEqual (result [f"{ prefix } _loss" ], expected_loss , places = 6 )
1190+ np .testing .assert_allclose (prediction .predictions , expected_predictions )
1191+ np .testing .assert_array_equal (prediction .label_ids , y )
1192+
11621193 def test_predict_iterable_dataset (self ):
11631194 config = RegressionModelConfig (a = 1.5 , b = 2.5 )
11641195 model = RegressionPreTrainedModel (config )
0 commit comments