@@ -274,7 +274,7 @@ def _test_ebc_resharding(
274274 )
275275
276276 sharded_m2 = sharder .shard (
277- module = m1 ,
277+ module = m2 ,
278278 params = module_sharding_plan ,
279279 env = env ,
280280 device = ctx .device ,
@@ -299,34 +299,30 @@ def _test_ebc_resharding(
299299 for table in tables :
300300 feature_keys .extend (table .feature_names )
301301
302- # For current test model and inputs, the prediction should be the exact same
303- # rtol = 0
304- # atol = 0
305-
306- for _ in range (world_size ):
307- # sharded model
308- # each rank gets a subbatch
309- sharded_m1_pred_kt_no_dict = sharded_m1 (kjt_input_per_rank [ctx .rank ])
310- resharded_m2_pred_kt_no_dict = resharded_m2 (kjt_input_per_rank [ctx .rank ])
302+ # sharded model
303+ # each rank gets a subbatch
304+ sharded_m1_pred_kt_no_dict = sharded_m1 (kjt_input_per_rank [ctx .rank ])
305+ resharded_m2_pred_kt_no_dict = resharded_m2 (kjt_input_per_rank [ctx .rank ])
311306
312- sharded_m1_pred_kt = sharded_m1_pred_kt_no_dict .to_dict ()
313- resharded_m2_pred_kt = resharded_m2_pred_kt_no_dict .to_dict ()
314- sharded_m1_pred = torch .stack (
315- [sharded_m1_pred_kt [feature ] for feature in feature_keys ]
316- )
307+ sharded_m1_pred_kt = sharded_m1_pred_kt_no_dict .to_dict ()
308+ resharded_m2_pred_kt = resharded_m2_pred_kt_no_dict .to_dict ()
309+ sharded_m1_pred = torch .stack (
310+ [sharded_m1_pred_kt [feature ] for feature in feature_keys ]
311+ )
317312
318- resharded_m2_pred = torch .stack (
319- [resharded_m2_pred_kt [feature ] for feature in feature_keys ]
320- )
321- # cast to CPU because when casting unsharded_model.to on the same module, there could some race conditions
322- # in normal author modelling code this won't be an issue because each rank would individually create
323- # their model. output from sharded_pred is correctly on the correct device.
313+ resharded_m2_pred = torch .stack (
314+ [resharded_m2_pred_kt [feature ] for feature in feature_keys ]
315+ )
316+ # cast to CPU because when casting unsharded_model.to on the same module, there could some race conditions
317+ # in normal author modelling code this won't be an issue because each rank would individually create
318+ # their model. output from sharded_pred is correctly on the correct device.
324319
325- # Compare predictions of sharded vs unsharded models.
326- torch .testing .assert_close (sharded_m1_pred .cpu (), resharded_m2_pred .cpu ())
320+ # Compare predictions of sharded vs unsharded models.
321+ torch .testing .assert_close (sharded_m1_pred .cpu (), resharded_m2_pred .cpu ())
327322
328- sharded_m1_pred .sum ().backward ()
329- resharded_m2_pred .sum ().backward ()
323+ # Smoke test: reshard must produce a trainable module (backward doesn't crash).
324+ sharded_m1_pred .sum ().backward ()
325+ resharded_m2_pred .sum ().backward ()
330326
331327
332328@skip_if_asan_class
@@ -497,7 +493,7 @@ def test_dynamic_sharding_ebc_cw(
497493 world_size , num_tables , ranks_per_tables
498494 )
499495 new_ranks = generate_rank_placements (
500- world_size , num_tables , ranks_per_tables
496+ world_size , num_tables , new_ranks_per_tables
501497 )
502498 per_param_sharding = {}
503499 new_per_param_sharding = {}
0 commit comments