Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 6 additions & 2 deletions train/tests/test_loss_fn_wiring.py
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,9 @@
def grad_cache_calls_in_main():
tree = ast.parse(SOURCE.read_text())
main = next(
node for node in tree.body if isinstance(node, ast.FunctionDef) and node.name == "main"
node
for node in tree.body
if isinstance(node, ast.FunctionDef) and node.name == "main"
)
return [
node
Expand All @@ -37,7 +39,9 @@ def test_every_grad_cache_call_passes_the_selected_loss():
call.lineno
for call in grad_cache_calls_in_main()
if not any(
kw.arg == "loss_fn" and isinstance(kw.value, ast.Name) and kw.value.id == "active_loss_fn"
kw.arg == "loss_fn"
and isinstance(kw.value, ast.Name)
and kw.value.id == "active_loss_fn"
for kw in call.keywords
)
]
Expand Down
11 changes: 7 additions & 4 deletions train/tests/test_loss_world_size.py
Original file line number Diff line number Diff line change
Expand Up @@ -23,7 +23,7 @@
TRAIN_DIR = str(Path(__file__).resolve().parent.parent)
sys.path.insert(0, TRAIN_DIR)

import train_contrastors as tc # noqa: E402
import train_contrastors as tc

WORLD = 2
QUERIES_PER_RANK = 4
Expand Down Expand Up @@ -79,8 +79,8 @@ def _worker(rank, port, case, out_dir):
loss_fn, hard_negatives = CASES[case]
q, d = global_batch(hard_negatives)
per_doc = QUERIES_PER_RANK * (1 + hard_negatives)
q = q[rank * QUERIES_PER_RANK:(rank + 1) * QUERIES_PER_RANK]
d = d[rank * per_doc:(rank + 1) * per_doc]
q = q[rank * QUERIES_PER_RANK : (rank + 1) * QUERIES_PER_RANK]
d = d[rank * per_doc : (rank + 1) * per_doc]

model = DDP(Tower(), broadcast_buffers=False)
scale = make_logit_scale()
Expand All @@ -107,7 +107,10 @@ def free_port():
@pytest.mark.parametrize("case", list(CASES))
def test_data_parallel_grads_match_global_batch(case, tmp_path):
mp.start_processes(
_worker, args=(free_port(), case, str(tmp_path)), nprocs=WORLD, start_method="spawn"
_worker,
args=(free_port(), case, str(tmp_path)),
nprocs=WORLD,
start_method="spawn",
)

loss_fn, hard_negatives = CASES[case]
Expand Down
Loading