Add validation for lora_dropout (#316)

* Add validation for lora_dropout

* Fix

* Fix eos

* Replace with llama3b

* Minimize the diff

* Version bump

Author: timofeev1995

pyproject.toml

@@ -12,7 +12,7 @@ build-backend = "poetry.masonry.api"
 
 [tool.poetry]
 name = "together"
-version = "1.5.10"
+version = "1.5.11"
 authors = ["Together AI <[email protected]>"]
 description = "Python client for Together's Cloud Platform!"
 readme = "README.md"

src/together/resources/finetune.py

@@ -101,6 +101,11 @@ def create_finetune_request(
             raise ValueError(
                 f"LoRA adapters are not supported for the selected model ({model_or_checkpoint})."
             )
+
+        if lora_dropout is not None:
+            if not 0 <= lora_dropout < 1.0:
+                raise ValueError("LoRA dropout must be in [0, 1) range.")
+
         lora_r = lora_r if lora_r is not None else model_limits.lora_training.max_rank
         lora_alpha = lora_alpha if lora_alpha is not None else lora_r * 2
         training_type = LoRATrainingType(

tests/unit/test_finetune_resources.py

@@ -85,6 +85,32 @@ def test_lora_request():
     assert request.batch_size == _MODEL_LIMITS.lora_training.max_batch_size
 
 
+@pytest.mark.parametrize("lora_dropout", [-1, 0, 0.5, 1.0, 10.0])
+def test_lora_request_with_lora_dropout(lora_dropout: float):
+
+    if 0 <= lora_dropout < 1:
+        request = create_finetune_request(
+            model_limits=_MODEL_LIMITS,
+            model=_MODEL_NAME,
+            training_file=_TRAINING_FILE,
+            lora=True,
+            lora_dropout=lora_dropout,
+        )
+        assert request.training_type.lora_dropout == lora_dropout
+    else:
+        with pytest.raises(
+            ValueError,
+            match=r"LoRA dropout must be in \[0, 1\) range.",
+        ):
+            create_finetune_request(
+                model_limits=_MODEL_LIMITS,
+                model=_MODEL_NAME,
+                training_file=_TRAINING_FILE,
+                lora=True,
+                lora_dropout=lora_dropout,
+            )
+
+
 def test_dpo_request_lora():
     request = create_finetune_request(
         model_limits=_MODEL_LIMITS,