qqp, mnli_mismatch: remove unlabled test sets (EleutherAI#1114)

* remove unlabled test sets * add note to readme
abacusai · Dec 13, 2023 · 057dc2d · 057dc2d
1 parent 9ef853a
commit 057dc2d
Show file tree

Hide file tree

Showing 3 changed files with 1 addition and 2 deletions.
diff --git a/lm_eval/tasks/glue/README.md b/lm_eval/tasks/glue/README.md
@@ -1,4 +1,5 @@
 # GLUE
+**NOTE**: GLUE benchmark tasks do not provide publicly accessible labels for their test sets, so we default to the validation sets for all sub-tasks.
 
 ### Paper
 

diff --git a/lm_eval/tasks/glue/mnli/mismatch.yaml b/lm_eval/tasks/glue/mnli/mismatch.yaml
@@ -1,4 +1,3 @@
 include: default.yaml
 task: mnli_mismatch
 validation_split: validation_mismatched
-test_split: test_mismatched
diff --git a/lm_eval/tasks/glue/qqp/default.yaml b/lm_eval/tasks/glue/qqp/default.yaml
@@ -5,7 +5,6 @@ dataset_name: qqp
 output_type: multiple_choice
 training_split: train
 validation_split: validation
-test_split: test
 doc_to_text: "\nSentence 1: {{question1}}\nSentence 2: {{question2}}\nAnswer:"
 doc_to_target: label
 doc_to_choice: ["no", "yes"]