-
Notifications
You must be signed in to change notification settings - Fork 2
Expand file tree
/
Copy pathrun_self_consistency.sh
More file actions
145 lines (101 loc) · 3.01 KB
/
Copy pathrun_self_consistency.sh
File metadata and controls
145 lines (101 loc) · 3.01 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
export PYTHONPATH=$PYTHONPATH:../
export TASK_PATH=tasks
export METHOD=self-prompt-cot
export SLEEP=40
export MODEL=gpt-3.5-turbo-0301
export OPENAI_API_KEY=sk-XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX
export OPENAI_API_BASE=https://api.openai.com/v1
export DATASET=cweb-qa
export DEMO_PATH=demos/$DATASET/knowledge-only_self-prompt-cot_retrieve_in_cluster_demos-8.json
python $TASK_PATH/self_consistency_async.py \
--task=$DATASET \
--input_file=$DEMO_PATH \
--method=$METHOD \
--model_name=$MODEL \
--answer_tokens=20 \
--do_eval \
--max_concurrency=50 \
--limit_dataset_size=0 \
--sleep_every_n_requests=$SLEEP \
--use_cache
#export DATASET=musique-qa
#export DEMO_PATH=demos/$DATASET/knowledge-only_self-prompt-cot_retrieve_in_cluster_demos-8.json
#python $TASK_PATH/self_consistency_async.py \
# --task=$DATASET \
# --input_file=$DEMO_PATH \
# --method=$METHOD \
# --model_name=$MODEL \
# --answer_tokens=20 \
# --do_eval \
# --max_concurrency=50 \
# --limit_dataset_size=0 \
# --sleep_every_n_requests=$SLEEP \
# --use_cache
#export DATASET=hotpot-qa
#export DEMO_PATH=demos/$DATASET/knowledge-only-test_self-prompt-cot_retrieve_in_cluster_demos-8_test.json
#python $TASK_PATH/self_consistency_async.py \
# --task=$DATASET \
# --input_file=$DEMO_PATH \
# --method=$METHOD \
# --model_name=$MODEL \
# --answer_tokens=20 \
# --do_eval \
# --max_concurrency=50 \
# --limit_dataset_size=0 \
# --sleep_every_n_requests=$SLEEP
## Manual CoT
#export DATASET=musique-qa
#python $TASK_PATH/inference_async.py \
# --task=$DATASET \
# --method=$METHOD \
# --model_name=$MODEL \
# --answer_tokens=20 \
# --do_eval \
# --max_concurrency=50 \
# --limit_dataset_size=0 \
# --sleep_every_n_requests=$SLEEP \
# --manual_cot_path=demos/manual-cot-random-hotpot.json \
# --flag=$FLAG
#
#
#export DATASET=cweb-qa
#python $TASK_PATH/inference_async.py \
# --task=$DATASET \
# --method=$METHOD \
# --model_name=$MODEL \
# --answer_tokens=20 \
# --do_eval \
# --max_concurrency=50 \
# --limit_dataset_size=0 \
# --sleep_every_n_requests=$SLEEP \
# --manual_cot_path=demos/manual-cot-random-hotpot.json \
# --flag=$FLAG
#
##
#export DATASET=hotpot-qa
#python $TASK_PATH/self_consistency_async.py \
# --task=$DATASET \
# --input_file=$DEMO_PATH \
# --method=$METHOD \
# --model_name=$MODEL \
# --answer_tokens=20 \
# --do_eval \
# --max_concurrency=50 \
# --limit_dataset_size=10 \
# --sleep_every_n_requests=$SLEEP \
# --manual_cot_path=demos/manual-cot-random-hotpot.json \
# --flag=$FLAG
#export DATASET=wikimh-qa
#python $TASK_PATH/inference_async.py \
# --task=$DATASET \
# --method=$METHOD \
# --model_name=$MODEL \
# --answer_tokens=20 \
# --do_eval \
# --max_concurrency=50 \
# --limit_dataset_size=0 \
# --sleep_every_n_requests=$SLEEP \
# --manual_cot_path=demos/manual-cot-random-hotpot.json \
# --flag=$FLAG \
# --use_cache
#