-
Notifications
You must be signed in to change notification settings - Fork 2
/
score_exp.sh
executable file
·203 lines (189 loc) · 9.15 KB
/
score_exp.sh
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
#!/usr/bin/env bash
set -e
tpu=$1
# mc
# uq_clean_test uq_clean_train test
# uq_clean_train_inp uq_clean_test_inp test_inp
# uq_clean_test_bt uq_clean_test_ret uq_clean_test_ret_bt uq_clean_test_ret_bt_inp
# uq_clean_train_bt uq_clean_train_ret uq_clean_train_ret_bt uq_clean_train_ret_bt_inp
# test_bt test_ret test_ret_bt test_ret_bt_inp
# uq_clean_test_bt_dedup uq_clean_test_ret_bt_dedup uq_clean_test_ret_bt_dedup_inp
# uq_clean_train_bt_dedup uq_clean_train_ret_bt_dedup uq_clean_train_ret_bt_dedup_inp
# test_bt_dedup test_ret_bt_dedup test_ret_bt_dedup_inp
# uq_clean_test_bt_dedup_top10 uq_clean_test_bt_dedup_top20
: '
for task in test test_inp test_bt_dedup test_ret test_ret_bt_dedup test_ret_bt_dedup_inp; do
for model in 11B; do
if [[ $task == 'uq_clean_test' ]]; then
output_root=output/exp/uq_clean_test/dev_nolennorm
mix=uq_clean_test_mix
split=dev
elif [[ $task == 'uq_clean_test_inp' ]]; then
output_root=output/exp/uq_clean_test/dev_nolennorm/inp
mix=uq_clean_test_inp_mix
split=dev
elif [[ $task == 'uq_clean_test_bt_dedup' ]]; then
output_root=output/exp/uq_clean_test/dev_nolennorm/bt_dedup
mix=uq_clean_test_bt_dedup_replace_mix
split=dev
elif [[ $task == 'uq_clean_test_bt_dedup_top20' ]]; then
output_root=output/exp/uq_clean_test/dev_nolennorm/bt_dedup_top20
mix=uq_clean_test_bt_dedup_top20_replace_mix
split=dev
elif [[ $task == 'uq_clean_test_ret' ]]; then
output_root=output/exp/uq_clean_test/dev_nolennorm/ret
mix=uq_clean_test_ret_drqa_3s_mix
split=dev
elif [[ $task == 'uq_clean_test_ret_bt_dedup' ]]; then
output_root=output/exp/uq_clean_test/dev_nolennorm/ret_bt_dedup
mix=uq_clean_test_ret_drqa_3s_bt_dedup_replace_mix
split=dev
elif [[ $task == 'uq_clean_test_ret_bt_dedup_inp' ]]; then
output_root=output/exp/uq_clean_test/dev_nolennorm/ret_bt_dedup_inp
mix=uq_clean_test_ret_drqa_3s_bt_dedup_replace_inp_mix
split=dev
elif [[ $task == 'uq_clean_train' ]]; then
output_root=output/exp/uq_clean_train/dev_nolennorm
mix=uq_clean_train_mix
split=dev
elif [[ $task == 'uq_clean_train_inp' ]]; then
output_root=output/exp/uq_clean_train/dev_nolennorm/inp
mix=uq_clean_train_inp_mix
split=dev
elif [[ $task == 'uq_clean_train_bt_dedup' ]]; then
output_root=output/exp/uq_clean_train/dev_nolennorm/bt_dedup
mix=uq_clean_train_bt_dedup_replace_mix
split=dev
elif [[ $task == 'uq_clean_train_ret' ]]; then
output_root=output/exp/uq_clean_train/dev_nolennorm/ret
mix=uq_clean_train_ret_drqa_3s_mix
split=dev
elif [[ $task == 'uq_clean_train_ret_bt_dedup' ]]; then
output_root=output/exp/uq_clean_train/dev_nolennorm/ret_bt_dedup
mix=uq_clean_train_ret_drqa_3s_bt_dedup_replace_mix
split=dev
elif [[ $task == 'uq_clean_train_ret_bt_dedup_inp' ]]; then
output_root=output/exp/uq_clean_train/dev_nolennorm/ret_bt_dedup_inp
mix=uq_clean_train_ret_drqa_3s_bt_dedup_replace_inp_mix
split=dev
elif [[ $task == 'test' ]]; then
output_root=output/exp/test/test_nolennorm
mix=test_mix
split=test
elif [[ $task == 'test_inp' ]]; then
output_root=output/exp/test/test_nolennorm/inp
mix=test_inp_mix
split=test
elif [[ $task == 'test_bt_dedup' ]]; then
output_root=output/exp/test/test_nolennorm/bt_dedup
mix=test_bt_dedup_replace_mix
split=test
elif [[ $task == 'test_ret' ]]; then
output_root=output/exp/test/test_nolennorm/ret
mix=test_ret_drqa_3s_mix
split=test
elif [[ $task == 'test_ret_bt_dedup' ]]; then
output_root=output/exp/test/test_nolennorm/ret_bt_dedup
mix=test_ret_drqa_3s_bt_dedup_replace_mix
split=test
elif [[ $task == 'test_ret_bt_dedup_inp' ]]; then
output_root=output/exp/test/test_nolennorm/ret_bt_dedup_inp
mix=test_ret_drqa_3s_bt_dedup_replace_inp_mix
split=test
fi
if [[ $model == '3B' ]]; then
step=1103000
elif [[ $model == '11B' ]]; then
step=1115000
fi
./score.sh $tpu ${output_root}/${model}/uq_ft_margin.txt ${model} unifiedqa/ft_models/${model}_margin_nolennorm $step $mix $split &> nohup.out
if [[ $task == 'uq_clean_test' ]] || [[ $task == 'uq_clean_train' ]] || [[ $task == 'test' ]]; then
./score.sh $tpu ${output_root}/${model}/uq_ft_softmax.txt ${model} unifiedqa/ft_models/${model}_softmax_nolennorm $step $mix $split &> nohup.out
#./score.sh $tpu ${output_root}/${model}/uq.txt ${model} unifiedqa/models/${model} 1100500 $mix $split &> nohup.out
#./score.sh $tpu ${output_root}/${model}/t5.txt ${model} t5-data/pretrained_models/${model} 1000000 $mix $split &> nohup.out
fi
done
done
'
# ext
# uq_ext_train uq_ext_test
# uq_ext_train_inp uq_ext_test_inp
# uq_ext_train_ret uq_ext_test_ret
# uq_ext_train_bt uq_ext_test_bt
# uq_ext_train_ret_bt uq_ext_test_ret_bt
# uq_ext_train_ret_bt_inp uq_ext_test_ret_bt_inp
# uq_ext uq_ext_first
# uq_ext_first_topk uq_ext_first_topk_bt
suffix='_span_topk_nogold'
for task in uq_ext_test_ret_bt_inp uq_ext_train uq_ext_train_inp uq_ext_train_ret uq_ext_train_bt uq_ext_train_ret_bt uq_ext_train_ret_bt_inp; do
for model in 3B; do
if [[ $task == 'uq_ext' ]]; then
output_root=output/exp/uq_ext/dev
mix=uq_ext_mix
split=dev
elif [[ $task == 'uq_ext_first' ]]; then
output_root=output/exp/uq_ext_first/dev
mix=uq_ext_first_decode_uq3B_mix
split=dev
elif [[ $task == 'uq_ext_train' ]]; then
output_root=output/exp/uq_ext_train${suffix}/dev
mix=uq_ext_decode_train_uq3B${suffix}_mix
split=dev
elif [[ $task == 'uq_ext_train_inp' ]]; then
output_root=output/exp/uq_ext_train${suffix}/dev/inp
mix=uq_ext_decode_train_uq3B${suffix}_inp_mix
split=dev
elif [[ $task == 'uq_ext_train_ret' ]]; then
output_root=output/exp/uq_ext_train${suffix}/dev/ret
mix=uq_ext_decode_train_uq3B${suffix}_ret_drqa_3s_mix
split=dev
elif [[ $task == 'uq_ext_train_bt' ]]; then
output_root=output/exp/uq_ext_train${suffix}/dev/bt
mix=uq_ext_decode_train_uq3B${suffix}_bt_mix
split=dev
elif [[ $task == 'uq_ext_train_ret_bt' ]]; then
output_root=output/exp/uq_ext_train${suffix}/dev/ret_bt
mix=uq_ext_decode_train_uq3B${suffix}_ret_drqa_3s_bt_mix
split=dev
elif [[ $task == 'uq_ext_train_ret_bt_inp' ]]; then
output_root=output/exp/uq_ext_train${suffix}/dev/ret_bt_inp
mix=uq_ext_decode_train_uq3B${suffix}_ret_drqa_3s_bt_inp_mix
split=dev
elif [[ $task == 'uq_ext_test' ]]; then
output_root=output/exp/uq_ext_test${suffix}/dev
mix=uq_ext_decode_test_uq3B${suffix}_mix
split=dev
elif [[ $task == 'uq_ext_test_inp' ]]; then
output_root=output/exp/uq_ext_test${suffix}/dev/inp
mix=uq_ext_decode_test_uq3B${suffix}_inp_mix
split=dev
elif [[ $task == 'uq_ext_test_ret' ]]; then
output_root=output/exp/uq_ext_test${suffix}/dev/ret
mix=uq_ext_decode_test_uq3B${suffix}_ret_drqa_3s_mix
split=dev
elif [[ $task == 'uq_ext_test_bt' ]]; then
output_root=output/exp/uq_ext_test${suffix}/dev/bt
mix=uq_ext_decode_test_uq3B${suffix}_bt_mix
split=dev
elif [[ $task == 'uq_ext_test_ret_bt' ]]; then
output_root=output/exp/uq_ext_test${suffix}/dev/ret_bt
mix=uq_ext_decode_test_uq3B${suffix}_ret_drqa_3s_bt_mix
split=dev
elif [[ $task == 'uq_ext_test_ret_bt_inp' ]]; then
output_root=output/exp/uq_ext_test${suffix}/dev/ret_bt_inp
mix=uq_ext_decode_test_uq3B${suffix}_ret_drqa_3s_bt_inp_mix
split=dev
fi
if [[ $model == '3B' ]]; then
step=1103000
elif [[ $model == '11B' ]]; then
step=1115000
fi
./score.sh $tpu ${output_root}/${model}/uq.txt ${model} unifiedqa/models/${model} 1100500 $mix $split &> nohup.out
if [[ $task == 'uq_ext_train' ]] || [[ $task == 'uq_ext_test' ]]; then
./score.sh $tpu ${output_root}/${model}/uq_ft_margin.txt ${model} unifiedqa/ft_models/${model}_ext_span_margin $step $mix $split &> nohup.out
./score.sh $tpu ${output_root}/${model}/uq_ft_softmax.txt ${model} unifiedqa/ft_models/${model}_ext_span_softmax $step $mix $split &> nohup.out
./score.sh $tpu ${output_root}/${model}/t5.txt ${model} t5-data/pretrained_models/${model} 1000000 $mix $split &> nohup.out
fi
done
done