1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
{
"K512_unpruned": {
"ops_tools": {
"ppl": 5.4696,
"ppl_se": 0.26012,
"sec": 42.9
},
"chat_json": {
"ppl": 6.9484,
"ppl_se": 0.38825,
"sec": 41.8
},
"ja": {
"ppl": 27.9434,
"ppl_se": 1.90485,
"sec": 43.4
}
},
"K320_reap_l2_sum": {
"ops_tools": {
"ppl": 5.7828,
"ppl_base": 5.4347,
"delta_ppl": 0.3481,
"mean_kld": 0.6273,
"mean_kld_se": 0.02492,
"same_top1_pct": 80.55,
"sec": 84.9
},
"chat_json": {
"ppl": 6.6057,
"ppl_base": 6.6732,
"delta_ppl": -0.0675,
"mean_kld": 0.30364,
"mean_kld_se": 0.01377,
"same_top1_pct": 84.29,
"sec": 60.4
},
"ja": {
"ppl": 21.0521,
"ppl_base": 26.1367,
"delta_ppl": -5.0846,
"mean_kld": 0.3225,
"mean_kld_se": 0.01243,
"same_top1_pct": 82.21,
"sec": 63.0
}
}
}