Skip to content

Commit 30f0435

Browse files
yl231yl231
authored
Sync leaderboard data from RouterArena@e95ba6764eb4a02d89a32c0cc7e858ec19b3ee6f (#48)
1 parent 1434253 commit 30f0435

2 files changed

Lines changed: 63 additions & 63 deletions

File tree

‎src/data/flip_labels/flip_labels_vllm-sr.json‎

Lines changed: 31 additions & 31 deletions
Original file line numberDiff line numberDiff line change
@@ -221,11 +221,11 @@
221221
},
222222
{
223223
"global index": "FinQA_56",
224-
"flip": 0
224+
"flip": 1
225225
},
226226
{
227227
"global index": "FinQA_60",
228-
"flip": 0
228+
"flip": 1
229229
},
230230
{
231231
"global index": "GSM8K_43",
@@ -297,39 +297,39 @@
297297
},
298298
{
299299
"global index": "GeoGraphyData_100k_42",
300-
"flip": 0
300+
"flip": 1
301301
},
302302
{
303303
"global index": "LiveCodeBench_105",
304-
"flip": 0
304+
"flip": 1
305305
},
306306
{
307307
"global index": "LiveCodeBench_114",
308-
"flip": 0
308+
"flip": 1
309309
},
310310
{
311311
"global index": "LiveCodeBench_118",
312-
"flip": 0
312+
"flip": 1
313313
},
314314
{
315315
"global index": "LiveCodeBench_131",
316-
"flip": 0
316+
"flip": 1
317317
},
318318
{
319319
"global index": "LiveCodeBench_136",
320-
"flip": 0
320+
"flip": 1
321321
},
322322
{
323323
"global index": "LiveCodeBench_181",
324-
"flip": 0
324+
"flip": 1
325325
},
326326
{
327327
"global index": "LiveCodeBench_237",
328328
"flip": 0
329329
},
330330
{
331331
"global index": "LiveCodeBench_271",
332-
"flip": 0
332+
"flip": 1
333333
},
334334
{
335335
"global index": "LiveCodeBench_350",
@@ -345,7 +345,7 @@
345345
},
346346
{
347347
"global index": "LiveCodeBench_43",
348-
"flip": 0
348+
"flip": 1
349349
},
350350
{
351351
"global index": "LiveCodeBench_431",
@@ -357,23 +357,23 @@
357357
},
358358
{
359359
"global index": "LiveCodeBench_476",
360-
"flip": 0
360+
"flip": 1
361361
},
362362
{
363363
"global index": "LiveCodeBench_485",
364-
"flip": 0
364+
"flip": 1
365365
},
366366
{
367367
"global index": "LiveCodeBench_49",
368-
"flip": 0
368+
"flip": 1
369369
},
370370
{
371371
"global index": "LiveCodeBench_491",
372-
"flip": 0
372+
"flip": 1
373373
},
374374
{
375375
"global index": "LiveCodeBench_499",
376-
"flip": 0
376+
"flip": 1
377377
},
378378
{
379379
"global index": "MATH_108",
@@ -437,7 +437,7 @@
437437
},
438438
{
439439
"global index": "MMLUPro_business_507",
440-
"flip": 1
440+
"flip": 0
441441
},
442442
{
443443
"global index": "MMLUPro_business_6",
@@ -645,7 +645,7 @@
645645
},
646646
{
647647
"global index": "MMLUPro_health_5144",
648-
"flip": 1
648+
"flip": 0
649649
},
650650
{
651651
"global index": "MMLUPro_health_5214",
@@ -841,7 +841,7 @@
841841
},
842842
{
843843
"global index": "MMLUPro_physics_8888",
844-
"flip": 1
844+
"flip": 0
845845
},
846846
{
847847
"global index": "MMLUPro_physics_9017",
@@ -961,7 +961,7 @@
961961
},
962962
{
963963
"global index": "MedMCQA_1005",
964-
"flip": 0
964+
"flip": 1
965965
},
966966
{
967967
"global index": "MedMCQA_1054",
@@ -1061,7 +1061,7 @@
10611061
},
10621062
{
10631063
"global index": "NarrativeQA_131",
1064-
"flip": 1
1064+
"flip": 0
10651065
},
10661066
{
10671067
"global index": "NarrativeQA_1683",
@@ -1357,7 +1357,7 @@
13571357
},
13581358
{
13591359
"global index": "QANTA_Fine Arts_1212",
1360-
"flip": 0
1360+
"flip": 1
13611361
},
13621362
{
13631363
"global index": "QANTA_Fine Arts_1702",
@@ -1369,7 +1369,7 @@
13691369
},
13701370
{
13711371
"global index": "QANTA_Fine Arts_865",
1372-
"flip": 0
1372+
"flip": 1
13731373
},
13741374
{
13751375
"global index": "QANTA_Geography_1023",
@@ -1385,15 +1385,15 @@
13851385
},
13861386
{
13871387
"global index": "QANTA_History_1084",
1388-
"flip": 0
1388+
"flip": 1
13891389
},
13901390
{
13911391
"global index": "QANTA_History_1154",
13921392
"flip": 0
13931393
},
13941394
{
13951395
"global index": "QANTA_History_433",
1396-
"flip": 0
1396+
"flip": 1
13971397
},
13981398
{
13991399
"global index": "QANTA_History_473",
@@ -1421,7 +1421,7 @@
14211421
},
14221422
{
14231423
"global index": "QANTA_Literature_1727",
1424-
"flip": 0
1424+
"flip": 1
14251425
},
14261426
{
14271427
"global index": "QANTA_Literature_1843",
@@ -1433,11 +1433,11 @@
14331433
},
14341434
{
14351435
"global index": "QANTA_Literature_408",
1436-
"flip": 0
1436+
"flip": 1
14371437
},
14381438
{
14391439
"global index": "QANTA_Literature_475",
1440-
"flip": 0
1440+
"flip": 1
14411441
},
14421442
{
14431443
"global index": "QANTA_Literature_833",
@@ -1461,7 +1461,7 @@
14611461
},
14621462
{
14631463
"global index": "QANTA_Science_1473",
1464-
"flip": 0
1464+
"flip": 1
14651465
},
14661466
{
14671467
"global index": "QANTA_Science_308",
@@ -1477,11 +1477,11 @@
14771477
},
14781478
{
14791479
"global index": "QANTA_Social Science_2",
1480-
"flip": 0
1480+
"flip": 1
14811481
},
14821482
{
14831483
"global index": "QANTA_Social Science_77",
1484-
"flip": 0
1484+
"flip": 1
14851485
},
14861486
{
14871487
"global index": "SocialiQA_13810",

‎src/data/routerMetrics/leaderboard.json‎

Lines changed: 32 additions & 32 deletions
Original file line numberDiff line numberDiff line change
@@ -1,15 +1,4 @@
11
[
2-
{
3-
"Router Name": "vllm",
4-
"Arena Score": 75.38,
5-
"Optimal Selection Score": 20.12,
6-
"Optimal Cost Score": 24.52,
7-
"Optimal Acc. Score": 89.87,
8-
"Robustness Score": 73.1,
9-
"Latency Score": null,
10-
"Accuracy": 75.97,
11-
"Cost per 1k": 0.11
12-
},
132
{
143
"Router Name": "Sqwish Router",
154
"Arena Score": 75.27,
@@ -32,17 +21,6 @@
3221
"Accuracy": 74.72,
3322
"Cost per 1k": 0.13
3423
},
35-
{
36-
"Router Name": "Nadir Router",
37-
"Arena Score": 73.33,
38-
"Optimal Selection Score": null,
39-
"Optimal Cost Score": null,
40-
"Optimal Acc. Score": null,
41-
"Robustness Score": 25.48,
42-
"Latency Score": null,
43-
"Accuracy": 74.87,
44-
"Cost per 1k": 0.29
45-
},
4624
{
4725
"Router Name": "Weave Router",
4826
"Arena Score": 72.82,
@@ -55,26 +33,37 @@
5533
"Cost per 1k": 0.94
5634
},
5735
{
58-
"Router Name": "OrcaRouter-Adaptive",
59-
"Arena Score": 72.08,
36+
"Router Name": "Nadir Router",
37+
"Arena Score": 72.29,
6038
"Optimal Selection Score": null,
6139
"Optimal Cost Score": null,
6240
"Optimal Acc. Score": null,
63-
"Robustness Score": 22.62,
41+
"Robustness Score": 25.48,
6442
"Latency Score": null,
65-
"Accuracy": 75.54,
66-
"Cost per 1k": 1.0
43+
"Accuracy": 75.01,
44+
"Cost per 1k": 0.68
6745
},
6846
{
69-
"Router Name": "azure",
70-
"Arena Score": 71.87,
47+
"Router Name": "vllm",
48+
"Arena Score": 72.15,
49+
"Optimal Selection Score": 20.77,
50+
"Optimal Cost Score": 17.19,
51+
"Optimal Acc. Score": 90.0,
52+
"Robustness Score": 73.1,
53+
"Latency Score": null,
54+
"Accuracy": 73.19,
55+
"Cost per 1k": 0.23
56+
},
57+
{
58+
"Router Name": "OrcaRouter-Adaptive",
59+
"Arena Score": 72.08,
7160
"Optimal Selection Score": null,
7261
"Optimal Cost Score": null,
7362
"Optimal Acc. Score": null,
74-
"Robustness Score": 71.43,
63+
"Robustness Score": 22.62,
7564
"Latency Score": null,
76-
"Accuracy": 72.82,
77-
"Cost per 1k": 0.22
65+
"Accuracy": 75.54,
66+
"Cost per 1k": 1.0
7867
},
7968
{
8069
"Router Name": "R2-Router",
@@ -87,6 +76,17 @@
8776
"Accuracy": 71.23,
8877
"Cost per 1k": 0.06
8978
},
79+
{
80+
"Router Name": "azure",
81+
"Arena Score": 70.42,
82+
"Optimal Selection Score": null,
83+
"Optimal Cost Score": null,
84+
"Optimal Acc. Score": null,
85+
"Robustness Score": 71.43,
86+
"Latency Score": null,
87+
"Accuracy": 72.94,
88+
"Cost per 1k": 0.73
89+
},
9090
{
9191
"Router Name": "Auto Router",
9292
"Arena Score": 70.05,

0 commit comments

Comments
 (0)