Skip to content

Commit 9f4811a

Browse files
authored
Sync leaderboard data from RouterArena@59559bde8f3408cc4dd5eb539dc3b9b1ad1dbab5 (#53)
1 parent 6d1021c commit 9f4811a

9 files changed

Lines changed: 13541 additions & 2991 deletions

‎src/data/flip_labels/flip_labels_asiainfo.json‎

Lines changed: 1682 additions & 0 deletions
Large diffs are not rendered by default.

‎src/data/flip_labels/flip_labels_barouter.json‎

Lines changed: 211 additions & 211 deletions
Large diffs are not rendered by default.

‎src/data/flip_labels/flip_labels_divyam.json‎

Lines changed: 1682 additions & 0 deletions
Large diffs are not rendered by default.

‎src/data/flip_labels/flip_labels_nadir-caliper.json‎

Lines changed: 1682 additions & 0 deletions
Large diffs are not rendered by default.

‎src/data/flip_labels/flip_labels_paix2.json‎

Lines changed: 1682 additions & 0 deletions
Large diffs are not rendered by default.

‎src/data/flip_labels/flip_labels_sqwish-router.json‎

Lines changed: 203 additions & 203 deletions
Large diffs are not rendered by default.

‎src/data/routerMetrics/category_scores.json‎

Lines changed: 6274 additions & 2546 deletions
Large diffs are not rendered by default.

‎src/data/routerMetrics/leaderboard.json‎

Lines changed: 75 additions & 31 deletions
Original file line numberDiff line numberDiff line change
@@ -1,47 +1,80 @@
11
[
2+
{
3+
"Router Name": "Paix2",
4+
"Arena Score": 77.63,
5+
"Optimal Selection Score": 89.67,
6+
"Optimal Cost Score": 62.92,
7+
"Optimal Acc. Score": 100.0,
8+
"Robustness Score": 77.86,
9+
"Latency Score": null,
10+
"Accuracy": 79.69,
11+
"Cost per 1k": 0.27
12+
},
13+
{
14+
"Router Name": "Sqwish Router",
15+
"Arena Score": 76.21,
16+
"Optimal Selection Score": 9.04,
17+
"Optimal Cost Score": 23.49,
18+
"Optimal Acc. Score": 94.07,
19+
"Robustness Score": 51.67,
20+
"Latency Score": null,
21+
"Accuracy": 79.76,
22+
"Cost per 1k": 0.7
23+
},
24+
{
25+
"Router Name": "Divyam",
26+
"Arena Score": 75.85,
27+
"Optimal Selection Score": 4.93,
28+
"Optimal Cost Score": 16.03,
29+
"Optimal Acc. Score": 93.43,
30+
"Robustness Score": 98.33,
31+
"Latency Score": null,
32+
"Accuracy": 78.59,
33+
"Cost per 1k": 0.48
34+
},
235
{
336
"Router Name": "Cross-Router",
4-
"Arena Score": 76.12,
37+
"Arena Score": 75.75,
538
"Optimal Selection Score": 17.66,
639
"Optimal Cost Score": 45.49,
740
"Optimal Acc. Score": 90.31,
841
"Robustness Score": 67.14,
942
"Latency Score": null,
1043
"Accuracy": 78.14,
11-
"Cost per 1k": 0.3
44+
"Cost per 1k": 0.4
45+
},
46+
{
47+
"Router Name": "Nadir-Tumbler",
48+
"Arena Score": 75.17,
49+
"Optimal Selection Score": null,
50+
"Optimal Cost Score": null,
51+
"Optimal Acc. Score": null,
52+
"Robustness Score": 66.43,
53+
"Latency Score": null,
54+
"Accuracy": 75.34,
55+
"Cost per 1k": 0.08
1256
},
1357
{
1458
"Router Name": "vllm",
15-
"Arena Score": 75.3,
59+
"Arena Score": 74.86,
1660
"Optimal Selection Score": 16.81,
1761
"Optimal Cost Score": 25.1,
1862
"Optimal Acc. Score": 89.37,
1963
"Robustness Score": 67.62,
2064
"Latency Score": null,
2165
"Accuracy": 77.18,
22-
"Cost per 1k": 0.3
66+
"Cost per 1k": 0.42
2367
},
2468
{
25-
"Router Name": "Sqwish Router",
26-
"Arena Score": 75.27,
27-
"Optimal Selection Score": 7.41,
28-
"Optimal Cost Score": 25.1,
29-
"Optimal Acc. Score": 90.47,
30-
"Robustness Score": 100.0,
31-
"Latency Score": null,
32-
"Accuracy": 76.4,
33-
"Cost per 1k": 0.18
34-
},
35-
{
36-
"Router Name": "Nadir-Tumbler",
37-
"Arena Score": 75.17,
69+
"Router Name": "nadir-caliper",
70+
"Arena Score": 74.55,
3871
"Optimal Selection Score": null,
3972
"Optimal Cost Score": null,
4073
"Optimal Acc. Score": null,
41-
"Robustness Score": 66.43,
74+
"Robustness Score": 79.76,
4275
"Latency Score": null,
43-
"Accuracy": 75.34,
44-
"Cost per 1k": 0.08
76+
"Accuracy": 75.84,
77+
"Cost per 1k": 0.22
4578
},
4679
{
4780
"Router Name": "AgentForge Router",
@@ -54,6 +87,17 @@
5487
"Accuracy": 74.72,
5588
"Cost per 1k": 0.13
5689
},
90+
{
91+
"Router Name": "BARouter",
92+
"Arena Score": 73.79,
93+
"Optimal Selection Score": 64.41,
94+
"Optimal Cost Score": 67.13,
95+
"Optimal Acc. Score": 93.84,
96+
"Robustness Score": 68.81,
97+
"Latency Score": null,
98+
"Accuracy": 75.72,
99+
"Cost per 1k": 0.36
100+
},
57101
{
58102
"Router Name": "Weave Router",
59103
"Arena Score": 72.82,
@@ -164,17 +208,6 @@
164208
"Accuracy": 68.41,
165209
"Cost per 1k": 0.29
166210
},
167-
{
168-
"Router Name": "BARouter",
169-
"Arena Score": 67.09,
170-
"Optimal Selection Score": 58.56,
171-
"Optimal Cost Score": 59.03,
172-
"Optimal Acc. Score": 87.91,
173-
"Robustness Score": 52.38,
174-
"Latency Score": null,
175-
"Accuracy": 68.8,
176-
"Cost per 1k": 0.63
177-
},
178211
{
179212
"Router Name": "mirt_bert",
180213
"Arena Score": 66.89,
@@ -197,6 +230,17 @@
197230
"Accuracy": 66.34,
198231
"Cost per 1k": 0.21
199232
},
233+
{
234+
"Router Name": "AsiaInfo-Router",
235+
"Arena Score": 65.87,
236+
"Optimal Selection Score": null,
237+
"Optimal Cost Score": null,
238+
"Optimal Acc. Score": null,
239+
"Robustness Score": 69.52,
240+
"Latency Score": null,
241+
"Accuracy": 75.2,
242+
"Cost per 1k": 8.54
243+
},
200244
{
201245
"Router Name": "gpt5",
202246
"Arena Score": 64.32,

‎src/data/routers.json‎

Lines changed: 50 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -486,5 +486,55 @@
486486
"qwen/qwen3-235b-a22b-2507",
487487
"z-ai/glm-4.7"
488488
]
489+
},
490+
"nadir-caliper": {
491+
"name": "nadir-caliper",
492+
"affiliation": "@doramirdor",
493+
"githubUrl": "https://github.com/doramirdor",
494+
"type": "open-source",
495+
"description": "Submitted by @doramirdor.",
496+
"modelPool": [
497+
"qwen/qwen3-235b-a22b-2507",
498+
"google/gemini-3.1-flash-lite",
499+
"grok-4.3",
500+
"deepseek/deepseek-v3.2"
501+
]
502+
},
503+
"Paix2": {
504+
"name": "Paix2",
505+
"affiliation": "@xufan866",
506+
"githubUrl": "https://github.com/xufan866",
507+
"type": "open-source",
508+
"description": "Submitted by @xufan866.",
509+
"modelPool": [
510+
"MiniMax-M3",
511+
"agnes-2.0-flash",
512+
"deepseek-ai/DeepSeek-R1-0528-Qwen3-8B",
513+
"THUDM/GLM-4-9B-0414"
514+
]
515+
},
516+
"Divyam": {
517+
"name": "Divyam",
518+
"affiliation": "@samikd",
519+
"githubUrl": "https://github.com/samikd",
520+
"type": "open-source",
521+
"description": "Submitted by @samikd.",
522+
"modelPool": [
523+
"qwen/qwen3-235b-a22b-2507",
524+
"Qwen/Qwen3-Coder-Next",
525+
"gemini-3-flash-preview",
526+
"google/gemini-3.1-flash-lite"
527+
]
528+
},
529+
"AsiaInfo-Router": {
530+
"name": "AsiaInfo-Router",
531+
"affiliation": "@Uncle-LL",
532+
"githubUrl": "https://github.com/Uncle-LL",
533+
"type": "open-source",
534+
"description": "Submitted by @Uncle-LL.",
535+
"modelPool": [
536+
"claude-opus-4-7",
537+
"deepseek/deepseek-v4-flash"
538+
]
489539
}
490540
}

0 commit comments

Comments
 (0)