Skip to content

Commit 4f0f568

Browse files
committed
field: self-describing manifests + language-colored charts; land arch, refresh litellm-python/tensorzero
Each gateway is now fully defined by its own dir. gateway.sh declares GW_DISPLAY / GW_LANG / GW_REPO; charts.py, run-all.sh and run-on-ec2.sh discover the field by scanning gateways/*/gateway.sh — add a dir and it appears in charts, tables and run lists; delete a dir and it's gone everywhere. No hard-coded gateway list to drift. Charts: bars colored by implementation language (Rust/Go/Python/Node/ Other), no winner highlight (rows already sort by the measured value). Report table links each gateway name to its GitHub repo. Results: arch lands (255ms p99 at c1, 0 sustained — archgw can't hold load); litellm-python memory fixed (idle 1339 / peak 2747 MiB, was a flat 257 that missed the uvicorn workers); tensorzero refreshed. New gateway agentgateway added (measurement pending).
1 parent c232734 commit 4f0f568

24 files changed

Lines changed: 310 additions & 70 deletions

File tree

‎charts.py‎

Lines changed: 74 additions & 28 deletions
Original file line numberDiff line numberDiff line change
@@ -16,6 +16,7 @@
1616
from __future__ import annotations
1717

1818
import json
19+
import re
1920
from dataclasses import dataclass
2021
from datetime import datetime, timezone
2122
from pathlib import Path
@@ -64,23 +65,57 @@ def _mpl():
6465
plt = _plt
6566
return plt
6667

67-
# display order + labels. A gateway appears in a chart only if it has a result file this run.
68-
GATEWAYS = {
69-
# Alphabetical by key — deliberately NOT busbar-first. Order here is only the input/load order;
70-
# every chart and the report tables sort by the MEASURED value, so placement never favors anyone.
71-
"apisix": "APISIX",
72-
"arch": "Arch",
73-
"bifrost": "Bifrost",
74-
"busbar": "Busbar",
75-
"gomodel": "GoModel",
76-
"helicone": "Helicone",
77-
"kong": "Kong",
78-
"litellm-python": "LiteLLM · Python",
79-
"litellm-rust": "LiteLLM · Rust",
80-
"one-api": "One-API",
81-
"portkey": "Portkey",
82-
"tensorzero": "TensorZero",
68+
# ── the field is discovered from the manifests, nothing is hard-coded here ─────────────────────────
69+
# Each gateway is fully defined by its own dir: gateways/<key>/gateway.sh declares GW_DISPLAY (label),
70+
# GW_LANG (color bucket), and GW_REPO (linked from the name in the report table). Add a dir → it shows
71+
# up in the charts/tables/run-lists; delete it → it's gone everywhere. A gateway only appears in a
72+
# chart once it also has a result file this run. Alphabetical by key — deliberately NOT busbar-first;
73+
# order here is only load order, every chart + table sorts by the MEASURED value.
74+
def _manifest_meta():
75+
out = {}
76+
for man in sorted((ROOT / "gateways").glob("*/gateway.sh")):
77+
key = man.parent.name
78+
txt = man.read_text()
79+
80+
def grab(var, default=""):
81+
m = re.search(rf'^{var}=(.*)$', txt, re.M)
82+
if not m:
83+
return default
84+
v = m.group(1).split("#", 1)[0].strip() # drop trailing comment
85+
return v.strip('"').strip("'").strip()
86+
87+
out[key] = {
88+
"display": grab("GW_DISPLAY", key),
89+
"lang": grab("GW_LANG", "Other"),
90+
"repo": grab("GW_REPO"),
91+
}
92+
return out
93+
94+
_META = _manifest_meta()
95+
GATEWAYS = {k: v["display"] for k, v in _META.items()} # key → display label
96+
LANGS = {k: v["lang"] for k, v in _META.items()} # key → language bucket
97+
REPOS = {k: v["repo"] for k, v in _META.items()} # key → GitHub URL (may be "")
98+
99+
100+
def _linked(key: str) -> str:
101+
"""Gateway display name for the report table, linked to its GitHub repo when the manifest gives one."""
102+
name = GATEWAYS.get(key, key)
103+
repo = REPOS.get(key)
104+
return f"[{name}]({repo})" if repo else name
105+
106+
# Bars are colored by the gateway's IMPLEMENTATION LANGUAGE — informative (you can see the Rust/Go/
107+
# Python clustering) and neutral (no "winner" highlight for the sponsor; the best is already the top
108+
# bar since rows are sorted). A gateway that didn't serve is drawn grey regardless.
109+
# Five buckets: Rust / Go / Python / Node / Other (Lua/OpenResty, Envoy/C++, … fold into Other).
110+
LANG_ORDER = ["Rust", "Go", "Python", "Node", "Other"]
111+
LANG_COLORS = {
112+
"Rust": "#c4602d", # orange
113+
"Go": "#00a0c6", # cyan
114+
"Python": "#3b6ea5", # blue
115+
"Node": "#c59b2d", # amber
116+
"Other": "#6b7280", # grey
83117
}
118+
LANG_DEFAULT = "#6b7280"
84119

85120

86121
@dataclass(frozen=True)
@@ -260,10 +295,10 @@ def _numlab(v: float) -> str:
260295
vals = [float(r.get(s.field, 0) or 0) for r in rows]
261296
rank = s.kind == "rank"
262297
if rank:
263-
# green = served winner; slate = served; muted = did-not-serve (bar is context only).
264-
colors = [BRAND if (best is not None and _served(r) and v == best)
265-
else (SLATE if _served(r) else MUTE)
266-
for r, v in zip(rows, vals)]
298+
# colored by implementation language (served); did-not-serve is drawn grey. No winner
299+
# highlight — the best is already the top bar (rows are sorted).
300+
colors = [LANG_COLORS.get(LANGS.get(r["_key"], ""), LANG_DEFAULT) if _served(r) else MUTE
301+
for r in rows]
267302
else:
268303
colors = [s.kind] * n
269304
# On a log axis a bar can't start at 0, and a negative/zero value can't be drawn at all.
@@ -323,17 +358,27 @@ def _numlab(v: float) -> str:
323358
ax.set_title(chart.title, fontsize=15, fontweight="bold", color=INK, loc="left", pad=38)
324359
ax.text(0, 1.035, chart.subtitle, transform=ax.transAxes, fontsize=10.5, color=GRAY, va="bottom")
325360

326-
# legend (only multi-series charts need it)
327-
if ns > 1:
328-
ax.legend(loc="lower right", fontsize=9, frameon=False, ncols=ns)
361+
# Language legend (swatch per language present) + a note for the secondary series (e.g. idle RAM).
362+
from matplotlib.patches import Patch
363+
present = [l for l in LANG_ORDER if any(_served(r) and LANGS.get(r["_key"]) == l for r in rows)]
364+
handles = [Patch(facecolor=LANG_COLORS[l], label=l) for l in present]
365+
if any(not _served(r) for r in rows):
366+
handles.append(Patch(facecolor=MUTE, label="did not serve"))
367+
if ns > 1: # a muted secondary series (idle RAM) — label it too
368+
handles.append(Patch(facecolor=MUTE, label=chart.series[1].legend))
369+
if handles:
370+
# "best" so the swatch box lands in whitespace (top bars are short on the log charts, longest
371+
# bars crowd the bottom) instead of sitting over a bar.
372+
ax.legend(handles=handles, loc="best", fontsize=8.5, frameon=False,
373+
ncols=min(len(handles), 6), title="colored by language")
329374

330375
meta = rows[0]
331376
bits = []
332377
if "hardware" in meta:
333378
bits.append(str(meta["hardware"]))
334379
if "concurrency" in meta and "payload_bytes" in meta:
335380
bits.append(f"{meta['concurrency']}× {int(meta['payload_bytes'])//1000}KB sustained")
336-
bits.append("highlighted = measured best")
381+
bits.append("bars colored by implementation language")
337382
fig.text(0.008, 0.012, " · ".join(bits) + f" github.com/GetBusbar/benchmarking — regenerated {RENDER_TS} from raw results",
338383
fontsize=7.3, color=GRAY)
339384

@@ -364,8 +409,9 @@ def _report_md(rows: list, title: str, charts: list, pending: tuple = (), chart_
364409
lines.append(f"**Ran on:** {hw} · {when}")
365410
lines.append("")
366411
lines.append("Every number below is regenerated from the raw `results/*.json` — re-run "
367-
"`run-all.sh` and this page updates. The highlighted bar in each chart = measured best. "
368-
"**Rows are sorted by added latency (p99), lowest first.**")
412+
"`run-all.sh` and this page updates. Chart bars are **colored by implementation "
413+
"language** (Rust / Go / Python / Node / Other). **Rows are sorted by added latency "
414+
"(p99), lowest first.**")
369415
lines.append("")
370416
lines.append("| Gateway | Added latency (p99) | Sustained RPS @20ms | Max proxy RPS | Idle RAM | Peak RAM | Built |")
371417
lines.append("|---|--:|--:|--:|--:|--:|---|")
@@ -407,7 +453,7 @@ def rps_cell(val, bound, served):
407453
if served is False and r.get("serve_error"):
408454
fail_notes.append((GATEWAYS[key], str(r.get("serve_error"))))
409455
lines.append(
410-
f"| {GATEWAYS[key]} "
456+
f"| {_linked(key)} "
411457
f"| {lat_cell} "
412458
f"| {llm} "
413459
f"| {proxy} "
@@ -418,7 +464,7 @@ def rps_cell(val, bound, served):
418464
# Gateways we intend to measure but haven't yet — shown so the field is transparent, never hidden.
419465
for key in pending:
420466
lines.append(
421-
f"| {GATEWAYS[key]} | ⏳ *pending* | — | — | — | — | *pending measurement* |"
467+
f"| {_linked(key)} | ⏳ *pending* | — | — | — | — | *pending measurement* |"
422468
)
423469
lines.append("")
424470
if pending:

‎gateways/README.md‎

Lines changed: 20 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -6,10 +6,21 @@ directory.** The runners (`memory/run.sh`, and friends) are gateway-agnostic: th
66

77
## The contract
88

9-
`gateways/<name>/gateway.sh` sets four variables and defines four functions:
9+
A gateway is defined **entirely by its own directory** — nothing about it is hard-coded in the
10+
runners, the charts, or the run lists. `run-all.sh`, `run-on-ec2.sh`, and `charts.py` all discover the
11+
field by scanning `gateways/*/gateway.sh`, so **adding a dir adds the gateway everywhere and deleting a
12+
dir removes it everywhere.** No list to keep in sync.
13+
14+
`gateways/<name>/gateway.sh` sets these variables and defines four functions:
1015

1116
```sh
1217
GW_KIND=native|docker # informational
18+
19+
# Self-describing metadata — charts.py + the report tables read these straight from the manifest.
20+
GW_DISPLAY="Busbar" # label shown in charts and the report table
21+
GW_LANG=Rust # implementation language → bar color bucket (Rust|Go|Python|Node|Other)
22+
GW_REPO=https://github.com/GetBusbar/busbar # the gateway name in the table links here
23+
1324
GW_PORT=8080 # port the gateway listens on
1425
GW_PATH=/v1/chat/completions # request path used to probe + load it
1526
GW_MODEL=gpt-4o-mini # model string put in the request body
@@ -21,6 +32,11 @@ gw_rss() { :; } # echo current resident memory in MiB
2132
gw_stop() { :; } # stop + clean up
2233
```
2334

35+
`GW_LANG` colors the bars by language (Rust / Go / Python / Node / Other — anything else, e.g.
36+
OpenResty/Lua or an Envoy/C++ data plane, folds into Other). There is **no winner highlight**: charts
37+
sort by the measured value, so the best is already the top bar. A gateway that didn't serve is drawn
38+
grey regardless of language.
39+
2440
The runner exports for you: `$MOCK_PORT` (deterministic mock upstream), `$CORES` (cpu pin),
2541
`$GW_DIR` (this gateway's directory, for config files).
2642

@@ -30,12 +46,12 @@ and it just works. The mock answers both shapes (OpenAI by default, Anthropic fo
3046

3147
## Shipped gateways
3248

33-
**In the default run** (serve the mock as a single-box drop-in):
34-
35-
Listed alphabetically — no gateway is seated first.
49+
**In the default run** (serve the mock as a single-box drop-in). This table is illustrative — the
50+
actual field is whatever dirs exist here; alphabetical, no gateway seated first.
3651

3752
| dir | what | notes |
3853
|---|---|---|
54+
| `agentgateway/` | agentgateway (Rust data plane, docker) | `ai` backend `hostOverride`/`pathOverride` → mock; no backendAuth/backendTLS; observability off |
3955
| `apisix/` | Apache APISIX + `ai-proxy` (docker, DB-less standalone) | `override.endpoint` → mock; no etcd; access log off, workers = pinned cores |
4056
| `arch/` | Arch (Katanemo, `archgw` CLI) | Envoy + Arch services in one arm64 container; egress-only config → mock; containers pinned to the gateway cores |
4157
| `bifrost/` | maximhq/bifrost (docker) | openai provider base_url → mock; runs its stock config |

‎gateways/agentgateway/gateway.sh‎

Lines changed: 62 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,62 @@
1+
#!/usr/bin/env bash
2+
# SPDX-License-Identifier: Apache-2.0
3+
# Gateway manifest: agentgateway (agentgateway/agentgateway, Rust data plane, docker).
4+
#
5+
# OpenAI-compatible bare proxy: the `ai` backend's openAI provider takes hostOverride + pathOverride
6+
# to point the upstream at the mock. No backendAuth (no key added) and no backendTLS (plain HTTP to the
7+
# mock). Tracing is off unless an OTLP endpoint is set; metrics are pull-only; admin/stats pinned to
8+
# loopback and RUST_LOG=error → clean overhead. AGENTGATEWAY_IMAGE is pinned in gateways/versions.env.
9+
GW_KIND=docker
10+
# Self-describing manifest metadata — charts.py + the run lists read these, so a gateway
11+
# is fully defined by its own dir (add/remove a dir → it appears/disappears everywhere).
12+
GW_DISPLAY="agentgateway" # label in charts + report tables
13+
GW_LANG=Rust # implementation language → bar color bucket
14+
GW_REPO=https://github.com/agentgateway/agentgateway # linked from the gateway name in the report table
15+
GW_PORT=8080
16+
GW_PATH=/v1/chat/completions
17+
GW_MODEL=gpt-4o-mini
18+
GW_AUTH=dummy
19+
AGENTGATEWAY_IMAGE="${AGENTGATEWAY_IMAGE:-ghcr.io/agentgateway/agentgateway:v1.3.1}"
20+
21+
gw_version() {
22+
local dg; dg=$(sudo docker inspect --format '{{index .RepoDigests 0}}' "$AGENTGATEWAY_IMAGE" 2>/dev/null)
23+
echo "${AGENTGATEWAY_IMAGE}${dg:+ (@${dg##*@})}"
24+
}
25+
26+
gw_build() {
27+
cat > "$GW_DIR/config.gen.yaml" <<YAML
28+
binds:
29+
- port: $GW_PORT
30+
listeners:
31+
- name: llm
32+
protocol: HTTP
33+
routes:
34+
- backends:
35+
- ai:
36+
name: openai
37+
hostOverride: "127.0.0.1:$MOCK_PORT"
38+
pathOverride: $GW_PATH
39+
provider:
40+
openAI:
41+
model: $GW_MODEL
42+
YAML
43+
sudo docker pull "$AGENTGATEWAY_IMAGE" >/dev/null 2>&1 || true
44+
}
45+
46+
gw_launch() {
47+
sudo docker rm -f agentgateway-bench >/dev/null 2>&1; sleep 1
48+
sudo docker run -d --name agentgateway-bench --network host --cpuset-cpus="$CORES" \
49+
-e ADMIN_ADDR=127.0.0.1:15000 -e STATS_ADDR=127.0.0.1:15020 -e RUST_LOG=error \
50+
-v "$GW_DIR/config.gen.yaml:/config.yaml:ro" \
51+
"$AGENTGATEWAY_IMAGE" -f /config.yaml >"$GW_DIR/launch.log" 2>&1 || true
52+
}
53+
54+
gw_rss() { container_rss_mib agentgateway-bench; } # summed process-tree VmRSS (same method as native)
55+
56+
gw_diag() {
57+
echo "container: $(sudo docker ps -a --filter name=agentgateway-bench --format '{{.Status}}' 2>/dev/null)"
58+
echo "run.log: $(cat "$GW_DIR/launch.log" 2>/dev/null | tr '\n' ' ' | head -c 300)"
59+
echo "logs:"; sudo docker logs --tail 25 agentgateway-bench 2>&1
60+
}
61+
62+
gw_stop() { sudo docker rm -f agentgateway-bench >/dev/null 2>&1; }

‎gateways/apisix/gateway.sh‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,11 @@
77
# logging off, worker_processes = pinned core count, no observability plugins → pure proxy overhead.
88
# APISIX_IMAGE is pinned in gateways/versions.env.
99
GW_KIND=docker
10+
# Self-describing manifest metadata — charts.py + the run lists read these, so a gateway
11+
# is fully defined by its own dir (add/remove a dir → it appears/disappears everywhere).
12+
GW_DISPLAY="APISIX" # label in charts + report tables
13+
GW_LANG=Other # implementation language → bar color bucket
14+
GW_REPO=https://github.com/apache/apisix # linked from the gateway name in the report table
1015
GW_PORT=9080
1116
GW_PATH=/v1/chat/completions
1217
GW_MODEL=gpt-4o-mini

‎gateways/arch/gateway.sh‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,11 @@
77
# prompt_targets / routing / guards, so the guard/router models never run (pure proxy overhead). The
88
# CLI runs the container(s), which we then pin to $CORES. ARCH_VERSION is in gateways/versions.env.
99
GW_KIND=docker
10+
# Self-describing manifest metadata — charts.py + the run lists read these, so a gateway
11+
# is fully defined by its own dir (add/remove a dir → it appears/disappears everywhere).
12+
GW_DISPLAY="Arch" # label in charts + report tables
13+
GW_LANG=Other # implementation language → bar color bucket
14+
GW_REPO=https://github.com/katanemo/archgw # linked from the gateway name in the report table
1015
GW_PORT=12000
1116
GW_PATH=/v1/chat/completions
1217
GW_MODEL=openai/gpt-4o-mini

‎gateways/bifrost/gateway.sh‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,11 @@
22
# SPDX-License-Identifier: Apache-2.0
33
# Gateway manifest: Bifrost (maximhq/bifrost, docker), its documented pool config.
44
GW_KIND=docker
5+
# Self-describing manifest metadata — charts.py + the run lists read these, so a gateway
6+
# is fully defined by its own dir (add/remove a dir → it appears/disappears everywhere).
7+
GW_DISPLAY="Bifrost" # label in charts + report tables
8+
GW_LANG=Go # implementation language → bar color bucket
9+
GW_REPO=https://github.com/maximhq/bifrost # linked from the gateway name in the report table
510
GW_PORT=8080
611
GW_PATH=/v1/chat/completions
712
GW_MODEL=gpt-4o-mini

‎gateways/busbar/gateway.sh‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,11 @@
77
# process so we measure the real process RSS/latency. Override with BUSBAR_BIN to point at a local
88
# working-tree build instead. Config: token auth, one model to the mock — no special setup.
99
GW_KIND=native
10+
# Self-describing manifest metadata — charts.py + the run lists read these, so a gateway
11+
# is fully defined by its own dir (add/remove a dir → it appears/disappears everywhere).
12+
GW_DISPLAY="Busbar" # label in charts + report tables
13+
GW_LANG=Rust # implementation language → bar color bucket
14+
GW_REPO=https://github.com/GetBusbar/busbar # linked from the gateway name in the report table
1015
GW_PORT=8080
1116
GW_PATH=/v1/chat/completions
1217
GW_MODEL=gpt-4o-mini

‎gateways/gomodel/gateway.sh‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,11 @@
77
# unset) for a pure proxy-overhead measurement — the default posture. Image pinned in
88
# gateways/versions.env; the resolved tag is recorded in the result.
99
GW_KIND=docker
10+
# Self-describing manifest metadata — charts.py + the run lists read these, so a gateway
11+
# is fully defined by its own dir (add/remove a dir → it appears/disappears everywhere).
12+
GW_DISPLAY="GoModel" # label in charts + report tables
13+
GW_LANG=Go # implementation language → bar color bucket
14+
GW_REPO=https://github.com/ENTERPILOT/GOModel # linked from the gateway name in the report table
1015
GW_PORT=8080
1116
GW_PATH=/v1/chat/completions
1217
GW_MODEL=gpt-4o-mini

‎gateways/helicone/gateway.sh‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,11 @@
88
# gateways/versions.env. Pure-proxy mode: helicone.features=none → no control plane, no auth, no key
99
# required; the built-in `openai` provider's base-url is overridden to the mock.
1010
GW_KIND=native
11+
# Self-describing manifest metadata — charts.py + the run lists read these, so a gateway
12+
# is fully defined by its own dir (add/remove a dir → it appears/disappears everywhere).
13+
GW_DISPLAY="Helicone" # label in charts + report tables
14+
GW_LANG=Rust # implementation language → bar color bucket
15+
GW_REPO=https://github.com/Helicone/ai-gateway # linked from the gateway name in the report table
1116
GW_PORT=8787
1217
GW_PATH=/router/default/chat/completions
1318
# Helicone's router requires the unified "{provider}/{model}" form (it errors on a bare model name);

‎gateways/kong/gateway.sh‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,11 @@
77
# DB-less declarative config, generated in gw_launch against the runner's mock port. KONG_IMAGE is
88
# pinned in gateways/versions.env.
99
GW_KIND=docker
10+
# Self-describing manifest metadata — charts.py + the run lists read these, so a gateway
11+
# is fully defined by its own dir (add/remove a dir → it appears/disappears everywhere).
12+
GW_DISPLAY="Kong" # label in charts + report tables
13+
GW_LANG=Other # implementation language → bar color bucket
14+
GW_REPO=https://github.com/Kong/kong # linked from the gateway name in the report table
1015
GW_PORT=8080
1116
GW_PATH=/v1/chat/completions
1217
GW_MODEL=gpt-4o-mini

0 commit comments

Comments
 (0)