Skip to content

Commit 77df628

Browse files
MattJacksonMattJackson
committed
bench: trainer-variance matrix defs (new-trainer x2, old-trainer rebuild)
1 parent 1344cb2 commit 77df628

15 files changed

Lines changed: 429 additions & 0 deletions

File tree

‎gateways/busbar-160-w123o2/config.gen.yaml.tmpl‎

Lines changed: 33 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,33 @@
1+
listen: 127.0.0.1:{GW_PORT}
2+
providers:
3+
mock-openai:
4+
api_key:
5+
env: BENCH_MOCK_KEY
6+
mock-responses:
7+
api_key:
8+
env: BENCH_MOCK_KEY
9+
mock-anthropic:
10+
api_key:
11+
env: BENCH_MOCK_KEY
12+
mock-gemini:
13+
api_key:
14+
env: BENCH_MOCK_KEY
15+
mock-cohere:
16+
api_key:
17+
env: BENCH_MOCK_KEY
18+
mock-bedrock:
19+
api_key:
20+
env: BENCH_BEDROCK_KEY
21+
models:
22+
gpt-4o-mini:
23+
provider: mock-openai
24+
gpt-4o-mini-responses:
25+
provider: mock-responses
26+
gpt-4o-mini-anthropic:
27+
provider: mock-anthropic
28+
gpt-4o-mini-gemini:
29+
provider: mock-gemini
30+
gpt-4o-mini-cohere:
31+
provider: mock-cohere
32+
gpt-4o-mini-bedrock:
33+
provider: mock-bedrock

‎gateways/busbar-160-w123o2/definition.json‎

Lines changed: 66 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,66 @@
1+
{
2+
"auth": "bench-token",
3+
"class": "Control plane",
4+
"config": [],
5+
"config_files": [
6+
{
7+
"output": "providers.gen.yaml",
8+
"template": "providers.gen.yaml.tmpl"
9+
},
10+
{
11+
"output": "config.gen.yaml",
12+
"template": "config.gen.yaml.tmpl"
13+
}
14+
],
15+
"display": "Busbar 1.6.0 w123o2",
16+
"egress": [
17+
"openai",
18+
"openai-responses",
19+
"anthropic",
20+
"gemini",
21+
"cohere",
22+
"bedrock"
23+
],
24+
"egress_models": {
25+
"anthropic": "gpt-4o-mini-anthropic",
26+
"bedrock": "gpt-4o-mini-bedrock",
27+
"cohere": "gpt-4o-mini-cohere",
28+
"gemini": "gpt-4o-mini-gemini",
29+
"openai": "gpt-4o-mini",
30+
"openai-responses": "gpt-4o-mini-responses"
31+
},
32+
"lang": "Rust",
33+
"launch": {
34+
"args": [],
35+
"image": "getbusbar/busbar:qa-160-w123o2-1f246e58",
36+
"kind": "docker",
37+
"mounts": [
38+
{
39+
"container_path": "/etc/busbar/config.yaml",
40+
"host_path": "config.gen.yaml"
41+
},
42+
{
43+
"container_path": "/etc/busbar/providers.yaml",
44+
"host_path": "providers.gen.yaml"
45+
}
46+
]
47+
},
48+
"matrix": [
49+
"111111",
50+
"111111",
51+
"111111",
52+
"111111",
53+
"111111",
54+
"111111"
55+
],
56+
"matrix_note": "busbar declares full 6x6 translation support",
57+
"model": "gpt-4o-mini",
58+
"name": "busbar-160-w123o2",
59+
"path": "/v1/chat/completions",
60+
"port": 8080,
61+
"repo": "https://github.com/GetBusbar/busbar",
62+
"runtime": {
63+
"container": "busbar-160-w123o2-bench",
64+
"kind": "docker"
65+
}
66+
}

‎gateways/busbar-160-w123o2/env‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,3 @@
1+
BENCH_MOCK_KEY=dummy
2+
BENCH_BEDROCK_KEY=AKIAMOCKACCESSKEY:mock-secret-access-key
3+
BUSBAR_STATE_FILE=

‎gateways/busbar-160-w123o2/providers.gen.yaml.tmpl‎

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,18 @@
1+
mock-openai:
2+
protocol: openai
3+
base_url: http://127.0.0.1:{MOCK_PORT}
4+
mock-responses:
5+
protocol: responses
6+
base_url: http://127.0.0.1:{MOCK_PORT}
7+
mock-anthropic:
8+
protocol: anthropic
9+
base_url: http://127.0.0.1:{MOCK_PORT}
10+
mock-gemini:
11+
protocol: gemini
12+
base_url: http://127.0.0.1:{MOCK_PORT}
13+
mock-cohere:
14+
protocol: cohere
15+
base_url: http://127.0.0.1:{MOCK_PORT}
16+
mock-bedrock:
17+
protocol: bedrock
18+
base_url: http://127.0.0.1:{MOCK_PORT}

‎gateways/busbar-160-w123o2/why‎

Lines changed: 23 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,23 @@
1+
# Why every setting, env var, header and command this gateway uses is here.
2+
#
3+
# The rule, from the published methodology: a setting may appear only if it is needed to boot the
4+
# process at all, to point an upstream at the test mock instead of a real provider, to expose an
5+
# ingress path the matrix exercises, or to bind the port and cores the rig requires. One claim per
6+
# line: <pattern> <boot|upstream|ingress|bind>. Anything unclaimed is a lint failure, and a claim
7+
# that matches nothing is a stale claim and also a failure.
8+
#
9+
# Each `boot` claim below was checked against the published image: the setting was removed, and kept
10+
# only where the image then failed to validate or run.
11+
listen bind # the port the harness drives
12+
providers boot # a required field: omitting it fails with "missing field `providers`"
13+
mock-* upstream # one provider per protocol, each base_url -> the mock
14+
protocol upstream # which upstream dialect that provider speaks
15+
base_url upstream # -> the mock
16+
api_key boot # the secret reference busbar reads; bedrock's SigV4 lane cannot sign without it
17+
env boot # names the variable holding that credential
18+
models ingress # the client-facing model table
19+
gpt-4o-mini* ingress # the model name each matrix column asks for; selects that column's provider
20+
provider upstream # which provider backs this model
21+
BENCH_MOCK_KEY boot # the credential api_key names (dummy; the mock ignores it)
22+
BENCH_BEDROCK_KEY boot # bedrock SigV4 ACCESS:SECRET pair, same
23+
BUSBAR_STATE_FILE boot # empty disables state persistence, whose default path is the read-only /etc/busbar

‎gateways/busbar-160-w123t/config.gen.yaml.tmpl‎

Lines changed: 33 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,33 @@
1+
listen: 127.0.0.1:{GW_PORT}
2+
providers:
3+
mock-openai:
4+
api_key:
5+
env: BENCH_MOCK_KEY
6+
mock-responses:
7+
api_key:
8+
env: BENCH_MOCK_KEY
9+
mock-anthropic:
10+
api_key:
11+
env: BENCH_MOCK_KEY
12+
mock-gemini:
13+
api_key:
14+
env: BENCH_MOCK_KEY
15+
mock-cohere:
16+
api_key:
17+
env: BENCH_MOCK_KEY
18+
mock-bedrock:
19+
api_key:
20+
env: BENCH_BEDROCK_KEY
21+
models:
22+
gpt-4o-mini:
23+
provider: mock-openai
24+
gpt-4o-mini-responses:
25+
provider: mock-responses
26+
gpt-4o-mini-anthropic:
27+
provider: mock-anthropic
28+
gpt-4o-mini-gemini:
29+
provider: mock-gemini
30+
gpt-4o-mini-cohere:
31+
provider: mock-cohere
32+
gpt-4o-mini-bedrock:
33+
provider: mock-bedrock

‎gateways/busbar-160-w123t/definition.json‎

Lines changed: 66 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,66 @@
1+
{
2+
"auth": "bench-token",
3+
"class": "Control plane",
4+
"config": [],
5+
"config_files": [
6+
{
7+
"output": "providers.gen.yaml",
8+
"template": "providers.gen.yaml.tmpl"
9+
},
10+
{
11+
"output": "config.gen.yaml",
12+
"template": "config.gen.yaml.tmpl"
13+
}
14+
],
15+
"display": "Busbar 1.6.0 w123t",
16+
"egress": [
17+
"openai",
18+
"openai-responses",
19+
"anthropic",
20+
"gemini",
21+
"cohere",
22+
"bedrock"
23+
],
24+
"egress_models": {
25+
"anthropic": "gpt-4o-mini-anthropic",
26+
"bedrock": "gpt-4o-mini-bedrock",
27+
"cohere": "gpt-4o-mini-cohere",
28+
"gemini": "gpt-4o-mini-gemini",
29+
"openai": "gpt-4o-mini",
30+
"openai-responses": "gpt-4o-mini-responses"
31+
},
32+
"lang": "Rust",
33+
"launch": {
34+
"args": [],
35+
"image": "getbusbar/busbar:qa-160-w123t-0bd9a74e",
36+
"kind": "docker",
37+
"mounts": [
38+
{
39+
"container_path": "/etc/busbar/config.yaml",
40+
"host_path": "config.gen.yaml"
41+
},
42+
{
43+
"container_path": "/etc/busbar/providers.yaml",
44+
"host_path": "providers.gen.yaml"
45+
}
46+
]
47+
},
48+
"matrix": [
49+
"111111",
50+
"111111",
51+
"111111",
52+
"111111",
53+
"111111",
54+
"111111"
55+
],
56+
"matrix_note": "busbar declares full 6x6 translation support",
57+
"model": "gpt-4o-mini",
58+
"name": "busbar-160-w123t",
59+
"path": "/v1/chat/completions",
60+
"port": 8080,
61+
"repo": "https://github.com/GetBusbar/busbar",
62+
"runtime": {
63+
"container": "busbar-160-w123t-bench",
64+
"kind": "docker"
65+
}
66+
}

‎gateways/busbar-160-w123t/env‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,3 @@
1+
BENCH_MOCK_KEY=dummy
2+
BENCH_BEDROCK_KEY=AKIAMOCKACCESSKEY:mock-secret-access-key
3+
BUSBAR_STATE_FILE=

‎gateways/busbar-160-w123t/providers.gen.yaml.tmpl‎

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,18 @@
1+
mock-openai:
2+
protocol: openai
3+
base_url: http://127.0.0.1:{MOCK_PORT}
4+
mock-responses:
5+
protocol: responses
6+
base_url: http://127.0.0.1:{MOCK_PORT}
7+
mock-anthropic:
8+
protocol: anthropic
9+
base_url: http://127.0.0.1:{MOCK_PORT}
10+
mock-gemini:
11+
protocol: gemini
12+
base_url: http://127.0.0.1:{MOCK_PORT}
13+
mock-cohere:
14+
protocol: cohere
15+
base_url: http://127.0.0.1:{MOCK_PORT}
16+
mock-bedrock:
17+
protocol: bedrock
18+
base_url: http://127.0.0.1:{MOCK_PORT}

‎gateways/busbar-160-w123t/why‎

Lines changed: 23 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,23 @@
1+
# Why every setting, env var, header and command this gateway uses is here.
2+
#
3+
# The rule, from the published methodology: a setting may appear only if it is needed to boot the
4+
# process at all, to point an upstream at the test mock instead of a real provider, to expose an
5+
# ingress path the matrix exercises, or to bind the port and cores the rig requires. One claim per
6+
# line: <pattern> <boot|upstream|ingress|bind>. Anything unclaimed is a lint failure, and a claim
7+
# that matches nothing is a stale claim and also a failure.
8+
#
9+
# Each `boot` claim below was checked against the published image: the setting was removed, and kept
10+
# only where the image then failed to validate or run.
11+
listen bind # the port the harness drives
12+
providers boot # a required field: omitting it fails with "missing field `providers`"
13+
mock-* upstream # one provider per protocol, each base_url -> the mock
14+
protocol upstream # which upstream dialect that provider speaks
15+
base_url upstream # -> the mock
16+
api_key boot # the secret reference busbar reads; bedrock's SigV4 lane cannot sign without it
17+
env boot # names the variable holding that credential
18+
models ingress # the client-facing model table
19+
gpt-4o-mini* ingress # the model name each matrix column asks for; selects that column's provider
20+
provider upstream # which provider backs this model
21+
BENCH_MOCK_KEY boot # the credential api_key names (dummy; the mock ignores it)
22+
BENCH_BEDROCK_KEY boot # bedrock SigV4 ACCESS:SECRET pair, same
23+
BUSBAR_STATE_FILE boot # empty disables state persistence, whose default path is the read-only /etc/busbar

0 commit comments

Comments
 (0)