Skip to content

Commit 9d798ff

Browse files
Louie Luclaude
andcommitted
Add model price profiles; exact price matching; charge unmatched models the max price
- model_cost/model_profiles.yaml is now the source of truth for prices: one profile per model with aliases; model_cost.json is generated from it by scripts/pricing/build_model_cost.py (--check verifies it is in sync). - Duplicate spellings are folded into aliases where that moves no score. Three models stay double-profiled with a conflict note (qwen3-235b, claude-haiku-4.5, qwen3-30b) because different board entries are charged different prices for them today. - Evaluator: price lookup is exact id/alias only (the substring fallback is removed). Spellings that relied on it are added as explicit aliases. - A model with no price profile is charged the highest input/output price in the table instead of $0, and an unknown model name is graded and charged rather than dropped. The submission check warns instead of failing. - Re-pricing all 265,568 rows in router_inference/predictions gives identical costs before and after. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
1 parent f371ef0 commit 9d798ff

7 files changed

Lines changed: 777 additions & 92 deletions

File tree

‎llm_evaluation/evaluate_models.py‎

Lines changed: 46 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -16,6 +16,7 @@
1616
import json
1717
import argparse
1818
import glob
19+
import threading
1920
from typing import Dict, List, Any, Optional
2021
import sys
2122

@@ -87,6 +88,10 @@ def __init__(
8788
str, Any
8889
] = {} # Store existing results for incremental evaluation
8990
self.cost_config: Dict[str, Any] = {} # Store cost configuration
91+
# Model names charged at the maximum price because no price profile
92+
# matched them -> number of rows.
93+
self.unpriced_models: Dict[str, int] = {}
94+
self._unpriced_lock = threading.Lock()
9095
self.num_workers = num_workers
9196

9297
# Load dataset configurations
@@ -207,21 +212,39 @@ def load_cost_config(self):
207212
self.cost_config = {}
208213

209214
def _lookup_cost_info(self, model_name: str):
210-
"""Find the pricing entry for a model name, trying an exact match first
211-
and then a substring fallback (historical behaviour). Returns the cost
212-
dict, or None if no price is known."""
215+
"""Find the pricing entry for a model name by exact match on a model id
216+
or alias (see model_cost/model_profiles.yaml). Returns the cost dict, or
217+
None if no price is known.
218+
219+
There is deliberately no fuzzy/substring fallback: it used to let a name
220+
pick up whichever price key happened to be a substring of it."""
213221
if not self.cost_config or not model_name:
214222
return None
215223
# Remove _batch suffix if present for cost lookup
216224
cost_lookup_name = (
217225
model_name[:-6] if model_name.endswith("_batch") else model_name
218226
)
219-
if cost_lookup_name in self.cost_config:
220-
return self.cost_config[cost_lookup_name]
221-
for config_name in self.cost_config.keys():
222-
if config_name in cost_lookup_name or cost_lookup_name in config_name:
223-
return self.cost_config[config_name]
224-
return None
227+
return self.cost_config.get(cost_lookup_name)
228+
229+
def max_price_info(self) -> Dict[str, float]:
230+
"""The highest input and output prices in the cost table. Used to charge
231+
rows whose model has no price profile, so an unregistered or misspelled
232+
model is never cheaper than a registered one."""
233+
if not self.cost_config:
234+
return {
235+
"input_token_price_per_million": 0.0,
236+
"output_token_price_per_million": 0.0,
237+
}
238+
return {
239+
"input_token_price_per_million": max(
240+
float(v.get("input_token_price_per_million", 0.0))
241+
for v in self.cost_config.values()
242+
),
243+
"output_token_price_per_million": max(
244+
float(v.get("output_token_price_per_million", 0.0))
245+
for v in self.cost_config.values()
246+
),
247+
}
225248

226249
def has_price(self, model_name: str) -> bool:
227250
"""Whether a price is known for this model name. Used to decide whether a
@@ -248,14 +271,22 @@ def calculate_inference_cost(
248271
cost_info = self._lookup_cost_info(model_name)
249272

250273
if not cost_info:
251-
print(
252-
f"Warning: No cost configuration found for model {model_name} (lookup: {cost_lookup_name})"
253-
)
254-
if len(self.cost_config) > 0:
274+
# Unmatched model: charge the highest price in the table rather than
275+
# dropping the cost, and warn once per model name.
276+
cost_info = self.max_price_info()
277+
with self._unpriced_lock:
278+
first_time = cost_lookup_name not in self.unpriced_models
279+
self.unpriced_models[cost_lookup_name] = (
280+
self.unpriced_models.get(cost_lookup_name, 0) + 1
281+
)
282+
if first_time:
255283
print(
256-
f"Available cost config keys (first 10): {list(self.cost_config.keys())[:10]}"
284+
f"Warning: No price profile for model {cost_lookup_name!r}; "
285+
"charging the maximum price "
286+
f"(${cost_info['input_token_price_per_million']}/"
287+
f"${cost_info['output_token_price_per_million']} per 1M tokens). "
288+
"Add it to model_cost/model_profiles.yaml."
257289
)
258-
return 0.0
259290

260291
# Calculate cost
261292
input_tokens = token_usage.get("input_tokens", 0) or 0

‎llm_evaluation/run.py‎

Lines changed: 20 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -362,14 +362,18 @@ def evaluate_single_prediction(
362362
)
363363
return False
364364

365-
# Convert to universal model name
365+
# Convert to universal model name. An unknown name is still graded (the score
366+
# does not depend on it) and is charged the maximum price below, so a router
367+
# cannot drop rows from its results by naming an unregistered model.
366368
try:
367369
universal_model_name = ModelNameManager.get_universal_name(model_name)
368370
except Exception as e:
369-
logger.error(
370-
f"Error converting model name '{model_name}' to universal name: {e}"
371+
# Logged once per name by the evaluator and summarised after the run.
372+
logger.debug(
373+
f"Unknown model name '{model_name}' ({e}); evaluating it anyway "
374+
"and charging the maximum price."
371375
)
372-
return False
376+
universal_model_name = model_name
373377

374378
# Determine dataset name from global_index
375379
dataset_name = evaluator.determine_dataset_from_global_index(global_index)
@@ -586,6 +590,18 @@ def save_callback():
586590
logger.info(
587591
f"Predictions saved to: ./router_inference/predictions/{router_name}.json"
588592
)
593+
if evaluator.unpriced_models:
594+
max_price = evaluator.max_price_info()
595+
logger.warning(
596+
"Charged at the maximum price "
597+
f"(${max_price['input_token_price_per_million']}/"
598+
f"${max_price['output_token_price_per_million']} per 1M tokens) "
599+
"because no price profile matched:"
600+
)
601+
for name, rows in sorted(
602+
evaluator.unpriced_models.items(), key=lambda kv: -kv[1]
603+
):
604+
logger.warning(f" {name}: {rows} rows")
589605
logger.info("=" * 60)
590606

591607
# Compute and display router-level metrics

0 commit comments

Comments
 (0)