docs: sanitize live deployment configuration

Require live Azure identifiers at runtime, document deployed-model execution, and keep retail pricing region configurable.

Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com>

Author: leestott

.env.example

@@ -16,6 +16,10 @@ AZURE_JUDGE_ENDPOINT=https://your-resource.openai.azure.com
 AZURE_JUDGE_KEY=your-azure-openai-api-key
 AZURE_JUDGE_DEPLOYMENT=gpt-5
 
+# ARM region of the Foundry resource, used to select matching Azure Retail
+# Prices API meters (for example: eastus, swedencentral, westeurope).
+AZURE_PRICING_REGION=eastus
+
 # Microsoft Foundry (optional — for cloud evaluation via Foundry SDK)
 AZURE_AI_PROJECT_ENDPOINT=https://your-project.services.ai.azure.com
 AZURE_AI_MODEL_DEPLOYMENT_NAME=gpt-5

README.md

@@ -69,6 +69,61 @@ bash scripts/demo.sh
 
 See [QUICKSTART.md](QUICKSTART.md) for the CLI quickstart details.
 
+### Run Against Deployed Models
+
+The live demo calls an existing Microsoft Foundry Model Router deployment,
+baseline deployment, and judge deployment. It consumes billable tokens.
+
+#### Prerequisites
+
+- Complete the installation steps below so `.venv` exists.
+- Install the [Azure CLI](https://learn.microsoft.com/cli/azure/install-azure-cli).
+- Sign in with `az login`.
+- Ensure the three deployments exist in the same Foundry resource.
+- Ensure your identity can list keys for that resource.
+
+Run the evaluation from PowerShell:
+
+```powershell
+az login
+
+.\scripts\demo.ps1 -Live `
+  -Subscription "<subscription-id>" `
+  -ResourceGroup "<resource-group>" `
+  -ResourceName "<foundry-resource-name>" `
+  -RouterDeployment "<model-router-deployment>" `
+  -BaselineDeployment "<baseline-model-deployment>" `
+  -JudgeDeployment "<judge-model-deployment>"
+```
+
+The script:
+
+1. Selects the Azure subscription.
+2. Reads the Foundry resource endpoint, access key, and region with Azure CLI.
+3. Refreshes supported model prices from the Azure Retail Prices API.
+4. Evaluates all 25 prompts in `datasets/zava_custom.jsonl`.
+5. Opens the generated `dashboard.html`.
+
+To resume an interrupted live evaluation, add `-Resume`.
+
+> The judge should ideally be a deployment distinct from both evaluated
+> endpoints to reduce self-preference bias.
+
+If the Router, baseline, and judge use different Azure resources, configure
+their endpoints and keys in `.env`, then use the standard command:
+
+```powershell
+Copy-Item .env.example .env
+# Edit .env with the three deployed endpoints, keys, deployment names,
+# and AZURE_PRICING_REGION.
+
+.\.venv\Scripts\python.exe scripts\run_eval.py `
+  --config configs\live_demo.yaml
+```
+
+See [Run a Live Evaluation](docs/how-to-run-live-eval.md) for dataset,
+configuration, security, and troubleshooting details.
+
 ### 1. Install
 
 **Prerequisites:** Python 3.9+, a Microsoft Foundry Model Router endpoint, and an Azure OpenAI baseline endpoint.
@@ -156,11 +211,13 @@ To swap the baseline or judge model, update the deployment name in `.env` or ove
 
 ```yaml
 # configs/default.yaml
-baseline:
-  deployment: gpt-5          # change to any Azure OpenAI deployment
+endpoints:
+  baseline:
+    deployment_name: gpt-5
 judge:
-  deployment: gpt-5          # model used for LLM-as-a-judge scoring
-  concurrency: 3             # parallel judge calls
+  endpoint:
+    deployment_name: gpt-5
+  max_parallel: 3
 ```
 
 See [configs/](configs/) for all presets (`quick_test`, `large_scale`, `foundry`).

configs/live_demo.yaml

@@ -1,4 +1,4 @@
-# Live evaluation using the <your-foundry-resource> Foundry deployments.
+# Live evaluation using user-supplied Foundry deployments.
 # Credentials are injected into the process environment by scripts/demo.ps1.
 evaluation:
   name: "live-model-router-demo"
@@ -27,7 +27,7 @@ pricing:
   model_router:
     input: 0.50
     output: 1.50
-  <your-baseline-deployment>:
+  baseline:
     input: 2.50
     output: 15.00
   gpt-5.4:
@@ -51,7 +51,7 @@ pricing_source:
   azure_retail:
     enabled: true
     currency: "USD"
-    region: "eastus"
+    region: "${AZURE_PRICING_REGION}"
     cache_file: ".cache/azure-retail-prices.json"
     cache_ttl_hours: 24
     timeout_seconds: 20

scripts/demo.ps1

@@ -3,13 +3,19 @@
 #   .\scripts\demo.ps1
 #   .\scripts\demo.ps1 -Live
 #   .\scripts\demo.ps1 -Live -Resume
+#   .\scripts\demo.ps1 -Live -Subscription <id> -ResourceGroup <rg> `
+#       -ResourceName <name> -RouterDeployment <router> `
+#       -BaselineDeployment <baseline> -JudgeDeployment <judge>
 
 param(
     [switch]$Live,
     [switch]$Resume,
-    [string]$Subscription = "<your-subscription-id>",
-    [string]$ResourceGroup = "<your-resource-group>",
-    [string]$ResourceName = "<your-foundry-resource>"
+    [string]$Subscription,
+    [string]$ResourceGroup,
+    [string]$ResourceName,
+    [string]$RouterDeployment,
+    [string]$BaselineDeployment,
+    [string]$JudgeDeployment
 )
 
 $ErrorActionPreference = "Stop"
@@ -46,10 +52,27 @@ if ($LASTEXITCODE -ne 0) {
 Push-Location $projectRoot
 try {
     if ($Live) {
+        $requiredParameters = @{
+            Subscription = $Subscription
+            ResourceGroup = $ResourceGroup
+            ResourceName = $ResourceName
+            RouterDeployment = $RouterDeployment
+            BaselineDeployment = $BaselineDeployment
+            JudgeDeployment = $JudgeDeployment
+        }
+        $missingParameters = @(
+            $requiredParameters.GetEnumerator() |
+                Where-Object { [string]::IsNullOrWhiteSpace([string]$_.Value) } |
+                ForEach-Object { "-$($_.Key)" }
+        )
+        if ($missingParameters.Count -gt 0) {
+            throw "Live mode requires: $($missingParameters -join ', ')."
+        }
+
         Write-Host "This calls real deployments and consumes billable tokens." -ForegroundColor Yellow
-        Write-Host "  Router:   <your-router-deployment>"
-        Write-Host "  Baseline: <your-baseline-deployment> (GPT-5.4)"
-        Write-Host "  Judge:    <your-baseline-deployment> (GPT-5.4)"
+        Write-Host "  Router:   $RouterDeployment"
+        Write-Host "  Baseline: $BaselineDeployment"
+        Write-Host "  Judge:    $JudgeDeployment"
         Write-Host ""
 
         $az = Get-Command az -ErrorAction SilentlyContinue
@@ -73,16 +96,26 @@ try {
             throw "Unable to retrieve an access key for Foundry resource '$ResourceName'."
         }
 
+        $resourceLocation = az cognitiveservices account show `
+            --resource-group $ResourceGroup `
+            --name $ResourceName `
+            --query location `
+            --output tsv
+        if ($LASTEXITCODE -ne 0 -or [string]::IsNullOrWhiteSpace($resourceLocation)) {
+            throw "Unable to determine the Azure region for Foundry resource '$ResourceName'."
+        }
+
         $endpoint = "https://$ResourceName.cognitiveservices.azure.com/"
         $env:AZURE_MODEL_ROUTER_ENDPOINT = $endpoint
         $env:AZURE_MODEL_ROUTER_KEY = $accountKey
-        $env:AZURE_MODEL_ROUTER_DEPLOYMENT = "<your-router-deployment>"
+        $env:AZURE_MODEL_ROUTER_DEPLOYMENT = $RouterDeployment
         $env:AZURE_OPENAI_ENDPOINT = $endpoint
         $env:AZURE_OPENAI_KEY = $accountKey
-        $env:AZURE_BASELINE_DEPLOYMENT = "<your-baseline-deployment>"
+        $env:AZURE_BASELINE_DEPLOYMENT = $BaselineDeployment
         $env:AZURE_JUDGE_ENDPOINT = $endpoint
         $env:AZURE_JUDGE_KEY = $accountKey
-        $env:AZURE_JUDGE_DEPLOYMENT = "<your-baseline-deployment>"
+        $env:AZURE_JUDGE_DEPLOYMENT = $JudgeDeployment
+        $env:AZURE_PRICING_REGION = $resourceLocation
 
         $evalArgs = @("scripts\run_eval.py", "--config", "configs\live_demo.yaml")
         if ($Resume) {
@@ -128,6 +161,7 @@ try {
     Remove-Item Env:\AZURE_MODEL_ROUTER_KEY -ErrorAction SilentlyContinue
     Remove-Item Env:\AZURE_OPENAI_KEY -ErrorAction SilentlyContinue
     Remove-Item Env:\AZURE_JUDGE_KEY -ErrorAction SilentlyContinue
+    Remove-Item Env:\AZURE_PRICING_REGION -ErrorAction SilentlyContinue
     Pop-Location
 }