From 6970a5279f0a3de83ecb5c655c90aee18d6e3547 Mon Sep 17 00:00:00 2001 From: mohessie Date: Fri, 21 Aug 2026 00:13:40 +0300 Subject: [PATCH 1/4] TEMP review slice: model the include on every entry core can splice --- .../extensions/azure.ai.evaluations/README.md | 194 ++ .../internal/cmd/catalog.go | 180 ++ .../internal/cmd/catalog_include_test.go | 93 + .../internal/messages/messages.go | 2783 +++++++++++++++++ .../internal/project/config_keys_test.go | 133 + .../internal/project/eval_config.go | 402 +++ .../project/ref_on_every_entry_test.go | 60 + .../internal/project/ref_resolution_test.go | 242 ++ .../project/service_config_strict_test.go | 86 + .../internal/project/service_target_eval.go | 473 +++ .../schemas/azure.ai.eval.json | 259 ++ 11 files changed, 4905 insertions(+) create mode 100644 cli/azd/extensions/azure.ai.evaluations/README.md create mode 100644 cli/azd/extensions/azure.ai.evaluations/internal/cmd/catalog.go create mode 100644 cli/azd/extensions/azure.ai.evaluations/internal/cmd/catalog_include_test.go create mode 100644 cli/azd/extensions/azure.ai.evaluations/internal/messages/messages.go create mode 100644 cli/azd/extensions/azure.ai.evaluations/internal/project/config_keys_test.go create mode 100644 cli/azd/extensions/azure.ai.evaluations/internal/project/eval_config.go create mode 100644 cli/azd/extensions/azure.ai.evaluations/internal/project/ref_on_every_entry_test.go create mode 100644 cli/azd/extensions/azure.ai.evaluations/internal/project/ref_resolution_test.go create mode 100644 cli/azd/extensions/azure.ai.evaluations/internal/project/service_config_strict_test.go create mode 100644 cli/azd/extensions/azure.ai.evaluations/internal/project/service_target_eval.go create mode 100644 cli/azd/extensions/azure.ai.evaluations/schemas/azure.ai.eval.json diff --git a/cli/azd/extensions/azure.ai.evaluations/README.md b/cli/azd/extensions/azure.ai.evaluations/README.md new file mode 100644 index 00000000000..cfd7f71ba27 --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/README.md @@ -0,0 +1,194 @@ +# Azure Developer CLI (azd) Evaluations Extension + +Define Foundry evaluations alongside your agent in `azure.yaml`, deploy them +with `azd up`, and run them from the terminal. + +```bash +azd ai eval init # scaffold evals/ next to your agent +azd ai eval generate # synthesize a rubric and dataset from the agent +azd up # register datasets and evaluators, create the eval group +azd ai eval run # run the evaluation and summarize the results +``` + +## What gets deployed + +Eval resources are one service entry in `azure.yaml`, normally a `$ref` to a +file under `evals/`: + +```yaml +# azure.yaml +services: + ai-project: + host: azure.ai.project + evals: + host: azure.ai.eval + uses: [ai-project] + $ref: ./evals/azure.eval.yaml +``` + +```yaml +# evals/azure.eval.yaml +datasets: + - name: support-golden + file: ./datasets/support-golden.jsonl + +evaluators: + - name: support-quality + source: ./evaluators/support-quality.json + +evals: + - name: support-quality + dataset: support-golden + evaluation_level: turn + evaluators: + - evaluator: builtin.task_adherence + initialization_parameters: + model: gpt-4.1-nano + - evaluator: support-quality + target: + type: agent + name: support-agent +``` + +`azd up` reconciles **datasets ΓåÆ evaluators ΓåÆ eval groups**, in that order, +because a group references the versions the first two resolve to. + +Relative paths inside the `$ref`'d configuration resolve against **that file's** +directory, so `./datasets/x.jsonl` above means `evals/datasets/x.jsonl`. + +That holds for the configuration as a whole. It does **not** hold for a `$ref` +on a single catalog entry: azd rebases only the path keys it owns, so a relative +`source:` written inside `evals/evaluators/quality.yaml` still resolves against +`azure.eval.yaml` and will not be found. An entry pulled in from its own file +should carry the rubric under `definition:` rather than point at a second file: + +```yaml +evaluators: + - $ref: ./evaluators/quality.json # the rubric itself, not a pointer to one + name: quality +``` + +### Repeated deploys do not create redundant versions + +Datasets are fingerprinted locally, because the dataset API exposes no content +hash and comparing against the service would mean downloading the blob on every +deploy. Evaluator definitions are compared against the service, but only on the +keys you authored ΓÇö the service adds `data_schema`, `init_parameters` and +`metrics` of its own. + +Eval groups are immutable, so a change to a group's evaluators, target or +sampling creates a new group and a new id. The id is cached in the azd +environment so repeat runs stay comparable. + +## Commands + +| Group | Commands | +|---|---| +| `azd ai eval` | `init` ┬╖ `generate` ┬╖ `run` | +| `azd ai eval dataset` | `create` ┬╖ `list` ┬╖ `show` ┬╖ `update` ┬╖ `delete` | +| `azd ai eval evaluator` | `upload` ┬╖ `list` ┬╖ `show` ┬╖ `update` ┬╖ `delete` ┬╖ `builtins` | +| `azd ai eval run` | `start` ┬╖ `list` ┬╖ `show` ┬╖ `cancel` | +| `azd ai eval results` | `show` ┬╖ `export` | + +`create` and `update` both publish a new immutable version; the server +auto-increments and nothing mutates in place. + +Every command supports `-o json` and `--no-prompt`, so the whole surface is +usable from CI. + +## Evaluators + +Built-ins need no declaration ΓÇö reference them as `builtin.` and list +them with `azd ai eval evaluator builtins`. + +Evaluators do not share an input contract, so the CLI reads each one's +published contract and shapes the request to match. An evaluator needing an +input your dataset does not carry is reported before the request is sent, with +the column named, rather than as a service-side rejection. + +A custom rubric is a JSON list of weighted dimensions: + +```json +{ + "dimensions": [ + { "id": "accuracy", "description": "The answer is factually correct.", "weight": 5 }, + { "id": "tone", "description": "The answer is polite and professional.", "weight": 2 } + ] +} +``` + +`weight` is an **integer from 1 to 10**. Weights do not need to sum to +anything. + +## Choosing a project + +The project endpoint is resolved in this order: + +1. `--project-endpoint` +2. `FOUNDRY_PROJECT_ENDPOINT` in the active azd environment, then + `AZURE_AI_PROJECT_ENDPOINT` there +3. `extensions.ai-agents.project.context.endpoint` in azd's global config, + which `azure.ai.agents` writes and this extension only reads +4. `FOUNDRY_PROJECT_ENDPOINT` in the host environment, then + `AZURE_AI_PROJECT_ENDPOINT` + +Level 3 is worth knowing about: it is machine-wide rather than per-project, so +a project context left behind by `azd ai agent` somewhere else takes precedence +over the variable exported in this shell. `--debug` prints which level answered. + +## Local development + +### Prerequisites + +- Go (the version in `go.mod`; `GOTOOLCHAIN=auto` fetches it) +- [azd](https://aka.ms/azd) and the extension developer kit: + `azd ext install microsoft.azd.extensions` + +### Build, test, install + +```bash +azd x build # compile and install into the local azd +azd x pack # package the artifacts +azd x publish # register in the local extension source +azd ext install azure.ai.evaluations --source local +``` + +```bash +go test ./internal/... # unit tests +``` + +### Live integration tests + +These talk to a real Foundry project, so they are excluded from the default +build by the `live` tag and additionally gated on an environment variable: + +```bash +export AZURE_AI_EVAL_E2E_LIVE=1 +export FOUNDRY_PROJECT_ENDPOINT=https://.services.ai.azure.com/api/projects/ +export AZURE_AI_EVAL_MODEL=gpt-4.1-nano # optional judge model +export AZURE_AI_EVAL_AGENT= # optional, enables the run phase + +go test -tags live ./internal/cmd/ ./tests/live/ +``` + +They clean up every resource they create. + +### Debug logging + +Request tracing is off by default. `--debug`, or `AZD_EXT_DEBUG=true`, writes +it to a dated log file rather than the terminal. + +## TODO before release + +Both are files the azd extensions team owns, so they are not changed here: + +- [ ] **`cli/azd/extensions/registry.json`** ΓÇö add the `azure.ai.evaluations` + entry. Until it exists `azd extension install azure.ai.evaluations` cannot + resolve, so the extension is only reachable through `azd x pack` + + `azd x publish` into the local source registry. +- [ ] **`.github/CODEOWNERS`** ΓÇö add `/cli/azd/extensions/azure.ai.evaluations/`. + Every sibling Foundry extension has an entry; without one, PRs here get no + reviewer routing. +- [ ] **`microsoft.foundry/extension.yaml`** ΓÇö add the dependency, but only + after the registry entry lands. Declaring a dependency that cannot resolve + breaks installing the bundle. diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/catalog.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/catalog.go new file mode 100644 index 00000000000..c2fe04aaca5 --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/catalog.go @@ -0,0 +1,180 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package cmd + +import ( + "fmt" + "path/filepath" + + "azureaieval/internal/messages" + "azureaieval/internal/project" + + "github.com/spf13/cobra" +) + +// Generation writes the artifact and then names it in the configuration, so +// what it produced is referenceable without a hand edit. Only the catalogs are +// touched: which evals use the artifact is the author's decision, and `init` is +// the command that makes it. + +// addDatasetToCatalog records a generated dataset in `datasets:`. +func addDatasetToCatalog(cmd *cobra.Command, evalDir string, ref *project.ArtifactRef) error { + if ref == nil { + return nil + } + return updateCatalog(cmd, evalDir, "dataset", ref, func(cfg *project.EvalConfig) bool { + for i := range cfg.Datasets { + if cfg.Datasets[i].Name == ref.Name { + // Regeneration overwrites the file in place, so the entry only + // changes when the artifact moved. + if cfg.Datasets[i].File == ref.Source { + return false + } + cfg.Datasets[i].File = ref.Source + return true + } + } + cfg.Datasets = append(cfg.Datasets, project.DatasetDecl{ + Name: ref.Name, + File: ref.Source, + }) + return true + }) +} + +// addEvaluatorToCatalog records a generated evaluator in `evaluators:`. +func addEvaluatorToCatalog(cmd *cobra.Command, evalDir string, ref *project.ArtifactRef) error { + if ref == nil { + return nil + } + return updateCatalog(cmd, evalDir, "evaluator", ref, func(cfg *project.EvalConfig) bool { + for i := range cfg.Evaluators { + if cfg.Evaluators[i].Name == ref.Name { + if cfg.Evaluators[i].Source == ref.Source { + return false + } + cfg.Evaluators[i].Source = ref.Source + return true + } + } + cfg.Evaluators = append(cfg.Evaluators, project.EvaluatorDecl{ + Name: ref.Name, + Source: ref.Source, + }) + return true + }) +} + +// checkNameNotBehindAnInclude refuses a name whose entry lives in another file. +// +// Two shapes reach this. A pure `$ref` has no name here at all, so the duplicate +// scan had nothing to match on and appended a second entry with the same name -- +// a collision that surfaced only on the next resolving read. A `$ref` carrying +// an overlay `name` does match, and updating it in place writes `source:` beside +// the directive, so resolution then produces a rubric and a source and the +// configuration is rejected for declaring it twice. Neither is editable here. +// +// A configuration that will not resolve is left to the commands that resolve it: +// failing a generate over an unrelated broken include would be its own surprise. +func checkNameNotBehindAnInclude(evalDir string, asWritten *project.EvalConfig, kind, name string) error { + if ref, ok := catalogEntryRef(asWritten, kind, name); ok { + if ref != "" { + return messages.CatalogNameBehindAnInclude(kind, name) + } + return nil + } + resolved, err := project.OpenEvalConfig(evalDir) + if err != nil || resolved == nil { + return nil + } + if _, ok := catalogEntryRef(resolved, kind, name); ok { + return messages.CatalogNameBehindAnInclude(kind, name) + } + return nil +} + +// catalogEntryRef returns the include this entry was written as, and whether the +// configuration names it at all. +func catalogEntryRef(cfg *project.EvalConfig, kind, name string) (string, bool) { + if cfg == nil { + return "", false + } + if kind == "dataset" { + if decl, ok := cfg.DatasetDeclaration(name); ok { + return decl.Ref, true + } + return "", false + } + if decl, ok := cfg.EvaluatorDeclaration(name); ok { + return decl.Ref, true + } + return "", false +} + +// updateCatalog applies a change to the configuration and writes it back. +// +// A missing configuration is created holding only the catalog. `generate` runs +// before `init` on the golden path, and a downloaded artifact nobody recorded +// is the one state that goes stale. The file it creates has no evals and no +// azure.yaml entry, so it stays inert until init wires one. +func updateCatalog( + cmd *cobra.Command, + evalDir string, + kind string, + ref *project.ArtifactRef, + apply func(*project.EvalConfig) bool, +) error { + // Held across the read and the write: two generates adding different + // entries would otherwise both read the same state, and the second write + // would drop the first one's entry while reporting success. + unlock, err := project.LockEvalConfig(cmd.Context(), evalDir) + if err != nil { + return err + } + defer unlock() + + cfg, err := project.OpenEvalConfigForEdit(evalDir) + if err != nil { + return err + } + created := cfg == nil + if created { + cfg = &project.EvalConfig{} + } + if err := checkNameNotBehindAnInclude(evalDir, cfg, kind, ref.Name); err != nil { + return err + } + if !apply(cfg) { + return nil + } + + if err := project.SaveEvalConfig(evalDir, cfg); err != nil { + return err + } + if !isJSON(cmd) { + // Resolved, not the current name: SaveEvalConfig writes back over a + // legacy file when that is what the project has, and the line has to + // name the file it actually wrote. + resolved, err := project.ResolveEvalConfigPath(evalDir) + if err != nil { + return err + } + path := filepath.ToSlash(resolved) + if created { + fmt.Fprint(cmd.OutOrStdout(), messages.CreatedCatalogFile(path)) + } + fmt.Fprint(cmd.OutOrStdout(), + messages.AddedToCatalog(kind, describeArtifact(ref), path)) + } + return nil +} + +// describeArtifact names what was recorded, with the published version when the +// job reported one, so a reader can pin it without going to look. +// +// Single-quoted to match the spec's transcripts; the surrounding Done: lines +// carry bare values, but a dataset name can hold a space and these cannot. +func describeArtifact(ref *project.ArtifactRef) string { + return messages.ArtifactDescription(ref.Name, ref.Version) +} diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/catalog_include_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/catalog_include_test.go new file mode 100644 index 00000000000..73426c28257 --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/catalog_include_test.go @@ -0,0 +1,93 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package cmd + +import ( + "os" + "path/filepath" + "testing" + + "azureaieval/internal/project" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// A name declared through a `$ref` is refused, not appended alongside. +// +// The editing read sees the directive rather than the entry behind it, so the +// duplicate scan had nothing to match on and appended a second entry with the +// same name. The collision then surfaced on the next resolving read, naming a +// duplicate the author never wrote and could not see in the file in front of +// them. +func TestGenerateRefusesANameAnIncludeAlreadyDeclares(t *testing.T) { + dir := t.TempDir() + require.NoError(t, os.MkdirAll(filepath.Join(dir, "parts"), 0o755)) + require.NoError(t, os.WriteFile(filepath.Join(dir, "parts", "quality.yaml"), + []byte("name: quality\nsource: ./quality.json\n"), 0o600)) + + require.NoError(t, os.WriteFile(filepath.Join(dir, project.EvalConfigBase), []byte(` +evaluators: + - $ref: ./parts/quality.yaml +`), 0o600)) + + err := checkNameNotBehindAnInclude( + dir, mustOpenForEdit(t, dir), "evaluator", "quality") + + require.Error(t, err, "the name is taken, even though this file does not show it") + assert.Contains(t, err.Error(), "quality") + assert.Contains(t, err.Error(), "$ref", "the reader has to be told where the name lives") +} + +// A name nothing declares is still free, so generation is not blocked by the +// mere presence of an include elsewhere. +func TestGenerateStillAddsANameNobodyDeclares(t *testing.T) { + dir := t.TempDir() + require.NoError(t, os.MkdirAll(filepath.Join(dir, "parts"), 0o755)) + require.NoError(t, os.WriteFile(filepath.Join(dir, "parts", "quality.yaml"), + []byte("name: quality\nsource: ./quality.json\n"), 0o600)) + + require.NoError(t, os.WriteFile(filepath.Join(dir, project.EvalConfigBase), []byte(` +evaluators: + - $ref: ./parts/quality.yaml +`), 0o600)) + + require.NoError(t, checkNameNotBehindAnInclude( + dir, mustOpenForEdit(t, dir), "evaluator", "tone")) +} + +// An include carrying an overlay `name` is refused too, even though the name is +// right there in the file. +// +// This is the shape the README recommends for a rubric. Updating it in place +// writes `source:` beside the directive, and resolution then yields both a +// spliced rubric and a source -- a catalog the next read rejects for declaring +// the rubric twice. The name being visible is what made this the easier one to +// miss. +func TestGenerateRefusesAnIncludeThatCarriesItsName(t *testing.T) { + dir := t.TempDir() + require.NoError(t, os.MkdirAll(filepath.Join(dir, "evaluators"), 0o755)) + require.NoError(t, os.WriteFile(filepath.Join(dir, "evaluators", "quality.json"), + []byte(`{"type":"rubric","dimensions":[{"id":"tone","weight":3}]}`), 0o600)) + + require.NoError(t, os.WriteFile(filepath.Join(dir, project.EvalConfigBase), []byte(` +evaluators: + - $ref: ./evaluators/quality.json + name: quality +`), 0o600)) + + err := checkNameNotBehindAnInclude( + dir, mustOpenForEdit(t, dir), "evaluator", "quality") + + require.Error(t, err, "the entry is an include, so it cannot be updated in place") + assert.Contains(t, err.Error(), "quality") +} + +func mustOpenForEdit(t *testing.T, dir string) *project.EvalConfig { + t.Helper() + cfg, err := project.OpenEvalConfigForEdit(dir) + require.NoError(t, err) + require.NotNil(t, cfg) + return cfg +} diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/messages/messages.go b/cli/azd/extensions/azure.ai.evaluations/internal/messages/messages.go new file mode 100644 index 00000000000..55159233d43 --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/internal/messages/messages.go @@ -0,0 +1,2783 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +// Package messages holds every string this extension shows a user. +// +// One file, so the whole voice of the CLI can be reviewed in one sitting and a +// wording change never has to be hunted through the command tree. The only +// extension package it imports is exterrors, which holds no wording of its own, +// so every other package can use this one. +// +// Conventions, so the set stays consistent: +// +// - Errors state what went wrong and, where there is one, the way out. +// Lowercase, no trailing period: azd renders them after "ERROR: ". +// - A name the user chose is quoted with %q; an identifier the service +// assigned is not, because it is already unmistakable. +// - A filesystem path goes through filepath.ToSlash first. %q escapes a +// Windows separator, so `evals\eval.yaml` prints as "evals\\eval.yaml" and +// a reader who copies it back gets a path that does not exist. +// - Progress and success lines are sentences with a capital and no period. +// - A printed line carries its own newlines, so a call site is a bare Fprint. +// - Nothing here decides *whether* to print. That stays at the call site. +package messages + +import ( + "errors" + "fmt" + "io/fs" + "net/http" + "path/filepath" + "sort" + "strconv" + "strings" + "time" + + "azureaieval/internal/exterrors" + + "github.com/Azure/azure-sdk-for-go/sdk/azcore" + "github.com/Azure/azure-sdk-for-go/sdk/azidentity" + "github.com/azure/azure-dev/cli/azd/pkg/azdext" +) + +// --------------------------------------------------------------------------- +// Running an eval +// --------------------------------------------------------------------------- + +// NoEvalToRun reports a run with nothing resolved to run. +func NoEvalToRun() error { + return errors.New("no eval to run") +} + +// EvalHasNoDataset reports an eval whose rows cannot be located. +// +// Named separately from the traces and responses cases because the way out is +// different: this one is answered by a dataset, not by a source block. +func EvalHasNoDataset(eval string) error { + return fmt.Errorf( + "eval %q references no dataset and declares no source. Add a dataset: to "+ + "score rows you supply, or a source: to score traces or stored responses", + eval) +} + +// DatasetFileEmpty reports a local dataset file that parsed but held no rows. +func DatasetFileEmpty(path string) error { + return fmt.Errorf("dataset file %q has no rows", filepath.ToSlash(path)) +} + +// DatasetOverrideNeedsDeclaredEval reports --dataset passed against a bare id. +func DatasetOverrideNeedsDeclaredEval() error { + return errors.New( + "--dataset overrides the dataset an eval declares, so it needs a " + + "declared eval; pass --eval with a name from the configuration") +} + +// DatasetNotInCatalog reports a --dataset the configuration does not declare. +func DatasetNotInCatalog(dataset, configPath string) error { + return fmt.Errorf("dataset %q is not in the catalog in %s", dataset, filepath.ToSlash(configPath)) +} + +// NothingToGenerateFrom refuses a generation request carrying no sources. +// +// The service answers one with "At least one source is required", wrapped in a +// 400 and thirty lines of JSON. Nothing about that names the two things a +// reader can actually supply. +func NothingToGenerateFrom() error { + return errors.New( + "nothing to generate from: pass --target to seed from an " + + "agent's instructions, or declare one under target: in the eval " + + "configuration. A trace-backed eval names its agent under source:, " + + "which selects traces to read and does not seed generation") +} + +// SelectEvaluatorsPrompt asks which references the eval grades on. +func SelectEvaluatorsPrompt() string { + return "Select evaluators to grade with:" +} + +// SelectingEvaluators reports a failed evaluator prompt. +func SelectingEvaluators(err error) error { + return fmt.Errorf("selecting evaluators: %w", err) +} + +// NoEvaluatorsChosen reports an eval that would grade on nothing. +func NoEvaluatorsChosen() error { + return errors.New( + "an eval has to grade on at least one evaluator: select one, or pass " + + "--evaluator") +} + +// GateNeedsTheWait refuses a gate on a run the command will not wait for. +// +// The two flags together read as "start it and tell me if it regressed", but +// the verdict does not exist yet when --no-wait returns, so the gate was +// silently dropped and the command exited 0 however the run turned out. +func GateNeedsTheWait() error { + return errors.New( + "--fail-on needs a result to judge, and --no-wait returns before there " + + "is one. Drop --no-wait, or reattach with `azd ai eval run show " + + " --wait --fail-on `") +} + +// GateOutlivedTheWait reports a gate that never got a verdict because the run +// outlived the wait. +// +// Without a gate this is not a failure and exits 0, which is why the run was +// reported and the reattach line printed. With one, exiting 0 tells a pipeline +// the gate passed when nothing was ever judged -- the same silent drop +// GateNeedsTheWait refuses up front, arrived at by running long instead. +func GateOutlivedTheWait(runID string, budget time.Duration) error { + return fmt.Errorf( + "run %s outlived the %s wait, so --fail-on never got a result to judge. "+ + "The run is still going: reattach with `azd ai eval run show %s "+ + "--wait --fail-on `", runID, budget, runID) +} + +// DatasetHasUnregisteredEdits reports local rows no deployed version holds. +func DatasetHasUnregisteredEdits(dataset, deployCmd string) error { + return fmt.Errorf( + "dataset %q has local edits that are not registered.\n"+ + " Run `%s` to register them, or `--eval ` to run against "+ + "an existing eval", + dataset, deployCmd) +} + +// StartingRun reports the service refusing to start the run. +func StartingRun(err error) error { + return fmt.Errorf("starting the evaluation run: %w", err) +} + +// RunStarted reports a submitted run that was not waited on. +func RunStarted(runID, status string) string { + return fmt.Sprintf("Started run %s (status: %s)\n", runID, status) +} + +// ReattachToRun says how to come back to a run started with --no-wait. +func ReattachToRun(runID, evalID string) string { + return fmt.Sprintf("Reattach with: azd ai eval run show %s --eval %s\n", runID, evalID) +} + +// ReadingPreviousRuns reports a failure to look up what an eval last ran. +func ReadingPreviousRuns(evalID string, err error) error { + return fmt.Errorf("reading previous runs of eval %s: %w", evalID, err) +} + +// EvalHasNoPreviousRun reports an eval named by id that has nothing to repeat. +func EvalHasNoPreviousRun(evalID string) error { + return fmt.Errorf( + "eval %s has no previous run to repeat, so there is no target or dataset "+ + "to reuse.\n"+ + " Run it from the config once with `azd ai eval run start`, or name an "+ + "eval that declares one with `--eval`", + evalID) +} + +// PollingRun reports a failure while waiting for a run to finish. +func PollingRun(runID string, err error) error { + return fmt.Errorf("polling run %s: %w", runID, err) +} + +// WaitBudgetSpent reports a run that outlived the foreground wait. +func WaitBudgetSpent(runID string, budget time.Duration) string { + return fmt.Sprintf( + "Run %s is still going after %s, so the wait stopped, not the run.\n", + runID, budget) +} + +// WaitInterrupted reports a wait cut short, naming the run still in flight. +func WaitInterrupted(runID string, err error) error { + return fmt.Errorf( + "stopped waiting on run %s, which is still running: %w. "+ + "Pick it back up with `azd ai eval run show %s`", + runID, err, runID) +} + +// RunStatusLine reports a status change seen while polling. +func RunStatusLine(status string) string { + return fmt.Sprintf(" status: %s\n", status) +} + +// RunFinishedWithStatus reports a run that ended in something other than completed. +func RunFinishedWithStatus(runID, status string) error { + return fmt.Errorf("run %s finished with status %s", runID, status) +} + +// OverallPassRate reports the share of the rows an evaluator scored that passed +// every evaluator. +// +// The denominator is named rather than left as a bare fraction. Rows nothing +// could grade are outside it, so a run that errored on most of its samples can +// report a high rate, and "of N scored" is what stops that reading as a verdict +// on the whole run. It is also the figure `--fail-on pass-rate` compares. +func OverallPassRate(rate string, passed, scored, unscored int) string { + if unscored > 0 { + return fmt.Sprintf("\nOverall pass rate: %s (%d of %d scored; %d not scored)\n", + rate, passed, scored, unscored) + } + return fmt.Sprintf("\nOverall pass rate: %s (%d/%d)\n", rate, passed, scored) +} + +// SamplesErrored reports rows the run could not score at all. +func SamplesErrored(errored int) string { + return fmt.Sprintf("%d sample(s) errored and were not scored.\n", errored) +} + +// ViewFailingSamples points at the command that lists the rows that failed. +func ViewFailingSamples() string { + return "\nView failing samples: azd ai eval run output list --failed-only\n" +} + +// ErroredNotScored annotates an evaluator's row with what it could not score. +func ErroredNotScored(errored int) string { + return fmt.Sprintf("(%d errored, not scored)", errored) +} + +// EvalNotDeployed reports an eval id the project does not hold. +func EvalNotDeployed(evalID, deployCmd string) error { + return fmt.Errorf( + "no eval %q in this project; "+ + "`%s` creates the ones your config declares", evalID, deployCmd) +} + +// NoEnvironmentToRememberEval reports an eval whose id had nowhere to be kept. +// +// `create` publishes the eval and records its id in the azd environment. With +// no environment there is nowhere to record it, so create reports success and +// the next command cannot find what it made. Saying "not deployed" there sends +// the reader to deploy it again, which lands in the same place. +func NoEnvironmentToRememberEval(eval string) error { + return fmt.Errorf( + "eval %q may exist in the project, but this directory has no azd "+ + "environment to have recorded its id in. Create one with "+ + "`azd env new ` and run `azd ai eval create` again, or name "+ + "the eval's id with --eval", eval) +} + +// EvalNotDeployedYet reports a declared eval that no deploy has created. +func EvalNotDeployedYet(eval, deployCmd string) error { + return fmt.Errorf( + "eval %q is declared but has not been deployed to this environment yet; "+ + "run `%s` first", eval, deployCmd) +} + +// NoEvalNamedOrDeclared reports a command with no eval to act on. +func NoEvalNamedOrDeclared(configPath string) error { + return fmt.Errorf( + "no eval was named and none is declared in %s; pass --eval with a name or an id", + filepath.ToSlash(configPath)) +} + +// ListingRuns reports a failure to list an eval's runs. +func ListingRuns(evalID string, err error) error { + return fmt.Errorf("listing runs for %q: %w", evalID, err) +} + +// EvalHasNoRunsLine reports an eval with no runs to list. +func EvalHasNoRunsLine(evalID string) string { + return fmt.Sprintf("Eval %s has no runs yet.\n", evalID) +} + +// EvalHasNoRuns reports an eval with no run to fall back on. +func EvalHasNoRuns(evalID string) error { + return fmt.Errorf("eval %s has no runs yet", evalID) +} + +// ReadingRun reports a failure to read the run the caller named. +func ReadingRun(runID string, err error) error { + return fmt.Errorf("reading run %s: %w", runID, err) +} + +// CountsSummary renders a run's verdict counts on one line. +func CountsSummary(passed, failed, errored int) string { + return fmt.Sprintf("%d passed, %d failed, %d errored", passed, failed, errored) +} + +// RunAlreadyFinished reports a cancel asked of a run that already ended. +func RunAlreadyFinished(runID, status string) error { + return fmt.Errorf("run %s already finished with status %q", runID, status) +} + +// CancellingRun reports the service refusing to cancel the run. +func CancellingRun(runID string, err error) error { + return fmt.Errorf("cancelling run %s: %w", runID, err) +} + +// RunIsNow reports the state a cancelled run moved to. +func RunIsNow(runID, status string) string { + return fmt.Sprintf("Run %s is now %s\n", runID, status) +} + +// RunNotFound reports a run id the eval does not hold. +func RunNotFound(runID, evalID string) error { + return fmt.Errorf("no run %q on eval %q", runID, evalID) +} + +// DeletingRun reports the service refusing to delete the run. +func DeletingRun(runID string, err error) error { + return fmt.Errorf("deleting run %s: %w", runID, err) +} + +// RunDeleted confirms a deleted run. +func RunDeleted(runID string) string { + return fmt.Sprintf("Deleted run %s\n", runID) +} + +// ReadingRunResults reports a failure to read a run's per-sample rows. +func ReadingRunResults(runID string, err error) error { + return fmt.Errorf("reading the results of run %s: %w", runID, err) +} + +// OutputItemNotFound reports an output item the run does not hold. +func OutputItemNotFound(itemID, runID string) error { + return fmt.Errorf( + "no output item %q on run %s; "+ + "`azd ai eval run output list` shows the ones there are", + itemID, runID) +} + +// ReadingOutputItem reports a failure to read one evaluated row. +func ReadingOutputItem(itemID string, err error) error { + return fmt.Errorf("reading output item %q: %w", itemID, err) +} + +// RunStatusHeading opens the per-sample view of a run. +func RunStatusHeading(runID, status string) string { + return fmt.Sprintf("Run %s status: %s\n", runID, status) +} + +// ResultTotals reports a run's verdict counts above the rows. +func ResultTotals(passed, failed, errored int) string { + return fmt.Sprintf("Totals: %d passed, %d failed, %d errored\n\n", passed, failed, errored) +} + +// NoFailingRows reports a --failed-only listing with nothing in it. +func NoFailingRows() string { + return "\nNo failing rows.\n" +} + +// NoRowsScored reports a run that has produced no rows yet. +func NoRowsScored() string { + return "\nNo rows have been scored yet.\n" +} + +// SamplesNeedingALook closes a --failed-only listing, holding the rows that +// failed apart from the rows nothing managed to score. +// +// One count covering both contradicted the totals printed two lines above it, +// which is what a reader compares it with: a run reporting 5 failed and 8 +// errored closed with "13 sample(s) failed at least one evaluator". +func SamplesNeedingALook(failed, unscored int) string { + if unscored == 0 { + return fmt.Sprintf("\n%d sample(s) failed at least one evaluator.\n", failed) + } + if failed == 0 { + return fmt.Sprintf("\n%d sample(s) could not be scored.\n", unscored) + } + return fmt.Sprintf( + "\n%d sample(s) failed at least one evaluator, and %d could not be scored.\n", + failed, unscored) +} + +// GateSawUnscoredRows warns that a pass-rate gate judged only part of the run. +// +// The rate excludes rows nothing could grade, so a run that errored on most of +// its samples can clear a threshold on the few that survived. The gate is the +// one place a pipeline is guaranteed to read, so it is said there rather than +// left for someone to notice in the summary. +func GateSawUnscoredRows(unscored, total int) error { + return fmt.Errorf( + "%d of %d samples were not scored, so the pass rate this gate read covers "+ + "only the rest; use --fail-on any-failure to count them against the run", + unscored, total) +} + +// GeneratedNameNotAFileName reports a generated artifact name that would not +// stay inside the output directory, or that would produce a file whose name is +// read as a flag by whatever the path is handed to next. +func GeneratedNameNotAFileName(kind, name string) error { + return fmt.Errorf( + "%s name %q cannot be used as a file name: remove any of / \\ : , "+ + "do not start with -, and do not use . or ..", + kind, name) +} + +// OutputItemEmpty reports a row the service acknowledged but returned nothing +// for, which is a service fault rather than a missing item. +func OutputItemEmpty() error { + return errors.New("the service returned no content for this output item") +} + +// NotARegularFile reports an --output-file that names a directory or a device. +func NotARegularFile(path string) error { + return fmt.Errorf("%s is not a regular file, so it will not be overwritten", filepath.ToSlash(path)) +} + +// CannotWriteInDirectory reports a destination directory that cannot be written +// to. A missing directory is reported as such: the wrapped error names the +// temporary file the writer chose, which the caller never asked for. +func CannotWriteInDirectory(dir string, err error) error { + if errors.Is(err, fs.ErrNotExist) { + return fmt.Errorf("%s does not exist", filepath.ToSlash(dir)) + } + return fmt.Errorf("cannot write in %s: %w", dir, err) +} + +// OutputItemVerdict is one evaluator's line in `run output show`. +func OutputItemVerdict(evaluator, score, verdict string) string { + return fmt.Sprintf("%s %s %s\n", evaluator, score, verdict) +} + +// OutputItemEvaluator heads the dimensions of a rubric that scored per metric. +func OutputItemEvaluator(evaluator string) string { + return evaluator + "\n" +} + +// OutputItemMetric is one scored dimension under its evaluator. +func OutputItemMetric(metric, score, verdict string) string { + return fmt.Sprintf(" %s %s %s\n", metric, score, verdict) +} + +// OutputItemReason is the judge's explanation, indented under its verdict. +func OutputItemReason(reason string) string { + return fmt.Sprintf(" %s\n", reason) +} + +// OutputFileCannotHoldBothArtifacts reports one --output-dir file for two +// artifacts. +// +// Both jobs would resolve to it and both would write it, concurrently, and the +// configuration would then name it as a dataset and as an evaluator. +func OutputFileCannotHoldBothArtifacts(outputDir string) error { + return fmt.Errorf( + "--output-dir %q names a file, and this generates a dataset and an evaluator; "+ + "name a directory, or add --dataset or --evaluator to generate one of them", + filepath.ToSlash(outputDir)) +} + +// UsingLastRun names the run a command chose when it was not given one. +// +// Written to stderr so it does not land in a redirected listing. +func UsingLastRun(runID string) string { + return fmt.Sprintf("Using last run: %s\n", runID) +} + +// PortalLinkAfterRows closes a per-sample listing with the run's one link. +// +// Labelled the way every other view labels it: the run's report page is in the +// portal, and a reader looking for the link should not have to know two words +// for it. +func PortalLinkAfterRows(url string) string { + return fmt.Sprintf("\nPortal: %s\n", url) +} + +// ExportFormatUnsupported reports an --format the export command cannot write. +func ExportFormatUnsupported(format, csv, json, jsonl string) error { + return fmt.Errorf( + "--format %q is not supported; use %s, %s or %s", + format, csv, json, jsonl) +} + +// FailOnInvalid reports a --fail-on value that is neither form of threshold. +func FailOnInvalid(spec string) error { + return fmt.Errorf("--fail-on must be any-failure or pass-rate=<0..1>, got %q", spec) +} + +// FailOnRateNotNumber reports a --fail-on pass rate that will not parse. +func FailOnRateNotNumber(rate string) error { + return fmt.Errorf("--fail-on pass-rate must be a number, got %q", rate) +} + +// FailOnRateOutOfRange reports a --fail-on pass rate outside 0..1. +func FailOnRateOutOfRange(value float64) error { + return fmt.Errorf("--fail-on pass-rate must be between 0 and 1, got %v", value) +} + +// GateNoResultCounts reports a gate that has nothing to measure against. +func GateNoResultCounts() string { + return "the run reported no result counts, so the threshold cannot be checked" +} + +// GateSamplesDidNotPass reports an any-failure gate that was breached. +func GateSamplesDidNotPass(unpassed, total int) string { + return fmt.Sprintf("%d of %d samples did not pass", unpassed, total) +} + +// GateNoRowsScored reports a pass-rate gate over a run that scored nothing. +func GateNoRowsScored() string { + return "the run scored no rows, so its pass rate is below any threshold" +} + +// GatePassRateBelow reports a pass-rate gate that was breached. +// +// One decimal, which is what the spec's hero scenario shows -- except when that +// rounds the actual rate onto the threshold. The gate compares exact values, so +// 7996/10000 breaches 0.8 while both read "80.0%", and the line would say a +// rate is below itself. Only that case is given more precision. +func GatePassRateBelow(actual, required float64) string { + shown := fmt.Sprintf("%.1f", actual*100) + if shown == fmt.Sprintf("%.1f", required*100) { + shown = strconv.FormatFloat(actual*100, 'f', -1, 64) + } + return fmt.Sprintf("pass rate %s%% is below the required %.1f%%", + shown, required*100) +} + +// GateBreached is the block a breached gate leaves in a pipeline's log. +func GateBreached(reason string) string { + return fmt.Sprintf("%s Evaluation gate: %s\n\nERROR: evaluation quality gate not met.\n", + failedMark, reason) +} + +// --------------------------------------------------------------------------- +// Generation +// --------------------------------------------------------------------------- + +// GeneratedNameNeedsATarget reports a generation that can name neither the +// artifact nor the agent to derive its name from. +func GeneratedNameNeedsATarget(kind string) error { + return fmt.Errorf( + "no name for the generated %s and no target to derive one from: "+ + "pass --%s-name, or --target", kind, kind) +} + +// GenerationFailed labels one half of a composite generate that did not finish. +// +// The label goes inside a structured error rather than around it: azd +// serializes a LocalError's own message and drops any wrapper, so wrapping +// would throw away the one word saying which job failed. +func GenerationFailed(kind string, err error) error { + var local *azdext.LocalError + if errors.As(err, &local) { + labelled := *local + labelled.Message = "generating the " + kind + ": " + local.Message + return &labelled + } + return fmt.Errorf("generating the %s: %w", kind, err) +} + +// multiError presents several failures as one line while keeping every cause +// reachable through errors.Is and errors.As. +// +// errors.Join would keep the causes but renders them one per line, and this is +// a single error the CLI prints after "ERROR: ". +type multiError struct { + msg string + causes []error +} + +func (m *multiError) Error() string { return m.msg } +func (m *multiError) Unwrap() []error { return m.causes } + +// SomeGenerationsFailed reports a composite generate where at least one job +// did not finish. The others may well have. +// +// Two structured failures of the same category stay structured, so an expired +// login still arrives as an auth error carrying its suggestion rather than as +// a flat string. +func SomeGenerationsFailed(failures []error) error { + if len(failures) == 1 { + return failures[0] + } + + parts := make([]string, 0, len(failures)) + for _, f := range failures { + parts = append(parts, f.Error()) + } + joined := strings.Join(parts, "; ") + + var first *azdext.LocalError + if !errors.As(failures[0], &first) { + return &multiError{msg: joined, causes: failures} + } + for _, f := range failures[1:] { + var other *azdext.LocalError + if !errors.As(f, &other) || other.Category != first.Category { + return &multiError{msg: joined, causes: failures} + } + } + merged := *first + merged.Message = joined + return &merged +} + +// GenerationStarting announces a job before it is submitted, so a long +// generation is not silent while it runs. +func GenerationStarting(kind, name string) string { + return fmt.Sprintf(" Starting %s generation for %q...\n", kind, name) +} + +// GenerationModelRequired reports a generation with no deployment to run on. +// +// Reached only when the target agent could not supply one either, so the flag +// is the whole of the way out. +func GenerationModelRequired() error { + return errors.New("a model deployment is required to generate: pass --generation-model") +} + +// ReadingInstructionFile reports an --agent-instruction-file that would not read. +func ReadingInstructionFile(path string, err error) error { + return fmt.Errorf("reading --agent-instruction-file %q: %w", filepath.ToSlash(path), err) +} + +// InstructionFileEmpty reports an --agent-instruction-file with nothing in it. +func InstructionFileEmpty(path string) error { + return fmt.Errorf("--agent-instruction-file %q is empty", filepath.ToSlash(path)) +} + +// ReadingInstructions reports a declared instructions file that would not read. +func ReadingInstructions(named string, err error) error { + return fmt.Errorf("reading instructions %q: %w", named, err) +} + +// SeedingFromFile names the local file generation was seeded from. +func SeedingFromFile(path string) string { + return fmt.Sprintf(" Seeding generation from %s.\n", filepath.ToSlash(path)) +} + +// SeedingFromAgent names the agent whose published instructions seeded generation. +func SeedingFromAgent(agent string) string { + return fmt.Sprintf(" Seeding generation from the instructions of agent %q.\n", agent) +} + +// WarningAgentUnreadable reports an agent that could not supply context. +// +// A misspelled --target is the common cause and answers 404, whose body is ten +// lines of URL, status rule and nested JSON for a fact that fits on one. +func WarningAgentUnreadable(agent string, err error) string { + if notFound(err) { + return fmt.Sprintf(" warning: no agent %q in this project, so generation "+ + "has no agent context to work from\n", agent) + } + return fmt.Sprintf(" warning: could not read agent %q for generation context: %v\n", + agent, err) +} + +// notFound reports a service answer of 404. +// +// Written here rather than imported from eval_api, because that package imports +// this one for its own wording and the dependency only goes one way. +func notFound(err error) bool { + var respErr *azcore.ResponseError + return errors.As(err, &respErr) && respErr.StatusCode == http.StatusNotFound +} + +// WarningAgentSeedFailedRetrying reports the retry that drops the agent source. +func WarningAgentSeedFailedRetrying(agent string) string { + return fmt.Sprintf( + " warning: generating from agent %q failed in the service; "+ + "retrying from the instruction alone.\n", agent) +} + +// GeneratingRubric reports a rubric generation job about to be submitted. +func GeneratingRubric(name string) string { + return fmt.Sprintf("Generating rubric %s...\n", name) +} + +// GeneratingDataset reports a dataset generation job about to be submitted. +func GeneratingDataset(name string, samples int) string { + return fmt.Sprintf("Generating dataset %s (%d samples)...\n", name, samples) +} + +// SubmittingRubricJob reports the service refusing the rubric job. +func SubmittingRubricJob(err error) error { + return fmt.Errorf("submitting the rubric generation job: %w", err) +} + +// SubmittingDataJob reports the service refusing the data generation job. +func SubmittingDataJob(err error) error { + return fmt.Errorf("submitting the data generation job: %w", err) +} + +// RubricGeneration reports a rubric job that did not finish successfully. +func RubricGeneration(err error) error { + return fmt.Errorf("rubric generation: %w", err) +} + +// DataGeneration reports a data job that did not finish successfully. +func DataGeneration(err error) error { + return fmt.Errorf("data generation: %w", err) +} + +// RubricJobReturnedNoResult reports a completed rubric job with nothing to write. +func RubricJobReturnedNoResult() error { + return errors.New("the rubric generation job returned no result") +} + +// DataJobReturnedNoDataset reports a completed data job with nothing to fetch. +func DataJobReturnedNoDataset() error { + return errors.New("the data generation job returned no dataset reference") +} + +// ReadingGeneratedDataset reports the generated dataset not being there to read. +func ReadingGeneratedDataset(name string, err error) error { + return fmt.Errorf("reading the generated dataset %q: %w", name, err) +} + +// DownloadingGeneratedDataset reports a failure to fetch the generated rows. +func DownloadingGeneratedDataset(name string, err error) error { + return fmt.Errorf("downloading the generated dataset %q: %w", name, err) +} + +// AgentSeededGenerationFailing explains the service-side failure that hits +// every agent, so the caller does not retry against a deterministic failure. +func AgentSeededGenerationFailing(err error, agent string) error { + return fmt.Errorf( + "%w\n\n"+ + "This job seeded generation from agent %q. Agent-seeded data generation is "+ + "currently failing in the service for every agent, so retrying will not help.\n"+ + "Workarounds: supply your own dataset with --dataset, or run without --target "+ + "to generate from the instruction alone.", + err, agent) +} + +// FromPromptNeedsInstruction reports --from prompt with nothing to prompt with. +func FromPromptNeedsInstruction() string { + return "--from prompt needs --agent-instruction or --agent-instruction-file" +} + +// FromAgentNeedsTarget reports --from agent with no agent to read. +func FromAgentNeedsTarget() string { + return "--from agent needs a target agent; pass --target, " + + "or declare one under target: in azure.eval.yaml" +} + +// FromFileNotASource reports --from file, which generation has no path for. +func FromFileNotASource() string { + return "--from file is not a generation source; " + + "register the file with `azd ai eval dataset create` instead" +} + +// FromNotBuildable reports a --from this plan cannot satisfy. +func FromNotBuildable(kind string) string { + return fmt.Sprintf("--from %s cannot be built from this plan", kind) +} + +// UnbuildableSources reports every --from the plan could not honour at once. +func UnbuildableSources(reasons []string) error { + return errors.New(strings.Join(reasons, "; ")) +} + +// JobSubmitted reports the id of a job started with --no-wait. +func JobSubmitted(jobID string) string { + return fmt.Sprintf(" submitted job %s\n", jobID) +} + +// ReattachToJob says how to come back to a job started with --no-wait. +// +// The selector is part of the line because `job` requires it: the two +// collections share an id shape, so an id alone does not say which to call. +func ReattachToJob(selector, jobID string) string { + return fmt.Sprintf("\nReattach with: azd ai eval job show %s --%s\n", jobID, selector) +} + +// WroteArtifact reports where a generated artifact landed. +func WroteArtifact(path string) string { + return fmt.Sprintf("%s Downloaded %s\n", doneMark, filepath.ToSlash(path)) +} + +// ArtifactExists reports a generation that would overwrite a checked-in file. +func ArtifactExists(path string) error { + return fmt.Errorf( + "%s already exists; pass --force to overwrite it, or --output-dir to write elsewhere", + path) +} + +// JobKindRequired reports a job command that does not say which collection. +func JobKindRequired() error { + return errors.New("pass --dataset or --evaluator to say which generation jobs to act on") +} + +// ListingJobs reports a failure to list one kind of generation job. +func ListingJobs(kind string, err error) error { + return fmt.Errorf("listing %s generation jobs: %w", kind, err) +} + +// NoJobs reports a project with no generation jobs of that kind. +func NoJobs(kind string) string { + return fmt.Sprintf("No %s generation jobs found.\n", kind) +} + +// JobLine renders one generation job in a listing or a detail view. +func JobLine(jobID, status string) string { + return fmt.Sprintf("%s %s\n", jobID, status) +} + +// JobErrorLine reports why a generation job failed. +func JobErrorLine(message string) string { + return fmt.Sprintf("error: %s\n", message) +} + +// JobCancelled confirms a cancelled generation job. +func JobCancelled(kind, jobID, status string) string { + return fmt.Sprintf("Cancelled %s generation job %s (%s)\n", kind, jobID, status) +} + +// JobDeleted confirms a deleted generation job record. +func JobDeleted(kind, jobID string) string { + return fmt.Sprintf("Deleted %s generation job %s\n", kind, jobID) +} + +// JobNotFound reports a job id that is not in this group, naming the other one. +// +// Phrased to avoid an article before the kind: "a evaluator" is what the +// obvious wording produces. +func JobNotFound(kind, jobID, other string) error { + return fmt.Errorf( + "no %s generation job %q in this project; try the %s job group", + kind, jobID, other) +} + +// JobActionFailed reports a job operation that was not a read, so the sentence +// names what was attempted. A delete that reports "reading" sends the reader +// looking for a read that never happened. +func JobActionFailed(action, kind, jobID string, err error) error { + return fmt.Errorf("%s %s generation job %s: %w", action, kind, jobID, err) +} + +// JobFailedWithReason reports a polled job that failed and said why. +func JobFailedWithReason(status, message string) string { + return fmt.Sprintf("job failed with status %q: %s", status, message) +} + +// JobFailed reports a polled job that failed without saying why. +func JobFailed(status string) string { + return fmt.Sprintf("job failed with status %q", status) +} + +// PollerTimedOut reports a job that was still running when polling gave up. +func PollerTimedOut(operationID string, attempts int) string { + return fmt.Sprintf("operation %s did not complete within %d attempts", + operationID, attempts) +} + +// OperationIDEmpty reports a poll with nothing to poll for. +func OperationIDEmpty() error { + return errors.New("operation ID is empty") +} + +// --------------------------------------------------------------------------- +// Datasets +// --------------------------------------------------------------------------- + +// ReadingDataset reports a dataset that could not be read, by name or by path. +func ReadingDataset(dataset string, err error) error { + return fmt.Errorf("reading dataset %q: %w", dataset, err) +} + +// DatasetHasNoVersionsToRead reports a registered dataset with nothing published. +func DatasetHasNoVersionsToRead(dataset string) error { + return fmt.Errorf("dataset %q has no versions to read", dataset) +} + +// ReadingDatasetVersion reports one version of a dataset failing to read. +func ReadingDatasetVersion(dataset, version string, err error) error { + return fmt.Errorf("reading dataset %q version %s: %w", dataset, version, err) +} + +// CheckingDataset reports the read that decides whether a name is already +// taken. It is worth its own message because that read is what separates +// `create` from `update`, and a failure answered as "not there" turns a create +// into a silent update. +func CheckingDataset(dataset string, err error) error { + return fmt.Errorf( + "checking whether dataset %q already exists: %w", dataset, err) +} + +// DatasetVersionEmpty reports a published version that holds no rows. +func DatasetVersionEmpty(dataset, version string) error { + return fmt.Errorf("dataset %q version %s has no rows", dataset, version) +} + +// JSONLLineInvalid reports a row that is not JSON, by line. +func JSONLLineInvalid(line int, err error) error { + return fmt.Errorf("line %d is not valid JSON: %w", line, err) +} + +// JSONLRowInvalid reports a row that is not JSON before the file is published. +func JSONLRowInvalid(path string, line int, err error) error { + return fmt.Errorf( + "%s line %d is not valid JSON: %w. Every line must be one JSON object", + path, line, err) +} + +// JSONLRowEmpty reports a row that parses to nothing to evaluate. +func JSONLRowEmpty(path string, line int) error { + return fmt.Errorf("%s line %d is an empty object, which evaluates to nothing", path, line) +} + +// JSONLNoRows reports a dataset file with nothing in it to evaluate. +func JSONLNoRows(path string) error { + return fmt.Errorf("%s has no rows to evaluate", filepath.ToSlash(path)) +} + +// ReadingFromFile reports a --from-file that would not stat. +// +// A path that is simply absent is reported as absent: the wrapped error is a +// syscall name that says nothing to the person who mistyped it. +func ReadingFromFile(path string, err error) error { + if errors.Is(err, fs.ErrNotExist) { + return fmt.Errorf("--from-file %q does not exist", filepath.ToSlash(path)) + } + return fmt.Errorf("reading --from-file %q: %w", filepath.ToSlash(path), err) +} + +// FromFileMustBeJSONL reports a --from-file that is not a dataset. +func FromFileMustBeJSONL(path string) error { + return fmt.Errorf( + "--from-file must be a .jsonl file or a directory containing one, got %q", + filepath.ToSlash(path)) +} + +// FromFileDirectoryHasNoJSONL reports a directory with nothing to upload. +func FromFileDirectoryHasNoJSONL(dir string) error { + return fmt.Errorf("no .jsonl file in %q; --from-file needs one to upload", filepath.ToSlash(dir)) +} + +// FromFileDirectoryIsAmbiguous refuses to guess which dataset was meant. +func FromFileDirectoryIsAmbiguous(dir string, names []string) error { + return fmt.Errorf( + "%q holds %d .jsonl files (%s); name the one to upload with --from-file", + filepath.ToSlash(dir), len(names), strings.Join(names, ", ")) +} + +// InvalidDatasetName reports a name the service will not accept. +func InvalidDatasetName(name string) error { + return invalidAssetName("dataset", name) +} + +// InvalidEvaluatorName reports a name the service will not accept. +func InvalidEvaluatorName(name string) error { + return invalidAssetName("evaluator", name) +} + +func invalidAssetName(kind, name string) error { + return fmt.Errorf( + "%s name %q is invalid: use letters, digits, dashes and underscores, "+ + "up to 255 characters", kind, name) +} + +// RegisteringDataset reports the service refusing to publish the dataset. +func RegisteringDataset(dataset string, err error) error { + return fmt.Errorf("registering dataset %q: %w", dataset, err) +} + +// DatasetRegistered confirms a published dataset version. +func DatasetRegistered(dataset, version string) string { + return fmt.Sprintf("Registered dataset %s version %s\n", dataset, version) +} + +// ListingDatasets reports a failure to list the project's datasets. +func ListingDatasets(err error) error { + return fmt.Errorf("listing datasets: %w", err) +} + +// ListingDatasetVersions reports a failure to list one dataset's versions. +func ListingDatasetVersions(dataset string, err error) error { + return fmt.Errorf("listing versions of dataset %q: %w", dataset, err) +} + +// NoDatasets reports a project with no datasets to list. +func NoDatasets() string { + return "No datasets found.\n" +} + +// NoDatasetVersions reports a name whose versions listed nothing. +// +// Listing a name that does not exist is not an error ΓÇö a delete is checked for +// idempotence this way ΓÇö so this has to read as an answer about that name +// rather than as a report about the project, which holds other datasets. +// +// The suggested command carries no placeholder, so it pastes and runs; the file +// is the one thing only the caller knows, and is named outside the command. +func NoDatasetVersions(dataset string) string { + return fmt.Sprintf("No versions of dataset %q. Publish one with "+ + "`azd ai eval dataset create %s` and a --from-file path.\n", dataset, shellArg(dataset)) +} + +// ResolvingLatestDatasetVersion reports a failure to find what "latest" means. +func ResolvingLatestDatasetVersion(dataset string, err error) error { + return fmt.Errorf("resolving the latest version of %q: %w", dataset, err) +} + +// DatasetNotFound reports a name the project does not hold. +// +// The service answers an unknown name with an empty version list rather than a +// 404, and a dataset cannot exist with no versions, so an empty list means the +// dataset is absent rather than empty. +func DatasetNotFound(dataset string) error { + return fmt.Errorf( + "no dataset %q in this project; "+ + "`azd ai eval dataset list` shows the ones there are", dataset) +} + +// DatasetVersionNotFoundWithHint reports a dataset version the project does not hold. +func DatasetVersionNotFoundWithHint(dataset, version string) error { + return fmt.Errorf( + "no dataset %q at version %q in this project; "+ + "`azd ai eval dataset versions list %s` shows the ones there are", + dataset, version, shellArg(dataset)) +} + +// DatasetVersionNotFound reports a dataset version there is nothing to delete at. +func DatasetVersionNotFound(dataset, version string) error { + return fmt.Errorf("no dataset %q at version %q in this project", dataset, version) +} + +// DeletingDatasetVersion reports the service refusing the delete. +func DeletingDatasetVersion(dataset, version string, err error) error { + return fmt.Errorf("deleting dataset %q version %q: %w", dataset, version, err) +} + +// DatasetDeleted confirms a deleted dataset version. +func DatasetDeleted(dataset, version string) string { + return fmt.Sprintf("Deleted dataset %s version %s\n", dataset, version) +} + +// DatasetProblem attributes a failure to the dataset it happened under. +func DatasetProblem(dataset string, err error) error { + return fmt.Errorf("dataset %q: %w", dataset, err) +} + +// DatasetSource reports a declared source that is not on disk. +func DatasetSource(path string, err error) error { + return fmt.Errorf("dataset source %q: %w", filepath.ToSlash(path), err) +} + +// DatasetNotGeneratedYet reports a declared dataset whose rows are not written +// yet. +// +// `init` declares the dataset it plans and names the command that produces it, +// so reaching a deploy without one is an ordering mistake rather than a broken +// configuration. Said plainly, because the bare stat failure underneath is a +// Windows syscall name and a path with doubled separators. +// +// Both callers wrap this with DatasetProblem, which names the dataset, so this +// does not name it again. +func DatasetNotGeneratedYet(dataset, path string) error { + return fmt.Errorf( + "its rows %s have not been generated yet. "+ + "Run `azd ai eval generate --dataset --dataset-name %s` to write them, "+ + "or point the declaration at a .jsonl you already have. "+ + "If this entry came from a `$ref`, note that a relative `file:` inside "+ + "the referenced file resolves against azure.eval.yaml rather than against "+ + "that file -- write the path relative to the configuration instead", + filepath.ToSlash(path), shellArg(dataset)) +} + +// DatasetNotLocalNorFound reports a source-less dataset the project rejected. +func DatasetNotLocalNorFound(dataset string, err error) error { + return fmt.Errorf( + "dataset %q has no local source and could not be found on the project: %w", + dataset, err) +} + +// DatasetNotLocalNorRegistered reports a source-less dataset nobody published. +func DatasetNotLocalNorRegistered(dataset string) error { + return fmt.Errorf( + "dataset %q has no local source and is not registered on the project", dataset) +} + +// DatasetVersionConflict reports a pinned version the local file disagrees with. +func DatasetVersionConflict(dataset, version string) error { + return fmt.Errorf( + "dataset %q version %s already exists and the local file differs from it. "+ + "Raise `version:` to publish the change, or drop it to let each "+ + "deploy take the next version", + dataset, version) +} + +// DatasetDrifted reports a version published outside this configuration since +// the last deploy. +// +// `azd ai eval dataset update` publishes without recording the per-dataset +// version the reconciler reads, so it is a likely cause and naming it saves the +// reader looking for a colleague who did nothing. "Pull the newer content +// locally" was the other half of the old advice and is a no-op when the bytes +// already match, which is the common case. +func DatasetDrifted(dataset, latest, recorded string) error { + return fmt.Errorf( + "dataset %q is at version %s on the project but %s was recorded at the last deploy; "+ + "something published outside this configuration, which `azd ai eval dataset update` "+ + "on the same dataset also does. Pin it with `version: %s` on the dataset to deploy "+ + "what is already there, or publish a new version from the configuration's source, "+ + "then deploy again", + dataset, latest, recorded, latest) +} + +// ReadingDatasetDirectory reports the upload scan failing to read the directory. +func ReadingDatasetDirectory(err error) error { + return fmt.Errorf("reading directory: %w", err) +} + +// DatasetFileHasNoRows reports an empty dataset file, refused before upload. +func DatasetFileHasNoRows(name string) error { + return fmt.Errorf( + "dataset file %q has no rows, so there would be nothing to evaluate", name) +} + +// NoJSONLInDirectory reports an upload directory holding no dataset. +func NoJSONLInDirectory(dir string) error { + return fmt.Errorf("no .jsonl file found in %s", filepath.ToSlash(dir)) +} + +// ReadingDatasetFromDir reports the upload failing to gather the local rows. +func ReadingDatasetFromDir(dir string, err error) error { + return fmt.Errorf("reading dataset from %s: %w", dir, err) +} + +// StartingPendingUpload reports the service refusing to open an upload. +func StartingPendingUpload(err error) error { + return fmt.Errorf("starting pending upload: %w", err) +} + +// NoUploadURI reports an accepted upload the service gave nowhere to write to. +func NoUploadURI() error { + return errors.New("no upload SAS URI returned from startPendingUpload") +} + +// NoBlobURI reports an accepted upload the service gave no way to finalize. +// +// Separate from NoUploadURI because they are different fields of the same +// response: the SAS says where to write, the blob URI says what to register, +// and a response can carry one without the other. +func NoBlobURI() error { + return errors.New("no blob URI returned from startPendingUpload, so there is nothing to register the upload as") +} + +// UploadingBlob reports the dataset content failing to upload. +func UploadingBlob(err error) error { + return fmt.Errorf("uploading blob: %w", err) +} + +// ReadingDownloadCredentials reports the service refusing to hand out a read URI. +func ReadingDownloadCredentials(dataset string, err error) error { + return fmt.Errorf("reading download credentials for %q: %w", dataset, err) +} + +// NoDownloadURI reports a dataset the service gave nowhere to read from. +func NoDownloadURI(dataset string) error { + return fmt.Errorf("no download URI returned for dataset %q", dataset) +} + +// ListingDatasetContent reports a failure to list what a dataset version holds. +func ListingDatasetContent(dataset string, err error) error { + return fmt.Errorf("listing the content of dataset %q: %w", dataset, err) +} + +// DatasetHasNoFile reports a dataset version with nothing to download. +func DatasetHasNoFile(dataset string) error { + return fmt.Errorf("dataset %q holds no downloadable file", dataset) +} + +// --------------------------------------------------------------------------- +// Evaluators +// --------------------------------------------------------------------------- + +// EvaluatorNeedsFields reports required inputs the dataset does not carry. +func EvaluatorNeedsFields(evaluator string, missing []string) error { + return fmt.Errorf( + "evaluator %q requires %s, which the dataset does not provide; "+ + "add %s to the dataset, or bind it with `data_mapping`", + evaluator, quoteList(missing), pluralColumns(missing)) +} + +// EvaluatorLevelUnsupported reports an evaluation level the evaluator refuses. +func EvaluatorLevelUnsupported(evaluator, level string, supported []string) error { + return fmt.Errorf( + "evaluator %q does not support evaluation level %q; it supports %s", + evaluator, level, quoteList(supported)) +} + +// EvaluatorNeedsInitParams reports required initialization parameters left unset. +func EvaluatorNeedsInitParams(evaluator string, missing []string) error { + return fmt.Errorf( + "evaluator %q requires %s; set it under the evaluator's "+ + "`initialization_parameters` in the eval config", + evaluator, quoteList(missing)) +} + +// ReadingEvaluator reports an evaluator that could not be read, by name or path. +func ReadingEvaluator(evaluator string, err error) error { + return fmt.Errorf("reading evaluator %q: %w", evaluator, err) +} + +// EvaluatorProblem attributes a failure to the evaluator it happened under. +func EvaluatorProblem(evaluator string, err error) error { + return fmt.Errorf("evaluator %q: %w", evaluator, err) +} + +// EvaluatorSource reports a declared source that is not on disk. +func EvaluatorSource(path string, err error) error { + return fmt.Errorf("evaluator source %q: %w", filepath.ToSlash(path), err) +} + +// EvaluatorNotGeneratedYet reports a declared evaluator whose definition has +// not been written yet. +// +// `init` declares the rubric it plans and names the command that produces it, +// so reaching a deploy without one is an ordering mistake rather than a broken +// configuration. Said plainly, because the bare stat failure underneath is a +// Windows syscall name and a path with doubled separators. +// +// Both callers wrap this with EvaluatorProblem, which names the evaluator, so +// this does not name it again. +func EvaluatorNotGeneratedYet(evaluator, path string) error { + return fmt.Errorf( + "its definition %s has not been generated yet. "+ + "Run `azd ai eval generate --evaluator --evaluator-name %s` to write it, "+ + "or drop the evaluator from azure.eval.yaml. "+ + "If this entry came from a `$ref`, note that a relative `source:` inside "+ + "the referenced file resolves against azure.eval.yaml rather than against "+ + "that file -- carry the rubric under `definition:` instead", + filepath.ToSlash(path), shellArg(evaluator)) +} + +// CheckingEvaluatorExists reports a failure to tell create from update. +func CheckingEvaluatorExists(evaluator string, err error) error { + return fmt.Errorf("checking whether evaluator %q exists: %w", evaluator, err) +} + +// RegisteringEvaluator reports the service refusing to publish the evaluator. +func RegisteringEvaluator(evaluator string, err error) error { + return fmt.Errorf("registering evaluator %q: %w", evaluator, err) +} + +// EvaluatorRegistered confirms a published evaluator version. +func EvaluatorRegistered(evaluator, version string) string { + return fmt.Sprintf("Registered evaluator %s version %s\n", evaluator, version) +} + +// AssetAlreadyExists reports `create` asked of a name already in use. +func AssetAlreadyExists(kind, name string) error { + return fmt.Errorf("%s %q already exists: use `update` to publish a new version", kind, name) +} + +// AssetDoesNotExist reports `update` asked of a name nobody registered. +func AssetDoesNotExist(kind, name string) error { + return fmt.Errorf("%s %q does not exist: use `create` to register it", kind, name) +} + +// DefinitionNotJSONObject reports an evaluator definition that is not an object. +func DefinitionNotJSONObject(err error) error { + return fmt.Errorf("the definition is not a JSON object: %w", err) +} + +// NotValidJSON reports an evaluator file that will not parse at all. +func NotValidJSON(err error) error { + return fmt.Errorf("not valid JSON: %w", err) +} + +// RubricMissingDimensions reports a file that is neither rubric nor document. +func RubricMissingDimensions() error { + return errors.New( + "expected a rubric definition with 'dimensions', or a document with 'definition'") +} + +// ListingEvaluators reports a failure to list the project's evaluators. +func ListingEvaluators(err error) error { + return fmt.Errorf("listing evaluators: %w", err) +} + +// ListingEvaluatorVersions reports a failure to list one evaluator's versions. +func ListingEvaluatorVersions(evaluator string, err error) error { + return fmt.Errorf("listing versions of evaluator %q: %w", evaluator, err) +} + +// NoEvaluators reports a project with no evaluators to list. +func NoEvaluators() string { + return "No evaluators found.\n" +} + +// EvaluatorNotFound reports an evaluator the project does not hold. +func EvaluatorNotFound(evaluator string) error { + return fmt.Errorf( + "no evaluator %q in this project; "+ + "`azd ai eval evaluator list` shows the ones there are", evaluator) +} + +// EvaluatorVersionNotFound reports an evaluator version there is nothing to delete at. +func EvaluatorVersionNotFound(evaluator, version string) error { + return fmt.Errorf("no evaluator %q at version %q in this project", evaluator, version) +} + +// DeletingEvaluatorVersion reports the service refusing the delete. +func DeletingEvaluatorVersion(evaluator, version string, err error) error { + return fmt.Errorf("deleting evaluator %q version %q: %w", evaluator, version, err) +} + +// EvaluatorDeleted confirms a deleted evaluator version. +func EvaluatorDeleted(evaluator, version string) string { + return fmt.Sprintf("Deleted evaluator %s version %s\n", evaluator, version) +} + +// EvaluatorNotLocalNorFound reports a source-less evaluator the project rejected. +func EvaluatorNotLocalNorFound(evaluator string, err error) error { + return fmt.Errorf( + "evaluator %q has no local source and could not be found on the project: %w", + evaluator, err) +} + +// EvaluatorDrifted reports a version published outside this configuration since +// the last deploy. +// +// `azd ai eval evaluator update` publishes without recording the version the +// reconciler reads, so it is a likely cause and naming it saves the reader +// looking for a colleague who did nothing. +func EvaluatorDrifted(evaluator, remote, recorded string) error { + return fmt.Errorf( + "evaluator %q is at version %s on the project but %s was recorded at the last "+ + "deploy, and the local definition does not match it: something published a "+ + "version outside this configuration, which `azd ai eval evaluator update` on "+ + "the same evaluator also does. Publishing over it would leave that change "+ + "behind, so bring version %s into the declared source and deploy again, or "+ + "delete that version if it was a mistake", + evaluator, remote, recorded, remote) +} + +// EvaluatorVersionNotAdvancing reports a publish the service kept answering with +// a version that already existed. +func EvaluatorVersionNotAdvancing(evaluator, version string, waited fmt.Stringer) error { + return fmt.Errorf( + "publishing evaluator %q kept returning version %s, which already "+ + "existed. The service was still assigning that version after %s, so "+ + "version %s now holds what was just published and any eval bound to "+ + "it is scoring against it", + evaluator, version, waited, version) +} + +// EvaluatorHasNoVersions reports an evaluator nothing was ever published under. +func EvaluatorHasNoVersions(evaluator string) error { + return fmt.Errorf("evaluator %q has no versions", evaluator) +} + +// EvaluatorHasNoUsableVersion reports versions none of which can be resolved. +func EvaluatorHasNoUsableVersion(evaluator string) error { + return fmt.Errorf("evaluator %q has no usable version", evaluator) +} + +// BareEvaluatorEntry reports an evaluators: entry written as a plain string. +func BareEvaluatorEntry(name string) error { + return fmt.Errorf( + "an evaluator entry is a mapping, not a bare string: "+ + "write `- evaluator: %s`", name) +} + +// EvaluatorsMustBeSequence reports an evaluators: block that is not a list. +func EvaluatorsMustBeSequence(kind string) error { + return fmt.Errorf("evaluators must be a list, got %s", kind) +} + +// EvaluatorsMustBeList reports an evaluators: block that is not a JSON array. +func EvaluatorsMustBeList(err error) error { + return fmt.Errorf("evaluators must be a list: %w", err) +} + +// DecodingEvaluatorName reports an evaluator entry whose name will not decode. +func DecodingEvaluatorName(err error) error { + return fmt.Errorf("decoding evaluator name: %w", err) +} + +// DecodingEvaluator reports an evaluator entry that will not decode. +func DecodingEvaluator(err error) error { + return fmt.Errorf("decoding evaluator: %w", err) +} + +// EvaluatorEntryMissingEvaluator reports an entry that names no evaluator. +func EvaluatorEntryMissingEvaluator() error { + return errors.New("evaluator entry is missing 'evaluator'") +} + +// EvaluatorEntryMustBeMapping reports an entry that is neither map nor string. +func EvaluatorEntryMustBeMapping(kind string) error { + return fmt.Errorf("evaluator entry must be a mapping, got %s", kind) +} + +// EvaluatorAliasIsCircular reports an anchor that contains its own alias. +// +// Expanding it has no end, so it is named rather than followed. +func EvaluatorAliasIsCircular(anchor string) error { + return fmt.Errorf("anchor %q refers to itself, so it cannot be expanded", anchor) +} + +// --------------------------------------------------------------------------- +// Deploy and reconcile +// --------------------------------------------------------------------------- + +// EvalConfigInvalid reports a configuration a deploy will not act on. +func EvalConfigInvalid(err error) error { + return fmt.Errorf("eval config is invalid: %w", err) +} + +// ServiceCarriesNoConfig reports an azure.yaml entry with nothing to deploy. +func ServiceCarriesNoConfig(service string) error { + return fmt.Errorf( + "service %q carries no eval configuration; expected evaluators, datasets, or evals", + service) +} + +// ResolvingServiceRefs reports a $ref that could not be followed. +func ResolvingServiceRefs(err error) error { + return fmt.Errorf("resolving $ref in the eval service configuration: %w", err) +} + +// RefNeedsAProjectRoot reports an include reached without a directory to +// resolve it against. +func RefNeedsAProjectRoot(service string) error { + return fmt.Errorf( + "service %q uses `$ref`, but the project directory could not be determined, "+ + "so the referenced file cannot be found. Run from inside the azd project, "+ + "or inline the configuration this service points at", + service) +} + +// CatalogNameBehindAnInclude reports a name declared through a `$ref`, which +// this command cannot edit in place. +func CatalogNameBehindAnInclude(kind, name string) error { + return fmt.Errorf( + "%s %q is already declared through a `$ref`, so this command cannot update "+ + "it here: adding a second entry would collide with the first only after "+ + "the include is resolved. Edit the referenced file, or generate under a "+ + "different name", + kind, name) +} + +// ReadingServiceConfig reports the service entry failing to serialize. +func ReadingServiceConfig(err error) error { + return fmt.Errorf("reading the eval service configuration: %w", err) +} + +// ReconcilingDataset reports the dataset a deploy has reached. +func ReconcilingDataset(dataset string) string { + return fmt.Sprintf("Reconciling dataset %s", dataset) +} + +// ReconcilingEvaluator reports the evaluator a deploy has reached. +func ReconcilingEvaluator(evaluator string) string { + return fmt.Sprintf("Reconciling evaluator %s", evaluator) +} + +// ReconcilingEval reports the eval a deploy has reached. +func ReconcilingEval(eval string) string { + return fmt.Sprintf("Reconciling eval %s", eval) +} + +// PublishedVersion reports an artifact a deploy published. +func PublishedVersion(kind, name, version string) string { + return fmt.Sprintf("Published %s %s version %s", kind, name, version) +} + +// UnchangedAtVersion reports an artifact a deploy left alone. +func UnchangedAtVersion(kind, name, version string) string { + return fmt.Sprintf("%s %s is unchanged at version %s", + strings.ToUpper(kind[:1])+kind[1:], name, version) +} + +// EvalProblem attributes a failure to the eval it happened under. +func EvalProblem(eval string, err error) error { + return fmt.Errorf("eval %q: %w", eval, err) +} + +// EvalCreatedProgress and EvalUnchangedProgress are the deploy-time equivalents +// of EvalCreated and EvalUnchanged, without the status marks azd adds itself. +func EvalCreatedProgress(eval, id string) string { + return fmt.Sprintf("Created eval %s (%s)", eval, id) +} + +func EvalUnchangedProgress(eval, id string) string { + return fmt.Sprintf("Eval %s is unchanged (%s)", eval, id) +} + +// EvalCreated confirms a single eval created outside a full deploy. +func EvalCreated(eval, id string) string { + return fmt.Sprintf("%s Created eval: %s (%s)\n", doneMark, eval, id) +} + +// EvalUnchanged reports an eval a create found already in place. +// +// An eval is immutable, so re-running create against an unedited declaration +// creates nothing. Saying "Created" there claims work that did not happen, and +// hides the one thing worth checking: that the id, and so the run history +// hanging off it, survived. +func EvalUnchanged(eval, id string) string { + return fmt.Sprintf("%s Eval %s is unchanged (%s)\n", skippedMark, eval, id) +} + +// ListingEvals reports a failure to list the project's evals. +func ListingEvals(err error) error { + return fmt.Errorf("listing evals: %w", err) +} + +// NoEvals reports a project with no evals to list. +func NoEvals() string { + return "No evals found.\n" +} + +// EvalNotFound reports an eval id the project does not hold. +func EvalNotFound(evalID string) error { + return fmt.Errorf( + "no eval %q in this project; "+ + "`azd ai eval list` shows the ones there are", evalID) +} + +// AmbiguousEvalName reports a name carried by more than one eval. +// +// An eval is immutable, so editing a declaration creates another under the same +// name and leaves the previous one holding its run history. Deleting takes the +// runs with it, so which one is meant has to be said rather than guessed. +func AmbiguousEvalName(name string, ids []string) error { + return fmt.Errorf( + "%d evals are named %q, and deleting one discards its runs, so name the "+ + "id instead: %s", len(ids), name, strings.Join(ids, ", ")) +} + +// EvalGone reports an eval id there is nothing to delete at. +func EvalGone(evalID string) error { + return fmt.Errorf("no eval %q in this project", evalID) +} + +// ReadingEval reports a failure to read one eval. +func ReadingEval(evalID string, err error) error { + return fmt.Errorf("reading eval %q: %w", evalID, err) +} + +// DeletingEval reports the service refusing the delete. +func DeletingEval(evalID string, err error) error { + return fmt.Errorf("deleting eval %q: %w", evalID, err) +} + +// EvalDeleted confirms a deleted eval. +func EvalDeleted(evalID string) string { + return fmt.Sprintf("Deleted eval %s\n", evalID) +} + +// Hashing reports a local artifact that could not be fingerprinted. +func Hashing(path string, err error) error { + return fmt.Errorf("hashing %q: %w", filepath.ToSlash(path), err) +} + +// HashingEval reports an eval declaration that could not be fingerprinted. +func HashingEval(eval string, err error) error { + return fmt.Errorf("hashing eval %q: %w", eval, err) +} + +// --------------------------------------------------------------------------- +// Config +// --------------------------------------------------------------------------- + +// NoAzdProject reports a command that found no project to attach to. +func NoAzdProject() error { + return errors.New( + "no azd project found in this directory. Run `azd init` first, " + + "or run this from the root of an existing one; the eval service is added to " + + "its azure.yaml") +} + +// ConnectingToAzd reports the azd daemon being unreachable. +func ConnectingToAzd(err error) error { + return fmt.Errorf("connecting to azd: %w", err) +} + +// CreatingCredential reports the Azure credential failing to build. +func CreatingCredential(err error) error { + return fmt.Errorf("creating Azure credential: %w", err) +} + +// ErrNoAzdEnvironment reports that there is no azd environment to persist into. +var ErrNoAzdEnvironment = errors.New("no active azd environment") + +// NoAzdEnvironmentToWrite reports a value with nowhere to be remembered. +func NoAzdEnvironmentToWrite(key string) error { + return fmt.Errorf("%w to write %s into", ErrNoAzdEnvironment, key) +} + +// WritingEnvValue reports the azd environment refusing a write. +func WritingEnvValue(key string, err error) error { + return fmt.Errorf("writing %s to the azd environment: %w", key, err) +} + +// BuildingServiceEntry reports the eval service entry failing to build. +func BuildingServiceEntry(err error) error { + return fmt.Errorf("building the eval service entry: %w", err) +} + +// AddingServiceTo reports azd refusing to add the eval service. +func AddingServiceTo(rootConfig string, err error) error { + return fmt.Errorf("adding the eval service to %s: %w", rootConfig, err) +} + +// SourceNotADataSource reports an --source that names nothing rows come from. +func SourceNotADataSource(source, dataset, traces string) error { + return fmt.Errorf("--source %q is not a data source; use %q or %q", source, dataset, traces) +} + +// TracesTakesNoDataset reports --dataset paired with a trace-backed eval. +func TracesTakesNoDataset() error { + return errors.New("--source traces reads production traces, so it takes no --dataset") +} + +// MaxTracesNeedsTraceSource reports --max-traces without a trace-backed eval. +func MaxTracesNeedsTraceSource() error { + return errors.New("--max-traces caps a trace-backed eval; pass --source traces") +} + +// MaxTracesMustBePositive reports a negative --max-traces. +func MaxTracesMustBePositive() error { + return errors.New("--max-traces must be positive") +} + +// EvalAlreadyDeclared reports an init that would overwrite a hand-tuned eval. +func EvalAlreadyDeclared(eval, configPath string) error { + return fmt.Errorf( + "an eval named %q already exists in %s; choose another name with --name, "+ + "or pass --force to replace it. `init` only adds: editing an eval is a file edit", + eval, configPath) +} + +// CreatingDatasetsDir reports the datasets directory failing to be created. +func CreatingDatasetsDir(err error) error { + return fmt.Errorf("creating the datasets directory: %w", err) +} + +// CreatingEvaluatorsDir reports the evaluators directory failing to be created. +func CreatingEvaluatorsDir(err error) error { + return fmt.Errorf("creating the evaluators directory: %w", err) +} + +// DetectedTarget reports the agent the scaffolded eval will evaluate. +func DetectedTarget(target string) string { + return fmt.Sprintf("%s Detected agent target: %s\n", doneMark, target) +} + +// NoAgentToEvaluate reports a project declaring no agent service. +func NoAgentToEvaluate() error { + return errors.New( + "this project declares no agent service to evaluate. Add one, or name an " + + "existing agent with --target") +} + +// AmbiguousAgentTarget reports several agents where only one can be scaffolded. +func AmbiguousAgentTarget(agents []string) error { + return fmt.Errorf( + "this project declares more than one agent (%s), so --target says which to "+ + "evaluate", strings.Join(agents, ", ")) +} + +// SelectAgentPrompt asks which agent the eval is for. +func SelectAgentPrompt() string { + return "Select the agent to evaluate:" +} + +// SelectingAgent reports a failed agent prompt. +func SelectingAgent(err error) error { + return fmt.Errorf("selecting an agent to evaluate: %w", err) +} + +// JudgeModelRequired reports a scaffold that has no deployment to judge with. +// +// Reached when the project declares no model deployment to read one from, so +// the flag is the whole of the way out. Failing here is deliberate: the judging +// built-ins declare the deployment as required, so a config written without one +// is rejected by the service later, far from the command that wrote it. +func JudgeModelRequired() error { + return errors.New( + "a model deployment is required to judge with: pass --judge-model. " + + "This project declares no deployments, and the azd environment sets no " + + "AZURE_AI_MODEL_DEPLOYMENT_NAME") +} + +// EvaluatorRefEmpty reports an --evaluator that carries no name, which is what +// a stray comma leaves behind. +func EvaluatorRefEmpty() error { + return errors.New("--evaluator was given an empty reference: name an evaluator, " + + "or use builtin. for a built-in") +} + +// EvaluatorRefMalformed reports a reference no evaluator can be found under. +func EvaluatorRefMalformed(ref string) error { + return fmt.Errorf("%q is not an evaluator reference: repeat --evaluator, or separate "+ + "them with commas, and use builtin. for a built-in", ref) +} + +// GateNeedsATerminalRun refuses to gate a run that is still moving. +// +// The counts are partial until the run stops, so a threshold read from them +// can fail a run that would have passed. Ignoring the flag instead would leave +// a pipeline believing it is gated when it is not. +func GateNeedsATerminalRun(runID, status string) error { + return fmt.Errorf( + "run %s is %s, so --fail-on has only partial results to judge: "+ + "add --wait to gate once it finishes", + runID, status) +} + +// InEvalAt says which declaration an error came from. +// +// The source rules are checked in one place and reported from two, because the +// same file is read when it is validated and again when a run is built. Only +// the first of those has an index to name. +func InEvalAt(i int, eval string, err error) error { + return fmt.Errorf("evals[%d] (%s): %w", i, eval, err) +} + +// InEval says which eval an error came from, where there is no index. +func InEval(eval string, err error) error { + return fmt.Errorf("eval %q: %w", eval, err) +} + +// TraceWindowNotATime reports a window bound that is not a timestamp. +func TraceWindowNotATime(field, value string) error { + return fmt.Errorf( + "source.%s is %q, which is not a time: use RFC 3339, "+ + "for example 2026-08-18T09:00:00Z", + field, value) +} + +// TraceWindowBoundUnusable reports a bound that parses but says nothing. +// +// Both this layer and the wire read a zero as "no bound", so a bound that +// resolves to one would be dropped from the request rather than applied. +func TraceWindowBoundUnusable(field, value string) error { + return fmt.Errorf( + "source.%s is %q, which is not a time any traces were recorded at: "+ + "give a time the agent was running", + field, value) +} + +// TraceWindowEndsBeforeItStarts reports a window that can hold no traces. +func TraceWindowEndsBeforeItStarts(start, end string) error { + return fmt.Errorf( + "source.end_time %q is not after source.start_time %q, "+ + "so the window holds no traces", + end, start) +} + +// TraceWindowOverSpecified reports a window declared twice over. +// +// lookback_hours measures back from where the window closes and start_time is +// an absolute bound, so a file carrying both does not say which was meant. +func TraceWindowOverSpecified() error { + return errors.New( + "source declares both start_time and lookback_hours, which are two ways " + + "of saying where the window opens: keep one") +} + +// NegativeLookbackHours reports a lookback that is not a length. +func NegativeLookbackHours(hours int) error { + return fmt.Errorf( + "source.lookback_hours is %d, and how far back to look cannot be "+ + "negative: give the hours to look back", + hours) +} + +// LookbackTooLarge reports a lookback beyond the span a window may cover. +func LookbackTooLarge(hours, limit int) error { + return fmt.Errorf( + "source.lookback_hours is %d, which is beyond the %d hours a window can "+ + "reach back: give a shorter lookback, or replace it with a start_time", + hours, limit) +} + +// MaxTracesUnusable reports a negative cap written into the file. +// +// The flag that writes it is already guarded; this catches the file being +// edited afterwards, where a negative value is sent as-is and the run comes +// back empty. +func MaxTracesUnusable(maxTraces int) error { + return fmt.Errorf( + "source.max_traces is %d: give a positive cap, or leave it out to use "+ + "the service's default", + maxTraces) +} + +// SourceFieldsNotRead reports fields the declared source type ignores. +func SourceFieldsNotRead(sourceType string, fields []string) error { + if len(fields) == 1 { + return fmt.Errorf( + "source declares %s, which a %q source does not read: "+ + "remove it, or change the type to one that does", + fields[0], sourceType) + } + return fmt.Errorf( + "source declares %s, which a %q source does not read: "+ + "remove them, or change the type to one that does", + strings.Join(fields, ", "), sourceType) +} + +// MaxTurnsUnusable reports a turn cap a run could not apply. +func MaxTurnsUnusable(maxTurns int) error { + return fmt.Errorf( + "source.max_turns is %d: give a positive cap, or leave it out to use "+ + "the service's default", + maxTurns) +} + +// LookbackReachesTooFarBack reports a lookback that lands on an unusable start. +func LookbackReachesTooFarBack(hours int) error { + return fmt.Errorf( + "source.lookback_hours is %d, which opens the window before any trace "+ + "was recorded: give a shorter lookback", + hours) +} + +// AmbiguousJudgeModel reports several deployments where only one can be used. +func AmbiguousJudgeModel(models []string) error { + return fmt.Errorf( + "this project declares more than one model deployment (%s), so "+ + "--judge-model says which the graders judge with", strings.Join(models, ", ")) +} + +// SelectJudgeModelPrompt asks which deployment the graders judge with. +func SelectJudgeModelPrompt() string { + return "Select the model deployment the graders judge with:" +} + +// SelectEvalPrompt asks which of the declared evals a command means. +func SelectEvalPrompt() string { + return "Select the eval to use:" +} + +// SelectingJudgeModel reports a failed judge model prompt. +func SelectingJudgeModel(err error) error { + return fmt.Errorf("selecting a judge model deployment: %w", err) +} + +// UsingTraceSource reports a scaffold that reads production traces. +// +// Naming Application Insights is a claim about the project, so it is only made +// when a connection was actually found. `init` makes no service calls and +// cannot verify one it did not see. +func UsingTraceSource(connected bool) string { + if connected { + return fmt.Sprintf("%s Using data source: traces (Application Insights)\n", doneMark) + } + return fmt.Sprintf( + "%s Using data source: traces. No Application Insights connection is recorded "+ + "in this environment, so the run finds rows only if the project has one\n", + doneMark) +} + +// JudgeModelDeployment reports the deployment the graders will judge with. +func JudgeModelDeployment(model string) string { + return fmt.Sprintf("%s Judge model deployment: %s\n", doneMark, model) +} + +// GradingWith reports the evaluators the scaffold settled on. +// +// Omitting --evaluator picks them, so without this the one thing `init` decided +// on the reader's behalf is the one thing it does not mention. +func GradingWith(evaluators []string) string { + return fmt.Sprintf("%s Grading with: %s\n", doneMark, strings.Join(evaluators, ", ")) +} + +// createdHeading opens the list of what a scaffold wrote. +func createdHeading() string { + return "\nCreated\n" +} + +// ScaffoldHeading opens the list of what a scaffold wrote. `init` appends to an +// existing configuration rather than replacing it, and a reader who sees +// "Created" over a file they already had reasonably fears it was overwritten. +func ScaffoldHeading(existed bool) string { + if existed { + return "\nUpdated\n" + } + return createdHeading() +} + +// createdConfigLine names the configuration a scaffold wrote. +func createdConfigLine(configPath string) string { + return fmt.Sprintf(" %-33s evaluation configuration\n", configPath) +} + +// ScaffoldConfigLine names the configuration a scaffold wrote or added to. +func ScaffoldConfigLine(configPath string, existed bool) string { + if existed { + return fmt.Sprintf(" %-33s evaluation configuration (eval added)\n", configPath) + } + return createdConfigLine(configPath) +} + +// AddedServiceLine reports the eval service being added to the root config. +func AddedServiceLine(rootConfig, service string) string { + return fmt.Sprintf(" %-33s added service '%s'\n", rootConfig, service) +} + +// AlreadyDeclaresServiceLine reports a root config that already referenced the eval. +func AlreadyDeclaresServiceLine(rootConfig, service string) string { + return fmt.Sprintf(" %-33s already declares service '%s'\n", rootConfig, service) +} + +// FirstNextStep opens the list of commands to run after a scaffold. +func FirstNextStep(step string) string { + return fmt.Sprintf("\nNext: %s\n", step) +} + +// FurtherNextStep continues the list of commands to run after a scaffold. +func FurtherNextStep(step string) string { + return fmt.Sprintf(" %s\n", step) +} + +// CreatedCatalogFile reports a configuration created to hold a catalog entry. +func CreatedCatalogFile(configPath string) string { + return fmt.Sprintf("%s Created %s with the catalog entry\n", doneMark, filepath.ToSlash(configPath)) +} + +// AddedToCatalog reports a generated artifact recorded in the configuration. +func AddedToCatalog(kind, artifact, configPath string) string { + return fmt.Sprintf("%s Added %s %s to %s\n", doneMark, kind, artifact, filepath.ToSlash(configPath)) +} + +// ArtifactDescription names a catalogued artifact, with its version when there is one. +func ArtifactDescription(name, version string) string { + if version == "" || version == "latest" { + return fmt.Sprintf("'%s'", name) + } + return fmt.Sprintf("'%s' (version %s)", name, version) +} + +// NoEvalsDeclared reports a configuration with nothing to act on. +// +// The same sentence wherever it is reached. `generate` writes the dataset and +// evaluator it made into the catalog but declares no eval, so a `create` or a +// run straight afterwards lands here, and both need to be told the same way +// out. +func NoEvalsDeclared() error { + return errors.New( + "no eval is declared; `azd ai eval init` declares one. " + + "`generate` only adds the dataset and evaluator it made") +} + +// SeveralEvalsDeclared reports an unnamed eval where guessing would be wrong. +// The two commands that hit this name their eval differently, so neither form +// can be recommended on its own: `create` takes it as an argument, the run +// commands take --eval. +func SeveralEvalsDeclared(count int, names []string) error { + return fmt.Errorf( + "this configuration declares %d evals (%s); name the one you mean, "+ + "as an argument to `create` or with --eval on the run commands", + count, strings.Join(names, ", ")) +} + +// EvalNotDeclared reports a name the configuration does not carry. +func EvalNotDeclared(eval string, names []string) error { + // "this configuration has" with nothing after it is a sentence that stops + // mid-clause, which is what an empty list produces. + if len(names) == 0 { + return fmt.Errorf("eval %q is not declared, and this configuration declares none", eval) + } + return fmt.Errorf( + "eval %q is not declared; this configuration has %s", + eval, strings.Join(names, ", ")) +} + +// AtLeastOneEvalRequired reports it on the way to deploying. +// +// One sentence for one fact: the deploy door and the run door reach this from +// different directions and both need the same way out. +func AtLeastOneEvalRequired() error { + return NoEvalsDeclared() +} + +// EvalNameRequired reports an eval entry with no name. +func EvalNameRequired(index int) error { + return fmt.Errorf("evals[%d]: 'name' is required", index) +} + +// DuplicateEvalName reports two evals answering to the same name. +func DuplicateEvalName(index int, eval string) error { + return fmt.Errorf("evals[%d]: duplicate eval name %q", index, eval) +} + +// EvalsIdenticalApartFromName reports two evals nothing can tell apart once deployed. +func EvalsIdenticalApartFromName(index int, eval, first string) error { + return fmt.Errorf( + "evals[%d] (%s): identical to %q apart from its name and description; "+ + "give them different evaluators, datasets or settings, or declare one", + index, eval, first) +} + +// DatasetNameRequired reports a catalog entry with no name. +func DatasetNameRequired(index int) error { + return fmt.Errorf("datasets[%d]: 'name' is required", index) +} + +// DuplicateDatasetName reports two catalog entries answering to the same name. +func DuplicateDatasetName(index int, dataset string) error { + return fmt.Errorf("datasets[%d]: duplicate dataset name %q", index, dataset) +} + +// EvaluatorNameRequired reports a catalog entry with no name. +func EvaluatorNameRequired(index int) error { + return fmt.Errorf("evaluators[%d]: 'name' is required", index) +} + +// DuplicateEvaluatorName reports two catalog entries answering to the same name. +func DuplicateEvaluatorName(index int, evaluator string) error { + return fmt.Errorf("evaluators[%d]: duplicate evaluator name %q", index, evaluator) +} + +// BuiltinNeedsNoCatalogEntry reports a built-in declared as though it were custom. +func BuiltinNeedsNoCatalogEntry(index int, evaluator string) error { + return fmt.Errorf( + "evaluators[%d] (%s): a built-in needs no catalog entry; reference it "+ + "straight from an eval", index, evaluator) +} + +// EvaluatorVersionWithSource reports a pin the service would assign anyway. +func EvaluatorVersionWithSource(index int, evaluator string) error { + return fmt.Errorf( + "evaluators[%d] (%s): `version` cannot be set with `source`, because the "+ + "service assigns the version when it publishes. Drop `version` to "+ + "publish this file, or drop `source` to reference a version already "+ + "on the project", index, evaluator) +} + +// EvaluatorVersionWithDefinition reports the same pin against a rubric written +// out in the configuration rather than named as a file. +func EvaluatorVersionWithDefinition(index int, evaluator string) error { + return fmt.Errorf( + "evaluators[%d] (%s): `version` cannot be set with `definition`, because "+ + "the service assigns the version when it publishes. Drop `version` to "+ + "publish this rubric, or drop `definition` to reference a version "+ + "already on the project", index, evaluator) +} + +// EvaluatorRubricDeclaredTwice reports a rubric both named and written out. +// +// Publishing uses the written one, so leaving this to the schema alone would +// mean the file quietly never got read. +func EvaluatorRubricDeclaredTwice(index int, evaluator string) error { + return fmt.Errorf( + "evaluators[%d] (%s): `source` and `definition` both give the rubric; "+ + "declare one. Keep `definition` to publish what is written here, or "+ + "keep `source` to publish the file it names", index, evaluator) +} + +// DatasetAndSourceDeclareTheSameThing reports it where there is no index. +func DatasetAndSourceDeclareTheSameThing() error { + return errors.New("`dataset` and `source` both say where rows come from; declare one") +} + +// NoEvalToValidate reports a declaration that is not there at all. +func NoEvalToValidate() error { + return errors.New("no eval declaration to check") +} + +// DatasetNotInDatasetsCatalog reports an eval naming a dataset nobody declared. +func DatasetNotInDatasetsCatalog(index int, eval, dataset string) error { + return InEvalAt(index, eval, DatasetNotDeclared(dataset)) +} + +// DatasetNotDeclared reports it where there is no index. +func DatasetNotDeclared(dataset string) error { + return fmt.Errorf("dataset %q is not in the datasets catalog", dataset) +} + +// SourceTypeMissing reports it where there is no index. +func SourceTypeMissing() error { + return errors.New("source.type is required") +} + +// SourceTypeNotSupported reports the same, where there is no index to name. +func SourceTypeNotSupported(got, traces, responses string) error { + return fmt.Errorf("source.type %q is not supported; use %q or %q", got, traces, responses) +} + +// TraceSourceNeedsAnAgent reports it where there is no index. +// +// A target names one too, unless it names a model: a deployment name matches +// no spans, so it is not an answer to whose conversations to read. +func TraceSourceNeedsAnAgent() error { + return errors.New( + "source.agent_name is required for a trace source, " + + "or declare an agent target.name") +} + +// ResponsesSourceNeedsResponseIDs reports it where there is no index. +func ResponsesSourceNeedsResponseIDs() error { + return errors.New("source.response_ids is required for a responses source") +} + +// AtLeastOneEvaluatorRequired reports an eval that scores nothing. +func AtLeastOneEvaluatorRequired(index int, eval string) error { + return fmt.Errorf("evals[%d] (%s): at least one evaluator is required", index, eval) +} + +// EvaluatorFieldRequired reports an evaluators: entry with no evaluator named. +func EvaluatorFieldRequired(evalIndex, refIndex int) error { + return fmt.Errorf("evals[%d].evaluators[%d]: 'evaluator' is required", evalIndex, refIndex) +} + +// DuplicateCriterion reports two result rows nothing could tell apart. +func DuplicateCriterion(evalIndex, refIndex int, criterion string) error { + return fmt.Errorf( + "evals[%d].evaluators[%d]: duplicate criterion %q; give one a `name`", + evalIndex, refIndex, criterion) +} + +// EvaluatorNotInCatalog reports a reference to an evaluator nobody declared. +func EvaluatorNotInCatalog(evalIndex, refIndex int, evaluator string) error { + return fmt.Errorf( + "evals[%d].evaluators[%d]: evaluator %q is not in the evaluators catalog", + evalIndex, refIndex, evaluator) +} + +// TargetTypeNotSupported reports it where there is no index. +func TargetTypeNotSupported(got, agent, model string) error { + return fmt.Errorf("target.type %q is not supported; use %q or %q", got, agent, model) +} + +// EvaluationLevelNotSupported reports it where there is no index. +func EvaluationLevelNotSupported(got, turn, conversation string) error { + return fmt.Errorf("evaluation_level %q is invalid; expected %q or %q", got, turn, conversation) +} + +// TraceSourceCannotReadAModelTarget reports a trace eval pointed at a deployment. +// +// Its own sentence, because the general advice is "declare an agent target", +// which here reads as an invitation to relabel the deployment -- producing a +// filter that matches no spans and a run that reports nothing. +func TraceSourceCannotReadAModelTarget(name string) error { + return fmt.Errorf( + "source.agent_name is required for a trace source: target %q is a model "+ + "deployment, and traces are recorded against an agent, not a deployment", + name) +} + +// TargetNameMissing reports it where there is no index. +func TargetNameMissing() error { + return errors.New( + "target.name is required; remove the target: to score the dataset as it stands") +} + +// AmbiguousEvalConfig reports a directory holding both configuration names. +func AmbiguousEvalConfig(current, legacy string) error { + return fmt.Errorf( + "%s and %s are both present, and azure.yaml can reference only one of them. "+ + "Keep %s and delete the other, or point the service's $ref at the one you want", + filepath.ToSlash(current), filepath.ToSlash(legacy), filepath.ToSlash(current)) +} + +// ReadingEvalConfig reports a configuration file that would not read. +func ReadingEvalConfig(path string, err error) error { + if errors.Is(err, fs.ErrNotExist) { + return noEvalConfig(path) + } + return fmt.Errorf("reading eval config %q: %w", filepath.ToSlash(path), err) +} + +// noEvalConfig reports a command run before anything scaffolded a config. +// +// The bare read failure underneath is a Windows syscall phrase about a path, +// which describes the symptom of running `create` before `init` without naming +// either command. +// +// Still unwraps to fs.ErrNotExist, because callers that tolerate an absent +// configuration ΓÇö OpenEvalConfig, and the reference resolution above it ΓÇö decide +// that by asking, and a nicer sentence that stopped answering would turn every +// one of those into a failure. +func noEvalConfig(path string) error { + return &missingFileError{ + msg: fmt.Sprintf( + "no eval configuration at %s; run `azd ai eval init` to scaffold one", + filepath.ToSlash(path)), + } +} + +type missingFileError struct{ msg string } + +func (e *missingFileError) Error() string { return e.msg } +func (e *missingFileError) Unwrap() error { return fs.ErrNotExist } + +// ParsingEvalConfig reports a configuration file that would not parse. +func ParsingEvalConfig(path string, err error) error { + return fmt.Errorf("parsing eval config %q: %w", filepath.ToSlash(path), err) +} + +// SerializingEvalConfig reports a configuration that would not serialize. +func SerializingEvalConfig(err error) error { + return fmt.Errorf("serializing eval config: %w", err) +} + +// WritingEvalConfig reports a configuration file that would not be written. +func WritingEvalConfig(path string, err error) error { + return fmt.Errorf("writing eval config %q: %w", filepath.ToSlash(path), err) +} + +// ErrAmbiguousAgentService reports that a target name matched more than one service. +var ErrAmbiguousAgentService = errors.New("more than one agent service matches") + +// AmbiguousAgentService reports a --target that names no single set of instructions. +func AmbiguousAgentService(agent string, matched []string) error { + return fmt.Errorf( + "%w %q: %s. Name one of them with --target, or pass the text with "+ + "--agent-instruction", + ErrAmbiguousAgentService, agent, strings.Join(matched, ", ")) +} + +// InstructionFileUnreadable reports optimize metadata pointing at a missing file. +func InstructionFileUnreadable(metadataPath, named string, err error) error { + return fmt.Errorf( + "%s names instruction_file %q, which could not be read: %w", + metadataPath, named, err) +} + +// ListingTruncated reports a page walk that stopped before the end. +// +// Worth saying out loud rather than logging: a short evaluator listing resolves +// the latest version from the pages that arrived, so a truncated one can pick +// an older version and report nothing unusual. +func ListingTruncated(pages int) error { + return fmt.Errorf( + "stopped reading the listing after %d pages, so it may be incomplete", pages) +} + +// ServiceRefPointsElsewhere reports an existing service entry wired to a +// different configuration than the one just scaffolded. +func ServiceRefPointsElsewhere(serviceName, have, want string) error { + return fmt.Errorf( + "service %q already points at %s, and the configuration just written is "+ + "%s; point the service's $ref at the one you want, or scaffold with "+ + "--path %s", serviceName, have, want, have) +} + +// InstructionFileOutsideProject reports metadata pointing outside the project. +// +// The pointer is read from a file in the checkout, so it is only as trustworthy +// as the checkout: an absolute path or one climbing out with `..` would read +// something the project does not contain and send it on as agent instructions. +func InstructionFileOutsideProject(metadataPath, named string) error { + return fmt.Errorf( + "%s names instruction_file %q, which is outside the project; "+ + "name a path inside it", metadataPath, named) +} + +// FromNotASource reports a --from value the generation service has no path for. +func FromNotASource(from string, sources []string) error { + return fmt.Errorf( + "--from %q is not a source; use one of %s", + from, strings.Join(sources, ", ")) +} + +// ConfigLockUnavailable reports a config lock that could not be taken. +// +// Not fatal, and said out loud for that reason: the work goes ahead unlocked, +// so a lost update afterwards has no other explanation on record. +func ConfigLockUnavailable(evalDir string, err error) error { + if err == nil { + return fmt.Errorf( + "another process is still updating %s, so this update is not "+ + "serialized against it", filepath.ToSlash(evalDir)) + } + return fmt.Errorf( + "could not lock %s, so this update is not serialized against other "+ + "processes: %w", filepath.ToSlash(evalDir), err) +} + +// InvalidNextLink reports a pagination link the service sent that will not parse. +func InvalidNextLink(link string, err error) error { + return fmt.Errorf("invalid nextLink %q: %w", link, err) +} + +// NextLinkOffOrigin reports a pagination link pointing somewhere other than the +// project endpoint. Following it would send the caller's token to that host. +func NextLinkOffOrigin(origin string) error { + return fmt.Errorf("refusing to follow nextLink to %s: it is not the project endpoint", origin) +} + +// PageLinkLeftTheService reports a paging link pointing somewhere else. +// +// The link arrives in a response body and this client sends an Authorization +// header, so following one to another host would send the token there. +func PageLinkLeftTheService(expected, got string) error { + return fmt.Errorf( + "the service returned a paging link for %q while this client is "+ + "talking to %q, so it was not followed", got, expected) +} + +// SampleSizeOutOfRange reports a row count the generation service would reject. +func SampleSizeOutOfRange(min, max, got int) error { + return fmt.Errorf("sample size must be between %d and %d, got %d", min, max, got) +} + +// MaxSamplesNegative reports a declared row cap below zero. +// +// Anything not above zero reads as "no cap", so this used to send the whole +// dataset to a run that is billed per row -- the opposite of what a cap asks +// for, and silent. +func MaxSamplesNegative(got int) error { + return fmt.Errorf( + "max_samples cannot be negative, got %d. "+ + "Remove it to send every row, or set the number of rows to send", got) +} + +// NegativeMaxSamplesFlag reports the same thing given on the command line. +func NegativeMaxSamplesFlag(got int) error { + return fmt.Errorf( + "--max-samples cannot be negative, got %d. "+ + "Omit it to send every row, or give the number of rows to send", got) +} + +// FlagDoesNotApply reports a flag given to a generate that produces nothing it +// could affect. +// +// Each of these is read while building one kind of artifact and ignored while +// building the other, so given for the wrong one they were accepted and +// dropped: `--evaluator --max-samples 50` produced a rubric and said nothing +// about the 50. +func FlagDoesNotApply(flag, narrowedBy string) error { + return fmt.Errorf( + "--%s has no effect on what %s generates. "+ + "Drop --%s, or drop %s to generate both", flag, narrowedBy, flag, narrowedBy) +} + +// NegativeTraceDays reports a trace window below zero. +// +// Zero already means "do not read traces", so a negative value has nothing +// left to mean; it used to be accepted and treated as zero, which silently +// produced a rubric with none of the trace seeding that was asked for. +func NegativeTraceDays(got int) error { + return fmt.Errorf( + "--trace-days cannot be negative, got %d. "+ + "Use 0 to seed the rubric from no traces, or the number of days to read", got) +} + +// OutputDirNeedsTheWait reports an output directory that nothing will be +// written to. +// +// --no-wait returns as soon as the job is submitted, so there is no artifact +// to place. Accepting both left the caller waiting for a file that was never +// coming. +func OutputDirNeedsTheWait() error { + return errors.New( + "--output-dir has nothing to write to with --no-wait, which returns " + + "before the artifact exists. Drop --no-wait, or collect the " + + "artifact later with `azd ai eval job show`") +} + +// EndpointEmpty reports a project endpoint given as blank. +func EndpointEmpty() error { + return exterrors.Validation( + exterrors.CodeInvalidParameter, + "project endpoint must not be empty", + "provide a Foundry project endpoint URL "+ + "(e.g. https://.services.ai.azure.com/api/projects/)", + ) +} + +// EndpointUnparseable reports a project endpoint that is not a URL. +func EndpointUnparseable(err error) error { + return exterrors.Validation( + exterrors.CodeInvalidParameter, + fmt.Sprintf("invalid project endpoint URL: %v", err), + "provide a valid https:// Foundry project endpoint URL", + ) +} + +// EndpointNotHTTPS reports a project endpoint on the wrong scheme. +func EndpointNotHTTPS() error { + return exterrors.Validation( + exterrors.CodeInvalidParameter, + "project endpoint must use https", + "provide an https:// URL", + ) +} + +// EndpointNotFoundryHost reports a project endpoint pointing somewhere else. +func EndpointNotFoundryHost(host, suffix string) error { + return exterrors.Validation( + exterrors.CodeInvalidParameter, + fmt.Sprintf( + "project endpoint host %q is not a recognized Foundry host (*%s)", + host, suffix, + ), + "the host must end with "+suffix, + ) +} + +// EndpointHasPort reports a project endpoint carrying an explicit port. +func EndpointHasPort(host string) error { + return exterrors.Validation( + exterrors.CodeInvalidParameter, + fmt.Sprintf("project endpoint host %q must not include a port", host), + "remove the explicit port from the URL", + ) +} + +// NoEndpoint reports a project endpoint that no source could supply. +func NoEndpoint() error { + return exterrors.Dependency( + exterrors.CodeMissingProjectEndpoint, + "no Foundry project endpoint resolved", + "persist a workspace default with `azd ai project set `, "+ + "or set FOUNDRY_PROJECT_ENDPOINT (or AZURE_AI_PROJECT_ENDPOINT) "+ + "in the active azd environment, "+ + "or export FOUNDRY_PROJECT_ENDPOINT (or AZURE_AI_PROJECT_ENDPOINT) in your shell", + ) +} + +// ProjectContextClient reports the config helper failing to build. +func ProjectContextClient(err error) error { + return fmt.Errorf("getProjectContext: %w", err) +} + +// ProjectContextRead reports the persisted project context failing to read. +func ProjectContextRead(err error) error { + return fmt.Errorf("getProjectContext: failed to read config: %w", err) +} + +// --------------------------------------------------------------------------- +// Output +// --------------------------------------------------------------------------- + +// Progress markers from the azd style guide, so the extension's lines sit +// alongside core's without a second vocabulary. +const ( + doneMark = "(Γ£ô) Done:" // finished successfully + skippedMark = "(-) Skipped:" // intentionally not done, not a failure + failedMark = "(x) Failed:" // the step did not complete +) + +// Warning reports a problem that is not worth failing the command over. +func Warning(err error) string { + return fmt.Sprintf("warning: %v\n", err) +} + +// PortalLink closes a detail view with the asset's portal URL. +func PortalLink(url string) string { + return fmt.Sprintf("Portal: %s\n", url) +} + +// FlagRequired reports a value the command needs and cannot settle itself. +// +// It used to add "(running with --no-prompt)", which was untrue at every call +// site: none of them prompts, so the parenthetical named a flag the caller had +// not passed and implied that dropping it would make the command ask. +func FlagRequired(name string) error { + return fmt.Errorf("--%s is required", name) +} + +// Creating reports a directory or file that could not be created. +func Creating(path string, err error) error { + return fmt.Errorf("creating %q: %w", filepath.ToSlash(path), err) +} + +// Serializing reports a value that could not be written out. +func Serializing(path string, err error) error { + return fmt.Errorf("serializing %q: %w", filepath.ToSlash(path), err) +} + +// Writing reports a file that could not be written. +func Writing(path string, err error) error { + return fmt.Errorf("writing %q: %w", filepath.ToSlash(path), err) +} + +// ReadingPath reports a file or directory that could not be read. +func ReadingPath(path string, err error) error { + return fmt.Errorf("reading %s: %w", path, err) +} + +// quoteList renders names as a readable "a", "b" and "c". +func quoteList(values []string) string { + if len(values) == 0 { + return "nothing" + } + quoted := make([]string, 0, len(values)) + for _, value := range values { + quoted = append(quoted, fmt.Sprintf("%q", value)) + } + sort.Strings(quoted) + if len(quoted) == 1 { + return quoted[0] + } + return strings.Join(quoted[:len(quoted)-1], ", ") + " and " + quoted[len(quoted)-1] +} + +// pluralColumns agrees with however many columns are missing. +func pluralColumns(values []string) string { + if len(values) == 1 { + return "that column" + } + return "those columns" +} + +// --------------------------------------------------------------------------- +// Talking to the service +// --------------------------------------------------------------------------- + +// InvalidEndpointURL reports a client built on an endpoint that will not parse. +func InvalidEndpointURL(err error) error { + return fmt.Errorf("invalid endpoint URL: %w", err) +} + +// InvalidRequestPath reports a request path that will not parse. +func InvalidRequestPath(path string, err error) error { + return fmt.Errorf("invalid request path %q: %w", path, err) +} + +// CreatingRequest reports a request that could not be built. +func CreatingRequest(err error) error { + return fmt.Errorf("failed to create request: %w", err) +} + +// MarshalingRequest reports a request body that would not serialize. +func MarshalingRequest(err error) error { + return fmt.Errorf("failed to marshal request: %w", err) +} + +// SettingRequestBody reports a request body that would not attach. +func SettingRequestBody(err error) error { + return fmt.Errorf("failed to set request body: %w", err) +} + +// RequestFailed reports a request that never reached an answer. +// +// A credential that cannot mint a token fails here rather than as a 401, and +// the SDK's own text for it names neither azd nor the way out. isCredentialFailure +// decides which is which; see it for how. +// +// The hint is in the message as well as the suggestion because the suggestion +// is not rendered on every surface, and it offers a retry first: this call +// shells out to `azd auth token`, which has been seen to fail transiently +// against a login that was perfectly valid -- measured once at over 70 seconds, +// long enough to lose to a deadline. +func RequestFailed(err error) error { + if isCredentialUnavailable(err) { + // Not an expired login, and `azd auth login` cannot be run to fix it. + return exterrors.Auth( + exterrors.CodeAuthFailed, + fmt.Sprintf( + "could not get a token for the Foundry project because azd itself "+ + "could not be run: %v", err), + "check that `azd` is installed and on PATH") + } + if isCredentialFailure(err) { + return exterrors.Auth( + exterrors.CodeLoginExpired, + fmt.Sprintf( + "could not get a token for the Foundry project: %v. "+ + "Try again; if it keeps failing, run `azd auth login`", err), + "try the command again, then `azd auth login` if it keeps failing") + } + return fmt.Errorf("HTTP request failed: %w", err) +} + +// isCredentialUnavailable reports the credential never having run at all, as +// opposed to running and being refused. +// +// azidentity's credentialUnavailableError is unexported, so this matches the +// two messages it carries for that case. Worth separating because the answer +// to both is not `azd auth login` -- you cannot log in with a tool that is not +// on PATH. +func isCredentialUnavailable(err error) bool { + if err == nil { + return false + } + text := err.Error() + return strings.Contains(text, "executable not found on path") || + strings.Contains(text, "is not recognized") +} + +// ServiceRefused turns an unauthorized answer into one that says what to do. +// Every other status is left as the service reported it. +func ServiceRefused(status int, err error) error { + if status == http.StatusUnauthorized || status == http.StatusForbidden { + return exterrors.Auth( + exterrors.CodeAuthFailed, + fmt.Sprintf( + "the Foundry project refused the request (HTTP %d): %v. "+ + "Run `azd auth login`, and check you have access to this project", + status, err), + "run `azd auth login`, and check you have access to this project") + } + return err +} + +// isCredentialFailure reports whether the request failed because no token could +// be minted, rather than for any of the other reasons a request fails. +// +// Decided on the SDK's own error types. This used to also match the phrase +// "failed to acquire a token" anywhere in the text, which any error is free to +// contain -- a service that could not acquire a token bucket lease was told its +// login had expired and to run `azd auth login`. +// +// The credential names stay as a fallback because credentialUnavailableError is +// unexported: a credential that never ran can only be recognized by the name it +// puts in its own message. +func isCredentialFailure(err error) bool { + if err == nil { + return false + } + + var authFailed *azidentity.AuthenticationFailedError + var authRequired *azidentity.AuthenticationRequiredError + if errors.As(err, &authFailed) || errors.As(err, &authRequired) { + return true + } + + text := err.Error() + for _, credential := range []string{ + "AzureDeveloperCLICredential", + "DefaultAzureCredential", + } { + if strings.Contains(text, credential) { + return true + } + } + return false +} + +// ReadingResponseBody reports a response that could not be read. +func ReadingResponseBody(err error) error { + return fmt.Errorf("failed to read response body: %w", err) +} + +// ParsingResponse reports a response that could not be parsed. +func ParsingResponse(err error) error { + return fmt.Errorf("failed to parse response: %w", err) +} + +// ParsingNumber reports a numeric field that did not arrive as a number. +func ParsingNumber(data string, err error) error { + return fmt.Errorf("parsing number %s: %w", data, err) +} + +// InvalidContainerURI reports a storage URI the service handed back unusable. +func InvalidContainerURI(err error) error { + return fmt.Errorf("invalid container SAS URI: %w", err) +} + +// CreatingUploadRequest reports the blob upload request failing to build. +func CreatingUploadRequest(err error) error { + return fmt.Errorf("failed to create upload request: %w", err) +} + +// UploadingBlobFailed reports the blob upload never reaching an answer. +func UploadingBlobFailed(err error) error { + return fmt.Errorf("failed to upload blob: %w", err) +} + +// BlobUploadStatus reports storage refusing the upload. +func BlobUploadStatus(status int, body string) error { + return fmt.Errorf("blob upload failed with status %d: %s", status, body) +} + +// CreatingDownloadRequest reports the dataset download request failing to build. +func CreatingDownloadRequest(err error) error { + return fmt.Errorf("failed to create download request: %w", err) +} + +// DownloadingDatasetBlob reports the dataset download never reaching an answer. +func DownloadingDatasetBlob(err error) error { + return fmt.Errorf("failed to download dataset from blob: %w", err) +} + +// BlobDownloadStatus reports storage refusing the download. +func BlobDownloadStatus(status int) error { + return fmt.Errorf("blob download failed with status %d", status) +} + +// ReadingDatasetContent reports a downloaded dataset that could not be read. +func ReadingDatasetContent(err error) error { + return fmt.Errorf("failed to read dataset content: %w", err) +} + +// CreatingListRequest reports the container listing request failing to build. +func CreatingListRequest(err error) error { + return fmt.Errorf("failed to create list request: %w", err) +} + +// ListingContainerBlobs reports the container listing never reaching an answer. +func ListingContainerBlobs(err error) error { + return fmt.Errorf("failed to list container blobs: %w", err) +} + +// ContainerListStatus reports storage refusing the listing. +func ContainerListStatus(status int) error { + return fmt.Errorf("container list failed with status %d", status) +} + +// ReadingListResponse reports a container listing that could not be read. +func ReadingListResponse(err error) error { + return fmt.Errorf("failed to read list response: %w", err) +} + +// CreatingBlobDownloadRequest reports the blob download request failing to build. +func CreatingBlobDownloadRequest(err error) error { + return fmt.Errorf("failed to create blob download request: %w", err) +} + +// DownloadingBlob reports one blob's download never reaching an answer. +func DownloadingBlob(err error) error { + return fmt.Errorf("failed to download blob: %w", err) +} + +// BlobDownloadStatusFor reports storage refusing one named blob. +func BlobDownloadStatusFor(status int, blobName string) error { + return fmt.Errorf("blob download failed with status %d for %s", status, blobName) +} + +// ReadingBlobContent reports a downloaded blob that could not be read. +func ReadingBlobContent(err error) error { + return fmt.Errorf("failed to read blob content: %w", err) +} + +// ParsingProjectResourceID reports a project ARM id that will not parse. +func ParsingProjectResourceID(err error) error { + return fmt.Errorf("failed to parse project resource ID: %w", err) +} + +// EncodingSubscriptionID reports a subscription that would not encode for a URL. +func EncodingSubscriptionID(err error) error { + return fmt.Errorf("failed to encode subscription ID: %w", err) +} + +// NotAFoundryProjectResourceID reports an ARM id that names something else. +func NotAFoundryProjectResourceID(resourceID string) error { + return fmt.Errorf( + "resource ID does not represent a Foundry project (missing parent account): %s", + resourceID) +} + +// InvalidSubscriptionID reports a subscription id that is not a GUID. +func InvalidSubscriptionID(err error) error { + return fmt.Errorf("invalid subscription ID format: %w", err) +} + +// CouldNotReadAgentForModel reports a target agent whose deployment could not +// be read, leaving generation without a default model. +func CouldNotReadAgentForModel(agent string, err error) string { + if notFound(err) { + return fmt.Sprintf(" warning: no agent %q in this project, so there is no "+ + "deployment to default to; pass --generation-model\n", agent) + } + return fmt.Sprintf(" warning: could not read agent %q for its deployment: %v\n", agent, err) +} + +// shellArg wraps a value a shell would otherwise read as more than one +// argument. +// +// A suggested command is written to be pasted and run. A name the caller chose +// -- `--name "my eval"` becomes the dataset's name -- turns +// `--dataset-name my eval` into one flag and a stray positional argument, which +// generate refuses without naming the cause. +// +// Double quotes are what cmd, PowerShell, bash and zsh all read the same way. +// A value containing $ or a backtick has no portable answer and is wrapped +// anyway: one argument that may expand still beats two that certainly break. +// Backslashes are left alone, so a Windows path comes back out as itself. +func shellArg(v string) string { + if v == "" { + return `""` + } + if !strings.ContainsAny(v, " \t\n\"'`$&|;<>()*?[]#~!") { + return v + } + return `"` + strings.ReplaceAll(v, `"`, `\"`) + `"` +} + +// ShellArg is shellArg for the command builders outside this package, so one +// rule decides how every printed command quotes what it carries. +func ShellArg(v string) string { + return shellArg(v) +} diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/project/config_keys_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/project/config_keys_test.go new file mode 100644 index 00000000000..a5b60cb4763 --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/internal/project/config_keys_test.go @@ -0,0 +1,133 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package project + +import ( + "reflect" + "strings" + "testing" + + "azureaieval/internal/pkg/evalcore" + + "github.com/stretchr/testify/assert" +) + +// eval.yaml is the file a user writes, so its keys are the contract. They are +// pinned whole rather than exercised through fixtures: a fixture that stops +// parsing says a test broke, not that a published key was renamed under +// everyone who already wrote one. +// +// The spec's configuration model is the source for every list here. Changing +// one means changing both. + +// yamlKeys reads the yaml tag names off a struct, in declaration order. +func yamlKeys(t *testing.T, v any) []string { + t.Helper() + typ := reflect.TypeOf(v) + for typ.Kind() == reflect.Pointer { + typ = typ.Elem() + } + + var keys []string + for field := range typ.Fields() { + tag := field.Tag.Get("yaml") + if tag == "" || tag == "-" { + continue + } + name := strings.Split(tag, ",")[0] + if name == "" { + continue + } + keys = append(keys, name) + } + return keys +} + +// The top level: catalogs first, then the evals defined over them. +func TestEvalConfigKeys(t *testing.T) { + assert.Equal(t, []string{"datasets", "evaluators", "evals"}, + yamlKeys(t, EvalConfig{}), + "the top-level shape is the spec's configuration model") +} + +// An eval names what it evaluates, what it reads, and how to grade it, or is +// pulled in whole with `$ref`. +func TestEvalKeys(t *testing.T) { + assert.ElementsMatch(t, + []string{ + "$ref", "name", "id", "description", "dataset", "source", + "evaluation_level", "max_samples", "evaluators", "target", + }, + yamlKeys(t, Eval{})) +} + +// Every entry in an eval's evaluators: list is a map keyed evaluator:, and the +// spec gives that map exactly five keys. +func TestEvaluatorRefKeys(t *testing.T) { + assert.ElementsMatch(t, + []string{"evaluator", "name", "version", "initialization_parameters", "data_mapping"}, + yamlKeys(t, evalcore.EvaluatorRef{}), + "the spec tabulates these five; a sixth is a promise it does not make") +} + +// source: says where rows come from when they are not a dataset. +func TestSourceDeclKeys(t *testing.T) { + assert.ElementsMatch(t, + []string{ + "type", "lookback_hours", "max_traces", "agent_name", "response_ids", "max_turns", + "agent_version", "start_time", "end_time", + }, + yamlKeys(t, SourceDecl{})) +} + +// The catalogs are named, reusable assets. A dataset says where its rows come +// from. An evaluator says where its rubric is -- named as a file, or written +// out under `definition` -- and may instead be pulled in with `$ref`, which is +// modelled rather than only resolved so that a command which reads, modifies +// and saves the file writes the author's include back out instead of inlining +// it. +func TestCatalogKeys(t *testing.T) { + assert.ElementsMatch(t, + []string{"$ref", "name", "file", "version"}, yamlKeys(t, DatasetDecl{})) + assert.ElementsMatch(t, + []string{"$ref", "name", "source", "version", "definition"}, + yamlKeys(t, EvaluatorDecl{})) +} + +// The spec's casing table: eval.yaml uses the API's snake_case throughout, so +// a camelCase key would be the one place a reader has to remember an exception. +func TestEveryKeyIsSnakeCase(t *testing.T) { + shapes := map[string]any{ + "EvalConfig": EvalConfig{}, + "Eval": Eval{}, + "SourceDecl": SourceDecl{}, + "Target": Target{}, + "DatasetDecl": DatasetDecl{}, + "EvaluatorDecl": EvaluatorDecl{}, + "EvaluatorRef": evalcore.EvaluatorRef{}, + } + + for name, shape := range shapes { + for _, key := range yamlKeys(t, shape) { + assert.Equalf(t, strings.ToLower(key), key, + "%s.%s is not snake_case; eval.yaml uses the API's spelling throughout", name, key) + assert.NotContainsf(t, key, "-", + "%s.%s uses a dash; the API's convention is underscores", name, key) + } + } +} + +// `target:` always means invoke and `source:` always means where rows come +// from. A trace-backed eval has no target, which is what agent_name under +// source: exists to say. +func TestTargetAndSourceAreDistinct(t *testing.T) { + assert.ElementsMatch(t, []string{"type", "name"}, yamlKeys(t, Target{}), + "the spec's target: is a type and a name; a version there would pin the "+ + "agent an eval invokes, which nothing asks for") + + assert.Contains(t, yamlKeys(t, SourceDecl{}), "agent_name", + "a trace run filters by agent rather than invoking one") + assert.NotContains(t, yamlKeys(t, Target{}), "agent_name", + "the target already names what it invokes") +} diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/project/eval_config.go b/cli/azd/extensions/azure.ai.evaluations/internal/project/eval_config.go new file mode 100644 index 00000000000..bb8e262b7c6 --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/internal/project/eval_config.go @@ -0,0 +1,402 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +// Package project models the eval configuration carried by the +// `host: azure.ai.eval` service entry in azure.yaml. +package project + +import ( + "strings" + + "azureaieval/internal/messages" + "azureaieval/internal/pkg/evalcore" +) + +// EvalConfig is one evaluation configuration: the catalogs of reusable assets, +// and every eval defined over them. +// +// It is the body of a single `azure.ai.eval` service entry, pulled in with +// $ref. One file rather than one per eval, because the catalogs are shared: +// two evals over the same dataset should name it once. +// +// How it is stored lives in eval_config_store.go. +type EvalConfig struct { + Datasets []DatasetDecl `yaml:"datasets,omitempty" json:"datasets,omitempty"` + Evaluators []EvaluatorDecl `yaml:"evaluators,omitempty" json:"evaluators,omitempty"` + Evals []Eval `yaml:"evals,omitempty" json:"evals,omitempty"` +} + +// DatasetDecl is a catalog entry. A local File is uploaded on deploy; without +// one the name must already resolve to a registered dataset. +// +// Deliberately no `$ref` in place of `file`: that directive replaces a +// definition with one loaded from a YAML or JSON file, and these rows are an +// artifact to publish. A `.jsonl` is neither, so there would be nothing to +// splice. +// +// Ref is the directive an author may still write to load this whole entry from +// its own file. Core resolves it on every node, so the commands that use the +// configuration never see it -- but the ones that read, modify and save do, and +// modelling it is what keeps them from refusing a file `azd up` accepts. +type DatasetDecl struct { + Ref string `yaml:"$ref,omitempty" json:"$ref,omitempty"` + Name string `yaml:"name,omitempty" json:"name,omitempty"` + File string `yaml:"file,omitempty" json:"file,omitempty"` + Version string `yaml:"version,omitempty" json:"version,omitempty"` +} + +// EvaluatorDecl is a catalog entry for a custom evaluator. Built-ins are +// referenced straight from an eval and never declared here. +// +// Source names a `.json` file holding a rubric: a list of weighted scoring +// dimensions. +// +// Ref carries the `$ref` an author wrote. Commands that use the configuration +// never see it -- resolution has already replaced the entry with the file's +// content by then -- but the commands that read, modify and save the file do, +// and modelling it is what lets the include survive being written back. Name is +// omitempty for the same reason: an entry that is only a `$ref` has no name of +// its own until the file it names supplies one. +// +// Definition is the rubric written out in place of naming a file, which is what +// lets a `$ref` name a rubric: resolution splices the file's keys in here, and +// they have to land on a field to survive strict decoding. It is deliberately +// one named key rather than a catch-all: a catch-all would swallow every +// misspelling in the entry and publish it to the service as rubric content. +type EvaluatorDecl struct { + Ref string `yaml:"$ref,omitempty" json:"$ref,omitempty"` + Name string `yaml:"name,omitempty" json:"name,omitempty"` + Source string `yaml:"source,omitempty" json:"source,omitempty"` + Version string `yaml:"version,omitempty" json:"version,omitempty"` + Definition map[string]any `yaml:"definition,omitempty" json:"definition,omitempty"` +} + +// Eval is one evaluation defined over the catalogs. +// +// Dataset and Source are alternatives: rows come from a catalog dataset, or +// from a source such as production traces. Target is what gets invoked, and is +// a separate axis ΓÇö an eval can read traces and invoke nothing. +type Eval struct { + Ref string `yaml:"$ref,omitempty" json:"$ref,omitempty"` + Name string `yaml:"name,omitempty" json:"name,omitempty"` + ID string `yaml:"id,omitempty" json:"id,omitempty"` + Description string `yaml:"description,omitempty" json:"description,omitempty"` + Dataset string `yaml:"dataset,omitempty" json:"dataset,omitempty"` + Source *SourceDecl `yaml:"source,omitempty" json:"source,omitempty"` + EvaluationLevel string `yaml:"evaluation_level,omitempty" json:"evaluation_level,omitempty"` + MaxSamples int `yaml:"max_samples,omitempty" json:"max_samples,omitempty"` + Evaluators evalcore.EvaluatorList `yaml:"evaluators,omitempty" json:"evaluators,omitempty"` + Target *Target `yaml:"target,omitempty" json:"target,omitempty"` +} + +// SourceDecl says where an eval's rows come from when they are not a dataset. +type SourceDecl struct { + Type string `yaml:"type" json:"type"` + LookbackHours int `yaml:"lookback_hours,omitempty" json:"lookback_hours,omitempty"` + MaxTraces int `yaml:"max_traces,omitempty" json:"max_traces,omitempty"` + AgentName string `yaml:"agent_name,omitempty" json:"agent_name,omitempty"` + ResponseIDs []string `yaml:"response_ids,omitempty" json:"response_ids,omitempty"` + MaxTurns int `yaml:"max_turns,omitempty" json:"max_turns,omitempty"` + // AgentVersion pins which deployment's spans are read. Without it the + // service chooses, and a redeployed agent is evaluated against whichever + // version it picked. + AgentVersion string `yaml:"agent_version,omitempty" json:"agent_version,omitempty"` + // StartTime and EndTime bound the window explicitly. LookbackHours stays + // supported, read as a start bound measured back from EndTime, or from now + // when nothing closes the window. + StartTime string `yaml:"start_time,omitempty" json:"start_time,omitempty"` + EndTime string `yaml:"end_time,omitempty" json:"end_time,omitempty"` +} + +// Source types an eval can read rows from. +const ( + SourceTypeTraces = "traces" + SourceTypeResponses = "responses" +) + +// DefaultScaffoldMaxTraces is the cap init writes on a trace-backed eval, so a +// first run is bounded rather than taking the service's own default of 1000. +// Deleting max_traces from the file restores that default. +const DefaultScaffoldMaxTraces = 20 + +// Target names what the run invokes. +type Target struct { + Type string `yaml:"type" json:"type"` + Name string `yaml:"name" json:"name"` +} + +// Target types the extension can invoke. Absent means nothing is invoked and +// the dataset already carries the answers. +const ( + TargetTypeAgent = "agent" + TargetTypeModel = "model" +) + +// Evaluation levels accepted by the service. The service default is turn. +const ( + EvaluationLevelTurn = "turn" + EvaluationLevelConversation = "conversation" +) + +// EvalNames lists the declared evals in declaration order. +func (c *EvalConfig) EvalNames() []string { + names := make([]string, 0, len(c.Evals)) + for _, e := range c.Evals { + names = append(names, e.Name) + } + return names +} + +// Eval returns the named eval. +// +// An empty name is only answered when the file declares exactly one, because +// guessing which eval a command meant is the kind of mistake that is noticed +// only after it has run. +func (c *EvalConfig) Eval(name string) (*Eval, error) { + if name == "" { + switch len(c.Evals) { + case 0: + return nil, messages.NoEvalsDeclared() + case 1: + return &c.Evals[0], nil + default: + return nil, messages.SeveralEvalsDeclared(len(c.Evals), c.EvalNames()) + } + } + + for i := range c.Evals { + if c.Evals[i].Name == name { + return &c.Evals[i], nil + } + } + return nil, messages.EvalNotDeclared(name, c.EvalNames()) +} + +// HasEval reports whether the named eval is declared. Unlike Eval it never +// falls back to "the only one", so callers checking for a collision cannot +// match a differently named entry. +func (c *EvalConfig) HasEval(name string) bool { + for i := range c.Evals { + if c.Evals[i].Name == name { + return true + } + } + return false +} + +// RemoveEval drops the named eval, reporting whether it was there. +func (c *EvalConfig) RemoveEval(name string) bool { + for i := range c.Evals { + if c.Evals[i].Name == name { + c.Evals = append(c.Evals[:i], c.Evals[i+1:]...) + return true + } + } + return false +} + +// DatasetDeclaration returns the catalog entry an eval's `dataset:` names. +func (c *EvalConfig) DatasetDeclaration(name string) (*DatasetDecl, bool) { + for i := range c.Datasets { + if c.Datasets[i].Name == name { + return &c.Datasets[i], true + } + } + return nil, false +} + +// EvaluatorDeclaration returns the catalog entry an evaluator reference names. +func (c *EvalConfig) EvaluatorDeclaration(name string) (*EvaluatorDecl, bool) { + for i := range c.Evaluators { + if c.Evaluators[i].Name == name { + return &c.Evaluators[i], true + } + } + return nil, false +} + +// CustomEvaluators are the catalog entries this configuration owns -- the ones +// carrying a rubric, either as a local source or written out under +// `definition`, published before the evals that name them. +func (c *EvalConfig) CustomEvaluators() []EvaluatorDecl { + var owned []EvaluatorDecl + for _, decl := range c.Evaluators { + if decl.Source == "" && decl.Definition == nil { + continue + } + owned = append(owned, decl) + } + return owned +} + +// LocalDatasets are the catalog entries carrying a file to upload. +func (c *EvalConfig) LocalDatasets() []DatasetDecl { + var owned []DatasetDecl + for _, decl := range c.Datasets { + if decl.File == "" { + continue + } + owned = append(owned, decl) + } + return owned +} + +// Validate checks the invariants the provider relies on before it calls the +// service, so failures surface as config errors rather than opaque 4xx. +func (c *EvalConfig) Validate() error { + return c.validate(true) +} + +// ValidateForLookup checks what resolving a declaration by name depends on: a +// readable set of catalogs, and a name that is present and not shared. +// +// What an eval says about itself is left to deploying it, and to the run door, +// which applies the same rules to the entry the run is actually about. Checking +// it here stranded commands that had already been told which eval they meant -- +// `run list --eval ` refused to list anything because a different entry +// was malformed, and the way out was to hand-edit a file the error did not +// mention. +// +// The catalogs stay, because they are the file's shared half: a duplicate +// dataset name makes the lookup this method exists to serve ambiguous, and no +// declaration can be read against a catalog that does not parse into one. +func (c *EvalConfig) ValidateForLookup() error { + return c.validate(false) +} + +func (c *EvalConfig) validate(deploying bool) error { + if err := c.validateCatalogs(); err != nil { + return err + } + // A catalog with no eval is what `generate` leaves behind, and it stays + // that way until `init` wires one. Refusing it on the way to a lookup + // stranded `run --eval ` in a project where `generate` ran first, over + // the absence of a declaration the id did not need. `Eval` answers for the + // case that does need one. + if deploying && len(c.Evals) == 0 { + return messages.AtLeastOneEvalRequired() + } + + seen := map[string]bool{} + substance := map[string]string{} + for i, eval := range c.Evals { + if eval.Name == "" { + return messages.EvalNameRequired(i) + } + if seen[eval.Name] { + return messages.DuplicateEvalName(i, eval.Name) + } + seen[eval.Name] = true + + if !deploying { + // Only what resolving a declaration by name depends on, which is a + // name that is present and not shared. Everything an eval says about + // itself is checked on the way to deploying it, and again at the run + // door on the entry the run is actually about. Enforcing it here + // stranded commands that had already been told which eval they + // meant: one malformed entry stopped `run list --eval ` + // listing anything, and the way out was to hand-edit a file the + // error did not mention. + continue + } + + if err := c.validateEval(i, eval); err != nil { + return err + } + + // Two evals that differ only by name are indistinguishable once + // deployed: the environment records an id against each eval's substance + // so a renamed declaration can find what it already deployed, and a + // shared substance makes that lookup ambiguous. + digest, err := FingerprintGroup(eval) + if err != nil { + return err + } + if first, clash := substance[digest]; clash { + return messages.EvalsIdenticalApartFromName(i, eval.Name, first) + } + substance[digest] = eval.Name + } + return nil +} + +func (c *EvalConfig) validateCatalogs() error { + datasets := map[string]bool{} + for i, d := range c.Datasets { + if d.Name == "" { + return messages.DatasetNameRequired(i) + } + if datasets[d.Name] { + return messages.DuplicateDatasetName(i, d.Name) + } + datasets[d.Name] = true + } + + evaluators := map[string]bool{} + for i, e := range c.Evaluators { + if e.Name == "" { + return messages.EvaluatorNameRequired(i) + } + if evaluators[e.Name] { + return messages.DuplicateEvaluatorName(i, e.Name) + } + evaluators[e.Name] = true + + if strings.HasPrefix(e.Name, evalcore.BuiltinPrefix) { + return messages.BuiltinNeedsNoCatalogEntry(i, e.Name) + } + // The service assigns an evaluator's version on publish, so a declared + // one cannot be honoured alongside a source: the upload lands on + // whatever comes next and the eval binds that, leaving the pin + // describing a version nothing uses. + if e.Source != "" && e.Version != "" { + return messages.EvaluatorVersionWithSource(i, e.Name) + } + if e.Definition != nil { + if e.Source != "" { + return messages.EvaluatorRubricDeclaredTwice(i, e.Name) + } + if e.Version != "" { + return messages.EvaluatorVersionWithDefinition(i, e.Name) + } + } + } + return nil +} + +func (c *EvalConfig) validateEval(i int, eval Eval) error { + if err := ValidateRunnable(&eval); err != nil { + return messages.InEvalAt(i, eval.Name, err) + } + if eval.Dataset != "" { + if _, ok := c.DatasetDeclaration(eval.Dataset); !ok { + return messages.DatasetNotInDatasetsCatalog(i, eval.Name, eval.Dataset) + } + } + + if len(eval.Evaluators) == 0 { + return messages.AtLeastOneEvaluatorRequired(i, eval.Name) + } + criteria := map[string]bool{} + for j, ref := range eval.Evaluators { + if ref.Evaluator == "" { + return messages.EvaluatorFieldRequired(i, j) + } + // The criterion name is what identifies a result row, so two rows that + // cannot be told apart are refused here rather than in the results. + criterion := ref.CriterionName() + if criteria[criterion] { + return messages.DuplicateCriterion(i, j, criterion) + } + criteria[criterion] = true + + if ref.IsBuiltin() { + continue + } + if _, ok := c.EvaluatorDeclaration(ref.Evaluator); !ok { + return messages.EvaluatorNotInCatalog(i, j, ref.Evaluator) + } + } + + return nil +} diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/project/ref_on_every_entry_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/project/ref_on_every_entry_test.go new file mode 100644 index 00000000000..747eac3a21d --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/internal/project/ref_on_every_entry_test.go @@ -0,0 +1,60 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package project + +import ( + "os" + "path/filepath" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// Core resolves `$ref` on every node, not just evaluator entries, so a dataset +// or an eval can be pulled in from its own file too. +// +// Only the evaluator entry modelled the directive, so a configuration that +// deployed and ran fine could not be opened by `init`, `generate` or the catalog +// writers at all: they read the file exactly as written, and the strict decoder +// refused the very key that pointed at the content. +func TestEveryEntryCoreCanSpliceCanAlsoBeEdited(t *testing.T) { + dir := t.TempDir() + require.NoError(t, os.MkdirAll(filepath.Join(dir, "parts"), 0o755)) + + require.NoError(t, os.WriteFile(filepath.Join(dir, "parts", "golden.yaml"), + // Relative to the configuration, not to this file: core rebases only the + // path keys it owns, so a path written beside this file would be resolved + // beside azure.eval.yaml and not found. + []byte("name: golden\nfile: ./datasets/golden.jsonl\n"), 0o600)) + require.NoError(t, os.WriteFile(filepath.Join(dir, "parts", "nightly.yaml"), + []byte("name: nightly\ndataset: golden\n"), 0o600)) + + path := filepath.Join(dir, EvalConfigBase) + require.NoError(t, os.WriteFile(path, []byte(` +datasets: + - $ref: ./parts/golden.yaml + +evals: + - $ref: ./parts/nightly.yaml +`), 0o600)) + + resolved, err := LoadEvalConfig(path) + require.NoError(t, err, "the resolving route has always accepted this") + require.Len(t, resolved.Datasets, 1) + assert.Equal(t, "golden", resolved.Datasets[0].Name) + assert.Equal(t, "./datasets/golden.jsonl", resolved.Datasets[0].File, + "the spliced path is resolved against the configuration, so it is written that way") + require.Len(t, resolved.Evals, 1) + assert.Equal(t, "nightly", resolved.Evals[0].Name) + + asWritten, err := OpenEvalConfigForEdit(dir) + require.NoError(t, err, + "a command that edits the file has to be able to open what azd up deploys") + require.Len(t, asWritten.Datasets, 1) + assert.Equal(t, "./parts/golden.yaml", asWritten.Datasets[0].Ref, + "the include survives an editing read, so saving writes it back") + require.Len(t, asWritten.Evals, 1) + assert.Equal(t, "./parts/nightly.yaml", asWritten.Evals[0].Ref) +} diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/project/ref_resolution_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/project/ref_resolution_test.go new file mode 100644 index 00000000000..f5f2aa43cd1 --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/internal/project/ref_resolution_test.go @@ -0,0 +1,242 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package project + +import ( + "os" + "path/filepath" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// `$ref` means the same thing to `azd up` and to the CLI commands. +// +// Core owns the resolver but does not run it for us -- it hands each extension +// the entry with `$ref` still in it. The service target called it and this path +// did not, so an include deployed fine and then failed every `azd ai eval` +// command with `unknown key "$ref"`: one file, two meanings, decided by which +// command opened it. +func TestRefResolvesOnTheCLIPathToo(t *testing.T) { + dir := t.TempDir() + + require.NoError(t, os.MkdirAll(filepath.Join(dir, "evaluators"), 0o755)) + require.NoError(t, os.WriteFile( + filepath.Join(dir, "evaluators", "quality.yaml"), + []byte("name: support-agent-quality\nsource: ./quality.json\n"), + 0o600)) + + path := filepath.Join(dir, "azure.eval.yaml") + require.NoError(t, os.WriteFile(path, []byte(` +datasets: + - name: golden + file: ./datasets/golden.jsonl + +evaluators: + - $ref: ./evaluators/quality.yaml + +evals: + - name: nightly + dataset: golden +`), 0o600)) + + cfg, err := LoadEvalConfig(path) + require.NoError(t, err, "the include the service target resolves has to resolve here too") + require.Len(t, cfg.Evaluators, 1) + assert.Equal(t, "support-agent-quality", cfg.Evaluators[0].Name, + "the referenced file's content replaces the directive") + // Verbatim, deliberately: core rebases only the two path keys it owns, so a + // relative `source:` written beside the referenced file arrives unchanged and + // is then resolved against azure.eval.yaml. That is a known limitation, not + // the behaviour this asserts -- carrying the rubric under `definition:` + // avoids it, and EvaluatorNotGeneratedYet names it when the path misses. + assert.Equal(t, "./quality.json", cfg.Evaluators[0].Source, + "a spliced path is not rebased; see the note above before changing this") +} + +// A `$ref` can name the rubric itself, not only a pointer to one. +// +// This is the shape the spec documents, and it works because resolution splices +// the referenced file's keys into the entry: they have to land on fields of the +// declaration or strict decoding rejects them. `definition` is that field. +func TestRefCanNameTheRubricItself(t *testing.T) { + dir := t.TempDir() + + require.NoError(t, os.MkdirAll(filepath.Join(dir, "evaluators"), 0o755)) + require.NoError(t, os.WriteFile( + filepath.Join(dir, "evaluators", "quality.json"), + []byte(`{"name":"support-agent-quality",`+ + `"definition":{"type":"rubric","dimensions":[{"name":"tone","weight":1}]}}`), + 0o600)) + + path := filepath.Join(dir, "azure.eval.yaml") + require.NoError(t, os.WriteFile(path, []byte(` +evaluators: + - $ref: ./evaluators/quality.json + +evals: + - name: nightly + dataset: golden +`), 0o600)) + + cfg, err := LoadEvalConfig(path) + require.NoError(t, err, "a rubric named by $ref has to decode") + require.Len(t, cfg.Evaluators, 1) + assert.Equal(t, "support-agent-quality", cfg.Evaluators[0].Name) + assert.Equal(t, "rubric", cfg.Evaluators[0].Definition["type"], + "the rubric travels with the declaration, so there is no second file to find") + assert.Empty(t, cfg.Evaluators[0].Source, + "a definition in hand is not a path to resolve against anything") +} + +// Both routes into a configuration have to agree about a `$ref`'d rubric. +// +// The deploy path resolves includes itself rather than going through +// LoadEvalConfig, so a rescue added on one side only would recreate the exact +// asymmetry that started this work -- an include `azd up` accepted and every +// CLI command refused, in mirror image. +func TestBothRoutesReadARefdRubricTheSameWay(t *testing.T) { + dir := t.TempDir() + + require.NoError(t, os.MkdirAll(filepath.Join(dir, "evaluators"), 0o755)) + require.NoError(t, os.WriteFile( + filepath.Join(dir, "evaluators", "quality.json"), + []byte(`{"type":"rubric","dimensions":[{"id":"tone","weight":3}]}`), + 0o600)) + + path := filepath.Join(dir, "azure.eval.yaml") + require.NoError(t, os.WriteFile(path, []byte(` +evaluators: + - $ref: ./evaluators/quality.json + name: quality + +evals: + - name: nightly + dataset: golden +`), 0o600)) + + fromDisk, err := LoadEvalConfig(path) + require.NoError(t, err) + + svc := serviceWith(t, map[string]any{ + "evaluators": []any{map[string]any{ + "$ref": "./evaluators/quality.json", + "name": "quality", + }}, + "evals": []any{map[string]any{"name": "nightly", "dataset": "golden"}}, + }) + fromService, err := EvalConfigFromService(svc, dir) + require.NoError(t, err, "`azd up` has to read what the CLI reads") + + assert.Equal(t, fromDisk.Evaluators, fromService.Evaluators, + "one file, one meaning, whichever command opened it") +} + +// A `$ref` can name a bare rubric file, which is the shape the spec documents +// and the shape `generate` downloads from the service. +// +// `$ref` splices the file's top-level keys into the entry, so `dimensions` and +// friends land beside `name` and used to be rejected outright. They are moved +// under `definition` instead. Wrapping the file would have been the smaller +// change and the wrong one: the tool writes that file, so the config has to +// read what the tool writes. +func TestRefCanNameABareRubricFile(t *testing.T) { + dir := t.TempDir() + + require.NoError(t, os.MkdirAll(filepath.Join(dir, "evaluators"), 0o755)) + require.NoError(t, os.WriteFile( + filepath.Join(dir, "evaluators", "support-agent-quality.json"), + []byte(`{"type":"rubric","pass_threshold":0.7,`+ + `"dimensions":[{"id":"resolves_issue","weight":9,"description":"Resolves it."}]}`), + 0o600)) + + path := filepath.Join(dir, "azure.eval.yaml") + require.NoError(t, os.WriteFile(path, []byte(` +evaluators: + - $ref: ./evaluators/support-agent-quality.json + name: support-agent-quality + +evals: + - name: nightly + dataset: golden +`), 0o600)) + + cfg, err := LoadEvalConfig(path) + require.NoError(t, err, "the spec's own example has to load") + require.Len(t, cfg.Evaluators, 1) + assert.Equal(t, "support-agent-quality", cfg.Evaluators[0].Name, + "the sibling name stays the author's, not a key from the rubric") + assert.Equal(t, "rubric", cfg.Evaluators[0].Definition["type"]) + assert.Equal(t, 0.7, cfg.Evaluators[0].Definition["pass_threshold"], + "every rubric key travels, not just the ones this decoder happens to know") + assert.Len(t, cfg.Evaluators[0].Definition["dimensions"], 1) +} + +// A configuration that uses no `$ref` is never rescued, so a misspelling in a +// hand-written entry is reported rather than filed away. +// +// The `dimensions` gate is what separates the rescue from a catch-all, and it is +// exercised in nested_ref_rubric_test.go; this pins the other half, that a +// document nobody spliced into is left strictly alone. +func TestAMisspelledEvaluatorKeyIsStillRejected(t *testing.T) { + dir := t.TempDir() + + path := filepath.Join(dir, "azure.eval.yaml") + require.NoError(t, os.WriteFile(path, []byte(` +evaluators: + - nmae: support-agent-quality + definition: + type: rubric +`), 0o600)) + + _, err := LoadEvalConfig(path) + require.Error(t, err) + assert.Contains(t, err.Error(), "nmae", "the error has to name the key that is wrong") +} + +// Sibling keys overlay the loaded file, which is what lets a name live in the +// configuration while the definition it names lives beside the code it grades. +func TestRefSiblingKeysOverlayTheLoadedFile(t *testing.T) { + dir := t.TempDir() + + require.NoError(t, os.MkdirAll(filepath.Join(dir, "evaluators"), 0o755)) + require.NoError(t, os.WriteFile( + filepath.Join(dir, "evaluators", "quality.yaml"), + []byte("name: from-the-file\nsource: ./quality.json\n"), + 0o600)) + + path := filepath.Join(dir, "azure.eval.yaml") + require.NoError(t, os.WriteFile(path, []byte(` +evaluators: + - $ref: ./evaluators/quality.yaml + name: from-the-configuration + +evals: + - name: nightly +`), 0o600)) + + cfg, err := LoadEvalConfig(path) + require.NoError(t, err) + require.Len(t, cfg.Evaluators, 1) + assert.Equal(t, "from-the-configuration", cfg.Evaluators[0].Name) +} + +// A configuration with no include is handed to the decoder untouched, so its +// diagnostics keep the line numbers of the file the author actually wrote. +func TestConfigWithoutRefIsNotRoundTripped(t *testing.T) { + dir := t.TempDir() + path := filepath.Join(dir, "azure.eval.yaml") + require.NoError(t, os.WriteFile(path, []byte(` +datasets: + - name: golden + fiel: ./datasets/golden.jsonl +`), 0o600)) + + _, err := LoadEvalConfig(path) + + require.Error(t, err) + assert.Contains(t, err.Error(), "line 4", + "a typo is reported where it was written, not where a re-marshal put it") +} diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/project/service_config_strict_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/project/service_config_strict_test.go new file mode 100644 index 00000000000..b59a80ef220 --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/internal/project/service_config_strict_test.go @@ -0,0 +1,86 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package project + +import ( + "testing" + + "github.com/azure/azure-dev/cli/azd/pkg/azdext" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "google.golang.org/protobuf/types/known/structpb" +) + +func serviceWith(t *testing.T, props map[string]any) *azdext.ServiceConfig { + t.Helper() + s, err := structpb.NewStruct(props) + require.NoError(t, err) + return &azdext.ServiceConfig{Name: "support-agent-evals", AdditionalProperties: s} +} + +// `azd up` reads the configuration through the service entry, not off disk, and +// that route used json.Unmarshal -- which drops unknown keys silently. So a +// misspelled key was named by `azd ai eval run` and ignored by `azd up`, and +// the setting the author thought they had wrote simply did not exist. +// +// Both routes now go through the same strict decoder. +func TestEvalConfigFromServiceRejectsAMistypedKey(t *testing.T) { + svc := serviceWith(t, map[string]any{ + "evals": []any{map[string]any{ + "name": "support-agent-eval", + "evaulators": []any{}, // the typo `azd ai eval run` already catches + "evaluation_level": "turn", + }}, + }) + + _, err := EvalConfigFromService(svc, "") + + require.Error(t, err, "a key this extension does not know is a typo, on either route") + assert.Contains(t, err.Error(), "evaulators") + assert.Contains(t, err.Error(), "evaluators", "the near miss is what makes it actionable") +} + +// The keys the schema does know still decode, so the strictness did not close +// the door on the authoring style it is meant to serve. +func TestEvalConfigFromServiceAcceptsADeclaredConfig(t *testing.T) { + svc := serviceWith(t, map[string]any{ + "datasets": []any{map[string]any{"name": "golden", "file": "./datasets/golden.jsonl"}}, + "evals": []any{map[string]any{ + "name": "support-agent-eval", + "dataset": "golden", + "evaluation_level": "turn", + }}, + }) + + cfg, err := EvalConfigFromService(svc, "") + + require.NoError(t, err) + require.Len(t, cfg.Evals, 1) + assert.Equal(t, "support-agent-eval", cfg.Evals[0].Name) + require.Len(t, cfg.Datasets, 1) + assert.Equal(t, "golden", cfg.Datasets[0].Name) +} + +// An include reached without a project directory is refused rather than +// discarded. +// +// Resolution was skipped when there was nowhere to resolve against, and the +// directive was then deleted so the strict decoder would not trip on it. This +// fixture shows the cost: a service mixing an include with inline content +// deployed only the inline half, and the failure surfaced as a missing eval +// rather than as the include nobody could read. +func TestARefWithoutAProjectRootIsRefused(t *testing.T) { + svc := serviceWith(t, map[string]any{ + "$ref": "./evals/azure.eval.yaml", + "evals": []any{map[string]any{ + "name": "support-agent-eval", + "evaluation_level": "turn", + }}, + }) + + _, err := EvalConfigFromService(svc, "") + + require.Error(t, err, "half a configuration is not a configuration") + assert.Contains(t, err.Error(), "$ref") +} diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/project/service_target_eval.go b/cli/azd/extensions/azure.ai.evaluations/internal/project/service_target_eval.go new file mode 100644 index 00000000000..80b7e7cf1ae --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/internal/project/service_target_eval.go @@ -0,0 +1,473 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package project + +import ( + "context" + "crypto/sha256" + "encoding/hex" + "encoding/json" + "os" + "path/filepath" + "strings" + + "azureaieval/internal/messages" + + "github.com/azure/azure-dev/cli/azd/pkg/azdext" + "go.yaml.in/yaml/v3" + "google.golang.org/protobuf/types/known/structpb" +) + +// EvalHost is the azure.yaml host this provider serves. +const EvalHost = "azure.ai.eval" + +// azd environment keys owned by this extension. +const ( + EnvKeyEvalID = "EVAL_ID" + EnvKeyDatasetVersion = "EVAL_DATASET_VERSION" + EnvKeyFingerprintPrefix = "EVAL_FINGERPRINT_" +) + +// Reconciler applies the eval configuration to the service. It is satisfied by +// the command layer, which owns the data-plane clients. +type Reconciler interface { + // EnsureDataset registers a new dataset version when the local content + // changed, returning the resolved version and whether anything was written. + EnsureDataset(ctx context.Context, decl DatasetDecl, localPath string) (version string, changed bool, err error) + // EnsureEvaluator registers a new evaluator version when the definition + // differs from what the service already holds. + EnsureEvaluator(ctx context.Context, decl EvaluatorDecl, localPath string) (version string, changed bool, err error) + // EnsureEval creates the group when it is absent or its resolved + // evaluators or options changed, returning its id. datasetPath is the local + // dataset backing the group, or empty when it is already registered; it lets + // the reconciler bind criteria to the columns that actually exist. + EnsureEval(ctx context.Context, group Eval, datasetPath string) (id string, created bool, err error) + // ReserveDeclared marks the evals these declarations already resolve to as + // spoken for, so no other declaration adopts one. Called once before + // reconciling, because adoption otherwise depends on the order the file + // lists them in. + ReserveDeclared(ctx context.Context, groups []Eval) +} + +// EvalServiceTargetProvider deploys eval resources during `azd up`. azd owns +// ordering across services through `uses:`; this provider owns only the order +// within the eval service itself. +type EvalServiceTargetProvider struct { + azdClient *azdext.AzdClient + newReconciler func(ctx context.Context) (Reconciler, error) + + serviceConfig *azdext.ServiceConfig +} + +// NewEvalServiceTargetProvider builds the provider. The reconciler is supplied +// lazily so the data-plane clients are only created when a deploy actually runs. +func NewEvalServiceTargetProvider( + azdClient *azdext.AzdClient, + newReconciler func(ctx context.Context) (Reconciler, error), +) *EvalServiceTargetProvider { + return &EvalServiceTargetProvider{azdClient: azdClient, newReconciler: newReconciler} +} + +func (p *EvalServiceTargetProvider) Initialize( + ctx context.Context, + serviceConfig *azdext.ServiceConfig, +) error { + p.serviceConfig = serviceConfig + return nil +} + +// Endpoints reports no endpoints: eval resources are not addressable. +func (p *EvalServiceTargetProvider) Endpoints( + ctx context.Context, + serviceConfig *azdext.ServiceConfig, + targetResource *azdext.TargetResource, +) ([]string, error) { + return nil, nil +} + +func (p *EvalServiceTargetProvider) GetTargetResource( + ctx context.Context, + subscriptionId string, + serviceConfig *azdext.ServiceConfig, + defaultResolver func() (*azdext.TargetResource, error), +) (*azdext.TargetResource, error) { + if defaultResolver != nil { + if target, err := defaultResolver(); err == nil { + return target, nil + } + } + // Eval resources live on the project data plane, so there is no ARM + // resource of our own to resolve. + return &azdext.TargetResource{SubscriptionId: subscriptionId}, nil +} + +// Package is a no-op: eval artifacts are plain files already on disk. +func (p *EvalServiceTargetProvider) Package( + ctx context.Context, + serviceConfig *azdext.ServiceConfig, + serviceContext *azdext.ServiceContext, + progress azdext.ProgressReporter, +) (*azdext.ServicePackageResult, error) { + return &azdext.ServicePackageResult{}, nil +} + +// Publish is a no-op: there is no artifact registry step for eval resources. +func (p *EvalServiceTargetProvider) Publish( + ctx context.Context, + serviceConfig *azdext.ServiceConfig, + serviceContext *azdext.ServiceContext, + targetResource *azdext.TargetResource, + publishOptions *azdext.PublishOptions, + progress azdext.ProgressReporter, +) (*azdext.ServicePublishResult, error) { + return &azdext.ServicePublishResult{}, nil +} + +// Deploy reconciles the eval configuration in a fixed order ΓÇö datasets, then +// evaluators, then evals ΓÇö because a group references the versions the +// first two resolve to. It fails fast; the next `azd up` resumes from wherever +// it stopped. +func (p *EvalServiceTargetProvider) Deploy( + ctx context.Context, + serviceConfig *azdext.ServiceConfig, + serviceContext *azdext.ServiceContext, + targetResource *azdext.TargetResource, + progress azdext.ProgressReporter, +) (*azdext.ServiceDeployResult, error) { + cfg, err := EvalConfigFromService(serviceConfig, p.projectRoot(ctx)) + if err != nil { + return nil, err + } + if err := cfg.Validate(); err != nil { + return nil, messages.EvalConfigInvalid(err) + } + + reconciler, err := p.newReconciler(ctx) + if err != nil { + return nil, err + } + + baseDir := p.evalBaseDir(ctx, serviceConfig) + + // 1. Datasets the configuration owns. Paths are kept so an eval that names + // one can derive its columns without reading the blob back. + // + // A declaration with no `file:` is included rather than skipped: it names + // a dataset that is already registered, and reconciling it is what confirms + // it is really there and settles which version a `version:` pin selected. + // Skipping it would leave a misspelled name to surface as a failed run. + datasetPaths := map[string]string{} + for _, decl := range cfg.Datasets { + report(progress, messages.ReconcilingDataset(decl.Name)) + localPath := ResolveSource(baseDir, decl.File) + datasetPaths[decl.Name] = localPath + version, changed, err := reconciler.EnsureDataset(ctx, decl, localPath) + if err != nil { + return nil, messages.DatasetProblem(decl.Name, err) + } + report(progress, describeResult("dataset", decl.Name, version, changed)) + } + + // 2. Evaluators this configuration owns. Built-ins and already-registered + // ones need no publish. + for _, decl := range cfg.CustomEvaluators() { + report(progress, messages.ReconcilingEvaluator(decl.Name)) + // A rubric written out in the configuration has no file to read. + localPath := "" + if decl.Source != "" { + localPath = ResolveSource(baseDir, decl.Source) + } + version, changed, err := reconciler.EnsureEvaluator(ctx, decl, localPath) + if err != nil { + return nil, messages.EvaluatorProblem(decl.Name, err) + } + report(progress, describeResult("evaluator", decl.Name, version, changed)) + } + + // 3. The evals. An eval is recreated only when its own declaration changed: + // the comparison covers what the entry declares, not what its references + // resolve to. An evaluator tracking latest that publishes a new version + // leaves every eval that runs it alone, which is what keeps a rubric edit + // comparable against the runs before it. + reconciler.ReserveDeclared(ctx, cfg.Evals) + + for i := range cfg.Evals { + eval := cfg.Evals[i] + report(progress, messages.ReconcilingEval(eval.Name)) + id, created, err := reconciler.EnsureEval(ctx, eval, datasetPaths[eval.Dataset]) + if err != nil { + return nil, messages.EvalProblem(eval.Name, err) + } + report(progress, describeEval(eval.Name, id, created)) + } + + return &azdext.ServiceDeployResult{}, nil +} + +// projectRoot is the directory `$ref` paths resolve against. It is the +// directory holding azure.yaml, which only azd can report. +func (p *EvalServiceTargetProvider) projectRoot(ctx context.Context) string { + if p.azdClient == nil { + return "" + } + resp, err := p.azdClient.Project().Get(ctx, &azdext.EmptyRequest{}) + if err != nil || resp.GetProject() == nil { + return "" + } + return resp.GetProject().GetPath() +} + +// evalBaseDir is the directory a declaration's `source:` resolves against. +// +// serviceRelativeDir answers relative to the project, because that is what the +// service's `$ref` and relativePath are written relative to. Left there it was +// resolved against this process's working directory instead, and azd neither +// changes it nor reports the project through it: azure.yaml is found by walking +// up from wherever the caller stood, and AZD_CWD carries the --cwd flag and +// nothing else. So `azd up` from any subdirectory of the project reported every +// dataset as not yet generated, and the remedy it offered would have billed a +// generation job to rewrite a file already on disk. +// +// The same join is what agent_instructions.go does with the same helper. +func (p *EvalServiceTargetProvider) evalBaseDir( + ctx context.Context, + serviceConfig *azdext.ServiceConfig, +) string { + return baseDirUnder(p.projectRoot(ctx), serviceConfig) +} + +// baseDirUnder places a service's directory under the project. +// +// azd does not re-root an absolute `$ref` or an absolute `project:`, so neither +// does this: joining one under the project produced /C:/shared/evals, +// which is the bug this fixes, reached from a different input. An empty root is +// azd having failed to name the project, where the relative path is what the +// extension did before and is better than resolving against nothing. +func baseDirUnder(projectRoot string, serviceConfig *azdext.ServiceConfig) string { + relative := serviceRelativeDir(serviceConfig) + if filepath.IsAbs(relative) || projectRoot == "" { + return relative + } + return filepath.Join(projectRoot, relative) +} + +// describeResult reports whether a version was published or reused, so a +// no-op deploy is visibly a no-op. +func describeResult(kind, name, version string, changed bool) string { + if changed { + return messages.PublishedVersion(kind, name, version) + } + return messages.UnchangedAtVersion(kind, name, version) +} + +// describeEval keeps a deploy's eval line saying the same thing the direct +// command says. Reporting the id either way left a deploy unable to answer +// whether it published anything. +func describeEval(name, id string, created bool) string { + if created { + return messages.EvalCreatedProgress(name, id) + } + return messages.EvalUnchangedProgress(name, id) +} + +func report(progress azdext.ProgressReporter, message string) { + if progress != nil { + progress(message) + } +} + +// EvalConfigFromService reads the eval configuration carried inline on the +// service entry. azd captures unknown keys into AdditionalProperties and hands +// them to the extension untouched. +// +// azd core deliberately does not resolve `$ref` includes for extensions ΓÇö it +// strips the ServiceConfig fields it owns and leaves `$ref` at the top of the +// map for the owning extension to resolve. Without this call a service written +// as `host: azure.ai.eval` + `$ref: ./evals/azure.yaml` deploys nothing at all, +// because the config parses to an empty set of datasets and groups. +func EvalConfigFromService(svc *azdext.ServiceConfig, projectRoot string) (*EvalConfig, error) { + props := serviceProps(svc) + if props == nil || len(props.GetFields()) == 0 { + return nil, messages.ServiceCarriesNoConfig(svc.GetName()) + } + + values := props.AsMap() + if projectRoot != "" { + resolved, err := resolveEvalRefs(values, projectRoot) + if err != nil { + return nil, err + } + values = resolved + } else if containsRefDirective(values) { + // Without a project root there is nothing to resolve paths against, and + // deleting the directive below would deploy a silently truncated + // configuration: an entry that mixes inline content with an include + // loses only the included half, and the failure then names a missing + // eval rather than the include nobody could read. + return nil, messages.RefNeedsAProjectRoot(svc.GetName()) + } + + // `$ref` is a directive, not configuration: ResolveFileRefs has already + // replaced it with the file's content. It only survives when resolution was + // skipped, and that config cannot deploy anyway. + delete(values, "$ref") + + // Decoded by the same strict reader the on-disk path uses, so `azd up` and + // `azd ai eval run` name a mistyped key identically instead of one + // explaining it and the other ignoring it. + raw, err := yaml.Marshal(values) + if err != nil { + return nil, messages.ReadingServiceConfig(err) + } + cfg, err := DecodeEvalConfig(raw, svc.GetName()) + if err != nil { + return nil, err + } + return cfg, nil +} + +// serviceProps prefers the inline properties, falling back to the nested +// config block. +func serviceProps(svc *azdext.ServiceConfig) *structpb.Struct { + if s := svc.GetAdditionalProperties(); s != nil && len(s.GetFields()) > 0 { + return s + } + return svc.GetConfig() +} + +// serviceRelativeDir returns the directory that `source:` paths resolve against. +// +// When the service is authored as `host:` + `$ref: ./evals/azure.yaml`, the +// paths inside that file are written relative to the file itself, so the +// include's own directory is the base. ResolveFileRefs inlines the content +// without rebasing paths, so the base has to be recovered from the `$ref` +// value before resolution. +func serviceRelativeDir(svc *azdext.ServiceConfig) string { + if svc == nil { + return "." + } + if props := serviceProps(svc); props != nil { + if ref, ok := props.AsMap()["$ref"].(string); ok && ref != "" { + if dir := filepath.Dir(filepath.FromSlash(ref)); dir != "" { + return dir + } + } + } + if p := svc.GetRelativePath(); p != "" { + return p + } + return "." +} + +// ResolveSource joins a declared source against the directory holding the +// configuration, leaving absolute paths and empty values alone. +// +// Exported because `eval create` resolves the same declarations as `azd up` +// and had grown its own copy that joined unconditionally, so an absolute +// source came out as evals/C:/data/rows.jsonl there while `azd up` handled it. +// One resolver is what stops the two drifting again. +func ResolveSource(baseDir, source string) string { + if source == "" { + return "" + } + if filepath.IsAbs(source) { + return source + } + return filepath.Join(baseDir, source) +} + +// Fingerprint hashes a local artifact so a later deploy can tell whether the +// content changed without downloading anything from the service. +// +// The dataset API returns no content hash or etag, so comparing against the +// service would mean downloading the blob on every deploy. Every artifact this +// applies to ΓÇö a dataset, a rubric, an evaluator script ΓÇö is a single file. +func Fingerprint(path string) (string, error) { + data, err := os.ReadFile(path) + if err != nil { + return "", messages.Hashing(path, err) + } + sum := sha256.Sum256(data) + return hex.EncodeToString(sum[:]), nil +} + +// FingerprintBytes hashes content that was never a file, which is how a rubric +// written in the configuration gets the change detection a rubric file has. +func FingerprintBytes(data []byte) string { + sum := sha256.Sum256(data) + return hex.EncodeToString(sum[:]) +} + +// FingerprintGroup hashes an eval's own declaration. +// +// Change detection on upstream artifacts is not sufficient: editing a group's +// evaluators, target, or options changes what the group means, and groups are +// immutable, so the group has to be recreated even when the dataset and +// evaluators are untouched. Without this a retargeted group keeps running +// against the old definition. +func FingerprintGroup(group Eval) (string, error) { + // Only substance is hashed. The id is server-assigned; name and description + // are what UpdateEvalParametersBody reaches, so an edit confined to them is + // pushed in place and must not cost the eval its id and its run history. + // Everything else ΓÇö dataset, source, evaluators, target, level ΓÇö is what + // makes this declaration the one it is. + name := group.Name + group.ID = "" + group.Name = "" + group.Description = "" + + data, err := json.Marshal(group) + if err != nil { + return "", messages.HashingEval(name, err) + } + sum := sha256.Sum256(data) + return hex.EncodeToString(sum[:]), nil +} + +// FingerprintDefinition hashes only what the service stores. +// +// max_samples and source: are applied per run, not at creation -- +// CreateOpenAIEvalRequest carries neither and buildEvalRequest reads neither. +// Recreating the eval when one of them changes points the declaration at a new +// id and leaves every run taken before it reachable only through the old one, +// for an edit the stored eval cannot even express. +// +// Kept separate from FingerprintGroup rather than folded into it, because that +// digest also answers "which eval was this declaration before it was renamed". +// Two evals over the same dataset and evaluators that differ only in their +// window are two evals, and one key for both hands the second the first one's +// id -- so the second is never created, and the first is renamed to whichever +// declaration came last. +// +// The cost is that renaming an eval and changing its window in one edit is read +// as a new eval rather than a rename. That forks a history, which is the +// conservative direction: the other way silently merges two. +func FingerprintDefinition(group Eval) (string, error) { + group.MaxSamples = 0 + group.Source = nil + return FingerprintGroup(group) +} + +// FingerprintKey is the azd environment key holding an artifact's fingerprint. +// +// The readable half is lossy: everything outside [A-Z0-9] becomes an +// underscore, so `quality-a`, `quality_a` and `quality a` all sanitize alike, +// as does any pair of names differing only outside ASCII. Two artifacts sharing +// a key overwrite each other's recorded fingerprint, version and id, which +// makes every deploy republish both. The trailing digest keeps them apart. +func FingerprintKey(kind, name string) string { + readable := strings.Map(func(r rune) rune { + switch { + case r >= 'A' && r <= 'Z', r >= '0' && r <= '9': + return r + case r >= 'a' && r <= 'z': + return r - 32 + default: + return '_' + } + }, kind+"_"+name) + + sum := sha256.Sum256([]byte(kind + "\x00" + name)) + return EnvKeyFingerprintPrefix + readable + "_" + strings.ToUpper(hex.EncodeToString(sum[:4])) +} diff --git a/cli/azd/extensions/azure.ai.evaluations/schemas/azure.ai.eval.json b/cli/azd/extensions/azure.ai.evaluations/schemas/azure.ai.eval.json new file mode 100644 index 00000000000..335182ed3f9 --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/schemas/azure.ai.eval.json @@ -0,0 +1,259 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "$id": "https://raw.githubusercontent.com/Azure/azure-dev/main/cli/azd/extensions/azure.ai.evaluations/schemas/azure.ai.eval.json", + "title": "Azure AI Foundry evaluation service", + "description": "Service-level configuration for a host: azure.ai.eval entry. The service key is the evaluation configuration name. The body is normally supplied with $ref to an evals/azure.eval.yaml holding the dataset and evaluator catalogs and every eval defined over them; relative paths inside that file resolve against the file's own directory.", + "type": "object", + "additionalProperties": true, + "properties": { + "datasets": { + "type": "array", + "description": "Catalog of named datasets. An entry with a file is uploaded on deploy; without one the name must already resolve to a registered dataset. An entry may instead be a $ref to a file holding the declaration.", + "items": { + "oneOf": [ + { "$ref": "#/definitions/DatasetDecl" }, + { "$ref": "#/definitions/FileRef" } + ] + } + }, + "evaluators": { + "type": "array", + "description": "Catalog of named custom evaluators. An entry is a declaration, or a $ref to a file holding one. Built-in evaluators are referenced straight from an eval and are never declared here.", + "items": { + "oneOf": [ + { "$ref": "#/definitions/EvaluatorDecl" }, + { "$ref": "#/definitions/FileRef" } + ] + } + }, + "evals": { + "type": "array", + "description": "The evaluations defined over the catalogs. A list, because one target is normally gated by more than one evaluation. An entry may instead be a $ref to a file holding the eval.", + "items": { + "oneOf": [ + { "$ref": "#/definitions/Eval" }, + { "$ref": "#/definitions/FileRef" } + ] + } + } + }, + "definitions": { + "FileRef": { + "type": "object", + "required": ["$ref"], + "additionalProperties": true, + "description": "Replace an inline definition with a reference to an external YAML or JSON file. Sibling properties on the same object act as overlay overrides on top of the loaded file. Cloned rather than referenced across extensions, following azure.ai.projects.", + "properties": { + "$ref": { + "type": "string", + "description": "Path to a YAML or JSON file containing the definition. Relative paths resolve from the file containing this $ref. Absolute paths are also accepted; remote URLs are not supported." + } + } + }, + "DatasetDecl": { + "type": "object", + "required": ["name"], + "additionalProperties": false, + "properties": { + "name": { + "type": "string", + "description": "Name the dataset is registered under. Letters, digits, dashes and underscores, up to 255 characters." + }, + "file": { + "type": "string", + "description": "Path to a local .jsonl whose rows are uploaded on deploy, relative to this file. Deliberately not a $ref: the rows are a data artifact to publish, not a definition to splice in, and a .jsonl is neither YAML nor JSON. Omit to use a dataset already registered under this name." + }, + "version": { + "type": "string", + "description": "Pin to an already-registered version. Omit to publish and track the newest." + } + } + }, + "EvaluatorDecl": { + "type": "object", + "required": ["name"], + "additionalProperties": false, + "properties": { + "name": { + "type": "string", + "description": "Name the evaluator is registered under. Letters, digits, dashes and underscores, up to 255 characters." + }, + "source": { + "type": "string", + "description": "Path to a local .json rubric -- a list of weighted scoring dimensions -- relative to this file. Published on deploy and fingerprinted locally so a later deploy can tell an edit here from a version published elsewhere." + }, + "definition": { + "type": "object", + "description": "The rubric written out here instead of named as a file. This is also what a $ref to a rubric file becomes once it is resolved. Its keys belong to the evaluator service, so they are not constrained here.", + "required": ["dimensions"], + "properties": { + "type": { + "type": "string", + "description": "Defaults to rubric when omitted." + }, + "dimensions": { + "type": "array", + "description": "The weighted scoring dimensions the evaluator grades on." + } + } + }, + "version": { + "type": "string", + "description": "Pin to an already-registered version. Omit to publish and track the newest." + } + }, + "allOf": [ + { + "$comment": "A rubric is named or written out, never both.", + "if": { "required": ["definition"] }, + "then": { "properties": { "source": false } } + } + ] + }, + "Eval": { + "type": "object", + "required": ["name"], + "additionalProperties": false, + "description": "One evaluation. dataset and source are alternatives for where rows come from; target is a separate axis and means what gets invoked, so an eval can read traces and invoke nothing.", + "properties": { + "name": { + "type": "string", + "description": "Name of the eval. The azd environment records the created eval id under it, which is what --eval resolves." + }, + "id": { + "type": "string", + "description": "An existing eval id, for an eval created outside this configuration." + }, + "description": { "type": "string" }, + "dataset": { + "type": "string", + "description": "Name of an entry in datasets[]. Mutually exclusive with source." + }, + "source": { + "$ref": "#/definitions/SourceDecl", + "description": "Where rows come from when they are not a catalog dataset. Mutually exclusive with dataset." + }, + "evaluation_level": { + "type": "string", + "enum": ["turn", "conversation"], + "description": "Whether each row is graded as a single turn or a whole conversation. The service default is turn." + }, + "max_samples": { + "type": "integer", + "minimum": 0, + "description": "Cap on rows graded per run." + }, + "evaluators": { + "type": "array", + "description": "The evaluators this eval grades with. Entries are either builtin. or a name declared in evaluators[].", + "items": { "$ref": "#/definitions/EvaluatorRef" } + }, + "target": { + "$ref": "#/definitions/Target", + "description": "What the run invokes to produce the responses being graded. Omit to grade the responses already present in the rows." + } + }, + "allOf": [ + { + "$comment": "Rows come from a catalog dataset or from a source, never both.", + "if": { "required": ["dataset"] }, + "then": { "properties": { "source": false } } + } + ] + }, + "SourceDecl": { + "type": "object", + "required": ["type"], + "additionalProperties": false, + "properties": { + "type": { + "type": "string", + "enum": ["traces", "responses"], + "description": "traces reads recorded spans from the project's Application Insights; responses reads named response ids." + }, + "lookback_hours": { + "type": "integer", + "minimum": 0, + "maximum": 87600, + "description": "How far back the trace window reaches from its end, capped at ten years (project.MaxLookbackHours). Zero means unset. Only for type: traces." + }, + "max_traces": { + "type": "integer", + "minimum": 0, + "description": "Cap on traces read. Only for type: traces. Removing it restores the service default." + }, + "agent_name": { + "type": "string", + "description": "Name of the agent whose traces are read. Only for type: traces." + }, + "agent_version": { + "type": "string", + "description": "Pin the traces to one agent version." + }, + "response_ids": { + "type": "array", + "items": { "type": "string" }, + "description": "Explicit response ids to grade. Only for type: responses." + }, + "max_turns": { + "type": "integer", + "minimum": 0, + "description": "Cap on turns taken from each conversation." + }, + "start_time": { + "type": "string", + "description": "Start of the window, RFC 3339. With end_time this pins an absolute window instead of a lookback." + }, + "end_time": { + "type": "string", + "description": "End of the window, RFC 3339." + } + } + }, + "EvaluatorRef": { + "type": "object", + "required": ["evaluator"], + "additionalProperties": false, + "properties": { + "evaluator": { + "type": "string", + "description": "builtin. for a built-in, or the name of an entry in evaluators[]." + }, + "name": { + "type": "string", + "description": "Label for this evaluator within the eval, when the same evaluator is used more than once." + }, + "version": { + "type": "string", + "description": "Pin this eval to one evaluator version, so a later publish does not change what it grades." + }, + "initialization_parameters": { + "type": "object", + "additionalProperties": true, + "description": "Passed to the evaluator when the run starts; model names the judge deployment." + }, + "data_mapping": { + "type": "object", + "additionalProperties": { "type": "string" }, + "description": "Binds the evaluator's required inputs to dataset columns, for inputs beyond the agent shape such as ground_truth or context." + } + } + }, + "Target": { + "type": "object", + "required": ["type", "name"], + "additionalProperties": false, + "properties": { + "type": { + "type": "string", + "enum": ["agent", "model"], + "description": "What kind of thing is invoked." + }, + "name": { + "type": "string", + "description": "Name of the agent service or the model deployment to invoke." + } + } + } + } +} From 94a04d58608c7cf51624d4787a57d7799488dcc3 Mon Sep 17 00:00:00 2001 From: mohessie Date: Fri, 21 Aug 2026 01:36:18 +0300 Subject: [PATCH 2/4] Refresh the slice with the review answered, blobs copied exactly --- .../extensions/azure.ai.evaluations/README.md | 33 +- .../internal/cmd/catalog.go | 58 +- .../internal/cmd/catalog_include_test.go | 99 +++- .../internal/cmd/eval_group.go | 351 ++++++++++++ .../internal/messages/messages.go | 35 +- .../internal/project/eval_config.go | 20 +- .../internal/project/eval_config_store.go | 513 ++++++++++++++++++ .../internal/project/one_ownership_test.go | 74 +++ .../internal/project/service_target_eval.go | 33 +- .../schemas/azure.ai.eval.json | 11 +- 10 files changed, 1158 insertions(+), 69 deletions(-) create mode 100644 cli/azd/extensions/azure.ai.evaluations/internal/cmd/eval_group.go create mode 100644 cli/azd/extensions/azure.ai.evaluations/internal/project/eval_config_store.go create mode 100644 cli/azd/extensions/azure.ai.evaluations/internal/project/one_ownership_test.go diff --git a/cli/azd/extensions/azure.ai.evaluations/README.md b/cli/azd/extensions/azure.ai.evaluations/README.md index cfd7f71ba27..a27c5be2706 100644 --- a/cli/azd/extensions/azure.ai.evaluations/README.md +++ b/cli/azd/extensions/azure.ai.evaluations/README.md @@ -1,4 +1,4 @@ -# Azure Developer CLI (azd) Evaluations Extension +# Azure Developer CLI (azd) Evaluations Extension Define Foundry evaluations alongside your agent in `azure.yaml`, deploy them with `azd up`, and run them from the terminal. @@ -50,7 +50,7 @@ evals: name: support-agent ``` -`azd up` reconciles **datasets ΓåÆ evaluators ΓåÆ eval groups**, in that order, +`azd up` reconciles **datasets → evaluators → eval groups**, in that order, because a group references the versions the first two resolve to. Relative paths inside the `$ref`'d configuration resolve against **that file's** @@ -60,7 +60,9 @@ That holds for the configuration as a whole. It does **not** hold for a `$ref` on a single catalog entry: azd rebases only the path keys it owns, so a relative `source:` written inside `evals/evaluators/quality.yaml` still resolves against `azure.eval.yaml` and will not be found. An entry pulled in from its own file -should carry the rubric under `definition:` rather than point at a second file: +should carry the rubric rather than point at a second file — either written out +under `definition:`, or as a `$ref` straight at the rubric, whose keys are +spliced in and become that `definition:`: ```yaml evaluators: @@ -68,12 +70,17 @@ evaluators: name: quality ``` +An entry declared this way is read and deployed normally, but it lives in the +referenced file, so `azd ai eval generate` will not update it in place and says +so rather than writing a second declaration of the same rubric beside the +directive. Edit the referenced file, or generate under a different name. + ### Repeated deploys do not create redundant versions Datasets are fingerprinted locally, because the dataset API exposes no content hash and comparing against the service would mean downloading the blob on every deploy. Evaluator definitions are compared against the service, but only on the -keys you authored ΓÇö the service adds `data_schema`, `init_parameters` and +keys you authored — the service adds `data_schema`, `init_parameters` and `metrics` of its own. Eval groups are immutable, so a change to a group's evaluators, target or @@ -84,11 +91,11 @@ environment so repeat runs stay comparable. | Group | Commands | |---|---| -| `azd ai eval` | `init` ┬╖ `generate` ┬╖ `run` | -| `azd ai eval dataset` | `create` ┬╖ `list` ┬╖ `show` ┬╖ `update` ┬╖ `delete` | -| `azd ai eval evaluator` | `upload` ┬╖ `list` ┬╖ `show` ┬╖ `update` ┬╖ `delete` ┬╖ `builtins` | -| `azd ai eval run` | `start` ┬╖ `list` ┬╖ `show` ┬╖ `cancel` | -| `azd ai eval results` | `show` ┬╖ `export` | +| `azd ai eval` | `init` · `generate` · `run` | +| `azd ai eval dataset` | `create` · `list` · `show` · `update` · `delete` | +| `azd ai eval evaluator` | `upload` · `list` · `show` · `update` · `delete` · `builtins` | +| `azd ai eval run` | `start` · `list` · `show` · `cancel` | +| `azd ai eval results` | `show` · `export` | `create` and `update` both publish a new immutable version; the server auto-increments and nothing mutates in place. @@ -98,7 +105,7 @@ usable from CI. ## Evaluators -Built-ins need no declaration ΓÇö reference them as `builtin.` and list +Built-ins need no declaration — reference them as `builtin.` and list them with `azd ai eval evaluator builtins`. Evaluators do not share an input contract, so the CLI reads each one's @@ -182,13 +189,13 @@ it to a dated log file rather than the terminal. Both are files the azd extensions team owns, so they are not changed here: -- [ ] **`cli/azd/extensions/registry.json`** ΓÇö add the `azure.ai.evaluations` +- [ ] **`cli/azd/extensions/registry.json`** — add the `azure.ai.evaluations` entry. Until it exists `azd extension install azure.ai.evaluations` cannot resolve, so the extension is only reachable through `azd x pack` + `azd x publish` into the local source registry. -- [ ] **`.github/CODEOWNERS`** ΓÇö add `/cli/azd/extensions/azure.ai.evaluations/`. +- [ ] **`.github/CODEOWNERS`** — add `/cli/azd/extensions/azure.ai.evaluations/`. Every sibling Foundry extension has an entry; without one, PRs here get no reviewer routing. -- [ ] **`microsoft.foundry/extension.yaml`** ΓÇö add the dependency, but only +- [ ] **`microsoft.foundry/extension.yaml`** — add the dependency, but only after the registry entry lands. Declaring a dependency that cannot resolve breaks installing the bundle. diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/catalog.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/catalog.go index c2fe04aaca5..552fa2b4af1 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/catalog.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/catalog.go @@ -1,4 +1,4 @@ -// Copyright (c) Microsoft Corporation. All rights reserved. +// Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT License. package cmd @@ -66,21 +66,29 @@ func addEvaluatorToCatalog(cmd *cobra.Command, evalDir string, ref *project.Arti }) } -// checkNameNotBehindAnInclude refuses a name whose entry lives in another file. +// checkCatalogEntryIsEditable refuses a name this command cannot rewrite in +// place without corrupting the entry. // -// Two shapes reach this. A pure `$ref` has no name here at all, so the duplicate -// scan had nothing to match on and appended a second entry with the same name -- -// a collision that surfaced only on the next resolving read. A `$ref` carrying -// an overlay `name` does match, and updating it in place writes `source:` beside -// the directive, so resolution then produces a rubric and a source and the -// configuration is rejected for declaring it twice. Neither is editable here. +// Three shapes reach this. A pure `$ref` has no name here at all, so the +// duplicate scan had nothing to match on and appended a second entry with the +// same name -- a collision that surfaced only on the next resolving read. A +// `$ref` carrying an overlay `name` does match, and updating it in place writes +// `source:` beside the directive, so resolution then produces a rubric and a +// source and the configuration is rejected for declaring it twice. An entry +// already carrying its rubric under `definition:` fails that same way with no +// include involved, because recording the generated file leaves both in one +// entry -- and it fails after the generation job has been billed and the file +// written, which is why it is refused here rather than left to the next read. // // A configuration that will not resolve is left to the commands that resolve it: // failing a generate over an unrelated broken include would be its own surprise. -func checkNameNotBehindAnInclude(evalDir string, asWritten *project.EvalConfig, kind, name string) error { - if ref, ok := catalogEntryRef(asWritten, kind, name); ok { - if ref != "" { +func checkCatalogEntryIsEditable(evalDir string, asWritten *project.EvalConfig, kind, name string) error { + if entry, ok := catalogEntryShapeOf(asWritten, kind, name); ok { + switch { + case entry.ref != "": return messages.CatalogNameBehindAnInclude(kind, name) + case entry.inlineRubric: + return messages.EvaluatorRubricWrittenInPlace(name) } return nil } @@ -88,28 +96,38 @@ func checkNameNotBehindAnInclude(evalDir string, asWritten *project.EvalConfig, if err != nil || resolved == nil { return nil } - if _, ok := catalogEntryRef(resolved, kind, name); ok { + if _, ok := catalogEntryShapeOf(resolved, kind, name); ok { return messages.CatalogNameBehindAnInclude(kind, name) } return nil } -// catalogEntryRef returns the include this entry was written as, and whether the +// catalogEntryShape is how an entry was written, for deciding whether this +// command may rewrite it. +type catalogEntryShape struct { + // ref is the `$ref` directive the entry carries, empty when it is written + // out here. + ref string + // inlineRubric is an evaluator holding its rubric under `definition:`. + inlineRubric bool +} + +// catalogEntryShapeOf returns how the entry was written, and whether the // configuration names it at all. -func catalogEntryRef(cfg *project.EvalConfig, kind, name string) (string, bool) { +func catalogEntryShapeOf(cfg *project.EvalConfig, kind, name string) (catalogEntryShape, bool) { if cfg == nil { - return "", false + return catalogEntryShape{}, false } if kind == "dataset" { if decl, ok := cfg.DatasetDeclaration(name); ok { - return decl.Ref, true + return catalogEntryShape{ref: decl.Ref}, true } - return "", false + return catalogEntryShape{}, false } if decl, ok := cfg.EvaluatorDeclaration(name); ok { - return decl.Ref, true + return catalogEntryShape{ref: decl.Ref, inlineRubric: decl.Definition != nil}, true } - return "", false + return catalogEntryShape{}, false } // updateCatalog applies a change to the configuration and writes it back. @@ -142,7 +160,7 @@ func updateCatalog( if created { cfg = &project.EvalConfig{} } - if err := checkNameNotBehindAnInclude(evalDir, cfg, kind, ref.Name); err != nil { + if err := checkCatalogEntryIsEditable(evalDir, cfg, kind, ref.Name); err != nil { return err } if !apply(cfg) { diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/catalog_include_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/catalog_include_test.go index 73426c28257..a664ffe39a6 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/catalog_include_test.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/catalog_include_test.go @@ -1,4 +1,4 @@ -// Copyright (c) Microsoft Corporation. All rights reserved. +// Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT License. package cmd @@ -32,7 +32,7 @@ evaluators: - $ref: ./parts/quality.yaml `), 0o600)) - err := checkNameNotBehindAnInclude( + err := checkCatalogEntryIsEditable( dir, mustOpenForEdit(t, dir), "evaluator", "quality") require.Error(t, err, "the name is taken, even though this file does not show it") @@ -53,7 +53,7 @@ evaluators: - $ref: ./parts/quality.yaml `), 0o600)) - require.NoError(t, checkNameNotBehindAnInclude( + require.NoError(t, checkCatalogEntryIsEditable( dir, mustOpenForEdit(t, dir), "evaluator", "tone")) } @@ -77,13 +77,104 @@ evaluators: name: quality `), 0o600)) - err := checkNameNotBehindAnInclude( + err := checkCatalogEntryIsEditable( dir, mustOpenForEdit(t, dir), "evaluator", "quality") require.Error(t, err, "the entry is an include, so it cannot be updated in place") assert.Contains(t, err.Error(), "quality") } +// The dataset branch of the guard is its own lookup, so it gets its own tests. +// +// Every case above is an evaluator, and `catalogEntryShapeOf` dispatches on kind +// before it looks anything up. A regression in the dataset branch would +// reintroduce the duplicate entries the guard exists to prevent while the +// evaluator tests stayed green. +func TestGenerateRefusesADatasetNameAnIncludeAlreadyDeclares(t *testing.T) { + dir := t.TempDir() + require.NoError(t, os.MkdirAll(filepath.Join(dir, "parts"), 0o755)) + require.NoError(t, os.WriteFile(filepath.Join(dir, "parts", "golden.yaml"), + []byte("name: golden\nfile: ./datasets/golden.jsonl\n"), 0o600)) + + require.NoError(t, os.WriteFile(filepath.Join(dir, project.EvalConfigBase), []byte(` +datasets: + - $ref: ./parts/golden.yaml +`), 0o600)) + + err := checkCatalogEntryIsEditable( + dir, mustOpenForEdit(t, dir), "dataset", "golden") + + require.Error(t, err, "the dataset name is taken by the included file") + assert.Contains(t, err.Error(), "golden") + assert.Contains(t, err.Error(), "dataset", "the message names the kind it refused") +} + +// A dataset include carrying an overlay `name`, the shape the evaluator test +// above covers, refused through the dataset branch. +func TestGenerateRefusesADatasetIncludeThatCarriesItsName(t *testing.T) { + dir := t.TempDir() + require.NoError(t, os.MkdirAll(filepath.Join(dir, "parts"), 0o755)) + require.NoError(t, os.WriteFile(filepath.Join(dir, "parts", "golden.yaml"), + []byte("file: ./datasets/golden.jsonl\n"), 0o600)) + + require.NoError(t, os.WriteFile(filepath.Join(dir, project.EvalConfigBase), []byte(` +datasets: + - $ref: ./parts/golden.yaml + name: golden +`), 0o600)) + + err := checkCatalogEntryIsEditable( + dir, mustOpenForEdit(t, dir), "dataset", "golden") + + require.Error(t, err, "the entry is an include, so it cannot be updated in place") + assert.Contains(t, err.Error(), "golden") +} + +// An evaluator already carrying its rubric under `definition:` is refused. +// +// No include is involved. Recording a generated file against it writes +// `source:` into an entry that already holds a `definition:`, and the next read +// rejects the whole configuration for declaring the rubric twice -- after the +// generation job has been billed and the file written. Refusing here is what +// keeps the failure ahead of the cost. +func TestGenerateRefusesAnEvaluatorThatAlreadyCarriesItsRubric(t *testing.T) { + dir := t.TempDir() + require.NoError(t, os.WriteFile(filepath.Join(dir, project.EvalConfigBase), []byte(` +evaluators: + - name: quality + definition: + type: rubric + dimensions: + - id: tone + weight: 3 +`), 0o600)) + + err := checkCatalogEntryIsEditable( + dir, mustOpenForEdit(t, dir), "evaluator", "quality") + + require.Error(t, err, "there is nowhere to record a file without declaring the rubric twice") + assert.Contains(t, err.Error(), "quality") + assert.Contains(t, err.Error(), "definition", "the reader has to be told which half is already there") +} + +// An entry written out here, with no include and no inline rubric, stays +// editable -- the case the guard must not catch. +func TestGenerateStillUpdatesAnEntryWrittenInThisFile(t *testing.T) { + dir := t.TempDir() + require.NoError(t, os.WriteFile(filepath.Join(dir, project.EvalConfigBase), []byte(` +datasets: + - name: golden + file: ./datasets/golden.jsonl +evaluators: + - name: quality + source: ./evaluators/quality.json +`), 0o600)) + + cfg := mustOpenForEdit(t, dir) + require.NoError(t, checkCatalogEntryIsEditable(dir, cfg, "evaluator", "quality")) + require.NoError(t, checkCatalogEntryIsEditable(dir, cfg, "dataset", "golden")) +} + func mustOpenForEdit(t *testing.T, dir string) *project.EvalConfig { t.Helper() cfg, err := project.OpenEvalConfigForEdit(dir) diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/eval_group.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/eval_group.go new file mode 100644 index 00000000000..56486220048 --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/eval_group.go @@ -0,0 +1,351 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package cmd + +import ( + "fmt" + "path/filepath" + "strings" + + "azureaieval/internal/messages" + "azureaieval/internal/pkg/eval_api" + "azureaieval/internal/project" + + "github.com/spf13/cobra" +) + +// Creation normally belongs to `azd up`, which owns reconciliation. `create` +// is the same path for a single eval outside a project, and takes the +// configuration rather than a wall of flags so there is never a second +// definition to maintain. + +// newEvalCreateCommand creates one declared eval without deploying the rest. +func newEvalCreateCommand() *cobra.Command { + var ( + fromFile string + evalDir string + endpointFlg string + ) + + cmd := &cobra.Command{ + Use: "create [name]", + Short: "Create one eval declared in the configuration.", + Long: "Create one eval declared in the configuration.\n\n" + + "`azd up` reconciles every eval in the file. This creates a single one, " + + "for a project that is not deployed as a whole — or, with --from-file, " + + "for no project at all.\n\n" + + "The name is optional while the configuration declares exactly one eval.", + Args: cobra.MaximumNArgs(1), + RunE: func(cmd *cobra.Command, args []string) error { + ctx := cmd.Context() + + path := fromFile + if path == "" { + dir, err := resolveEvalDir(ctx, evalDir) + if err != nil { + return err + } + if path, err = project.ResolveEvalConfigPath(dir); err != nil { + return err + } + } + cfg, err := project.LoadEvalConfig(path) + if err != nil { + return err + } + if err := cfg.Validate(); err != nil { + return err + } + + eval, err := cfg.Eval(chooseEval(cmd, cfg, firstArg(args))) + if err != nil { + return err + } + + ec, err := newEvalContext(ctx, endpointFlg) + if err != nil { + return err + } + defer ec.Close() + + // Local sources resolve against the file, not the working directory, + // so the columns are read from where the declaration points. + baseDir := filepath.Dir(path) + datasetPath := "" + if decl, ok := cfg.DatasetDeclaration(eval.Dataset); ok { + datasetPath = project.ResolveSource(baseDir, decl.File) + } + + reconciler := &evalReconciler{ec: ec} + // Every eval the file declares, not only the one being created: an + // eval another declaration already owns must not be adopted here. + reconciler.ReserveDeclared(ctx, cfg.Evals) + out := cmd.OutOrStdout() + + // Before anything is pushed. Publishing is not free -- a dataset + // version is immutable and the number climbs on every attempt -- so + // a declaration the evaluators cannot satisfy is refused first. + if err := checkEvaluatorRequirements(eval, ec.evaluatorSchemas(ctx)); err != nil { + return err + } + // Reported per artifact, because "publishes nothing when nothing + // changed" is the contract a reader is checking here and a single + // closing line cannot show it. Silent under -o json. + say := func(kind, name, version string, changed bool) { + if isJSON(cmd) { + return + } + if changed { + fmt.Fprintln(out, messages.PublishedVersion(kind, name, version)) + } else { + fmt.Fprintln(out, messages.UnchangedAtVersion(kind, name, version)) + } + } + + // The eval names its dataset and evaluators, and the service resolves + // those names when the eval is created, so they have to be published + // first. `azd up` reconciles the whole file; this reconciles only what + // this eval refers to, which is also what makes a rubric edit reach + // the service without a full deploy. + if decl, ok := cfg.DatasetDeclaration(eval.Dataset); ok { + version, changed, err := reconciler.EnsureDataset(ctx, *decl, datasetPath) + if err != nil { + return messages.DatasetProblem(decl.Name, err) + } + say("dataset", decl.Name, version, changed) + } + for _, ref := range eval.Evaluators { + decl, ok := cfg.EvaluatorDeclaration(ref.Evaluator) + // A built-in, or one already registered, has nothing local to publish. + if !ok || !decl.CarriesItsRubric() { + continue + } + // A rubric written out in the configuration has no file to read. + local := "" + if decl.Source != "" { + local = project.ResolveSource(baseDir, decl.Source) + } + version, changed, err := reconciler.EnsureEvaluator(ctx, *decl, local) + if err != nil { + return messages.EvaluatorProblem(decl.Name, err) + } + say("evaluator", decl.Name, version, changed) + } + + id, created, err := reconciler.EnsureEval(ctx, *eval, datasetPath) + if err != nil { + return err + } + + if isJSON(cmd) { + return emitJSON(cmd.OutOrStdout(), map[string]string{ + "id": id, "name": eval.Name, + }) + } + if created { + fmt.Fprint(out, messages.EvalCreated(eval.Name, id)) + } else { + fmt.Fprint(out, messages.EvalUnchanged(eval.Name, id)) + } + return nil + }, + } + + cmd.Flags().StringVar(&fromFile, "from-file", "", + "Read the configuration from this path instead of the eval directory.") + cmd.Flags().StringVar(&evalDir, "path", "", + "Directory holding the evaluation configuration. Defaults to the directory "+ + "`init` scaffolded, otherwise ./evals.") + cmd.Flags().StringVar(&endpointFlg, "project-endpoint", "", "Foundry project endpoint.") + return cmd +} + +func newEvalListCommand() *cobra.Command { + var ( + limit int + endpointFlg string + ) + + cmd := &cobra.Command{ + Use: "list", + Short: "List the project's evals.", + Args: cobra.NoArgs, + RunE: func(cmd *cobra.Command, args []string) error { + ctx := cmd.Context() + ec, err := newEvalContext(ctx, endpointFlg) + if err != nil { + return err + } + defer ec.Close() + + list, err := ec.evalClient.ListOpenAIEvals(ctx, limit) + if err != nil { + return messages.ListingEvals(err) + } + + if isJSON(cmd) { + return emitJSONList(cmd.OutOrStdout(), list.Data) + } + if len(list.Data) == 0 { + fmt.Fprint(cmd.OutOrStdout(), messages.NoEvals()) + return nil + } + rows := make([][]string, 0, len(list.Data)) + for _, e := range list.Data { + rows = append(rows, []string{e.ID, e.Name}) + } + return emitTable(cmd.OutOrStdout(), []string{"EVAL ID", "NAME"}, rows) + }, + } + + cmd.Flags().IntVar(&limit, "limit", 0, "Cap the number of evals returned.") + cmd.Flags().StringVar(&endpointFlg, "project-endpoint", "", "Foundry project endpoint.") + return cmd +} + +func newEvalShowCommand() *cobra.Command { + var endpointFlg string + + cmd := &cobra.Command{ + Use: "show ", + Short: "Show an eval definition.", + Args: cobra.ExactArgs(1), + RunE: func(cmd *cobra.Command, args []string) error { + evalID := args[0] + + ctx := cmd.Context() + ec, err := newEvalContext(ctx, endpointFlg) + if err != nil { + return err + } + defer ec.Close() + + group, err := ec.evalClient.GetOpenAIEval(ctx, evalID) + if err != nil && eval_api.IsNotFound(err) { + // The argument reads as an id. `list` reports names, and this + // refused the very name it points the reader at, so a name is + // resolved before giving up. + if resolved := ec.evalIDNamed(ctx, evalID); resolved != "" { + group, err = ec.evalClient.GetOpenAIEval(ctx, resolved) + } + } + if err != nil { + if eval_api.IsNotFound(err) { + return messages.EvalNotFound(evalID) + } + return messages.ReadingEval(evalID, err) + } + if isJSON(cmd) { + return emitJSON(cmd.OutOrStdout(), group) + } + detail := []field{ + {"Id", group.ID}, + {"Name", group.Name}, + // CreatedAt is `any` because the service sends epoch seconds here + // and RFC3339 elsewhere; fmt.Sprint on the former prints a float + // in scientific notation. + {"Created", timestampString(group.CreatedAt)}, + {"Created By", group.CreatedBy}, + } + // Without this the command answers "does this id exist", which is + // not what a definition is, nor what its own help promises. + if graders := evalGraders(group); graders != "" { + detail = append(detail, field{"Evaluators", graders}) + } + return emitDetail(cmd.OutOrStdout(), detail) + }, + } + + cmd.Flags().StringVar(&endpointFlg, "project-endpoint", "", "Foundry project endpoint.") + return cmd +} + +// evalGraders lists the evaluators the eval grades with, preferring the +// reference a caller would recognize over the criterion label. +// +// data_source_config is deliberately not shown beside it: every eval this +// extension creates carries type "custom", which describes the item schema +// rather than where the rows come from, so a "Source" row would read as an +// answer while always saying the same thing. +func evalGraders(group *eval_api.OpenAIEval) string { + if group == nil { + return "" + } + names := make([]string, 0, len(group.TestingCriteria)) + for _, c := range group.TestingCriteria { + name := c.EvaluatorName + if name == "" { + name = c.Name + } + if name == "" { + continue + } + if c.EvaluatorVersion != "" { + name += " (" + c.EvaluatorVersion + ")" + } + names = append(names, name) + } + return strings.Join(names, ", ") +} + +func newEvalDeleteCommand() *cobra.Command { + var endpointFlg string + + cmd := &cobra.Command{ + Use: "delete ", + Short: "Delete an eval and everything under it.", + Long: "Delete an eval and everything under it.\n\n" + + "An eval owns its runs, so deleting one discards their results too.", + Args: cobra.ExactArgs(1), + RunE: func(cmd *cobra.Command, args []string) error { + evalID := args[0] + + ctx := cmd.Context() + ec, err := newEvalContext(ctx, endpointFlg) + if err != nil { + return err + } + defer ec.Close() + + err = ec.evalClient.DeleteOpenAIEval(ctx, evalID) + if err != nil && eval_api.IsNotFound(err) { + // `list` reports names, so a name is what a reader has to hand. + // An eval is immutable, though, so editing a declaration leaves + // another under the same name, and this deletes the runs under + // whichever it picks: with more than one it asks rather than guesses. + ids, listErr := ec.evalIDsNamed(ctx, evalID) + if listErr != nil { + // Reporting the eval gone on a listing we could not + // read would be a delete silently doing nothing. + return listErr + } + switch len(ids) { + case 0: + case 1: + evalID = ids[0] + err = ec.evalClient.DeleteOpenAIEval(ctx, evalID) + default: + return messages.AmbiguousEvalName(evalID, ids) + } + } + if err != nil { + if eval_api.IsNotFound(err) { + return messages.EvalGone(evalID) + } + return messages.DeletingEval(evalID, err) + } + + if isJSON(cmd) { + return emitJSON(cmd.OutOrStdout(), map[string]string{ + "id": evalID, "status": "deleted", + }) + } + fmt.Fprint(cmd.OutOrStdout(), messages.EvalDeleted(evalID)) + return nil + }, + } + + cmd.Flags().StringVar(&endpointFlg, "project-endpoint", "", "Foundry project endpoint.") + return cmd +} diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/messages/messages.go b/cli/azd/extensions/azure.ai.evaluations/internal/messages/messages.go index 55159233d43..0704856c12d 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/messages/messages.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/messages/messages.go @@ -1,4 +1,4 @@ -// Copyright (c) Microsoft Corporation. All rights reserved. +// Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT License. // Package messages holds every string this extension shows a user. @@ -983,8 +983,8 @@ func NoDatasets() string { // NoDatasetVersions reports a name whose versions listed nothing. // -// Listing a name that does not exist is not an error ΓÇö a delete is checked for -// idempotence this way ΓÇö so this has to read as an answer about that name +// Listing a name that does not exist is not an error — a delete is checked for +// idempotence this way — so this has to read as an answer about that name // rather than as a report about the project, which holds other datasets. // // The suggested command carries no placeholder, so it pastes and runs; the file @@ -1432,15 +1432,32 @@ func RefNeedsAProjectRoot(service string) error { // CatalogNameBehindAnInclude reports a name declared through a `$ref`, which // this command cannot edit in place. +// +// One message for two shapes, so the cause is stated as what they share: the +// entry lives in the referenced file. Naming only the duplicate-on-resolve case +// would misdescribe an overlay `name`, which collides with nothing and instead +// ends up declaring the rubric twice. func CatalogNameBehindAnInclude(kind, name string) error { return fmt.Errorf( - "%s %q is already declared through a `$ref`, so this command cannot update "+ - "it here: adding a second entry would collide with the first only after "+ - "the include is resolved. Edit the referenced file, or generate under a "+ - "different name", + "%s %q is declared in a file pulled in with `$ref`, so this command cannot "+ + "update it here: an entry written beside the directive takes effect only "+ + "once the include is resolved, and not as it reads. Edit the referenced "+ + "file, or generate under a different name", kind, name) } +// EvaluatorRubricWrittenInPlace reports an evaluator whose rubric is already +// written out under `definition:`, so there is nowhere to record a generated +// file without declaring the rubric twice. +func EvaluatorRubricWrittenInPlace(name string) error { + return fmt.Errorf( + "evaluator %q already carries its rubric under `definition:`, so this "+ + "command cannot record a generated file against it: an entry holding "+ + "both a `definition:` and a `source:` is refused on the next read. Edit "+ + "the rubric in place, or generate under a different name", + name) +} + // ReadingServiceConfig reports the service entry failing to serialize. func ReadingServiceConfig(err error) error { return fmt.Errorf("reading the eval service configuration: %w", err) @@ -2173,7 +2190,7 @@ func ReadingEvalConfig(path string, err error) error { // either command. // // Still unwraps to fs.ErrNotExist, because callers that tolerate an absent -// configuration ΓÇö OpenEvalConfig, and the reference resolution above it ΓÇö decide +// configuration — OpenEvalConfig, and the reference resolution above it — decide // that by asking, and a nicer sentence that stopped answering would turn every // one of those into a failure. func noEvalConfig(path string) error { @@ -2433,7 +2450,7 @@ func ProjectContextRead(err error) error { // Progress markers from the azd style guide, so the extension's lines sit // alongside core's without a second vocabulary. const ( - doneMark = "(Γ£ô) Done:" // finished successfully + doneMark = "(✓) Done:" // finished successfully skippedMark = "(-) Skipped:" // intentionally not done, not a failure failedMark = "(x) Failed:" // the step did not complete ) diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/project/eval_config.go b/cli/azd/extensions/azure.ai.evaluations/internal/project/eval_config.go index bb8e262b7c6..ffb38bd20ca 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/project/eval_config.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/project/eval_config.go @@ -1,4 +1,4 @@ -// Copyright (c) Microsoft Corporation. All rights reserved. +// Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT License. // Package project models the eval configuration carried by the @@ -75,7 +75,7 @@ type EvaluatorDecl struct { // // Dataset and Source are alternatives: rows come from a catalog dataset, or // from a source such as production traces. Target is what gets invoked, and is -// a separate axis ΓÇö an eval can read traces and invoke nothing. +// a separate axis — an eval can read traces and invoke nothing. type Eval struct { Ref string `yaml:"$ref,omitempty" json:"$ref,omitempty"` Name string `yaml:"name,omitempty" json:"name,omitempty"` @@ -215,13 +215,27 @@ func (c *EvalConfig) EvaluatorDeclaration(name string) (*EvaluatorDecl, bool) { return nil, false } +// CarriesItsRubric reports whether this configuration owns the evaluator and +// has to publish it, rather than referring to a built-in or to one already +// registered under this name. +// +// Both fields have to be tested, and this is the only place that should test +// them. Validation forbids declaring the rubric twice, so `definition` implies +// an empty `source`: a selector written as `source == ""` reads as "nothing +// local to publish" but silently drops every evaluator carrying its rubric +// inline. That shipped once already -- the eval was created bound to an +// evaluator the service had never been told about. +func (d EvaluatorDecl) CarriesItsRubric() bool { + return d.Source != "" || d.Definition != nil +} + // CustomEvaluators are the catalog entries this configuration owns -- the ones // carrying a rubric, either as a local source or written out under // `definition`, published before the evals that name them. func (c *EvalConfig) CustomEvaluators() []EvaluatorDecl { var owned []EvaluatorDecl for _, decl := range c.Evaluators { - if decl.Source == "" && decl.Definition == nil { + if !decl.CarriesItsRubric() { continue } owned = append(owned, decl) diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/project/eval_config_store.go b/cli/azd/extensions/azure.ai.evaluations/internal/project/eval_config_store.go new file mode 100644 index 00000000000..823729e2b96 --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/internal/project/eval_config_store.go @@ -0,0 +1,513 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package project + +import ( + "bytes" + "errors" + "io" + "io/fs" + "os" + "path/filepath" + "runtime" + "slices" + "syscall" + "time" + + "azureaieval/internal/messages" + + "github.com/azure/azure-dev/cli/azd/pkg/foundry" + "go.yaml.in/yaml/v3" +) + +// This file is the only place that knows how the configuration is stored: the +// directory it lives in, what the file is called, and how it is parsed and +// serialized. Everything else works with *EvalConfig, so changing the on-disk +// shape stays a local edit. + +// DefaultEvalDir is where init writes the configuration and its artifacts. +const DefaultEvalDir = "evals" + +// EvalConfigBase is the single configuration file inside that directory. +// +// Prefixed for azd, the way azure.yaml is: eval.yaml is generic enough to +// collide with an unrelated tool's file in the same folder, and the prefix says +// whose it is. +const EvalConfigBase = "azure.eval.yaml" + +// LegacyEvalConfigBase is what the file was called before it was named for azd. +// Read, never written: a project that already has one keeps working, and does +// not silently grow a second configuration beside it. +const LegacyEvalConfigBase = "eval.yaml" + +// EvalConfigPath is the configuration file at a location. It is exported for +// error messages and for the azure.yaml $ref; readers should prefer +// OpenEvalConfig. +// +// A location is normally the eval directory, and the file inside it is named by +// convention. It may also be the file itself, because azure.yaml's `$ref` names +// one by name rather than by directory: a project is free to declare +// `./config/nightly.yaml`, and looking for `azure.eval.yaml` beside it would +// report the configuration missing while `azd up` deployed it. +func EvalConfigPath(location string) string { + if namesAFile(location) { + return location + } + return filepath.Join(location, EvalConfigBase) +} + +// EvalDirOf is the directory a location's relative paths resolve against. +func EvalDirOf(location string) string { + if namesAFile(location) { + return filepath.Dir(location) + } + return location +} + +// namesAFile reports whether a location is the configuration file rather than +// the directory holding it. A path that does not exist is read as a directory, +// which is what `init` is given before it writes anything. +func namesAFile(location string) bool { + info, err := os.Stat(location) + return err == nil && !info.IsDir() +} + +// ResolveEvalConfigPath is the configuration this location actually holds: +// the current name, or the legacy one when that is the only file there. +// +// It refuses a directory holding both, rather than leaving that to the caller. +// The rule used to live in OpenEvalConfig alone, so `eval create` -- which +// needs the path rather than the parsed configuration -- resolved one silently +// while `run`, `init` and `generate` all refused. Returning an error is what +// makes the guard unavoidable: there is no longer a way to ask this question +// and not be told. +func ResolveEvalConfigPath(location string) (string, error) { + if err := checkOneConfig(location); err != nil { + return "", err + } + return resolvedConfigPath(location), nil +} + +// resolvedConfigPath is the naming rule on its own, for the two functions that +// have already applied the guard. +func resolvedConfigPath(location string) string { + if namesAFile(location) { + return location + } + current := EvalConfigPath(location) + if _, err := os.Stat(current); err == nil { + return current + } + legacy := filepath.Join(location, LegacyEvalConfigBase) + if _, err := os.Stat(legacy); err == nil { + return legacy + } + return current +} + +// checkOneConfig refuses a directory holding both names. +// +// Preferring one silently is the dangerous answer: `azure.yaml` `$ref`s a +// single file by name, so the CLI would edit one configuration while `azd up` +// deployed the other, and nothing would say so. A location that already names +// the file has nothing to disambiguate. +func checkOneConfig(location string) error { + if namesAFile(location) { + return nil + } + current := EvalConfigPath(location) + legacy := filepath.Join(location, LegacyEvalConfigBase) + if _, err := os.Stat(current); err != nil { + return nil + } + if _, err := os.Stat(legacy); err != nil { + return nil + } + return messages.AmbiguousEvalConfig(current, legacy) +} + +// OpenEvalConfig reads the configuration at a location, with `$ref` includes +// resolved. This is the reader for commands that *use* the configuration. +// +// A missing file returns (nil, nil): generate runs before init, so "no +// configuration yet" is an ordinary state rather than a failure. +// +// Commands that write the configuration back must use OpenEvalConfigForEdit +// instead. Resolution and editing do not mix: what comes back here is the +// configuration with every include spliced in, and saving that replaces the +// author's `$ref` with its content. +func OpenEvalConfig(location string) (*EvalConfig, error) { + return openEvalConfig(location, true) +} + +// OpenEvalConfigForEdit reads the configuration exactly as written, leaving +// `$ref` directives alone. +// +// `init` and `generate` read, modify and write the same file. Handing them a +// resolved configuration and saving the result inlined the author's includes, +// orphaned the files they named, and left the paths inside those files +// resolving against the wrong directory -- a `source: ./quality.json` written +// beside `evaluators/quality.yaml` came back pointing at the project root. +// None of it was reported, because from the writer's point of view it had +// simply saved what it read. +func OpenEvalConfigForEdit(location string) (*EvalConfig, error) { + return openEvalConfig(location, false) +} + +func openEvalConfig(location string, resolve bool) (*EvalConfig, error) { + if err := checkOneConfig(location); err != nil { + return nil, err + } + cfg, err := loadEvalConfig(resolvedConfigPath(location), resolve) + if errors.Is(err, fs.ErrNotExist) { + return nil, nil + } + return cfg, err +} + +// LoadEvalConfig reads a configuration from an explicit path, with `$ref` +// includes resolved. The path is used verbatim, relative to the process working +// directory — never re-rooted. +// +// Decoded strictly: a key this extension does not know is a typo, and reading +// it as nothing leaves a configuration that looks fine and fails later +// somewhere else. `agent:` written under `target:` instead of `type:`/`name:` +// used to produce an empty target and a run that complained about the target. +func LoadEvalConfig(path string) (*EvalConfig, error) { + return loadEvalConfig(path, true) +} + +func loadEvalConfig(path string, resolve bool) (*EvalConfig, error) { + data, err := ReadFileNoBOM(path) + if err != nil { + return nil, messages.ReadingEvalConfig(path, err) + } + if resolve { + data, err = resolveConfigRefs(data, filepath.Dir(path), path) + if err != nil { + return nil, err + } + } + return DecodeEvalConfig(data, path) +} + +// resolveConfigRefs expands `$ref` includes before the strict decode. +// +// Core owns the resolver but does not run it for us: it hands each extension +// the entry with `$ref` still in it. The service target has always called it, +// and this path did not, so `azd up` accepted an include that every CLI command +// then refused as an unknown key — the same file meaning two different things +// depending on which command opened it. +// +// A configuration with no `$ref` is returned untouched rather than round-tripped +// through a map, so the overwhelmingly common case keeps the decoder's own line +// numbers in its diagnostics. +func resolveConfigRefs(data []byte, baseDir, name string) ([]byte, error) { + if !bytes.Contains(data, []byte("$ref")) { + return data, nil + } + + var raw map[string]any + if err := yaml.Unmarshal(data, &raw); err != nil { + return nil, messages.ParsingEvalConfig(name, err) + } + if raw == nil { + return data, nil + } + + resolved, err := resolveEvalRefs(raw, baseDir) + if err != nil { + return nil, err + } + + out, err := yaml.Marshal(resolved) + if err != nil { + return nil, messages.ParsingEvalConfig(name, err) + } + return out, nil +} + +// resolveEvalRefs is the one place `$ref` is resolved, so the CLI and `azd up` +// cannot disagree about what an include means. +// +// They read the configuration by different routes -- off disk, and out of the +// service entry -- and each used to resolve for itself. Every rule then had to +// be added twice, and twice it was not: an include `azd up` accepted and every +// CLI command refused, and later the reverse. Callers differ in how they obtain +// the map and what they do with it; everything between is here. +func resolveEvalRefs(values map[string]any, baseDir string) (map[string]any, error) { + // Read before resolution, which removes the directive. Both routes gate the + // rescue on it so they cannot disagree: without it, the CLI's no-`$ref` fast + // path would skip nesting while the deploy path still applied it, and a + // hand-written entry carrying rubric keys would deploy and then be refused + // by every command that reads it. + spliced := containsRefDirective(values) + + resolved, err := foundry.ResolveFileRefs(values, baseDir) + if err != nil { + return nil, messages.ResolvingServiceRefs(err) + } + // `$ref` is a directive rather than configuration, and the strict decoder + // would report the leftover as a mistyped key. + delete(resolved, "$ref") + if spliced { + nestSplicedRubrics(resolved) + } + return resolved, nil +} + +// containsRefDirective reports whether the document uses `$ref` anywhere. +// +// Structural rather than a text scan: the byte "$ref" also appears in comments +// and in prose values, and letting those decide whether an unrelated entry is +// rescued would make one entry's meaning depend on another's wording. +func containsRefDirective(value any) bool { + switch typed := value.(type) { + case map[string]any: + if _, ok := typed[refDirective]; ok { + return true + } + for _, child := range typed { + if containsRefDirective(child) { + return true + } + } + case []any: + return slices.ContainsFunc(typed, containsRefDirective) + } + return false +} + +// refDirective is the include key azd core owns. +const refDirective = "$ref" + +// evaluatorDeclKeys are the keys an evaluator entry declares in its own right. +// Anything else at entry level was spliced in by a `$ref`. +var evaluatorDeclKeys = map[string]bool{ + "$ref": true, "name": true, "source": true, "version": true, "definition": true, +} + +// nestSplicedRubrics moves a rubric that a `$ref` spliced in at entry level +// down under `definition`. +// +// `$ref` splices the referenced file's top-level keys into the entry, and a +// rubric file is a bare `{type, dimensions}` -- the shape `generate` downloads +// from the service -- so its keys land beside `name` and the strict decoder +// rejects them. Moving them is what lets a `$ref` name a rubric. +// +// `dimensions` is what marks the leftovers as a rubric rather than a typo, and +// it is the same key normalizeRubricBody insists on before it will treat a +// document as a definition. Without that gate this would be a catch-all by +// another name, filing a misspelled `name` as rubric content and publishing it +// to the service instead of reporting it. +// +// Structural rather than positional on purpose: an earlier version marked +// entries by index before resolution, which cannot see the evaluators inside a +// config that is itself behind a `$ref` -- the layout the README documents. +func nestSplicedRubrics(resolved map[string]any) { + entries, _ := resolved["evaluators"].([]any) + for _, entry := range entries { + m, ok := entry.(map[string]any) + if !ok { + continue + } + // A file already shaped `{name, definition}` needs no rescue, and + // merging into it would guess at which one the author meant. + if _, has := m["definition"]; has { + continue + } + if _, isRubric := m["dimensions"]; !isRubric { + continue + } + rubric := map[string]any{} + for key, value := range m { + if !evaluatorDeclKeys[key] { + rubric[key] = value + delete(m, key) + } + } + m["definition"] = rubric + } +} + +// DecodeEvalConfig is the one strict decoder, so every route into a +// configuration reports a mistyped key the same way. +// +// `azd up` reads the configuration through the service entry rather than off +// disk, and that route used json.Unmarshal, which drops unknown keys in +// silence. The same typo was therefore named by `azd ai eval run` and ignored +// by `azd up`. The name is what the diagnostic points at. +func DecodeEvalConfig(data []byte, name string) (*EvalConfig, error) { + decoder := yaml.NewDecoder(bytes.NewReader(data)) + decoder.KnownFields(true) + + var cfg EvalConfig + if err := decoder.Decode(&cfg); err != nil { + // An empty file is a configuration with nothing in it, not a parse + // failure: `generate` writes one before it has anything to record. + if errors.Is(err, io.EOF) { + return &cfg, nil + } + return nil, messages.ParsingEvalConfig(name, explainUnknownKeys(err)) + } + return &cfg, nil +} + +// SaveEvalConfig writes cfg as the configuration under evalDir, creating the +// directory when it does not exist yet. +// +// Writes back over a legacy eval.yaml when that is the file the project has, so +// a generate into an existing project updates the configuration it already +// references rather than leaving an inert second one beside it. +func SaveEvalConfig(evalDir string, cfg *EvalConfig) error { + if err := checkOneConfig(evalDir); err != nil { + return err + } + if err := os.MkdirAll(evalDir, 0o750); err != nil { + return messages.Creating(evalDir, err) + } + return SaveEvalConfigTo(resolvedConfigPath(evalDir), cfg) +} + +// SaveEvalConfigTo writes cfg over an explicit path, for callers that already +// resolved one. +// +// The replacement is atomic because os.WriteFile truncates first, and this file +// is read by other processes. A reader landing inside that window sees zero +// bytes, and a zero-byte config parses as a valid empty one rather than as an +// error, so it would go on to write back a configuration with every eval +// missing. Renaming into place means a reader sees either the whole old file or +// the whole new one. +func SaveEvalConfigTo(path string, cfg *EvalConfig) error { + body, err := yaml.Marshal(cfg) + if err != nil { + return messages.SerializingEvalConfig(err) + } + + dir := filepath.Dir(path) + tmp, err := os.CreateTemp(dir, ".azd-eval-config-*") + if err != nil { + return messages.WritingEvalConfig(path, err) + } + tmpName := tmp.Name() + defer os.Remove(tmpName) + + if _, err := tmp.Write(body); err != nil { + _ = tmp.Close() + return messages.WritingEvalConfig(path, err) + } + if err := tmp.Chmod(0o600); err != nil { + _ = tmp.Close() + return messages.WritingEvalConfig(path, err) + } + if err := tmp.Close(); err != nil { + return messages.WritingEvalConfig(path, err) + } + // Straight over the destination, and never by unlinking it first. Windows + // refuses a rename while a reader holds the destination open, so the + // obvious fallback -- remove, then rename -- turns a collision into a + // window where the config does not exist, and OpenEvalConfig reports a + // missing file as "no configuration yet", which callers answer by writing a + // fresh one. That is the same data loss this function exists to prevent. + // Contention is measured in microseconds, so it is waited out instead. + if err := ReplaceFile(tmpName, path); err != nil { + return messages.WritingEvalConfig(path, err) + } + return nil +} + +// ReplaceFile moves a freshly written temporary file over a destination. +// +// Never by unlinking the destination first, which is the obvious shape and is +// wrong twice over. Windows refuses a rename while a reader holds the +// destination open, so remove-then-rename turns a collision into a window where +// the file does not exist -- and a config that momentarily does not exist reads +// as "no configuration yet", which callers answer by writing a fresh empty one. +// Contention is measured in microseconds, so it is waited out instead. +// +// The unlink was doing one thing worth keeping: os.Remove clears a read-only +// attribute and retries, so a file marked read-only (a Perforce or TFVC +// checkout, `attrib +R`, some archive extractions) could still be replaced. +// Windows reports a rename onto a read-only destination with the same errno as +// one a reader holds open, so the two cannot be told apart before the wait. +func ReplaceFile(from, to string) error { + err := renameOverContention(from, to) + if err == nil { + return nil + } + if !clearReadOnly(to) { + return err + } + return os.Rename(from, to) +} + +// clearReadOnly drops a read-only attribute, reporting whether it had one to +// drop. os.Chmod is what carries FILE_ATTRIBUTE_READONLY on Windows. +func clearReadOnly(path string) bool { + info, err := os.Stat(path) + if err != nil || info.Mode().Perm()&0o200 != 0 { + return false + } + return os.Chmod(path, info.Mode().Perm()|0o200) == nil +} + +// The budgets are deliberately different. A replacement window is measured in +// microseconds, so neither needs to be generous -- and every millisecond here +// is also charged to a file that is genuinely unreadable, because Windows +// reports "someone has this open" and "you may not have this" as one errno. +const ( + renameRetryBudget = 500 * time.Millisecond + readRetryBudget = 250 * time.Millisecond +) + +func renameOverContention(from, to string) error { + deadline := time.Now().Add(renameRetryBudget) + delay := time.Millisecond + for { + err := os.Rename(from, to) + if err == nil || !isSharingContention(err) || time.Now().After(deadline) { + return err + } + time.Sleep(delay) + if delay < 16*time.Millisecond { + delay *= 2 + } + } +} + +// isSharingContention reports the errors Windows raises while another handle is +// open. It cannot be precise: renaming onto a destination a reader holds open +// and renaming onto one the caller may not touch both report ERROR_ACCESS_DENIED, +// so a genuine permission failure is waited on before it is reported. The +// budget is what keeps that wait short enough to be worth the trade. +func isSharingContention(err error) bool { + if err == nil || errors.Is(err, os.ErrNotExist) { + return false + } + if runtime.GOOS != "windows" { + return false + } + var errno syscall.Errno + if !errors.As(err, &errno) { + return false + } + // ERROR_ACCESS_DENIED and ERROR_SHARING_VIOLATION. + return errno == 5 || errno == 32 +} + +// readFileOverContention reads a file that another process may be replacing. +func readFileOverContention(path string) ([]byte, error) { + deadline := time.Now().Add(readRetryBudget) + delay := time.Millisecond + for { + body, err := os.ReadFile(path) + if err == nil || !isSharingContention(err) || time.Now().After(deadline) { + return body, err + } + time.Sleep(delay) + if delay < 16*time.Millisecond { + delay *= 2 + } + } +} diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/project/one_ownership_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/project/one_ownership_test.go new file mode 100644 index 00000000000..7657a5f34b0 --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/internal/project/one_ownership_test.go @@ -0,0 +1,74 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package project + +import ( + "os" + "path/filepath" + "strings" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// Whether this configuration owns an evaluator is decided in exactly one place. +// +// Two publish loops ask it -- `azd up` through CustomEvaluators, and `eval +// create` over the evaluators one eval names. Each carried its own copy of the +// test, and the first version of both read `source == ""`. That is wrong in a +// way nothing surfaces: validation forbids declaring the rubric twice, so an +// evaluator carrying its rubric under `definition:` has an empty `source`, was +// read as "nothing local to publish", and was skipped. The eval was then created +// bound to an evaluator the service had never been told about. +// +// A second copy of the predicate is how that returns, and it returns quietly -- +// the config still decodes, the deploy still reports success. So the shape is +// worth failing the build over rather than trusting a reviewer to spot it. +func TestEvaluatorOwnershipIsDecidedInOnePlace(t *testing.T) { + const predicate = "CarriesItsRubric" + + // Either half of the pair, written inline. `Definition == nil` on its own is + // legitimate -- the reconciler branches on it to choose what to publish -- + // so it is the pairing with a `Source` test that means someone has + // re-derived ownership. + sightings := map[string][]int{} + require.NoError(t, filepath.WalkDir("../..", func(path string, d os.DirEntry, err error) error { + if err != nil || d.IsDir() || !strings.HasSuffix(path, ".go") { + return err + } + if strings.HasSuffix(path, "_test.go") { + return nil // including this file, which spells out the shape it is looking for + } + body, err := os.ReadFile(path) + if err != nil { + return err + } + insidePredicate := false + for i, line := range strings.Split(string(body), "\n") { + // The predicate is the one place allowed to say this, so skip its body. + if strings.Contains(line, predicate+"() bool {") { + insidePredicate = true + continue + } + if insidePredicate { + if strings.HasPrefix(line, "}") { + insidePredicate = false + } + continue + } + source := strings.Contains(line, `Source == ""`) || strings.Contains(line, `Source != ""`) + definition := strings.Contains(line, "Definition == nil") || strings.Contains(line, "Definition != nil") + if source && definition { + sightings[path] = append(sightings[path], i+1) + } + } + return nil + })) + + assert.Empty(t, sightings, + "ownership is re-derived at %v; call %s instead, or the next rule about "+ + "what this configuration publishes will land on one loop and not the other", + sightings, predicate) +} diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/project/service_target_eval.go b/cli/azd/extensions/azure.ai.evaluations/internal/project/service_target_eval.go index 80b7e7cf1ae..f4812538cdd 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/project/service_target_eval.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/project/service_target_eval.go @@ -1,4 +1,4 @@ -// Copyright (c) Microsoft Corporation. All rights reserved. +// Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT License. package project @@ -124,8 +124,8 @@ func (p *EvalServiceTargetProvider) Publish( return &azdext.ServicePublishResult{}, nil } -// Deploy reconciles the eval configuration in a fixed order ΓÇö datasets, then -// evaluators, then evals ΓÇö because a group references the versions the +// Deploy reconciles the eval configuration in a fixed order — datasets, then +// evaluators, then evals — because a group references the versions the // first two resolve to. It fails fast; the next `azd up` resumes from wherever // it stopped. func (p *EvalServiceTargetProvider) Deploy( @@ -135,7 +135,13 @@ func (p *EvalServiceTargetProvider) Deploy( targetResource *azdext.TargetResource, progress azdext.ProgressReporter, ) (*azdext.ServiceDeployResult, error) { - cfg, err := EvalConfigFromService(serviceConfig, p.projectRoot(ctx)) + // Asked once and reused: a second call could fail where the first + // succeeded, and the empty root that comes back is indistinguishable from a + // project that has none. The include guard below would have passed while + // artifact paths quietly resolved against this process's directory instead. + projectRoot := p.projectRoot(ctx) + + cfg, err := EvalConfigFromService(serviceConfig, projectRoot) if err != nil { return nil, err } @@ -148,7 +154,7 @@ func (p *EvalServiceTargetProvider) Deploy( return nil, err } - baseDir := p.evalBaseDir(ctx, serviceConfig) + baseDir := baseDirUnder(projectRoot, serviceConfig) // 1. Datasets the configuration owns. Paths are kept so an eval that names // one can derive its columns without reading the blob back. @@ -218,7 +224,8 @@ func (p *EvalServiceTargetProvider) projectRoot(ctx context.Context) string { return resp.GetProject().GetPath() } -// evalBaseDir is the directory a declaration's `source:` resolves against. +// baseDirUnder places a service's directory under the project -- the directory +// a declaration's `source:` resolves against. // // serviceRelativeDir answers relative to the project, because that is what the // service's `$ref` and relativePath are written relative to. Left there it was @@ -230,14 +237,6 @@ func (p *EvalServiceTargetProvider) projectRoot(ctx context.Context) string { // generation job to rewrite a file already on disk. // // The same join is what agent_instructions.go does with the same helper. -func (p *EvalServiceTargetProvider) evalBaseDir( - ctx context.Context, - serviceConfig *azdext.ServiceConfig, -) string { - return baseDirUnder(p.projectRoot(ctx), serviceConfig) -} - -// baseDirUnder places a service's directory under the project. // // azd does not re-root an absolute `$ref` or an absolute `project:`, so neither // does this: joining one under the project produced /C:/shared/evals, @@ -281,7 +280,7 @@ func report(progress azdext.ProgressReporter, message string) { // service entry. azd captures unknown keys into AdditionalProperties and hands // them to the extension untouched. // -// azd core deliberately does not resolve `$ref` includes for extensions ΓÇö it +// azd core deliberately does not resolve `$ref` includes for extensions — it // strips the ServiceConfig fields it owns and leaves `$ref` at the top of the // map for the owning extension to resolve. Without this call a service written // as `host: azure.ai.eval` + `$ref: ./evals/azure.yaml` deploys nothing at all, @@ -382,7 +381,7 @@ func ResolveSource(baseDir, source string) string { // // The dataset API returns no content hash or etag, so comparing against the // service would mean downloading the blob on every deploy. Every artifact this -// applies to ΓÇö a dataset, a rubric, an evaluator script ΓÇö is a single file. +// applies to — a dataset, a rubric, an evaluator script — is a single file. func Fingerprint(path string) (string, error) { data, err := os.ReadFile(path) if err != nil { @@ -410,7 +409,7 @@ func FingerprintGroup(group Eval) (string, error) { // Only substance is hashed. The id is server-assigned; name and description // are what UpdateEvalParametersBody reaches, so an edit confined to them is // pushed in place and must not cost the eval its id and its run history. - // Everything else ΓÇö dataset, source, evaluators, target, level ΓÇö is what + // Everything else — dataset, source, evaluators, target, level — is what // makes this declaration the one it is. name := group.Name group.ID = "" diff --git a/cli/azd/extensions/azure.ai.evaluations/schemas/azure.ai.eval.json b/cli/azd/extensions/azure.ai.evaluations/schemas/azure.ai.eval.json index 335182ed3f9..f980d734e7d 100644 --- a/cli/azd/extensions/azure.ai.evaluations/schemas/azure.ai.eval.json +++ b/cli/azd/extensions/azure.ai.evaluations/schemas/azure.ai.eval.json @@ -1,4 +1,4 @@ -{ +{ "$schema": "http://json-schema.org/draft-07/schema#", "$id": "https://raw.githubusercontent.com/Azure/azure-dev/main/cli/azd/extensions/azure.ai.evaluations/schemas/azure.ai.eval.json", "title": "Azure AI Foundry evaluation service", @@ -61,7 +61,7 @@ }, "file": { "type": "string", - "description": "Path to a local .jsonl whose rows are uploaded on deploy, relative to this file. Deliberately not a $ref: the rows are a data artifact to publish, not a definition to splice in, and a .jsonl is neither YAML nor JSON. Omit to use a dataset already registered under this name." + "description": "Path to a local .jsonl whose rows are uploaded on deploy, resolved against the evaluation configuration -- also when this entry arrived through a $ref, because a spliced path is not rebased to the file it was written in. Deliberately not a $ref: the rows are a data artifact to publish, not a definition to splice in, and a .jsonl is neither YAML nor JSON. Omit to use a dataset already registered under this name." }, "version": { "type": "string", @@ -80,7 +80,7 @@ }, "source": { "type": "string", - "description": "Path to a local .json rubric -- a list of weighted scoring dimensions -- relative to this file. Published on deploy and fingerprinted locally so a later deploy can tell an edit here from a version published elsewhere." + "description": "Path to a local .json rubric -- a list of weighted scoring dimensions -- resolved against the evaluation configuration, also when this entry arrived through a $ref, because a spliced path is not rebased; an entry pulled in from its own file should carry the rubric under definition instead. Published on deploy and fingerprinted locally so a later deploy can tell an edit here from a version published elsewhere." }, "definition": { "type": "object", @@ -107,6 +107,11 @@ "$comment": "A rubric is named or written out, never both.", "if": { "required": ["definition"] }, "then": { "properties": { "source": false } } + }, + { + "$comment": "A version pins one already registered, so there is nothing local to publish alongside it. Both shapes are refused when the configuration is read; stating them here is what makes the editor agree.", + "if": { "required": ["version"] }, + "then": { "properties": { "source": false, "definition": false } } } ] }, From e1364f2b54992bf073037a8e7e12f89142e1695e Mon Sep 17 00:00:00 2001 From: mohessie Date: Fri, 21 Aug 2026 01:38:22 +0300 Subject: [PATCH 3/4] Copy the remaining slice files exactly --- .../azure.ai.evaluations/internal/project/config_keys_test.go | 2 +- .../internal/project/ref_on_every_entry_test.go | 2 +- .../internal/project/ref_resolution_test.go | 2 +- .../internal/project/service_config_strict_test.go | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/project/config_keys_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/project/config_keys_test.go index a5b60cb4763..f109334026a 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/project/config_keys_test.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/project/config_keys_test.go @@ -1,4 +1,4 @@ -// Copyright (c) Microsoft Corporation. All rights reserved. +// Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT License. package project diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/project/ref_on_every_entry_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/project/ref_on_every_entry_test.go index 747eac3a21d..648378fba83 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/project/ref_on_every_entry_test.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/project/ref_on_every_entry_test.go @@ -1,4 +1,4 @@ -// Copyright (c) Microsoft Corporation. All rights reserved. +// Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT License. package project diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/project/ref_resolution_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/project/ref_resolution_test.go index f5f2aa43cd1..7e6a2b9a9f9 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/project/ref_resolution_test.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/project/ref_resolution_test.go @@ -1,4 +1,4 @@ -// Copyright (c) Microsoft Corporation. All rights reserved. +// Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT License. package project diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/project/service_config_strict_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/project/service_config_strict_test.go index b59a80ef220..ad54453ef9a 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/project/service_config_strict_test.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/project/service_config_strict_test.go @@ -1,4 +1,4 @@ -// Copyright (c) Microsoft Corporation. All rights reserved. +// Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT License. package project From a18fd7785230ad042a4814bf35c81d58d3bfa822 Mon Sep 17 00:00:00 2001 From: mohessie Date: Fri, 21 Aug 2026 02:08:12 +0300 Subject: [PATCH 4/4] Gate the rubric rescue on the entry's own include --- .../internal/project/eval_config_store.go | 71 ++++++++++++++----- .../internal/project/neighbour_ref_test.go | 66 +++++++++++++++++ 2 files changed, 118 insertions(+), 19 deletions(-) create mode 100644 cli/azd/extensions/azure.ai.evaluations/internal/project/neighbour_ref_test.go diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/project/eval_config_store.go b/cli/azd/extensions/azure.ai.evaluations/internal/project/eval_config_store.go index 823729e2b96..cdd60f1a47c 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/project/eval_config_store.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/project/eval_config_store.go @@ -237,12 +237,12 @@ func resolveConfigRefs(data []byte, baseDir, name string) ([]byte, error) { // CLI command refused, and later the reverse. Callers differ in how they obtain // the map and what they do with it; everything between is here. func resolveEvalRefs(values map[string]any, baseDir string) (map[string]any, error) { - // Read before resolution, which removes the directive. Both routes gate the - // rescue on it so they cannot disagree: without it, the CLI's no-`$ref` fast - // path would skip nesting while the deploy path still applied it, and a - // hand-written entry carrying rubric keys would deploy and then be refused - // by every command that reads it. - spliced := containsRefDirective(values) + // Read before resolution, which consumes the directive. Both routes gate the + // rescue on the same answer so they cannot disagree: without it, the CLI's + // no-`$ref` fast path would skip nesting while the deploy path still applied + // it, and a hand-written entry carrying rubric keys would deploy and then be + // refused by every command that reads it. + spliced, visible := splicedEvaluators(values) resolved, err := foundry.ResolveFileRefs(values, baseDir) if err != nil { @@ -251,12 +251,41 @@ func resolveEvalRefs(values map[string]any, baseDir string) (map[string]any, err // `$ref` is a directive rather than configuration, and the strict decoder // would report the leftover as a mistyped key. delete(resolved, "$ref") - if spliced { - nestSplicedRubrics(resolved) - } + nestSplicedRubrics(resolved, spliced, visible) return resolved, nil } +// splicedEvaluators reports which evaluator entries carry an include of their +// own, and whether the list could be read at all. +// +// Entry level rather than document level. Asking only whether the document used +// `$ref` anywhere made one entry's meaning depend on another's: a directive on +// an unrelated dataset switched the rescue on for the whole file, so a +// hand-written `dimensions:` -- a mistake the strict decoder exists to report -- +// was filed as rubric content and published instead. The same evaluator was +// then refused or accepted according to a neighbour. +func splicedEvaluators(values map[string]any) (map[int]bool, bool) { + entries, ok := values["evaluators"].([]any) + if !ok { + // The configuration is itself behind a `$ref`, so its entries do not + // exist yet and nothing here was hand-written to protect. + return nil, false + } + spliced := map[int]bool{} + for i, entry := range entries { + m, ok := entry.(map[string]any) + if !ok { + continue + } + if _, has := m[refDirective]; has { + spliced[i] = true + } + } + // Position survives resolution: entries are replaced in place, never added + // or dropped. + return spliced, true +} + // containsRefDirective reports whether the document uses `$ref` anywhere. // // Structural rather than a text scan: the byte "$ref" also appears in comments @@ -296,18 +325,22 @@ var evaluatorDeclKeys = map[string]bool{ // from the service -- so its keys land beside `name` and the strict decoder // rejects them. Moving them is what lets a `$ref` name a rubric. // -// `dimensions` is what marks the leftovers as a rubric rather than a typo, and -// it is the same key normalizeRubricBody insists on before it will treat a -// document as a definition. Without that gate this would be a catch-all by -// another name, filing a misspelled `name` as rubric content and publishing it -// to the service instead of reporting it. +// Only the entries that carried a directive are touched. `dimensions` then +// marks the leftovers as a rubric rather than a typo, and it is the same key +// normalizeRubricBody insists on before it will treat a document as a +// definition. Without both gates this is a catch-all by another name, filing a +// misspelled `name` as rubric content and publishing it to the service instead +// of reporting it. // -// Structural rather than positional on purpose: an earlier version marked -// entries by index before resolution, which cannot see the evaluators inside a -// config that is itself behind a `$ref` -- the layout the README documents. -func nestSplicedRubrics(resolved map[string]any) { +// visible is false when the configuration is itself behind a `$ref`, where the +// entries only exist after resolution and there is nothing written here to tell +// them apart from. That is the layout the README documents. +func nestSplicedRubrics(resolved map[string]any, spliced map[int]bool, visible bool) { entries, _ := resolved["evaluators"].([]any) - for _, entry := range entries { + for i, entry := range entries { + if visible && !spliced[i] { + continue + } m, ok := entry.(map[string]any) if !ok { continue diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/project/neighbour_ref_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/project/neighbour_ref_test.go new file mode 100644 index 00000000000..35f87e742a0 --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/internal/project/neighbour_ref_test.go @@ -0,0 +1,66 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package project + +import ( + "os" + "path/filepath" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// A `$ref` on one entry does not change what a different entry means. +// +// The rescue that moves spliced rubric keys under `definition` used to be gated +// on "this document uses `$ref` somewhere". A directive on an unrelated dataset +// therefore switched it on for every evaluator in the file, and a hand-written +// `dimensions:` -- a mistake the strict decoder exists to report -- was silently +// filed as rubric content and published to the service instead. +// +// The same evaluator, refused in one file and accepted in another because of a +// neighbour, is the shape this whole mechanism is supposed to rule out. +func TestARefOnOneEntryDoesNotRescueAnother(t *testing.T) { + withoutRef := ` +datasets: + - name: golden + file: ./datasets/golden.jsonl +evaluators: + - name: quality + dimensions: + - id: tone + weight: 3 +` + withUnrelatedRef := ` +datasets: + - $ref: ./parts/golden.yaml +evaluators: + - name: quality + dimensions: + - id: tone + weight: 3 +` + + refused := func(t *testing.T, body string) error { + t.Helper() + dir := t.TempDir() + require.NoError(t, os.MkdirAll(filepath.Join(dir, "parts"), 0o755)) + require.NoError(t, os.WriteFile(filepath.Join(dir, "parts", "golden.yaml"), + []byte("name: golden\nfile: ./datasets/golden.jsonl\n"), 0o600)) + require.NoError(t, os.WriteFile(filepath.Join(dir, EvalConfigBase), []byte(body), 0o600)) + _, err := OpenEvalConfig(dir) + return err + } + + baseline := refused(t, withoutRef) + require.Error(t, baseline, + "a rubric key written at entry level is a mistake, and the strict decoder reports it") + assert.Contains(t, baseline.Error(), "dimensions") + + neighbour := refused(t, withUnrelatedRef) + require.Error(t, neighbour, + "the dataset's `$ref` says nothing about this evaluator, so the same entry is still a mistake") + assert.Contains(t, neighbour.Error(), "dimensions") +}