Declare the azure.ai.eval schema, and make $ref mean one thing

cli/azd/extensions/azure.ai.evaluations/README.md

@@ -23,30 +23,31 @@ services:
   evals:
     host: azure.ai.eval
     uses: [ai-project]
-    $ref: ./evals/azure.yaml
+    $ref: ./evals/azure.eval.yaml
 ```
 
 ```yaml
-# evals/azure.yaml
+# evals/azure.eval.yaml
 datasets:
   - name: support-golden
-    source: ./datasets/support-golden.jsonl
+    file: ./datasets/support-golden.jsonl
 
 evaluators:
   - name: support-quality
     source: ./evaluators/support-quality.json
 
-evalGroups:
+evals:
   - name: support-quality
     dataset: support-golden
+    evaluation_level: turn
     evaluators:
-      - builtin.task_adherence
-      - support-quality
+      - evaluator: builtin.task_adherence
+        initialization_parameters:
+          model: gpt-4.1-nano
+      - evaluator: support-quality
     target:
       type: agent
       name: support-agent
-    options:
-      eval_model: gpt-4.1-nano
 ```
 
 `azd up` reconciles **datasets → evaluators → eval groups**, in that order,
@@ -64,7 +65,7 @@ keys you authored — the service adds `data_schema`, `init_parameters` and
 `metrics` of its own.
 
 Eval groups are immutable, so a change to a group's evaluators, target or
-options creates a new group and a new id. The id is cached in the azd
+sampling creates a new group and a new id. The id is cached in the azd
 environment so repeat runs stay comparable.
 
 ## Commands

cli/azd/extensions/azure.ai.evaluations/internal/cmd/catalog.go

@@ -28,16 +28,16 @@ func addDatasetToCatalog(cmd *cobra.Command, evalDir string, ref *project.Artifa
 			if cfg.Datasets[i].Name == ref.Name {
 				// Regeneration overwrites the file in place, so the entry only
 				// changes when the artifact moved.
-				if cfg.Datasets[i].Source == ref.Source {
+				if cfg.Datasets[i].File == ref.Source {
 					return false
 				}
-				cfg.Datasets[i].Source = ref.Source
+				cfg.Datasets[i].File = ref.Source
 				return true
 			}
 		}
 		cfg.Datasets = append(cfg.Datasets, project.DatasetDecl{
-			Name:   ref.Name,
-			Source: ref.Source,
+			Name: ref.Name,
+			File: ref.Source,
 		})
 		return true
 	})
@@ -88,7 +88,7 @@ func updateCatalog(
 	}
 	defer unlock()
 
-	cfg, err := project.OpenEvalConfig(evalDir)
+	cfg, err := project.OpenEvalConfigForEdit(evalDir)
 	if err != nil {
 		return err
 	}

cli/azd/extensions/azure.ai.evaluations/internal/cmd/context.go

@@ -9,6 +9,7 @@ import (
 	"fmt"
 	"log"
 	"os"
+	"path/filepath"
 	"strings"
 
 	"azureaieval/internal/foundry/projectctx"
@@ -417,12 +418,21 @@ const (
 //
 //  1. --path
 //  2. the path `init` recorded in the azd environment
-//  3. ./evals
+//  3. the `$ref` on the `azure.ai.eval` service in azure.yaml
+//  4. ./evals
 //
-// The middle level is what stops `--path` from having to be repeated on every
-// later command. Without it, `init --path ./quality` wrote a configuration that
-// `run` then looked for under ./evals and reported as missing -- while
-// azure.yaml's $ref pointed at it correctly the whole time.
+// The middle levels are what stop `--path` from having to be repeated on every
+// later command. Without the recorded one, `init --path ./quality` wrote a
+// configuration that `run` then looked for under ./evals and reported as
+// missing -- while azure.yaml's $ref pointed at it correctly the whole time.
+//
+// That $ref is now read rather than only written, which is what makes the rule
+// survive a fresh clone. The recorded path lives in the azd environment, and an
+// azd environment is not in the repository: check the project out somewhere
+// else and level 2 is empty, so a configuration the project declares perfectly
+// well under ./config was reported missing by every command while `azd up`
+// deployed it. Reading the declaration is also what keeps one answer to "where
+// is the configuration" instead of one for deploy and one for everything else.
 //
 // This is the whole rule, and every command that reads the configuration goes
 // through it. Stating it here and applying it on only some paths is how
@@ -432,10 +442,17 @@ const (
 //
 // recorded tells absence apart from failure, and the two get different
 // answers. A project with no azd environment has genuinely recorded nothing,
-// so ./evals is right. An azd that could not be asked has said nothing at all,
-// and defaulting on that would write the second configuration all over again --
-// this time for a reason nobody could reproduce.
-func evalDirCascade(flagValue string, recorded func() (string, error)) (string, error) {
+// so the next level is right. An azd that could not be asked has said nothing
+// at all, and defaulting on that would write the second configuration all over
+// again -- this time for a reason nobody could reproduce.
+//
+// declared is best-effort by contrast: outside an azd project there is no
+// azure.yaml to read, which is ordinary rather than a failure.
+func evalDirCascade(
+	flagValue string,
+	recorded func() (string, error),
+	declared func() string,
+) (string, error) {
 	if flagValue != "" {
 		return flagValue, nil
 	}
@@ -446,16 +463,57 @@ func evalDirCascade(flagValue string, recorded func() (string, error)) (string,
 	if path != "" {
 		return path, nil
 	}
+	if declared != nil {
+		if dir := declared(); dir != "" {
+			return dir, nil
+		}
+	}
 	return project.DefaultEvalDir, nil
 }
 
+// declaredEvalConfig reads the location azure.yaml's `$ref` points at.
+//
+// The service entry is the project's own statement of where its evaluation
+// configuration lives, and `azd up` has always deployed from it. The full path
+// is returned rather than its directory: the `$ref` names a file, and a project
+// declaring `./config/nightly.yaml` means that file, not whatever
+// `azure.eval.yaml` happens to sit beside it.
+//
+// Returns empty outside an azd project, or when nothing declares the eval host.
+func declaredEvalConfig(ctx context.Context, azdClient *azdext.AzdClient) string {
+	if azdClient == nil {
+		return ""
+	}
+	resp, err := azdClient.Project().Get(ctx, &azdext.EmptyRequest{})
+	if err != nil || resp.GetProject() == nil {
+		return ""
+	}
+	for _, svc := range resp.GetProject().GetServices() {
+		if svc.GetHost() != project.EvalHost {
+			continue
+		}
+		props := svc.GetAdditionalProperties()
+		if props == nil {
+			continue
+		}
+		ref, _ := props.AsMap()["$ref"].(string)
+		if ref == "" {
+			continue
+		}
+		return filepath.Clean(filepath.FromSlash(ref))
+	}
+	return ""
+}
+
 // evalDir is the cascade for a command that already holds an azd connection.
 func (ec *evalContext) evalDir(ctx context.Context, flagValue string) (string, error) {
 	return evalDirCascade(flagValue, func() (string, error) {
 		if ec.envName == "" || ec.azdClient == nil {
 			return "", nil
 		}
 		return readRecordedEvalPath(ctx, ec.azdClient, ec.envName)
+	}, func() string {
+		return declaredEvalConfig(ctx, ec.azdClient)
 	})
 }
 
@@ -494,6 +552,13 @@ func resolveEvalDir(ctx context.Context, flagValue string) (string, error) {
 			return "", nil
 		}
 		return readRecordedEvalPath(ctx, azdClient, env.GetEnvironment().GetName())
+	}, func() string {
+		azdClient, err := azdext.NewAzdClient()
+		if err != nil {
+			return ""
+		}
+		defer azdClient.Close()
+		return declaredEvalConfig(ctx, azdClient)
 	})
 }
 

cli/azd/extensions/azure.ai.evaluations/internal/cmd/eval_group.go

@@ -74,7 +74,7 @@ func newEvalCreateCommand() *cobra.Command {
 			baseDir := filepath.Dir(path)
 			datasetPath := ""
 			if decl, ok := cfg.DatasetDeclaration(eval.Dataset); ok {
-				datasetPath = project.ResolveSource(baseDir, decl.Source)
+				datasetPath = project.ResolveSource(baseDir, decl.File)
 			}
 
 			reconciler := &evalReconciler{ec: ec}

cli/azd/extensions/azure.ai.evaluations/internal/cmd/evaldir_test.go

@@ -82,20 +82,68 @@ func TestEvalDirCascadeAnswersInOrder(t *testing.T) {
 		t.Run(tc.name, func(t *testing.T) {
 			got, err := evalDirCascade(tc.flag, func() (string, error) {
 				return tc.recorded, nil
-			})
+			}, nil)
 			require.NoError(t, err)
 			assert.Equal(t, tc.want, got)
 		})
 	}
 }
 
+// azure.yaml's `$ref` is read, not only written.
+//
+// The recorded path lives in the azd environment, and an azd environment is not
+// in the repository. Check the project out somewhere else and that level is
+// empty, so a configuration the project declares under ./config was reported
+// missing by every command while `azd up` deployed it from the same `$ref`.
+func TestEvalDirCascadeReadsTheDeclaredRef(t *testing.T) {
+	got, err := evalDirCascade("",
+		func() (string, error) { return "", nil },
+		func() string { return "config" })
+
+	require.NoError(t, err)
+	assert.Equal(t, "config", got)
+}
+
+// The recorded path is what `--path` wrote on this machine, so it answers over
+// a declaration that may predate it.
+func TestEvalDirCascadePrefersTheRecordedPathOverTheDeclaredRef(t *testing.T) {
+	got, err := evalDirCascade("",
+		func() (string, error) { return "quality", nil },
+		func() string { return "config" })
+
+	require.NoError(t, err)
+	assert.Equal(t, "quality", got)
+}
+
+// Outside an azd project there is no azure.yaml to read, which is ordinary.
+func TestEvalDirCascadeFallsBackWhenNothingIsDeclared(t *testing.T) {
+	got, err := evalDirCascade("",
+		func() (string, error) { return "", nil },
+		func() string { return "" })
+
+	require.NoError(t, err)
+	assert.Equal(t, project.DefaultEvalDir, got)
+}
+
+// A --path that was given is the answer on its own, so neither level is asked.
+func TestEvalDirCascadeSkipsBothLookupsWhenPathWasGiven(t *testing.T) {
+	var declaredAsked int
+	got, err := evalDirCascade("./given",
+		func() (string, error) { return "recorded", nil },
+		func() string { declaredAsked++; return "config" })
+
+	require.NoError(t, err)
+	assert.Equal(t, "./given", got)
+	assert.Equal(t, 0, declaredAsked, "a --path that was given should not cost a round trip")
+}
+
 // A read that failed is not a project that recorded nothing. Defaulting on it
 // is how `generate` would write a second configuration under ./evals for a
 // reason nobody could reproduce, so the failure has to come back out.
 func TestEvalDirCascadeDoesNotDefaultOnAFailedRead(t *testing.T) {
 	boom := errors.New("the environment could not be read")
 
-	got, err := evalDirCascade("", func() (string, error) { return "", boom })
+	got, err := evalDirCascade("", func() (string, error) { return "", boom }, nil)
 
 	require.ErrorIs(t, err, boom)
 	assert.Empty(t, got, "a failed read must not answer with the default")
@@ -106,7 +154,7 @@ func TestEvalDirCascadeDoesNotDefaultOnAFailedRead(t *testing.T) {
 func TestEvalDirCascadeIgnoresAFailedReadWhenPathWasGiven(t *testing.T) {
 	got, err := evalDirCascade("./given", func() (string, error) {
 		return "", errors.New("the environment could not be read")
-	})
+	}, nil)
 
 	require.NoError(t, err)
 	assert.Equal(t, "./given", got)
@@ -118,7 +166,7 @@ func TestEvalDirCascadeAsksForTheRecordedPathOnce(t *testing.T) {
 	got, err := evalDirCascade("", func() (string, error) {
 		asked++
 		return "", nil
-	})
+	}, nil)
 
 	require.NoError(t, err)
 	assert.Equal(t, project.DefaultEvalDir, got)
@@ -128,7 +176,7 @@ func TestEvalDirCascadeAsksForTheRecordedPathOnce(t *testing.T) {
 	_, err = evalDirCascade("./given", func() (string, error) {
 		asked++
 		return "", nil
-	})
+	}, nil)
 	require.NoError(t, err)
 	assert.Equal(t, 0, asked, "a --path that was given should not cost a round trip")
 }

cli/azd/extensions/azure.ai.evaluations/internal/cmd/generate_commands.go

@@ -90,7 +90,7 @@ func resolvePlan(f *generateFlags, name string, defaultOutputDir string) (genera
 		Agent:       firstNonEmpty(f.target, declaredTarget(f.path)),
 		Model:       f.model,
 		Instruction: instruction,
-		BaseDir:     f.path,
+		BaseDir:     project.EvalDirOf(f.path),
 		OutputDir:   firstNonEmpty(f.outputDir, "./"+defaultOutputDir),
 	}
 	if plan.Model == "" && plan.Agent == "" {

cli/azd/extensions/azure.ai.evaluations/internal/cmd/init.go

@@ -137,7 +137,7 @@ func newInitCommand() *cobra.Command {
 			// reporting it as created would claim a file it only added to.
 			_, configExistedErr := os.Stat(configPath)
 			configExisted := configExistedErr == nil
-			cfg, err := project.OpenEvalConfig(path)
+			cfg, err := project.OpenEvalConfigForEdit(path)
 			if err != nil {
 				return err
 			}
@@ -181,7 +181,7 @@ func newInitCommand() *cobra.Command {
 			}
 			defer unlockConfig()
 
-			cfg, err = project.OpenEvalConfig(path)
+			cfg, err = project.OpenEvalConfigForEdit(path)
 			if err != nil {
 				return err
 			}
@@ -196,10 +196,13 @@ func newInitCommand() *cobra.Command {
 				cfg.RemoveEval(evalName)
 			}
 
-			if err := os.MkdirAll(filepath.Join(path, project.DefaultDatasetsDir), 0o750); err != nil {
+			// The location may be the file azure.yaml names rather than the
+			// directory holding it, and artifacts sit beside the configuration.
+			evalDir := project.EvalDirOf(path)
+			if err := os.MkdirAll(filepath.Join(evalDir, project.DefaultDatasetsDir), 0o750); err != nil {
 				return messages.CreatingDatasetsDir(err)
 			}
-			if err := os.MkdirAll(filepath.Join(path, project.DefaultEvaluatorsDir), 0o750); err != nil {
+			if err := os.MkdirAll(filepath.Join(evalDir, project.DefaultEvaluatorsDir), 0o750); err != nil {
 				return messages.CreatingEvaluatorsDir(err)
 			}
 
@@ -236,8 +239,8 @@ func newInitCommand() *cobra.Command {
 					"eval":          evalName,
 					"evalConfig":    configPath,
 					"service":       serviceName,
-					"datasetsDir":   filepath.Join(path, project.DefaultDatasetsDir),
-					"evaluatorsDir": filepath.Join(path, project.DefaultEvaluatorsDir),
+					"datasetsDir":   filepath.Join(evalDir, project.DefaultDatasetsDir),
+					"evaluatorsDir": filepath.Join(evalDir, project.DefaultEvaluatorsDir),
 					"rootConfig":    rootWiring,
 					"target":        target,
 					"source":        source,
@@ -401,7 +404,7 @@ func planScaffold(in scaffoldInput) scaffold {
 		}
 		eval.Dataset = datasetName
 		out.datasetName = datasetName
-		addDatasetDecl(cfg, project.DatasetDecl{Name: datasetName, Source: datasetSource})
+		addDatasetDecl(cfg, project.DatasetDecl{Name: datasetName, File: datasetSource})
 	}
 
 	// Every evaluator carries the judge deployment, because that is where the