diff --git a/cli/azd/extensions/azure.ai.evaluations/CHANGELOG.md b/cli/azd/extensions/azure.ai.evaluations/CHANGELOG.md index 69b395018cf..532ebf558da 100644 --- a/cli/azd/extensions/azure.ai.evaluations/CHANGELOG.md +++ b/cli/azd/extensions/azure.ai.evaluations/CHANGELOG.md @@ -22,6 +22,8 @@ First release of the Foundry evaluations extension. declaration changes. - Atomic commands for every operation: `dataset`, `evaluator`, `run` and `run output` subcommands, all supporting `-o json` and `--no-prompt`. +- `dataset --version` names the version to publish, on `create` and `update` + alike, and means the same thing as `version:` in the configuration. - Testing criteria are shaped from each evaluator's published contract, so evaluators requiring inputs beyond the agent shape — `ground_truth`, `context`, `instruction_id_list` — work by binding them to dataset columns. diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/dataset.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/dataset.go index c79f6cb2ff4..de0c8d03944 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/dataset.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/dataset.go @@ -142,9 +142,21 @@ func newDatasetWriteCommand(verb, short string) *cobra.Command { return err } - ds, err := ec.datasetClient.UploadNextVersion( - ctx, name, version, localSource, ProjectEndpointAPIVersion, - ) + // A declared version is the version to publish, never one to count + // from, so it is written exactly as given. Only an omitted version is + // derived, and only that path walks past a conflict: a version the + // author named and the service already holds is theirs to resolve, + // and stepping past it would publish one they did not ask for. + var ds *dataset_api.Dataset + if version != "" { + ds, err = ec.datasetClient.UploadVersion( + ctx, name, version, localSource, ProjectEndpointAPIVersion, + ) + } else { + ds, err = ec.datasetClient.UploadNextVersion( + ctx, name, "", localSource, ProjectEndpointAPIVersion, + ) + } if err != nil { return messages.RegisteringDataset(name, err) } @@ -170,14 +182,8 @@ func newDatasetWriteCommand(verb, short string) *cobra.Command { cmd.Flags().StringVar(&fromFile, "from-file", "", "Path to a .jsonl file, or a directory containing one.") - // Only on update. create publishes a first version, and the upload derives - // the next version from whatever this holds, so `create --version 4.0` - // would publish 5.0 -- and leave the existence probe, which looks for the - // versions a first publish can carry, unable to find what it wrote. - if verb == "update" { - cmd.Flags().StringVar(&version, "version", "", - "Current version to increment from. Omit to increment from the latest registered version.") - } + cmd.Flags().StringVar(&version, "version", "", + "Version to publish. Omit to publish the next version after the latest registered.") cmd.Flags().StringVar(&endpointFlg, "project-endpoint", "", "Foundry project endpoint.") return cmd } diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/pkg/dataset_api/upload_version_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/pkg/dataset_api/upload_version_test.go index 164cc6d415b..e66ecb39457 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/pkg/dataset_api/upload_version_test.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/pkg/dataset_api/upload_version_test.go @@ -103,6 +103,63 @@ func TestUploadNextVersionWalksPastAStaleListing(t *testing.T) { "the version just refused is proof it exists, so the next one is tried") } +// A declared version is the version published, not one to count from. +// +// `--version` used to reach the incrementing path, so `--version 7.0` published +// 8.0 while `version: 7.0` in configuration published 7.0 -- one word, two +// answers, decided by where it was written. +func TestUploadVersionPublishesTheVersionDeclared(t *testing.T) { + server := &uploadServer{taken: map[string]bool{}, listing: []string{"1.0", "2.0"}} + httpServer := func() *httptest.Server { + var s *httptest.Server + s = httptest.NewServer(server.handler(t, func() string { return s.URL })) + return s + }() + t.Cleanup(httpServer.Close) + + client := NewDatasetClientFromPipeline( + httpServer.URL, runtime.NewPipeline("test", "v1", runtime.PipelineOptions{}, nil)) + + dir := t.TempDir() + require.NoError(t, os.WriteFile( + filepath.Join(dir, "rows.jsonl"), []byte("{\"query\":\"q\"}\n"), 0o600)) + + ds, err := client.UploadVersion(context.Background(), "ds", "7.0", dir, "2025-11-15-preview") + require.NoError(t, err) + assert.Equal(t, "7.0", ds.Version, "the version asked for is the version written") + assert.Equal(t, []string{"7.0"}, server.attempts, + "a declared version is published as given, not counted from") +} + +// A declared version the service already holds is refused rather than stepped +// past. +// +// The conflict walk exists because the listing lags behind a publish, which +// makes it right for a version the CLI derived. Applying it to one an author +// named would publish a version they did not ask for, and report success. +func TestUploadVersionDoesNotWalkPastAConflict(t *testing.T) { + server := &uploadServer{taken: map[string]bool{"7.0": true}, listing: []string{"7.0"}} + httpServer := func() *httptest.Server { + var s *httptest.Server + s = httptest.NewServer(server.handler(t, func() string { return s.URL })) + return s + }() + t.Cleanup(httpServer.Close) + + client := NewDatasetClientFromPipeline( + httpServer.URL, runtime.NewPipeline("test", "v1", runtime.PipelineOptions{}, nil)) + + dir := t.TempDir() + require.NoError(t, os.WriteFile( + filepath.Join(dir, "rows.jsonl"), []byte("{\"query\":\"q\"}\n"), 0o600)) + + _, err := client.UploadVersion(context.Background(), "ds", "7.0", dir, "2025-11-15-preview") + require.Error(t, err, "the version the author named is taken, and that is theirs to resolve") + assert.True(t, IsVersionConflict(err), "the refusal has to read as a conflict") + assert.Equal(t, []string{"7.0"}, server.attempts, + "nothing beyond the declared version is attempted") +} + // When the listing has caught up and is further ahead than the refused // version, it is the better answer: it skips versions somebody else published. func TestUploadNextVersionPrefersACaughtUpListing(t *testing.T) {