[WIP] Implement config verbs for OTE

cmd/example-tests/main.go

@@ -1,6 +1,7 @@
 package main
 
 import (
+	"context"
 	"fmt"
 	"os"
 
@@ -9,6 +10,7 @@ import (
 
 	"github.com/openshift-eng/openshift-tests-extension/pkg/cmd"
 	e "github.com/openshift-eng/openshift-tests-extension/pkg/extension"
+	"github.com/openshift-eng/openshift-tests-extension/pkg/flags"
 	g "github.com/openshift-eng/openshift-tests-extension/pkg/ginkgo"
 
 	// If using ginkgo, import your tests here
@@ -48,6 +50,20 @@ func main() {
 		},
 	})
 
+	// Register configs that tests may require
+	ext.RegisterConfig(e.Config{
+		Name:        "example-config",
+		Description: "An example configuration demonstrating the config feature",
+		Apply: func(ctx context.Context, envFlags flags.EnvironmentalFlags) error {
+			fmt.Fprintf(os.Stderr, "Example config applied (platform: %s)\n", envFlags.Platform)
+			return nil
+		},
+		Remove: func(ctx context.Context, envFlags flags.EnvironmentalFlags) error {
+			fmt.Fprintf(os.Stderr, "Example config removed\n")
+			return nil
+		},
+	})
+
 	// If using Ginkgo, build test specs automatically
 	specs, err := g.BuildExtensionTestSpecsFromOpenShiftGinkgoSuite()
 	if err != nil {

pkg/cmd/cmd.go

@@ -3,6 +3,7 @@ package cmd
 import (
 	"github.com/spf13/cobra"
 
+	"github.com/openshift-eng/openshift-tests-extension/pkg/cmd/cmdconfig"
 	"github.com/openshift-eng/openshift-tests-extension/pkg/cmd/cmdimages"
 	"github.com/openshift-eng/openshift-tests-extension/pkg/cmd/cmdinfo"
 	"github.com/openshift-eng/openshift-tests-extension/pkg/cmd/cmdlist"
@@ -19,5 +20,6 @@ func DefaultExtensionCommands(registry *extension.Registry) []*cobra.Command {
 		cmdinfo.NewInfoCommand(registry),
 		cmdupdate.NewUpdateCommand(registry),
 		cmdimages.NewImagesCommand(registry),
+		cmdconfig.NewConfigCommand(registry),
 	}
 }

pkg/cmd/cmdconfig/apply.go

@@ -0,0 +1,65 @@
+package cmdconfig
+
+import (
+	"context"
+	"fmt"
+
+	"github.com/spf13/cobra"
+
+	"github.com/openshift-eng/openshift-tests-extension/pkg/extension"
+	"github.com/openshift-eng/openshift-tests-extension/pkg/flags"
+)
+
+func NewApplyCommand(registry *extension.Registry) *cobra.Command {
+	componentFlags := flags.NewComponentFlags()
+	envFlags := flags.NewEnvironmentalFlags()
+	var configName string
+
+	cmd := &cobra.Command{
+		Use:          "apply",
+		Short:        "Apply a configuration",
+		SilenceUsage: true,
+		RunE: func(cmd *cobra.Command, args []string) error {
+			ext := registry.Get(componentFlags.Component)
+			if ext == nil {
+				return fmt.Errorf("couldn't find the component %q", componentFlags.Component)
+			}
+
+			if configName == "" {
+				return fmt.Errorf("--config flag is required")
+			}
+
+			// Find the config by name
+			var config *extension.Config
+			for i := range ext.Configs {
+				if ext.Configs[i].Name == configName {
+					config = &ext.Configs[i]
+					break
+				}
+			}
+
+			if config == nil {
+				return fmt.Errorf("config %q not found", configName)
+			}
+
+			if config.Apply == nil {
+				return fmt.Errorf("config %q does not have an Apply function", configName)
+			}
+
+			ctx := context.Background()
+			if err := config.Apply(ctx, *envFlags); err != nil {
+				return fmt.Errorf("failed to apply config %q: %w", configName, err)
+			}
+
+			fmt.Printf("Successfully applied config %q\n", configName)
+			return nil
+		},
+	}
+
+	componentFlags.BindFlags(cmd.Flags())
+	envFlags.BindFlags(cmd.Flags())
+	cmd.Flags().StringVar(&configName, "config", "", "Name of the configuration to apply")
+
+	return cmd
+}
+

pkg/cmd/cmdconfig/config.go

@@ -0,0 +1,23 @@
+package cmdconfig
+
+import (
+	"github.com/spf13/cobra"
+
+	"github.com/openshift-eng/openshift-tests-extension/pkg/extension"
+)
+
+func NewConfigCommand(registry *extension.Registry) *cobra.Command {
+	cmd := &cobra.Command{
+		Use:          "config",
+		Short:        "Manage extension configurations",
+		SilenceUsage: true,
+	}
+
+	cmd.AddCommand(
+		NewApplyCommand(registry),
+		NewRemoveCommand(registry),
+	)
+
+	return cmd
+}
+

pkg/cmd/cmdconfig/remove.go

@@ -0,0 +1,65 @@
+package cmdconfig
+
+import (
+	"context"
+	"fmt"
+
+	"github.com/spf13/cobra"
+
+	"github.com/openshift-eng/openshift-tests-extension/pkg/extension"
+	"github.com/openshift-eng/openshift-tests-extension/pkg/flags"
+)
+
+func NewRemoveCommand(registry *extension.Registry) *cobra.Command {
+	componentFlags := flags.NewComponentFlags()
+	envFlags := flags.NewEnvironmentalFlags()
+	var configName string
+
+	cmd := &cobra.Command{
+		Use:          "remove",
+		Short:        "Remove a configuration",
+		SilenceUsage: true,
+		RunE: func(cmd *cobra.Command, args []string) error {
+			ext := registry.Get(componentFlags.Component)
+			if ext == nil {
+				return fmt.Errorf("couldn't find the component %q", componentFlags.Component)
+			}
+
+			if configName == "" {
+				return fmt.Errorf("--config flag is required")
+			}
+
+			// Find the config by name
+			var config *extension.Config
+			for i := range ext.Configs {
+				if ext.Configs[i].Name == configName {
+					config = &ext.Configs[i]
+					break
+				}
+			}
+
+			if config == nil {
+				return fmt.Errorf("config %q not found", configName)
+			}
+
+			if config.Remove == nil {
+				return fmt.Errorf("config %q does not have a Remove function", configName)
+			}
+
+			ctx := context.Background()
+			if err := config.Remove(ctx, *envFlags); err != nil {
+				return fmt.Errorf("failed to remove config %q: %w", configName, err)
+			}
+
+			fmt.Printf("Successfully removed config %q\n", configName)
+			return nil
+		},
+	}
+
+	componentFlags.BindFlags(cmd.Flags())
+	envFlags.BindFlags(cmd.Flags())
+	cmd.Flags().StringVar(&configName, "config", "", "Name of the configuration to remove")
+
+	return cmd
+}
+

pkg/cmd/cmdrun/runsuite.go

@@ -5,6 +5,7 @@ import (
 	"fmt"
 	"os"
 	"os/signal"
+	"strings"
 	"syscall"
 	"time"
 
@@ -14,18 +15,21 @@ import (
 	"github.com/openshift-eng/openshift-tests-extension/pkg/extension"
 	"github.com/openshift-eng/openshift-tests-extension/pkg/extension/extensiontests"
 	"github.com/openshift-eng/openshift-tests-extension/pkg/flags"
+	"github.com/openshift-eng/openshift-tests-extension/pkg/util/sets"
 )
 
 func NewRunSuiteCommand(registry *extension.Registry) *cobra.Command {
 	opts := struct {
 		componentFlags   *flags.ComponentFlags
 		outputFlags      *flags.OutputFlags
 		concurrencyFlags *flags.ConcurrencyFlags
+		envFlags         *flags.EnvironmentalFlags
 		junitPath        string
 	}{
 		componentFlags:   flags.NewComponentFlags(),
 		outputFlags:      flags.NewOutputFlags(),
 		concurrencyFlags: flags.NewConcurrencyFlags(),
+		envFlags:         flags.NewEnvironmentalFlags(),
 		junitPath:        "",
 	}
 
@@ -97,18 +101,136 @@ func NewRunSuiteCommand(registry *extension.Registry) *cobra.Command {
 			}
 			compositeWriter.AddWriter(jsonWriter)
 
-			specs, err := ext.GetSpecs().Filter(suite.Qualifiers)
-			if err != nil {
-				return errors.Wrap(err, "couldn't filter specs")
+		specs, err := ext.GetSpecs().Filter(suite.Qualifiers)
+		if err != nil {
+			return errors.Wrap(err, "couldn't filter specs")
+		}
+
+		// Group specs by required configs
+		specsByConfig := groupSpecsByConfig(specs)
+
+		// Track overall errors
+		var runErrors []error
+
+		// First, run specs that don't require any config
+		if noConfigSpecs, ok := specsByConfig[""]; ok && len(noConfigSpecs) > 0 {
+			fmt.Fprintf(os.Stderr, "Running %d test(s) without config requirements...\n", len(noConfigSpecs))
+			if err := noConfigSpecs.Run(ctx, compositeWriter, opts.concurrencyFlags.MaxConcurency); err != nil {
+				runErrors = append(runErrors, err)
 			}
+			delete(specsByConfig, "")
+		}
+
+		// Then run specs grouped by config
+		for configName, configSpecs := range specsByConfig {
+			if err := runSpecsWithConfig(
+				ctx,
+				ext,
+				configName,
+				configSpecs,
+				compositeWriter,
+				opts.concurrencyFlags.MaxConcurency,
+				*opts.envFlags,
+			); err != nil {
+				runErrors = append(runErrors, err)
+			}
+		}
+
+		// Return combined errors if any
+		if len(runErrors) > 0 {
+			return fmt.Errorf("%d test group(s) failed", len(runErrors))
+		}
 
-			return specs.Run(ctx, compositeWriter, opts.concurrencyFlags.MaxConcurency)
+		return nil
 		},
 	}
 	opts.componentFlags.BindFlags(cmd.Flags())
 	opts.outputFlags.BindFlags(cmd.Flags())
 	opts.concurrencyFlags.BindFlags(cmd.Flags())
+	opts.envFlags.BindFlags(cmd.Flags())
 	cmd.Flags().StringVarP(&opts.junitPath, "junit-path", "j", opts.junitPath, "write results to junit XML")
 
 	return cmd
 }
+
+// extractRequiredConfigs extracts the set of config names required by the given specs
+func extractRequiredConfigs(specs extensiontests.ExtensionTestSpecs) sets.Set[string] {
+	configs := sets.New[string]()
+	for _, spec := range specs {
+		for label := range spec.Labels {
+			if strings.HasPrefix(label, "Config:") {
+				configName := strings.TrimPrefix(label, "Config:")
+				configs.Insert(configName)
+			}
+		}
+	}
+	return configs
+}
+
+// groupSpecsByConfig groups specs by the configs they require. Specs with no config requirement
+// are grouped under an empty string key.
+func groupSpecsByConfig(specs extensiontests.ExtensionTestSpecs) map[string]extensiontests.ExtensionTestSpecs {
+	groups := make(map[string]extensiontests.ExtensionTestSpecs)
+
+	for _, spec := range specs {
+		var configName string
+		// Find the config requirement for this spec
+		for label := range spec.Labels {
+			if strings.HasPrefix(label, "Config:") {
+				configName = strings.TrimPrefix(label, "Config:")
+				break
+			}
+		}
+
+		// Group by config name (empty string for no config)
+		groups[configName] = append(groups[configName], spec)
+	}
+
+	return groups
+}
+
+// runSpecsWithConfig applies a config, runs the specs, then removes the config
+func runSpecsWithConfig(
+	ctx context.Context,
+	ext *extension.Extension,
+	configName string,
+	specs extensiontests.ExtensionTestSpecs,
+	writer extensiontests.ResultWriter,
+	maxConcurrency int,
+	envFlags flags.EnvironmentalFlags,
+) error {
+	// Find the config
+	var config *extension.Config
+	for i := range ext.Configs {
+		if ext.Configs[i].Name == configName {
+			config = &ext.Configs[i]
+			break
+		}
+	}
+
+	if config == nil {
+		return fmt.Errorf("config %q not found but required by tests", configName)
+	}
+
+	// Apply the config
+	if config.Apply != nil {
+		fmt.Fprintf(os.Stderr, "Applying config %q for %d test(s)...\n", configName, len(specs))
+		if err := config.Apply(ctx, envFlags); err != nil {
+			return fmt.Errorf("failed to apply config %q: %w", configName, err)
+		}
+	}
+
+	// Run the specs
+	err := specs.Run(ctx, writer, maxConcurrency)
+
+	// Remove the config
+	if config.Remove != nil {
+		fmt.Fprintf(os.Stderr, "Removing config %q...\n", configName)
+		if removeErr := config.Remove(ctx, envFlags); removeErr != nil {
+			// Log warning but don't fail the overall run
+			fmt.Fprintf(os.Stderr, "Warning: failed to remove config %q: %v\n", configName, removeErr)
+		}
+	}
+
+	return err
+}

pkg/extension/extension.go

@@ -135,6 +135,11 @@ func (e *Extension) RegisterImage(image Image) *Extension {
 	return e
 }
 
+func (e *Extension) RegisterConfig(config Config) *Extension {
+	e.Configs = append(e.Configs, config)
+	return e
+}
+
 func (e *Extension) FindSpecsByName(names ...string) (et.ExtensionTestSpecs, error) {
 	var specs et.ExtensionTestSpecs
 	var notFound []string