From 46404211949e2bd70f557a34156d919029ba03c2 Mon Sep 17 00:00:00 2001 From: Michael Armijo Date: Wed, 14 May 2025 16:49:34 -0600 Subject: [PATCH 1/5] platform/azure: support additional data disks Add support for attaching additional data disks to instances created in Azure. Disks are defined through the machines options as the Size in GB and the 'sku', or storage type, e.g. '["100G:sku=UltraSSD_LRS"]' for NVMe disks. --- mantle/cmd/kola/kola.go | 30 ++++--- mantle/cmd/kola/options.go | 1 + mantle/platform/api/azure/disk.go | 107 +++++++++++++++++++++++ mantle/platform/api/azure/instance.go | 58 +++++++++++- mantle/platform/api/azure/options.go | 15 ++-- mantle/platform/machine/azure/cluster.go | 5 +- 6 files changed, 190 insertions(+), 26 deletions(-) create mode 100644 mantle/platform/api/azure/disk.go diff --git a/mantle/cmd/kola/kola.go b/mantle/cmd/kola/kola.go index d96f341dda..25a6bbc075 100644 --- a/mantle/cmd/kola/kola.go +++ b/mantle/cmd/kola/kola.go @@ -300,13 +300,14 @@ func writeProps() error { InstanceType string `json:"type"` } type Azure struct { - DiskURI string `json:"diskUri"` - Publisher string `json:"publisher"` - Offer string `json:"offer"` - Sku string `json:"sku"` - Version string `json:"version"` - Location string `json:"location"` - Size string `json:"size"` + DiskURI string `json:"diskUri"` + Publisher string `json:"publisher"` + Offer string `json:"offer"` + Sku string `json:"sku"` + Version string `json:"version"` + Location string `json:"location"` + Size string `json:"size"` + AvailabilityZone string `json:"availability_zone"` } type DO struct { Region string `json:"region"` @@ -355,13 +356,14 @@ func writeProps() error { InstanceType: kola.AWSOptions.InstanceType, }, Azure: Azure{ - DiskURI: kola.AzureOptions.DiskURI, - Publisher: kola.AzureOptions.Publisher, - Offer: kola.AzureOptions.Offer, - Sku: kola.AzureOptions.Sku, - Version: kola.AzureOptions.Version, - Location: kola.AzureOptions.Location, - Size: kola.AzureOptions.Size, + DiskURI: kola.AzureOptions.DiskURI, + Publisher: kola.AzureOptions.Publisher, + Offer: kola.AzureOptions.Offer, + Sku: kola.AzureOptions.Sku, + Version: kola.AzureOptions.Version, + Location: kola.AzureOptions.Location, + Size: kola.AzureOptions.Size, + AvailabilityZone: kola.AzureOptions.AvailabilityZone, }, DO: DO{ Region: kola.DOOptions.Region, diff --git a/mantle/cmd/kola/options.go b/mantle/cmd/kola/options.go index e3f24699f9..c8804b0358 100644 --- a/mantle/cmd/kola/options.go +++ b/mantle/cmd/kola/options.go @@ -100,6 +100,7 @@ func init() { sv(&kola.AzureOptions.Version, "azure-version", "", "Azure image version") sv(&kola.AzureOptions.Location, "azure-location", "westus", "Azure location (default \"westus\"") sv(&kola.AzureOptions.Size, "azure-size", "Standard_D2_v2", "Azure machine size (default \"Standard_D2_v2\")") + sv(&kola.AzureOptions.AvailabilityZone, "azure-availability-zone", "1", "Azure Availability Zone (default \"1\")") // do-specific options sv(&kola.DOOptions.ConfigPath, "do-config-file", "", "DigitalOcean config file (default \"~/"+auth.DOConfigPath+"\")") diff --git a/mantle/platform/api/azure/disk.go b/mantle/platform/api/azure/disk.go new file mode 100644 index 0000000000..f1112227ea --- /dev/null +++ b/mantle/platform/api/azure/disk.go @@ -0,0 +1,107 @@ +// Copyright 2025 Red Hat +// Copyright 2016 CoreOS, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package azure + +import ( + "context" + "fmt" + "strings" + + "github.com/Azure/azure-sdk-for-go/sdk/azcore/to" + "github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/compute/armcompute" + + "github.com/coreos/coreos-assembler/mantle/util" +) + +// CreateDisk provisions a new managed disk in the specified Azure resource group using +// the given name, size (in GiB), and SKU (e.g., Premium_LRS). The disk is created in +// the location and availability zone specified in the API options. +func (a *API) CreateDisk(name, resourceGroup string, sizeGB int32, sku armcompute.DiskStorageAccountTypes) (string, error) { + ctx := context.Background() + poller, err := a.diskClient.BeginCreateOrUpdate(ctx, resourceGroup, name, armcompute.Disk{ + Location: &a.opts.Location, + Zones: []*string{&a.opts.AvailabilityZone}, + Tags: map[string]*string{ + "createdBy": to.Ptr("mantle"), + }, + SKU: &armcompute.DiskSKU{ + Name: to.Ptr(sku), + }, + Properties: &armcompute.DiskProperties{ + DiskSizeGB: to.Ptr(sizeGB), + CreationData: &armcompute.CreationData{ + CreateOption: to.Ptr(armcompute.DiskCreateOptionEmpty), + }, + }, + }, nil) + + if err != nil { + return "", fmt.Errorf("failed to create azure disk %v", err) + } + + diskResponse, err := poller.PollUntilDone(context.Background(), nil) + if err != nil { + return "", err + } + + if diskResponse.Disk.ID == nil { + return "", fmt.Errorf("failed to get azure disk id") + } + + return *diskResponse.Disk.ID, nil +} + +// DeleteDisk deletes a managed disk by name from the specified Azure resource group. +func (a *API) DeleteDisk(name, resourceGroup string) error { + ctx := context.Background() + poller, err := a.diskClient.BeginDelete(ctx, resourceGroup, name, nil) + if err != nil { + return err + } + _, err = poller.PollUntilDone(ctx, nil) + return err +} + +// ParseDisk parses a disk specification string from a kola test and returns the +// disk size and the Azure disk SKU. The spec format is ":sku=", +// e.g., ["10G:sku=UltraSSD_LRS"] for NVMe disks. If no SKU is specified, "Standard_LRS" is used. +func (a *API) ParseDisk(spec string) (int64, armcompute.DiskStorageAccountTypes, error) { + sku := armcompute.DiskStorageAccountTypes(armcompute.DiskStorageAccountTypesStandardLRS) + size, diskmap, err := util.ParseDiskSpec(spec, false) + if err != nil { + return size, sku, fmt.Errorf("failed to parse disk spec %q: %w", spec, err) + } + for key, value := range diskmap { + switch key { + case "sku": + normalizedSku := strings.ToUpper(value) + foundSku := false + for _, validSku := range armcompute.PossibleDiskStorageAccountTypesValues() { + if strings.EqualFold(normalizedSku, string(validSku)) { + sku = validSku + foundSku = true + break + } + } + if !foundSku { + return size, sku, fmt.Errorf("unsupported disk sku: %s", value) + } + default: + return size, sku, fmt.Errorf("invalid key: %s", key) + } + } + return size, sku, nil +} diff --git a/mantle/platform/api/azure/instance.go b/mantle/platform/api/azure/instance.go index 8a3ef46171..d7d80d4830 100644 --- a/mantle/platform/api/azure/instance.go +++ b/mantle/platform/api/azure/instance.go @@ -30,6 +30,7 @@ import ( "github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/compute/armcompute" "github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/network/armnetwork" + "github.com/coreos/coreos-assembler/mantle/platform" "github.com/coreos/coreos-assembler/mantle/util" ) @@ -108,6 +109,7 @@ func (a *API) getVMParameters(name, userdata, sshkey, storageAccountURI string, return armcompute.VirtualMachine{ Name: &name, Location: &a.opts.Location, + Zones: []*string{&a.opts.AvailabilityZone}, Tags: map[string]*string{ "createdBy": to.Ptr("mantle"), }, @@ -142,7 +144,7 @@ func (a *API) getVMParameters(name, userdata, sshkey, storageAccountURI string, } } -func (a *API) CreateInstance(name, userdata, sshkey, resourceGroup, storageAccount string) (*Machine, error) { +func (a *API) CreateInstance(name, userdata, sshkey, resourceGroup, storageAccount string, opts platform.MachineOptions) (*Machine, error) { subnet, err := a.getSubnet(resourceGroup) if err != nil { return nil, fmt.Errorf("preparing network resources: %v", err) @@ -206,6 +208,18 @@ func (a *API) CreateInstance(name, userdata, sshkey, resourceGroup, storageAccou return nil, fmt.Errorf("couldn't get VM ID") } + for i, spec := range opts.AdditionalDisks { + size, sku, err := a.ParseDisk(spec) + if err != nil { + return nil, fmt.Errorf("error parsing additional disk: %v", err) + } + diskName := util.RandomName(fmt.Sprintf("disk-%d", i)) + err = a.CreateAndAttachDiskToInstance(*vm.Name, resourceGroup, diskName, sku, int32(size), int32(i)) + if err != nil { + return nil, fmt.Errorf("failed to attach disk to vm: %v", err) + } + } + publicaddr, privaddr, err := a.GetIPAddresses(*nic.Name, *ip.Name, resourceGroup) if err != nil { return nil, err @@ -272,3 +286,45 @@ func (a *API) GetConsoleOutput(name, resourceGroup, storageAccount string) ([]by return io.ReadAll(data) } + +func (a *API) CreateAndAttachDiskToInstance(instanceName, resourceGroup, diskName string, sku armcompute.DiskStorageAccountTypes, sizeGB int32, lun int32) error { + ctx := context.Background() + + vm, err := a.getInstance(instanceName, resourceGroup) + if err != nil { + return err + } + + diskID, err := a.CreateDisk(diskName, resourceGroup, sizeGB, sku) + if err != nil { + return err + } + + newDisk := armcompute.DataDisk{ + Lun: to.Ptr(lun), + Name: to.Ptr(diskName), + CreateOption: to.Ptr(armcompute.DiskCreateOptionTypesAttach), + ManagedDisk: &armcompute.ManagedDiskParameters{ + ID: to.Ptr(diskID), + }, + } + + if vm.Properties.StorageProfile.DataDisks == nil { + vm.Properties.StorageProfile.DataDisks = []*armcompute.DataDisk{} + } + vm.Properties.StorageProfile.DataDisks = append(vm.Properties.StorageProfile.DataDisks, &newDisk) + + poller, err := a.compClient.BeginUpdate(ctx, resourceGroup, instanceName, armcompute.VirtualMachineUpdate{ + Properties: &armcompute.VirtualMachineProperties{ + StorageProfile: &armcompute.StorageProfile{ + DataDisks: vm.Properties.StorageProfile.DataDisks, + }, + }, + }, nil) + if err != nil { + return fmt.Errorf("failed to attach disk to VM %s: %v", instanceName, err) + } + + _, err = poller.PollUntilDone(ctx, nil) + return err +} diff --git a/mantle/platform/api/azure/options.go b/mantle/platform/api/azure/options.go index ea6cd0eebf..cd773313e9 100644 --- a/mantle/platform/api/azure/options.go +++ b/mantle/platform/api/azure/options.go @@ -25,13 +25,14 @@ type Options struct { AzureCredentials string AzureSubscription string - DiskURI string - Publisher string - Offer string - Sku string - Version string - Size string - Location string + DiskURI string + Publisher string + Offer string + Sku string + Version string + Size string + Location string + AvailabilityZone string SubscriptionName string SubscriptionID string diff --git a/mantle/platform/machine/azure/cluster.go b/mantle/platform/machine/azure/cluster.go index 87d3d78a0c..2b1f846050 100644 --- a/mantle/platform/machine/azure/cluster.go +++ b/mantle/platform/machine/azure/cluster.go @@ -46,9 +46,6 @@ func (ac *cluster) NewMachine(userdata *conf.UserData) (platform.Machine, error) } func (ac *cluster) NewMachineWithOptions(userdata *conf.UserData, options platform.MachineOptions) (platform.Machine, error) { - if len(options.AdditionalDisks) > 0 { - return nil, errors.New("platform azure does not yet support additional disks") - } if options.MultiPathDisk { return nil, errors.New("platform azure does not support multipathed disks") } @@ -69,7 +66,7 @@ func (ac *cluster) NewMachineWithOptions(userdata *conf.UserData, options platfo return nil, err } - instance, err := ac.flight.api.CreateInstance(ac.vmname(), conf.String(), ac.sshKey, ac.ResourceGroup, ac.StorageAccount) + instance, err := ac.flight.api.CreateInstance(ac.vmname(), conf.String(), ac.sshKey, ac.ResourceGroup, ac.StorageAccount, options) if err != nil { return nil, err } From 02a2b620503ed379f805960560d68bab6e2c6b66 Mon Sep 17 00:00:00 2001 From: Michael Armijo Date: Wed, 14 May 2025 17:26:58 -0600 Subject: [PATCH 2/5] azure: add --azure-hyper-v-generation flag Add a new flag to allow specifying the Hyper-V generation (V1 or V2) when creating Azure images. This enables support for both Gen1 and Gen2 image creation. --- mantle/cmd/kola/kola.go | 2 ++ mantle/cmd/kola/options.go | 1 + mantle/cmd/ore/azure/azure.go | 3 +++ mantle/platform/api/azure/image.go | 2 +- mantle/platform/api/azure/options.go | 1 + 5 files changed, 8 insertions(+), 1 deletion(-) diff --git a/mantle/cmd/kola/kola.go b/mantle/cmd/kola/kola.go index 25a6bbc075..e9d27d6c9d 100644 --- a/mantle/cmd/kola/kola.go +++ b/mantle/cmd/kola/kola.go @@ -308,6 +308,7 @@ func writeProps() error { Location string `json:"location"` Size string `json:"size"` AvailabilityZone string `json:"availability_zone"` + HyperVGeneration string `json:"hyper_v_generation"` } type DO struct { Region string `json:"region"` @@ -364,6 +365,7 @@ func writeProps() error { Location: kola.AzureOptions.Location, Size: kola.AzureOptions.Size, AvailabilityZone: kola.AzureOptions.AvailabilityZone, + HyperVGeneration: kola.AzureOptions.HyperVGeneration, }, DO: DO{ Region: kola.DOOptions.Region, diff --git a/mantle/cmd/kola/options.go b/mantle/cmd/kola/options.go index c8804b0358..277035ef85 100644 --- a/mantle/cmd/kola/options.go +++ b/mantle/cmd/kola/options.go @@ -101,6 +101,7 @@ func init() { sv(&kola.AzureOptions.Location, "azure-location", "westus", "Azure location (default \"westus\"") sv(&kola.AzureOptions.Size, "azure-size", "Standard_D2_v2", "Azure machine size (default \"Standard_D2_v2\")") sv(&kola.AzureOptions.AvailabilityZone, "azure-availability-zone", "1", "Azure Availability Zone (default \"1\")") + sv(&kola.AzureOptions.HyperVGeneration, "azure-hyper-v-generation", "V1", "Azure Hyper-V Generation (default \"V1\")") // do-specific options sv(&kola.DOOptions.ConfigPath, "do-config-file", "", "DigitalOcean config file (default \"~/"+auth.DOConfigPath+"\")") diff --git a/mantle/cmd/ore/azure/azure.go b/mantle/cmd/ore/azure/azure.go index 63d1dd6b45..53e5f549e7 100644 --- a/mantle/cmd/ore/azure/azure.go +++ b/mantle/cmd/ore/azure/azure.go @@ -34,6 +34,7 @@ var ( azureCredentials string azureLocation string + azureHyperVGen string api *azure.API ) @@ -44,6 +45,7 @@ func init() { sv := Azure.PersistentFlags().StringVar sv(&azureCredentials, "azure-credentials", "", "Azure credentials file location (default \"~/"+auth.AzureCredentialsPath+"\")") sv(&azureLocation, "azure-location", "westus", "Azure location (default \"westus\")") + sv(&azureHyperVGen, "azure-hyper-v-generation", "V1", "Azure Hypervisor Generation") } func preauth(cmd *cobra.Command, args []string) error { @@ -52,6 +54,7 @@ func preauth(cmd *cobra.Command, args []string) error { a, err := azure.New(&azure.Options{ AzureCredentials: azureCredentials, Location: azureLocation, + HyperVGeneration: azureHyperVGen, }) if err != nil { plog.Fatalf("Failed to create Azure API: %v", err) diff --git a/mantle/platform/api/azure/image.go b/mantle/platform/api/azure/image.go index 2df2b10a00..7d2b588cc1 100644 --- a/mantle/platform/api/azure/image.go +++ b/mantle/platform/api/azure/image.go @@ -28,7 +28,7 @@ func (a *API) CreateImage(name, resourceGroup, blobURI string) (armcompute.Image Name: &name, Location: &a.opts.Location, Properties: &armcompute.ImageProperties{ - HyperVGeneration: to.Ptr(armcompute.HyperVGenerationTypesV1), + HyperVGeneration: to.Ptr(armcompute.HyperVGenerationTypes(a.opts.HyperVGeneration)), StorageProfile: &armcompute.ImageStorageProfile{ OSDisk: &armcompute.ImageOSDisk{ OSType: to.Ptr(armcompute.OperatingSystemTypesLinux), diff --git a/mantle/platform/api/azure/options.go b/mantle/platform/api/azure/options.go index cd773313e9..1aee5454ea 100644 --- a/mantle/platform/api/azure/options.go +++ b/mantle/platform/api/azure/options.go @@ -33,6 +33,7 @@ type Options struct { Size string Location string AvailabilityZone string + HyperVGeneration string SubscriptionName string SubscriptionID string From f056a44dbd6dcdece93fa2e688347e3033474a94 Mon Sep 17 00:00:00 2001 From: Michael Armijo Date: Wed, 14 May 2025 17:58:40 -0600 Subject: [PATCH 3/5] ore/azure: add options to create and delete Shared Image Gallery Images Add a new `create-gallery-image` ore command to Support creating images within Azure Shared Image Galleries (Gallery Images). The command creates image definitions and versions in Azure Shared Image Galleries. Gallery images can be created from either a blob URL or an existing managed image. A `--azure-publisher` flag is added to assign a publisher to the gallery image. `delete-gallery-image` is also added to delete individual gallery images or an entire Shared Image Gallery. --- mantle/cmd/ore/azure/azure.go | 3 + mantle/cmd/ore/azure/create-gallery-image.go | 95 +++++++++ mantle/cmd/ore/azure/delete-gallery-image.go | 77 ++++++++ mantle/platform/api/azure/api.go | 50 ++++- mantle/platform/api/azure/gallery.go | 195 +++++++++++++++++++ mantle/platform/api/azure/instance.go | 17 +- mantle/platform/api/azure/network.go | 73 ++++++- 7 files changed, 497 insertions(+), 13 deletions(-) create mode 100644 mantle/cmd/ore/azure/create-gallery-image.go create mode 100644 mantle/cmd/ore/azure/delete-gallery-image.go create mode 100644 mantle/platform/api/azure/gallery.go diff --git a/mantle/cmd/ore/azure/azure.go b/mantle/cmd/ore/azure/azure.go index 53e5f549e7..952930e714 100644 --- a/mantle/cmd/ore/azure/azure.go +++ b/mantle/cmd/ore/azure/azure.go @@ -35,6 +35,7 @@ var ( azureCredentials string azureLocation string azureHyperVGen string + azurePublisher string api *azure.API ) @@ -46,6 +47,7 @@ func init() { sv(&azureCredentials, "azure-credentials", "", "Azure credentials file location (default \"~/"+auth.AzureCredentialsPath+"\")") sv(&azureLocation, "azure-location", "westus", "Azure location (default \"westus\")") sv(&azureHyperVGen, "azure-hyper-v-generation", "V1", "Azure Hypervisor Generation") + sv(&azurePublisher, "azure-publisher", "CoreOS", "Azure image publisher") } func preauth(cmd *cobra.Command, args []string) error { @@ -55,6 +57,7 @@ func preauth(cmd *cobra.Command, args []string) error { AzureCredentials: azureCredentials, Location: azureLocation, HyperVGeneration: azureHyperVGen, + Publisher: azurePublisher, }) if err != nil { plog.Fatalf("Failed to create Azure API: %v", err) diff --git a/mantle/cmd/ore/azure/create-gallery-image.go b/mantle/cmd/ore/azure/create-gallery-image.go new file mode 100644 index 0000000000..15047c027e --- /dev/null +++ b/mantle/cmd/ore/azure/create-gallery-image.go @@ -0,0 +1,95 @@ +// Copyright 2025 Red Hat +// Copyright 2018 CoreOS, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package azure + +import ( + "encoding/json" + "fmt" + "os" + + "github.com/spf13/cobra" +) + +var ( + cmdCreateGalleryImage = &cobra.Command{ + Use: "create-gallery-image", + Short: "Create Azure Gallery image", + Long: "Create Azure Gallery image from a blob url", + RunE: runCreateGalleryImage, + Aliases: []string{"create-gallery-image-arm"}, + + SilenceUsage: true, + } + + galleryImageName string + galleryName string +) + +func init() { + sv := cmdCreateGalleryImage.Flags().StringVar + + sv(&galleryImageName, "gallery-image-name", "", "gallery image name") + sv(&galleryName, "gallery-name", "kola", "gallery name") + sv(&blobUrl, "image-blob", "", "source blob url") + sv(&resourceGroup, "resource-group", "kola", "resource group name") + + Azure.AddCommand(cmdCreateGalleryImage) +} + +func runCreateGalleryImage(cmd *cobra.Command, args []string) error { + if blobUrl == "" { + fmt.Fprintf(os.Stderr, "must supply --image-blob\n") + os.Exit(1) + } + + if err := api.SetupClients(); err != nil { + fmt.Fprintf(os.Stderr, "setting up clients: %v\n", err) + os.Exit(1) + } + + img, err := api.CreateImage(galleryImageName, resourceGroup, blobUrl) + if err != nil { + fmt.Fprintf(os.Stderr, "Couldn't create Azure image: %v\n", err) + os.Exit(1) + } + if img.ID == nil { + fmt.Fprintf(os.Stderr, "received nil image\n") + os.Exit(1) + } + sourceImageId := *img.ID + + galleryImage, err := api.CreateGalleryImage(galleryImageName, galleryName, resourceGroup, sourceImageId) + if err != nil { + fmt.Fprintf(os.Stderr, "Couldn't create Azure Shared Image Gallery image: %v\n", err) + os.Exit(1) + } + if galleryImage.ID == nil { + fmt.Fprintf(os.Stderr, "received nil gallery image\n") + os.Exit(1) + } + err = json.NewEncoder(os.Stdout).Encode(&struct { + ID *string + Location *string + }{ + ID: galleryImage.ID, + Location: galleryImage.Location, + }) + if err != nil { + fmt.Fprintf(os.Stderr, "Couldn't encode result: %v\n", err) + os.Exit(1) + } + return nil +} diff --git a/mantle/cmd/ore/azure/delete-gallery-image.go b/mantle/cmd/ore/azure/delete-gallery-image.go new file mode 100644 index 0000000000..d2b3aec9cb --- /dev/null +++ b/mantle/cmd/ore/azure/delete-gallery-image.go @@ -0,0 +1,77 @@ +// Copyright 2025 Red Hat +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package azure + +import ( + "fmt" + + "github.com/spf13/cobra" +) + +var ( + cmdDeleteGalleryImage = &cobra.Command{ + Use: "delete-gallery-image", + Short: "Delete Azure Gallery image", + Long: "Remove a Shared Image Gallery image from Azure.", + RunE: runDeleteGalleryImage, + Aliases: []string{"delete-gallery-image-arm"}, + + SilenceUsage: true, + } + + deleteGallery bool +) + +func init() { + sv := cmdDeleteGalleryImage.Flags().StringVar + bv := cmdDeleteGalleryImage.Flags().BoolVar + + sv(&imageName, "gallery-image-name", "", "gallery image name") + sv(&resourceGroup, "resource-group", "kola", "resource group name") + sv(&galleryName, "gallery-name", "kola", "gallery name") + bv(&deleteGallery, "delete-entire-gallery", false, "delete entire gallery") + + Azure.AddCommand(cmdDeleteGalleryImage) +} + +func runDeleteGalleryImage(cmd *cobra.Command, args []string) error { + if err := api.SetupClients(); err != nil { + return fmt.Errorf("setting up clients: %v\n", err) + } + + if deleteGallery { + err := api.DeleteGallery(galleryName, resourceGroup) + if err != nil { + return fmt.Errorf("Couldn't delete gallery: %v\n", err) + } + plog.Printf("Gallery %q in resource group %q removed", galleryName, resourceGroup) + return nil + } + + err := api.DeleteGalleryImage(imageName, resourceGroup, galleryName) + if err != nil { + return fmt.Errorf("Couldn't delete gallery image: %v\n", err) + } + + // Gallery image versions are backed by managed images with the same name, + // so we can easily identify and delete them together. + err = api.DeleteImage(imageName, resourceGroup) + if err != nil { + return fmt.Errorf("Couldn't delete image: %v\n", err) + } + + plog.Printf("Image %q in gallery %q in resource group %q removed", imageName, galleryName, resourceGroup) + return nil +} diff --git a/mantle/platform/api/azure/api.go b/mantle/platform/api/azure/api.go index b13bfdc924..f09ca30b0b 100644 --- a/mantle/platform/api/azure/api.go +++ b/mantle/platform/api/azure/api.go @@ -31,16 +31,21 @@ import ( ) type API struct { - azIdCred *azidentity.DefaultAzureCredential - rgClient *armresources.ResourceGroupsClient - imgClient *armcompute.ImagesClient - compClient *armcompute.VirtualMachinesClient - netClient *armnetwork.VirtualNetworksClient - subClient *armnetwork.SubnetsClient - ipClient *armnetwork.PublicIPAddressesClient - intClient *armnetwork.InterfacesClient - accClient *armstorage.AccountsClient - opts *Options + azIdCred *azidentity.DefaultAzureCredential + rgClient *armresources.ResourceGroupsClient + imgClient *armcompute.ImagesClient + compClient *armcompute.VirtualMachinesClient + galClient *armcompute.GalleriesClient + galImgClient *armcompute.GalleryImagesClient + galImgVerClient *armcompute.GalleryImageVersionsClient + diskClient *armcompute.DisksClient + netClient *armnetwork.VirtualNetworksClient + subClient *armnetwork.SubnetsClient + ipClient *armnetwork.PublicIPAddressesClient + intClient *armnetwork.InterfacesClient + nsgClient *armnetwork.SecurityGroupsClient + accClient *armstorage.AccountsClient + opts *Options } // New creates a new Azure client. If no publish settings file is provided or @@ -89,6 +94,26 @@ func (a *API) SetupClients() error { return err } + a.galClient, err = armcompute.NewGalleriesClient(a.opts.SubscriptionID, a.azIdCred, nil) + if err != nil { + return err + } + + a.galImgClient, err = armcompute.NewGalleryImagesClient(a.opts.SubscriptionID, a.azIdCred, nil) + if err != nil { + return err + } + + a.galImgVerClient, err = armcompute.NewGalleryImageVersionsClient(a.opts.SubscriptionID, a.azIdCred, nil) + if err != nil { + return err + } + + a.diskClient, err = armcompute.NewDisksClient(a.opts.SubscriptionID, a.azIdCred, nil) + if err != nil { + return err + } + a.netClient, err = armnetwork.NewVirtualNetworksClient(a.opts.SubscriptionID, a.azIdCred, nil) if err != nil { return err @@ -109,6 +134,11 @@ func (a *API) SetupClients() error { return err } + a.nsgClient, err = armnetwork.NewSecurityGroupsClient(a.opts.SubscriptionID, a.azIdCred, nil) + if err != nil { + return err + } + a.accClient, err = armstorage.NewAccountsClient(a.opts.SubscriptionID, a.azIdCred, nil) return err } diff --git a/mantle/platform/api/azure/gallery.go b/mantle/platform/api/azure/gallery.go new file mode 100644 index 0000000000..a32322df13 --- /dev/null +++ b/mantle/platform/api/azure/gallery.go @@ -0,0 +1,195 @@ +// Copyright 2025 Red Hat +// Copyright 2016 CoreOS, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package azure + +import ( + "context" + "fmt" + "strings" + "time" + + "github.com/Azure/azure-sdk-for-go/sdk/azcore/to" + "github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/compute/armcompute" + + "github.com/coreos/coreos-assembler/mantle/util" +) + +func (a *API) CreateGalleryImage(name, galleryName, resourceGroup, sourceImageID string) (armcompute.GalleryImageVersion, error) { + ctx := context.Background() + + // Ensure the Azure Shared Image Gallery exists. BeginCreateOrUpdate will create the gallery + // in the specified resource group if it doesn't already exist, or update it if it does. + // Since no properties are being changed here, this acts as a no-op if the gallery does exist. + // Note: the gallery's location is immutable. If a gallery with the same name exists in a different + // location within the same resource group, the operation will fail. + galleryPoller, err := a.galClient.BeginCreateOrUpdate(ctx, resourceGroup, galleryName, armcompute.Gallery{ + Location: &a.opts.Location, + }, nil) + if err != nil { + return armcompute.GalleryImageVersion{}, err + } + _, err = galleryPoller.PollUntilDone(context.Background(), nil) + if err != nil { + return armcompute.GalleryImageVersion{}, err + } + + // enable NVMe support for Gen2 images only. NVMe support is not available on Gen1 images. + // DiskControllerTypes is set to SCSI by default for Gen1 images. + var galleryImageFeatures []*armcompute.GalleryImageFeature + if strings.EqualFold(a.opts.HyperVGeneration, string(armcompute.HyperVGenerationV2)) { + galleryImageFeatures = []*armcompute.GalleryImageFeature{ + { + Name: to.Ptr("DiskControllerTypes"), + Value: to.Ptr("SCSI,NVMe"), + }, + } + } else { + galleryImageFeatures = nil + } + + // Create a Gallery Image Definition with the specified Hyper-V generation (V1 or V2). + galleryImagePoller, err := a.galImgClient.BeginCreateOrUpdate(ctx, resourceGroup, galleryName, name, armcompute.GalleryImage{ + Location: &a.opts.Location, + Properties: &armcompute.GalleryImageProperties{ + OSState: to.Ptr(armcompute.OperatingSystemStateTypesGeneralized), + OSType: to.Ptr(armcompute.OperatingSystemTypesLinux), + HyperVGeneration: to.Ptr(armcompute.HyperVGeneration(a.opts.HyperVGeneration)), + Identifier: &armcompute.GalleryImageIdentifier{ + Publisher: &a.opts.Publisher, + Offer: to.Ptr(name), + SKU: to.Ptr(util.RandomName("sku")), + }, + Features: galleryImageFeatures, + }, + }, nil) + if err != nil { + return armcompute.GalleryImageVersion{}, err + } + _, err = galleryImagePoller.PollUntilDone(context.Background(), nil) + if err != nil { + return armcompute.GalleryImageVersion{}, err + } + + // Create a Gallery Image Version + versionName := "1.0.0" + imageVersionPoller, err := a.galImgVerClient.BeginCreateOrUpdate(ctx, resourceGroup, galleryName, name, versionName, armcompute.GalleryImageVersion{ + Location: &a.opts.Location, + Properties: &armcompute.GalleryImageVersionProperties{ + StorageProfile: &armcompute.GalleryImageVersionStorageProfile{ + Source: &armcompute.GalleryArtifactVersionSource{ + ID: to.Ptr(sourceImageID), + }, + }, + }, + }, nil) + if err != nil { + return armcompute.GalleryImageVersion{}, err + } + imageVersionResponse, err := imageVersionPoller.PollUntilDone(context.Background(), nil) + if err != nil { + return armcompute.GalleryImageVersion{}, err + } + + return imageVersionResponse.GalleryImageVersion, nil +} + +func (a *API) DeleteGalleryImage(imageName, resourceGroup, galleryName string) error { + ctx := context.Background() + + timeout := 5 * time.Minute + delay := 5 * time.Second + // There is sometimes a delay in the azure backend where deleted gallery images versions + // still show within the image definition causing a failure during image deletion. We'll + // retry the delete command again until a specified timeout to ensure the image is deleted. + err := util.RetryUntilTimeout(timeout, delay, func() error { + // Find all image versions in the image definition and delete them. + // Gallery images can only be deleted if they have no nested resources. + versionPager := a.galImgVerClient.NewListByGalleryImagePager(resourceGroup, galleryName, imageName, nil) + for versionPager.More() { + versionPage, err := versionPager.NextPage(ctx) + if err != nil { + return fmt.Errorf("failed to list image versions for %s: %v", imageName, err) + } + for _, version := range versionPage.Value { + poller, err := a.galImgVerClient.BeginDelete(ctx, resourceGroup, galleryName, imageName, *version.Name, nil) + if err != nil { + return err + } + _, err = poller.PollUntilDone(ctx, nil) + if err != nil { + return err + } + } + } + + // delete the gallery image + poller, err := a.galImgClient.BeginDelete(ctx, resourceGroup, galleryName, imageName, nil) + if err != nil { + return err + } + _, err = poller.PollUntilDone(ctx, nil) + if err != nil { + return err + } + + return nil + }) + + return err + +} + +func (a *API) DeleteGallery(galleryName, resourceGroup string) error { + ctx := context.Background() + + timeout := 10 * time.Minute + delay := 5 * time.Second + // There is sometimes a delay in the azure backend where deleted gallery images still show + // within the gallery causing a failure during gallery deletion. We'll retry the delete + // command again until a specified timeout to ensure the gallery is deleted. + err := util.RetryUntilTimeout(timeout, delay, func() error { + // Find all images in the gallery and delete them. + // Galleries can only be deleted if they have no nested resources. + imagePager := a.galImgClient.NewListByGalleryPager(resourceGroup, galleryName, nil) + for imagePager.More() { + page, err := imagePager.NextPage(ctx) + if err != nil { + return fmt.Errorf("failed to get image definitions") + } + for _, image := range page.Value { + err := a.DeleteGalleryImage(*image.Name, resourceGroup, galleryName) + if err != nil { + return fmt.Errorf("Couldn't delete gallery image: %v\n", err) + } + } + } + + // delete the gallery + poller, err := a.galClient.BeginDelete(ctx, resourceGroup, galleryName, nil) + if err != nil { + return err + } + _, err = poller.PollUntilDone(ctx, nil) + if err != nil { + return err + } + + return nil + }) + + return err + +} diff --git a/mantle/platform/api/azure/instance.go b/mantle/platform/api/azure/instance.go index d7d80d4830..8ec5cf94f5 100644 --- a/mantle/platform/api/azure/instance.go +++ b/mantle/platform/api/azure/instance.go @@ -24,6 +24,7 @@ import ( "math" "math/big" "regexp" + "strings" "time" "github.com/Azure/azure-sdk-for-go/sdk/azcore/to" @@ -106,6 +107,15 @@ func (a *API) getVMParameters(name, userdata, sshkey, storageAccountURI string, Version: &a.opts.Version, } } + // UltraSSDEnabled=true is required for NVMe support on Gen2 VMs + var additionalCapabilities *armcompute.AdditionalCapabilities + if strings.EqualFold(a.opts.HyperVGeneration, string(armcompute.HyperVGenerationV2)) { + additionalCapabilities = &armcompute.AdditionalCapabilities{ + UltraSSDEnabled: to.Ptr(true), + } + } else { + additionalCapabilities = nil + } return armcompute.VirtualMachine{ Name: &name, Location: &a.opts.Location, @@ -140,6 +150,7 @@ func (a *API) getVMParameters(name, userdata, sshkey, storageAccountURI string, StorageURI: &storageAccountURI, }, }, + AdditionalCapabilities: additionalCapabilities, }, } } @@ -157,8 +168,12 @@ func (a *API) CreateInstance(name, userdata, sshkey, resourceGroup, storageAccou if ip.Name == nil { return nil, fmt.Errorf("couldn't get public IP name") } + nsg, err := a.CreateNSG(resourceGroup) + if err != nil { + return nil, fmt.Errorf("creating network security group: %v", err) + } - nic, err := a.createNIC(ip, &subnet, resourceGroup) + nic, err := a.createNIC(ip, &subnet, &nsg, resourceGroup) if err != nil { return nil, fmt.Errorf("creating nic: %v", err) } diff --git a/mantle/platform/api/azure/network.go b/mantle/platform/api/azure/network.go index f0e03bcebd..f65ad5ad31 100644 --- a/mantle/platform/api/azure/network.go +++ b/mantle/platform/api/azure/network.go @@ -18,8 +18,10 @@ package azure import ( "context" "fmt" + "strings" "github.com/Azure/azure-sdk-for-go/sdk/azcore/to" + "github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/compute/armcompute" "github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/network/armnetwork" "github.com/coreos/coreos-assembler/mantle/util" @@ -84,8 +86,36 @@ func (a *API) createPublicIP(resourceGroup string) (armnetwork.PublicIPAddress, name := util.RandomName("ip") ctx := context.Background() + var ipSKU *armnetwork.PublicIPAddressSKU + var ipProperties *armnetwork.PublicIPAddressPropertiesFormat + var ipZones []*string + + // set SKU=Standard, Allocation Method=Static and Availability Zone on public IPs when creating Gen2 images + if strings.EqualFold(a.opts.HyperVGeneration, string(armcompute.HyperVGenerationV2)) { + ipSKU = &armnetwork.PublicIPAddressSKU{ + Name: to.Ptr(armnetwork.PublicIPAddressSKUNameStandard), + } + ipProperties = &armnetwork.PublicIPAddressPropertiesFormat{ + PublicIPAllocationMethod: to.Ptr(armnetwork.IPAllocationMethodStatic), + } + ipZones = []*string{to.Ptr(a.opts.AvailabilityZone)} + // gen 1 + } else { + ipSKU = &armnetwork.PublicIPAddressSKU{ + Name: to.Ptr(armnetwork.PublicIPAddressSKUNameBasic), + } + ipProperties = &armnetwork.PublicIPAddressPropertiesFormat{ + PublicIPAllocationMethod: to.Ptr(armnetwork.IPAllocationMethodDynamic), + } + // No Zones for Gen1 + ipZones = nil + } + poller, err := a.ipClient.BeginCreateOrUpdate(ctx, resourceGroup, name, armnetwork.PublicIPAddress{ - Location: to.Ptr(a.opts.Location), + Location: to.Ptr(a.opts.Location), + Zones: ipZones, + SKU: ipSKU, + Properties: ipProperties, }, nil) if err != nil { return armnetwork.PublicIPAddress{}, err @@ -145,7 +175,7 @@ func (a *API) GetPrivateIP(interfaceName, resourceGroup string) (string, error) return "", fmt.Errorf("no private configurations found") } -func (a *API) createNIC(ip armnetwork.PublicIPAddress, subnet *armnetwork.Subnet, resourceGroup string) (armnetwork.Interface, error) { +func (a *API) createNIC(ip armnetwork.PublicIPAddress, subnet *armnetwork.Subnet, nsg *armnetwork.SecurityGroup, resourceGroup string) (armnetwork.Interface, error) { name := util.RandomName("nic") ipconf := util.RandomName("nic-ipconf") ctx := context.Background() @@ -153,6 +183,7 @@ func (a *API) createNIC(ip armnetwork.PublicIPAddress, subnet *armnetwork.Subnet poller, err := a.intClient.BeginCreateOrUpdate(ctx, resourceGroup, name, armnetwork.Interface{ Location: to.Ptr(a.opts.Location), Properties: &armnetwork.InterfacePropertiesFormat{ + NetworkSecurityGroup: nsg, IPConfigurations: []*armnetwork.InterfaceIPConfiguration{ { Name: to.Ptr(ipconf), @@ -178,3 +209,41 @@ func (a *API) createNIC(ip armnetwork.PublicIPAddress, subnet *armnetwork.Subnet return nic, nil } + +func (a *API) CreateNSG(resourceGroup string) (armnetwork.SecurityGroup, error) { + name := util.RandomName("nsg") + ctx := context.Background() + + sshRule := &armnetwork.SecurityRule{ + Name: to.Ptr("allow_ssh"), + Properties: &armnetwork.SecurityRulePropertiesFormat{ + Access: to.Ptr(armnetwork.SecurityRuleAccessAllow), + Direction: to.Ptr(armnetwork.SecurityRuleDirectionInbound), + Protocol: to.Ptr(armnetwork.SecurityRuleProtocolTCP), + SourcePortRange: to.Ptr("*"), + DestinationPortRange: to.Ptr("22"), + SourceAddressPrefix: to.Ptr("*"), + DestinationAddressPrefix: to.Ptr("*"), + Priority: to.Ptr(int32(1000)), + }, + } + + nsgParams := armnetwork.SecurityGroup{ + Location: to.Ptr(a.opts.Location), + Properties: &armnetwork.SecurityGroupPropertiesFormat{ + SecurityRules: []*armnetwork.SecurityRule{sshRule}, + }, + } + + poller, err := a.nsgClient.BeginCreateOrUpdate(ctx, resourceGroup, name, nsgParams, nil) + if err != nil { + return armnetwork.SecurityGroup{}, err + } + + resp, err := poller.PollUntilDone(ctx, nil) + if err != nil { + return armnetwork.SecurityGroup{}, err + } + + return resp.SecurityGroup, nil +} From 5b40b1f5ae9a039f1214a69d2b246688698b8fa0 Mon Sep 17 00:00:00 2001 From: Michael Armijo Date: Fri, 16 May 2025 17:11:02 -0600 Subject: [PATCH 4/5] mantle/kola: add InstanceType to PlatformOptions for external tests Add an `InstanceType` field to `PlatformOptions` to allow external tests to override the instance type used in `kola run`. This is useful for cases where a specific test needs to run on a different (potentially more expensive) instance type. Support for this is currently limited to the Azure Platform. Also, fix the `MultiPathDisk` check for the qemu-iso platform. The check is now correctly performed in the `NewMachineWithOptions` function, since `MultiPathDisk` is part of `platform.MachineOptions`. --- mantle/kola/harness.go | 3 +++ mantle/kola/register/register.go | 4 ++++ mantle/platform/api/azure/instance.go | 16 +++++++++++++--- mantle/platform/machine/aws/cluster.go | 4 ++++ mantle/platform/machine/do/cluster.go | 3 +++ mantle/platform/machine/esx/cluster.go | 3 +++ mantle/platform/machine/gcloud/cluster.go | 3 +++ mantle/platform/machine/openstack/cluster.go | 3 +++ mantle/platform/machine/qemu/cluster.go | 3 +++ mantle/platform/machine/qemuiso/cluster.go | 9 ++++++--- mantle/platform/platform.go | 1 + 11 files changed, 46 insertions(+), 6 deletions(-) diff --git a/mantle/kola/harness.go b/mantle/kola/harness.go index 1a44fb2c32..e3e0876154 100644 --- a/mantle/kola/harness.go +++ b/mantle/kola/harness.go @@ -1027,6 +1027,7 @@ type externalTestMeta struct { Conflicts []string `json:"conflicts" yaml:"conflicts"` AllowConfigWarnings bool `json:"allowConfigWarnings" yaml:"allowConfigWarnings"` NoInstanceCreds bool `json:"noInstanceCreds" yaml:"noInstanceCreds"` + InstanceType string `json:"instanceType" yaml:"instanceType"` Description string `json:"description" yaml:"description"` } @@ -1236,6 +1237,7 @@ ExecStart=%s AdditionalNics: targetMeta.AdditionalNics, AppendKernelArgs: targetMeta.AppendKernelArgs, AppendFirstbootKernelArgs: targetMeta.AppendFirstbootKernelArgs, + InstanceType: targetMeta.InstanceType, NonExclusive: !targetMeta.Exclusive, Conflicts: targetMeta.Conflicts, @@ -1757,6 +1759,7 @@ func runTest(h *harness.H, t *register.Test, pltfrm string, flight platform.Flig AppendKernelArgs: t.AppendKernelArgs, AppendFirstbootKernelArgs: t.AppendFirstbootKernelArgs, SkipStartMachine: true, + InstanceType: t.InstanceType, } // Providers sometimes fail to bring up a machine within a diff --git a/mantle/kola/register/register.go b/mantle/kola/register/register.go index a2ede2e674..454f810429 100644 --- a/mantle/kola/register/register.go +++ b/mantle/kola/register/register.go @@ -117,6 +117,10 @@ type Test struct { // Conflicts is non-empty iff nonexclusive is true // Contains the tests that conflict with this particular test Conflicts []string + + // If provided, this test will be run on the target instance type. + // This overrides the instance type set with `kola run` + InstanceType string } // Registered tests that run as part of `kola run` live here. Mapping of names diff --git a/mantle/platform/api/azure/instance.go b/mantle/platform/api/azure/instance.go index 8ec5cf94f5..9356b5dd59 100644 --- a/mantle/platform/api/azure/instance.go +++ b/mantle/platform/api/azure/instance.go @@ -51,7 +51,7 @@ func (a *API) getInstance(name, resourceGroup string) (armcompute.VirtualMachine return resp.VirtualMachine, nil } -func (a *API) getVMParameters(name, userdata, sshkey, storageAccountURI string, ip armnetwork.PublicIPAddress, nic armnetwork.Interface) armcompute.VirtualMachine { +func (a *API) getVMParameters(name, userdata, sshkey, storageAccountURI, size string, ip armnetwork.PublicIPAddress, nic armnetwork.Interface) armcompute.VirtualMachine { // Azure requires that either a username/password be set or an SSH key. // @@ -125,7 +125,7 @@ func (a *API) getVMParameters(name, userdata, sshkey, storageAccountURI string, }, Properties: &armcompute.VirtualMachineProperties{ HardwareProfile: &armcompute.HardwareProfile{ - VMSize: to.Ptr(armcompute.VirtualMachineSizeTypes(a.opts.Size)), + VMSize: to.Ptr(armcompute.VirtualMachineSizeTypes(size)), }, StorageProfile: &armcompute.StorageProfile{ ImageReference: imgRef, @@ -181,7 +181,17 @@ func (a *API) CreateInstance(name, userdata, sshkey, resourceGroup, storageAccou return nil, fmt.Errorf("couldn't get NIC name") } - vmParams := a.getVMParameters(name, userdata, sshkey, fmt.Sprintf("https://%s.blob.core.windows.net/", storageAccount), ip, nic) + // Override the vm size with the one specified in the external kola test config. + // This is useful for cases where a specific test needs to run on a different + // (potentially more expensive) instance type. + var size string + if opts.InstanceType != "" { + size = opts.InstanceType + } else { + size = a.opts.Size + } + + vmParams := a.getVMParameters(name, userdata, sshkey, fmt.Sprintf("https://%s.blob.core.windows.net/", storageAccount), size, ip, nic) ctx, cancel := context.WithTimeout(context.Background(), 10*time.Minute) defer cancel() diff --git a/mantle/platform/machine/aws/cluster.go b/mantle/platform/machine/aws/cluster.go index fd0f60ada0..5a16412051 100644 --- a/mantle/platform/machine/aws/cluster.go +++ b/mantle/platform/machine/aws/cluster.go @@ -57,6 +57,10 @@ func (ac *cluster) NewMachineWithOptions(userdata *conf.UserData, options platfo return nil, errors.New("platform aws does not support appending firstboot kernel arguments") } + if options.InstanceType != "" { + return nil, errors.New("platform aws does not support changing instance types") + } + conf, err := ac.RenderUserData(userdata, map[string]string{ "$public_ipv4": "${COREOS_EC2_IPV4_PUBLIC}", "$private_ipv4": "${COREOS_EC2_IPV4_LOCAL}", diff --git a/mantle/platform/machine/do/cluster.go b/mantle/platform/machine/do/cluster.go index 87d06569d1..77d39c234b 100644 --- a/mantle/platform/machine/do/cluster.go +++ b/mantle/platform/machine/do/cluster.go @@ -52,6 +52,9 @@ func (dc *cluster) NewMachineWithOptions(userdata *conf.UserData, options platfo if options.AppendFirstbootKernelArgs != "" { return nil, errors.New("platform do does not support appending firstboot kernel arguments") } + if options.InstanceType != "" { + return nil, errors.New("platform do does not support changing instance types") + } conf, err := dc.RenderUserData(userdata, map[string]string{ "$public_ipv4": "${COREOS_DIGITALOCEAN_IPV4_PUBLIC_0}", diff --git a/mantle/platform/machine/esx/cluster.go b/mantle/platform/machine/esx/cluster.go index 2a7c7f6838..86135e4700 100644 --- a/mantle/platform/machine/esx/cluster.go +++ b/mantle/platform/machine/esx/cluster.go @@ -58,6 +58,9 @@ func (ec *cluster) NewMachineWithOptions(userdata *platformConf.UserData, option if options.AppendFirstbootKernelArgs != "" { return nil, errors.New("platform esx does not support appending firstboot kernel arguments") } + if options.InstanceType != "" { + return nil, errors.New("platform esx does not support changing instance types") + } conf, err := ec.RenderUserData(userdata, map[string]string{ "$public_ipv4": "${COREOS_ESX_IPV4_PUBLIC_0}", diff --git a/mantle/platform/machine/gcloud/cluster.go b/mantle/platform/machine/gcloud/cluster.go index 5a3ed5f121..0eabc741f1 100644 --- a/mantle/platform/machine/gcloud/cluster.go +++ b/mantle/platform/machine/gcloud/cluster.go @@ -49,6 +49,9 @@ func (gc *cluster) NewMachineWithOptions(userdata *conf.UserData, options platfo if options.AppendFirstbootKernelArgs != "" { return nil, errors.New("platform gcp does not support appending firstboot kernel arguments") } + if options.InstanceType != "" { + return nil, errors.New("platform gcp does not support changing instance types") + } conf, err := gc.RenderUserData(userdata, map[string]string{ "$public_ipv4": "${COREOS_GCE_IP_EXTERNAL_0}", diff --git a/mantle/platform/machine/openstack/cluster.go b/mantle/platform/machine/openstack/cluster.go index a2aced336e..60255fe63a 100644 --- a/mantle/platform/machine/openstack/cluster.go +++ b/mantle/platform/machine/openstack/cluster.go @@ -50,6 +50,9 @@ func (oc *cluster) NewMachineWithOptions(userdata *conf.UserData, options platfo if options.AppendFirstbootKernelArgs != "" { return nil, errors.New("platform openstack does not support appending firstboot kernel arguments") } + if options.InstanceType != "" { + return nil, errors.New("platform openstack does not support changing instance types") + } conf, err := oc.RenderUserData(userdata, map[string]string{ "$public_ipv4": "${COREOS_OPENSTACK_IPV4_PUBLIC}", diff --git a/mantle/platform/machine/qemu/cluster.go b/mantle/platform/machine/qemu/cluster.go index c23c6ffef8..817c733e24 100644 --- a/mantle/platform/machine/qemu/cluster.go +++ b/mantle/platform/machine/qemu/cluster.go @@ -49,6 +49,9 @@ func (qc *Cluster) NewMachine(userdata *conf.UserData) (platform.Machine, error) } func (qc *Cluster) NewMachineWithOptions(userdata *conf.UserData, options platform.MachineOptions) (platform.Machine, error) { + if options.InstanceType != "" { + return nil, errors.New("platform qemu does not support changing instance types") + } return qc.NewMachineWithQemuOptions(userdata, platform.QemuMachineOptions{ MachineOptions: options, }) diff --git a/mantle/platform/machine/qemuiso/cluster.go b/mantle/platform/machine/qemuiso/cluster.go index 9223192149..b14df90ff7 100644 --- a/mantle/platform/machine/qemuiso/cluster.go +++ b/mantle/platform/machine/qemuiso/cluster.go @@ -47,15 +47,18 @@ func (qc *Cluster) NewMachine(userdata *conf.UserData) (platform.Machine, error) } func (qc *Cluster) NewMachineWithOptions(userdata *conf.UserData, options platform.MachineOptions) (platform.Machine, error) { + if options.InstanceType != "" { + return nil, errors.New("platform qemu-iso does not support changing instance types") + } + if options.MultiPathDisk { + return nil, errors.New("platform qemu-iso does not support multipathed primary disks") + } return qc.NewMachineWithQemuOptions(userdata, platform.QemuMachineOptions{ MachineOptions: options, }) } func (qc *Cluster) NewMachineWithQemuOptions(userdata *conf.UserData, options platform.QemuMachineOptions) (platform.Machine, error) { - if options.MultiPathDisk { - return nil, errors.New("platform qemu-iso does not support multipathed primary disks") - } id := uuid.New() dir := filepath.Join(qc.RuntimeConf().OutputDir, id) diff --git a/mantle/platform/platform.go b/mantle/platform/platform.go index 591fa60c4e..22461fbb92 100644 --- a/mantle/platform/platform.go +++ b/mantle/platform/platform.go @@ -163,6 +163,7 @@ type MachineOptions struct { AppendKernelArgs string AppendFirstbootKernelArgs string SkipStartMachine bool // Skip platform.StartMachine on machine bringup + InstanceType string } // SystemdDropin is a userdata type agnostic struct representing a systemd dropin From ecc856985925ce3511d84d1291ada70f282a52bc Mon Sep 17 00:00:00 2001 From: Michael Armijo Date: Wed, 28 May 2025 15:08:36 -0600 Subject: [PATCH 5/5] azure: only build and run Hyper-V Gen2 images Remove the option to specify the Hyper-V Generation during image creation and when running instances through kola. Instead, only build traditional and gallery images using Hyper-V Gen2. --- mantle/cmd/kola/kola.go | 2 -- mantle/cmd/kola/options.go | 1 - mantle/cmd/ore/azure/azure.go | 3 --- mantle/platform/api/azure/gallery.go | 18 ++++++------------ mantle/platform/api/azure/image.go | 2 +- mantle/platform/api/azure/instance.go | 10 ++-------- mantle/platform/api/azure/network.go | 26 ++++++-------------------- mantle/platform/api/azure/options.go | 1 - 8 files changed, 15 insertions(+), 48 deletions(-) diff --git a/mantle/cmd/kola/kola.go b/mantle/cmd/kola/kola.go index e9d27d6c9d..25a6bbc075 100644 --- a/mantle/cmd/kola/kola.go +++ b/mantle/cmd/kola/kola.go @@ -308,7 +308,6 @@ func writeProps() error { Location string `json:"location"` Size string `json:"size"` AvailabilityZone string `json:"availability_zone"` - HyperVGeneration string `json:"hyper_v_generation"` } type DO struct { Region string `json:"region"` @@ -365,7 +364,6 @@ func writeProps() error { Location: kola.AzureOptions.Location, Size: kola.AzureOptions.Size, AvailabilityZone: kola.AzureOptions.AvailabilityZone, - HyperVGeneration: kola.AzureOptions.HyperVGeneration, }, DO: DO{ Region: kola.DOOptions.Region, diff --git a/mantle/cmd/kola/options.go b/mantle/cmd/kola/options.go index 277035ef85..c8804b0358 100644 --- a/mantle/cmd/kola/options.go +++ b/mantle/cmd/kola/options.go @@ -101,7 +101,6 @@ func init() { sv(&kola.AzureOptions.Location, "azure-location", "westus", "Azure location (default \"westus\"") sv(&kola.AzureOptions.Size, "azure-size", "Standard_D2_v2", "Azure machine size (default \"Standard_D2_v2\")") sv(&kola.AzureOptions.AvailabilityZone, "azure-availability-zone", "1", "Azure Availability Zone (default \"1\")") - sv(&kola.AzureOptions.HyperVGeneration, "azure-hyper-v-generation", "V1", "Azure Hyper-V Generation (default \"V1\")") // do-specific options sv(&kola.DOOptions.ConfigPath, "do-config-file", "", "DigitalOcean config file (default \"~/"+auth.DOConfigPath+"\")") diff --git a/mantle/cmd/ore/azure/azure.go b/mantle/cmd/ore/azure/azure.go index 952930e714..11886b0063 100644 --- a/mantle/cmd/ore/azure/azure.go +++ b/mantle/cmd/ore/azure/azure.go @@ -34,7 +34,6 @@ var ( azureCredentials string azureLocation string - azureHyperVGen string azurePublisher string api *azure.API @@ -46,7 +45,6 @@ func init() { sv := Azure.PersistentFlags().StringVar sv(&azureCredentials, "azure-credentials", "", "Azure credentials file location (default \"~/"+auth.AzureCredentialsPath+"\")") sv(&azureLocation, "azure-location", "westus", "Azure location (default \"westus\")") - sv(&azureHyperVGen, "azure-hyper-v-generation", "V1", "Azure Hypervisor Generation") sv(&azurePublisher, "azure-publisher", "CoreOS", "Azure image publisher") } @@ -56,7 +54,6 @@ func preauth(cmd *cobra.Command, args []string) error { a, err := azure.New(&azure.Options{ AzureCredentials: azureCredentials, Location: azureLocation, - HyperVGeneration: azureHyperVGen, Publisher: azurePublisher, }) if err != nil { diff --git a/mantle/platform/api/azure/gallery.go b/mantle/platform/api/azure/gallery.go index a32322df13..4ee0c8266b 100644 --- a/mantle/platform/api/azure/gallery.go +++ b/mantle/platform/api/azure/gallery.go @@ -18,7 +18,6 @@ package azure import ( "context" "fmt" - "strings" "time" "github.com/Azure/azure-sdk-for-go/sdk/azcore/to" @@ -48,16 +47,11 @@ func (a *API) CreateGalleryImage(name, galleryName, resourceGroup, sourceImageID // enable NVMe support for Gen2 images only. NVMe support is not available on Gen1 images. // DiskControllerTypes is set to SCSI by default for Gen1 images. - var galleryImageFeatures []*armcompute.GalleryImageFeature - if strings.EqualFold(a.opts.HyperVGeneration, string(armcompute.HyperVGenerationV2)) { - galleryImageFeatures = []*armcompute.GalleryImageFeature{ - { - Name: to.Ptr("DiskControllerTypes"), - Value: to.Ptr("SCSI,NVMe"), - }, - } - } else { - galleryImageFeatures = nil + galleryImageFeatures := []*armcompute.GalleryImageFeature{ + { + Name: to.Ptr("DiskControllerTypes"), + Value: to.Ptr("SCSI,NVMe"), + }, } // Create a Gallery Image Definition with the specified Hyper-V generation (V1 or V2). @@ -66,7 +60,7 @@ func (a *API) CreateGalleryImage(name, galleryName, resourceGroup, sourceImageID Properties: &armcompute.GalleryImageProperties{ OSState: to.Ptr(armcompute.OperatingSystemStateTypesGeneralized), OSType: to.Ptr(armcompute.OperatingSystemTypesLinux), - HyperVGeneration: to.Ptr(armcompute.HyperVGeneration(a.opts.HyperVGeneration)), + HyperVGeneration: to.Ptr(armcompute.HyperVGeneration(armcompute.HyperVGenerationV2)), Identifier: &armcompute.GalleryImageIdentifier{ Publisher: &a.opts.Publisher, Offer: to.Ptr(name), diff --git a/mantle/platform/api/azure/image.go b/mantle/platform/api/azure/image.go index 7d2b588cc1..12ad628be3 100644 --- a/mantle/platform/api/azure/image.go +++ b/mantle/platform/api/azure/image.go @@ -28,7 +28,7 @@ func (a *API) CreateImage(name, resourceGroup, blobURI string) (armcompute.Image Name: &name, Location: &a.opts.Location, Properties: &armcompute.ImageProperties{ - HyperVGeneration: to.Ptr(armcompute.HyperVGenerationTypes(a.opts.HyperVGeneration)), + HyperVGeneration: to.Ptr(armcompute.HyperVGenerationTypes(armcompute.HyperVGenerationTypesV2)), StorageProfile: &armcompute.ImageStorageProfile{ OSDisk: &armcompute.ImageOSDisk{ OSType: to.Ptr(armcompute.OperatingSystemTypesLinux), diff --git a/mantle/platform/api/azure/instance.go b/mantle/platform/api/azure/instance.go index 9356b5dd59..06adf64239 100644 --- a/mantle/platform/api/azure/instance.go +++ b/mantle/platform/api/azure/instance.go @@ -24,7 +24,6 @@ import ( "math" "math/big" "regexp" - "strings" "time" "github.com/Azure/azure-sdk-for-go/sdk/azcore/to" @@ -108,13 +107,8 @@ func (a *API) getVMParameters(name, userdata, sshkey, storageAccountURI, size st } } // UltraSSDEnabled=true is required for NVMe support on Gen2 VMs - var additionalCapabilities *armcompute.AdditionalCapabilities - if strings.EqualFold(a.opts.HyperVGeneration, string(armcompute.HyperVGenerationV2)) { - additionalCapabilities = &armcompute.AdditionalCapabilities{ - UltraSSDEnabled: to.Ptr(true), - } - } else { - additionalCapabilities = nil + additionalCapabilities := &armcompute.AdditionalCapabilities{ + UltraSSDEnabled: to.Ptr(true), } return armcompute.VirtualMachine{ Name: &name, diff --git a/mantle/platform/api/azure/network.go b/mantle/platform/api/azure/network.go index f65ad5ad31..6b5f69bba3 100644 --- a/mantle/platform/api/azure/network.go +++ b/mantle/platform/api/azure/network.go @@ -18,10 +18,8 @@ package azure import ( "context" "fmt" - "strings" "github.com/Azure/azure-sdk-for-go/sdk/azcore/to" - "github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/compute/armcompute" "github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/network/armnetwork" "github.com/coreos/coreos-assembler/mantle/util" @@ -91,25 +89,13 @@ func (a *API) createPublicIP(resourceGroup string) (armnetwork.PublicIPAddress, var ipZones []*string // set SKU=Standard, Allocation Method=Static and Availability Zone on public IPs when creating Gen2 images - if strings.EqualFold(a.opts.HyperVGeneration, string(armcompute.HyperVGenerationV2)) { - ipSKU = &armnetwork.PublicIPAddressSKU{ - Name: to.Ptr(armnetwork.PublicIPAddressSKUNameStandard), - } - ipProperties = &armnetwork.PublicIPAddressPropertiesFormat{ - PublicIPAllocationMethod: to.Ptr(armnetwork.IPAllocationMethodStatic), - } - ipZones = []*string{to.Ptr(a.opts.AvailabilityZone)} - // gen 1 - } else { - ipSKU = &armnetwork.PublicIPAddressSKU{ - Name: to.Ptr(armnetwork.PublicIPAddressSKUNameBasic), - } - ipProperties = &armnetwork.PublicIPAddressPropertiesFormat{ - PublicIPAllocationMethod: to.Ptr(armnetwork.IPAllocationMethodDynamic), - } - // No Zones for Gen1 - ipZones = nil + ipSKU = &armnetwork.PublicIPAddressSKU{ + Name: to.Ptr(armnetwork.PublicIPAddressSKUNameStandard), + } + ipProperties = &armnetwork.PublicIPAddressPropertiesFormat{ + PublicIPAllocationMethod: to.Ptr(armnetwork.IPAllocationMethodStatic), } + ipZones = []*string{to.Ptr(a.opts.AvailabilityZone)} poller, err := a.ipClient.BeginCreateOrUpdate(ctx, resourceGroup, name, armnetwork.PublicIPAddress{ Location: to.Ptr(a.opts.Location), diff --git a/mantle/platform/api/azure/options.go b/mantle/platform/api/azure/options.go index 1aee5454ea..cd773313e9 100644 --- a/mantle/platform/api/azure/options.go +++ b/mantle/platform/api/azure/options.go @@ -33,7 +33,6 @@ type Options struct { Size string Location string AvailabilityZone string - HyperVGeneration string SubscriptionName string SubscriptionID string