feat: add ability to configure platforms of the deployed images

.gitignore

@@ -27,4 +27,3 @@ qodana.yaml
 
 # Pipenv
 Pipfile*
-

cmd/cluster/clusterCreate.go

@@ -251,6 +251,9 @@ func NewCmdClusterCreate() *cobra.Command {
 	cmd.Flags().StringArrayP("runtime-ulimit", "", nil, "Add ulimit to container runtime (Format: `NAME[=SOFT]:[HARD]`\n - Example: `k3d cluster create --agents 2 --runtime-ulimit \"nofile=1024:1024\" --runtime-ulimit \"noproc=1024:1024\"`")
 	_ = ppViper.BindPFlag("cli.runtime-ulimits", cmd.Flags().Lookup("runtime-ulimit"))
 
+	cmd.Flags().StringArrayP("runtime-platform", "", nil, "Add platform to container runtime (Format: `<os>[(<OSVersion>)]|<arch>|<os>[(<OSVersion>)]/<arch>[/<variant>][@NODEFILTER[;NODEFILTER...]]`\n - Example: `k3d cluster create --agents 2 --runtime-platform \"linux/amd64@agent:0,1\" --runtime-platform \"linux/arm64/v8@server:0\"`")
+	_ = ppViper.BindPFlag("cli.runtime-platform", cmd.Flags().Lookup("runtime-platform"))
+
 	cmd.Flags().String("registry-create", "", "Create a k3d-managed registry and connect it to the cluster (Format: `NAME[:HOST][:HOSTPORT]`\n - Example: `k3d cluster create --registry-create mycluster-registry:0.0.0.0:5432`")
 	_ = ppViper.BindPFlag("cli.registries.create", cmd.Flags().Lookup("registry-create"))
 
@@ -518,6 +521,33 @@ func applyCLIOverrides(cfg conf.SimpleConfig) (conf.SimpleConfig, error) {
 		cfg.Options.Runtime.Ulimits = append(cfg.Options.Runtime.Ulimits, *cliutil.ParseRuntimeUlimit[conf.Ulimit](ulimit))
 	}
 
+	// --runtime-platform
+	// runtimePlatform will add container platform configuration to applied node filters
+	runtimePlatformFilterMap := make(map[string][]string, 1)
+	for _, platformFlag := range ppViper.GetStringSlice("cli.runtime-platform") {
+		// split node filter from the specified platform
+		platform, nodeFilters, err := cliutil.SplitFiltersFromFlag(platformFlag)
+		if err != nil {
+			l.Log().Fatalln(err)
+		}
+
+		// create new entry or append filter to existing entry
+		if _, exists := runtimePlatformFilterMap[platform]; exists {
+			runtimePlatformFilterMap[platform] = append(runtimePlatformFilterMap[platform], nodeFilters...)
+		} else {
+			runtimePlatformFilterMap[platform] = nodeFilters
+		}
+	}
+
+	for platform, nodeFilters := range runtimePlatformFilterMap {
+		cfg.Options.Runtime.Platforms = append(cfg.Options.Runtime.Platforms, conf.PlatformWithNodeFilters{
+			Platform:    platform,
+			NodeFilters: nodeFilters,
+		})
+	}
+
+	l.Log().Tracef("RuntimePlatformFilterMap: %+v", runtimePlatformFilterMap)
+
 	// --env
 	// envFilterMap will add container env vars to applied node filters
 	envFilterMap := make(map[string][]string, 1)

cmd/node/nodeCreate.go

@@ -82,6 +82,7 @@ func NewCmdNodeCreate() *cobra.Command {
 
 	cmd.Flags().StringSliceP("runtime-label", "", []string{}, "Specify container runtime labels in format \"foo=bar\"")
 	cmd.Flags().StringSliceP("runtime-ulimit", "", []string{}, "Specify container runtime ulimit in format \"ulimit=soft:hard\"")
+	cmd.Flags().StringP("runtime-platform", "", "", "Specify container platform in format \"linux/amd64\"")
 	cmd.Flags().StringSliceP("k3s-node-label", "", []string{}, "Specify k3s node labels in format \"foo=bar\"")
 
 	cmd.Flags().StringSliceP("network", "n", []string{}, "Add node to (another) runtime network")
@@ -166,6 +167,13 @@ func parseCreateNodeCmd(cmd *cobra.Command, args []string) ([]*k3d.Node, string)
 	for index, ulimit := range runtimeUlimitsFlag {
 		runtimeUlimits[index] = cliutil.ParseRuntimeUlimit[dockerunits.Ulimit](ulimit)
 	}
+
+	// runtime-platform
+	platform, err := cmd.Flags().GetString("runtime-platform")
+	if err != nil {
+		l.Log().Fatalf("No runtime-platform specified: %v", err)
+	}
+
 	// --k3s-node-label
 	k3sNodeLabelsFlag, err := cmd.Flags().GetStringSlice("k3s-node-label")
 	if err != nil {
@@ -204,6 +212,7 @@ func parseCreateNodeCmd(cmd *cobra.Command, args []string) ([]*k3d.Node, string)
 			K3sNodeLabels:  k3sNodeLabels,
 			RuntimeLabels:  runtimeLabels,
 			RuntimeUlimits: runtimeUlimits,
+			Platform:       platform,
 			Restart:        true,
 			Memory:         memory,
 			Networks:       networks,

docs/usage/configfile.md

@@ -148,6 +148,11 @@ options:
       - name: nofile
         soft: 26677
         hard: 26677
+    platforms: # overwrite the default platforms for the cluster to pass it as an runtime argument
+      - platform: linux/arm64/v8
+        nodeFilters:
+          - server:*
+          - agent:*
 
 ```
 

go.mod

@@ -3,6 +3,7 @@ module github.com/k3d-io/k3d/v5
 go 1.24.4
 
 require (
+	github.com/containerd/platforms v0.2.1
 	github.com/goodhosts/hostsfile v0.1.6
 	github.com/google/go-containerregistry v0.20.6
 	github.com/rancher/wharfie v0.6.2
@@ -76,7 +77,7 @@ require (
 	github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
 	github.com/modern-go/reflect2 v1.0.2 // indirect
 	github.com/opencontainers/go-digest v1.0.0 // indirect
-	github.com/opencontainers/image-spec v1.1.1 // indirect
+	github.com/opencontainers/image-spec v1.1.1
 	github.com/pelletier/go-toml/v2 v2.1.0 // indirect
 	github.com/pkg/errors v0.9.1
 	github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect

go.sum

@@ -32,6 +32,8 @@ github.com/containerd/errdefs/pkg v0.3.0 h1:9IKJ06FvyNlexW690DXuQNx2KA2cUJXx151X
 github.com/containerd/errdefs/pkg v0.3.0/go.mod h1:NJw6s9HwNuRhnjJhM7pylWwMyAkmCQvQ4GpJHEqRLVk=
 github.com/containerd/log v0.1.0 h1:TCJt7ioM2cr/tfR8GPbGf9/VRAX8D2B4PjzCpfX540I=
 github.com/containerd/log v0.1.0/go.mod h1:VRRf09a7mHDIRezVKTRCrOq78v577GXq3bSa3EhrzVo=
+github.com/containerd/platforms v0.2.1 h1:zvwtM3rz2YHPQsF2CHYM8+KtB5dvhISiXh5ZpSBQv6A=
+github.com/containerd/platforms v0.2.1/go.mod h1:XHCb+2/hzowdiut9rkudds9bE5yJ7npe7dG/wG+uFPw=
 github.com/containerd/stargz-snapshotter/estargz v0.16.3 h1:7evrXtoh1mSbGj/pfRccTampEyKpjpOnS3CyiV1Ebr8=
 github.com/containerd/stargz-snapshotter/estargz v0.16.3/go.mod h1:uyr4BfYfOj3G9WBVE8cOlQmXAbPN9VEQpBBeJIuOipU=
 github.com/corpix/uarand v0.0.0-20170723150923-031be390f409 h1:9A+mfQmwzZ6KwUXPc8nHxFtKgn9VIvO3gXAOspIcE3s=

pkg/client/node.go

@@ -42,6 +42,7 @@ import (
 	"golang.org/x/sync/errgroup"
 	"sigs.k8s.io/yaml"
 
+	"github.com/containerd/platforms"
 	"github.com/k3d-io/k3d/v5/pkg/actions"
 	l "github.com/k3d-io/k3d/v5/pkg/logger"
 	"github.com/k3d-io/k3d/v5/pkg/runtimes"
@@ -53,6 +54,7 @@ import (
 	"github.com/k3d-io/k3d/v5/pkg/types/k3s"
 	"github.com/k3d-io/k3d/v5/pkg/util"
 	"github.com/k3d-io/k3d/v5/version"
+	ocispecv1 "github.com/opencontainers/image-spec/specs-go/v1"
 )
 
 // NodeAddToCluster adds a node to an existing cluster
@@ -650,7 +652,15 @@ func NodeCreate(ctx context.Context, runtime runtimes.Runtime, node *k3d.Node, c
 			}
 			node.Volumes = append(node.Volumes, fmt.Sprintf("%s:%s:ro", fakemempath, util.MemInfoPath))
 			// mount empty edac folder, but only if it exists
-			exists, err := docker.CheckIfDirectoryExists(ctx, node.Image, util.EdacFolderPath)
+			var platform *ocispecv1.Platform
+			if node.Platform != "" {
+				p, err := platforms.Parse(node.Platform)
+				if err != nil {
+					return fmt.Errorf("failed to parse platform '%s': %w", node.Platform, err)
+				}
+				platform = &p
+			}
+			exists, err := docker.CheckIfDirectoryExists(ctx, node.Image, platform, util.EdacFolderPath)
 			if err != nil {
 				return fmt.Errorf("failed to check for the existence of edac folder: %w", err)
 			}

pkg/config/transform.go

@@ -252,6 +252,22 @@ func TransformSimpleToClusterConfig(ctx context.Context, runtime runtimes.Runtim
 		}
 	}
 
+	// -> RUNTIME PLATFORMS
+	for _, runtimePlatformWithNodeFilters := range simpleConfig.Options.Runtime.Platforms {
+		if len(runtimePlatformWithNodeFilters.NodeFilters) == 0 && nodeCount > 1 {
+			return nil, fmt.Errorf("RuntimePlatformmapping '%s' lacks a node filter, but there's more than one node", runtimePlatformWithNodeFilters.Platform)
+		}
+
+		nodes, err := util.FilterNodes(nodeList, runtimePlatformWithNodeFilters.NodeFilters)
+		if err != nil {
+			return nil, fmt.Errorf("failed to filter nodes for runtime platform mapping '%s': %w", runtimePlatformWithNodeFilters.Platform, err)
+		}
+
+		for _, node := range nodes {
+			node.Platform = runtimePlatformWithNodeFilters.Platform
+		}
+	}
+
 	// -> ENV
 	for _, envVarWithNodeFilters := range simpleConfig.Env {
 		if len(envVarWithNodeFilters.NodeFilters) == 0 && nodeCount > 1 {

pkg/config/v1alpha5/types.go

@@ -74,6 +74,11 @@ type LabelWithNodeFilters struct {
 	NodeFilters []string `mapstructure:"nodeFilters" json:"nodeFilters,omitempty"`
 }
 
+type PlatformWithNodeFilters struct {
+	Platform    string   `mapstructure:"platform" json:"platform,omitempty"`
+	NodeFilters []string `mapstructure:"nodeFilters" json:"nodeFilters,omitempty"`
+}
+
 type EnvVarWithNodeFilters struct {
 	EnvVar      string   `mapstructure:"envVar" json:"envVar,omitempty"`
 	NodeFilters []string `mapstructure:"nodeFilters" json:"nodeFilters,omitempty"`
@@ -115,12 +120,13 @@ type SimpleConfigOptions struct {
 }
 
 type SimpleConfigOptionsRuntime struct {
-	GPURequest    string                 `mapstructure:"gpuRequest" json:"gpuRequest,omitempty"`
-	ServersMemory string                 `mapstructure:"serversMemory" json:"serversMemory,omitempty"`
-	AgentsMemory  string                 `mapstructure:"agentsMemory" json:"agentsMemory,omitempty"`
-	HostPidMode   bool                   `mapstructure:"hostPidMode" yjson:"hostPidMode,omitempty"`
-	Labels        []LabelWithNodeFilters `mapstructure:"labels" json:"labels,omitempty"`
-	Ulimits       []Ulimit               `mapstructure:"ulimits" json:"ulimits,omitempty"`
+	GPURequest    string                    `mapstructure:"gpuRequest" json:"gpuRequest,omitempty"`
+	ServersMemory string                    `mapstructure:"serversMemory" json:"serversMemory,omitempty"`
+	AgentsMemory  string                    `mapstructure:"agentsMemory" json:"agentsMemory,omitempty"`
+	HostPidMode   bool                      `mapstructure:"hostPidMode" yjson:"hostPidMode,omitempty"`
+	Labels        []LabelWithNodeFilters    `mapstructure:"labels" json:"labels,omitempty"`
+	Ulimits       []Ulimit                  `mapstructure:"ulimits" json:"ulimits,omitempty"`
+	Platforms     []PlatformWithNodeFilters `mapstructure:"platforms" json:"platforms,omitempty"`
 }
 
 type Ulimit struct {

pkg/runtimes/docker/container.go

@@ -27,19 +27,21 @@ import (
 	"fmt"
 	"io"
 
+	"github.com/containerd/platforms"
 	"github.com/docker/docker/api/types"
 	"github.com/docker/docker/api/types/container"
 	"github.com/docker/docker/api/types/filters"
 	dockerimage "github.com/docker/docker/api/types/image"
 	"github.com/docker/docker/client"
 	l "github.com/k3d-io/k3d/v5/pkg/logger"
 	k3d "github.com/k3d-io/k3d/v5/pkg/types"
+	ocispecv1 "github.com/opencontainers/image-spec/specs-go/v1"
 	"github.com/sirupsen/logrus"
 )
 
 // createContainer creates a new docker container from translated specs
 func createContainer(ctx context.Context, dockerNode *NodeInDocker, name string) (string, error) {
-	l.Log().Tracef("Creating docker container with translated config\n%+v\n", dockerNode)
+	l.Log().Tracef("Creating docker container with translated config: %s\n%+v\n", name, dockerNode)
 
 	// initialize docker client
 	docker, err := GetDockerClient()
@@ -51,10 +53,10 @@ func createContainer(ctx context.Context, dockerNode *NodeInDocker, name string)
 	// create container
 	var resp container.CreateResponse
 	for {
-		resp, err = docker.ContainerCreate(ctx, &dockerNode.ContainerConfig, &dockerNode.HostConfig, &dockerNode.NetworkingConfig, nil, name)
+		resp, err = docker.ContainerCreate(ctx, &dockerNode.ContainerConfig, &dockerNode.HostConfig, &dockerNode.NetworkingConfig, dockerNode.PlatformConfig, name)
 		if err != nil {
 			if client.IsErrNotFound(err) {
-				if err := pullImage(ctx, docker, dockerNode.ContainerConfig.Image); err != nil {
+				if err := pullImage(ctx, docker, dockerNode.ContainerConfig.Image, dockerNode.PlatformConfig); err != nil {
 					return "", fmt.Errorf("docker failed to pull image '%s': %w", dockerNode.ContainerConfig.Image, err)
 				}
 				continue
@@ -105,8 +107,15 @@ func removeContainer(ctx context.Context, ID string) error {
 }
 
 // pullImage pulls a container image and outputs progress if --verbose flag is set
-func pullImage(ctx context.Context, docker client.APIClient, image string) error {
-	resp, err := docker.ImagePull(ctx, image, dockerimage.PullOptions{})
+func pullImage(ctx context.Context, docker client.APIClient, image string, platform *ocispecv1.Platform) error {
+	opts := dockerimage.PullOptions{}
+
+	if platform != nil {
+		// if a platform is specified, use it to pull the image
+		opts.Platform = platforms.Format(*platform)
+	}
+
+	resp, err := docker.ImagePull(ctx, image, opts)
 	if err != nil {
 		return fmt.Errorf("docker failed to pull the image '%s': %w", image, err)
 	}
@@ -168,7 +177,7 @@ func getNodeContainer(ctx context.Context, node *k3d.Node) (*types.Container, er
 
 // executes an arbitrary command in a container while returning its exit code.
 // useful to check something in docker env
-func executeCheckInContainer(ctx context.Context, image string, cmd []string) (int64, error) {
+func executeCheckInContainer(ctx context.Context, image string, platform *ocispecv1.Platform, cmd []string) (int64, error) {
 	docker, err := GetDockerClient()
 	if err != nil {
 		return -1, fmt.Errorf("failed to create docker client: %w", err)
@@ -186,7 +195,7 @@ func executeCheckInContainer(ctx context.Context, image string, cmd []string) (i
 		}, nil, nil, nil, "")
 		if err != nil {
 			if client.IsErrNotFound(err) {
-				if err := pullImage(ctx, docker, image); err != nil {
+				if err := pullImage(ctx, docker, image, platform); err != nil {
 					return -1, fmt.Errorf("docker failed to pull image '%s': %w", image, err)
 				}
 				continue
@@ -219,11 +228,11 @@ func executeCheckInContainer(ctx context.Context, image string, cmd []string) (i
 }
 
 // CheckIfDirectoryExists checks for the existence of a given path inside the docker environment
-func CheckIfDirectoryExists(ctx context.Context, image string, dir string) (bool, error) {
+func CheckIfDirectoryExists(ctx context.Context, image string, platform *ocispecv1.Platform, dir string) (bool, error) {
 	l.Log().Tracef("checking if dir %s exists in docker environment...", dir)
 	shellCmd := fmt.Sprintf("[ -d \"%s\" ] && exit 0 || exit 1", dir)
 	cmd := []string{"sh", "-c", shellCmd}
-	exitCode, err := executeCheckInContainer(ctx, image, cmd)
+	exitCode, err := executeCheckInContainer(ctx, image, platform, cmd)
 	l.Log().Tracef("check dir container returned %d exit code", exitCode)
 	return exitCode == 0, err
 }

pkg/runtimes/docker/translate.go

@@ -30,13 +30,15 @@ import (
 	"strconv"
 	"strings"
 
+	"github.com/containerd/platforms"
 	"github.com/docker/docker/api/types"
 	docker "github.com/docker/docker/api/types/container"
 	"github.com/docker/docker/api/types/network"
 	"github.com/docker/go-connections/nat"
 	l "github.com/k3d-io/k3d/v5/pkg/logger"
 	runtimeErr "github.com/k3d-io/k3d/v5/pkg/runtimes/errors"
 	k3d "github.com/k3d-io/k3d/v5/pkg/types"
+	ocispecv1 "github.com/opencontainers/image-spec/specs-go/v1"
 
 	dockercliopts "github.com/docker/cli/opts"
 	dockerunits "github.com/docker/go-units"
@@ -122,6 +124,16 @@ func TranslateNodeToContainer(node *k3d.Node) (*NodeInDocker, error) {
 		hostConfig.Memory = memory
 	}
 
+	var platformConfig *ocispecv1.Platform
+	if node.Platform != "" {
+		// parse platform string
+		p, err := platforms.Parse(node.Platform)
+		if err != nil {
+			return nil, fmt.Errorf("Failed to parse platform string '%s': %+v", node.Platform, err)
+		}
+		platformConfig = &p
+	}
+
 	/* They have to run in privileged mode */
 	// TODO: can we replace this by a reduced set of capabilities?
 	hostConfig.Privileged = true
@@ -178,6 +190,7 @@ func TranslateNodeToContainer(node *k3d.Node) (*NodeInDocker, error) {
 		ContainerConfig:  containerConfig,
 		HostConfig:       hostConfig,
 		NetworkingConfig: networkingConfig,
+		PlatformConfig:   platformConfig,
 	}, nil
 }
 

pkg/runtimes/docker/types.go

@@ -24,11 +24,13 @@ package docker
 import (
 	"github.com/docker/docker/api/types/container"
 	"github.com/docker/docker/api/types/network"
+	ocispecv1 "github.com/opencontainers/image-spec/specs-go/v1"
 )
 
 // NodeInDocker represents everything that we need to represent a k3d node in docker
 type NodeInDocker struct {
 	ContainerConfig  container.Config // TODO: do we need this as pointers?
 	HostConfig       container.HostConfig
 	NetworkingConfig network.NetworkingConfig
+	PlatformConfig   *ocispecv1.Platform
 }

pkg/types/fixes/assets/k3d-entrypoint-dns.sh

@@ -15,12 +15,12 @@ echo "[$(date -Iseconds)] [DNS Fix] Use the detected Gateway IP $gateway instead
 
 # Change iptables rules added by Docker to route traffic to out Gateway IP instead of Docker's embedded DNS
 echo "[$(date -Iseconds)] [DNS Fix] > Changing iptables rules ..."
-iptables-save \
+iptables-legacy-save \
   | sed \
     -e "s/-d ${docker_dns}/-d ${gateway}/g" \
     -e 's/-A OUTPUT \(.*\) -j DOCKER_OUTPUT/\0\n-A PREROUTING \1 -j DOCKER_OUTPUT/' \
     -e "s/--to-source :53/--to-source ${gateway}:53/g"\
-  | iptables-restore
+  | iptables-legacy-restore
 
 # Update resolv.conf to use the Gateway IP if needed: this will also make CoreDNS use it via k3s' default `forward . /etc/resolv.conf` rule in the CoreDNS config
 if grep -q "${docker_dns}" /etc/resolv.conf; then

pkg/types/fixes/assets/k3d-entrypoint.sh

@@ -7,14 +7,14 @@ LOGFILE="/var/log/k3d-entrypoints_$(date "+%y%m%d%H%M%S").log"
 
 touch "$LOGFILE"
 
-echo "[$(date -Iseconds)] Running k3d entrypoints..." >> "$LOGFILE"
+echo "[$(date -Iseconds)] Running k3d entrypoints..." | tee -a "$LOGFILE"
 
-for entrypoint in /bin/k3d-entrypoint-*.sh ; do
-  echo "[$(date -Iseconds)] Running $entrypoint"  >> "$LOGFILE"
-  "$entrypoint"  >> "$LOGFILE" 2>&1 || exit 1
+for entrypoint in /bin/k3d-entrypoint-*.sh; do
+  echo "[$(date -Iseconds)] Running $entrypoint" | tee -a "$LOGFILE"
+  eval "$entrypoint" 2>&1 | tee "$LOGFILE" || exit 1
 done
 
-echo "[$(date -Iseconds)] Finished k3d entrypoint scripts!" >> "$LOGFILE"
+echo "[$(date -Iseconds)] Finished k3d entrypoint scripts!" | tee -a "$LOGFILE"
 
 /bin/k3s "$@" &
 k3s_pid=$!

pkg/types/types.go

@@ -292,6 +292,7 @@ type Node struct {
 	Cmd            []string              // filled automatically based on role
 	Args           []string              `json:"extraArgs,omitempty"`
 	Files          []File                `json:"files,omitempty"`
+	Platform       string                `json:"platform,omitempty"`
 	Ports          nat.PortMap           `json:"portMappings,omitempty"`
 	Restart        bool                  `json:"restart,omitempty"`
 	Created        string                `json:"created,omitempty"`