diff --git a/cmd/minikube/cmd/start_flags.go b/cmd/minikube/cmd/start_flags.go index aac3a8a270b9..ac931d6c1137 100644 --- a/cmd/minikube/cmd/start_flags.go +++ b/cmd/minikube/cmd/start_flags.go @@ -30,6 +30,7 @@ import ( "k8s.io/klog/v2" "k8s.io/minikube/pkg/drivers/kic" "k8s.io/minikube/pkg/drivers/kic/oci" + "k8s.io/minikube/pkg/drivers/vmnet" "k8s.io/minikube/pkg/minikube/bootstrapper/bsutil" "k8s.io/minikube/pkg/minikube/bootstrapper/bsutil/kverify" "k8s.io/minikube/pkg/minikube/cni" @@ -197,7 +198,7 @@ func initMinikubeFlags() { startCmd.Flags().Bool(noKubernetes, false, "If set, minikube VM/container will start without starting or configuring Kubernetes. (only works on new clusters)") startCmd.Flags().Bool(deleteOnFailure, false, "If set, delete the current cluster if start fails and try again. Defaults to false.") startCmd.Flags().Bool(forceSystemd, false, "If set, force the container runtime to use systemd as cgroup manager. Defaults to false.") - startCmd.Flags().String(network, "", "network to run minikube with. Now it is used by docker/podman and KVM drivers. If left empty, minikube will create a new network.") + startCmd.Flags().String(network, "", "network to run minikube with. Used by docker/podman, qemu, kvm, and vfkit drivers. If left empty, minikube will create a new network.") startCmd.Flags().StringVarP(&outputFormat, "output", "o", "text", "Format to print stdout in. Options include: [text,json]") startCmd.Flags().String(trace, "", "Send trace events. Options include: [gcp]") startCmd.Flags().Int(extraDisks, 0, "Number of extra disks created and attached to the minikube VM (currently only implemented for hyperkit, kvm2, and qemu2 drivers)") @@ -469,9 +470,15 @@ func getCNIConfig(cmd *cobra.Command) string { func getNetwork(driverName string) string { n := viper.GetString(network) - if !driver.IsQEMU(driverName) { - return n + if driver.IsQEMU(driverName) { + return validateQemuNetwork(n) + } else if driver.IsVFKit(driverName) { + return validateVfkitNetwork(n) } + return n +} + +func validateQemuNetwork(n string) string { switch n { case "socket_vmnet": if runtime.GOOS != "darwin" { @@ -503,6 +510,27 @@ func getNetwork(driverName string) string { return n } +func validateVfkitNetwork(n string) string { + if runtime.GOOS != "darwin" { + exit.Message(reason.Usage, "The vfkit driver is only supported on macOS") + } + switch n { + case "nat": + // always available + case "vmnet-shared": + // "vment-shared" provides access between machines, with lower performance compared to "nat". + if !vmnet.HelperAvailable() { + exit.Message(reason.NotFoundVmnetHelper, "\n\n") + } + case "": + // Default to nat since it is always available and provides the best performance. + n = "nat" + default: + exit.Message(reason.Usage, "--network with vfkit must be 'nat' or 'vmnet-shared'") + } + return n +} + // generateNewConfigFromFlags generate a config.ClusterConfig based on flags func generateNewConfigFromFlags(cmd *cobra.Command, k8sVersion string, rtime string, drvName string) config.ClusterConfig { var cc config.ClusterConfig @@ -513,8 +541,8 @@ func generateNewConfigFromFlags(cmd *cobra.Command, k8sVersion string, rtime str out.WarningT("With --network-plugin=cni, you will need to provide your own CNI. See --cni flag as a user-friendly alternative") } - if !(driver.IsKIC(drvName) || driver.IsKVM(drvName) || driver.IsQEMU(drvName)) && viper.GetString(network) != "" { - out.WarningT("--network flag is only valid with the docker/podman, KVM and Qemu drivers, it will be ignored") + if viper.GetString(network) != "" && !driver.SupportsNetworkFlag(drvName) { + out.WarningT("--network flag is only valid with the docker/podman, qemu, kvm, and vfkit drivers, it will be ignored") } validateHANodeCount(cmd) diff --git a/pkg/drivers/vfkit/vfkit.go b/pkg/drivers/vfkit/vfkit.go index de1749fb25c6..b0fb8e920399 100644 --- a/pkg/drivers/vfkit/vfkit.go +++ b/pkg/drivers/vfkit/vfkit.go @@ -43,6 +43,7 @@ import ( "k8s.io/klog/v2" pkgdrivers "k8s.io/minikube/pkg/drivers" + "k8s.io/minikube/pkg/drivers/vmnet" "k8s.io/minikube/pkg/minikube/exit" "k8s.io/minikube/pkg/minikube/firewall" "k8s.io/minikube/pkg/minikube/out" @@ -67,8 +68,10 @@ type Driver struct { CPU int Memory int Cmdline string - MACAddress string ExtraDisks int + Network string // "", "nat", "vmnet-shared" + MACAddress string // For network=nat, network="" + VmnetHelper *vmnet.Helper // For network=vmnet-shared } func NewDriver(hostName, storePath string) drivers.Driver { @@ -136,7 +139,7 @@ func (d *Driver) GetIP() (string, error) { return d.IPAddress, nil } -func (d *Driver) GetState() (state.State, error) { +func (d *Driver) getVfkitState() (state.State, error) { pidfile := d.pidfilePath() pid, err := process.ReadPidfile(pidfile) if err != nil { @@ -159,6 +162,24 @@ func (d *Driver) GetState() (state.State, error) { return state.Running, nil } +func (d *Driver) getVmnetHelperState() (state.State, error) { + if d.VmnetHelper == nil { + return state.Stopped, nil + } + return d.VmnetHelper.GetState() +} + +// GetState returns driver state. Since vfkit driver may use 2 processes +// (vmnet-helper, vfkit), this returns combined state of both processes. +func (d *Driver) GetState() (state.State, error) { + if vfkitState, err := d.getVfkitState(); err != nil { + return state.Error, err + } else if vfkitState == state.Running { + return state.Running, nil + } + return d.getVmnetHelperState() +} + func (d *Driver) Create() error { var err error if d.SSHPort, err = d.GetSSHPort(); err != nil { @@ -200,6 +221,40 @@ func (d *Driver) Create() error { } func (d *Driver) Start() error { + var helperSock, vfkitSock *os.File + var err error + + if d.VmnetHelper != nil { + helperSock, vfkitSock, err = vmnet.Socketpair() + if err != nil { + return err + } + defer helperSock.Close() + defer vfkitSock.Close() + + if err := d.VmnetHelper.Start(helperSock); err != nil { + return err + } + + d.MACAddress = d.VmnetHelper.GetMACAddress() + } + + if err := d.startVfkit(vfkitSock); err != nil { + return err + } + + if err := d.setupIP(d.MACAddress); err != nil { + return err + } + + log.Infof("Waiting for VM to start (ssh -p %d docker@%s)...", d.SSHPort, d.IPAddress) + + return WaitForTCPWithDelay(fmt.Sprintf("%s:%d", d.IPAddress, d.SSHPort), time.Second) +} + +// startVfkit starts the vfkit child process. If vfkitSock is non nil, vfkit is +// connected to the vmnet network via the socket instead of "nat" network. +func (d *Driver) startVfkit(vfkitSock *os.File) error { machineDir := filepath.Join(d.StorePath, "machines", d.GetMachineName()) var startCmd []string @@ -212,9 +267,15 @@ func (d *Driver) Start() error { startCmd = append(startCmd, "--device", fmt.Sprintf("virtio-blk,path=%s", isoPath)) - var mac = d.MACAddress - startCmd = append(startCmd, - "--device", fmt.Sprintf("virtio-net,nat,mac=%s", mac)) + if vfkitSock != nil { + // The guest will be able to access other guests in the vmnet network. + startCmd = append(startCmd, + "--device", fmt.Sprintf("virtio-net,fd=%d,mac=%s", vfkitSock.Fd(), d.MACAddress)) + } else { + // The guest will not be able to access other guests. + startCmd = append(startCmd, + "--device", fmt.Sprintf("virtio-net,nat,mac=%s", d.MACAddress)) + } startCmd = append(startCmd, "--device", "virtio-rng") @@ -245,16 +306,7 @@ func (d *Driver) Start() error { if err := cmd.Start(); err != nil { return err } - if err := process.WritePidfile(d.pidfilePath(), cmd.Process.Pid); err != nil { - return err - } - if err := d.setupIP(mac); err != nil { - return err - } - - log.Infof("Waiting for VM to start (ssh -p %d docker@%s)...", d.SSHPort, d.IPAddress) - - return WaitForTCPWithDelay(fmt.Sprintf("%s:%d", d.IPAddress, d.SSHPort), time.Second) + return process.WritePidfile(d.pidfilePath(), cmd.Process.Pid) } func (d *Driver) setupIP(mac string) error { @@ -295,7 +347,7 @@ func isBootpdError(err error) bool { return strings.Contains(err.Error(), "could not find an IP address") } -func (d *Driver) Stop() error { +func (d *Driver) stopVfkit() error { if err := d.SetVFKitState("Stop"); err != nil { // vfkit may be already stopped, shutting down, or not listening. // We don't fallback to "HardStop" since it typically fails due to @@ -324,6 +376,20 @@ func (d *Driver) Stop() error { return nil } +func (d *Driver) stopVmnetHelper() error { + if d.VmnetHelper == nil { + return nil + } + return d.VmnetHelper.Stop() +} + +func (d *Driver) Stop() error { + if err := d.stopVfkit(); err != nil { + return err + } + return d.stopVmnetHelper() +} + func (d *Driver) Remove() error { s, err := d.GetState() if err != nil { @@ -367,7 +433,7 @@ func (d *Driver) extractKernel(isoPath string) error { return nil } -func (d *Driver) Kill() error { +func (d *Driver) killVfkit() error { if err := d.SetVFKitState("HardStop"); err != nil { // Typically fails with EOF due to https://github.com/crc-org/vfkit/issues/277. log.Debugf("Failed to set vfkit state to 'HardStop': %s", err) @@ -394,6 +460,20 @@ func (d *Driver) Kill() error { return nil } +func (d *Driver) killVmnetHelper() error { + if d.VmnetHelper == nil { + return nil + } + return d.VmnetHelper.Kill() +} + +func (d *Driver) Kill() error { + if err := d.killVfkit(); err != nil { + return err + } + return d.killVmnetHelper() +} + func (d *Driver) StartDocker() error { return fmt.Errorf("hosts without a driver cannot start docker") } @@ -530,7 +610,7 @@ func (d *Driver) SetVFKitState(state string) error { if err != nil { return err } - log.Debugf("set state: %+v", vmstate) + log.Infof("Set vfkit state: %+v", vmstate) return nil } diff --git a/pkg/drivers/vmnet/vmnet.go b/pkg/drivers/vmnet/vmnet.go new file mode 100644 index 000000000000..4fae1694336b --- /dev/null +++ b/pkg/drivers/vmnet/vmnet.go @@ -0,0 +1,253 @@ +//go:build darwin + +/* +Copyright 2025 The Kubernetes Authors All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// package vmnet provides the helper process connecting virtual machines to the +// vmnet network. +package vmnet + +import ( + "encoding/json" + "errors" + "fmt" + "os" + "os/exec" + "path/filepath" + "syscall" + + "github.com/docker/machine/libmachine/log" + "github.com/docker/machine/libmachine/state" + "k8s.io/minikube/pkg/minikube/process" +) + +const ( + pidfileName = "vmnet-helper.pid" + logfileName = "vmnet-helper.log" + executablePath = "/opt/vmnet-helper/bin/vmnet-helper" +) + +// Helper manages the vmnet-helper process. +type Helper struct { + // The pidfile and log are stored here. + MachineDir string + + // InterfaceID is a random UUID for the vmnet interface. Using the same UUID + // will obtain the same MAC address from vmnet. + InterfaceID string + + // Set when vmnet interface is started. + macAddress string +} + +type interfaceInfo struct { + MACAddress string `json:"vmnet_mac_address"` +} + +// HelperAvailable tells if vmnet-helper executable is installed and configured +// correctly. +func HelperAvailable() bool { + version, err := exec.Command("sudo", "--non-interactive", executablePath, "--version").Output() + if err != nil { + log.Debugf("Failed to run vmnet-helper: %w", err) + return false + } + log.Debugf("Using vmnet-helper version %q", version) + return true +} + +// Start the vmnet-helper child process, creating the vmnet interface for the +// machine. sock is a connected unix datagram socket to pass the helper child +// process. +func (h *Helper) Start(sock *os.File) error { + cmd := exec.Command( + "sudo", + "--non-interactive", + "--close-from", fmt.Sprintf("%d", sock.Fd()+1), + executablePath, + "--fd", fmt.Sprintf("%d", sock.Fd()), + "--interface-id", h.InterfaceID, + ) + + cmd.ExtraFiles = []*os.File{sock} + + // Create vmnet-helper in a new process group so it is not harmed when + // terminating the minikube process group. + cmd.SysProcAttr = &syscall.SysProcAttr{Setpgid: true} + + logfile, err := h.openLogfile() + if err != nil { + return fmt.Errorf("failed to open helper logfile: %w", err) + } + defer logfile.Close() + cmd.Stderr = logfile + + stdout, err := cmd.StdoutPipe() + if err != nil { + return fmt.Errorf("failed to create helper stdout pipe: %w", err) + } + defer stdout.Close() + + if err := cmd.Start(); err != nil { + return fmt.Errorf("failed to start vmnet-helper: %w", err) + } + + log.Infof("Started vmnet-helper (pid=%v)", cmd.Process.Pid) + + if err := process.WritePidfile(h.pidfilePath(), cmd.Process.Pid); err != nil { + return fmt.Errorf("failed to write vmnet-helper pidfile: %w", err) + } + + var info interfaceInfo + if err := json.NewDecoder(stdout).Decode(&info); err != nil { + return fmt.Errorf("failed to decode vmnet interface info: %w", err) + } + + log.Infof("Got mac address %q", info.MACAddress) + h.macAddress = info.MACAddress + + return nil +} + +// GetMACAddress reutuns the mac address assigned by vmnet framework. +func (h *Helper) GetMACAddress() string { + return h.macAddress +} + +// Stop terminates sudo, which will terminate vmnet-helper. +func (h *Helper) Stop() error { + log.Info("Stop vmnet-helper") + pidfile := h.pidfilePath() + pid, err := process.ReadPidfile(pidfile) + if err != nil { + if !errors.Is(err, os.ErrNotExist) { + return err + } + // No pidfile. + return nil + } + log.Debugf("Terminate sudo (pid=%v)", pid) + if err := process.Terminate(pid, "sudo"); err != nil { + if err != os.ErrProcessDone { + return err + } + // No process, stale pidfile. + if err := os.Remove(pidfile); err != nil { + log.Debugf("failed to remove %q: %s", pidfile, err) + } + } + return nil +} + +// Kill both sudo and vmnet-helper by killing the process group. +func (h *Helper) Kill() error { + log.Info("Kill vmnet-helper") + pidfile := h.pidfilePath() + pid, err := process.ReadPidfile(pidfile) + if err != nil { + if !errors.Is(err, os.ErrNotExist) { + return err + } + // No pidfile. + return nil + } + exists, err := process.Exists(pid, "sudo") + if err != nil { + return err + } + if !exists { + // No process, stale pidfile. + if err := os.Remove(pidfile); err != nil { + log.Debugf("failed to remove %q: %s", pidfile, err) + } + return nil + } + log.Debugf("Kill vmnet-helper process group (pgid=%v)", pid) + if err := syscall.Kill(-pid, syscall.SIGKILL); err != nil { + if err != syscall.ESRCH { + return err + } + // No process, stale pidfile. + if err := os.Remove(pidfile); err != nil { + log.Debugf("failed to remove %q: %s", pidfile, err) + } + } + return nil +} + +// GetState returns the sudo child process state. +func (h *Helper) GetState() (state.State, error) { + pidfile := h.pidfilePath() + pid, err := process.ReadPidfile(pidfile) + if err != nil { + if !errors.Is(err, os.ErrNotExist) { + return state.Error, err + } + // No pidfile. + return state.Stopped, nil + } + exists, err := process.Exists(pid, "sudo") + if err != nil { + return state.Error, err + } + if !exists { + // No process, stale pidfile. + if err := os.Remove(pidfile); err != nil { + log.Debugf("failed to remove %q: %s", pidfile, err) + } + return state.Stopped, nil + } + return state.Running, nil +} + +func (h *Helper) openLogfile() (*os.File, error) { + logfile := filepath.Join(h.MachineDir, logfileName) + return os.OpenFile(logfile, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0o600) +} + +func (h *Helper) pidfilePath() string { + return filepath.Join(h.MachineDir, pidfileName) +} + +// Apple recommends sizing the receive buffer at 4 times the size of the send +// buffer, and other projects typically use a 1 MiB send buffer and a 4 MiB +// receive buffer. However the send buffer size is not used to allocate a buffer +// in datagram sockets, it only limits the maximum packet size. We use 65 KiB +// buffer to allow the largest possible packet size (65550 bytes) when using the +// vmnet_enable_tso option. +const sendBufferSize = 65 * 1024 + +// The receive buffer size determines how many packets can be queued by the +// peer. Testing shows good performance with a 2 MiB receive buffer. We use a 4 +// MiB buffer to make ENOBUFS errors less likely for the peer and allowing to +// queue more packets when using the vmnet_enable_tso option. +const recvBufferSize = 4 * 1024 * 1024 + +// Socketpair returns a pair of connected unix datagram sockets that can be used +// to connect the helper and a vm. Pass one socket to the helper child process +// and the other to the vm child process. +func Socketpair() (*os.File, *os.File, error) { + fds, err := syscall.Socketpair(syscall.AF_UNIX, syscall.SOCK_DGRAM, 0) + if err != nil { + return nil, nil, err + } + // Setting buffer size is an optimization - don't fail on errors. + for _, fd := range fds { + _ = syscall.SetsockoptInt(fd, syscall.SOL_SOCKET, syscall.SO_SNDBUF, sendBufferSize) + _ = syscall.SetsockoptInt(fd, syscall.SOL_SOCKET, syscall.SO_RCVBUF, recvBufferSize) + } + return os.NewFile(uintptr(fds[0]), "sock1"), os.NewFile(uintptr(fds[1]), "sock2"), nil +} diff --git a/pkg/drivers/vmnet/vmnet_stub.go b/pkg/drivers/vmnet/vmnet_stub.go new file mode 100644 index 000000000000..6def9d5ed1d3 --- /dev/null +++ b/pkg/drivers/vmnet/vmnet_stub.go @@ -0,0 +1,23 @@ +//go:build !darwin + +/* +Copyright 2024 The Kubernetes Authors All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package vmnet + +func HelperAvailable() bool { + return false +} diff --git a/pkg/minikube/driver/driver.go b/pkg/minikube/driver/driver.go index 7f721e5c955f..7df9c359f882 100644 --- a/pkg/minikube/driver/driver.go +++ b/pkg/minikube/driver/driver.go @@ -173,6 +173,11 @@ func IsQEMU(name string) bool { return name == QEMU2 || name == QEMU } +// IsVFKit checks if the driver is vfkit +func IsVFKit(name string) bool { + return name == VFKit +} + // IsVM checks if the driver is a VM func IsVM(name string) bool { if IsKIC(name) || BareMetal(name) { @@ -206,6 +211,11 @@ func IsHyperV(name string) bool { return name == HyperV } +// SupportsNetworkFlag reutuns if driver supports the --network flag +func SupportsNetworkFlag(name string) bool { + return IsKIC(name) || IsKVM(name) || IsQEMU(name) || IsVFKit(name) +} + // AllowsPreload returns if preload is allowed for the driver func AllowsPreload(driverName string) bool { return !BareMetal(driverName) && !IsSSH(driverName) diff --git a/pkg/minikube/reason/reason.go b/pkg/minikube/reason/reason.go index afdf136014af..f254769f8540 100644 --- a/pkg/minikube/reason/reason.go +++ b/pkg/minikube/reason/reason.go @@ -229,7 +229,7 @@ var ( ID: "RSRC_DOCKER_STORAGE", ExitCode: ExInsufficientStorage, Advice: translate.T(`Try one or more of the following to free up space on the device: - + 1. Run "docker system prune" to remove unused Docker data (optionally with "-a") 2. Increase the storage allocated to Docker for Desktop by clicking on: Docker icon > Preferences > Resources > Disk Image Size @@ -241,7 +241,7 @@ var ( ID: "RSRC_PODMAN_STORAGE", ExitCode: ExInsufficientStorage, Advice: translate.T(`Try one or more of the following to free up space on the device: - + 1. Run "sudo podman system prune" to remove unused podman data 2. Run "minikube ssh -- docker system prune" if using the Docker container runtime`), Issues: []int{9024}, @@ -360,7 +360,7 @@ var ( ID: "GUEST_MOUNT_COULD_NOT_CONNECT", ExitCode: ExGuestError, Advice: translate.T(`If the host has a firewall: - + 1. Allow a port through the firewall 2. Specify "--port=" for "minikube mount"`), } @@ -490,16 +490,16 @@ var ( ID: "K8S_DOWNGRADE_UNSUPPORTED", ExitCode: ExControlPlaneUnsupported, Advice: translate.T(`1) Recreate the cluster with Kubernetes {{.new}}, by running: - + minikube delete{{.profile}} minikube start{{.profile}} --kubernetes-version={{.prefix}}{{.new}} - + 2) Create a second cluster with Kubernetes {{.new}}, by running: - + minikube start -p {{.suggestedName}} --kubernetes-version={{.prefix}}{{.new}} - + 3) Use the existing cluster at version Kubernetes {{.old}}, by running: - + minikube start{{.profile}} --kubernetes-version={{.prefix}}{{.old}} `), Style: style.SeeNoEvil, @@ -509,7 +509,7 @@ var ( ID: "NOT_FOUND_CRI_DOCKERD", ExitCode: ExProgramNotFound, Advice: translate.T(`The none driver with Kubernetes v1.24+ and the docker container-runtime requires cri-dockerd. - + Please install cri-dockerd using these instructions: https://github.com/Mirantis/cri-dockerd`), @@ -519,7 +519,7 @@ var ( ID: "NOT_FOUND_DOCKERD", ExitCode: ExProgramNotFound, Advice: translate.T(`The none driver with Kubernetes v1.24+ and the docker container-runtime requires dockerd. - + Please install dockerd using these instructions: https://docs.docker.com/engine/install/`), @@ -549,4 +549,18 @@ var ( minikube start{{.profile}} --driver qemu --network user`), Style: style.SeeNoEvil, } + NotFoundVmnetHelper = Kind{ + ID: "NOT_FOUND_VMNET_HELPER", + ExitCode: ExProgramNotFound, + Advice: translate.T(`vmnet-helper was not found on the system, resolve by: + + Option 1) Installing vment-helper: + + https://github.com/nirs/vmnet-helper#installation + + Option 2) Using the nat network: + + minikube start{{.profile}} --driver vfkit --network nat`), + Style: style.SeeNoEvil, + } ) diff --git a/pkg/minikube/registry/drvs/vfkit/vfkit.go b/pkg/minikube/registry/drvs/vfkit/vfkit.go index 1f2bb8c094dc..63d289a13489 100644 --- a/pkg/minikube/registry/drvs/vfkit/vfkit.go +++ b/pkg/minikube/registry/drvs/vfkit/vfkit.go @@ -22,10 +22,13 @@ import ( "crypto/rand" "fmt" "os/exec" + "path/filepath" "github.com/docker/machine/libmachine/drivers" + "github.com/google/uuid" "k8s.io/minikube/pkg/drivers/vfkit" + "k8s.io/minikube/pkg/drivers/vmnet" "k8s.io/minikube/pkg/minikube/config" "k8s.io/minikube/pkg/minikube/download" "k8s.io/minikube/pkg/minikube/driver" @@ -51,24 +54,50 @@ func init() { } func configure(cfg config.ClusterConfig, n config.Node) (interface{}, error) { - mac, err := generateMACAddress() - if err != nil { - return nil, fmt.Errorf("generating MAC address: %v", err) + var mac string + var helper *vmnet.Helper + + machineName := config.MachineName(cfg, n) + storePath := localpath.MiniPath() + + switch cfg.Network { + case "nat", "": + // We generate a random mac address. + var err error + mac, err = generateMACAddress() + if err != nil { + return nil, fmt.Errorf("generating MAC address: %v", err) + } + case "vmnet-shared": + // We generate a random UUID (or use a user provided one). vment-helper + // will obtain a mac address from the vmnet framework using the UUID. + u := cfg.UUID + if u == "" { + u = uuid.NewString() + } + helper = &vmnet.Helper{ + MachineDir: filepath.Join(storePath, "machines", machineName), + InterfaceID: u, + } + default: + return nil, fmt.Errorf("unsupported network: %q", cfg.Network) } return &vfkit.Driver{ BaseDriver: &drivers.BaseDriver{ - MachineName: config.MachineName(cfg, n), - StorePath: localpath.MiniPath(), + MachineName: machineName, + StorePath: storePath, SSHUser: "docker", }, Boot2DockerURL: download.LocalISOResource(cfg.MinikubeISO), DiskSize: cfg.DiskSize, Memory: cfg.Memory, CPU: cfg.CPUs, - MACAddress: mac, Cmdline: "", ExtraDisks: cfg.ExtraDisks, + Network: cfg.Network, + MACAddress: mac, + VmnetHelper: helper, }, nil } diff --git a/site/content/en/docs/drivers/vfkit.md b/site/content/en/docs/drivers/vfkit.md index 934db8c39471..14a6b001142a 100644 --- a/site/content/en/docs/drivers/vfkit.md +++ b/site/content/en/docs/drivers/vfkit.md @@ -7,7 +7,78 @@ aliases: ## Overview -[VFKit](https://github.com/crc-org/vfkit) is an open-source program for macOS virtualization, optimized for lightweight virtual machines and container deployment. +[VFKit](https://github.com/crc-org/vfkit) is an open-source program for +macOS virtualization, optimized for lightweight virtual machines and +container deployment. + +## Networking + +The vfkit driver has two networking options: `nat` and `vmnet-shared`. +The `nat` network is always available, but it does not provide access +between minikube clusters. To access other clusters or run multi-node +cluster, you need the `vmnet-shared` network. The `vmnet-shared` network +requires [vmnet-helper](https://github.com/nirs/vmnet-helper), see +installation instructions bellow. + +{{% tabs %}} +{{% tab vmnet-shared %}} + +### Requirements + +- Requires macOS 10.15 or later +- Requires minikube version 1.36.0 or later. +- Requires [vmnet-helper](https://github.com/nirs/vmnet-helper). + +### Install vment-helper + +```shell +tag="$(curl -fsSL https://api.github.com/repos/nirs/vmnet-helper/releases/latest | jq -r .tag_name)" +machine="$(uname -m)" +archive="vmnet-helper-$tag-$machine.tar.gz" +curl -LOf "https://github.com/nirs/vmnet-helper/releases/download/$tag/$archive" +sudo tar xvf "$archive" -C / opt/vmnet-helper +rm "$archive" +``` + +The command downloads the latest release from github and installs it to +`/opt/vmnet-helper`. + +**IMPORTANT**: The vmnet-helper executable and the directory where it is +installed must be owned by root and may not be modifiable by +unprivileged users. + +### Grant permission to run vmnet-helper + +The vment-helper process must run as root to create a vmnet interface. +To allow users in the `staff` group to run the vmnet helper without a +password, you can install the default sudoers rule: + +```shell +sudo install -m 0640 /opt/vmnet-helper/share/doc/vmnet-helper/sudoers.d/vmnet-helper /etc/sudoers.d/ +``` + +You can change the sudoers configuration to allow access to specific +users or other groups. + +### Usage + +```shell +minikube start --driver vfkit --network vmnet-shared +``` + +{{% /tab %}} +{{% tab builtin %}} +### Usage + +```shell +minikube start --driver vfkit [--network nat] +```` + +The `nat` network is used by default if the `--network` option is not +specified. + +{{% /tab %}} +{{% /tabs %}} ## Issues @@ -31,3 +102,21 @@ New updates to macOS often require an updated vfkit driver. To upgrade: * To check your current version, run: `vfkit -v` * If the version didn't change after upgrading verify the correct VFKit is in the path. run: `which vfkit` +### Troubleshooting the vmnet-shared network + +Check for errors in vment-helper log: + +```shell +$MINIKUBE_HOME/.minikube/machines/MACHINE-NAME/vmnet-helper.log +``` + +Check that the `vmnet-helper` process is running: + +```shell +ps au | grep vmnet-helper | grep -v grep +``` + +If the helper is not running restart the minikube cluster. + +For help with vment-helper please use the +[discussions](https://github.com/nirs/vmnet-helper/discussions).