Shared network for vfkit driver using vmnet-helper

cmd/minikube/cmd/start_flags.go

@@ -30,6 +30,7 @@ import (
 	"k8s.io/klog/v2"
 	"k8s.io/minikube/pkg/drivers/kic"
 	"k8s.io/minikube/pkg/drivers/kic/oci"
+	"k8s.io/minikube/pkg/drivers/vmnet"
 	"k8s.io/minikube/pkg/minikube/bootstrapper/bsutil"
 	"k8s.io/minikube/pkg/minikube/bootstrapper/bsutil/kverify"
 	"k8s.io/minikube/pkg/minikube/cni"
@@ -197,7 +198,7 @@ func initMinikubeFlags() {
 	startCmd.Flags().Bool(noKubernetes, false, "If set, minikube VM/container will start without starting or configuring Kubernetes. (only works on new clusters)")
 	startCmd.Flags().Bool(deleteOnFailure, false, "If set, delete the current cluster if start fails and try again. Defaults to false.")
 	startCmd.Flags().Bool(forceSystemd, false, "If set, force the container runtime to use systemd as cgroup manager. Defaults to false.")
-	startCmd.Flags().String(network, "", "network to run minikube with. Now it is used by docker/podman and KVM drivers. If left empty, minikube will create a new network.")
+	startCmd.Flags().String(network, "", "network to run minikube with. Used by docker/podman, qemu, kvm, and vfkit drivers. If left empty, minikube will create a new network.")
 	startCmd.Flags().StringVarP(&outputFormat, "output", "o", "text", "Format to print stdout in. Options include: [text,json]")
 	startCmd.Flags().String(trace, "", "Send trace events. Options include: [gcp]")
 	startCmd.Flags().Int(extraDisks, 0, "Number of extra disks created and attached to the minikube VM (currently only implemented for hyperkit, kvm2, and qemu2 drivers)")
@@ -469,9 +470,15 @@ func getCNIConfig(cmd *cobra.Command) string {
 
 func getNetwork(driverName string) string {
 	n := viper.GetString(network)
-	if !driver.IsQEMU(driverName) {
-		return n
+	if driver.IsQEMU(driverName) {
+		return validateQemuNetwork(n)
+	} else if driver.IsVFKit(driverName) {
+		return validateVfkitNetwork(n)
 	}
+	return n
+}
+
+func validateQemuNetwork(n string) string {
 	switch n {
 	case "socket_vmnet":
 		if runtime.GOOS != "darwin" {
@@ -503,6 +510,27 @@ func getNetwork(driverName string) string {
 	return n
 }
 
+func validateVfkitNetwork(n string) string {
+	if runtime.GOOS != "darwin" {
+		exit.Message(reason.Usage, "The vfkit driver is only supported on macOS")
+	}
+	switch n {
+	case "nat":
+		// always available
+	case "vmnet-shared":
+		// "vment-shared" provides access between machines, with lower performance compared to "nat".
+		if !vmnet.HelperAvailable() {
+			exit.Message(reason.NotFoundVmnetHelper, "\n\n")
+		}
+	case "":
+		// Default to nat since it is always available and provides the best performance.
+		n = "nat"
+	default:
+		exit.Message(reason.Usage, "--network with vfkit must be 'nat' or 'vmnet-shared'")
+	}
+	return n
+}
+
 // generateNewConfigFromFlags generate a config.ClusterConfig based on flags
 func generateNewConfigFromFlags(cmd *cobra.Command, k8sVersion string, rtime string, drvName string) config.ClusterConfig {
 	var cc config.ClusterConfig
@@ -513,8 +541,8 @@ func generateNewConfigFromFlags(cmd *cobra.Command, k8sVersion string, rtime str
 		out.WarningT("With --network-plugin=cni, you will need to provide your own CNI. See --cni flag as a user-friendly alternative")
 	}
 
-	if !(driver.IsKIC(drvName) || driver.IsKVM(drvName) || driver.IsQEMU(drvName)) && viper.GetString(network) != "" {
-		out.WarningT("--network flag is only valid with the docker/podman, KVM and Qemu drivers, it will be ignored")
+	if viper.GetString(network) != "" && !driver.SupportsNetworkFlag(drvName) {
+		out.WarningT("--network flag is only valid with the docker/podman, qemu, kvm, and vfkit drivers, it will be ignored")
 	}
 
 	validateHANodeCount(cmd)

pkg/drivers/vfkit/vfkit.go

@@ -43,6 +43,7 @@ import (
 
 	"k8s.io/klog/v2"
 	pkgdrivers "k8s.io/minikube/pkg/drivers"
+	"k8s.io/minikube/pkg/drivers/vmnet"
 	"k8s.io/minikube/pkg/minikube/exit"
 	"k8s.io/minikube/pkg/minikube/firewall"
 	"k8s.io/minikube/pkg/minikube/out"
@@ -67,8 +68,10 @@ type Driver struct {
 	CPU            int
 	Memory         int
 	Cmdline        string
-	MACAddress     string
 	ExtraDisks     int
+	Network        string        // "", "nat", "vmnet-shared"
+	MACAddress     string        // For network=nat, network=""
+	VmnetHelper    *vmnet.Helper // For network=vmnet-shared
 }
 
 func NewDriver(hostName, storePath string) drivers.Driver {
@@ -136,7 +139,7 @@ func (d *Driver) GetIP() (string, error) {
 	return d.IPAddress, nil
 }
 
-func (d *Driver) GetState() (state.State, error) {
+func (d *Driver) getVfkitState() (state.State, error) {
 	pidfile := d.pidfilePath()
 	pid, err := process.ReadPidfile(pidfile)
 	if err != nil {
@@ -159,6 +162,24 @@ func (d *Driver) GetState() (state.State, error) {
 	return state.Running, nil
 }
 
+func (d *Driver) getVmnetHelperState() (state.State, error) {
+	if d.VmnetHelper == nil {
+		return state.Stopped, nil
+	}
+	return d.VmnetHelper.GetState()
+}
+
+// GetState returns driver state. Since vfkit driver may use 2 processes
+// (vmnet-helper, vfkit), this returns combined state of both processes.
+func (d *Driver) GetState() (state.State, error) {
+	if vfkitState, err := d.getVfkitState(); err != nil {
+		return state.Error, err
+	} else if vfkitState == state.Running {
+		return state.Running, nil
+	}
+	return d.getVmnetHelperState()
+}
+
 func (d *Driver) Create() error {
 	var err error
 	if d.SSHPort, err = d.GetSSHPort(); err != nil {
@@ -200,6 +221,40 @@ func (d *Driver) Create() error {
 }
 
 func (d *Driver) Start() error {
+	var helperSock, vfkitSock *os.File
+	var err error
+
+	if d.VmnetHelper != nil {
+		helperSock, vfkitSock, err = vmnet.Socketpair()
+		if err != nil {
+			return err
+		}
+		defer helperSock.Close()
+		defer vfkitSock.Close()
+
+		if err := d.VmnetHelper.Start(helperSock); err != nil {
+			return err
+		}
+
+		d.MACAddress = d.VmnetHelper.GetMACAddress()
+	}
+
+	if err := d.startVfkit(vfkitSock); err != nil {
+		return err
+	}
+
+	if err := d.setupIP(d.MACAddress); err != nil {
+		return err
+	}
+
+	log.Infof("Waiting for VM to start (ssh -p %d docker@%s)...", d.SSHPort, d.IPAddress)
+
+	return WaitForTCPWithDelay(fmt.Sprintf("%s:%d", d.IPAddress, d.SSHPort), time.Second)
+}
+
+// startVfkit starts the vfkit child process. If vfkitSock is non nil, vfkit is
+// connected to the vmnet network via the socket instead of "nat" network.
+func (d *Driver) startVfkit(vfkitSock *os.File) error {
 	machineDir := filepath.Join(d.StorePath, "machines", d.GetMachineName())
 
 	var startCmd []string
@@ -212,9 +267,15 @@ func (d *Driver) Start() error {
 	startCmd = append(startCmd,
 		"--device", fmt.Sprintf("virtio-blk,path=%s", isoPath))
 
-	var mac = d.MACAddress
-	startCmd = append(startCmd,
-		"--device", fmt.Sprintf("virtio-net,nat,mac=%s", mac))
+	if vfkitSock != nil {
+		// The guest will be able to access other guests in the vmnet network.
+		startCmd = append(startCmd,
+			"--device", fmt.Sprintf("virtio-net,fd=%d,mac=%s", vfkitSock.Fd(), d.MACAddress))
+	} else {
+		// The guest will not be able to access other guests.
+		startCmd = append(startCmd,
+			"--device", fmt.Sprintf("virtio-net,nat,mac=%s", d.MACAddress))
+	}
 
 	startCmd = append(startCmd,
 		"--device", "virtio-rng")
@@ -245,16 +306,7 @@ func (d *Driver) Start() error {
 	if err := cmd.Start(); err != nil {
 		return err
 	}
-	if err := process.WritePidfile(d.pidfilePath(), cmd.Process.Pid); err != nil {
-		return err
-	}
-	if err := d.setupIP(mac); err != nil {
-		return err
-	}
-
-	log.Infof("Waiting for VM to start (ssh -p %d docker@%s)...", d.SSHPort, d.IPAddress)
-
-	return WaitForTCPWithDelay(fmt.Sprintf("%s:%d", d.IPAddress, d.SSHPort), time.Second)
+	return process.WritePidfile(d.pidfilePath(), cmd.Process.Pid)
 }
 
 func (d *Driver) setupIP(mac string) error {
@@ -295,7 +347,7 @@ func isBootpdError(err error) bool {
 	return strings.Contains(err.Error(), "could not find an IP address")
 }
 
-func (d *Driver) Stop() error {
+func (d *Driver) stopVfkit() error {
 	if err := d.SetVFKitState("Stop"); err != nil {
 		// vfkit may be already stopped, shutting down, or not listening.
 		// We don't fallback to "HardStop" since it typically fails due to
@@ -324,6 +376,20 @@ func (d *Driver) Stop() error {
 	return nil
 }
 
+func (d *Driver) stopVmnetHelper() error {
+	if d.VmnetHelper == nil {
+		return nil
+	}
+	return d.VmnetHelper.Stop()
+}
+
+func (d *Driver) Stop() error {
+	if err := d.stopVfkit(); err != nil {
+		return err
+	}
+	return d.stopVmnetHelper()
+}
+
 func (d *Driver) Remove() error {
 	s, err := d.GetState()
 	if err != nil {
@@ -367,7 +433,7 @@ func (d *Driver) extractKernel(isoPath string) error {
 	return nil
 }
 
-func (d *Driver) Kill() error {
+func (d *Driver) killVfkit() error {
 	if err := d.SetVFKitState("HardStop"); err != nil {
 		// Typically fails with EOF due to https://github.com/crc-org/vfkit/issues/277.
 		log.Debugf("Failed to set vfkit state to 'HardStop': %s", err)
@@ -394,6 +460,20 @@ func (d *Driver) Kill() error {
 	return nil
 }
 
+func (d *Driver) killVmnetHelper() error {
+	if d.VmnetHelper == nil {
+		return nil
+	}
+	return d.VmnetHelper.Kill()
+}
+
+func (d *Driver) Kill() error {
+	if err := d.killVfkit(); err != nil {
+		return err
+	}
+	return d.killVmnetHelper()
+}
+
 func (d *Driver) StartDocker() error {
 	return fmt.Errorf("hosts without a driver cannot start docker")
 }
@@ -530,7 +610,7 @@ func (d *Driver) SetVFKitState(state string) error {
 	if err != nil {
 		return err
 	}
-	log.Debugf("set state: %+v", vmstate)
+	log.Infof("Set vfkit state: %+v", vmstate)
 	return nil
 }