check vpn connection state asynchronous (#343)

cmd/metal-api/internal/headscale/client.go

@@ -119,13 +119,20 @@ func (h *HeadscaleClient) CreatePreAuthKey(namespace string, expiration time.Tim
 	return resp.PreAuthKey.Key, nil
 }
 
-func (h *HeadscaleClient) DescribeMachine(machineID, projectID string) (connected bool, err error) {
-	machine, err := h.getMachine(machineID, projectID)
-	if err != nil || machine == nil {
-		return false, err
+type connectedMap map[string]bool
+
+func (h *HeadscaleClient) MachinesConnected() (connectedMap, error) {
+	resp, err := h.client.ListMachines(h.ctx, &headscalev1.ListMachinesRequest{})
+	if err != nil || resp == nil {
+		return nil, fmt.Errorf("failed to list machines: %w", err)
+	}
+	result := connectedMap{}
+	for _, m := range resp.Machines {
+		connected := m.LastSeen.AsTime().After(time.Now().Add(-5 * time.Minute))
+		result[m.Name] = connected
 	}
 
-	return machine.LastSeen.AsTime().After(time.Now().Add(-5 * time.Minute)), nil
+	return result, nil
 }
 
 // DeleteMachine removes the node entry from headscale DB

cmd/metal-api/internal/metal/machine.go

@@ -419,4 +419,5 @@ type FirmwareUpdate struct {
 type MachineVPN struct {
 	ControlPlaneAddress string `rethinkdb:"address" json:"address"`
 	AuthKey             string `rethinkdb:"auth_key" json:"auth_key"`
+	Connected           bool   `rethinkdb:"connected" json:"connected"`
 }

cmd/metal-api/internal/service/firewall-service.go

@@ -135,15 +135,7 @@ func (r *firewallResource) findFirewall(request *restful.Request, response *rest
 		return
 	}
 
-	var connected bool
-	if fw.Allocation != nil && fw.Allocation.VPN != nil && r.headscaleClient != nil {
-		connected, err = r.headscaleClient.DescribeMachine(fw.ID, fw.Allocation.Project)
-		if err != nil {
-			r.log.Errorw("unable to get headscale connected state, ignoring", "firewall", fw.ID)
-		}
-	}
-
-	resp, err := makeFirewallResponse(fw, connected, r.ds)
+	resp, err := makeFirewallResponse(fw, r.ds)
 	if err != nil {
 		r.sendError(request, response, defaultError(err))
 		return
@@ -262,16 +254,11 @@ func (r firewallResource) setVPNConfigInSpec(allocationSpec *machineAllocationSp
 	return nil
 }
 
-func makeFirewallResponse(fw *metal.Machine, connectedToVPN bool, ds *datastore.RethinkStore) (*v1.FirewallResponse, error) {
+func makeFirewallResponse(fw *metal.Machine, ds *datastore.RethinkStore) (*v1.FirewallResponse, error) {
 	ms, err := makeMachineResponse(fw, ds)
 	if err != nil {
 		return nil, err
 	}
-
-	if ms.VPN == nil {
-		ms.VPN = &v1.MachineVPN{}
-	}
-	ms.VPN.Connected = connectedToVPN
 	return &v1.FirewallResponse{MachineResponse: *ms}, nil
 }
 

cmd/metal-api/internal/service/v1/machine.go

@@ -22,7 +22,6 @@ type MachineBase struct {
 	Liveliness               string                          `json:"liveliness" description:"the liveliness of this machine"`
 	RecentProvisioningEvents MachineRecentProvisioningEvents `json:"events" description:"recent events of this machine during provisioning"`
 	Tags                     []string                        `json:"tags" description:"tags for this machine"`
-	VPN                      *MachineVPN                     `json:"vpn" description:"vpn connection info for machine" optional:"true"`
 }
 
 type MachineAllocation struct {
@@ -559,5 +558,6 @@ func NewMachineVPN(m *metal.MachineVPN) *MachineVPN {
 	return &MachineVPN{
 		ControlPlaneAddress: m.ControlPlaneAddress,
 		AuthKey:             m.AuthKey,
+		Connected:           m.Connected,
 	}
 }

cmd/metal-api/main.go

@@ -34,6 +34,7 @@ import (
 	"github.com/go-openapi/spec"
 	"github.com/spf13/cobra"
 	"github.com/spf13/viper"
+	"go.uber.org/multierr"
 	"go.uber.org/zap"
 	"go.uber.org/zap/zapcore"
 
@@ -166,6 +167,16 @@ var machineLiveliness = &cobra.Command{
 		return evaluateLiveliness()
 	},
 }
+var machineConnectedToVPN = &cobra.Command{
+	Use:     "machines-vpn-connected",
+	Short:   "evaluates whether machines connected to vpn",
+	Version: v.V.String(),
+	RunE: func(cmd *cobra.Command, args []string) error {
+		initLogging()
+		initHeadscale()
+		return evaluateVPNConnected()
+	},
+}
 
 var deleteOrphanImagesCmd = &cobra.Command{
 	Use:     "delete-orphan-images",
@@ -201,6 +212,7 @@ func init() {
 		resurrectMachines,
 		machineLiveliness,
 		deleteOrphanImagesCmd,
+		machineConnectedToVPN,
 	)
 
 	rootCmd.Flags().StringP("config", "c", "", "alternative path to config file")
@@ -842,6 +854,46 @@ func evaluateLiveliness() error {
 	return nil
 }
 
+func evaluateVPNConnected() error {
+	err := connectDataStore()
+	if err != nil {
+		return err
+	}
+
+	ms, err := ds.ListMachines()
+	if err != nil {
+		return err
+	}
+
+	connectedMap, err := headscaleClient.MachinesConnected()
+	if err != nil {
+		return err
+	}
+
+	var updateErr error
+	for _, m := range ms {
+		m := m
+		if m.Allocation == nil || m.Allocation.VPN == nil {
+			continue
+		}
+		connected := connectedMap[m.ID]
+		if m.Allocation.VPN.Connected == connected {
+			continue
+		}
+
+		old := m
+		m.Allocation.VPN.Connected = connected
+		err := ds.UpdateMachine(&old, &m)
+		if err != nil {
+			updateErr = multierr.Append(updateErr, err)
+			logger.Errorw("unable to update vpn connected state, continue anyway", "machine", m.ID, "error", err)
+			continue
+		}
+		logger.Infow("updated vpn connected state", "machine", m.ID, "connected", connected)
+	}
+	return updateErr
+}
+
 func run() error {
 	initRestServices(true)
 

spec/metal-api.json

@@ -1204,10 +1204,6 @@
             "type": "string"
           },
           "type": "array"
-        },
-        "vpn": {
-          "$ref": "#/definitions/v1.MachineVPN",
-          "description": "vpn connection info for machine"
         }
       },
       "required": [
@@ -1986,10 +1982,6 @@
             "type": "string"
           },
           "type": "array"
-        },
-        "vpn": {
-          "$ref": "#/definitions/v1.MachineVPN",
-          "description": "vpn connection info for machine"
         }
       },
       "required": [
@@ -2462,10 +2454,6 @@
             "type": "string"
           },
           "type": "array"
-        },
-        "vpn": {
-          "$ref": "#/definitions/v1.MachineVPN",
-          "description": "vpn connection info for machine"
         }
       },
       "required": [
@@ -2794,10 +2782,6 @@
             "type": "string"
           },
           "type": "array"
-        },
-        "vpn": {
-          "$ref": "#/definitions/v1.MachineVPN",
-          "description": "vpn connection info for machine"
         }
       },
       "required": [