Skip to content

Commit a201efa

Browse files
hakmanhakman
committed
azure: Allow independent VMs to join the cluster
Signed-off-by: Ciprian Hacman <[email protected]>
1 parent 8f61cc0 commit a201efa

File tree

2 files changed

+107
-48
lines changed

2 files changed

+107
-48
lines changed

‎upup/pkg/fi/cloudup/azure/authenticator.go‎

Lines changed: 16 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -23,6 +23,7 @@ import (
2323
"net/http"
2424
"strings"
2525

26+
"k8s.io/klog/v2"
2627
"k8s.io/kops/pkg/bootstrap"
2728
)
2829

@@ -42,22 +43,27 @@ func (h *azureAuthenticator) CreateToken(body []byte) (string, error) {
4243
if err != nil {
4344
return "", fmt.Errorf("querying instance metadata: %w", err)
4445
}
45-
46-
vmId := m.Compute.VMID
47-
if vmId == "" {
46+
if m.Compute == nil || m.Compute.VMID == "" {
4847
return "", fmt.Errorf("missing virtual machine ID")
4948
}
5049

51-
// The fully qualified VMSS VM resource ID format is:
52-
// /subscriptions/SUBSCRIPTION_ID/resourceGroups/RESOURCE_GROUP_NAME/providers/Microsoft.Compute/virtualMachineScaleSets/VMSS_NAME/virtualMachines/VMSS_INDEX
50+
token := m.Compute.VMID
51+
52+
// This should be a fully qualified VM resource ID.
5353
r := strings.Split(m.Compute.ResourceID, "/")
54-
if len(r) != 11 || r[7] != "virtualMachineScaleSets" || r[9] != "virtualMachines" {
55-
return "", fmt.Errorf("unexpected resource ID format: %q", m.Compute.ResourceID)
54+
switch len(r) {
55+
case 11:
56+
if r[7] == "virtualMachineScaleSets" && r[9] == "virtualMachines" {
57+
token += " " + r[8] + " " + r[10]
58+
}
59+
case 9:
60+
if r[7] == "virtualMachines" {
61+
token += " " + r[8]
62+
}
5663
}
57-
vmssName := r[8]
58-
vmssIndex := r[10]
5964

60-
return AzureAuthenticationTokenPrefix + vmId + " " + vmssName + " " + vmssIndex, nil
65+
klog.Info(token)
66+
return AzureAuthenticationTokenPrefix + token, nil
6167
}
6268

6369
type instanceComputeMetadata struct {

‎upup/pkg/fi/cloudup/azure/verifier.go‎

Lines changed: 91 additions & 38 deletions
Original file line numberDiff line numberDiff line change
@@ -31,6 +31,11 @@ import (
3131
"k8s.io/kops/pkg/wellknownports"
3232
)
3333

34+
const (
35+
// InstanceGroupNameTag is the key of the tag used to identify an instance group that VM belongs to.
36+
InstanceGroupNameTag = "kops.k8s.io_instancegroup"
37+
)
38+
3439
type AzureVerifierOptions struct {
3540
ClusterName string `json:"clusterName,omitempty"`
3641
}
@@ -63,52 +68,94 @@ func (a azureVerifier) VerifyToken(ctx context.Context, rawRequest *http.Request
6368
return nil, bootstrap.ErrNotThisVerifier
6469
}
6570

71+
var nodeName, igName string
72+
var addrs, challengeEndpoints []string
73+
6674
v := strings.Split(strings.TrimPrefix(token, AzureAuthenticationTokenPrefix), " ")
67-
if len(v) != 3 {
68-
return nil, fmt.Errorf("incorrect token format")
69-
}
70-
vmId := v[0]
71-
vmssName := v[1]
72-
vmssIndex := v[2]
75+
switch len(v) {
76+
case 2:
77+
vmId := v[0]
78+
vmName := v[1]
79+
80+
vm, err := a.client.vmsClient.Get(ctx, a.client.resourceGroup, vmName, nil)
81+
if err != nil {
82+
return nil, fmt.Errorf("getting info for VM %q: %w", vmName, err)
83+
}
84+
if vm.Properties == nil || vm.Properties.VMID == nil {
85+
return nil, fmt.Errorf("determining VMID for VM %q", vmName)
86+
}
87+
if vmId != *vm.Properties.VMID {
88+
return nil, fmt.Errorf("matching VMID %q to VM %q", vmId, vmName)
89+
}
90+
if vm.Properties.OSProfile == nil || vm.Properties.OSProfile.ComputerName == nil || *vm.Properties.OSProfile.ComputerName == "" {
91+
return nil, fmt.Errorf("determining ComputerName for VM %q", vmName)
92+
}
7393

74-
if !strings.HasSuffix(vmssName, "."+a.clusterName) {
75-
return nil, fmt.Errorf("matching cluster name %q to VMSS %q", a.clusterName, vmssName)
76-
}
77-
igName := strings.TrimSuffix(vmssName, "."+a.clusterName)
94+
nodeName = strings.ToLower(*vm.Properties.OSProfile.ComputerName)
7895

79-
vm, err := a.client.vmsClient.Get(ctx, a.client.resourceGroup, vmssName, vmssIndex, nil)
80-
if err != nil {
81-
return nil, fmt.Errorf("getting info for VMSS virtual machine %q #%s: %w", vmssName, vmssIndex, err)
82-
}
83-
if vm.Properties == nil || vm.Properties.VMID == nil {
84-
return nil, fmt.Errorf("determining VMID for VMSS %q virtual machine #%s", vmssName, vmssIndex)
85-
}
86-
if vmId != *vm.Properties.VMID {
87-
return nil, fmt.Errorf("matching VMID %q to VMSS %q virtual machine #%s", vmId, vmssName, vmssIndex)
88-
}
89-
if vm.Properties.OSProfile == nil || vm.Properties.OSProfile.ComputerName == nil || *vm.Properties.OSProfile.ComputerName == "" {
90-
return nil, fmt.Errorf("determining ComputerName for VMSS %q virtual machine #%s", vmssName, vmssIndex)
91-
}
92-
nodeName := strings.ToLower(*vm.Properties.OSProfile.ComputerName)
96+
if v, ok := vm.Tags[InstanceGroupNameTag]; !ok || v == nil {
97+
return nil, fmt.Errorf("determining IG name for VM %q", vmName)
98+
}
99+
igName = *vm.Tags[InstanceGroupNameTag]
93100

94-
ni, err := a.client.nisClient.GetVirtualMachineScaleSetNetworkInterface(ctx, a.client.resourceGroup, vmssName, vmssIndex, vmssName, nil)
95-
if err != nil {
96-
return nil, fmt.Errorf("getting info for VMSS network interface %q #%s: %w", vmssName, vmssIndex, err)
97-
}
101+
ni, err := a.client.nisClient.Get(ctx, a.client.resourceGroup, nodeName, nil)
102+
if err != nil {
103+
return nil, fmt.Errorf("getting info for VM network interface %q: %w", vmName, err)
104+
}
105+
106+
for _, ipc := range ni.Properties.IPConfigurations {
107+
if ipc.Properties != nil && ipc.Properties.PrivateIPAddress != nil {
108+
addrs = append(addrs, *ipc.Properties.PrivateIPAddress)
109+
challengeEndpoints = append(challengeEndpoints, net.JoinHostPort(*ipc.Properties.PrivateIPAddress, strconv.Itoa(wellknownports.NodeupChallenge)))
110+
}
111+
}
112+
113+
case 3:
114+
vmId := v[0]
115+
vmssName := v[1]
116+
vmssIndex := v[2]
117+
118+
if !strings.HasSuffix(vmssName, "."+a.clusterName) {
119+
return nil, fmt.Errorf("matching cluster name %q to VMSS %q", a.clusterName, vmssName)
120+
}
121+
igName = strings.TrimSuffix(vmssName, "."+a.clusterName)
122+
123+
vm, err := a.client.vmssVMsClient.Get(ctx, a.client.resourceGroup, vmssName, vmssIndex, nil)
124+
if err != nil {
125+
return nil, fmt.Errorf("getting info for VMSS VM %q #%s: %w", vmssName, vmssIndex, err)
126+
}
127+
if vm.Properties == nil || vm.Properties.VMID == nil {
128+
return nil, fmt.Errorf("determining VMID for VMSS %q VM #%s", vmssName, vmssIndex)
129+
}
130+
if vmId != *vm.Properties.VMID {
131+
return nil, fmt.Errorf("matching VMID %q to VMSS %q VM #%s", vmId, vmssName, vmssIndex)
132+
}
133+
if vm.Properties.OSProfile == nil || vm.Properties.OSProfile.ComputerName == nil || *vm.Properties.OSProfile.ComputerName == "" {
134+
return nil, fmt.Errorf("determining ComputerName for VMSS %q VM #%s", vmssName, vmssIndex)
135+
}
136+
nodeName = strings.ToLower(*vm.Properties.OSProfile.ComputerName)
98137

99-
var addrs []string
100-
var challengeEndpoints []string
101-
for _, ipc := range ni.Properties.IPConfigurations {
102-
if ipc.Properties != nil && ipc.Properties.PrivateIPAddress != nil {
103-
addrs = append(addrs, *ipc.Properties.PrivateIPAddress)
104-
challengeEndpoints = append(challengeEndpoints, net.JoinHostPort(*ipc.Properties.PrivateIPAddress, strconv.Itoa(wellknownports.NodeupChallenge)))
138+
ni, err := a.client.nisClient.GetVirtualMachineScaleSetNetworkInterface(ctx, a.client.resourceGroup, vmssName, vmssIndex, vmssName, nil)
139+
if err != nil {
140+
return nil, fmt.Errorf("getting info for VMSS VM network interface %q #%s: %w", vmssName, vmssIndex, err)
105141
}
142+
143+
for _, ipc := range ni.Properties.IPConfigurations {
144+
if ipc.Properties != nil && ipc.Properties.PrivateIPAddress != nil {
145+
addrs = append(addrs, *ipc.Properties.PrivateIPAddress)
146+
challengeEndpoints = append(challengeEndpoints, net.JoinHostPort(*ipc.Properties.PrivateIPAddress, strconv.Itoa(wellknownports.NodeupChallenge)))
147+
}
148+
}
149+
150+
default:
151+
return nil, fmt.Errorf("incorrect token format")
106152
}
153+
107154
if len(addrs) == 0 {
108-
return nil, fmt.Errorf("determining challenge endpoint for VMSS %q virtual machine #%s", vmssName, vmssIndex)
155+
return nil, fmt.Errorf("determining certificate alternate names for node %q", nodeName)
109156
}
110157
if len(challengeEndpoints) == 0 {
111-
return nil, fmt.Errorf("determining challenge endpoint for VMSS %q virtual machine #%s", vmssName, vmssIndex)
158+
return nil, fmt.Errorf("determining challenge endpoint for node %q", nodeName)
112159
}
113160

114161
result := &bootstrap.VerifyResult{
@@ -125,7 +172,8 @@ func (a azureVerifier) VerifyToken(ctx context.Context, rawRequest *http.Request
125172
type client struct {
126173
resourceGroup string
127174
nisClient *network.InterfacesClient
128-
vmsClient *compute.VirtualMachineScaleSetVMsClient
175+
vmsClient *compute.VirtualMachinesClient
176+
vmssVMsClient *compute.VirtualMachineScaleSetVMsClient
129177
}
130178

131179
// newClient returns a new Client.
@@ -150,7 +198,11 @@ func newClient() (*client, error) {
150198
if err != nil {
151199
return nil, fmt.Errorf("creating interfaces client: %w", err)
152200
}
153-
vmsClient, err := compute.NewVirtualMachineScaleSetVMsClient(m.Compute.SubscriptionID, cred, nil)
201+
vmsClient, err := compute.NewVirtualMachinesClient(m.Compute.SubscriptionID, cred, nil)
202+
if err != nil {
203+
return nil, fmt.Errorf("creating VMs client: %w", err)
204+
}
205+
vmssVMsClient, err := compute.NewVirtualMachineScaleSetVMsClient(m.Compute.SubscriptionID, cred, nil)
154206
if err != nil {
155207
return nil, fmt.Errorf("creating VMSSVMs client: %w", err)
156208
}
@@ -159,5 +211,6 @@ func newClient() (*client, error) {
159211
resourceGroup: m.Compute.ResourceGroupName,
160212
nisClient: nisClient,
161213
vmsClient: vmsClient,
214+
vmssVMsClient: vmssVMsClient,
162215
}, nil
163216
}

0 commit comments

Comments
 (0)