fix(controller): remove finaliser in controller if k8s node owning the volume is not present

Signed-off-by: Abhilash Shetty <[email protected]>

Author: abhilashshetty04

pkg/driver/controller.go

@@ -19,6 +19,7 @@ import (
 	corev1 "k8s.io/api/core/v1"
 	k8serror "k8s.io/apimachinery/pkg/api/errors"
 	apimeta "k8s.io/apimachinery/pkg/api/meta"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/labels"
 	kubeinformers "k8s.io/client-go/informers"
 	"k8s.io/client-go/kubernetes"
@@ -365,7 +366,7 @@ func CreateLVMVolume(ctx context.Context, req *csi.CreateVolumeRequest,
 	}
 	klog.Infof("scheduling the volume %s/%s on node %s",
 		params.VgPattern.String(), volName, owner)
-		
+	
 	volObj, err := volbuilder.NewBuilder().
 		WithName(volName).
 		WithCapacity(capacity).
@@ -472,6 +473,28 @@ func (cs *controller) DeleteVolume(
 	return csipayload.NewDeleteVolumeResponseBuilder().Build(), nil
 }
 
+// Checks if k8s node is present. If the check fails with any other reason then NotFound.
+// We will return true and assume node is present.
+func isNodePresent(nodeId string) bool {
+	cfg, err := k8sapi.Config().Get()
+	if err != nil {
+		return true
+	}
+
+	kubeClient, err := kubernetes.NewForConfig(cfg)
+	if err != nil {
+		return true
+	}
+
+	_, err = kubeClient.CoreV1().Nodes().Get(context.TODO(), nodeId, metav1.GetOptions{})
+	if err != nil {
+		if k8serror.IsNotFound(err) {
+			return false
+		}
+	}
+	return true
+}
+
 func (cs *controller) deleteVolume(ctx context.Context, volumeID string) error {
 	klog.Infof("received request to delete volume %q", volumeID)
 	vol, err := lvm.GetLVMVolume(volumeID)
@@ -482,7 +505,6 @@ func (cs *controller) deleteVolume(ctx context.Context, volumeID string) error {
 		return errors.Wrapf(err,
 			"failed to get volume for {%s}", volumeID)
 	}
-
 	// if volume is not already triggered for deletion, delete the volume.
 	// otherwise, just wait for the existing deletion operation to complete.
 	if vol.GetDeletionTimestamp() == nil {
@@ -491,6 +513,17 @@ func (cs *controller) deleteVolume(ctx context.Context, volumeID string) error {
 				"failed to handle delete volume request for {%s}", volumeID)
 		}
 	}
+	present := isNodePresent(vol.Spec.OwnerNodeID)
+	if !present {
+		klog.Warningf("Removing finalizer as node %s is not present in cluster", vol.Spec.OwnerNodeID)
+		if err = lvm.RemoveVolFinalizer(vol); err != nil {
+			return err
+		}
+		if err = lvm.VerifyLVMVolumeDestroy(volumeID); err != nil {
+			return err
+		}
+		return nil
+	}
 	if err = lvm.WaitForLVMVolumeDestroy(ctx, volumeID); err != nil {
 		return err
 	}

pkg/lvm/volume.go

@@ -104,7 +104,7 @@ func ProvisionVolume(vol *apis.LVMVolume) (*apis.LVMVolume, error) {
 func DeleteVolume(volumeID string) (err error) {
 	err = volbuilder.NewKubeclient().WithNamespace(LvmNamespace).Delete(volumeID)
 	if err == nil {
-		klog.Infof("deprovisioned volume %s", volumeID)
+		klog.Infof("deprovisioning volume %s", volumeID)
 	}
 
 	return
@@ -164,6 +164,23 @@ func WaitForLVMVolumeDestroy(ctx context.Context, volumeID string) error {
 	}
 }
 
+// Verifies if lvmvolume CR got removed after removing finaliser.
+func VerifyLVMVolumeDestroy(volumeID string) error {
+	for i := 0; i < 10; i++ {
+		_, err := GetLVMVolume(volumeID)
+		if err != nil {
+			if k8serror.IsNotFound(err) {
+				return nil
+			}
+			return status.Errorf(codes.Internal,
+				"lvmvolume: destroy wait failed, not able to get the volume %s %s", volumeID, err.Error())
+		}
+		time.Sleep(time.Duration(100 * time.Millisecond))
+	}
+	return status.Errorf(codes.FailedPrecondition,
+		"lvmvolume: %s not removed after removing finaliser", volumeID)
+}
+
 // GetLVMVolumeState returns LVMVolume OwnerNode and State for
 // the given volume. CreateVolume request may call it again and
 // again until volume is "Ready".
@@ -219,6 +236,9 @@ func RemoveVolFinalizer(vol *apis.LVMVolume) error {
 	vol.Finalizers = nil
 
 	_, err := volbuilder.NewKubeclient().WithNamespace(LvmNamespace).Update(vol)
+	if err != nil {
+		klog.Infof("Finalizer removed successfully for %s", vol.Name)
+	}
 	return err
 }