support resizing pvc only

Signed-off-by: Abner-1 <[email protected]>
openkruise · Sep 19, 2024 · 2379fe1 · 2379fe1
1 parent 6d57029
commit 2379fe1
Show file tree

Hide file tree

Showing 6 changed files with 306 additions and 8 deletions.
diff --git a/.github/workflows/e2e-1.18.yaml b/.github/workflows/e2e-1.18.yaml
@@ -79,7 +79,7 @@ jobs:
           export KUBECONFIG=/home/runner/.kube/config
           make ginkgo
           set +e
-          ./bin/ginkgo -timeout 60m -v --focus='\[apps\] AppStatefulSetStorage' test/e2e
+          ./bin/ginkgo -timeout 60m -p -v --focus='\[apps\] AppStatefulSetStorage' test/e2e
           retVal=$?
           restartCount=$(kubectl get pod -n kruise-system -l control-plane=controller-manager --no-headers | awk '{print $4}')
           if [ "${restartCount}" -eq "0" ];then

diff --git a/.github/workflows/e2e-1.24.yaml b/.github/workflows/e2e-1.24.yaml
@@ -80,7 +80,7 @@ jobs:
           export KUBECONFIG=/home/runner/.kube/config
           make ginkgo
           set +e
-          ./bin/ginkgo -timeout 60m -v --focus='\[apps\] AppStatefulSetStorage' test/e2e
+          ./bin/ginkgo -timeout 60m -p -v --focus='\[apps\] AppStatefulSetStorage' test/e2e
           retVal=$?
           restartCount=$(kubectl get pod -n kruise-system -l control-plane=controller-manager --no-headers | awk '{print $4}')
           if [ "${restartCount}" -eq "0" ];then

diff --git a/.github/workflows/e2e-1.26.yaml b/.github/workflows/e2e-1.26.yaml
@@ -79,7 +79,7 @@ jobs:
           export KUBECONFIG=/home/runner/.kube/config
           make ginkgo
           set +e
-          ./bin/ginkgo -timeout 60m -v --focus='\[apps\] AppStatefulSetStorage' test/e2e
+          ./bin/ginkgo -timeout 60m -p -v --focus='\[apps\] AppStatefulSetStorage' test/e2e
           retVal=$?
           restartCount=$(kubectl get pod -n kruise-system -l control-plane=controller-manager --no-headers | awk '{print $4}')
           if [ "${restartCount}" -eq "0" ];then

diff --git a/.github/workflows/e2e-1.28.yaml b/.github/workflows/e2e-1.28.yaml
@@ -79,7 +79,7 @@ jobs:
           export KUBECONFIG=/home/runner/.kube/config
           make ginkgo
           set +e
-          ./bin/ginkgo -timeout 60m -v --focus='\[apps\] AppStatefulSetStorage' test/e2e
+          ./bin/ginkgo -timeout 60m -p -v --focus='\[apps\] AppStatefulSetStorage' test/e2e
           retVal=$?
           restartCount=$(kubectl get pod -n kruise-system -l control-plane=controller-manager --no-headers | awk '{print $4}')
           if [ "${restartCount}" -eq "0" ];then

diff --git a/pkg/controller/statefulset/stateful_set_control.go b/pkg/controller/statefulset/stateful_set_control.go
@@ -619,6 +619,8 @@ func (ssc *defaultStatefulSetControl) rollingUpdateStatefulsetPods(
 				klog.V(4).ErrorS(err, "StatefulSet check owned pvcs unready",
 					"statefulSet", klog.KObj(set), "pod", klog.KObj(replicas[target]))
 			}
+			klog.V(4).InfoS("owned pvcs unready, added in unavailable Pods",
+				"statefulSet", klog.KObj(set), "pod", klog.KObj(replicas[target]))
 			unavailablePods.Insert(replicas[target].Name)
 		}
 	}
@@ -634,8 +636,16 @@ func (ssc *defaultStatefulSetControl) rollingUpdateStatefulsetPods(
 	klog.V(3).InfoS("Prepare to update pods indexes for StatefulSet", "statefulSet", klog.KObj(set), "podIndexes", updateIndexes)
 	// update pods in sequence
 	for _, target := range updateIndexes {
+		var pvcMatched bool = true
+		if utilfeature.DefaultFeatureGate.Enabled(features.StatefulSetAutoResizePVCGate) &&
+			set.Spec.VolumeClaimUpdateStrategy.Type == appsv1beta1.OnPodRollingUpdateVolumeClaimUpdateStrategyType {
+			if pvcMatched, err = ssc.podControl.IsClaimsCompatible(set, replicas[target]); err != nil {
+				return status, err
+			}
+		}
+
 		// the target is already up-to-date, go to next
-		if getPodRevision(replicas[target]) == updateRevision.Name {
+		if getPodRevision(replicas[target]) == updateRevision.Name && pvcMatched {
 			continue
 		}
 
@@ -653,9 +663,7 @@ func (ssc *defaultStatefulSetControl) rollingUpdateStatefulsetPods(
 		if utilfeature.DefaultFeatureGate.Enabled(features.StatefulSetAutoResizePVCGate) &&
 			set.Spec.VolumeClaimUpdateStrategy.Type == appsv1beta1.OnPodRollingUpdateVolumeClaimUpdateStrategyType {
 			// resize pvc if necessary and wait for resize completed
-			if match, err := ssc.podControl.IsClaimsCompatible(set, replicas[target]); err != nil {
-				return status, err
-			} else if !match {
+			if !pvcMatched {
 				err = ssc.podControl.TryPatchPVC(set, replicas[target])
 				if err != nil {
 					return status, err

diff --git a/test/e2e/apps/statefulset.go b/test/e2e/apps/statefulset.go
@@ -385,6 +385,248 @@ var _ = SIGDescribe("AppStatefulSetStorage", func() {
 		})
 	})
 
+	ginkgo.Describe("Resize PVC only", func() {
+		oldSize, newSize := "1Gi", "2Gi"
+		injectSC := func(podUpdatePolicy appsv1beta1.PodUpdateStrategyType, ss *appsv1beta1.StatefulSet, volumeClaimUpdateStrategy appsv1beta1.VolumeClaimUpdateStrategyType, scNames ...string) {
+			if podUpdatePolicy == appsv1beta1.InPlaceIfPossiblePodUpdateStrategyType {
+				ss.Spec.UpdateStrategy.RollingUpdate = &appsv1beta1.RollingUpdateStatefulSetStrategy{
+					PodUpdatePolicy: podUpdatePolicy,
+				}
+				ss.Spec.Template.Spec.ReadinessGates = append(ss.Spec.Template.Spec.ReadinessGates, v1.PodReadinessGate{ConditionType: appspub.InPlaceUpdateReady})
+			}
+
+			ss.Spec.VolumeClaimUpdateStrategy = appsv1beta1.VolumeClaimUpdateStrategy{
+				Type: volumeClaimUpdateStrategy,
+			}
+			if len(ss.Spec.VolumeClaimTemplates) != len(scNames) {
+				return
+			}
+			quantity, _ := resource.ParseQuantity(oldSize)
+			for i := range scNames {
+				ss.Spec.VolumeClaimTemplates[i].Spec.StorageClassName = &scNames[i]
+				ss.Spec.VolumeClaimTemplates[i].Spec.Resources.Requests[v1.ResourceStorage] = quantity
+			}
+		}
+		resizeVCT := func(ss *appsv1beta1.StatefulSet, size string, resizeElementSize int) {
+			quantity, _ := resource.ParseQuantity(size)
+			for i := range ss.Spec.VolumeClaimTemplates {
+				if i >= resizeElementSize {
+					return
+				}
+				ss.Spec.VolumeClaimTemplates[i].Spec.Resources.Requests[v1.ResourceStorage] = quantity
+			}
+		}
+		ssName := "ss"
+		labels := map[string]string{
+			"foo": "bar",
+			"baz": "blah",
+		}
+		headlessSvcName := "test"
+		var ss *appsv1beta1.StatefulSet
+		var sst *framework.StatefulSetTester
+
+		ginkgo.BeforeEach(func() {
+			ginkgo.By("Creating service " + headlessSvcName + " in namespace " + ns)
+			headlessService := framework.CreateServiceSpec(headlessSvcName, "", true, labels)
+			_, err := c.CoreV1().Services(ns).Create(context.TODO(), headlessService, metav1.CreateOptions{})
+			framework.ExpectNoError(err)
+			sst = framework.NewStatefulSetTester(c, kc)
+		})
+
+		ginkgo.AfterEach(func() {
+			if ginkgo.CurrentGinkgoTestDescription().Failed {
+				framework.DumpDebugInfo(c, ns)
+			}
+			framework.Logf("Deleting all statefulset in ns %v", ns)
+			framework.DeleteAllStatefulSets(c, kc, ns)
+		})
+		validateExpandVCT := func(vctNumber int, injectSCFn func(ss *appsv1beta1.StatefulSet), updateFn func(ss *appsv1beta1.StatefulSet), expectErr bool) {
+			ctx := context.TODO()
+			ginkgo.By("Creating statefulset " + ssName + " in namespace " + ns)
+			vms := []v1.VolumeMount{}
+			for i := 0; i < vctNumber; i++ {
+				vms = append(vms, v1.VolumeMount{
+					Name:      fmt.Sprintf("data%d", i),
+					MountPath: fmt.Sprintf("/data%d", i),
+				})
+			}
+			ss = framework.NewStatefulSet(ssName, ns, headlessSvcName, 3, vms, nil, labels)
+			injectSCFn(ss)
+
+			_, err := kc.AppsV1beta1().StatefulSets(ns).Create(ctx, ss, metav1.CreateOptions{})
+			framework.ExpectNoError(err)
+
+			sst = framework.NewStatefulSetTester(c, kc)
+			waitForStatus(ctx, c, kc, ss)
+			sst.WaitForStatusReplicas(ss, 3)
+			sst.WaitForStatusReadyReplicas(ss, 3)
+
+			ginkgo.By("expand volume claim size")
+			ss, err = updateStatefulSetWithRetries(ctx, kc, ns, ss.Name, updateFn)
+			if expectErr {
+				// error is expected
+				if err == nil {
+					framework.Failf("unexpected to update pvc with sc can not expand, but get error %v", err)
+				}
+				return
+			} else {
+				framework.ExpectNoError(err)
+			}
+
+			// we need to ensure we wait for all the new ones to show up, not
+			// just for any random 3
+			waitForStatus(ctx, c, kc, ss)
+			waitForPVCCapacity(ctx, c, kc, ss, func(pvc, template resource.Quantity) bool {
+				return pvc.Cmp(template) == 0
+			})
+
+			ginkgo.By("Confirming 3 pvc capacity consistent with spec")
+			sst.WaitForStatusReplicas(ss, 3)
+			sst.WaitForStatusReadyReplicas(ss, 3)
+			sst.WaitForStatusPVCReadyReplicas(ss, 3)
+		}
+
+		ginkgo.It("recreate_with_sc_can_expand", func() {
+			injectFn := func(ss *appsv1beta1.StatefulSet) {
+				injectSC(appsv1beta1.RecreatePodUpdateStrategyType, ss, appsv1beta1.OnPodRollingUpdateVolumeClaimUpdateStrategyType, canExpandSC)
+			}
+			updateFn := func(update *appsv1beta1.StatefulSet) {
+				resizeVCT(update, newSize, 2)
+			}
+			validateExpandVCT(1, injectFn, updateFn, false)
+		})
+
+		ginkgo.It("recreate_with_sc_cannot_expand", func() {
+			injectFn := func(ss *appsv1beta1.StatefulSet) {
+				injectSC(appsv1beta1.RecreatePodUpdateStrategyType, ss, appsv1beta1.OnPodRollingUpdateVolumeClaimUpdateStrategyType, cannotExpandSC)
+			}
+			updateFn := func(update *appsv1beta1.StatefulSet) {
+				resizeVCT(update, newSize, 2)
+			}
+			validateExpandVCT(1, injectFn, updateFn, true)
+		})
+
+		ginkgo.It("recreate_with_2sc_can_expand", func() {
+			injectFn := func(ss *appsv1beta1.StatefulSet) {
+				injectSC(appsv1beta1.RecreatePodUpdateStrategyType, ss, appsv1beta1.OnPodRollingUpdateVolumeClaimUpdateStrategyType, canExpandSC, canExpandSC)
+			}
+			updateFn := func(update *appsv1beta1.StatefulSet) {
+				resizeVCT(update, newSize, 2)
+			}
+			validateExpandVCT(2, injectFn, updateFn, false)
+		})
+
+		ginkgo.It("recreate_with_2sc_cannot_expand", func() {
+			injectFn := func(ss *appsv1beta1.StatefulSet) {
+				injectSC(appsv1beta1.RecreatePodUpdateStrategyType, ss, appsv1beta1.OnPodRollingUpdateVolumeClaimUpdateStrategyType, cannotExpandSC, cannotExpandSC)
+			}
+			updateFn := func(update *appsv1beta1.StatefulSet) {
+				resizeVCT(update, newSize, 2)
+			}
+			validateExpandVCT(2, injectFn, updateFn, true)
+		})
+
+		ginkgo.It("recreate_expand_only_can_with_mixed_sc", func() {
+			injectFn := func(ss *appsv1beta1.StatefulSet) {
+				injectSC(appsv1beta1.RecreatePodUpdateStrategyType, ss, appsv1beta1.OnPodRollingUpdateVolumeClaimUpdateStrategyType, canExpandSC, cannotExpandSC)
+			}
+			updateFn := func(update *appsv1beta1.StatefulSet) {
+				resizeVCT(update, newSize, 1)
+			}
+			validateExpandVCT(2, injectFn, updateFn, false)
+		})
+
+		ginkgo.It("recreate_expand_only_cannot_with_mixed_sc", func() {
+			injectFn := func(ss *appsv1beta1.StatefulSet) {
+				injectSC(appsv1beta1.RecreatePodUpdateStrategyType, ss, appsv1beta1.OnPodRollingUpdateVolumeClaimUpdateStrategyType, cannotExpandSC, canExpandSC)
+			}
+			updateFn := func(update *appsv1beta1.StatefulSet) {
+				resizeVCT(update, newSize, 1)
+			}
+			validateExpandVCT(2, injectFn, updateFn, true)
+		})
+
+		ginkgo.It("recreate_expand_both_cannot_with_mixed_sc", func() {
+			injectFn := func(ss *appsv1beta1.StatefulSet) {
+				injectSC(appsv1beta1.RecreatePodUpdateStrategyType, ss, appsv1beta1.OnPodRollingUpdateVolumeClaimUpdateStrategyType, cannotExpandSC, canExpandSC)
+			}
+			updateFn := func(update *appsv1beta1.StatefulSet) {
+				resizeVCT(update, newSize, 2)
+			}
+			validateExpandVCT(2, injectFn, updateFn, true)
+		})
+
+		ginkgo.It("inplace_with_sc_can_expand", func() {
+			injectFn := func(ss *appsv1beta1.StatefulSet) {
+				injectSC(appsv1beta1.InPlaceIfPossiblePodUpdateStrategyType, ss, appsv1beta1.OnPodRollingUpdateVolumeClaimUpdateStrategyType, canExpandSC)
+			}
+			updateFn := func(update *appsv1beta1.StatefulSet) {
+				resizeVCT(update, newSize, 2)
+			}
+			validateExpandVCT(1, injectFn, updateFn, false)
+		})
+
+		ginkgo.It("inplace_with_sc_cannot_expand", func() {
+			injectFn := func(ss *appsv1beta1.StatefulSet) {
+				injectSC(appsv1beta1.InPlaceIfPossiblePodUpdateStrategyType, ss, appsv1beta1.OnPodRollingUpdateVolumeClaimUpdateStrategyType, cannotExpandSC)
+			}
+			updateFn := func(update *appsv1beta1.StatefulSet) {
+				resizeVCT(update, newSize, 2)
+			}
+			validateExpandVCT(1, injectFn, updateFn, true)
+		})
+
+		ginkgo.It("inplace_with_2sc_can_expand", func() {
+			injectFn := func(ss *appsv1beta1.StatefulSet) {
+				injectSC(appsv1beta1.InPlaceIfPossiblePodUpdateStrategyType, ss, appsv1beta1.OnPodRollingUpdateVolumeClaimUpdateStrategyType, canExpandSC, canExpandSC)
+			}
+			updateFn := func(update *appsv1beta1.StatefulSet) {
+				resizeVCT(update, newSize, 2)
+			}
+			validateExpandVCT(2, injectFn, updateFn, false)
+		})
+
+		ginkgo.It("inplace_with_2sc_cannot_expand", func() {
+			injectFn := func(ss *appsv1beta1.StatefulSet) {
+				injectSC(appsv1beta1.InPlaceIfPossiblePodUpdateStrategyType, ss, appsv1beta1.OnPodRollingUpdateVolumeClaimUpdateStrategyType, cannotExpandSC, cannotExpandSC)
+			}
+			updateFn := func(update *appsv1beta1.StatefulSet) {
+				resizeVCT(update, newSize, 2)
+			}
+			validateExpandVCT(2, injectFn, updateFn, true)
+		})
+
+		ginkgo.It("inplace_expand_only_can_with_mixed_sc", func() {
+			injectFn := func(ss *appsv1beta1.StatefulSet) {
+				injectSC(appsv1beta1.InPlaceIfPossiblePodUpdateStrategyType, ss, appsv1beta1.OnPodRollingUpdateVolumeClaimUpdateStrategyType, canExpandSC, cannotExpandSC)
+			}
+			updateFn := func(update *appsv1beta1.StatefulSet) {
+				resizeVCT(update, newSize, 1)
+			}
+			validateExpandVCT(2, injectFn, updateFn, false)
+		})
+
+		ginkgo.It("inplace_expand_only_cannot_with_mixed_sc", func() {
+			injectFn := func(ss *appsv1beta1.StatefulSet) {
+				injectSC(appsv1beta1.InPlaceIfPossiblePodUpdateStrategyType, ss, appsv1beta1.OnPodRollingUpdateVolumeClaimUpdateStrategyType, cannotExpandSC, canExpandSC)
+			}
+			updateFn := func(update *appsv1beta1.StatefulSet) {
+				resizeVCT(update, newSize, 1)
+			}
+			validateExpandVCT(2, injectFn, updateFn, true)
+		})
+
+		ginkgo.It("inplace_expand_both_cannot_with_mixed_sc", func() {
+			injectFn := func(ss *appsv1beta1.StatefulSet) {
+				injectSC(appsv1beta1.InPlaceIfPossiblePodUpdateStrategyType, ss, appsv1beta1.OnPodRollingUpdateVolumeClaimUpdateStrategyType, cannotExpandSC, canExpandSC)
+			}
+			updateFn := func(update *appsv1beta1.StatefulSet) {
+				resizeVCT(update, newSize, 2)
+			}
+			validateExpandVCT(2, injectFn, updateFn, true)
+		})
+	})
+
 	ginkgo.Describe("Resize PVC with rollback", func() {
 		oldSize, newSize := "1Gi", "2Gi"
 		injectSC := func(podUpdatePolicy appsv1beta1.PodUpdateStrategyType, ss *appsv1beta1.StatefulSet, volumeClaimUpdateStrategy appsv1beta1.VolumeClaimUpdateStrategyType, scNames ...string) {
@@ -518,6 +760,54 @@ var _ = SIGDescribe("AppStatefulSetStorage", func() {
 			sst.WaitForStatusReadyReplicas(ss, 3)
 		}
 
+		// support reconcile when vct resize only
+		ginkgo.It("partition 2, rollback only image", func() {
+			injectFn := func(ss *appsv1beta1.StatefulSet) {
+				injectSC(appsv1beta1.RecreatePodUpdateStrategyType, ss, appsv1beta1.OnPodRollingUpdateVolumeClaimUpdateStrategyType, canExpandSC)
+			}
+			updateFn := func(update *appsv1beta1.StatefulSet) {
+				update.Spec.Template.Spec.Containers[0].Image = newImage
+				resizeVCT(update, newSize, 2)
+				partition := int32(2)
+				update.Spec.UpdateStrategy.RollingUpdate.Partition = &partition
+			}
+			rollbackFn := func(update *appsv1beta1.StatefulSet) {
+				ginkgo.By("rollback only image, remain new pvc size")
+				update.Spec.Template.Spec.Containers[0].Image = oldImage
+				partition := int32(0)
+				update.Spec.UpdateStrategy.RollingUpdate.Partition = &partition
+			}
+			validateUpdateVCTAndRollback(1, injectFn, updateFn, rollbackFn)
+			ctx := context.TODO()
+			waitForPVCCapacity(ctx, c, kc, ss, func(pvc, template resource.Quantity) bool {
+				return pvc.Cmp(template) == 0
+			})
+
+			ginkgo.By("Confirming 3 pvc capacity consistent with spec")
+			sst.WaitForStatusReplicas(ss, 3)
+			sst.WaitForStatusReadyReplicas(ss, 3)
+			sst.WaitForStatusPVCReadyReplicas(ss, 3)
+
+			// update to another version
+			ginkgo.By("After rollbacked, continue to expand pvc size")
+			var err error
+			ss, err = updateStatefulSetWithRetries(ctx, kc, ns, ss.Name, func(set *appsv1beta1.StatefulSet) {
+				resizeVCT(set, "3Gi", 2)
+			})
+			framework.ExpectNoError(err)
+
+			sst.WaitForStatusReadyReplicas(ss, 3)
+			waitForStatus(ctx, c, kc, ss)
+			waitForPVCCapacity(ctx, c, kc, ss, func(pvc, template resource.Quantity) bool {
+				return pvc.Cmp(template) == 0
+			})
+
+			ginkgo.By("Confirming 3 pvc capacity consistent with spec")
+			sst.WaitForStatusReplicas(ss, 3)
+			sst.WaitForStatusReadyReplicas(ss, 3)
+			sst.WaitForStatusPVCReadyReplicas(ss, 3)
+		})
+
 		ginkgo.It("partition 2, rollback only image and update to another image", func() {
 			injectFn := func(ss *appsv1beta1.StatefulSet) {
 				injectSC(appsv1beta1.RecreatePodUpdateStrategyType, ss, appsv1beta1.OnPodRollingUpdateVolumeClaimUpdateStrategyType, canExpandSC)