diff --git a/apis/apps/pub/inplace_update.go b/apis/apps/pub/inplace_update.go index 8274473e21..e08efd4675 100644 --- a/apis/apps/pub/inplace_update.go +++ b/apis/apps/pub/inplace_update.go @@ -45,6 +45,12 @@ const ( // RuntimeContainerMetaKey is a key in pod annotations. Kruise-daemon should report the // states of runtime containers into its value, which is a structure JSON of RuntimeContainerMetaSet type. RuntimeContainerMetaKey = "apps.kruise.io/runtime-containers-meta" + + // InPlaceUpdatePodRestartKey records the count of pod restarts + InPlaceUpdatePodRestartKey = "apps.kruise.io/inplace-update-pod-restart-count" + + // InPlaceUpdateContainersRestartKey records the count of containers restarts + InPlaceUpdateContainersRestartKey = "apps.kruise.io/inplace-update-containers-restart-count" ) // InPlaceUpdateState records latest inplace-update state, including old statuses of containers. @@ -101,6 +107,16 @@ type InPlaceUpdateStrategy struct { GracePeriodSeconds int32 `json:"gracePeriodSeconds,omitempty"` } +// InPlaceUpdateContainerRestartCount record the restart count of containers +type InPlaceUpdateContainerRestartCount struct { + // Revision is the updated revision hash. + Revision string `json:"revision,omitempty"` + // RestartCount is the container restart count + RestartCount int `json:"restartCount,omitempty"` + // Timestamp is the time for this update + Timestamp metav1.Time `json:"timestamp,omitempty"` +} + func GetInPlaceUpdateState(obj metav1.Object) (string, bool) { if v, ok := obj.GetAnnotations()[InPlaceUpdateStateKey]; ok { return v, ok diff --git a/apis/apps/pub/zz_generated.deepcopy.go b/apis/apps/pub/zz_generated.deepcopy.go index cda5d19b79..aa8131a746 100644 --- a/apis/apps/pub/zz_generated.deepcopy.go +++ b/apis/apps/pub/zz_generated.deepcopy.go @@ -46,6 +46,22 @@ func (in *InPlaceUpdateContainerBatch) DeepCopy() *InPlaceUpdateContainerBatch { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *InPlaceUpdateContainerRestartCount) DeepCopyInto(out *InPlaceUpdateContainerRestartCount) { + *out = *in + in.Timestamp.DeepCopyInto(&out.Timestamp) +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new InPlaceUpdateContainerRestartCount. +func (in *InPlaceUpdateContainerRestartCount) DeepCopy() *InPlaceUpdateContainerRestartCount { + if in == nil { + return nil + } + out := new(InPlaceUpdateContainerRestartCount) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *InPlaceUpdateContainerStatus) DeepCopyInto(out *InPlaceUpdateContainerStatus) { *out = *in diff --git a/pkg/controller/cloneset/sync/cloneset_update.go b/pkg/controller/cloneset/sync/cloneset_update.go index 40089214ec..517aa6c27f 100644 --- a/pkg/controller/cloneset/sync/cloneset_update.go +++ b/pkg/controller/cloneset/sync/cloneset_update.go @@ -50,7 +50,6 @@ func (c *realControl) Update(cs *appsv1alpha1.CloneSet, key := clonesetutils.GetControllerKey(cs) coreControl := clonesetcore.New(cs) - // 1. refresh states for all pods var modified bool for _, pod := range pods { @@ -249,7 +248,6 @@ func (c *realControl) updatePod(cs *appsv1alpha1.CloneSet, coreControl clonesetc break } } - if c.inplaceControl.CanUpdateInPlace(oldRevision, updateRevision, coreControl.GetUpdateOptions()) { switch state := lifecycle.GetPodLifecycleState(pod); state { case "", appspub.LifecycleStatePreparingNormal, appspub.LifecycleStateNormal: diff --git a/pkg/controller/cloneset/sync/cloneset_update_test.go b/pkg/controller/cloneset/sync/cloneset_update_test.go index 17add8342a..7ed983d31d 100644 --- a/pkg/controller/cloneset/sync/cloneset_update_test.go +++ b/pkg/controller/cloneset/sync/cloneset_update_test.go @@ -307,7 +307,12 @@ func TestUpdate(t *testing.T) { UpdateTimestamp: now, LastContainerStatuses: map[string]appspub.InPlaceUpdateContainerStatus{"c1": {ImageID: "image-id-xyz"}}, ContainerBatchesRecord: []appspub.InPlaceUpdateContainerBatch{{Timestamp: now, Containers: []string{"c1"}}}, - })}, + }), + appspub.InPlaceUpdatePodRestartKey: "1", + appspub.InPlaceUpdateContainersRestartKey: util.DumpJSON(map[string]appspub.InPlaceUpdateContainerRestartCount{ + "c1": {RestartCount: 1, Revision: "rev_new", Timestamp: now}, + }), + }, ResourceVersion: "2", }, Spec: v1.PodSpec{ @@ -555,6 +560,10 @@ func TestUpdate(t *testing.T) { LastContainerStatuses: map[string]appspub.InPlaceUpdateContainerStatus{"c1": {ImageID: "image-id-xyz"}}, ContainerBatchesRecord: []appspub.InPlaceUpdateContainerBatch{{Timestamp: now, Containers: []string{"c1"}}}, }), + appspub.InPlaceUpdatePodRestartKey: "1", + appspub.InPlaceUpdateContainersRestartKey: util.DumpJSON(map[string]appspub.InPlaceUpdateContainerRestartCount{ + "c1": {RestartCount: 1, Revision: "rev_new", Timestamp: now}, + }), }, ResourceVersion: "1", }, diff --git a/pkg/util/inplaceupdate/inplace_update.go b/pkg/util/inplaceupdate/inplace_update.go index 81b8966ef9..f3e7fd0c84 100644 --- a/pkg/util/inplaceupdate/inplace_update.go +++ b/pkg/util/inplaceupdate/inplace_update.go @@ -20,13 +20,10 @@ import ( "encoding/json" "fmt" "regexp" + "strconv" "strings" "time" - appspub "github.com/openkruise/kruise/apis/apps/pub" - "github.com/openkruise/kruise/pkg/util" - "github.com/openkruise/kruise/pkg/util/podadapter" - "github.com/openkruise/kruise/pkg/util/revisionadapter" apps "k8s.io/api/apps/v1" v1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -36,6 +33,11 @@ import ( "k8s.io/client-go/util/retry" "k8s.io/klog/v2" "sigs.k8s.io/controller-runtime/pkg/client" + + appspub "github.com/openkruise/kruise/apis/apps/pub" + "github.com/openkruise/kruise/pkg/util" + "github.com/openkruise/kruise/pkg/util/podadapter" + "github.com/openkruise/kruise/pkg/util/revisionadapter" ) var ( @@ -88,7 +90,6 @@ type UpdateSpec struct { OldTemplate *v1.PodTemplateSpec `json:"oldTemplate,omitempty"` NewTemplate *v1.PodTemplateSpec `json:"newTemplate,omitempty"` } - type realControl struct { podAdapter podadapter.Adapter revisionAdapter revisionadapter.Interface @@ -138,7 +139,6 @@ func (c *realControl) Refresh(pod *v1.Pod, opts *UpdateOptions) RefreshResult { klog.V(5).Infof("Pod %s/%s in-place update pre-check not passed: %v", pod.Namespace, pod.Name, checkErr) return RefreshResult{} } - // do update the next containers if updated, err := c.updateNextBatch(pod, opts); err != nil { return RefreshResult{RefreshErr: err} @@ -216,13 +216,13 @@ func (c *realControl) finishGracePeriod(pod *v1.Pod, opts *UpdateOptions) (time. delayDuration = roundupSeconds(graceDuration - span) return nil } - if clone, err = opts.PatchSpecToPod(clone, &spec, &updateState); err != nil { return err } + // record restart count of the pod(containers) + recordPodAndContainersRestartCount(clone, &spec, updateState.Revision) appspub.RemoveInPlaceUpdateGrace(clone) } - _, err = c.podAdapter.UpdatePod(clone) return err }) @@ -258,7 +258,8 @@ func (c *realControl) updateNextBatch(pod *v1.Pod, opts *UpdateOptions) (bool, e if clone, err = opts.PatchSpecToPod(clone, &spec, &state); err != nil { return err } - + // record restart count of the pod(containers) + recordPodAndContainersRestartCount(clone, &spec, state.Revision) updated = true _, err = c.podAdapter.UpdatePod(clone) return err @@ -273,7 +274,6 @@ func (c *realControl) CanUpdateInPlace(oldRevision, newRevision *apps.Controller func (c *realControl) Update(pod *v1.Pod, oldRevision, newRevision *apps.ControllerRevision, opts *UpdateOptions) UpdateResult { opts = SetOptionsDefaults(opts) - // 1. calculate inplace update spec spec := opts.CalculateSpec(oldRevision, newRevision, opts) if spec == nil { @@ -338,12 +338,13 @@ func (c *realControl) updatePodInPlace(pod *v1.Pod, spec *UpdateSpec, opts *Upda if clone, err = opts.PatchSpecToPod(clone, spec, &inPlaceUpdateState); err != nil { return err } + // record restart count of the pod(containers) + recordPodAndContainersRestartCount(clone, spec, inPlaceUpdateState.Revision) appspub.RemoveInPlaceUpdateGrace(clone) } else { inPlaceUpdateSpecJSON, _ := json.Marshal(spec) clone.Annotations[appspub.InPlaceUpdateGraceKey] = string(inPlaceUpdateSpecJSON) } - newPod, updateErr := c.podAdapter.UpdatePod(clone) if updateErr == nil { newResourceVersion = newPod.ResourceVersion @@ -419,3 +420,59 @@ func hasEqualCondition(pod *v1.Pod, newCondition *v1.PodCondition) bool { oldCondition.Reason == newCondition.Reason && oldCondition.Message == newCondition.Message return isEqual } + +// recordPodAndContainersRestartCount record the count of pod(containers) restarts +func recordPodAndContainersRestartCount(pod *v1.Pod, spec *UpdateSpec, revision string) { + if spec.ContainerImages == nil && !spec.UpdateEnvFromMetadata { + return + } + var currentPodRestartCount int64 + containersRestartCount := make(map[string]appspub.InPlaceUpdateContainerRestartCount) + + if v, ok := pod.Annotations[appspub.InPlaceUpdatePodRestartKey]; ok { + currentPodRestartCount, _ = strconv.ParseInt(v, 10, 64) + } + + if v, ok := pod.Annotations[appspub.InPlaceUpdateContainersRestartKey]; ok { + if err := json.Unmarshal([]byte(v), &containersRestartCount); err != nil { + return + } + } + calculateRestartCountFunc := func(cname string) { + containerRestartCount, ok := containersRestartCount[cname] + if ok && revision != containerRestartCount.Revision { + containerRestartCount.RestartCount += 1 + containerRestartCount.Revision = revision + containerRestartCount.Timestamp = metav1.NewTime(Clock.Now()) + currentPodRestartCount += 1 // pod restart + } else if !ok { + containerRestartCount = appspub.InPlaceUpdateContainerRestartCount{ + Revision: revision, + RestartCount: 1, + Timestamp: metav1.NewTime(Clock.Now()), + } + currentPodRestartCount += 1 // pod restart + } + containersRestartCount[cname] = containerRestartCount // containers restart + } + + containers := make(map[string]bool) + if spec.ContainerImages != nil { + for cname := range spec.ContainerImages { + containers[cname] = true + calculateRestartCountFunc(cname) + } + } + + if spec.UpdateEnvFromMetadata { + for cname := range spec.ContainerRefMetadata { + if !containers[cname] { + calculateRestartCountFunc(cname) + } + } + } + + containersRestartCountJson, _ := json.Marshal(containersRestartCount) + pod.Annotations[appspub.InPlaceUpdateContainersRestartKey] = string(containersRestartCountJson) + pod.Annotations[appspub.InPlaceUpdatePodRestartKey] = strconv.FormatInt(currentPodRestartCount, 10) +} diff --git a/pkg/util/inplaceupdate/inplace_update_test.go b/pkg/util/inplaceupdate/inplace_update_test.go index 75353a8ae9..224887b47e 100644 --- a/pkg/util/inplaceupdate/inplace_update_test.go +++ b/pkg/util/inplaceupdate/inplace_update_test.go @@ -23,11 +23,6 @@ import ( "testing" "time" - appspub "github.com/openkruise/kruise/apis/apps/pub" - "github.com/openkruise/kruise/pkg/features" - "github.com/openkruise/kruise/pkg/util" - utilfeature "github.com/openkruise/kruise/pkg/util/feature" - "github.com/openkruise/kruise/pkg/util/revisionadapter" apps "k8s.io/api/apps/v1" v1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -35,6 +30,12 @@ import ( "k8s.io/apimachinery/pkg/types" "k8s.io/apimachinery/pkg/util/clock" "sigs.k8s.io/controller-runtime/pkg/client/fake" + + appspub "github.com/openkruise/kruise/apis/apps/pub" + "github.com/openkruise/kruise/pkg/features" + "github.com/openkruise/kruise/pkg/util" + utilfeature "github.com/openkruise/kruise/pkg/util/feature" + "github.com/openkruise/kruise/pkg/util/revisionadapter" ) func TestCalculateInPlaceUpdateSpec(t *testing.T) { @@ -451,6 +452,10 @@ func TestRefresh(t *testing.T) { LastContainerStatuses: map[string]appspub.InPlaceUpdateContainerStatus{"c1": {ImageID: "img01"}}, ContainerBatchesRecord: []appspub.InPlaceUpdateContainerBatch{{Timestamp: aHourAgo, Containers: []string{"main"}}}, }), + appspub.InPlaceUpdatePodRestartKey: "1", + appspub.InPlaceUpdateContainersRestartKey: util.DumpJSON(map[string]appspub.InPlaceUpdateContainerRestartCount{ + "main": {RestartCount: 1, Revision: "new-revision", Timestamp: aHourAgo}, + }), }, ResourceVersion: "1", }, @@ -588,6 +593,10 @@ func TestRefresh(t *testing.T) { LastContainerStatuses: map[string]appspub.InPlaceUpdateContainerStatus{"c1": {ImageID: "c1-img1-ID"}, "c2": {ImageID: "c2-img1-ID"}}, ContainerBatchesRecord: []appspub.InPlaceUpdateContainerBatch{{Timestamp: aHourAgo, Containers: []string{"c1"}}, {Timestamp: aHourAgo, Containers: []string{"c2"}}}, }), + appspub.InPlaceUpdatePodRestartKey: "1", + appspub.InPlaceUpdateContainersRestartKey: util.DumpJSON(map[string]appspub.InPlaceUpdateContainerRestartCount{ + "c2": {RestartCount: 1, Revision: "new-revision", Timestamp: aHourAgo}, + }), }, }, Spec: v1.PodSpec{