Skip to content

Commit 1bc8d85

Browse files
zmbergzmberg
authored
sidecarset support k8s 1.28 sidecarContainers (openkruise#1613)
Signed-off-by: liheng.zms <[email protected]>
1 parent b969432 commit 1bc8d85

File tree

10 files changed

+504
-58
lines changed

10 files changed

+504
-58
lines changed

‎.gitignore

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -28,3 +28,5 @@ test/e2e/generated/bindata.go
2828
.vscode
2929

3030
.DS_Store
31+
32+
vendor/

‎pkg/control/sidecarcontrol/hash.go

Lines changed: 14 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -43,6 +43,9 @@ func SidecarSetHashWithoutImage(sidecarSet *appsv1alpha1.SidecarSet) (string, er
4343
for i := range ss.Spec.Containers {
4444
ss.Spec.Containers[i].Image = ""
4545
}
46+
for i := range ss.Spec.InitContainers {
47+
ss.Spec.InitContainers[i].Image = ""
48+
}
4649
encoded, err := encodeSidecarSet(ss)
4750
if err != nil {
4851
return "", err
@@ -53,7 +56,17 @@ func SidecarSetHashWithoutImage(sidecarSet *appsv1alpha1.SidecarSet) (string, er
5356
func encodeSidecarSet(sidecarSet *appsv1alpha1.SidecarSet) (string, error) {
5457
// json.Marshal sorts the keys in a stable order in the encoding
5558
m := map[string]interface{}{"containers": sidecarSet.Spec.Containers}
56-
//m["initContainers"] = sidecarSet.Spec.InitContainers
59+
// when k8s 1.28, if initContainer restartPolicy = Always, indicates it is sidecar container, so the hash needs to contain it.
60+
initContainer := make([]appsv1alpha1.SidecarContainer, 0)
61+
for i := range sidecarSet.Spec.InitContainers {
62+
container := &sidecarSet.Spec.InitContainers[i]
63+
if IsSidecarContainer(container.Container) {
64+
initContainer = append(initContainer, *container)
65+
}
66+
}
67+
if len(initContainer) > 0 {
68+
m["initContainers"] = sidecarSet.Spec.InitContainers
69+
}
5770
data, err := json.Marshal(m)
5871
if err != nil {
5972
return "", err

‎pkg/control/sidecarcontrol/hash_test.go

Lines changed: 186 additions & 54 deletions
Original file line numberDiff line numberDiff line change
@@ -8,78 +8,210 @@ import (
88
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
99
)
1010

11+
var always = corev1.ContainerRestartPolicyAlways
12+
1113
func TestSidecarSetHash(t *testing.T) {
12-
sidecarSet := &appsv1alpha1.SidecarSet{
13-
ObjectMeta: metav1.ObjectMeta{
14-
Name: "test-sidecar-set",
14+
cases := []struct {
15+
name string
16+
getSidecarSet func() *appsv1alpha1.SidecarSet
17+
expectHash string
18+
}{
19+
{
20+
name: "containers",
21+
getSidecarSet: func() *appsv1alpha1.SidecarSet {
22+
return &appsv1alpha1.SidecarSet{
23+
ObjectMeta: metav1.ObjectMeta{
24+
Name: "test-sidecar-set",
25+
},
26+
Spec: appsv1alpha1.SidecarSetSpec{
27+
Containers: []appsv1alpha1.SidecarContainer{
28+
{
29+
Container: corev1.Container{
30+
Name: "container1",
31+
Image: "test-image",
32+
},
33+
},
34+
},
35+
},
36+
}
37+
},
38+
expectHash: "w26c4x8fz245642fdv499b464248f974xddx4x55z5dw55bc6x66464fxz77dc78",
1539
},
16-
Spec: appsv1alpha1.SidecarSetSpec{
17-
Containers: []appsv1alpha1.SidecarContainer{
18-
{
19-
Container: corev1.Container{
20-
Name: "container1",
21-
Image: "test-image",
40+
{
41+
name: "containers and initContainers",
42+
getSidecarSet: func() *appsv1alpha1.SidecarSet {
43+
return &appsv1alpha1.SidecarSet{
44+
ObjectMeta: metav1.ObjectMeta{
45+
Name: "test-sidecar-set",
46+
},
47+
Spec: appsv1alpha1.SidecarSetSpec{
48+
Containers: []appsv1alpha1.SidecarContainer{
49+
{
50+
Container: corev1.Container{
51+
Name: "container1",
52+
Image: "test-image",
53+
},
54+
},
55+
},
56+
InitContainers: []appsv1alpha1.SidecarContainer{
57+
{
58+
Container: corev1.Container{
59+
Name: "container1",
60+
Image: "test-image",
61+
},
62+
},
63+
},
2264
},
23-
},
65+
}
2466
},
67+
expectHash: "w26c4x8fz245642fdv499b464248f974xddx4x55z5dw55bc6x66464fxz77dc78",
68+
},
69+
{
70+
name: "containers and initContainers with restartPolicy=Always",
71+
getSidecarSet: func() *appsv1alpha1.SidecarSet {
72+
return &appsv1alpha1.SidecarSet{
73+
ObjectMeta: metav1.ObjectMeta{
74+
Name: "test-sidecar-set",
75+
},
76+
Spec: appsv1alpha1.SidecarSetSpec{
77+
Containers: []appsv1alpha1.SidecarContainer{
78+
{
79+
Container: corev1.Container{
80+
Name: "container1",
81+
Image: "test-image",
82+
},
83+
},
84+
},
85+
InitContainers: []appsv1alpha1.SidecarContainer{
86+
{
87+
Container: corev1.Container{
88+
Name: "container1",
89+
Image: "test-image",
90+
RestartPolicy: &always,
91+
},
92+
},
93+
},
94+
},
95+
}
96+
},
97+
expectHash: "4xwx4d4844vd4v9x79wb4xbxf4xb29475cc4446v8cz2c2f2f5c5bw448vd42z8w",
2598
},
2699
}
27100

28-
hash, err := SidecarSetHash(sidecarSet)
29-
if err != nil {
30-
t.Fatalf("Unexpected error: %v", err)
31-
}
32-
33-
if hash == "" {
34-
t.Fatalf("Expected non-empty hash")
35-
}
36-
37-
// Change sidecar set and expect different hash
38-
sidecarSet.Spec.Containers[0].Image = "new-image"
39-
newHash, err := SidecarSetHash(sidecarSet)
40-
if err != nil {
41-
t.Fatalf("Unexpected error: %v", err)
42-
}
101+
for _, cs := range cases {
102+
t.Run(cs.name, func(t *testing.T) {
103+
sidecarSet := cs.getSidecarSet()
104+
hash1, err := SidecarSetHash(sidecarSet)
105+
if err != nil {
106+
t.Fatalf("Unexpected error: %v", err)
107+
} else if hash1 == "" {
108+
t.Fatalf("Expected non-empty hash")
109+
}
110+
if cs.expectHash != hash1 {
111+
t.Fatalf("expect(%s), but get(%s)", cs.expectHash, hash1)
112+
}
43113

44-
if newHash == hash {
45-
t.Fatalf("Expected different hashes for different SidecarSets")
114+
// Change sidecar set and expect different hash
115+
sidecarSet.Spec.Containers[0].Image = "new-image"
116+
newHash, err := SidecarSetHash(sidecarSet)
117+
if err != nil {
118+
t.Fatalf("Unexpected error: %v", err)
119+
} else if newHash == hash1 {
120+
t.Fatalf("Expected different hashes for different SidecarSets")
121+
}
122+
})
46123
}
47124
}
48125

49126
func TestSidecarSetHashWithoutImage(t *testing.T) {
50-
sidecarSet := &appsv1alpha1.SidecarSet{
51-
ObjectMeta: metav1.ObjectMeta{
52-
Name: "test-sidecar-set",
127+
cases := []struct {
128+
name string
129+
getSidecarSet func() *appsv1alpha1.SidecarSet
130+
expectHash string
131+
}{
132+
{
133+
name: "containers and initContainers",
134+
getSidecarSet: func() *appsv1alpha1.SidecarSet {
135+
return &appsv1alpha1.SidecarSet{
136+
ObjectMeta: metav1.ObjectMeta{
137+
Name: "test-sidecar-set",
138+
},
139+
Spec: appsv1alpha1.SidecarSetSpec{
140+
Containers: []appsv1alpha1.SidecarContainer{
141+
{
142+
Container: corev1.Container{
143+
Name: "container1",
144+
Image: "test-image",
145+
},
146+
},
147+
},
148+
InitContainers: []appsv1alpha1.SidecarContainer{
149+
{
150+
Container: corev1.Container{
151+
Name: "container1",
152+
Image: "test-image",
153+
},
154+
},
155+
},
156+
},
157+
}
158+
},
159+
expectHash: "8wzddb4dvv9c6x8zdc77z4z75987424f457dfv6724ddw6zbdx467wz5x24fc759",
53160
},
54-
Spec: appsv1alpha1.SidecarSetSpec{
55-
Containers: []appsv1alpha1.SidecarContainer{
56-
{
57-
Container: corev1.Container{
58-
Name: "container1",
59-
Image: "test-image",
161+
{
162+
name: "containers and initContainers with restartPolicy=Always",
163+
getSidecarSet: func() *appsv1alpha1.SidecarSet {
164+
return &appsv1alpha1.SidecarSet{
165+
ObjectMeta: metav1.ObjectMeta{
166+
Name: "test-sidecar-set",
167+
},
168+
Spec: appsv1alpha1.SidecarSetSpec{
169+
Containers: []appsv1alpha1.SidecarContainer{
170+
{
171+
Container: corev1.Container{
172+
Name: "container1",
173+
Image: "test-image",
174+
},
175+
},
176+
},
177+
InitContainers: []appsv1alpha1.SidecarContainer{
178+
{
179+
Container: corev1.Container{
180+
Name: "container1",
181+
Image: "test-image",
182+
RestartPolicy: &always,
183+
},
184+
},
185+
},
60186
},
61-
},
187+
}
62188
},
189+
expectHash: "5725fw8bwbx249bw57v5892c847dzf48bww9zb7c86xb95264fdz26654847b2c8",
63190
},
64191
}
65192

66-
hash, err := SidecarSetHashWithoutImage(sidecarSet)
67-
if err != nil {
68-
t.Fatalf("Unexpected error: %v", err)
69-
}
70-
71-
if hash == "" {
72-
t.Fatalf("Expected non-empty hash")
73-
}
74-
75-
// Change sidecar set image and expect same hash
76-
sidecarSet.Spec.Containers[0].Image = "new-image"
77-
newHash, err := SidecarSetHashWithoutImage(sidecarSet)
78-
if err != nil {
79-
t.Fatalf("Unexpected error: %v", err)
80-
}
193+
for _, cs := range cases {
194+
t.Run(cs.name, func(t *testing.T) {
195+
sidecarSet := cs.getSidecarSet()
196+
hash1, err := SidecarSetHashWithoutImage(sidecarSet)
197+
if err != nil {
198+
t.Fatalf("Unexpected error: %v", err)
199+
} else if hash1 == "" {
200+
t.Fatalf("Expected non-empty hash")
201+
}
202+
if cs.expectHash != hash1 {
203+
t.Fatalf("expect(%s), but get(%s)", cs.expectHash, hash1)
204+
}
81205

82-
if newHash != hash {
83-
t.Fatalf("Expected same hashes for SidecarSets with different images")
206+
// Change sidecar set and expect different hash
207+
sidecarSet.Spec.Containers[0].Image = "new-image"
208+
sidecarSet.Spec.InitContainers[0].Image = "new-image"
209+
newHash, err := SidecarSetHashWithoutImage(sidecarSet)
210+
if err != nil {
211+
t.Fatalf("Unexpected error: %v", err)
212+
} else if newHash != hash1 {
213+
t.Fatalf("Expected same hashes for different SidecarSets")
214+
}
215+
})
84216
}
85217
}

‎pkg/control/sidecarcontrol/util.go

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -246,6 +246,9 @@ func UpdatePodSidecarSetHash(pod *corev1.Pod, sidecarSet *appsv1alpha1.SidecarSe
246246
for _, sidecar := range sidecarSet.Spec.Containers {
247247
sidecarList.Insert(sidecar.Name)
248248
}
249+
for _, sidecar := range sidecarSet.Spec.InitContainers {
250+
sidecarList.Insert(sidecar.Name)
251+
}
249252

250253
sidecarSetHash[sidecarSet.Name] = SidecarSetUpgradeSpec{
251254
UpdateTimestamp: metav1.Now(),
@@ -564,3 +567,11 @@ func matchRegKey(key string, regs []*regexp.Regexp) bool {
564567
}
565568
return false
566569
}
570+
571+
// IsSidecarContainer check whether initContainer is sidecar container in k8s 1.28.
572+
func IsSidecarContainer(container corev1.Container) bool {
573+
if container.RestartPolicy != nil && *container.RestartPolicy == corev1.ContainerRestartPolicyAlways {
574+
return true
575+
}
576+
return false
577+
}

‎pkg/webhook/pod/mutating/sidecarset.go

Lines changed: 14 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -152,6 +152,7 @@ func (h *PodCreateHandler) sidecarsetMutatingPod(ctx context.Context, req admiss
152152
sort.SliceStable(sidecarInitContainers, func(i, j int) bool {
153153
return sidecarInitContainers[i].Name < sidecarInitContainers[j].Name
154154
})
155+
// TODO, implement PodInjectPolicy for initContainers
155156
for _, initContainer := range sidecarInitContainers {
156157
pod.Spec.InitContainers = append(pod.Spec.InitContainers, initContainer.Container)
157158
}
@@ -368,11 +369,13 @@ func buildSidecars(isUpdated bool, pod *corev1.Pod, oldPod *corev1.Pod, matchedS
368369
}
369370

370371
isInjecting := false
372+
sidecarList := sets.NewString()
371373
//process initContainers
372374
//only when created pod, inject initContainer and pullSecrets
373375
if !isUpdated {
374376
for i := range sidecarSet.Spec.InitContainers {
375377
initContainer := &sidecarSet.Spec.InitContainers[i]
378+
sidecarList.Insert(initContainer.Name)
376379
// volumeMounts that injected into sidecar container
377380
// when volumeMounts SubPathExpr contains expansions, then need copy container EnvVars(injectEnvs)
378381
injectedMounts, injectedEnvs := sidecarcontrol.GetInjectedVolumeMountsAndEnvs(control, initContainer, pod)
@@ -393,13 +396,22 @@ func buildSidecars(isUpdated bool, pod *corev1.Pod, oldPod *corev1.Pod, matchedS
393396
// merged Env from sidecar.Env and transfer envs
394397
initContainer.Env = util.MergeEnvVar(initContainer.Env, transferEnvs)
395398
isInjecting = true
396-
sidecarInitContainers = append(sidecarInitContainers, initContainer)
399+
400+
// when sidecar container UpgradeStrategy is HotUpgrade
401+
if sidecarcontrol.IsSidecarContainer(initContainer.Container) && sidecarcontrol.IsHotUpgradeContainer(initContainer) {
402+
hotContainers, annotations := injectHotUpgradeContainers(hotUpgradeWorkInfo, initContainer)
403+
sidecarInitContainers = append(sidecarInitContainers, hotContainers...)
404+
for k, v := range annotations {
405+
injectedAnnotations[k] = v
406+
}
407+
} else {
408+
sidecarInitContainers = append(sidecarInitContainers, initContainer)
409+
}
397410
}
398411
//process imagePullSecrets
399412
sidecarSecrets = append(sidecarSecrets, sidecarSet.Spec.ImagePullSecrets...)
400413
}
401414

402-
sidecarList := sets.NewString()
403415
//process containers
404416
for i := range sidecarSet.Spec.Containers {
405417
sidecarContainer := &sidecarSet.Spec.Containers[i]

0 commit comments

Comments
 (0)