From 57ce3d3ecaeaef02e3edffe9eaca38334a13398d Mon Sep 17 00:00:00 2001
From: pixiake <guofeng@yunify.com>
Date: Wed, 6 Mar 2024 12:32:08 +0800
Subject: [PATCH 01/22] Create issue_comment_webhook.yml

---
 .github/workflows/issue_comment_webhook.yml | 45 +++++++++++++++++++++
 1 file changed, 45 insertions(+)
 create mode 100644 .github/workflows/issue_comment_webhook.yml

diff --git a/.github/workflows/issue_comment_webhook.yml b/.github/workflows/issue_comment_webhook.yml
new file mode 100644
index 000000000..60e9992a6
--- /dev/null
+++ b/.github/workflows/issue_comment_webhook.yml
@@ -0,0 +1,45 @@
+name: Issue Comment WeCom Webhook
+
+on:
+  issues:
+    types: [opened, edited]
+  issue_comment:
+    types: [created, edited]
+
+jobs:
+  send_to_webhook:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/setup-node@v3
+        with:
+          node-version: "20.x"
+      - run: npm install axios
+      - name: Send issue/comment to WeCom webhook
+        uses: actions/github-script@v7
+        env:
+          WEBHOOK_URL: ${{ secrets.WEBHOOK_URL }}
+        with:
+          script: |
+            console.log(context);
+            if (context.payload.sender.login === "ks-ci-bot") return;
+            const axios = require('axios');
+            const issue = context.payload.issue;
+            const comment = context.payload.comment;
+            var subject = {};
+            var action = '';
+            if (comment) {
+              action = "comment";
+              subject = comment;
+            } else {
+              action = "issue";
+              subject = issue;
+            };
+            const payload = {
+              msgtype: 'markdown',
+              markdown: {
+                content: `[${context.payload.sender.login}](${context.payload.sender.html_url}) ${context.payload.action} ${action} [${issue.title}](${subject.html_url})\n${subject.body}`,
+              },
+            };
+            const formattedPayload = JSON.stringify(payload, null, 2);
+            console.log(formattedPayload);
+            await axios.post(process.env.WEBHOOK_URL, payload);

From 3cd588b678d3506d105b74c3dd933dc73e62d928 Mon Sep 17 00:00:00 2001
From: stark <stark@yunify.com>
Date: Wed, 6 Mar 2024 21:11:02 +0800
Subject: [PATCH 02/22] fix start docker error when init harbor

---
 cmd/kk/pkg/bootstrap/registry/module.go | 35 ++++++++++++++++++++++---
 1 file changed, 31 insertions(+), 4 deletions(-)

diff --git a/cmd/kk/pkg/bootstrap/registry/module.go b/cmd/kk/pkg/bootstrap/registry/module.go
index 284b88bac..f17be620e 100644
--- a/cmd/kk/pkg/bootstrap/registry/module.go
+++ b/cmd/kk/pkg/bootstrap/registry/module.go
@@ -170,6 +170,20 @@ func InstallHarbor(i *InstallRegistryModule) []task.Interface {
 		Retry:    2,
 	}
 
+	generateContainerdService := &task.RemoteTask{
+		Name:  "GenerateContainerdService",
+		Desc:  "Generate containerd service",
+		Hosts: i.Runtime.GetHostsByRole(common.K8s),
+		Prepare: &prepare.PrepareCollection{
+			&container.ContainerdExist{Not: true},
+		},
+		Action: &action.Template{
+			Template: docker_template.ContainerdService,
+			Dst:      filepath.Join("/etc/systemd/system", docker_template.ContainerdService.Name()),
+		},
+		Parallel: true,
+	}
+
 	generateDockerService := &task.RemoteTask{
 		Name:  "GenerateDockerService",
 		Desc:  "Generate docker service",
@@ -202,6 +216,17 @@ func InstallHarbor(i *InstallRegistryModule) []task.Interface {
 		Parallel: true,
 	}
 
+	enableContainerdForDocker := &task.RemoteTask{
+		Name:  "EnableContainerd",
+		Desc:  "Enable containerd",
+		Hosts: i.Runtime.GetHostsByRole(common.K8s),
+		Prepare: &prepare.PrepareCollection{
+			&container.ContainerdExist{Not: true},
+		},
+		Action:   new(container.EnableContainerdForDocker),
+		Parallel: true,
+	}
+
 	enableDocker := &task.RemoteTask{
 		Name:  "EnableDocker",
 		Desc:  "Enable docker",
@@ -250,10 +275,10 @@ func InstallHarbor(i *InstallRegistryModule) []task.Interface {
 	}
 
 	generateHarborConfig := &task.RemoteTask{
-		Name:   "GenerateHarborConfig",
-		Desc:   "Generate harbor config",
-		Hosts:  i.Runtime.GetHostsByRole(common.Registry),
-		Action: new(GenerateHarborConfig),
+		Name:     "GenerateHarborConfig",
+		Desc:     "Generate harbor config",
+		Hosts:    i.Runtime.GetHostsByRole(common.Registry),
+		Action:   new(GenerateHarborConfig),
 		Parallel: true,
 		Retry:    1,
 	}
@@ -269,8 +294,10 @@ func InstallHarbor(i *InstallRegistryModule) []task.Interface {
 
 	return []task.Interface{
 		syncBinaries,
+		generateContainerdService,
 		generateDockerService,
 		generateDockerConfig,
+		enableContainerdForDocker,
 		enableDocker,
 		installDockerCompose,
 		syncHarborPackage,

From 4b367548eab1047d251d6c42e2f43836bdf18862 Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Wed, 6 Mar 2024 16:34:15 +0000
Subject: [PATCH 03/22] build(deps): bump actions/setup-node from 3 to 4

Bumps [actions/setup-node](https://github.com/actions/setup-node) from 3 to 4.
- [Release notes](https://github.com/actions/setup-node/releases)
- [Commits](https://github.com/actions/setup-node/compare/v3...v4)

---
updated-dependencies:
- dependency-name: actions/setup-node
  dependency-type: direct:production
  update-type: version-update:semver-major
...

Signed-off-by: dependabot[bot] <support@github.com>
---
 .github/workflows/issue_comment_webhook.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/issue_comment_webhook.yml b/.github/workflows/issue_comment_webhook.yml
index 60e9992a6..189cb9569 100644
--- a/.github/workflows/issue_comment_webhook.yml
+++ b/.github/workflows/issue_comment_webhook.yml
@@ -10,7 +10,7 @@ jobs:
   send_to_webhook:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/setup-node@v3
+      - uses: actions/setup-node@v4
         with:
           node-version: "20.x"
       - run: npm install axios

From 10d5dc93198497b5c5222931e2985026a2cb958b Mon Sep 17 00:00:00 2001
From: joyceliu <joyceliu@yunify.com>
Date: Thu, 7 Mar 2024 11:55:14 +0800
Subject: [PATCH 04/22] fix: Relax artifact strategy Signed-off-by: joyceliu
 <joyceliu@yunify.com>

Signed-off-by: joyceliu <joyceliu@yunify.com>
---
 cmd/kk/pkg/binaries/kubernetes.go       | 58 +++++++++++++++++++------
 cmd/kk/pkg/binaries/tasks.go            |  4 ++
 cmd/kk/pkg/pipelines/artifact_export.go |  2 +-
 3 files changed, 50 insertions(+), 14 deletions(-)

diff --git a/cmd/kk/pkg/binaries/kubernetes.go b/cmd/kk/pkg/binaries/kubernetes.go
index 98eeafe99..1f43fda86 100644
--- a/cmd/kk/pkg/binaries/kubernetes.go
+++ b/cmd/kk/pkg/binaries/kubernetes.go
@@ -101,18 +101,25 @@ func K8sFilesDownloadHTTP(kubeConf *common.KubeConf, path, version, arch string,
 	return nil
 }
 
-func KubernetesArtifactBinariesDownload(manifest *common.ArtifactManifest, path, arch, k8sVersion string) error {
+func KubernetesComponentBinariesDownload(manifest *common.ArtifactManifest, path, arch string) error {
 	m := manifest.Spec
+	var binaries []*files.KubeBinary
 
-	etcd := files.NewKubeBinary("etcd", arch, m.Components.ETCD.Version, path, manifest.Arg.DownloadCommand)
-	kubeadm := files.NewKubeBinary("kubeadm", arch, k8sVersion, path, manifest.Arg.DownloadCommand)
-	kubelet := files.NewKubeBinary("kubelet", arch, k8sVersion, path, manifest.Arg.DownloadCommand)
-	kubectl := files.NewKubeBinary("kubectl", arch, k8sVersion, path, manifest.Arg.DownloadCommand)
-	kubecni := files.NewKubeBinary("kubecni", arch, m.Components.CNI.Version, path, manifest.Arg.DownloadCommand)
-	helm := files.NewKubeBinary("helm", arch, m.Components.Helm.Version, path, manifest.Arg.DownloadCommand)
-	crictl := files.NewKubeBinary("crictl", arch, m.Components.Crictl.Version, path, manifest.Arg.DownloadCommand)
-	calicoctl := files.NewKubeBinary("calicoctl", arch, m.Components.Calicoctl.Version, path, manifest.Arg.DownloadCommand)
-	binaries := []*files.KubeBinary{kubeadm, kubelet, kubectl, helm, kubecni, etcd, calicoctl}
+	if m.Components.ETCD.Version != "" {
+		binaries = append(binaries, files.NewKubeBinary("etcd", arch, m.Components.ETCD.Version, path, manifest.Arg.DownloadCommand))
+	}
+	if m.Components.CNI.Version != "" {
+		binaries = append(binaries, files.NewKubeBinary("kubecni", arch, m.Components.CNI.Version, path, manifest.Arg.DownloadCommand))
+	}
+	if m.Components.Helm.Version != "" {
+		binaries = append(binaries, files.NewKubeBinary("helm", arch, m.Components.Helm.Version, path, manifest.Arg.DownloadCommand))
+	}
+	if m.Components.Crictl.Version != "" {
+		binaries = append(binaries, files.NewKubeBinary("crictl", arch, m.Components.Crictl.Version, path, manifest.Arg.DownloadCommand))
+	}
+	if m.Components.Calicoctl.Version != "" {
+		binaries = append(binaries, files.NewKubeBinary("calicoctl", arch, m.Components.Calicoctl.Version, path, manifest.Arg.DownloadCommand))
+	}
 
 	containerManagerArr := make([]*files.KubeBinary, 0, 0)
 	containerManagerVersion := make(map[string]struct{})
@@ -128,11 +135,36 @@ func KubernetesArtifactBinariesDownload(manifest *common.ArtifactManifest, path,
 		}
 	}
 
-	binaries = append(binaries, containerManagerArr...)
-	if m.Components.Crictl.Version != "" {
-		binaries = append(binaries, crictl)
+	for _, binary := range binaries {
+		if err := binary.CreateBaseDir(); err != nil {
+			return errors.Wrapf(errors.WithStack(err), "create file %s base dir failed", binary.FileName)
+		}
+
+		logger.Log.Messagef(common.LocalHost, "downloading %s %s %s ...", arch, binary.ID, binary.Version)
+
+		if util.IsExist(binary.Path()) {
+			// download it again if it's incorrect
+			if err := binary.SHA256Check(); err != nil {
+				_ = exec.Command("/bin/sh", "-c", fmt.Sprintf("rm -f %s", binary.Path())).Run()
+			} else {
+				continue
+			}
+		}
+
+		if err := binary.Download(); err != nil {
+			return fmt.Errorf("Failed to download %s binary: %s error: %w ", binary.ID, binary.GetCmd(), err)
+		}
 	}
 
+	return nil
+}
+
+func KubernetesArtifactBinariesDownload(manifest *common.ArtifactManifest, path, arch, k8sVersion string) error {
+	kubeadm := files.NewKubeBinary("kubeadm", arch, k8sVersion, path, manifest.Arg.DownloadCommand)
+	kubelet := files.NewKubeBinary("kubelet", arch, k8sVersion, path, manifest.Arg.DownloadCommand)
+	kubectl := files.NewKubeBinary("kubectl", arch, k8sVersion, path, manifest.Arg.DownloadCommand)
+	binaries := []*files.KubeBinary{kubeadm, kubelet, kubectl}
+
 	for _, binary := range binaries {
 		if err := binary.CreateBaseDir(); err != nil {
 			return errors.Wrapf(errors.WithStack(err), "create file %s base dir failed", binary.FileName)
diff --git a/cmd/kk/pkg/binaries/tasks.go b/cmd/kk/pkg/binaries/tasks.go
index 7d829cb79..1c6b3620c 100644
--- a/cmd/kk/pkg/binaries/tasks.go
+++ b/cmd/kk/pkg/binaries/tasks.go
@@ -168,6 +168,10 @@ func (a *ArtifactDownload) Execute(runtime connector.Runtime) error {
 			}
 		}
 
+		if err := KubernetesComponentBinariesDownload(a.Manifest, basePath, arch); err != nil {
+			return err
+		}
+
 		if err := RegistryBinariesDownload(a.Manifest, basePath, arch); err != nil {
 			return err
 		}
diff --git a/cmd/kk/pkg/pipelines/artifact_export.go b/cmd/kk/pkg/pipelines/artifact_export.go
index 6e730deb5..a47db6857 100644
--- a/cmd/kk/pkg/pipelines/artifact_export.go
+++ b/cmd/kk/pkg/pipelines/artifact_export.go
@@ -117,7 +117,7 @@ func ArtifactExport(args common.ArtifactArgument, downloadCmd string) error {
 	}
 
 	if len(runtime.Spec.KubernetesDistributions) == 0 {
-		return errors.New("the length of kubernetes distributions can't be 0")
+		return NewArtifactExportPipeline(runtime)
 	}
 
 	pre := runtime.Spec.KubernetesDistributions[0].Type

From b89d2fca87282ee47d469e9b42f9448cf80252fc Mon Sep 17 00:00:00 2001
From: stark <stark@yunify.com>
Date: Mon, 11 Mar 2024 17:16:25 +0800
Subject: [PATCH 05/22] Add ubuntu2204 cuda for iso repo

---
 hack/gen-repository-iso/dockerfile.ubuntu2204 | 14 ++++++++++++++
 hack/gen-repository-iso/packages.yaml         |  3 +++
 2 files changed, 17 insertions(+)

diff --git a/hack/gen-repository-iso/dockerfile.ubuntu2204 b/hack/gen-repository-iso/dockerfile.ubuntu2204
index 7a92912a6..42ebb4d94 100644
--- a/hack/gen-repository-iso/dockerfile.ubuntu2204
+++ b/hack/gen-repository-iso/dockerfile.ubuntu2204
@@ -1,5 +1,6 @@
 FROM ubuntu:22.04 as ubuntu2204
 ARG TARGETARCH
+ARG DISTRO=ubuntu2204
 ARG OS_RELEASE=jammy
 ARG DIR=ubuntu-22.04-${TARGETARCH}-debs
 ARG PKGS=.common[],.debs[],.ubuntu[],.ubuntu2204[]
@@ -15,6 +16,19 @@ RUN apt update -qq \
     && echo "deb [arch=$TARGETARCH] https://download.docker.com/linux/ubuntu ${OS_RELEASE} stable" > /etc/apt/sources.list.d/docker.list\
     && apt update -qq
 
+# install NVIDIA CUDA
+RUN if [ "${TARGETARCH}" = "amd64" ]; then \
+        ARCH=x86_64; \
+    else \
+        ARCH=${TARGETARCH}; \
+    fi \
+    && wget https://developer.download.nvidia.com/compute/cuda/repos/${DISTRO}/${ARCH}/cuda-archive-keyring.gpg \
+    && mv cuda-archive-keyring.gpg /usr/share/keyrings/cuda-archive-keyring.gpg \
+    && echo "deb [signed-by=/usr/share/keyrings/cuda-archive-keyring.gpg] https://developer.download.nvidia.com/compute/cuda/repos/${DISTRO}/${ARCH}/ /" | tee /etc/apt/sources.list.d/cuda-${DISTRO}-${ARCH}.list \
+    && wget https://developer.download.nvidia.com/compute/cuda/repos/${DISTRO}/${ARCH}/cuda-${DISTRO}.pin \
+    && mv cuda-${DISTRO}.pin /etc/apt/preferences.d/cuda-repository-pin-600 \
+    && apt-get update
+
 WORKDIR /package
 COPY packages.yaml .
 
diff --git a/hack/gen-repository-iso/packages.yaml b/hack/gen-repository-iso/packages.yaml
index 65d89a3eb..4708bd564 100644
--- a/hack/gen-repository-iso/packages.yaml
+++ b/hack/gen-repository-iso/packages.yaml
@@ -39,6 +39,9 @@ debs:
   - openssh-server
   - software-properties-common
   - sudo
+  - cuda-toolkit-12-4
+  - nvidia-driver-550-open
+  - cuda-drivers-550
 
 centos:
   - containerd.io

From dd390d02a7991d8490d8e1dbd497634c0f2ce350 Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Mon, 11 Mar 2024 16:01:59 +0000
Subject: [PATCH 06/22] build(deps): bump softprops/action-gh-release from 1 to
 2

Bumps [softprops/action-gh-release](https://github.com/softprops/action-gh-release) from 1 to 2.
- [Release notes](https://github.com/softprops/action-gh-release/releases)
- [Changelog](https://github.com/softprops/action-gh-release/blob/master/CHANGELOG.md)
- [Commits](https://github.com/softprops/action-gh-release/compare/v1...v2)

---
updated-dependencies:
- dependency-name: softprops/action-gh-release
  dependency-type: direct:production
  update-type: version-update:semver-major
...

Signed-off-by: dependabot[bot] <support@github.com>
---
 .github/workflows/gen-repository-iso.yaml | 2 +-
 .github/workflows/release.yaml            | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/gen-repository-iso.yaml b/.github/workflows/gen-repository-iso.yaml
index 27fc7d193..c97562754 100644
--- a/.github/workflows/gen-repository-iso.yaml
+++ b/.github/workflows/gen-repository-iso.yaml
@@ -66,7 +66,7 @@ jobs:
 
       - name: Release and upload packages
         if: startsWith(github.ref, 'refs/tags/')
-        uses: softprops/action-gh-release@v1
+        uses: softprops/action-gh-release@v2
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
         with:
diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml
index 205c60fe9..accbf0a51 100644
--- a/.github/workflows/release.yaml
+++ b/.github/workflows/release.yaml
@@ -24,7 +24,7 @@ jobs:
         run: |
           make release
       - name: Release
-        uses: softprops/action-gh-release@v1
+        uses: softprops/action-gh-release@v2
         with:
           draft: true
           files: out/*

From 84a33a310d0aacee2226e1badb296702af3f44e2 Mon Sep 17 00:00:00 2001
From: pixiake <guofeng@yunify.com>
Date: Tue, 12 Mar 2024 10:21:35 +0800
Subject: [PATCH 07/22] upgrade sha256 for cri-docker

Signed-off-by: pixiake <guofeng@yunify.com>
---
 version/components.json | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/version/components.json b/version/components.json
index 185ebf316..8cdb05ac6 100644
--- a/version/components.json
+++ b/version/components.json
@@ -979,10 +979,10 @@
     },
     "cri-dockerd": {
         "amd64": {
-            "0.3.10": "46bc9594c634e1bae9b4c7dfaba1b4b34917b407c3275c08425f93d193024dab"
+            "0.3.10": "3e19ef525e02d2d1dfd42e8d661ee45b4bc8a49a6dcafd8baa578bdb3a23aeb6"
         },
         "arm64": {
-            "0.3.10": "8684425e696dc52c3a10086af34af1ce6a7412de0ef819c3994f11e0e17d7fb9"
+            "0.3.10": "24d2d9cdbb4ed4bda4b0838edb52104ac7a4e2212a0ee05b177de0ae5b6a4a9a"
         }
     },
     "containerd": {

From 17d18f5c1b8ff580bff0b915f098ba03a114791b Mon Sep 17 00:00:00 2001
From: wenwutang <1218040628@qq.com>
Date: Wed, 13 Mar 2024 15:25:09 +0800
Subject: [PATCH 08/22] fix bug: add missing Skip for InstallCriDockerdModule

---
 cmd/kk/pkg/container/module.go | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/cmd/kk/pkg/container/module.go b/cmd/kk/pkg/container/module.go
index 5c732b02a..09c109370 100644
--- a/cmd/kk/pkg/container/module.go
+++ b/cmd/kk/pkg/container/module.go
@@ -281,6 +281,10 @@ type InstallCriDockerdModule struct {
 	Skip bool
 }
 
+func (m *InstallCriDockerdModule) IsSkip() bool {
+	return m.Skip
+}
+
 func (m *InstallCriDockerdModule) Init() {
 	m.Name = "InstallCriDockerdModule"
 	m.Desc = "Install cri-dockerd"

From a21d7ee0b6365de4a80a1c29b3bfb5b5ab3e7770 Mon Sep 17 00:00:00 2001
From: wenwutang <1218040628@qq.com>
Date: Mon, 18 Mar 2024 16:00:55 +0800
Subject: [PATCH 09/22] fix bug: not generate crictl config

---
 cmd/kk/pkg/container/module.go | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/cmd/kk/pkg/container/module.go b/cmd/kk/pkg/container/module.go
index 09c109370..36eaddeac 100644
--- a/cmd/kk/pkg/container/module.go
+++ b/cmd/kk/pkg/container/module.go
@@ -254,7 +254,7 @@ func InstallContainerd(m *InstallContainerModule) []task.Interface {
 		Hosts: m.Runtime.GetHostsByRole(common.K8s),
 		Prepare: &prepare.PrepareCollection{
 			&kubernetes.NodeInCluster{Not: true},
-			&ContainerdExist{Not: true},
+			&CrictlExist{Not: false},
 		},
 		Action: &action.Template{
 			Template: templates.CrictlConfig,

From 427c5061041437ff62c3039f0c1df8907c070148 Mon Sep 17 00:00:00 2001
From: pixiake <guofeng@yunify.com>
Date: Wed, 20 Mar 2024 08:18:16 +0800
Subject: [PATCH 10/22] chore: block message notifications from PRs and bots

---
 .github/workflows/issue_comment_webhook.yml | 13 +++++++------
 1 file changed, 7 insertions(+), 6 deletions(-)

diff --git a/.github/workflows/issue_comment_webhook.yml b/.github/workflows/issue_comment_webhook.yml
index 189cb9569..b0a148fca 100644
--- a/.github/workflows/issue_comment_webhook.yml
+++ b/.github/workflows/issue_comment_webhook.yml
@@ -20,9 +20,10 @@ jobs:
           WEBHOOK_URL: ${{ secrets.WEBHOOK_URL }}
         with:
           script: |
-            console.log(context);
+            console.log(JSON.stringify(context, null, 2));
             if (context.payload.sender.login === "ks-ci-bot") return;
-            const axios = require('axios');
+            if (context.payload.sender.type === 'Bot') return;
+            if (context.payload.issue.html_url.indexOf('/pull/') > 0) return;
             const issue = context.payload.issue;
             const comment = context.payload.comment;
             var subject = {};
@@ -34,12 +35,12 @@ jobs:
               action = "issue";
               subject = issue;
             };
-            const payload = {
+            const webhook_body = {
               msgtype: 'markdown',
               markdown: {
                 content: `[${context.payload.sender.login}](${context.payload.sender.html_url}) ${context.payload.action} ${action} [${issue.title}](${subject.html_url})\n${subject.body}`,
               },
             };
-            const formattedPayload = JSON.stringify(payload, null, 2);
-            console.log(formattedPayload);
-            await axios.post(process.env.WEBHOOK_URL, payload);
+            console.log(JSON.stringify(webhook_body, null, 2));
+            const axios = require('axios');
+            await axios.post(process.env.WEBHOOK_URL, webhook_body);

From 725e8fb9f6c0d9d5cb34435b0b95e9c0b850602d Mon Sep 17 00:00:00 2001
From: joyceliu <joyceliu@yunify.com>
Date: Wed, 20 Mar 2024 15:00:21 +0800
Subject: [PATCH 11/22] fix: Relax artifact strategy. add containerManager

Signed-off-by: joyceliu <joyceliu@yunify.com>
---
 cmd/kk/pkg/binaries/kubernetes.go | 5 ++---
 1 file changed, 2 insertions(+), 3 deletions(-)

diff --git a/cmd/kk/pkg/binaries/kubernetes.go b/cmd/kk/pkg/binaries/kubernetes.go
index 1f43fda86..01f516743 100644
--- a/cmd/kk/pkg/binaries/kubernetes.go
+++ b/cmd/kk/pkg/binaries/kubernetes.go
@@ -121,16 +121,15 @@ func KubernetesComponentBinariesDownload(manifest *common.ArtifactManifest, path
 		binaries = append(binaries, files.NewKubeBinary("calicoctl", arch, m.Components.Calicoctl.Version, path, manifest.Arg.DownloadCommand))
 	}
 
-	containerManagerArr := make([]*files.KubeBinary, 0, 0)
 	containerManagerVersion := make(map[string]struct{})
 	for _, c := range m.Components.ContainerRuntimes {
 		if _, ok := containerManagerVersion[c.Type+c.Version]; !ok {
 			containerManagerVersion[c.Type+c.Version] = struct{}{}
 			containerManager := files.NewKubeBinary(c.Type, arch, c.Version, path, manifest.Arg.DownloadCommand)
-			containerManagerArr = append(containerManagerArr, containerManager)
+			binaries = append(binaries, containerManager)
 			if c.Type == "containerd" {
 				runc := files.NewKubeBinary("runc", arch, kubekeyapiv1alpha2.DefaultRuncVersion, path, manifest.Arg.DownloadCommand)
-				containerManagerArr = append(containerManagerArr, runc)
+				binaries = append(binaries, runc)
 			}
 		}
 	}

From a2ae6c56751b53fbac6426c905f5adf6b00dac06 Mon Sep 17 00:00:00 2001
From: wenwutang <1218040628@qq.com>
Date: Thu, 21 Mar 2024 14:36:04 +0800
Subject: [PATCH 12/22] fix bug: upgrade k8s , kubeadm return coredns start
 version unsupported

---
 cmd/kk/pkg/kubernetes/module.go | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/cmd/kk/pkg/kubernetes/module.go b/cmd/kk/pkg/kubernetes/module.go
index 9123472e2..027ddbd40 100644
--- a/cmd/kk/pkg/kubernetes/module.go
+++ b/cmd/kk/pkg/kubernetes/module.go
@@ -563,6 +563,7 @@ func (p *ProgressiveUpgradeModule) Init() {
 		Desc:  "Generate coredns manifests",
 		Hosts: p.Runtime.GetHostsByRole(common.Master),
 		Prepare: &prepare.PrepareCollection{
+			new(NotEqualPlanVersion),
 			new(common.OnlyFirstMaster),
 		},
 		Action:   new(dns.GenerateCorednsmanifests),
@@ -574,6 +575,7 @@ func (p *ProgressiveUpgradeModule) Init() {
 		Desc:  "Deploy coredns",
 		Hosts: p.Runtime.GetHostsByRole(common.Master),
 		Prepare: &prepare.PrepareCollection{
+			new(NotEqualPlanVersion),
 			new(common.OnlyFirstMaster),
 		},
 		Action:   new(dns.DeployCoreDNS),
@@ -585,6 +587,7 @@ func (p *ProgressiveUpgradeModule) Init() {
 		Desc:  "Generate nodelocaldns",
 		Hosts: p.Runtime.GetHostsByRole(common.Master),
 		Prepare: &prepare.PrepareCollection{
+			new(NotEqualPlanVersion),
 			new(common.OnlyFirstMaster),
 			new(dns.EnableNodeLocalDNS),
 		},
@@ -604,6 +607,7 @@ func (p *ProgressiveUpgradeModule) Init() {
 		Desc:  "Deploy nodelocaldns",
 		Hosts: p.Runtime.GetHostsByRole(common.Master),
 		Prepare: &prepare.PrepareCollection{
+			new(NotEqualPlanVersion),
 			new(common.OnlyFirstMaster),
 			new(dns.EnableNodeLocalDNS)},
 		Action:   new(dns.DeployNodeLocalDNS),

From 5e899b6070bb73427962d5f1fab161e702191e70 Mon Sep 17 00:00:00 2001
From: pixiake <guofeng@yunify.com>
Date: Thu, 21 Mar 2024 17:19:04 +0800
Subject: [PATCH 13/22] add kubekey-system to system workspace

Signed-off-by: pixiake <guofeng@yunify.com>
---
 cmd/kk/pkg/kubernetes/tasks.go | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/cmd/kk/pkg/kubernetes/tasks.go b/cmd/kk/pkg/kubernetes/tasks.go
index 86703000c..ad625591e 100644
--- a/cmd/kk/pkg/kubernetes/tasks.go
+++ b/cmd/kk/pkg/kubernetes/tasks.go
@@ -1027,7 +1027,8 @@ func (s *SaveKubeConfig) Execute(runtime connector.Runtime) error {
 
 	namespace := &corev1.Namespace{
 		ObjectMeta: metav1.ObjectMeta{
-			Name: "kubekey-system",
+			Name:   "kubekey-system",
+			Labels: map[string]string{"kubesphere.io/workspace": "system-workspace"},
 		},
 	}
 	if _, err := clientsetForCluster.

From c16d1658f84c8c75956765d36f67910e18cfdc82 Mon Sep 17 00:00:00 2001
From: baikjy0215 <110450904+baikjy0215@users.noreply.github.com>
Date: Sat, 23 Mar 2024 17:48:19 +1300
Subject: [PATCH 14/22] Debugged calico.tmpl to support IPv6

Added support for setting CALICO_IPV6POOL_BLOCK_SIZE, CALICO_IPV6POOL_VXLAN, CALICO_IPV6POOL_NAT_OUTGOING and FELIX_IPV6SUPPORT.

Hard coded CALICO_IPV6POOL_BLOCK_SIZE as 120. Need to make CALICO_IPV6POOL_BLOCK_SIZE and CALICO_IPV4POOL_BLOCK_SIZE configurable by having something like network.calico.ipv6PoolBlockSize and network.calico.ipv4PoolBlockSize.
---
 cmd/kk/pkg/plugins/network/templates/calico.tmpl | 16 ++++++++++++++--
 1 file changed, 14 insertions(+), 2 deletions(-)

diff --git a/cmd/kk/pkg/plugins/network/templates/calico.tmpl b/cmd/kk/pkg/plugins/network/templates/calico.tmpl
index 73c391022..a612fccf8 100644
--- a/cmd/kk/pkg/plugins/network/templates/calico.tmpl
+++ b/cmd/kk/pkg/plugins/network/templates/calico.tmpl
@@ -4982,9 +4982,19 @@ spec:
             - name: CALICO_IPV4POOL_NAT_OUTGOING
               value: "false"
 {{- end }}
+{{- if .IPv6Support }}
+            # Enable or Disable VXLAN on the default IPv6 IP pool.
+            - name: CALICO_IPV6POOL_VXLAN
+              value: "Always"
+            - name: CALICO_IPV6POOL_NAT_OUTGOING
+              value: "true"
+{{- else }}
             # Enable or Disable VXLAN on the default IPv6 IP pool.
             - name: CALICO_IPV6POOL_VXLAN
               value: "Never"
+            - name: CALICO_IPV6POOL_NAT_OUTGOING
+              value: "false"
+{{- end }}
             # Set MTU for tunnel device used if ipip is enabled
             - name: FELIX_IPINIPMTU
               valueFrom:
@@ -5014,6 +5024,8 @@ spec:
 {{- if .IPv6Support }}
             - name: CALICO_IPV6POOL_CIDR
               value: "{{ .KubePodsV6CIDR }}"
+            - name: CALICO_IPV6POOL_BLOCK_SIZE
+              value: "120"
 {{- end }}
 {{- else }}
             - name: NO_DEFAULT_POOLS
@@ -5033,10 +5045,10 @@ spec:
             # Disable IPv6 on Kubernetes.
 {{- if .IPv6Support }}
             - name: FELIX_IPV6SUPPORT
-              value: "false"
+              value: "true"
 {{- else }}
             - name: FELIX_IPV6SUPPORT
-              value: "true"
+              value: "false"
 {{- end }}
             - name: FELIX_HEALTHENABLED
               value: "true"

From e12e9de088a0ff02329a0e1142f1625dec72641f Mon Sep 17 00:00:00 2001
From: "allcontributors[bot]"
 <46447321+allcontributors[bot]@users.noreply.github.com>
Date: Tue, 26 Mar 2024 10:13:03 +0000
Subject: [PATCH 15/22] update README.md [skip ci]

---
 README.md | 1 +
 1 file changed, 1 insertion(+)

diff --git a/README.md b/README.md
index 3a2c29d06..9600381e6 100644
--- a/README.md
+++ b/README.md
@@ -401,6 +401,7 @@ Thanks goes to these wonderful people ([emoji key](https://allcontributors.org/d
     </tr>
     <tr>
       <td align="center" valign="top" width="14.28%"><a href="https://github.com/xrwang8"><img src="https://avatars.githubusercontent.com/u/68765051?v=4?s=100" width="100px;" alt="Ronald Fletcher"/><br /><sub><b>Ronald Fletcher</b></sub></a><br /><a href="https://github.com/kubesphere/kubekey/commits?author=xrwang8" title="Code">💻</a></td>
+      <td align="center" valign="top" width="14.28%"><a href="https://github.com/baikjy0215"><img src="https://avatars.githubusercontent.com/u/110450904?v=4?s=100" width="100px;" alt="baikjy0215"/><br /><sub><b>baikjy0215</b></sub></a><br /><a href="https://github.com/kubesphere/kubekey/commits?author=baikjy0215" title="Code">💻</a></td>
     </tr>
   </tbody>
 </table>

From 5916799ff08b14c300a254391c8dff8bc145bc41 Mon Sep 17 00:00:00 2001
From: "allcontributors[bot]"
 <46447321+allcontributors[bot]@users.noreply.github.com>
Date: Tue, 26 Mar 2024 10:13:04 +0000
Subject: [PATCH 16/22] update CONTRIBUTORS.md [skip ci]

---
 CONTRIBUTORS.md | 1 +
 1 file changed, 1 insertion(+)

diff --git a/CONTRIBUTORS.md b/CONTRIBUTORS.md
index 1a75af7e8..d882bdd97 100644
--- a/CONTRIBUTORS.md
+++ b/CONTRIBUTORS.md
@@ -117,6 +117,7 @@ Contributions of any kind are welcome! Thanks goes to these wonderful contributo
     </tr>
     <tr>
       <td align="center" valign="top" width="14.28%"><a href="https://github.com/xrwang8"><img src="https://avatars.githubusercontent.com/u/68765051?v=4?s=100" width="100px;" alt="Ronald Fletcher"/><br /><sub><b>Ronald Fletcher</b></sub></a><br /><a href="https://github.com/kubesphere/kubekey/commits?author=xrwang8" title="Code">💻</a></td>
+      <td align="center" valign="top" width="14.28%"><a href="https://github.com/baikjy0215"><img src="https://avatars.githubusercontent.com/u/110450904?v=4?s=100" width="100px;" alt="baikjy0215"/><br /><sub><b>baikjy0215</b></sub></a><br /><a href="https://github.com/kubesphere/kubekey/commits?author=baikjy0215" title="Code">💻</a></td>
     </tr>
   </tbody>
 </table>

From 7fe1b09a5d6579e2f3a6b80f6e43b52e6eb9fe1d Mon Sep 17 00:00:00 2001
From: "allcontributors[bot]"
 <46447321+allcontributors[bot]@users.noreply.github.com>
Date: Tue, 26 Mar 2024 10:13:05 +0000
Subject: [PATCH 17/22] update README_zh-CN.md [skip ci]

---
 README_zh-CN.md | 1 +
 1 file changed, 1 insertion(+)

diff --git a/README_zh-CN.md b/README_zh-CN.md
index 62c4844f2..39619500c 100644
--- a/README_zh-CN.md
+++ b/README_zh-CN.md
@@ -418,6 +418,7 @@ kubectl completion bash >/etc/bash_completion.d/kubectl
     </tr>
     <tr>
       <td align="center" valign="top" width="14.28%"><a href="https://github.com/xrwang8"><img src="https://avatars.githubusercontent.com/u/68765051?v=4?s=100" width="100px;" alt="Ronald Fletcher"/><br /><sub><b>Ronald Fletcher</b></sub></a><br /><a href="https://github.com/kubesphere/kubekey/commits?author=xrwang8" title="Code">💻</a></td>
+      <td align="center" valign="top" width="14.28%"><a href="https://github.com/baikjy0215"><img src="https://avatars.githubusercontent.com/u/110450904?v=4?s=100" width="100px;" alt="baikjy0215"/><br /><sub><b>baikjy0215</b></sub></a><br /><a href="https://github.com/kubesphere/kubekey/commits?author=baikjy0215" title="Code">💻</a></td>
     </tr>
   </tbody>
 </table>

From 7d1c35cbaf374f7d08249c0fede87c930bf2181c Mon Sep 17 00:00:00 2001
From: "allcontributors[bot]"
 <46447321+allcontributors[bot]@users.noreply.github.com>
Date: Tue, 26 Mar 2024 10:13:06 +0000
Subject: [PATCH 18/22] update .all-contributorsrc [skip ci]

---
 .all-contributorsrc | 9 +++++++++
 1 file changed, 9 insertions(+)

diff --git a/.all-contributorsrc b/.all-contributorsrc
index ea9bf6fe9..ea78cea7f 100644
--- a/.all-contributorsrc
+++ b/.all-contributorsrc
@@ -787,6 +787,15 @@
       "contributions": [
         "code"
       ]
+    },
+    {
+      "login": "baikjy0215",
+      "name": "baikjy0215",
+      "avatar_url": "https://avatars.githubusercontent.com/u/110450904?v=4",
+      "profile": "https://github.com/baikjy0215",
+      "contributions": [
+        "code"
+      ]
     }
   ],
   "contributorsPerLine": 7,

From 1122bd528ea68b6e699efc9c8cc500d7689bca10 Mon Sep 17 00:00:00 2001
From: pixiake <guofeng@yunify.com>
Date: Thu, 28 Mar 2024 10:55:02 +0800
Subject: [PATCH 19/22] fix some action mistakes

Signed-off-by: pixiake <guofeng@yunify.com>
---
 hack/gen-repository-iso/dockerfile.ubuntu1604 | 33 -------------------
 hack/gen-repository-iso/dockerfile.ubuntu2204 | 13 --------
 hack/gen-repository-iso/packages.yaml         |  7 ----
 hack/sync-components.sh                       |  2 +-
 4 files changed, 1 insertion(+), 54 deletions(-)
 delete mode 100644 hack/gen-repository-iso/dockerfile.ubuntu1604

diff --git a/hack/gen-repository-iso/dockerfile.ubuntu1604 b/hack/gen-repository-iso/dockerfile.ubuntu1604
deleted file mode 100644
index 719698198..000000000
--- a/hack/gen-repository-iso/dockerfile.ubuntu1604
+++ /dev/null
@@ -1,33 +0,0 @@
-FROM ubuntu:16.04 as ubuntu1604
-ARG TARGETARCH
-ARG OS_RELEASE=xenial
-ARG DIR=ubuntu-16.04-${TARGETARCH}-debs
-ARG PKGS=.common[],.debs[],.ubuntu[],.ubuntu1604[]
-ARG BUILD_TOOLS="apt-transport-https software-properties-common ca-certificates curl wget gnupg dpkg-dev genisoimage"
-ENV DEBIAN_FRONTEND=noninteractive
-
-# dump system package list
-RUN dpkg --get-selections | grep -v deinstall | cut -f1 | cut -d ':' -f1 > packages.list
-RUN apt update -qq \
-    && apt install -y --no-install-recommends $BUILD_TOOLS \
-    && add-apt-repository ppa:gluster/glusterfs-7 -y \
-    && curl -fsSL "https://download.docker.com/linux/ubuntu/gpg" | apt-key add -qq - \
-    && echo "deb [arch=$TARGETARCH] https://download.docker.com/linux/ubuntu ${OS_RELEASE} stable" > /etc/apt/sources.list.d/docker.list\
-    && apt update -qq
-
-WORKDIR /package
-COPY packages.yaml .
-
-COPY --from=mikefarah/yq:4.11.1 /usr/bin/yq /usr/bin/yq
-RUN yq eval "${PKGS}" packages.yaml >> packages.list \
-    && sort -u packages.list | xargs apt-get install --yes --reinstall --print-uris | awk -F "'" '{print $2}' | grep -v '^$' | sort -u > packages.urls
-
-RUN mkdir -p ${DIR} \
-    && wget -q -x -P ${DIR} -i packages.urls \
-    && cd ${DIR} \
-    && dpkg-scanpackages ./ /dev/null | gzip -9c > ./Packages.gz
-
-RUN genisoimage -r -o ${DIR}.iso ${DIR}
-
-FROM scratch
-COPY --from=ubuntu1604 /package/*.iso /
diff --git a/hack/gen-repository-iso/dockerfile.ubuntu2204 b/hack/gen-repository-iso/dockerfile.ubuntu2204
index 42ebb4d94..67522316c 100644
--- a/hack/gen-repository-iso/dockerfile.ubuntu2204
+++ b/hack/gen-repository-iso/dockerfile.ubuntu2204
@@ -16,19 +16,6 @@ RUN apt update -qq \
     && echo "deb [arch=$TARGETARCH] https://download.docker.com/linux/ubuntu ${OS_RELEASE} stable" > /etc/apt/sources.list.d/docker.list\
     && apt update -qq
 
-# install NVIDIA CUDA
-RUN if [ "${TARGETARCH}" = "amd64" ]; then \
-        ARCH=x86_64; \
-    else \
-        ARCH=${TARGETARCH}; \
-    fi \
-    && wget https://developer.download.nvidia.com/compute/cuda/repos/${DISTRO}/${ARCH}/cuda-archive-keyring.gpg \
-    && mv cuda-archive-keyring.gpg /usr/share/keyrings/cuda-archive-keyring.gpg \
-    && echo "deb [signed-by=/usr/share/keyrings/cuda-archive-keyring.gpg] https://developer.download.nvidia.com/compute/cuda/repos/${DISTRO}/${ARCH}/ /" | tee /etc/apt/sources.list.d/cuda-${DISTRO}-${ARCH}.list \
-    && wget https://developer.download.nvidia.com/compute/cuda/repos/${DISTRO}/${ARCH}/cuda-${DISTRO}.pin \
-    && mv cuda-${DISTRO}.pin /etc/apt/preferences.d/cuda-repository-pin-600 \
-    && apt-get update
-
 WORKDIR /package
 COPY packages.yaml .
 
diff --git a/hack/gen-repository-iso/packages.yaml b/hack/gen-repository-iso/packages.yaml
index 4708bd564..3dcc1c4b6 100644
--- a/hack/gen-repository-iso/packages.yaml
+++ b/hack/gen-repository-iso/packages.yaml
@@ -39,9 +39,6 @@ debs:
   - openssh-server
   - software-properties-common
   - sudo
-  - cuda-toolkit-12-4
-  - nvidia-driver-550-open
-  - cuda-drivers-550
 
 centos:
   - containerd.io
@@ -65,10 +62,6 @@ debian11:
 ubuntu:
   - containerd.io
 
-ubuntu1604:
-  - docker-ce=5:20.10.8~3-0~ubuntu-xenial
-  - docker-ce-cli=5:20.10.8~3-0~ubuntu-xenial
-
 ubuntu1804:
   - docker-ce=5:20.10.8~3-0~ubuntu-bionic
   - docker-ce-cli=5:20.10.8~3-0~ubuntu-bionic
diff --git a/hack/sync-components.sh b/hack/sync-components.sh
index d57955217..5236eb2aa 100755
--- a/hack/sync-components.sh
+++ b/hack/sync-components.sh
@@ -95,7 +95,7 @@ if [ $KUBERNETES_VERSION ]; then
 
        qsctl cp binaries/kube/$KUBERNETES_VERSION/$arch/$binary \
              qs://kubernetes-release/release/$KUBERNETES_VERSION/bin/linux/$arch/$binary \
-             -c qsctl-config.yaml -f
+             -c qsctl-config.yaml
      done
    done
 

From 0938e59c9275e94a84a2a7b1072098c7ad24c209 Mon Sep 17 00:00:00 2001
From: stark <stark@yunify.com>
Date: Thu, 28 Mar 2024 16:07:41 +0800
Subject: [PATCH 20/22] fix config and start containerd on no registry node

---
 cmd/kk/pkg/bootstrap/registry/module.go | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/cmd/kk/pkg/bootstrap/registry/module.go b/cmd/kk/pkg/bootstrap/registry/module.go
index 2b38a1be1..7fc7ebf27 100644
--- a/cmd/kk/pkg/bootstrap/registry/module.go
+++ b/cmd/kk/pkg/bootstrap/registry/module.go
@@ -173,7 +173,7 @@ func InstallHarbor(i *InstallRegistryModule) []task.Interface {
 	generateContainerdService := &task.RemoteTask{
 		Name:  "GenerateContainerdService",
 		Desc:  "Generate containerd service",
-		Hosts: i.Runtime.GetHostsByRole(common.K8s),
+		Hosts: i.Runtime.GetHostsByRole(common.Registry),
 		Prepare: &prepare.PrepareCollection{
 			&container.ContainerdExist{Not: true},
 		},
@@ -219,7 +219,7 @@ func InstallHarbor(i *InstallRegistryModule) []task.Interface {
 	enableContainerdForDocker := &task.RemoteTask{
 		Name:  "EnableContainerd",
 		Desc:  "Enable containerd",
-		Hosts: i.Runtime.GetHostsByRole(common.K8s),
+		Hosts: i.Runtime.GetHostsByRole(common.Registry),
 		Prepare: &prepare.PrepareCollection{
 			&container.ContainerdExist{Not: true},
 		},

From ce434173e7be9ddfa70bc89681864fa3c3275bfd Mon Sep 17 00:00:00 2001
From: ks-ci-bot <47586280+ks-ci-bot@users.noreply.github.com>
Date: Fri, 29 Mar 2024 02:05:28 +0000
Subject: [PATCH 21/22] Add new kubernetes version

---
 docs/kubernetes-versions.md |  8 +++++
 version/components.json     | 62 ++++++++++++++++++++++++++++++++-----
 2 files changed, 63 insertions(+), 7 deletions(-)

diff --git a/docs/kubernetes-versions.md b/docs/kubernetes-versions.md
index 6d3d40ad2..d98d3982a 100644
--- a/docs/kubernetes-versions.md
+++ b/docs/kubernetes-versions.md
@@ -108,6 +108,8 @@
 | v1.26.11 | :white_check_mark: |
 | v1.26.12 | :white_check_mark: |
 | v1.26.13 | :white_check_mark: |
+| v1.26.14 | :white_check_mark: |
+| v1.26.15 | :white_check_mark: |
 | v1.27.0 | :white_check_mark: |
 | v1.27.1 | :white_check_mark: |
 | v1.27.2 | :white_check_mark: |
@@ -119,6 +121,8 @@
 | v1.27.8 | :white_check_mark: |
 | v1.27.9 | :white_check_mark: |
 | v1.27.10 | :white_check_mark: |
+| v1.27.11 | :white_check_mark: |
+| v1.27.12 | :white_check_mark: |
 | v1.28.0 | :white_check_mark: |
 | v1.28.1 | :white_check_mark: |
 | v1.28.2 | :white_check_mark: |
@@ -126,5 +130,9 @@
 | v1.28.4 | :white_check_mark: |
 | v1.28.5 | :white_check_mark: |
 | v1.28.6 | :white_check_mark: |
+| v1.28.7 | :white_check_mark: |
+| v1.28.8 | :white_check_mark: |
 | v1.29.0 | :white_check_mark: |
 | v1.29.1 | :white_check_mark: |
+| v1.29.2 | :white_check_mark: |
+| v1.29.3 | :white_check_mark: |
diff --git a/version/components.json b/version/components.json
index 8cdb05ac6..7222fcc82 100644
--- a/version/components.json
+++ b/version/components.json
@@ -108,6 +108,8 @@
             "v1.26.11": "58f886e39e517ba1a92493f136e80f1b6ea9362966ad9d2accdf2133004161f2",
             "v1.26.12": "5a5d65acefb50010859be8ffba8e6e059d552ae357e3101c12c62e747a9416a2",
             "v1.26.13": "62c0e49f749d230f0c27fa53cbe4851894d9002757fd3653fc5da36aa7c68032",
+            "v1.26.14": "fd092c1dc3a62ec37f416d1c07e2ae45a8aa4136c8c02b83e568031f6b521018",
+            "v1.26.15": "7e0f5bd34ddc6fac932cf9a137c57ade150ec53f9558d693e336ed9948d6ba76",
             "v1.27.0": "78d0e04705a7bdb76a514d60f60c073b16334b15f57ee87f064354ca8a233e80",
             "v1.27.1": "c7d32d698e99b90f877025104cb4a9f3f8c707e99e6817940f260135b6d1ad0a",
             "v1.27.2": "95c4bfb7929900506a42de4d92280f06efe6b47e0a32cbc1f5a1ed737592977a",
@@ -119,6 +121,8 @@
             "v1.27.8": "f8864769b8b2d7a14f53eb983f23317ff14d68ab76aba71e9de17ce84c38d4eb",
             "v1.27.9": "78dddac376fa2f04116022cb44ed39ccb9cb0104e05c5b21b220d5151e5c0f86",
             "v1.27.10": "23985e958443ac1aabdbeeedc675358abc0638eb580707829fd42b0996a0aae5",
+            "v1.27.11": "31bf446a712fb08190838c35d1f4c93b0f975708c59634a5dc3d8915a241c83e",
+            "v1.27.12": "06ee36cc80cfdfc01c937d750783d3ca6169a3da76382c7af3dd172d9f6bfa4e",
             "v1.28.0": "12ea68bfef0377ccedc1a7c98a05ea76907decbcf1e1ec858a60a7b9b73211bb",
             "v1.28.1": "6134dbc92dcb83c3bae1a8030f7bb391419b5d13ea94badd3a79b7ece75b2736",
             "v1.28.2": "6a4808230661c69431143db2e200ea2d021c7f1b1085e6353583075471310d00",
@@ -126,8 +130,12 @@
             "v1.28.4": "b4d2531b7cddf782f59555436bc098485b5fa6c05afccdeecf0d62d21d84f5bd",
             "v1.28.5": "2b54078c5ea9e85b27f162f508e0bf834a2753e52a57e896812ec3dca92fe9cd",
             "v1.28.6": "bda3eda8d51e8746a42b535b7eab7df52b091a796227c3212dc30909a8f1b431",
+            "v1.28.7": "8aa005bdf6af43e47fc818b26f4cb9f361aae8ec4390519e8d4033be65fbef2b",
+            "v1.28.8": "c11946cbfd962e1197062534514226cfd70230349e6343ff3ecebfca5476ee64",
             "v1.29.0": "629d4630657caace9c819fd3797f4a70c397fbd41a2a7e464a0507dad675d52c",
-            "v1.29.1": "d4d81d9020b550c896376fb9e0586a9f15a332175890d061619b52b3e9bc6cbd"
+            "v1.29.1": "d4d81d9020b550c896376fb9e0586a9f15a332175890d061619b52b3e9bc6cbd",
+            "v1.29.2": "2d4e4fa8685bcbfb661cb41050cd4756f50a7aa147f68492d51a99f9cdfd69ac",
+            "v1.29.3": "6abaa1208bf40b6d1f49e518bd68c8ae4a1be0c5b7d3e45d87979999ab070d8b"
         },
         "arm64": {
             "v1.19.0": "db1c432646e6e6484989b6f7191f3610996ac593409f12574290bfc008ea11f5",
@@ -237,6 +245,8 @@
             "v1.26.11": "a13318c1493e58a9f7c4359c79443f3c86a690ec601bcc76308c809d8d61edb8",
             "v1.26.12": "2dee03d460e8a1b3f30e5ef48ce94f5eeb4ff4550e65860e6e2d94b368c2cde6",
             "v1.26.13": "2cce047abb5df5c9e1e48405e1d4822f2ccc6609668db49e95d9ccbbeaa71a7a",
+            "v1.26.14": "42183dce0508a26a8708c1eeb296abfcbf06f0ae4f6de64e4a7317c0be371d1d",
+            "v1.26.15": "881a76cc98da0fe91394cb26f909186d0dcbf378262d11803ccf76f2a84a9039",
             "v1.27.0": "acd805c6783b678ee0068b9dd8165bbfd879c345fd9c25d6a978dbc965f48544",
             "v1.27.1": "024a59cd6fc76784b597c0c1cf300526e856e8c9fefa5fa7948158929b739551",
             "v1.27.2": "8f01f363f7c7f92de2f2276124a895503cdc5a60ff549440170880f296b087eb",
@@ -248,6 +258,8 @@
             "v1.27.8": "0d0f5b2781d663d314e785d14361aa5a09cfaf6e1694aa3cc731f4f06342ec13",
             "v1.27.9": "d3d022842b0b8e4661222e8873249f5acafdbef52fd1bfb98152a582352b3c40",
             "v1.27.10": "ed0447155a7e967ae23480b06b31b2c0aaa871e7c59dfd82ae25b03a1eccf6e6",
+            "v1.27.11": "b8452d6c3f1331beb3d5fa42466a9bc96638a76c40980dba9822300f230c0858",
+            "v1.27.12": "e74d47c14b5a251cff961dcce92cd632abcfd0fba4a07e78f0a5a5b2796e4b84",
             "v1.28.0": "b9b473d2d9136559b19eb465006af77df45c09862cd7ce6673a33aae517ff5ab",
             "v1.28.1": "7d2f68917470a5d66bd2a7d62897f59cb4afaeffb2f26c028afa119acd8c3fc8",
             "v1.28.2": "010789a94cf512d918ec4a3ef8ec734dea0061d89a8293059ef9101ca1bf6bff",
@@ -255,8 +267,12 @@
             "v1.28.4": "a4422780020954436b8e76ab1c59b68c5581a54432dd3e566c4709bb40c8d4f9",
             "v1.28.5": "22bb6b3377204e93d008f33ac4924d77adca1478f1ae3b515c03476ba54f1adc",
             "v1.28.6": "4298cad464e92eec19cdf3e6a607a82a1d626ae70fedba7956175152ab983457",
+            "v1.28.7": "f556e49494737f97a15bf15bb4b27d45f8747b477302cdfd22dd61816bc02203",
+            "v1.28.8": "e0f47adc69ef84e2f6c42cc341b8a790904a929ad10ed1c23c2e822ec804e247",
             "v1.29.0": "bbddee2d46d2e1643ae3623698b45b13aa2e858616d61c642f2f49e5bb14c980",
-            "v1.29.1": "3bff8c50c104c45e416cce9991706c6ac46365f0defbcd54f8cf4ace0fa68dcf"
+            "v1.29.1": "3bff8c50c104c45e416cce9991706c6ac46365f0defbcd54f8cf4ace0fa68dcf",
+            "v1.29.2": "e05720feb9d2d67eff25b0156a5c22e2de37be2ffab4e1f4d31e8c526fafd0e1",
+            "v1.29.3": "ce2e4c230f954e59ae77e34c4ff2ae08cad3970505ae1e21b6337e6d83b21682"
         }
     },
     "kubelet": {
@@ -368,6 +384,8 @@
             "v1.26.11": "a62953f20fa9fedff50c6c5423e68981e3382d92cf04174d5bca5f4d084de0c5",
             "v1.26.12": "aed0a351b01f1e6a84a0992ef1265bb0c9994b900162c075df58d0d02517d3df",
             "v1.26.13": "05860ef65deb594dc72034c0614f93d7ba9d1f229ff73b43c484298e22465f1b",
+            "v1.26.14": "37f83734208f88afba0f042c4436a44356451127602b06a153bfcc766f8c2af3",
+            "v1.26.15": "20db3c21d8e7215a581e48393be5b924267f1bb82ac69e22bd701a10df9f3974",
             "v1.27.0": "0b4ed4fcd75d33f5dff3ba17776e6089847fc83064d3f7a3ad59a34e94e60a29",
             "v1.27.1": "cb2845fff0ce41c400489393da73925d28fbee54cfeb7834cd4d11e622cbd3a7",
             "v1.27.2": "a0d12afcab3b2836de4a427558d067bebdff040e9b306b0512c93d9d2a066579",
@@ -379,6 +397,8 @@
             "v1.27.8": "2e0557b38c5b9a1263eed25a0b84d741453ed9c0c7bd916f80eadaf7edfb7784",
             "v1.27.9": "ede60eea3acbac3f35dbb23d7b148f45cf169ebbb20af102d3ce141fc0bac60c",
             "v1.27.10": "25a34bf98bb8a296ea07f1ebbcb496b1e6b6c6da3247695288a7c99fc8c1be2c",
+            "v1.27.11": "2ce92a5d8985b93bd8ffc4f5519cd79bf2f844590aa38228a3d809c5bf5986e0",
+            "v1.27.12": "aae861a21913c274228ccdad1609b370e5198c9f4b39b8924b20a7ffe7f148e0",
             "v1.28.0": "bfb6b977100963f2879a33e5fbaa59a5276ba829a957a6819c936e9c1465f981",
             "v1.28.1": "2bc22332f44f8fcd3fce57879fd873f977949ebd261571fbae31fbb2713a5dd3",
             "v1.28.2": "17edb866636f14eceaad58c56eab12af7ab3be3c78400aff9680635d927f1185",
@@ -386,8 +406,12 @@
             "v1.28.4": "db2a473b73c3754d4011590f2f0aa877657608499590c6b0f8b40bec96a3e9ba",
             "v1.28.5": "bf37335da58182783a8c63866ec1f895b4c436e3ed96bdd87fe3f8ae8004ba1d",
             "v1.28.6": "8506df1f20a5f8bba0592f5a4cf5d0cc541047708e664cb88580735400d0b26f",
+            "v1.28.7": "120b1495babc4364f7e16a9d0f8b8e6b6f78316d047e4f6de77b5569b05813c7",
+            "v1.28.8": "049b412a5861255cd3922f612acb79ab51135e166c5d80acf12fba9179eebf0c",
             "v1.29.0": "e1c38137db8d8777eed8813646b59bf4d22d19b9011ab11dc28e2e34f6b80a05",
-            "v1.29.1": "1b1975c58d38be1a99a8bcba4564ac489afd223b0abe9f2ab08bbde89d2412a3"
+            "v1.29.1": "1b1975c58d38be1a99a8bcba4564ac489afd223b0abe9f2ab08bbde89d2412a3",
+            "v1.29.2": "f71a85039b71fe08f1c063a93d61a1c952dc8f9a8c6be9b13fbdac8f0d9ff960",
+            "v1.29.3": "d8b55a2f8a87c8cd2cbf867d76d1d7f98b7198a740db19bad6ed7b8b813de771"
         },
         "arm64": {
             "v1.19.0": "d8fa5a9739ecc387dfcc55afa91ac6f4b0ccd01f1423c423dbd312d787bbb6bf",
@@ -497,6 +521,8 @@
             "v1.26.11": "ff8940394446028e75a2b8155e22eccf635f6a128f45dee41e293493d2743d17",
             "v1.26.12": "d4406ed5bfd12768c03fc4fbe011a01e5c91b74d1d4b526fe3ac320d13295ffb",
             "v1.26.13": "4925b769336df7cf5e7a7f33d82ee5c69ac67a7680ac1f7064534f971d9a79ab",
+            "v1.26.14": "1c3f0f42d59966e8eb89ab37ba57299fa44e3e28b858b48329b340a326d31148",
+            "v1.26.15": "4de64168aa08fe2149ec4a29cfaa683947a8ad866090aaf2c75336ce547b85b0",
             "v1.27.0": "37aa2edc7c0c4b3e488518c6a4b44c8aade75a55010534ee2be291220c73d157",
             "v1.27.1": "dbb09d297d924575654db38ed2fc627e35913c2d4000c34613ac6de4995457d0",
             "v1.27.2": "810cd9a611e9f084e57c9ee466e33c324b2228d4249ff38c2588a0cc3224f10d",
@@ -508,6 +534,8 @@
             "v1.27.8": "71849182ceb018dc084f499ad28b7b1afb7f23e35ccaf8421941dd5dafef0d4c",
             "v1.27.9": "8a14bc3739f5ca3b23d08301c2e769ee58c8d1cecb7243b46b1c098ae77effd7",
             "v1.27.10": "0edadc44ef36be8d8106cad9972360c0477540e2d8c0bbeb38fd97fd1d7801d5",
+            "v1.27.11": "e81987a864fb47afe14f65fa4e93760bc19c424335e0f0540c6c725b727ce22a",
+            "v1.27.12": "0d7d2d25c8b909d6cec7c1c2a5bfe51428ec33eaa5e8b209c718b77983e9dcba",
             "v1.28.0": "05dd12e35783cab4960e885ec0e7d0e461989b94297e7bea9018ccbd15c4dce9",
             "v1.28.1": "9b7fa64b2785da4a38768377961e227f8da629c56a5df43ca1b665dd07b56f3c",
             "v1.28.2": "32269e9ec38c561d028b65c3048ea6a100e1292cbe9e505565222455c8096577",
@@ -515,8 +543,12 @@
             "v1.28.4": "bf203989dd9b3987b8a0d2331dcce6319f834b57df810fafba5a4805d54823ac",
             "v1.28.5": "28ddb696eb6e076f2a2f59ccaa2e409785a63346e5bda819717c6e0f58297702",
             "v1.28.6": "ee2c060deff330d3338e24aec9734c9e5d5aea4fea1905c0795bccff6997a65e",
+            "v1.28.7": "e2c98b39b0b0745ef3e30febaeb8eaaf31ec721012405bd0dcf25e84026c221e",
+            "v1.28.8": "90d61f40b7bb061b0fc6d08b8b9ddae51f90863c899b098e19eaa89dc855f2c0",
             "v1.29.0": "0e0e4544c2a0a3475529154b7534d0d58683466efa04a2bb2e763b476db0bb16",
-            "v1.29.1": "e46417ab1ceae995f0e00d4177959a36ed34b807829422bc9dda70b263fe5c5d"
+            "v1.29.1": "e46417ab1ceae995f0e00d4177959a36ed34b807829422bc9dda70b263fe5c5d",
+            "v1.29.2": "9b4aa572d4cd51a41b1067161d961423d0d12b120fb636ea887a12a975d4b19a",
+            "v1.29.3": "891dce19ed0eae34050c2eca0454204892e97bfe1a926f988cd044a987a9c7c9"
         }
     },
     "kubectl": {
@@ -628,6 +660,8 @@
             "v1.26.11": "27c34a0870230d9dd723e1e01114634e396cd2a3d25ced263b769a4bd53e4edd",
             "v1.26.12": "8e6af8d68e7b9d2a1eb43255c0da793276e549a34a2b9c3c87a9c26438e7fd71",
             "v1.26.13": "e4bad4273431f9f5f05f27f5c2054cbbad6d9ee00b85e0810cb4ef0489b02571",
+            "v1.26.14": "afd9be91832a0400d8d9cc3da1cf2a395aa9f13b8ab245883869788786166ec3",
+            "v1.26.15": "b75f359e6fad3cdbf05a0ee9d5872c43383683bb8527a9e078bb5b8a44350a41",
             "v1.27.0": "71a78259d70da9c5540c4cf4cff121f443e863376f68f89a759d90cef3f51e87",
             "v1.27.1": "7fe3a762d926fb068bae32c399880e946e8caf3d903078bea9b169dcd5c17f6d",
             "v1.27.2": "4f38ee903f35b300d3b005a9c6bfb9a46a57f92e89ae602ef9c129b91dc6c5a5",
@@ -639,6 +673,8 @@
             "v1.27.8": "027b3161e99fa0a7fa529e8f17f73ee2c0807c81c721ca7cf307f6b41c17bc57",
             "v1.27.9": "d0caae91072297b2915dd65f6ef3055d27646dce821ec67d18da35ba9a8dc85b",
             "v1.27.10": "bfb219643c28d9842fceae51590776f06987835d93fc3cb9b0149c9111c741ac",
+            "v1.27.11": "7ae327978a1edb43700070c86f5fd77215792c6b58a7ea70192647e0da848e29",
+            "v1.27.12": "d639eda39be2dce42fbec21e038942ab5734541715e3ea5fb29c9ad76686bd7f",
             "v1.28.0": "4717660fd1466ec72d59000bb1d9f5cdc91fac31d491043ca62b34398e0799ce",
             "v1.28.1": "e7a7d6f9d06fab38b4128785aa80f65c54f6675a0d2abef655259ddd852274e1",
             "v1.28.2": "c922440b043e5de1afa3c1382f8c663a25f055978cbc6e8423493ec157579ec5",
@@ -646,8 +682,12 @@
             "v1.28.4": "893c92053adea6edbbd4e959c871f5c21edce416988f968bec565d115383f7b8",
             "v1.28.5": "2a44c0841b794d85b7819b505da2ff3acd5950bd1bcd956863714acc80653574",
             "v1.28.6": "c8351fe0611119fd36634dd3f53eb94ec1a2d43ef9e78b92b4846df5cc7aa7e3",
+            "v1.28.7": "aff42d3167685e4d8e86fda0ad9c6ce6ec6c047bc24d608041d54717a18192ba",
+            "v1.28.8": "e02aad5c0bac52c970700b814645b62c4f18b634144398ac344875dbaf1072f8",
             "v1.29.0": "0e03ab096163f61ab610b33f37f55709d3af8e16e4dcc1eb682882ef80f96fd5",
-            "v1.29.1": "69ab3a931e826bf7ac14d38ba7ca637d66a6fcb1ca0e3333a2cafdf15482af9f"
+            "v1.29.1": "69ab3a931e826bf7ac14d38ba7ca637d66a6fcb1ca0e3333a2cafdf15482af9f",
+            "v1.29.2": "7816d067740f47f949be826ac76943167b7b3a38c4f0c18b902fffa8779a5afa",
+            "v1.29.3": "89c0435cec75278f84b62b848b8c0d3e15897d6947b6c59a49ddccd93d7312bf"
         },
         "arm64": {
             "v1.19.0": "d4adf1b6b97252025cb2f7febf55daa3f42dc305822e3da133f77fd33071ec2f",
@@ -757,6 +797,8 @@
             "v1.26.11": "4a6d2b7204af3cf84cd0e2c670fbb211501050c9a288de49de3c6363d4e0a63e",
             "v1.26.12": "b9cfdb6c9a53146330d5694e711c48febb7ec022aea8d16a51b5e12d761580cc",
             "v1.26.13": "4abebc34c114111b81bbf5222f0810e6899937d04bc453d9ccd77046643bbcda",
+            "v1.26.14": "64b21431e5ff18a4999699d53815236ccb06a16d76c00cb812461110a5bd6318",
+            "v1.26.15": "1396313f0f8e84ab1879757797992f1af043e1050283532e0fd8469902632216",
             "v1.27.0": "f8e09630211f2b7c6a8cc38835e7dea94708d401f5c84b23a37c70c604602ddc",
             "v1.27.1": "fd3cb8f16e6ed8aee9955b76e3027ac423b6d1cc7356867310d128082e2db916",
             "v1.27.2": "1b0966692e398efe71fe59f913eaec44ffd4468cc1acd00bf91c29fa8ff8f578",
@@ -768,6 +810,8 @@
             "v1.27.8": "97ed6739e2803e63fd2d9de78be22d5ba6205bb63179a16ec773063526525a8e",
             "v1.27.9": "bda475539fdeda9d8a85a84b967af361af264d0826c121b23b0b62ee9b00cd2d",
             "v1.27.10": "2e1996379d5a8b132e0606fcd3df3c8689e11882630b75cca3b7135126847871",
+            "v1.27.11": "d30e1aa873e78eb376ddee3c785aa78c44eddc56ce2ef901dac1ce0c2c4f50b0",
+            "v1.27.12": "bfc6cb71041ebc0f048402988eccc107cfff2b866c864231c9ada05ab328e5bf",
             "v1.28.0": "f5484bd9cac66b183c653abed30226b561f537d15346c605cc81d98095f1717c",
             "v1.28.1": "46954a604b784a8b0dc16754cfc3fa26aabca9fd4ffd109cd028bfba99d492f6",
             "v1.28.2": "ea6d89b677a8d9df331a82139bb90d9968131530b94eab26cee561531eff4c53",
@@ -775,8 +819,12 @@
             "v1.28.4": "edf1e17b41891ec15d59dd3cc62bcd2cdce4b0fd9c2ee058b0967b17534457d7",
             "v1.28.5": "f87fe017ae3ccfd93df03bf17edd4089672528107f230563b8c9966909661ef2",
             "v1.28.6": "0de705659a80c3fef01df43cc0926610fe31482f728b0f992818abd9bdcd2cb9",
+            "v1.28.7": "13d547495bdea49b223fe06bffb6d2bef96436634847f759107655aa80fc990e",
+            "v1.28.8": "93d60dd36093b4c719f1f1bafcf59437c17cb2209341c7c94771e7dd9acdab33",
             "v1.29.0": "8f7a4bd6bae900a4ddab12bd1399aa652c0d59ea508f39b910e111d248893ff7",
-            "v1.29.1": "96d6dc7b2bdcd344ce58d17631c452225de5bbf59b83fd3c89c33c6298fb5d8b"
+            "v1.29.1": "96d6dc7b2bdcd344ce58d17631c452225de5bbf59b83fd3c89c33c6298fb5d8b",
+            "v1.29.2": "3507ecb4224cf05ae2151a98d4932253624e7762159936d5347b19fe037655ca",
+            "v1.29.3": "191a96b27e3c6ae28b330da4c9bfefc9592762670727df4fcf124c9f1d5a466a"
         }
     },
     "etcd": {
@@ -1216,4 +1264,4 @@
             "v3.27.2": "0fd1f65a511338cf9940835987d420c94ab95b5386288ba9673b736a4d347463"
         }
     }
-}
+}
\ No newline at end of file

From 1d70273286f44c613a69a3287945c10a274e205d Mon Sep 17 00:00:00 2001
From: pixiake <guofeng@yunify.com>
Date: Fri, 29 Mar 2024 21:58:45 +0800
Subject: [PATCH 22/22] system performance optimization

Signed-off-by: pixiake <guofeng@yunify.com>
---
 .../pkg/bootstrap/os/templates/init_script.go | 64 ++++++++++---------
 cmd/kk/pkg/etcd/templates/etcd_service.go     |  5 +-
 .../pkg/plugins/network/templates/calico.tmpl |  4 +-
 3 files changed, 40 insertions(+), 33 deletions(-)

diff --git a/cmd/kk/pkg/bootstrap/os/templates/init_script.go b/cmd/kk/pkg/bootstrap/os/templates/init_script.go
index ca671010c..574041038 100644
--- a/cmd/kk/pkg/bootstrap/os/templates/init_script.go
+++ b/cmd/kk/pkg/bootstrap/os/templates/init_script.go
@@ -76,6 +76,9 @@ echo 'net.ipv4.neigh.default.gc_thresh3 = 4096' >> /etc/sysctl.conf
 echo 'net.ipv4.tcp_retries2 = 15' >> /etc/sysctl.conf
 echo 'net.ipv4.tcp_max_tw_buckets = 1048576' >> /etc/sysctl.conf
 echo 'net.ipv4.tcp_max_orphans = 65535' >> /etc/sysctl.conf
+echo 'net.ipv4.tcp_keepalive_time = 600' >> /etc/sysctl.conf
+echo 'net.ipv4.tcp_keepalive_intvl = 30' >> /etc/sysctl.conf
+echo 'net.ipv4.tcp_keepalive_probes = 10' >> /etc/sysctl.conf
 echo 'net.ipv4.udp_rmem_min = 131072' >> /etc/sysctl.conf
 echo 'net.ipv4.udp_wmem_min = 131072' >> /etc/sysctl.conf
 echo 'net.ipv4.conf.all.rp_filter = 1' >> /etc/sysctl.conf
@@ -106,37 +109,40 @@ sed -r -i "s@#{0,}?net.ipv4.tcp_tw_recycle ?= ?(0|1|2)@net.ipv4.tcp_tw_recycle =
 sed -r -i "s@#{0,}?net.ipv4.tcp_tw_reuse ?= ?(0|1)@net.ipv4.tcp_tw_reuse = 0@g" /etc/sysctl.conf
 sed -r -i "s@#{0,}?net.ipv4.conf.all.rp_filter ?= ?(0|1|2)@net.ipv4.conf.all.rp_filter = 1@g" /etc/sysctl.conf
 sed -r -i "s@#{0,}?net.ipv4.conf.default.rp_filter ?= ?(0|1|2)@net.ipv4.conf.default.rp_filter = 1@g" /etc/sysctl.conf
-sed -r -i  "s@#{0,}?net.ipv4.ip_forward ?= ?(0|1)@net.ipv4.ip_forward = 1@g" /etc/sysctl.conf
-sed -r -i  "s@#{0,}?net.bridge.bridge-nf-call-arptables ?= ?(0|1)@net.bridge.bridge-nf-call-arptables = 1@g" /etc/sysctl.conf
-sed -r -i  "s@#{0,}?net.bridge.bridge-nf-call-ip6tables ?= ?(0|1)@net.bridge.bridge-nf-call-ip6tables = 1@g" /etc/sysctl.conf
-sed -r -i  "s@#{0,}?net.bridge.bridge-nf-call-iptables ?= ?(0|1)@net.bridge.bridge-nf-call-iptables = 1@g" /etc/sysctl.conf
-sed -r -i  "s@#{0,}?net.ipv4.ip_local_reserved_ports ?= ?([0-9]{1,}-{0,1},{0,1}){1,}@net.ipv4.ip_local_reserved_ports = 30000-32767@g" /etc/sysctl.conf
-sed -r -i  "s@#{0,}?vm.max_map_count ?= ?([0-9]{1,})@vm.max_map_count = 262144@g" /etc/sysctl.conf
-sed -r -i  "s@#{0,}?vm.swappiness ?= ?([0-9]{1,})@vm.swappiness = 0@g" /etc/sysctl.conf
-sed -r -i  "s@#{0,}?fs.inotify.max_user_instances ?= ?([0-9]{1,})@fs.inotify.max_user_instances = 524288@g" /etc/sysctl.conf
-sed -r -i  "s@#{0,}?kernel.pid_max ?= ?([0-9]{1,})@kernel.pid_max = 65535@g" /etc/sysctl.conf
+sed -r -i "s@#{0,}?net.ipv4.ip_forward ?= ?(0|1)@net.ipv4.ip_forward = 1@g" /etc/sysctl.conf
+sed -r -i "s@#{0,}?net.bridge.bridge-nf-call-arptables ?= ?(0|1)@net.bridge.bridge-nf-call-arptables = 1@g" /etc/sysctl.conf
+sed -r -i "s@#{0,}?net.bridge.bridge-nf-call-ip6tables ?= ?(0|1)@net.bridge.bridge-nf-call-ip6tables = 1@g" /etc/sysctl.conf
+sed -r -i "s@#{0,}?net.bridge.bridge-nf-call-iptables ?= ?(0|1)@net.bridge.bridge-nf-call-iptables = 1@g" /etc/sysctl.conf
+sed -r -i "s@#{0,}?net.ipv4.ip_local_reserved_ports ?= ?([0-9]{1,}-{0,1},{0,1}){1,}@net.ipv4.ip_local_reserved_ports = 30000-32767@g" /etc/sysctl.conf
+sed -r -i "s@#{0,}?vm.max_map_count ?= ?([0-9]{1,})@vm.max_map_count = 262144@g" /etc/sysctl.conf
+sed -r -i "s@#{0,}?vm.swappiness ?= ?([0-9]{1,})@vm.swappiness = 0@g" /etc/sysctl.conf
+sed -r -i "s@#{0,}?fs.inotify.max_user_instances ?= ?([0-9]{1,})@fs.inotify.max_user_instances = 524288@g" /etc/sysctl.conf
+sed -r -i "s@#{0,}?kernel.pid_max ?= ?([0-9]{1,})@kernel.pid_max = 65535@g" /etc/sysctl.conf
 sed -r -i "s@#{0,}?vm.overcommit_memory ?= ?(0|1|2)@vm.overcommit_memory = 0@g" /etc/sysctl.conf
-sed -r -i  "s@#{0,}?fs.inotify.max_user_watches ?= ?([0-9]{1,})@fs.inotify.max_user_watches = 524288@g" /etc/sysctl.conf
-sed -r -i  "s@#{0,}?fs.pipe-max-size ?= ?([0-9]{1,})@fs.pipe-max-size = 4194304@g" /etc/sysctl.conf
-sed -r -i  "s@#{0,}?net.core.netdev_max_backlog ?= ?([0-9]{1,})@net.core.netdev_max_backlog = 65535@g" /etc/sysctl.conf
-sed -r -i  "s@#{0,}?net.core.rmem_max ?= ?([0-9]{1,})@net.core.rmem_max = 33554432@g" /etc/sysctl.conf
-sed -r -i  "s@#{0,}?net.core.wmem_max ?= ?([0-9]{1,})@net.core.wmem_max = 33554432@g" /etc/sysctl.conf
-sed -r -i  "s@#{0,}?net.ipv4.tcp_max_syn_backlog ?= ?([0-9]{1,})@net.ipv4.tcp_max_syn_backlog = 1048576@g" /etc/sysctl.conf
-sed -r -i  "s@#{0,}?net.ipv4.neigh.default.gc_thresh1 ?= ?([0-9]{1,})@net.ipv4.neigh.default.gc_thresh1 = 512@g" /etc/sysctl.conf
-sed -r -i  "s@#{0,}?net.ipv4.neigh.default.gc_thresh2 ?= ?([0-9]{1,})@net.ipv4.neigh.default.gc_thresh2 = 2048@g" /etc/sysctl.conf
-sed -r -i  "s@#{0,}?net.ipv4.neigh.default.gc_thresh3 ?= ?([0-9]{1,})@net.ipv4.neigh.default.gc_thresh3 = 4096@g" /etc/sysctl.conf
-sed -r -i  "s@#{0,}?net.core.somaxconn ?= ?([0-9]{1,})@net.core.somaxconn = 32768@g" /etc/sysctl.conf
+sed -r -i "s@#{0,}?fs.inotify.max_user_watches ?= ?([0-9]{1,})@fs.inotify.max_user_watches = 524288@g" /etc/sysctl.conf
+sed -r -i "s@#{0,}?fs.pipe-max-size ?= ?([0-9]{1,})@fs.pipe-max-size = 4194304@g" /etc/sysctl.conf
+sed -r -i "s@#{0,}?net.core.netdev_max_backlog ?= ?([0-9]{1,})@net.core.netdev_max_backlog = 65535@g" /etc/sysctl.conf
+sed -r -i "s@#{0,}?net.core.rmem_max ?= ?([0-9]{1,})@net.core.rmem_max = 33554432@g" /etc/sysctl.conf
+sed -r -i "s@#{0,}?net.core.wmem_max ?= ?([0-9]{1,})@net.core.wmem_max = 33554432@g" /etc/sysctl.conf
+sed -r -i "s@#{0,}?net.ipv4.tcp_max_syn_backlog ?= ?([0-9]{1,})@net.ipv4.tcp_max_syn_backlog = 1048576@g" /etc/sysctl.conf
+sed -r -i "s@#{0,}?net.ipv4.neigh.default.gc_thresh1 ?= ?([0-9]{1,})@net.ipv4.neigh.default.gc_thresh1 = 512@g" /etc/sysctl.conf
+sed -r -i "s@#{0,}?net.ipv4.neigh.default.gc_thresh2 ?= ?([0-9]{1,})@net.ipv4.neigh.default.gc_thresh2 = 2048@g" /etc/sysctl.conf
+sed -r -i "s@#{0,}?net.ipv4.neigh.default.gc_thresh3 ?= ?([0-9]{1,})@net.ipv4.neigh.default.gc_thresh3 = 4096@g" /etc/sysctl.conf
+sed -r -i "s@#{0,}?net.core.somaxconn ?= ?([0-9]{1,})@net.core.somaxconn = 32768@g" /etc/sysctl.conf
 sed -r -i "s@#{0,}?net.ipv4.conf.eth0.arp_accept ?= ?(0|1)@net.ipv4.conf.eth0.arp_accept = 1@g" /etc/sysctl.conf
-sed -r -i  "s@#{0,}?fs.aio-max-nr ?= ?([0-9]{1,})@fs.aio-max-nr = 262144@g" /etc/sysctl.conf
-sed -r -i  "s@#{0,}?net.ipv4.tcp_retries2 ?= ?([0-9]{1,})@net.ipv4.tcp_retries2 = 15@g" /etc/sysctl.conf
-sed -r -i  "s@#{0,}?net.ipv4.tcp_max_tw_buckets ?= ?([0-9]{1,})@net.ipv4.tcp_max_tw_buckets = 1048576@g" /etc/sysctl.conf
-sed -r -i  "s@#{0,}?net.ipv4.tcp_max_orphans ?= ?([0-9]{1,})@net.ipv4.tcp_max_orphans = 65535@g" /etc/sysctl.conf
-sed -r -i  "s@#{0,}?net.ipv4.udp_rmem_min ?= ?([0-9]{1,})@net.ipv4.udp_rmem_min = 131072@g" /etc/sysctl.conf
-sed -r -i  "s@#{0,}?net.ipv4.udp_wmem_min ?= ?([0-9]{1,})@net.ipv4.udp_wmem_min = 131072@g" /etc/sysctl.conf
-sed -r -i  "s@#{0,}?net.ipv4.conf.all.arp_ignore ?= ??(0|1|2)@net.ipv4.conf.all.arp_ignore = 1@g" /etc/sysctl.conf
-sed -r -i  "s@#{0,}?net.ipv4.conf.default.arp_ignore ?= ??(0|1|2)@net.ipv4.conf.default.arp_ignore = 1@g" /etc/sysctl.conf
-sed -r -i  "s@#{0,}?kernel.watchdog_thresh ?= ?([0-9]{1,})@kernel.watchdog_thresh = 5@g" /etc/sysctl.conf
-sed -r -i  "s@#{0,}?kernel.hung_task_timeout_secs ?= ?([0-9]{1,})@kernel.hung_task_timeout_secs = 5@g" /etc/sysctl.conf
+sed -r -i "s@#{0,}?fs.aio-max-nr ?= ?([0-9]{1,})@fs.aio-max-nr = 262144@g" /etc/sysctl.conf
+sed -r -i "s@#{0,}?net.ipv4.tcp_retries2 ?= ?([0-9]{1,})@net.ipv4.tcp_retries2 = 15@g" /etc/sysctl.conf
+sed -r -i "s@#{0,}?net.ipv4.tcp_max_tw_buckets ?= ?([0-9]{1,})@net.ipv4.tcp_max_tw_buckets = 1048576@g" /etc/sysctl.conf
+sed -r -i "s@#{0,}?net.ipv4.tcp_max_orphans ?= ?([0-9]{1,})@net.ipv4.tcp_max_orphans = 65535@g" /etc/sysctl.conf
+sed -r -i "s@#{0,}?net.ipv4.tcp_keepalive_time ?= ?([0-9]{1,})@net.ipv4.tcp_keepalive_time = 600@g" /etc/sysctl.conf
+sed -r -i "s@#{0,}?net.ipv4.tcp_keepalive_intvl ?= ?([0-9]{1,})@net.ipv4.tcp_keepalive_intvl = 30@g" /etc/sysctl.conf
+sed -r -i "s@#{0,}?net.ipv4.tcp_keepalive_probes ?= ?([0-9]{1,})@net.ipv4.tcp_keepalive_probes = 10@g" /etc/sysctl.conf
+sed -r -i "s@#{0,}?net.ipv4.udp_rmem_min ?= ?([0-9]{1,})@net.ipv4.udp_rmem_min = 131072@g" /etc/sysctl.conf
+sed -r -i "s@#{0,}?net.ipv4.udp_wmem_min ?= ?([0-9]{1,})@net.ipv4.udp_wmem_min = 131072@g" /etc/sysctl.conf
+sed -r -i "s@#{0,}?net.ipv4.conf.all.arp_ignore ?= ??(0|1|2)@net.ipv4.conf.all.arp_ignore = 1@g" /etc/sysctl.conf
+sed -r -i "s@#{0,}?net.ipv4.conf.default.arp_ignore ?= ??(0|1|2)@net.ipv4.conf.default.arp_ignore = 1@g" /etc/sysctl.conf
+sed -r -i "s@#{0,}?kernel.watchdog_thresh ?= ?([0-9]{1,})@kernel.watchdog_thresh = 5@g" /etc/sysctl.conf
+sed -r -i "s@#{0,}?kernel.hung_task_timeout_secs ?= ?([0-9]{1,})@kernel.hung_task_timeout_secs = 5@g" /etc/sysctl.conf
 
 tmpfile="$$.tmp"
 awk ' !x[$0]++{print > "'$tmpfile'"}' /etc/sysctl.conf
diff --git a/cmd/kk/pkg/etcd/templates/etcd_service.go b/cmd/kk/pkg/etcd/templates/etcd_service.go
index 78f2b788c..de3159abb 100644
--- a/cmd/kk/pkg/etcd/templates/etcd_service.go
+++ b/cmd/kk/pkg/etcd/templates/etcd_service.go
@@ -17,8 +17,9 @@
 package templates
 
 import (
-	"github.com/lithammer/dedent"
 	"text/template"
+
+	"github.com/lithammer/dedent"
 )
 
 var (
@@ -31,6 +32,8 @@ After=network.target
 [Service]
 User=root
 Type=notify
+Nice=-20
+OOMScoreAdjust=-1000
 EnvironmentFile=/etc/etcd.env
 ExecStart=/usr/local/bin/etcd
 NotifyAccess=all
diff --git a/cmd/kk/pkg/plugins/network/templates/calico.tmpl b/cmd/kk/pkg/plugins/network/templates/calico.tmpl
index a612fccf8..6012037b7 100644
--- a/cmd/kk/pkg/plugins/network/templates/calico.tmpl
+++ b/cmd/kk/pkg/plugins/network/templates/calico.tmpl
@@ -89,9 +89,7 @@ data:
           "nodename": "__KUBERNETES_NODE_NAME__",
           "mtu": __CNI_MTU__,
           "ipam": {
-              "type": "calico-ipam",
-              "assign_ipv4": "true",
-              "assign_ipv6": "true"
+              "type": "calico-ipam"
           },
           "policy": {
               "type": "k8s"