Add url and token params for kubernetes templates

hack/bats/extras/k8s.bats

@@ -26,8 +26,8 @@ get_num_nodes() {
 
 local_setup() {
     local nodes=$(get_num_nodes)
-    local join_command=""
-    for ((i=0; i<nodes; i++)); do
+    local params=""
+    for ((i=0; i<1; i++)); do
         limactl delete --force "${NAME}-$i" || :
         local limactl_start_flags="--tty=false --name "${NAME}-$i""
         # Multi-node setup requires user-v2 network for VM-to-VM communication
@@ -40,15 +40,26 @@ local_setup() {
     # Multi-node setup
     if [[ $nodes -gt 1 ]]; then
         for ((i=0; i<nodes; i++)); do
-            if [[ $i -eq 0 ]]; then
+            if [[ $i -eq 0 && "${TEMPLATE}" == "k8s" ]]; then
                 # Get the join command from the first node
                 join_command=$(limactl shell "${NAME}-0" sudo kubeadm token create --print-join-command)
+                # kubeadm join ADDRESS --token TOKEN --discovery-token-ca-cert-hash DISCOVERY_TOKEN_CA_CERT_HASH
+                read -ra words <<< "$join_command"
+                assert_equal "${words[1]} ${words[3]} ${words[5]}" "join --token --discovery-token-ca-cert-hash"
+                params=".param.url=\"https://${words[2]}\"|.param.token=\"${words[4]}\"|.param.discoveryTokenCaCertHash=\"${words[6]}\""
+            elif [[ $i -eq 0 && "${TEMPLATE}" == "k3s" ]]; then
+                url=$(printf "https://lima-%s.internal:6443\n" "${NAME}-0")
+                token=$(limactl shell "${NAME}-0" sudo cat /var/lib/rancher/k3s/server/node-token)
+                params=".param.url=\"${url}\"|.param.token=\"${token}\""
             else
                 # Execute the join command on worker nodes
-                limactl shell "${NAME}-$i" sudo bash -euxc "kubeadm reset --force ; ip link delete cni0 ; ip link delete flannel.1 ; rm -rf /var/lib/cni /etc/cni"
-                limactl shell "${NAME}-$i" sudo ${join_command}
+                limactl delete --force "${NAME}-$i" || :
+                local limactl_start_flags="--tty=false --name "${NAME}-$i""
+                limactl_start_flags+=" --network lima:user-v2 --set $params"
+                limactl start ${limactl_start_flags} "template:${TEMPLATE}" 3>&- 4>&- &
             fi
         done
+        wait $(jobs -p)
     fi
     for node in $(k get node -o name); do
 	    k wait --timeout=5m --for=condition=ready "${node}"

pkg/store/instance.go

@@ -124,6 +124,7 @@ func Inspect(ctx context.Context, instName string) (*limatype.Instance, error) {
 			inst.Errors = append(inst.Errors, fmt.Errorf("cannot add global fields to instance data: %w", err))
 			inst.Status = limatype.StatusBroken
 		} else {
+			data.Param = y.Param
 			var message strings.Builder
 			err = tmpl.Execute(&message, data)
 			if err != nil {

templates/k3s.yaml

@@ -10,6 +10,18 @@
 # NAME       STATUS   ROLES                  AGE   VERSION
 # lima-k3s   Ready    control-plane,master   69s   v1.21.1+k3s1
 
+# A multi-node cluster can be created by starting multiple instances of this template
+# connected via the `lima:user-v2` network.
+#
+# $ limactl start --name k3s-0 --network lima:user-v2 template:k3s
+# $ printf "https://lima-%s.internal:6443\n" k3s-0
+# (The url for the start command printed here)
+# $ limactl shell k3s-0 sudo cat /var/lib/rancher/k3s/server/node-token
+# (The token for the start command printed here)
+#
+# $ limactl start --name k3s-1 --network lima:user-v2 template:k3s \
+#                 --set '.param.url="<URL_FROM_ABOVE>" | .param.token="<TOKEN_FROM_ABOVE>"'
+
 minimumLimaVersion: 2.0.0
 
 base: template:_images/ubuntu-lts
@@ -25,16 +37,25 @@ provision:
   script: |
     #!/bin/sh
     if [ ! -d /var/lib/rancher/k3s ]; then
+    {{if not ( and .Param.url .Param.token )}}
             curl -sfL https://get.k3s.io | INSTALL_K3S_EXEC="server --write-kubeconfig-mode 644" sh -
+    {{else}}
+            curl -sfL https://get.k3s.io | K3S_URL={{.Param.url}} K3S_TOKEN={{.Param.token}} sh -
+    {{end}}
     fi
 probes:
 - script: |
     #!/bin/bash
     set -eux -o pipefail
+    {{if not ( and .Param.url .Param.token )}}
     if ! timeout 30s bash -c "until test -f /etc/rancher/k3s/k3s.yaml; do sleep 3; done"; then
             echo >&2 "k3s is not running yet"
             exit 1
     fi
+    {{else}}
+    # create an empty file so that the "copyToHost" does not fail
+    sudo mkdir -p /etc/rancher/k3s && sudo touch /etc/rancher/k3s/k3s.yaml
+    {{end}}
   hint: |
     The k3s kubeconfig file has not yet been created.
     Run "limactl shell k3s sudo journalctl -u k3s" to check the log.
@@ -44,8 +65,13 @@ copyToHost:
   host: "{{.Dir}}/copied-from-guest/kubeconfig.yaml"
   deleteOnStop: true
 message: |
+  {{- if not ( and .Param.url .Param.token ) -}}
   To run `kubectl` on the host (assumes kubectl is installed), run the following commands:
   ------
   export KUBECONFIG="{{.Dir}}/copied-from-guest/kubeconfig.yaml"
   kubectl ...
   ------
+  {{end}}
+param:
+  url: ""
+  token: ""

templates/k8s.yaml

@@ -15,11 +15,11 @@
 #
 # $ limactl start --name k8s-0 --network lima:user-v2 template:k8s
 # $ limactl shell k8s-0 sudo kubeadm token create --print-join-command
-# (The join command printed here)
+# (The parameters for the start command printed here)
 #
-# $ limactl start --name k8s-1 --network lima:user-v2 template:k8s
-# $ limactl shell k8s-1 sudo bash -euxc "kubeadm reset --force ; ip link delete cni0 ; ip link delete flannel.1 ; rm -rf /var/lib/cni /etc/cni"
-# $ limactl shell k8s-1 sudo <JOIN_COMMAND_FROM_ABOVE>
+# $ limactl start --name k8s-1 --network lima:user-v2 template:k8s \
+#                 --set '.param.url="https://<ADDRESS_FROM_ABOVE>" | .param.token="<TOKEN_FROM_ABOVE>" | \
+#                        .param.discoveryTokenCaCertHash="<DISCOVERY_TOKEN_CA_CERT_HASH_FROM_ABOVE>"'
 
 minimumLimaVersion: 2.0.0
 
@@ -54,9 +54,7 @@ provision:
     export DEBIAN_FRONTEND=noninteractive
     apt-get update
     apt-get install -y apt-transport-https ca-certificates curl
-    # 1.35 fails: https://github.com/lima-vm/lima/issues/4473
-    VERSION=1.34
-    # VERSION=$(curl -L -s https://dl.k8s.io/release/stable.txt | sed -e 's/v//' | cut -d'.' -f1-2)
+    VERSION=$(curl -L -s https://dl.k8s.io/release/stable.txt | sed -e 's/v//' | cut -d'.' -f1-2)
     echo "deb [signed-by=/etc/apt/keyrings/kubernetes-apt-keyring.gpg] https://pkgs.k8s.io/core:/stable:/v${VERSION}/deb/ /" | sudo tee /etc/apt/sources.list.d/kubernetes.list
     curl -fsSL https://pkgs.k8s.io/core:/stable:/v${VERSION}/deb/Release.key | sudo gpg --dearmor -o /etc/apt/keyrings/kubernetes-apt-keyring.gpg
     apt-get update
@@ -93,6 +91,7 @@ provision:
     set -eux -o pipefail
     test -e /etc/kubernetes/admin.conf && exit 0
     export KUBECONFIG=/etc/kubernetes/admin.conf
+    {{if not ( and .Param.url .Param.token )}}
     systemctl stop kubelet
     kubeadm config images list
     kubeadm config images pull --cri-socket=unix:///run/containerd/containerd.sock
@@ -117,6 +116,18 @@ provision:
     cgroupDriver: systemd
     EOF
     kubeadm init --config kubeadm-config.yaml
+    {{else}}
+    cat <<EOF >kubeadm-config.yaml
+    kind: JoinConfiguration
+    apiVersion: kubeadm.k8s.io/v1beta4
+    nodeRegistration:
+      criSocket: unix:///run/containerd/containerd.sock
+    EOF
+    kubeadm join --config kubeadm-config.yaml {{.Param.url}} --token {{.Param.token}} \
+        --discovery-token-ca-cert-hash {{.Param.discoveryTokenCaCertHash}}
+    {{end}}
+
+    {{if not ( and .Param.url .Param.token )}}
     # Installing a Pod network add-on
     kubectl apply -f https://github.com/flannel-io/flannel/releases/download/v0.27.4/kube-flannel.yml
     # Control plane node isolation
@@ -127,6 +138,7 @@ provision:
     # The original kubeconfig is kept unmodified, so that `kubeadm token create --print-join-command`
     # can still print the reachable address.
     sed -e "/server:/ s|https://.*:\([0-9]*\)$|https://127.0.0.1:\1|" $KUBECONFIG >/root/.kube/config.localhost
+    {{end}}
 - mode: system
   script: |
     #!/bin/bash
@@ -150,40 +162,57 @@ probes:
   script: |
     #!/bin/bash
     set -eux -o pipefail
+    {{if not ( and .Param.url .Param.token )}}
     if ! timeout 30s bash -c "images=\"$(kubeadm config images list)\"; until for image in \$images; do sudo ctr -n k8s.io image inspect \$image >/dev/null; done; do sleep 3; done"; then
       echo >&2 "k8s images are not pulled yet"
       exit 1
     fi
+    {{end}}
 - description: "kubeadm to be completed"
   script: |
     #!/bin/bash
     set -eux -o pipefail
+    {{if not ( and .Param.url .Param.token )}}
     if ! timeout 300s bash -c "until test -f /etc/kubernetes/admin.conf; do sleep 3; done"; then
       echo >&2 "k8s is not running yet"
       exit 1
     fi
+    {{else}}
+    # create an empty file so that the "copyToHost" does not fail
+    sudo mkdir -p /root/.kube && sudo touch /root/.kube/config.localhost
+    {{end}}
   hint: |
     The k8s kubeconfig file has not yet been created.
 - description: "kubernetes cluster to be running"
   script: |
     #!/bin/bash
     set -eux -o pipefail
+    {{if not ( and .Param.url .Param.token )}}
     if ! timeout 300s bash -c "until kubectl version >/dev/null 2>&1; do sleep 3; done"; then
       echo >&2 "kubernetes cluster is not up and running yet"
       exit 1
     fi
+    {{end}}
 - description: "coredns deployment to be running"
   script: |
     #!/bin/bash
     set -eux -o pipefail
+    {{if not ( and .Param.url .Param.token )}}
     kubectl wait -n kube-system --timeout=180s --for=condition=available deploy coredns
+    {{end}}
 copyToHost:
 - guest: "/root/.kube/config.localhost"
   host: "{{.Dir}}/copied-from-guest/kubeconfig.yaml"
   deleteOnStop: true
 message: |
+  {{- if not ( and .Param.url .Param.token )}}
   To run `kubectl` on the host (assumes kubectl is installed), run the following commands:
   ------
   export KUBECONFIG="{{.Dir}}/copied-from-guest/kubeconfig.yaml"
   kubectl ...
   ------
+  {{end -}}
+param:
+  url: ""
+  token: ""
+  discoveryTokenCaCertHash: ""

website/content/en/docs/examples/containers/kubernetes/_index.md

@@ -66,8 +66,12 @@ See also <https://github.com/rootless-containers/usernetes>.
 
 A multi-node cluster can be created by creating multiple VMs connected via the [`lima:user-v2`](../../../config/network/user-v2.md) network.
 
-As of Lima v2.0, the built-in `k8s` template is designed to support multi-node mode.
+The following templates are designed to support multi-node mode:
+- `k8s` (since Lima v2.0)
+- `k3s` (since Lima v2.1)
 
+{{< tabpane text=true >}}
+{{% tab header="Lima v2.0" %}}
 ```bash
 limactl start --name k8s-0 --network lima:user-v2 template:k8s
 limactl shell k8s-0 sudo kubeadm token create --print-join-command
@@ -78,4 +82,33 @@ limactl shell k8s-0 sudo kubeadm token create --print-join-command
 limactl start --name k8s-1 --network lima:user-v2 template:k8s
 limactl shell k8s-1 sudo bash -euxc "kubeadm reset --force ; ip link delete cni0 ; ip link delete flannel.1 ; rm -rf /var/lib/cni /etc/cni"
 limactl shell k8s-1 sudo <JOIN_COMMAND_FROM_ABOVE>
-```
+```
+{{% /tab %}}
+{{% tab header="Lima v2.1 (k8s)" %}}
+```bash
+limactl start --name k8s-0 --network lima:user-v2 template:k8s
+limactl shell k8s-0 sudo kubeadm token create --print-join-command
+# (The parameters for the start command printed here)
+```
+
+```bash
+limactl start --name k8s-1 --network lima:user-v2 template:k8s \
+              --set '.param.url="https://<ADDRESS_FROM_ABOVE>" | .param.token="<TOKEN_FROM_ABOVE>" | \
+                     .param.discoveryTokenCaCertHash="<DISCOVERY_TOKEN_CA_CERT_HASH_FROM_ABOVE>"'
+```
+{{% /tab %}}
+{{% tab header="Lima v2.1 (k3s)" %}}
+```bash
+limactl start --name k3s-0 --network lima:user-v2 template:k3s
+printf "https://lima-%s.internal:6443\n" k3s-0
+# (The url for the start command printed here)
+limactl shell k3s-0 sudo cat /var/lib/rancher/k3s/server/node-token
+# (The token for the start command printed here)
+```
+
+```bash
+limactl start --name k3s-1 --network lima:user-v2 template:k3s \
+              --set '.param.url="<URL_FROM_ABOVE>" | .param.token="<TOKEN_FROM_ABOVE>"'
+```
+{{% /tab %}}
+{{< /tabpane >}}