Add MetalLB to provide a loadbalancer for services.

(defaults to on; profile also requests a single public IP.)

manifest-to-publicaddrs.py

@@ -0,0 +1,27 @@
+#!/usr/bin/env python
+
+from __future__ import print_function
+import sys
+import lxml.etree
+
+f = open(sys.argv[1],'r')
+contents = f.read()
+f.close()
+root = lxml.etree.fromstring(contents)
+
+mycluster = None
+if len(sys.argv) > 2:
+    mycluster = sys.argv[2]
+
+# Find all the public IP addresses:
+for elm in root.getchildren():
+    if not elm.tag.endswith("}routable_pool"):
+        continue
+    name = elm.get("client_id")
+    if mycluster and not name.endswith("-%s" % (mycluster,)):
+        continue
+    for elm2 in elm.getchildren():
+        if elm2.tag.endswith("}ipv4"):
+            print("%s/%s" % (elm2.get("address"),elm2.get("netmask")))
+
+sys.exit(0)

profile.py

@@ -118,10 +118,14 @@
     longDescription="The subnet containing service addresses.",
     advanced=True)
 pc.defineParameter(
-    "kubeAccessIp","Kubespray Access IP",
-    portal.ParameterType.STRING,"mgmt",
-    [("mgmt","mgmt"),("external","external")],
-    longDescription="If you choose external here, your NodePorts and LoadBalancers will be able to allocate ports on the public control net interface of your nodes.",
+    "kubeDoMetalLB","Kubespray Enable MetalLB",
+    portal.ParameterType.BOOLEAN,True,
+    longDescription="We enable MetalLB by default, so that users can use an \"external\" load balancer service type.  You need at least one public IP address for this option because it doesn't make sense without one.",
+    advanced=True)
+pc.defineParameter(
+    "publicIPCount", "Number of public IP addresses",
+    portal.ParameterType.INTEGER,1,
+    longDescription="Set the number of public IP addresses you will need for externally-published services (e.g., via a load balancer like MetalLB.",
     advanced=True)
 pc.defineParameter(
     "kubeFeatureGates","Kubernetes Feature Gate List",
@@ -156,6 +160,17 @@
 #
 params = pc.bindParameters()
 
+if params.publicIPCount > 8:
+    perr = portal.ParameterWarning(
+        "You cannot request more than 8 public IP addresses, at least not without creating your own modified version of this profile!",
+        ["publicIPCount"])
+    pc.reportWarning(perr)
+if params.kubeDoMetalLB and params.publicIPCount < 1:
+    perr = portal.ParameterWarning(
+        "If you enable MetalLB, you must request at least one public IP address!",
+        ["kubeDoMetalLB","publicIPCount"])
+    pc.reportWarning(perr)
+
 #
 # Give the library a chance to return nice JSON-formatted exception(s) and/or
 # warnings; this might sys.exit().
@@ -254,4 +269,14 @@ def _write(self, root):
 adminPassResource = EmulabEncrypt()
 rspec.addResource(adminPassResource)
 
+#
+# Grab a few public IP addresses.
+#
+apool = IG.AddressPool("node-0",params.publicIPCount)
+try:
+    apool.Site("1")
+except:
+    pass
+rspec.addResource(apool)
+
 pc.printRequestRSpec(rspec)

setup-kubespray.sh

@@ -19,6 +19,14 @@ fi
 
 logtstart "kubespray"
 
+# First, we need yq.
+are_packages_installed yq
+if [ ! $? -eq 1 ]; then
+    apt-key adv --keyserver keyserver.ubuntu.com --recv-keys CC86BB64
+    add-apt-repository -y ppa:rmescandon/yq
+    maybe_install_packages yq
+fi
+
 cd $OURDIR
 if [ -e kubespray ]; then
     rm -rf kubespray
@@ -272,22 +280,62 @@ kube_users:
 EOF
 #kube_api_anonymous_auth: false
 
+#
+# Add MetalLB support.
+#
+METALLB_PLAYBOOK=
+if [ "$KUBEDOMETALLB" = "1" -a $PUBLICADDRCOUNT -gt 0 ]; then
+    echo "kube_proxy_strict_arp: true" >> $INVDIR/group_vars/k8s-cluster/k8s-cluster.yml
+    METALLB_PLAYBOOK=contrib/metallb/metallb.yml
+    cat contrib/metallb/roles/provision/defaults/main.yml >>$INVDIR/groups_vars/k8s-cluster/addons.yml
+    echo "metallb:" >/tmp/metallb.yml
+    mi=0
+    for pip in $PUBLICADDRS ; do
+	if [ $mi -eq 0 ]; then
+	    cat <<EOF >>/tmp/metallb.yml
+  ip_range:
+    - "$pip-$pip"
+  protocol: "layer2"
+EOF
+	else
+	    if [ $mi -eq 1 ]; then
+		cat <<EOF >>/tmp/metallb.yml
+  additional_address_pools:
+EOF
+	    fi
+	    cat <<EOF >>/tmp/metallb.yml
+    kube_service_pool_$mi:
+      ip_range:
+        - "$pip-$pip"
+      protocol: "layer2"
+      auto_assign: true
+EOF
+	fi
+	mi=`expr $mi + 1`
+    done
+    yq m --inplace --overwrite $INVDIR/group_vars/k8s-cluster/addons.yml /tmp/metallb.yml
+    rm -f /tmp/metallb.yml
+fi
+
 #
 # Run ansible to build our kubernetes cluster.
 #
+cd $OURDIR/kubespray
 ansible-playbook -i $INVDIR/inventory.ini \
-    kubespray/cluster.yml -b -v
+    cluster.yml $METALLB_PLAYBOOK -b -v
 
 if [ ! $? -eq 0 ]; then
+    cd ..
     echo "ERROR: ansible-playbook failed; check logfiles!"
     exit 1
 fi
+cd ..
 
 mkdir -p /root/.kube
-mkdir -p ~$SWAPPER/.kube
+mkdir -p /users/$SWAPPER/.kube
 cp -p $INVDIR/artifacts/admin.conf /root/.kube/config
-cp -p $INVDIR/artifacts/admin.conf ~$SWAPPER/.kube/config
-chown -R $SWAPPER ~$SWAPPER/.kube
+cp -p $INVDIR/artifacts/admin.conf /users/$SWAPPER/.kube/config
+chown -R $SWAPPER /users/$SWAPPER/.kube
 
 kubectl wait pod -n kube-system --for=condition=Ready --all
 

setup-lib.sh

@@ -81,6 +81,7 @@ KUBEENABLEMULTUS=0
 KUBEPROXYMODE="ipvs"
 KUBEPODSSUBNET="192.168.0.0/17"
 KUBESERVICEADDRESSES="192.168.128.0/17"
+KUBEDOMETALLB=1
 KUBEACCESSIP="mgmt"
 KUBEFEATUREGATES=""
 KUBELETCUSTOMFLAGS=""
@@ -264,6 +265,22 @@ if [ ! -e $OURDIR/parameters ]; then
 fi
 . $OURDIR/parameters
 
+#
+# Grab our public addrs.
+#
+if [ ! -f $OURDIR/publicaddrs ]; then
+    $PYTHON $DIRNAME/manifest-to-publicaddrs.py $OURDIR/manifests.0.xml $CLUSTER > $OURDIR/publicaddrs.raw
+    PUBLICADDRS=`cat $OURDIR/publicaddrs.raw | sed -e 's|^\([^/]*\)/.*$|\1|' | xargs`
+    PUBLICADDRCOUNT=`cat $OURDIR/publicaddrs.raw | wc -l`
+    PUBLICADDRNETMASK=`cat $OURDIR/publicaddrs.raw | sed -e 's|^[^/]*/\(.*\)$|\1|' | head -1`
+    cat <<EOF > $OURDIR/publicaddrs
+PUBLICADDRS="$PUBLICADDRS"
+PUBLICADDRCOUNT=$PUBLICADDRCOUNT
+PUBLICADDRNETMASK="$PUBLICADDRNETMASK"
+EOF
+fi
+. $OURDIR/publicaddrs
+
 #
 # Ok, to be absolutely safe, if the ADMIN_PASS_HASH we got from params was "",
 # and if admin pass wasn't sent as an encrypted string to us, we have we have