testing infra updates

infra/README.md

@@ -5,74 +5,35 @@
 3. `terraform plan --var-file=your.tfvars`
 4. `terraform apply --var-file=your.tfvars`
 5. Login via serial and figure out the IPs that were recieved from DHCP
-6. SSH into the master node and setup
-```
-curl -sfL https://get.k3s.io | sh -s - server --cluster-init --disable servicelb
+6. One time provisioning for the master node
 
-echo "cluster-init: true" >> /etc/rancher/k3s/config.yaml
-echo "disable: servicelb" >> /etc/rancher/k3s/config.yaml
 ```
-
-7. Install metallb on master node
-
+target_host="<MGR IP>"
+scp infra/mgr_provision.sh ubuntu@$target_host:/home/ubuntu/mgr_provision.sh
+ssh -t ubuntu@$target_host "sudo bash /home/ubuntu/mgr_provision.sh"
 ```
-IP_RANGE="10.70.90.80/29"
-cat <<EOF > /var/lib/rancher/k3s/server/manifests/metallb.yaml
-apiVersion: v1
-kind: Namespace
-metadata:
-  name: metallb-system
----
-apiVersion: helm.cattle.io/v1
-kind: HelmChart
-metadata:
-  name: metallb
-  namespace: metallb-system
-spec:
-  repo: https://metallb.github.io/metallb
-  chart: metallb
-  targetNamespace: metallb-system
-
----
-apiVersion: metallb.io/v1beta1
-kind: IPAddressPool
-metadata:
-  name: pool-1
-  namespace: metallb-system
-spec:
-  addresses:
-  - $IP_RANGE
-
----
-apiVersion: metallb.io/v1beta1
-kind: L2Advertisement
-metadata:
-  name: k3s-l2
-  namespace: metallb-system
-spec:
-  ipAddressPools:
-  - pool-1
-EOF
 
+7. Set the IP range for metallb, such as `10.70.90.80/29`, in `/opt/meshdb_mgmt/cluster_local.tfvars` and then deploy metallb and longhorn from the manager
+```
+cd /opt/meshdb_mgmt/meshdb/infra/cluster/
+cat ../../cluster_local.tfvars
+terraform init
+terraform plan --var-file=../../cluster_local.tfvars
+terraform apply --var-file=../../cluster_local.tfvars
 ```
 
 8. Setup each node (from the manager)
 
-`bash setup_node.sh <NODE IP>`
-
 ```
-#!/bin/bash
-# setup_node.sh
-MASTER_IP="$(ip addr show eth0 | grep "inet\b" | awk '{print $2}' | cut -d/ -f1)"
-NODE_TOKEN="$(cat /var/lib/rancher/k3s/server/node-token)"
+cd /opt/meshdb_mgmt/meshdb/infra/
+declare -a target_nodes=("10.70.90.XX" "10.70.90.YY" "10.70.90.ZZ")
 
-target_host="$1"
-
-ssh -t ubuntu@$target_host "curl -sfL https://get.k3s.io>k3s; sudo bash k3s --server https://${MASTER_IP}:6443 --token $NODE_TOKEN;sudo apt-get update && sudo apt-get install nfs-common -y"
+for n in "${target_nodes[@]}"
+do
+  bash setup_node.sh $n
+done
 ```
 
-9. Install longhorn `kubectl apply -f https://raw.githubusercontent.com/longhorn/longhorn/v1.6.0/deploy/longhorn.yaml`
-
 10. `kubectl create namespace meshdbdev0 && helm template . -f values.yaml -f secret.values.yaml | kubectl apply -f -`
 
 11. If you need a superuser: `kubectl exec -it -n meshdbdev0 service/meshdb-meshweb bash` and `python manage.py createsuperuser`

infra/cluster/cluster_example.tfvars

@@ -0,0 +1 @@
+metallb_ip_address_range = "10.70.90.80/29"

infra/cluster/longhorn.yaml

@@ -0,0 +1,14 @@
+apiVersion: v1
+kind: Namespace
+metadata:
+  name: longhorn-system
+---
+apiVersion: helm.cattle.io/v1
+kind: HelmChart
+metadata:
+  name: longhorn
+  namespace: longhorn-system
+spec:
+  repo: https://charts.longhorn.io
+  chart: longhorn
+  targetNamespace: longhorn-system

infra/cluster/main.tf

@@ -0,0 +1,34 @@
+provider "kubernetes" {
+  config_path = "/etc/rancher/k3s/k3s.yaml"
+}
+
+# Read metallb yaml
+data "local_file" "yaml_file" {
+  filename = "./metallb.yaml"
+}
+
+# Parse the Kubernetes config file
+data "yamldecode" "metallb_kubernetes_config" {
+  # swap a single variable (IP range)
+  input = replace(data.local_file.yaml_file.content, "CHANGE_ME_IP_RANGE", var.metallb_ip_address_range)
+}
+
+# Create metallb with the manifest
+resource "kubernetes_manifest" "metallb" {
+  manifest = data.yamldecode.metallb_kubernetes_config
+}
+
+# Read longhorn yaml
+data "local_file" "longhorn_yaml_file" {
+  filename = "./longhorn.yaml"
+}
+
+# Parse the Kubernetes config file
+data "yamldecode" "longhorn_kubernetes_config" {
+  input = data.local_file.longhorn_yaml_file.content
+}
+
+# Create longhorn with the manifest
+resource "kubernetes_manifest" "longhorn" {
+  manifest = data.yamldecode.longhorn_kubernetes_config
+}

infra/cluster/metallb.yaml

@@ -0,0 +1,32 @@
+apiVersion: v1
+kind: Namespace
+metadata:
+  name: metallb-system
+---
+apiVersion: helm.cattle.io/v1
+kind: HelmChart
+metadata:
+  name: metallb
+  namespace: metallb-system
+spec:
+  repo: https://metallb.github.io/metallb
+  chart: metallb
+  targetNamespace: metallb-system
+---
+apiVersion: metallb.io/v1beta1
+kind: IPAddressPool
+metadata:
+  name: pool-1
+  namespace: metallb-system
+spec:
+  addresses:
+  - "CHANGE_ME_IP_RANGE"
+---
+apiVersion: metallb.io/v1beta1
+kind: L2Advertisement
+metadata:
+  name: k3s-l2
+  namespace: metallb-system
+spec:
+  ipAddressPools:
+  - pool-1

infra/cluster/vars.tf

@@ -0,0 +1,4 @@
+variable "metallb_ip_address_range" {
+  type        = string
+  description = "ip range to be used by metallb"
+}

infra/mgr_provision.sh

@@ -0,0 +1,28 @@
+#!/bin/bash
+
+# Create meshdb_mgmt directory
+MGMT_DIR="/opt/meshdb_mgmt"
+mkdir -p $MGMT_DIR
+cd $MGMT_DIR
+
+# Clone the repo
+apt-get update && apt-get install -y git
+git clone https://github.com/nycmeshnet/meshdb.git
+
+# JBO TODO REMOVE DEBUG
+cd meshdb
+git checkout james/infra_updates
+cd ..
+# END DEBUG
+
+# Setup secret files (will need to be modified)
+cp meshdb/infra/helm/meshdb/secret.values.yaml ./secret.values.yaml
+cp meshdb/infra/helm/meshdb/values.yaml ./values.yaml
+cp meshdb/infra/tf/example.tfvars ./local.tfvars
+cp meshdb/infra/cluster/cluster_example.tfvars ./cluster_local.tfvars
+
+# Setup k3s
+curl -sfL https://get.k3s.io | sh -s - server --cluster-init --disable servicelb
+
+echo "cluster-init: true" >> /etc/rancher/k3s/config.yaml
+echo "disable: servicelb" >> /etc/rancher/k3s/config.yaml

infra/setup_node.sh

@@ -0,0 +1,8 @@
+#!/bin/bash
+# setup_node.sh
+MASTER_IP="$(ip addr show eth0 | grep "inet\b" | awk '{print $2}' | cut -d/ -f1)"
+NODE_TOKEN="$(cat /var/lib/rancher/k3s/server/node-token)"
+
+target_host="$1"
+
+ssh -t ubuntu@$target_host "curl -sfL https://get.k3s.io>k3s; sudo bash k3s --server https://${MASTER_IP}:6443 --token $NODE_TOKEN;sudo apt-get update && sudo apt-get install nfs-common -y"0