From c3cf9694b36a4221471d411376a5ab746ca6864e Mon Sep 17 00:00:00 2001 From: james-otten Date: Sun, 28 Apr 2024 14:49:31 -0400 Subject: [PATCH] testing infra updates --- infra/README.md | 73 +++++++--------------------- infra/cluster/cluster_example.tfvars | 1 + infra/cluster/longhorn.yaml | 14 ++++++ infra/cluster/main.tf | 34 +++++++++++++ infra/cluster/metallb.yaml | 32 ++++++++++++ infra/cluster/vars.tf | 4 ++ infra/mgr_provision.sh | 28 +++++++++++ infra/setup_node.sh | 8 +++ 8 files changed, 138 insertions(+), 56 deletions(-) create mode 100644 infra/cluster/cluster_example.tfvars create mode 100644 infra/cluster/longhorn.yaml create mode 100644 infra/cluster/main.tf create mode 100644 infra/cluster/metallb.yaml create mode 100644 infra/cluster/vars.tf create mode 100644 infra/mgr_provision.sh create mode 100644 infra/setup_node.sh diff --git a/infra/README.md b/infra/README.md index 8e44a973..4d190ff0 100644 --- a/infra/README.md +++ b/infra/README.md @@ -5,74 +5,35 @@ 3. `terraform plan --var-file=your.tfvars` 4. `terraform apply --var-file=your.tfvars` 5. Login via serial and figure out the IPs that were recieved from DHCP -6. SSH into the master node and setup -``` -curl -sfL https://get.k3s.io | sh -s - server --cluster-init --disable servicelb +6. One time provisioning for the master node -echo "cluster-init: true" >> /etc/rancher/k3s/config.yaml -echo "disable: servicelb" >> /etc/rancher/k3s/config.yaml ``` - -7. Install metallb on master node - +target_host="" +scp infra/mgr_provision.sh ubuntu@$target_host:/home/ubuntu/mgr_provision.sh +ssh -t ubuntu@$target_host "sudo bash /home/ubuntu/mgr_provision.sh" ``` -IP_RANGE="10.70.90.80/29" -cat < /var/lib/rancher/k3s/server/manifests/metallb.yaml -apiVersion: v1 -kind: Namespace -metadata: - name: metallb-system ---- -apiVersion: helm.cattle.io/v1 -kind: HelmChart -metadata: - name: metallb - namespace: metallb-system -spec: - repo: https://metallb.github.io/metallb - chart: metallb - targetNamespace: metallb-system - ---- -apiVersion: metallb.io/v1beta1 -kind: IPAddressPool -metadata: - name: pool-1 - namespace: metallb-system -spec: - addresses: - - $IP_RANGE - ---- -apiVersion: metallb.io/v1beta1 -kind: L2Advertisement -metadata: - name: k3s-l2 - namespace: metallb-system -spec: - ipAddressPools: - - pool-1 -EOF +7. Set the IP range for metallb, such as `10.70.90.80/29`, in `/opt/meshdb_mgmt/cluster_local.tfvars` and then deploy metallb and longhorn from the manager +``` +cd /opt/meshdb_mgmt/meshdb/infra/cluster/ +cat ../../cluster_local.tfvars +terraform init +terraform plan --var-file=../../cluster_local.tfvars +terraform apply --var-file=../../cluster_local.tfvars ``` 8. Setup each node (from the manager) -`bash setup_node.sh ` - ``` -#!/bin/bash -# setup_node.sh -MASTER_IP="$(ip addr show eth0 | grep "inet\b" | awk '{print $2}' | cut -d/ -f1)" -NODE_TOKEN="$(cat /var/lib/rancher/k3s/server/node-token)" +cd /opt/meshdb_mgmt/meshdb/infra/ +declare -a target_nodes=("10.70.90.XX" "10.70.90.YY" "10.70.90.ZZ") -target_host="$1" - -ssh -t ubuntu@$target_host "curl -sfL https://get.k3s.io>k3s; sudo bash k3s --server https://${MASTER_IP}:6443 --token $NODE_TOKEN;sudo apt-get update && sudo apt-get install nfs-common -y" +for n in "${target_nodes[@]}" +do + bash setup_node.sh $n +done ``` -9. Install longhorn `kubectl apply -f https://raw.githubusercontent.com/longhorn/longhorn/v1.6.0/deploy/longhorn.yaml` - 10. `kubectl create namespace meshdbdev0 && helm template . -f values.yaml -f secret.values.yaml | kubectl apply -f -` 11. If you need a superuser: `kubectl exec -it -n meshdbdev0 service/meshdb-meshweb bash` and `python manage.py createsuperuser` diff --git a/infra/cluster/cluster_example.tfvars b/infra/cluster/cluster_example.tfvars new file mode 100644 index 00000000..1338f22e --- /dev/null +++ b/infra/cluster/cluster_example.tfvars @@ -0,0 +1 @@ +metallb_ip_address_range = "10.70.90.80/29" diff --git a/infra/cluster/longhorn.yaml b/infra/cluster/longhorn.yaml new file mode 100644 index 00000000..61b1c547 --- /dev/null +++ b/infra/cluster/longhorn.yaml @@ -0,0 +1,14 @@ +apiVersion: v1 +kind: Namespace +metadata: + name: longhorn-system +--- +apiVersion: helm.cattle.io/v1 +kind: HelmChart +metadata: + name: longhorn + namespace: longhorn-system +spec: + repo: https://charts.longhorn.io + chart: longhorn + targetNamespace: longhorn-system \ No newline at end of file diff --git a/infra/cluster/main.tf b/infra/cluster/main.tf new file mode 100644 index 00000000..346cc24a --- /dev/null +++ b/infra/cluster/main.tf @@ -0,0 +1,34 @@ +provider "kubernetes" { + config_path = "/etc/rancher/k3s/k3s.yaml" +} + +# Read metallb yaml +data "local_file" "yaml_file" { + filename = "./metallb.yaml" +} + +# Parse the Kubernetes config file +data "yamldecode" "metallb_kubernetes_config" { + # swap a single variable (IP range) + input = replace(data.local_file.yaml_file.content, "CHANGE_ME_IP_RANGE", var.metallb_ip_address_range) +} + +# Create metallb with the manifest +resource "kubernetes_manifest" "metallb" { + manifest = data.yamldecode.metallb_kubernetes_config +} + +# Read longhorn yaml +data "local_file" "longhorn_yaml_file" { + filename = "./longhorn.yaml" +} + +# Parse the Kubernetes config file +data "yamldecode" "longhorn_kubernetes_config" { + input = data.local_file.longhorn_yaml_file.content +} + +# Create longhorn with the manifest +resource "kubernetes_manifest" "longhorn" { + manifest = data.yamldecode.longhorn_kubernetes_config +} diff --git a/infra/cluster/metallb.yaml b/infra/cluster/metallb.yaml new file mode 100644 index 00000000..b5503cde --- /dev/null +++ b/infra/cluster/metallb.yaml @@ -0,0 +1,32 @@ +apiVersion: v1 +kind: Namespace +metadata: + name: metallb-system +--- +apiVersion: helm.cattle.io/v1 +kind: HelmChart +metadata: + name: metallb + namespace: metallb-system +spec: + repo: https://metallb.github.io/metallb + chart: metallb + targetNamespace: metallb-system +--- +apiVersion: metallb.io/v1beta1 +kind: IPAddressPool +metadata: + name: pool-1 + namespace: metallb-system +spec: + addresses: + - "CHANGE_ME_IP_RANGE" +--- +apiVersion: metallb.io/v1beta1 +kind: L2Advertisement +metadata: + name: k3s-l2 + namespace: metallb-system +spec: + ipAddressPools: + - pool-1 diff --git a/infra/cluster/vars.tf b/infra/cluster/vars.tf new file mode 100644 index 00000000..7dbf3812 --- /dev/null +++ b/infra/cluster/vars.tf @@ -0,0 +1,4 @@ +variable "metallb_ip_address_range" { + type = string + description = "ip range to be used by metallb" +} diff --git a/infra/mgr_provision.sh b/infra/mgr_provision.sh new file mode 100644 index 00000000..869ae4e9 --- /dev/null +++ b/infra/mgr_provision.sh @@ -0,0 +1,28 @@ +#!/bin/bash + +# Create meshdb_mgmt directory +MGMT_DIR="/opt/meshdb_mgmt" +mkdir -p $MGMT_DIR +cd $MGMT_DIR + +# Clone the repo +apt-get update && apt-get install -y git +git clone https://github.com/nycmeshnet/meshdb.git + +# JBO TODO REMOVE DEBUG +cd meshdb +git checkout james/infra_updates +cd .. +# END DEBUG + +# Setup secret files (will need to be modified) +cp meshdb/infra/helm/meshdb/secret.values.yaml ./secret.values.yaml +cp meshdb/infra/helm/meshdb/values.yaml ./values.yaml +cp meshdb/infra/tf/example.tfvars ./local.tfvars +cp meshdb/infra/cluster/cluster_example.tfvars ./cluster_local.tfvars + +# Setup k3s +curl -sfL https://get.k3s.io | sh -s - server --cluster-init --disable servicelb + +echo "cluster-init: true" >> /etc/rancher/k3s/config.yaml +echo "disable: servicelb" >> /etc/rancher/k3s/config.yaml diff --git a/infra/setup_node.sh b/infra/setup_node.sh new file mode 100644 index 00000000..25970615 --- /dev/null +++ b/infra/setup_node.sh @@ -0,0 +1,8 @@ +#!/bin/bash +# setup_node.sh +MASTER_IP="$(ip addr show eth0 | grep "inet\b" | awk '{print $2}' | cut -d/ -f1)" +NODE_TOKEN="$(cat /var/lib/rancher/k3s/server/node-token)" + +target_host="$1" + +ssh -t ubuntu@$target_host "curl -sfL https://get.k3s.io>k3s; sudo bash k3s --server https://${MASTER_IP}:6443 --token $NODE_TOKEN;sudo apt-get update && sudo apt-get install nfs-common -y"0