updates

nycmeshnet · Apr 28, 2024 · 2b80d6d · 2b80d6d
1 parent c3cf969
commit 2b80d6d
Show file tree

Hide file tree

Showing 9 changed files with 69 additions and 66 deletions.
diff --git a/infra/README.md b/infra/README.md
@@ -2,27 +2,33 @@
 
 1. Configure a user for the [proxmox provider](https://registry.terraform.io/providers/Telmate/proxmox/latest/docs) and setup env vars.
 2. Setup tfvars + ssh keys
-3. `terraform plan --var-file=your.tfvars`
-4. `terraform apply --var-file=your.tfvars`
-5. Login via serial and figure out the IPs that were recieved from DHCP
-6. One time provisioning for the master node
+3. Create the VMs that will host k3s
+```
+cd meshdb/infra/tf/
+terraform init
+terraform plan --var-file=your.tfvars
+terraform apply --var-file=your.tfvars
+```
+4. Login via serial and figure out the IPs that were recieved from DHCP
+5. One time provisioning for the master node
 
 ```
 target_host="<MGR IP>"
 scp infra/mgr_provision.sh ubuntu@$target_host:/home/ubuntu/mgr_provision.sh
 ssh -t ubuntu@$target_host "sudo bash /home/ubuntu/mgr_provision.sh"
 ```
 
-7. Set the IP range for metallb, such as `10.70.90.80/29`, in `/opt/meshdb_mgmt/cluster_local.tfvars` and then deploy metallb and longhorn from the manager
+6. Set the IP range for metallb, such as `10.70.90.80/29`, in `/opt/meshdb_mgmt/meshdb/infra/cluster/metallb_extra.yaml` and then deploy metallb and longhorn from the manager
 ```
 cd /opt/meshdb_mgmt/meshdb/infra/cluster/
-cat ../../cluster_local.tfvars
 terraform init
-terraform plan --var-file=../../cluster_local.tfvars
-terraform apply --var-file=../../cluster_local.tfvars
+terraform plan
+terraform apply
+# update address block in /opt/meshdb_mgmt/meshdb/infra/cluster/metallb_extra.yaml
+kubectl apply -f /opt/meshdb_mgmt/meshdb/infra/cluster/metallb_extra.yaml
 ```
 
-8. Setup each node (from the manager)
+7. Setup each node (from the manager)
 
 ```
 cd /opt/meshdb_mgmt/meshdb/infra/
@@ -34,6 +40,16 @@ do
 done
 ```
 
-10. `kubectl create namespace meshdbdev0 && helm template . -f values.yaml -f secret.values.yaml | kubectl apply -f -`
+8. Update values + secrets in `/opt/meshdb_mgmt/values.yaml` and `/opt/meshdb_mgmt/secret.values.yaml`
+
+9. Deploy helm chart. Create the namespace you indicated in `/opt/meshdb_mgmt/values.yaml`
+
+```
+your_ns="meshdbdev0"
+cd /opt/meshdb_mgmt/meshdb/infra/helm/meshdb/
+kubectl create namespace $your_ns
+helm template . -f ../../../../values.yaml -f ../../../../secret.values.yaml | kubectl apply -f -
+kubectl get all -n $your_ns
+```
 
-11. If you need a superuser: `kubectl exec -it -n meshdbdev0 service/meshdb-meshweb bash` and `python manage.py createsuperuser`
+10. If you need a superuser: `kubectl exec -it -n meshdbdev0 service/meshdb-meshweb bash` and `python manage.py createsuperuser`
diff --git a/infra/cluster/cluster_example.tfvars b/infra/cluster/cluster_example.tfvars
diff --git a/infra/cluster/longhorn.yaml b/infra/cluster/longhorn.yaml
@@ -1,8 +1,3 @@
-apiVersion: v1
-kind: Namespace
-metadata:
-  name: longhorn-system
----
 apiVersion: helm.cattle.io/v1
 kind: HelmChart
 metadata:

diff --git a/infra/cluster/main.tf b/infra/cluster/main.tf
@@ -2,33 +2,32 @@ provider "kubernetes" {
   config_path = "/etc/rancher/k3s/k3s.yaml"
 }
 
-# Read metallb yaml
-data "local_file" "yaml_file" {
-  filename = "./metallb.yaml"
-}
-
-# Parse the Kubernetes config file
-data "yamldecode" "metallb_kubernetes_config" {
-  # swap a single variable (IP range)
-  input = replace(data.local_file.yaml_file.content, "CHANGE_ME_IP_RANGE", var.metallb_ip_address_range)
+# Create metallb-system
+resource "kubernetes_namespace" "metallb-system-ns" {
+  metadata {
+    name = "metallb-system"
+  }
 }
 
 # Create metallb with the manifest
 resource "kubernetes_manifest" "metallb" {
-  manifest = data.yamldecode.metallb_kubernetes_config
-}
-
-# Read longhorn yaml
-data "local_file" "longhorn_yaml_file" {
-  filename = "./longhorn.yaml"
+  manifest = yamldecode(file("./metallb.yaml"))
+  depends_on = [
+    kubernetes_namespace.metallb-system-ns
+  ]
 }
 
-# Parse the Kubernetes config file
-data "yamldecode" "longhorn_kubernetes_config" {
-  input = data.local_file.longhorn_yaml_file.content
+# Create longhorn-system
+resource "kubernetes_namespace" "longhorn-system-ns" {
+  metadata {
+    name = "longhorn-system"
+  }
 }
 
 # Create longhorn with the manifest
 resource "kubernetes_manifest" "longhorn" {
-  manifest = data.yamldecode.longhorn_kubernetes_config
+  manifest = yamldecode(file("./longhorn.yaml"))
+  depends_on = [
+    kubernetes_namespace.longhorn-system-ns
+  ]
 }
diff --git a/infra/cluster/metallb.yaml b/infra/cluster/metallb.yaml
@@ -1,8 +1,3 @@
-apiVersion: v1
-kind: Namespace
-metadata:
-  name: metallb-system
----
 apiVersion: helm.cattle.io/v1
 kind: HelmChart
 metadata:
@@ -12,21 +7,3 @@ spec:
   repo: https://metallb.github.io/metallb
   chart: metallb
   targetNamespace: metallb-system
----
-apiVersion: metallb.io/v1beta1
-kind: IPAddressPool
-metadata:
-  name: pool-1
-  namespace: metallb-system
-spec:
-  addresses:
-  - "CHANGE_ME_IP_RANGE"
----
-apiVersion: metallb.io/v1beta1
-kind: L2Advertisement
-metadata:
-  name: k3s-l2
-  namespace: metallb-system
-spec:
-  ipAddressPools:
-  - pool-1
diff --git a/infra/cluster/metallb_extra.yaml b/infra/cluster/metallb_extra.yaml
@@ -0,0 +1,17 @@
+apiVersion: metallb.io/v1beta1
+kind: IPAddressPool
+metadata:
+  name: pool-1
+  namespace: metallb-system
+spec:
+  addresses:
+  - "10.70.90.80/29"
+---
+apiVersion: metallb.io/v1beta1
+kind: L2Advertisement
+metadata:
+  name: k3s-l2
+  namespace: metallb-system
+spec:
+  ipAddressPools:
+  - pool-1
diff --git a/infra/cluster/vars.tf b/infra/cluster/vars.tf
diff --git a/infra/helm/meshdb/templates/meshweb.yaml b/infra/helm/meshdb/templates/meshweb.yaml
@@ -167,7 +167,7 @@ spec:
         - name: static-content-vol
           persistentVolumeClaim:
             claimName: {{ .Values.meshweb.static_pvc_name }}
-      {{- with .Values.meshweb.meshweb.nodeSelector }}
+      {{- with .Values.meshweb.nodeSelector }}
       nodeSelector:
         {{- toYaml . | nindent 8 }}
       {{- end }}

diff --git a/infra/mgr_provision.sh b/infra/mgr_provision.sh
@@ -6,9 +6,14 @@ mkdir -p $MGMT_DIR
 cd $MGMT_DIR
 
 # Clone the repo
-apt-get update && apt-get install -y git
+apt-get update && apt-get install -y git unzip
 git clone https://github.com/nycmeshnet/meshdb.git
 
+# Install tf
+wget https://releases.hashicorp.com/terraform/1.8.2/terraform_1.8.2_linux_amd64.zip
+unzip terraform_*
+mv terraform /usr/bin/
+
 # JBO TODO REMOVE DEBUG
 cd meshdb
 git checkout james/infra_updates
@@ -19,7 +24,6 @@ cd ..
 cp meshdb/infra/helm/meshdb/secret.values.yaml ./secret.values.yaml
 cp meshdb/infra/helm/meshdb/values.yaml ./values.yaml
 cp meshdb/infra/tf/example.tfvars ./local.tfvars
-cp meshdb/infra/cluster/cluster_example.tfvars ./cluster_local.tfvars
 
 # Setup k3s
 curl -sfL https://get.k3s.io | sh -s - server --cluster-init --disable servicelb