From 2b80d6da3b83a5bb88c1c05dac7bd32720a83923 Mon Sep 17 00:00:00 2001 From: james-otten Date: Sun, 28 Apr 2024 16:18:24 -0400 Subject: [PATCH] updates --- infra/README.md | 38 +++++++++++++++++------- infra/cluster/cluster_example.tfvars | 1 - infra/cluster/longhorn.yaml | 5 ---- infra/cluster/main.tf | 37 +++++++++++------------ infra/cluster/metallb.yaml | 23 -------------- infra/cluster/metallb_extra.yaml | 17 +++++++++++ infra/cluster/vars.tf | 4 --- infra/helm/meshdb/templates/meshweb.yaml | 2 +- infra/mgr_provision.sh | 8 +++-- 9 files changed, 69 insertions(+), 66 deletions(-) delete mode 100644 infra/cluster/cluster_example.tfvars create mode 100644 infra/cluster/metallb_extra.yaml delete mode 100644 infra/cluster/vars.tf diff --git a/infra/README.md b/infra/README.md index 4d190ff0..58ea7ce9 100644 --- a/infra/README.md +++ b/infra/README.md @@ -2,10 +2,15 @@ 1. Configure a user for the [proxmox provider](https://registry.terraform.io/providers/Telmate/proxmox/latest/docs) and setup env vars. 2. Setup tfvars + ssh keys -3. `terraform plan --var-file=your.tfvars` -4. `terraform apply --var-file=your.tfvars` -5. Login via serial and figure out the IPs that were recieved from DHCP -6. One time provisioning for the master node +3. Create the VMs that will host k3s +``` +cd meshdb/infra/tf/ +terraform init +terraform plan --var-file=your.tfvars +terraform apply --var-file=your.tfvars +``` +4. Login via serial and figure out the IPs that were recieved from DHCP +5. One time provisioning for the master node ``` target_host="" @@ -13,16 +18,17 @@ scp infra/mgr_provision.sh ubuntu@$target_host:/home/ubuntu/mgr_provision.sh ssh -t ubuntu@$target_host "sudo bash /home/ubuntu/mgr_provision.sh" ``` -7. Set the IP range for metallb, such as `10.70.90.80/29`, in `/opt/meshdb_mgmt/cluster_local.tfvars` and then deploy metallb and longhorn from the manager +6. Set the IP range for metallb, such as `10.70.90.80/29`, in `/opt/meshdb_mgmt/meshdb/infra/cluster/metallb_extra.yaml` and then deploy metallb and longhorn from the manager ``` cd /opt/meshdb_mgmt/meshdb/infra/cluster/ -cat ../../cluster_local.tfvars terraform init -terraform plan --var-file=../../cluster_local.tfvars -terraform apply --var-file=../../cluster_local.tfvars +terraform plan +terraform apply +# update address block in /opt/meshdb_mgmt/meshdb/infra/cluster/metallb_extra.yaml +kubectl apply -f /opt/meshdb_mgmt/meshdb/infra/cluster/metallb_extra.yaml ``` -8. Setup each node (from the manager) +7. Setup each node (from the manager) ``` cd /opt/meshdb_mgmt/meshdb/infra/ @@ -34,6 +40,16 @@ do done ``` -10. `kubectl create namespace meshdbdev0 && helm template . -f values.yaml -f secret.values.yaml | kubectl apply -f -` +8. Update values + secrets in `/opt/meshdb_mgmt/values.yaml` and `/opt/meshdb_mgmt/secret.values.yaml` + +9. Deploy helm chart. Create the namespace you indicated in `/opt/meshdb_mgmt/values.yaml` + +``` +your_ns="meshdbdev0" +cd /opt/meshdb_mgmt/meshdb/infra/helm/meshdb/ +kubectl create namespace $your_ns +helm template . -f ../../../../values.yaml -f ../../../../secret.values.yaml | kubectl apply -f - +kubectl get all -n $your_ns +``` -11. If you need a superuser: `kubectl exec -it -n meshdbdev0 service/meshdb-meshweb bash` and `python manage.py createsuperuser` +10. If you need a superuser: `kubectl exec -it -n meshdbdev0 service/meshdb-meshweb bash` and `python manage.py createsuperuser` diff --git a/infra/cluster/cluster_example.tfvars b/infra/cluster/cluster_example.tfvars deleted file mode 100644 index 1338f22e..00000000 --- a/infra/cluster/cluster_example.tfvars +++ /dev/null @@ -1 +0,0 @@ -metallb_ip_address_range = "10.70.90.80/29" diff --git a/infra/cluster/longhorn.yaml b/infra/cluster/longhorn.yaml index 61b1c547..da07719d 100644 --- a/infra/cluster/longhorn.yaml +++ b/infra/cluster/longhorn.yaml @@ -1,8 +1,3 @@ -apiVersion: v1 -kind: Namespace -metadata: - name: longhorn-system ---- apiVersion: helm.cattle.io/v1 kind: HelmChart metadata: diff --git a/infra/cluster/main.tf b/infra/cluster/main.tf index 346cc24a..8f387d91 100644 --- a/infra/cluster/main.tf +++ b/infra/cluster/main.tf @@ -2,33 +2,32 @@ provider "kubernetes" { config_path = "/etc/rancher/k3s/k3s.yaml" } -# Read metallb yaml -data "local_file" "yaml_file" { - filename = "./metallb.yaml" -} - -# Parse the Kubernetes config file -data "yamldecode" "metallb_kubernetes_config" { - # swap a single variable (IP range) - input = replace(data.local_file.yaml_file.content, "CHANGE_ME_IP_RANGE", var.metallb_ip_address_range) +# Create metallb-system +resource "kubernetes_namespace" "metallb-system-ns" { + metadata { + name = "metallb-system" + } } # Create metallb with the manifest resource "kubernetes_manifest" "metallb" { - manifest = data.yamldecode.metallb_kubernetes_config -} - -# Read longhorn yaml -data "local_file" "longhorn_yaml_file" { - filename = "./longhorn.yaml" + manifest = yamldecode(file("./metallb.yaml")) + depends_on = [ + kubernetes_namespace.metallb-system-ns + ] } -# Parse the Kubernetes config file -data "yamldecode" "longhorn_kubernetes_config" { - input = data.local_file.longhorn_yaml_file.content +# Create longhorn-system +resource "kubernetes_namespace" "longhorn-system-ns" { + metadata { + name = "longhorn-system" + } } # Create longhorn with the manifest resource "kubernetes_manifest" "longhorn" { - manifest = data.yamldecode.longhorn_kubernetes_config + manifest = yamldecode(file("./longhorn.yaml")) + depends_on = [ + kubernetes_namespace.longhorn-system-ns + ] } diff --git a/infra/cluster/metallb.yaml b/infra/cluster/metallb.yaml index b5503cde..72e8a1e0 100644 --- a/infra/cluster/metallb.yaml +++ b/infra/cluster/metallb.yaml @@ -1,8 +1,3 @@ -apiVersion: v1 -kind: Namespace -metadata: - name: metallb-system ---- apiVersion: helm.cattle.io/v1 kind: HelmChart metadata: @@ -12,21 +7,3 @@ spec: repo: https://metallb.github.io/metallb chart: metallb targetNamespace: metallb-system ---- -apiVersion: metallb.io/v1beta1 -kind: IPAddressPool -metadata: - name: pool-1 - namespace: metallb-system -spec: - addresses: - - "CHANGE_ME_IP_RANGE" ---- -apiVersion: metallb.io/v1beta1 -kind: L2Advertisement -metadata: - name: k3s-l2 - namespace: metallb-system -spec: - ipAddressPools: - - pool-1 diff --git a/infra/cluster/metallb_extra.yaml b/infra/cluster/metallb_extra.yaml new file mode 100644 index 00000000..741b8a17 --- /dev/null +++ b/infra/cluster/metallb_extra.yaml @@ -0,0 +1,17 @@ +apiVersion: metallb.io/v1beta1 +kind: IPAddressPool +metadata: + name: pool-1 + namespace: metallb-system +spec: + addresses: + - "10.70.90.80/29" +--- +apiVersion: metallb.io/v1beta1 +kind: L2Advertisement +metadata: + name: k3s-l2 + namespace: metallb-system +spec: + ipAddressPools: + - pool-1 diff --git a/infra/cluster/vars.tf b/infra/cluster/vars.tf deleted file mode 100644 index 7dbf3812..00000000 --- a/infra/cluster/vars.tf +++ /dev/null @@ -1,4 +0,0 @@ -variable "metallb_ip_address_range" { - type = string - description = "ip range to be used by metallb" -} diff --git a/infra/helm/meshdb/templates/meshweb.yaml b/infra/helm/meshdb/templates/meshweb.yaml index 74e6cd8e..e6e298fb 100644 --- a/infra/helm/meshdb/templates/meshweb.yaml +++ b/infra/helm/meshdb/templates/meshweb.yaml @@ -167,7 +167,7 @@ spec: - name: static-content-vol persistentVolumeClaim: claimName: {{ .Values.meshweb.static_pvc_name }} - {{- with .Values.meshweb.meshweb.nodeSelector }} + {{- with .Values.meshweb.nodeSelector }} nodeSelector: {{- toYaml . | nindent 8 }} {{- end }} diff --git a/infra/mgr_provision.sh b/infra/mgr_provision.sh index 869ae4e9..dd8b6067 100644 --- a/infra/mgr_provision.sh +++ b/infra/mgr_provision.sh @@ -6,9 +6,14 @@ mkdir -p $MGMT_DIR cd $MGMT_DIR # Clone the repo -apt-get update && apt-get install -y git +apt-get update && apt-get install -y git unzip git clone https://github.com/nycmeshnet/meshdb.git +# Install tf +wget https://releases.hashicorp.com/terraform/1.8.2/terraform_1.8.2_linux_amd64.zip +unzip terraform_* +mv terraform /usr/bin/ + # JBO TODO REMOVE DEBUG cd meshdb git checkout james/infra_updates @@ -19,7 +24,6 @@ cd .. cp meshdb/infra/helm/meshdb/secret.values.yaml ./secret.values.yaml cp meshdb/infra/helm/meshdb/values.yaml ./values.yaml cp meshdb/infra/tf/example.tfvars ./local.tfvars -cp meshdb/infra/cluster/cluster_example.tfvars ./cluster_local.tfvars # Setup k3s curl -sfL https://get.k3s.io | sh -s - server --cluster-init --disable servicelb