thesis deployment

README.md

@@ -0,0 +1 @@
+# fl-deploy

docker-compose/filebeat/filebeat.compose.yml

@@ -37,4 +37,4 @@ processors:
   - drop_fields:
       fields: ["message_old"]
 
-
+

kind/core-secret-kibana-password.yml

@@ -0,0 +1,11 @@
+apiVersion: v1
+kind: Secret
+metadata:
+  name: fl-core-secret-kibana-password
+  namespace: fl
+type: Opaque
+data:
+  # the value is the base64 encoding of "kibanapassword"
+  # this value __MUST__ be overwritten in production
+  secret_key_password: aRgY7Odesha8qHfvRzWB # cannot set
+

kind/core.yml

@@ -49,6 +49,11 @@ spec:
             secretKeyRef:
               name: fl-core-secret-postgres-password
               key: secret_key_password
+        - name: KIBANA_PASSWORD
+          valueFrom:
+            secretKeyRef:
+              name: fl-core-secret-kibana-password
+              key: secret_key_password
         - name: NODE_IP
           valueFrom:
             fieldRef:

kind/elasticsearch.yml

@@ -0,0 +1,36 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: fl-elasticsearch
+  namespace: fl
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app: fl-elasticsearch
+  template:
+    metadata:
+      labels:
+        app: fl-elasticsearch
+    spec:
+      serviceAccountName: fl-svc-account
+      restartPolicy: "Always"
+      affinity:
+        nodeAffinity:
+          requiredDuringSchedulingIgnoredDuringExecution:
+            nodeSelectorTerms:
+            - matchExpressions:
+              - key: fl
+                operator: In
+                values:
+                - "core"
+      containers:
+        - name: elasticsearch
+          image: docker.elastic.co/elasticsearch/elasticsearch:8.8.0
+          ports:
+            - containerPort: 9200
+          env:
+            - name: discovery.type
+              value: single-node
+            - name: xpack.security.enabled
+              value: "true"

kind/filebeat-config.yml

@@ -0,0 +1,18 @@
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: fl-filebeat-config
+  namespace: fl
+data:
+  filebeat.yml: |
+    filebeat.inputs:
+    - type: container
+      paths:
+        - /var/log/containers/*.log
+      processors:
+        - add_kubernetes_metadata:
+            in_cluster: true
+
+    output.elasticsearch:
+      hosts: ['10.244.2.17:9200']
+

kind/filebeat.yml

@@ -0,0 +1,47 @@
+apiVersion: apps/v1
+kind: DaemonSet
+metadata:
+  name: fl-filebeat
+  namespace: fl
+spec:
+  selector:
+    matchLabels:
+      app: fl-filebeat
+  template:
+    metadata:
+      labels:
+        app: fl-filebeat
+    spec:
+      serviceAccountName: fl-svc-account
+      restartPolicy: "Always"
+      affinity:
+        nodeAffinity:
+          requiredDuringSchedulingIgnoredDuringExecution:
+            nodeSelectorTerms:
+            - matchExpressions:
+              - key: fl
+                operator: In
+                values:
+                - "core"
+      containers:
+      - name: filebeat
+        image: docker.elastic.co/beats/filebeat:8.8.0
+        args: ["--strict.perms=false"]
+        securityContext:
+          runAsUser: 0
+        volumeMounts:
+        - name: config
+          mountPath: /usr/share/filebeat/filebeat.yml
+          subPath: filebeat.yml
+          readOnly: true
+        - name: varlogcontainers
+          mountPath: /var/log/containers
+          readOnly: true
+      volumes:
+      - name: config
+        configMap:
+          name: fl-filebeat-config
+      - name: varlogcontainers
+        hostPath:
+          path: /var/log/containers
+

kind/kibana.yml

@@ -0,0 +1,66 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: fl-kibana
+  namespace: fl
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app: fl-kibana
+  template:
+    metadata:
+      labels:
+        app: fl-kibana
+    spec:
+      serviceAccountName: fl-svc-account
+      restartPolicy: "Always"
+      affinity:
+        nodeAffinity:
+          requiredDuringSchedulingIgnoredDuringExecution:
+            nodeSelectorTerms:
+            - matchExpressions:
+              - key: fl
+                operator: In
+                values:
+                - "core"
+      containers:
+        - name: kibana
+          image: docker.elastic.co/kibana/kibana:8.8.0
+          ports:
+            - containerPort: 5601
+          env:
+            - name: ELASTICSEARCH_HOSTS   
+              value: "http://10-244-1-18.default.pod.cluster.local:9200" # set elastic ip
+            - name: XPACK_SECURITY_ENABLED
+              value: "true"
+            - name: ELASTICSEARCH_USERNAME
+              value: "kibana_system"
+            - name: ELASTICSEARCH_PASSWORD
+              value: "aRgY7Odesha8qHfvRzWB"
+              # value: ${KIBANA_PASSWORD}
+            - name: XPACK_REPORTING_ROLES_ENABLED
+              value: "false"
+            - name: XPACK_FLEET_REGISTRYURL
+              value: "https://epr.elastic.co"
+            - name: XPACK_FLEET_AGENTS_ENABLED
+              value: "true"
+            - name: xpack.encryptedSavedObjects.encryptionKey
+              value: "true"
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: fl-kibana
+  namespace: fl
+  labels:
+    app: fl-kibana
+spec:
+  ports:
+    - name: http
+      port: 5601
+      targetPort: 5601
+  selector:
+    app: fl-kibana
+
+

kind/postgres.yml

@@ -1,3 +1,37 @@
+apiVersion: v1
+kind: PersistentVolume
+metadata:
+  name: fl-postgres-pv
+  namespace: fl
+  labels:
+    app: fl-postgres
+spec:
+  capacity:
+    storage: 2Gi
+  accessModes:
+    - ReadWriteOnce
+  persistentVolumeReclaimPolicy: Retain
+  storageClassName: standard
+  hostPath:
+    path: /data/postgres
+
+---
+
+apiVersion: v1
+kind: PersistentVolumeClaim
+metadata:
+  name: fl-postgres-pvc
+  namespace: fl
+spec:
+  accessModes:
+    - ReadWriteOnce
+  resources:
+    requests:
+      storage: 2Gi
+  storageClassName: standard
+
+---
+
 apiVersion: apps/v1
 kind: Deployment
 metadata:
@@ -45,6 +79,13 @@ spec:
               key: secret_key_user
         - name: POSTGRES_HOST_AUTH_METHOD
           value: trust
+        volumeMounts:
+          - name: fl-postgres-data
+            mountPath: /var/lib/postgresql/data
+      volumes:
+        - name: fl-postgres-data
+          persistentVolumeClaim:
+            claimName: fl-postgres-pvc
 
 ---
 
@@ -111,3 +152,4 @@ spec:
       port: 5432
       targetPort: 5432
       protocol: TCP
+

kind/prometheus.yml

@@ -1,3 +1,35 @@
+apiVersion: v1
+kind: PersistentVolume
+metadata:
+  name: fl-prometheus-pv
+  namespace: fl
+spec:
+  capacity:
+    storage: 2Gi
+  accessModes:
+    - ReadWriteOnce
+  persistentVolumeReclaimPolicy: Retain
+  storageClassName: standard
+  hostPath:
+    path: /data/prometheus
+
+---
+
+apiVersion: v1
+kind: PersistentVolumeClaim
+metadata:
+  name: fl-prometheus-pvc
+  namespace: fl
+spec:
+  accessModes:
+    - ReadWriteOnce
+  resources:
+    requests:
+      storage: 2Gi
+  storageClassName: standard
+
+---
+
 apiVersion: apps/v1
 kind: Deployment
 metadata:
@@ -21,27 +53,32 @@ spec:
         nodeAffinity:
           requiredDuringSchedulingIgnoredDuringExecution:
             nodeSelectorTerms:
-            - matchExpressions:
-              - key: fl
-                operator: In
-                values:
-                - "core"
+              - matchExpressions:
+                - key: fl
+                  operator: In
+                  values:
+                    - "core"
+      initContainers:
+        - name: create-data-dir
+          image: busybox
+          command: ["mkdir", "-p", "/data/prometheus"]
+          volumeMounts:
+            - name: fl-prometheus-data
+              mountPath: /data/prometheus
       volumes:
-        - name: fl-prometheus-config-volume
-          configMap:
-            name: fl-prometheus-configmap
+        - name: fl-prometheus-data
+          persistentVolumeClaim:
+            claimName: fl-prometheus-pvc
       containers:
-      - name: prometheus
-        image: "prom/prometheus:latest"
-        imagePullPolicy: IfNotPresent
-        ports:
-        - containerPort: 9090
-        volumeMounts:
-        - name: fl-prometheus-config-volume
-          mountPath: /etc/prometheus/
-
+        - name: prometheus
+          image: prom/prometheus:latest
+          imagePullPolicy: IfNotPresent
+          ports:
+            - containerPort: 9090
+          volumeMounts:
+            - name: fl-prometheus-data
+              mountPath: /prometheus/data
 ---
-
 apiVersion: v1
 kind: Service
 metadata:
@@ -57,4 +94,5 @@ spec:
     - name: http
       port: 9090
       targetPort: 9090
-      protocol: TCP
+      protocol: TCP
+

kind/start_kind.sh

@@ -21,3 +21,8 @@ kubectl apply -f postgres.yml
 kubectl apply -f core-secret-key-base.yml
 kubectl apply -f core.yml
 kubectl apply -f worker.yml
+kubectl apply -f core-secret-kibana-password.yml
+kubectl apply -f elasticsearch.ym
+kubectl apply -f filebeat-config.yml
+kubectl apply -f filebeat.yml
+kubectl apply -f kibana.yml

nomad-ansible/README.md

@@ -0,0 +1,3 @@
+## nomad-ansible
+
+Ansible to configure Nomad

nomad-ansible/ansible.cfg

@@ -0,0 +1,6 @@
+[defaults]
+inventory = hosts
+stratehy = free
+
+[ssh_connections]
+pipelining = true