서버 모니터링용 인프라 작업 (#237)

* feat: add monitoring helm charts

* chore: inject open telemetry autoinstrumentation to scc server dev

* feat: modify values for chart

* feat: add secret deploy

infra/helm/scc-monitoring/Chart.lock

@@ -1,6 +1,9 @@
 dependencies:
-- name: kube-prometheus-stack
-  repository: https://prometheus-community.github.io/helm-charts
-  version: 56.6.2
-digest: sha256:9cabf099ca6b6d6267fe9fbe4db9b1c2ef311795b0a4cbf94cf781276c2ff600
-generated: "2024-02-05T17:55:13.166201+09:00"
+- name: opentelemetry-operator
+  repository: https://open-telemetry.github.io/opentelemetry-helm-charts
+  version: 0.48.0
+- name: openobserve-collector
+  repository: https://charts.openobserve.ai
+  version: 0.3.1
+digest: sha256:bc6f82f23e468cf4df9ca5ede2859aefc069711b37d84b765236dc00225968e5
+generated: "2024-03-11T23:30:51.414395+09:00"

infra/helm/scc-monitoring/Chart.yaml

@@ -23,6 +23,9 @@ version: 0.1.0
 # It is recommended to use it with quotes.
 appVersion: "1.16.0"
 dependencies:
-  - name: kube-prometheus-stack
-    version: 56.6.2
-    repository: https://prometheus-community.github.io/helm-charts
+  - name: opentelemetry-operator
+    version: 0.48.0
+    repository: https://open-telemetry.github.io/opentelemetry-helm-charts
+  - name: openobserve-collector
+    version: 0.3.1
+    repository: https://charts.openobserve.ai

infra/helm/scc-monitoring/files/secret.yaml

@@ -0,0 +1,24 @@
+exporters:
+    otlphttp/openobserve:
+        endpoint: ENC[AES256_GCM,data:AoY67QmmyjS2hHW0tCUTELcPVhKH2Tr0AJQMJ61QtFtPK2+rT3096Ev+3TOUWSc1k31KOzPG/KtOJikAmxIRT+fbwIkH4z2+gA==,iv:qWvBVw6eJ+DdSqhXQ5PirK1qCVi+F37hxlOUcJwXL1U=,tag:AZ43F5merGHcg5Pf+VqzLw==,type:str]
+        headers:
+            Authorization: ENC[AES256_GCM,data:85/QsH/gga1KYOHlLGNnw++32bgNgEauoL1xPCld43ibFDrwZDEc5Wvy0BGnWb3EdBiP6s+MXPUzEL7P7SWWtoKG,iv:8bJDzzXzcUt8W3NYgb6k+LL4OeY6pLVp0p65wMpZeD8=,tag:6t4gTvom7g4+CF5hPTZAvw==,type:str]
+    otlphttp/openobserve_k8s_events:
+        endpoint: ENC[AES256_GCM,data:zDXFFXabYKAnsYkCeRWrhlhQwjBspGT5pydjixLeMlpEzA6UdFreunmMT8fDDGSIuQ9NsmKZClSSkeUv8wVPVK6canEwjK3C7g==,iv:AXgjuDI6AJkpxv3/IpK78AZti5g7oDiH0TkHgnq3iE8=,tag:MAR9evbPMuUwDNPvMJpE5g==,type:str]
+        headers:
+            Authorization: ENC[AES256_GCM,data:IS7jWLahY+lX1b+RGposrm1GBXQb1JdZu/aQ7frB0LKdXNV09aOkAEiNvs+yzh+Uc4KuERhLrYvzD3BYzYhsi2kI,iv:u3l8EO7+Fq/CPo4TATTAY9eQ5rN3Q9tj1U+/MFHWZWY=,tag:MHQL/EBiNCBwQwnTZCPqkA==,type:str]
+sops:
+    kms:
+        - arn: arn:aws:kms:ap-northeast-2:291889421067:alias/sops
+          created_at: "2023-07-31T15:54:45Z"
+          enc: AQICAHgF8bYnZL94LzSwlUWA75seDxMTpMHltwauy+q73c/QBwFniCIYq6aVHk8/6aLA+AJtAAAAfjB8BgkqhkiG9w0BBwagbzBtAgEAMGgGCSqGSIb3DQEHATAeBglghkgBZQMEAS4wEQQM1YMNm0wSD1pN7dRxAgEQgDtFpVZMaweOLSqjHlVDJClQ4G+1D3jpzLa/zFQEgdtDelP/Xo4MrMfHd++njCmbf6asBTNRorBCKvbcsg==
+          aws_profile: ""
+    gcp_kms: []
+    azure_kv: []
+    hc_vault: []
+    age: []
+    lastmodified: "2024-03-13T17:21:22Z"
+    mac: ENC[AES256_GCM,data:V8YtUVw896AaSKc6ukVnPm1fyXXz0J31f5jMqfkluktq53UFjPScztk4jW07hlSW99q3lkHEf9t0Jb8vbTW323aEE7Va/5QVfh0hAeil2jSch+THA7gVyB51LmaiRWrW+La13VLqYlzKnc5d+VAxAGAMoKg71gd3dEJkPycDi5A=,iv:eF/41v3dIua+yjYp5jdgw89KQTGSCujms8VSDqF3tw8=,tag:q1MUPiu5GjRTf/HCZB7Lgg==,type:str]
+    pgp: []
+    unencrypted_suffix: _unencrypted
+    version: 3.8.1

infra/helm/scc-monitoring/templates/configmap-for-secret.yaml

@@ -0,0 +1,12 @@
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: {{ include "scc-monitoring.fullname" . }}-for-secret
+  labels:
+    {{- include "scc-monitoring.labels" . | nindent 4 }}
+  annotations:
+    "helm.sh/hook": pre-install,pre-upgrade
+    "helm.sh/hook-weight": "10"
+data:
+  secret-raw.yaml: |-
+{{ printf "%s/secret.yaml" .Values.filesDir | .Files.Get | indent 4 }}

infra/helm/scc-monitoring/templates/deploy-secret-job.yaml

@@ -0,0 +1,68 @@
+apiVersion: batch/v1
+kind: Job
+metadata:
+  name: {{ include "scc-monitoring.fullname" . }}-deploy-secret-job
+  annotations:
+    "helm.sh/hook": pre-install,pre-upgrade
+    "helm.sh/hook-weight": "20"
+spec:
+  template:
+    spec:
+      serviceAccountName: {{ include "scc-monitoring.serviceAccountName" . }}-deploy-secret
+      securityContext:
+        {{- toYaml .Values.podSecurityContext | nindent 8 }}
+      containers:
+        - name: {{ .Chart.Name }}
+          image: "mozilla/sops:v3-alpine"
+          imagePullPolicy: IfNotPresent
+          command: ["/bin/sh"]
+          args:
+            - -c
+            - |-
+              wget "https://dl.k8s.io/release/$(wget https://dl.k8s.io/release/stable.txt -O-)/bin/linux/amd64/kubectl" &&
+              chmod u+x kubectl &&
+              wget https://github.com/mikefarah/yq/releases/latest/download/yq_linux_amd64 -O /usr/bin/yq &&
+              chmod +x /usr/bin/yq &&
+              mkdir secret &&
+              sops -d /app/conf/secret-raw.yaml > secret/secret.yaml &&
+              echo 'apiVersion: v1
+              kind: Secret
+              metadata:
+                name: scc-monitoring-secret
+                namespace: scc-monitoring
+              type: Opaque
+              stringData: {}' > scc-monitoring-secret.yaml &&
+              yq -i ".stringData += $(yq secret/secret.yaml -o json)" scc-monitoring-secret.yaml &&
+              ./kubectl apply -f scc-monitoring-secret.yaml
+#              ./kubectl create secret generic {{ include "scc-monitoring.fullname" . }}-secret --from-file=secret/secret.yaml --dry-run=client -o yaml | ./kubectl apply -f -
+#              TODO: kubectl을 이미지에 미리 깔아놓기
+          env:
+            - name: AWS_STS_REGIONAL_ENDPOINTS
+              value: regional
+            - name: AWS_REGION
+              value: ap-northeast-2
+            - name: AWS_ROLE_ARN
+              value: "{{ .Values.deploySecret.awsRoleArn }}"
+            - name: AWS_WEB_IDENTITY_TOKEN_FILE
+              value: /var/run/secrets/eks.amazonaws.com/serviceaccount/token
+          volumeMounts:
+            - name: aws-iam-token
+              mountPath: /var/run/secrets/eks.amazonaws.com/serviceaccount
+              readOnly: true
+            - name: config-volume
+              mountPath: /app/conf
+              readOnly: true
+      volumes:
+        - name: aws-iam-token
+          projected:
+            defaultMode: 420
+            sources:
+              - serviceAccountToken:
+                  audience: sts.amazonaws.com
+                  expirationSeconds: 86400
+                  path: token
+        - name: config-volume
+          configMap:
+            name: {{ include "scc-monitoring.fullname" . }}-for-secret
+      restartPolicy: Never
+  backoffLimit: 3

infra/helm/scc-monitoring/templates/deploy-secret-serviceaccount.yaml

@@ -0,0 +1,34 @@
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: {{ include "scc-monitoring.serviceAccountName" . }}-deploy-secret
+  annotations:
+    "helm.sh/hook": pre-install,pre-upgrade
+    "helm.sh/hook-weight": "10"
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: Role
+metadata:
+  name: create-secrets
+  annotations:
+    "helm.sh/hook": pre-install,pre-upgrade
+    "helm.sh/hook-weight": "10"
+rules:
+  - apiGroups: [""] # "" indicates the core API group
+    resources: ["secrets"]
+    verbs: ["create", "patch", "get"]
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: RoleBinding
+metadata:
+  name: create-secrets-to-{{ include "scc-monitoring.serviceAccountName" . }}-deploy-secret
+  annotations:
+    "helm.sh/hook": pre-install,pre-upgrade
+    "helm.sh/hook-weight": "10"
+subjects:
+  - kind: ServiceAccount
+    name: {{ include "scc-monitoring.serviceAccountName" . }}-deploy-secret
+roleRef:
+  kind: Role
+  name: create-secrets
+  apiGroup: rbac.authorization.k8s.io