[모니터링] 배포 자동화 (#423)

feat: automate monitoring deploy
Stair-Crusher-Club · Nov 10, 2024 · 9f978d9 · 9f978d9
1 parent d08485f
commit 9f978d9
Show file tree

Hide file tree

Showing 3 changed files with 83 additions and 15 deletions.
diff --git a/.github/workflows/monitoring-cd.yaml b/.github/workflows/monitoring-cd.yaml
@@ -0,0 +1,67 @@
+name: monitoring-cd
+on:
+  push:
+    branches:
+      - main
+    paths:
+      - infra/helm/monitoring/**
+  workflow_dispatch: # for manual trigger
+
+permissions:
+  id-token: write
+
+jobs:
+  cd:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+
+      - name: Notify CD start to slack
+        run: |
+          curl -X POST -H 'Content-type: application/json' --data "{\"attachments\":[{\"color\":\"#46AE74\",\"title\":\"[Monitoring] Deploy Started\", \"text\":\"version - latest\"}]}" ${{ secrets.SLACK_URL_SCC_SERVER_CHANNEL }} -v
+
+      - name: Install kubeconfig
+        env:
+          SCC_K3S_KUBECONFIG: ${{ secrets.SCC_K3S_KUBECONFIG }}
+        run: |
+          mkdir -p ~/.kube && echo "$SCC_K3S_KUBECONFIG" > ~/.kube/config
+
+      - uses: azure/setup-helm@v1
+        with:
+          version: '3.8.2'
+        id: install-helm
+
+      - uses: azure/setup-kubectl@v3
+        with:
+          version: 'v1.24.1'
+        id: install-kubectl
+
+      - name: Upgrade monitoring helm chart
+        working-directory: infra/helm/monitoring
+        env:
+          MONITORING_ENDPOINT: ${{ secrets.MONITORING_ENDPOINT }}
+          MONITORING_AUTHORIZATION_HEADER: ${{ secrets.MONITORING_AUTHORIZATION_HEADER }}
+        run: |
+          helm upgrade --install monitoring ./ \
+            --namespace monitoring \
+            -f values.yaml \
+            --set openobserve-collector.exporters."otlphttp/openobserve".endpoint="$MONITORING_ENDPOINT" \
+            --set openobserve-collector.exporters."otlphttp/openobserve".headers.Authorization="$MONITORING_AUTHORIZATION_HEADER" \
+            --set openobserve-collector.exporters."otlphttp/openobserve_k8s_events".endpoint="$MONITORING_ENDPOINT" \
+            --set openobserve-collector.exporters."otlphttp/openobserve_k8s_events".headers.Authorization="$MONITORING_AUTHORIZATION_HEADER"
+
+      - name: Wait for monitoring rollout
+        run: |
+          kubectl rollout status ds/monitoring-openobserve-collector-agent-collector -n monitoring
+
+      - name: Notify CD failure to slack
+        if: failure()
+        run: |
+          CURRENT_GITHUB_ACTION_RUN_URL="${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}"
+          curl -X POST -H 'Content-type: application/json' --data "{\"attachments\":[{\"color\":\"#FF0000\",\"title\":\"[Monitoring] Deployment Failed\", \"text\":\"<!subteam^S052B9W7129> (<$CURRENT_GITHUB_ACTION_RUN_URL|github action run url>)\"}]}" ${{ secrets.SLACK_URL_SCC_SERVER_CHANNEL }} -v
+
+      - name: Notify CD success to slack
+        if: success()
+        run: |
+          CURRENT_GITHUB_ACTION_RUN_URL="${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}"
+          curl -X POST -H 'Content-type: application/json' --data "{\"attachments\":[{\"color\":\"#0000FF\",\"title\":\"[Monitoring] Deploy Success\"}]}" ${{ secrets.SLACK_URL_SCC_SERVER_CHANNEL }} -v
diff --git a/infra/helm/monitoring/files/secret.yaml b/infra/helm/monitoring/files/secret.yaml
@@ -1,12 +1,13 @@
-exporters:
-    otlphttp/openobserve:
-        endpoint: ENC[AES256_GCM,data:AoY67QmmyjS2hHW0tCUTELcPVhKH2Tr0AJQMJ61QtFtPK2+rT3096Ev+3TOUWSc1k31KOzPG/KtOJikAmxIRT+fbwIkH4z2+gA==,iv:qWvBVw6eJ+DdSqhXQ5PirK1qCVi+F37hxlOUcJwXL1U=,tag:AZ43F5merGHcg5Pf+VqzLw==,type:str]
+openobserve-collector:
+    exporters:
+        otlphttp/openobserve:
+            endpoint: ENC[AES256_GCM,data:aEL/+Jt9vue/1M1qDNV6vJSHRtDOcrCoGa/YBnSEZs+CTH3hqooECNVUxQF6twX3hwNxZLP3PgT051GOIFkWk/SoHCABUxjdpw==,iv:iBMmFNh7d04bC5ZRBSTrRBCuDmZbscgLDKFA8jz7pT0=,tag:Uwf4RqDJzlwL1ZNnSXT/eQ==,type:str]
+            headers:
+                Authorization: ENC[AES256_GCM,data:+2Up4PBCYbaRClI6RUWAVOO5pSsn2YHSyD6NpB/Jq0m9YTYea/rSGMN0eP4l9mh5vyug3CHNTXhAYYV8dndkNYFt,iv:5DrEJ5QKbz+R2mn0Lx6UOVIoI9Yx63/1CHx18LEBdJY=,tag:aG/YacUnMwnZGffeIqcEzA==,type:str]
+        otlphttp/openobserve_k8s_events:
+            endpoint: ENC[AES256_GCM,data:UVGiji5G9g4juBGm7/jp/wsBF4RNyIpw/YNYSO5fQDE2YlimPk07kj7kSx6c/+qibdNL+yqqqaxriE0mVxJcDcnOKskPGlM5+w==,iv:RBcT7BzC0ztngQ2Eshcik+kKGjGi2YfWpvBxLntqxB8=,tag:dXC11717ARvI4hTLOVSrgw==,type:str]
         headers:
-            Authorization: ENC[AES256_GCM,data:85/QsH/gga1KYOHlLGNnw++32bgNgEauoL1xPCld43ibFDrwZDEc5Wvy0BGnWb3EdBiP6s+MXPUzEL7P7SWWtoKG,iv:8bJDzzXzcUt8W3NYgb6k+LL4OeY6pLVp0p65wMpZeD8=,tag:6t4gTvom7g4+CF5hPTZAvw==,type:str]
-    otlphttp/openobserve_k8s_events:
-        endpoint: ENC[AES256_GCM,data:zDXFFXabYKAnsYkCeRWrhlhQwjBspGT5pydjixLeMlpEzA6UdFreunmMT8fDDGSIuQ9NsmKZClSSkeUv8wVPVK6canEwjK3C7g==,iv:AXgjuDI6AJkpxv3/IpK78AZti5g7oDiH0TkHgnq3iE8=,tag:MAR9evbPMuUwDNPvMJpE5g==,type:str]
-        headers:
-            Authorization: ENC[AES256_GCM,data:IS7jWLahY+lX1b+RGposrm1GBXQb1JdZu/aQ7frB0LKdXNV09aOkAEiNvs+yzh+Uc4KuERhLrYvzD3BYzYhsi2kI,iv:u3l8EO7+Fq/CPo4TATTAY9eQ5rN3Q9tj1U+/MFHWZWY=,tag:MHQL/EBiNCBwQwnTZCPqkA==,type:str]
+            Authorization: ENC[AES256_GCM,data:3khq7at2YheV4Y6pn4TDZv8CKaOHxoWZk4340IkT2/KcbWViK3qdY1mFnepOWCqa9SwL86JNxdHj4LJy9A+MDXrj,iv:fu1iyL3ud9jd6zH8vqqcxiK1cclE2P9xUkQLVN2wL7g=,tag:6uysQ8/EHcOYX9Y9fIeVRA==,type:str]
 sops:
     kms:
         - arn: arn:aws:kms:ap-northeast-2:291889421067:alias/sops
@@ -17,8 +18,8 @@ sops:
     azure_kv: []
     hc_vault: []
     age: []
-    lastmodified: "2024-03-13T17:21:22Z"
-    mac: ENC[AES256_GCM,data:V8YtUVw896AaSKc6ukVnPm1fyXXz0J31f5jMqfkluktq53UFjPScztk4jW07hlSW99q3lkHEf9t0Jb8vbTW323aEE7Va/5QVfh0hAeil2jSch+THA7gVyB51LmaiRWrW+La13VLqYlzKnc5d+VAxAGAMoKg71gd3dEJkPycDi5A=,iv:eF/41v3dIua+yjYp5jdgw89KQTGSCujms8VSDqF3tw8=,tag:q1MUPiu5GjRTf/HCZB7Lgg==,type:str]
+    lastmodified: "2024-11-07T08:05:14Z"
+    mac: ENC[AES256_GCM,data:tg22CuckuVg6jZ2ZigtWvydAuYJxq5IU+gH96xqRGlB23bgqbEL+439/BKEU+AOoi2mZA9aafVIW9VdgD0nWbO7QB64/HN4zuxm0KlwrTR8wxEnT6Zi2wYJJzPVGzR2z/7q+nrnKtYpe8fwoI2J2FA1iUbK7s79eS9eWaurc2RY=,iv:EtRI+fMvRJVOXpiJNX0obtJDk5c6u+P8LiwEJuOUfd4=,tag:TOXDpS7Xy8+AnUYUFv5ZwQ==,type:str]
     pgp: []
     unencrypted_suffix: _unencrypted
-    version: 3.8.1
+    version: 3.9.1
diff --git a/infra/helm/monitoring/values.yaml b/infra/helm/monitoring/values.yaml
@@ -393,11 +393,11 @@ openobserve-collector:
       pipelines:
         logs:
           receivers: [filelog/std]
-          processors: [batch, k8sattributes, filter/keep-scc]
+          processors: [batch, k8sattributes]
           exporters: [otlphttp/openobserve]
         metrics:
           receivers: [kubeletstats]
-          processors: [batch, k8sattributes, filter/keep-scc]
+          processors: [batch, k8sattributes]
           exporters: [otlphttp/openobserve]
         traces:
           receivers: [otlp]
@@ -794,14 +794,14 @@ openobserve-collector:
       pipelines:
         logs/k8s_events:
           receivers: [k8s_events]
-          processors: [batch, k8sattributes, resourcedetection, filter/keep-scc]
+          processors: [batch, k8sattributes, resourcedetection]
           exporters: [otlphttp/openobserve_k8s_events]
         metrics:
           # receivers: [k8s_cluster, prometheus, spanmetrics, servicegraph]
           # spanmetrics 에서 health check 를 포함한 모든 api 콜의 span metrics 를 처리하다 보니
           # 너무 많은 양의 데이터가 ingest 되어서 필터링 로직을 붙이기 전까지 일단 제외한다
           receivers: [k8s_cluster, servicegraph, otlp]
-          processors: [batch, k8sattributes, resourcedetection, filter/keep-scc]
+          processors: [batch, k8sattributes, resourcedetection]
           exporters: [otlphttp/openobserve]
         traces:
           receivers: [otlp]