[모니터링] 실섭만 수집하도록 필터 설정 & 자잘한 업데이트 (#246)

* chore: change namespace for monitoring

* fix: monitoring pods

* feat: add filtering to metrics

infra/helm/scc-monitoring/Chart.lock → infra/helm/monitoring/Chart.lock

@@ -1,9 +1,9 @@
 dependencies:
 - name: opentelemetry-operator
   repository: https://open-telemetry.github.io/opentelemetry-helm-charts
-  version: 0.48.0
+  version: 0.49.1
 - name: openobserve-collector
   repository: https://charts.openobserve.ai
   version: 0.3.1
-digest: sha256:bc6f82f23e468cf4df9ca5ede2859aefc069711b37d84b765236dc00225968e5
-generated: "2024-03-11T23:30:51.414395+09:00"
+digest: sha256:cb8db2c32a350c3b14da84a5e73d65f6475bb3535a4970ec096d5b5d8327d052
+generated: "2024-03-19T09:17:44.003518+09:00"

infra/helm/scc-monitoring/Chart.yaml → infra/helm/monitoring/Chart.yaml

@@ -1,5 +1,5 @@
 apiVersion: v2
-name: scc-monitoring
+name: monitoring
 description: A Helm chart for Kubernetes
 
 # A chart can be either an 'application' or a 'library' chart.
@@ -24,7 +24,7 @@ version: 0.1.0
 appVersion: "1.16.0"
 dependencies:
   - name: opentelemetry-operator
-    version: 0.48.0
+    version: 0.49.1
     repository: https://open-telemetry.github.io/opentelemetry-helm-charts
   - name: openobserve-collector
     version: 0.3.1

infra/helm/scc-monitoring/README.md → infra/helm/monitoring/README.md

@@ -1,11 +1,21 @@
 # 모니터링 Chart 배포
 
+## 필요한 CRD 설치
+```bash
+kubectl apply -f manifests/crd-podmonitors.yaml
+kubectl apply -f manifests/crd-servicemonitors.yaml
+```
+[이 PR](https://github.com/openobserve/openobserve/pull/2884)에서 고쳐졌지만 아직 Helm chart 에 적용되진 않아서 직접 설치합니다
+
+
+## Helm Chart 설치
+
 secret 을 helm value 혹은 ENV 로 주입하여 배포되도록 하려고 했으나 마땅한 방법이 없어서 일단 --set 옵션으로 배포합니다.
 
 ```bash
-helm --namespace scc-monitoring \
+helm upgrade --install monitoring ./ \
+  --namespace monitoring \
   -f values.yaml \
-  upgrade --install scc-monitoring ./ \
   --set openobserve-collector.exporters."otlphttp/openobserve".endpoint={secret.yaml 참조}  \
   --set openobserve-collector.exporters."otlphttp/openobserve".headers.Authorization="{secret.yaml 참조}"  \
   --set openobserve-collector.exporters."otlphttp/openobserve_k8s_events".endpoint={secret.yaml 참조}  \

infra/helm/scc-monitoring/templates/_helpers.tpl → infra/helm/monitoring/templates/_helpers.tpl

@@ -1,7 +1,7 @@
 {{/*
 Expand the name of the chart.
 */}}
-{{- define "scc-monitoring.name" -}}
+{{- define "monitoring.name" -}}
 {{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" }}
 {{- end }}
 
@@ -10,7 +10,7 @@ Create a default fully qualified app name.
 We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec).
 If release name contains chart name it will be used as a full name.
 */}}
-{{- define "scc-monitoring.fullname" -}}
+{{- define "monitoring.fullname" -}}
 {{- if .Values.fullnameOverride }}
 {{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" }}
 {{- else }}
@@ -26,16 +26,16 @@ If release name contains chart name it will be used as a full name.
 {{/*
 Create chart name and version as used by the chart label.
 */}}
-{{- define "scc-monitoring.chart" -}}
+{{- define "monitoring.chart" -}}
 {{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" }}
 {{- end }}
 
 {{/*
 Common labels
 */}}
-{{- define "scc-monitoring.labels" -}}
-helm.sh/chart: {{ include "scc-monitoring.chart" . }}
-{{ include "scc-monitoring.selectorLabels" . }}
+{{- define "monitoring.labels" -}}
+helm.sh/chart: {{ include "monitoring.chart" . }}
+{{ include "monitoring.selectorLabels" . }}
 {{- if .Chart.AppVersion }}
 app.kubernetes.io/version: {{ .Chart.AppVersion | quote }}
 {{- end }}
@@ -45,17 +45,17 @@ app.kubernetes.io/managed-by: {{ .Release.Service }}
 {{/*
 Selector labels
 */}}
-{{- define "scc-monitoring.selectorLabels" -}}
-app.kubernetes.io/name: {{ include "scc-monitoring.name" . }}
+{{- define "monitoring.selectorLabels" -}}
+app.kubernetes.io/name: {{ include "monitoring.name" . }}
 app.kubernetes.io/instance: {{ .Release.Name }}
 {{- end }}
 
 {{/*
 Create the name of the service account to use
 */}}
-{{- define "scc-monitoring.serviceAccountName" -}}
+{{- define "monitoring.serviceAccountName" -}}
 {{- if .Values.serviceAccount.create }}
-{{- default (include "scc-monitoring.fullname" .) .Values.serviceAccount.name }}
+{{- default (include "monitoring.fullname" .) .Values.serviceAccount.name }}
 {{- else }}
 {{- default "default" .Values.serviceAccount.name }}
 {{- end }}

infra/helm/scc-monitoring/values.yaml → infra/helm/monitoring/values.yaml

@@ -1,8 +1,3 @@
-deploySecret:
-  awsRoleArn: arn:aws:iam::291889421067:role/scc-deploy-secret
-
-filesDir: files
-
 serviceAccount:
   # Specifies whether a service account should be created
   create: true
@@ -280,6 +275,15 @@ openobserve-collector:
           k8s.pod.memory_request_utilization:
             enabled: true
     processors:
+      filter/keep-scc:
+        error_mode: ignore
+        logs:
+          log_record:
+            - resource.attributes["k8s.namespace.name"]!=nil and resource.attributes["k8s.namespace.name"]!="scc"
+        metrics:
+          metric:
+            - resource.attributes["k8s.namespace.name"]!=nil and resource.attributes["k8s.namespace.name"]!="scc"
+            - resource.attributes["namespace"]!=nil and resource.attributes["namespace"]!="scc"
       resourcedetection:
         detectors: [system, env, k8snode]
         override: true
@@ -353,15 +357,15 @@ openobserve-collector:
       pipelines:
         logs:
           receivers: [filelog/std]
-          processors: [batch, k8sattributes]
+          processors: [batch, k8sattributes, filter/keep-scc]
           exporters: [otlphttp/openobserve]
         metrics:
           receivers: [kubeletstats]
-          processors: [batch, k8sattributes]
+          processors: [batch, k8sattributes, filter/keep-scc]
           exporters: [otlphttp/openobserve]
         traces:
           receivers: [otlp]
-          processors: [batch, k8sattributes]
+          processors: [batch, k8sattributes, filter/keep-scc]
           exporters: [otlphttp/openobserve]
 
   gateway:
@@ -545,6 +549,15 @@ openobserve-collector:
                   target_label: metrics_path
 
     processors:
+      filter/keep-scc:
+        error_mode: ignore
+        logs:
+          log_record:
+            - resource.attributes["k8s.namespace.name"]!="scc"
+        metrics:
+          metric:
+            - resource.attributes["k8s.namespace.name"]!="scc"
+            - resource.attributes["namespace"]!="scc"
       resourcedetection:
         detectors: [env]
         override: true
@@ -670,13 +683,13 @@ openobserve-collector:
       pipelines:
         logs/k8s_events:
           receivers: [k8s_events]
-          processors: [batch, k8sattributes, resourcedetection]
+          processors: [batch, k8sattributes, resourcedetection, filter/keep-scc]
           exporters: [otlphttp/openobserve_k8s_events]
         metrics:
           receivers: [k8s_cluster, prometheus, spanmetrics, servicegraph]
-          processors: [batch, k8sattributes, resourcedetection]
+          processors: [batch, k8sattributes, resourcedetection, filter/keep-scc]
           exporters: [otlphttp/openobserve]
         traces:
           receivers: [otlp]
-          processors: [batch, k8sattributes, resourcedetection]
+          processors: [batch, k8sattributes, resourcedetection, filter/keep-scc]
           exporters: [otlphttp/openobserve, spanmetrics, servicegraph]

infra/helm/scc-server/values-dev.yaml

@@ -30,7 +30,7 @@ serviceAccount:
   name: ""
 
 podAnnotations:
-  instrumentation.opentelemetry.io/inject-java: "openobserve-collector/openobserve-java"
+  instrumentation.opentelemetry.io/inject-java: "monitoring/openobserve-java"
 
 podSecurityContext:
   fsGroup: 2000