feat(k8s-infra): introduce prometheus scraper using pod annotations

charts/k8s-infra/templates/_config.tpl

@@ -71,6 +71,9 @@ Build config file for deployment OpenTelemetry Collector: OtelDeployment
 {{- if .Values.presets.clusterMetrics.enabled }}
 {{- $config = (include "opentelemetry-collector.applyClusterMetricsConfig" (dict "Values" $data "config" $config) | fromYaml) }}
 {{- end }}
+{{- if and .Values.presets.prometheusScraper.enabled (or .Values.presets.prometheusScraper.signoz.enabled .Values.presets.prometheusScraper.prometheus.enabled) }}
+{{- $config = (include "opentelemetry-collector.applyPrometheusScraperConfig" (dict "Values" $data "config" $config) | fromYaml) }}
+{{- end }}
 {{- if .Values.presets.loggingExporter.enabled }}
 {{- $config = (include "opentelemetry-collector.applyLoggingExporterConfig" (dict "Values" $data "config" $config) | fromYaml) }}
 {{- end }}
@@ -80,6 +83,9 @@ Build config file for deployment OpenTelemetry Collector: OtelDeployment
 {{- if or (eq (len (index (index $config.service.pipelines "metrics/internal") "receivers")) 0) (eq (len (index (index $config.service.pipelines "metrics/internal") "exporters")) 0) }}
 {{- $_ := unset $config.service.pipelines "metrics/internal" }}
 {{- end }}
+{{- if or (eq (len (index (index $config.service.pipelines "metrics/scraper") "receivers")) 0) (eq (len (index (index $config.service.pipelines "metrics/scraper") "exporters")) 0) }}
+{{- $_ := unset $config.service.pipelines "metrics/scraper" }}
+{{- end }}
 {{- tpl (toYaml $config) . }}
 {{- end }}
 
@@ -97,6 +103,9 @@ Build config file for deployment OpenTelemetry Collector: OtelDeployment
 {{- if index $config.service.pipelines "metrics/internal" }}
 {{- $_ := set (index $config.service.pipelines "metrics/internal") "exporters" (prepend (index (index $config.service.pipelines "metrics/internal") "exporters") "logging" | uniq)  }}
 {{- end }}
+{{- if index $config.service.pipelines "metrics/scraper" }}
+{{- $_ := set (index $config.service.pipelines "metrics/scraper") "exporters" (prepend (index (index $config.service.pipelines "metrics/scraper") "exporters") "logging" | uniq)  }}
+{{- end }}
 {{- $config | toYaml }}
 {{- end }}
 
@@ -125,6 +134,9 @@ exporters:
 {{- if index $config.service.pipelines "metrics/internal" }}
 {{- $_ := set (index $config.service.pipelines "metrics/internal") "exporters" (prepend (index (index $config.service.pipelines "metrics/internal") "exporters") "otlp" | uniq)  }}
 {{- end }}
+{{- if index $config.service.pipelines "metrics/scraper" }}
+{{- $_ := set (index $config.service.pipelines "metrics/scraper") "exporters" (prepend (index (index $config.service.pipelines "metrics/scraper") "exporters") "otlp" | uniq)  }}
+{{- end }}
 {{- $config | toYaml }}
 {{- end }}
 
@@ -164,6 +176,158 @@ receivers:
       {{- toYaml .Values.presets.clusterMetrics.allocatableTypesToReport | nindent 6 }}
 {{- end }}
 
+{{- define "opentelemetry-collector.applyPrometheusScraperConfig" -}}
+{{- $config := mustMergeOverwrite (include "opentelemetry-collector.prometheusScraperConfig" .Values | fromYaml) .config }}
+{{- if index $config.service.pipelines "metrics/internal" }}
+{{- $_ := set (index $config.service.pipelines "metrics/internal") "receivers" (append (index (index $config.service.pipelines "metrics/internal") "receivers") "prometheus/scraper" | uniq)  }}
+{{- end }}
+{{- $config | toYaml }}
+{{- end }}
+
+{{- define "opentelemetry-collector.prometheusScraperConfig" -}}
+receivers:
+  prometheus/scraper:
+    config:
+      scrape_configs:
+        {{- if .Values.presets.prometheusScraper.signoz.enabled }}
+        - job_name: "signoz-scraper"
+          scrape_interval: {{ .Values.presets.prometheusScraper.signoz.scrapeInterval }}
+          kubernetes_sd_configs:
+            - role: pod
+              {{- if or .Values.presets.prometheusScraper.signoz.namespaceScoped (len .Values.presets.prometheusScraper.signoz.namespaces) }}
+              namespaces:
+                {{- if .Values.presets.prometheusScraper.signoz.namespaceScoped }}
+                own_namespace: true
+                {{- end }}
+                {{- if .Values.presets.prometheusScraper.signoz.namespaces }}
+                names: {{ toYaml .Values.presets.prometheusScraper.signoz.namespaces | nindent 16 }}
+                {{- end }}
+              {{- end }}
+          relabel_configs:
+            - source_labels: [__meta_kubernetes_pod_annotation_signoz_io_scrape]
+              action: keep
+              regex: true
+            - source_labels: [__meta_kubernetes_pod_annotation_signoz_io_path]
+              action: replace
+              target_label: __metrics_path__
+              regex: (.+)
+            - source_labels: [__meta_kubernetes_pod_ip, __meta_kubernetes_pod_annotation_signoz_io_port]
+              action: replace
+              separator: ":"
+              target_label: __address__
+            - target_label: job_name
+              replacement: signoz-scraper
+            {{- if .Values.presets.prometheusScraper.signoz.includePodLabel }}
+            - action: labelmap
+              regex: __meta_kubernetes_pod_label_(.+)
+            {{- end }}
+            - source_labels: [__meta_kubernetes_pod_label_app_kubernetes_io_name]
+              action: replace
+              target_label: signoz_k8s_name
+            - source_labels: [__meta_kubernetes_pod_label_app_kubernetes_io_instance]
+              action: replace
+              target_label: signoz_k8s_instance
+            - source_labels: [__meta_kubernetes_pod_label_app_kubernetes_io_component]
+              action: replace
+              target_label: signoz_k8s_component
+            - source_labels: [__meta_kubernetes_namespace]
+              action: replace
+              target_label: k8s_namespace_name
+            - source_labels: [__meta_kubernetes_pod_name]
+              action: replace
+              target_label: k8s_pod_name
+            - source_labels: [__meta_kubernetes_pod_uid]
+              action: replace
+              target_label: k8s_pod_uid
+            {{- if .Values.presets.prometheusScraper.signoz.includeContainerName }}
+            - source_labels: [__meta_kubernetes_pod_container_name]
+              action: replace
+              target_label: k8s_container_name
+            - source_labels: [__meta_kubernetes_pod_container_name]
+              regex: (.+)-init
+              action: drop
+            {{- end }}
+            - source_labels: [__meta_kubernetes_pod_node_name]
+              action: replace
+              target_label: k8s_node_name
+            - source_labels: [__meta_kubernetes_pod_ready]
+              action: replace
+              target_label: k8s_pod_ready
+            - source_labels: [__meta_kubernetes_pod_phase]
+              action: replace
+              target_label: k8s_pod_phase
+        {{- end }}
+        {{- if .Values.presets.prometheusScraper.prometheus.enabled }}
+        - job_name: "prometheus-scraper"
+          scrape_interval: {{ .Values.presets.prometheusScraper.prometheus.scrapeInterval }}
+          kubernetes_sd_configs:
+            - role: pod
+              {{- if or .Values.presets.prometheusScraper.prometheus.namespaceScoped (len .Values.presets.prometheusScraper.prometheus.namespaces) }}
+              namespaces:
+                {{- if .Values.presets.prometheusScraper.prometheus.namespaceScoped }}
+                own_namespace: true
+                {{- end }}
+                {{- if.Values.presets.prometheusScraper.prometheus.namespaces }}
+                names:
+                  {{- toYaml .Values.presets.prometheusScraper.prometheus.namespaces | nindent 16 }}
+                {{- end }}
+              {{- end }}
+          relabel_configs:
+            - source_labels: [__meta_kubernetes_pod_annotation_prometheus_io_scrape]
+              action: keep
+              regex: true
+            - source_labels: [__meta_kubernetes_pod_annotation_prometheus_io_path]
+              action: replace
+              target_label: __metrics_path__
+              regex: (.+)
+            - source_labels: [__meta_kubernetes_pod_ip, __meta_kubernetes_pod_annotation_prometheus_io_port]
+              action: replace
+              separator: ":"
+              target_label: __address__
+            - target_label: job_name
+              replacement: prometheus-scraper
+            {{- if .Values.presets.prometheusScraper.prometheus.includePodLabel }}
+            - action: labelmap
+              regex: __meta_kubernetes_pod_label_(.+)
+            {{- end }}
+            - source_labels: [__meta_kubernetes_pod_label_app_kubernetes_io_name]
+              action: replace
+              target_label: signoz_k8s_name
+            - source_labels: [__meta_kubernetes_pod_label_app_kubernetes_io_instance]
+              action: replace
+              target_label: signoz_k8s_instance
+            - source_labels: [__meta_kubernetes_pod_label_app_kubernetes_io_component]
+              action: replace
+              target_label: signoz_k8s_component
+            - source_labels: [__meta_kubernetes_namespace]
+              action: replace
+              target_label: k8s_namespace_name
+            - source_labels: [__meta_kubernetes_pod_name]
+              action: replace
+              target_label: k8s_pod_name
+            - source_labels: [__meta_kubernetes_pod_uid]
+              action: replace
+              target_label: k8s_pod_uid
+            {{- if .Values.presets.prometheusScraper.prometheus.includeContainerName }}
+            - source_labels: [__meta_kubernetes_pod_container_name]
+              action: replace
+              target_label: k8s_container_name
+            - source_labels: [__meta_kubernetes_pod_container_name]
+              regex: (.+)-init
+              action: drop
+            {{- end }}
+            - source_labels: [__meta_kubernetes_pod_node_name]
+              action: replace
+              target_label: k8s_node_name
+            - source_labels: [__meta_kubernetes_pod_ready]
+              action: replace
+              target_label: k8s_pod_ready
+            - source_labels: [__meta_kubernetes_pod_phase]
+              action: replace
+              target_label: k8s_pod_phase
+        {{- end }}
+{{- end }}
+
 {{- define "opentelemetry-collector.applyHostMetricsConfig" -}}
 {{- $config := mustMergeOverwrite (include "opentelemetry-collector.hostMetricsConfig" .Values | fromYaml) .config }}
 {{- if index $config.service.pipelines "metrics/internal" }}

charts/k8s-infra/templates/otel-agent/tests/test-connection.yaml

@@ -4,6 +4,7 @@ apiVersion: v1
 kind: Pod
 metadata:
   name: "{{ include "otelAgent.fullname" . }}-test-connection"
+  namespace: {{ include "k8s-infra.namespace" . }}
   annotations:
     "helm.sh/hook": test
 spec:

charts/k8s-infra/templates/otel-deployment/tests/test-connection.yaml

@@ -4,6 +4,7 @@ apiVersion: v1
 kind: Pod
 metadata:
   name: "{{ include "otelDeployment.fullname" . }}-test-connection"
+  namespace: {{ include "k8s-infra.namespace" . }}
   labels:
     {{- include "otelDeployment.labels" . | nindent 4 }}
   annotations:

charts/k8s-infra/values.yaml

@@ -270,6 +270,40 @@ presets:
       - memory
       # - ephemeral-storage
       # - storage
+  prometheusScraper:
+    enabled: true
+    signoz:
+      # -- Whether to enable metrics scraping using `signoz.io/scrape` pod annotation
+      enabled: true
+      # -- How often to scrape metrics
+      scrapeInterval: 60s
+      # -- Whether to only scrape metrics from pods in the same namespace
+      namespaceScoped: false
+      # -- If set, only scrape metrics from pods in the specified namespaces
+      namespaces: []
+      # -- This will include all pod labels in the metrics, could potentially cause
+      # performance issues with large number of pods with many labels
+      includePodLabel: false
+      # -- This is not recommended in case of multiple containers or init containers in a pod
+      # Since it will create multiple timeseries for the same pod metrics with different container names
+      # containers with `-init` suffix in the name will be ignored
+      includeContainerName: false
+    prometheus:
+      # -- Whether to enable metrics scraping using `prometheus.io/scrape` pod annotation
+      enabled: true
+      # -- How often to scrape metrics
+      scrapeInterval: 60s
+      # -- If enabled, only scrape metrics from pods in the same namespace
+      namespaceScoped: false
+      # -- If set, only scrape metrics from pods in the specified namespaces
+      namespaces: []
+      # -- This will include all pod labels in the metrics, could potentially cause
+      # performance issues with large number of pods with many labels
+      includePodLabel: false
+      # -- This is not recommended to be enabled in case of multiple containers or init containers in a pod
+      # Since it will create multiple timeseries for the same pod metrics with different container names
+      # containers with `-init` suffix in the name will be ignored
+      includeContainerName: false
   resourceDetectionInternal:
     enabled: true
     timeout: 2s
@@ -931,3 +965,7 @@ otelDeployment:
           receivers: []
           processors: [batch]
           exporters: []
+        metrics/scraper:
+          receivers: []
+          processors: [batch]
+          exporters: []