From 126ea166ae5ac43903389e2e3de2fdd3267c4ef0 Mon Sep 17 00:00:00 2001 From: Matej Gera Date: Mon, 27 Nov 2023 16:59:03 +0100 Subject: [PATCH 1/5] Remove spanmetricsprocessor; add preset Signed-off-by: Matej Gera --- otel-integration/k8s-helm/values.yaml | 15 ++------------- 1 file changed, 2 insertions(+), 13 deletions(-) diff --git a/otel-integration/k8s-helm/values.yaml b/otel-integration/k8s-helm/values.yaml index 5dfd2778..81546156 100644 --- a/otel-integration/k8s-helm/values.yaml +++ b/otel-integration/k8s-helm/values.yaml @@ -89,6 +89,8 @@ opentelemetry-agent: enabled: true kubeletMetrics: enabled: true + spanMetrics: + enabled: false config: extensions: @@ -153,18 +155,6 @@ opentelemetry-agent: - "k8s.job.name" - "k8s.pod.name" - "k8s.node.name" - spanmetrics: - metrics_exporter: coralogix - dimensions: - # replace the below by `k8s.deployment.name` after https://github.com/open-telemetry/opentelemetry-collector-contrib/issues/23067 - - name: "k8s.replicaset.name" - - name: "k8s.statefulset.name" - - name: "k8s.daemonset.name" - - name: "k8s.cronjob.name" - - name: "k8s.job.name" - - name: "k8s.container.name" - - name: "k8s.node.name" - - name: "k8s.namespace.name" # Will get the k8s resource limits memory_limiter: null @@ -220,7 +210,6 @@ opentelemetry-agent: - resourcedetection/env - resourcedetection/region - memory_limiter - - spanmetrics - batch receivers: - otlp From 9594b09127439a405250c8b29be6496f3e4ed584 Mon Sep 17 00:00:00 2001 From: Matej Gera Date: Mon, 27 Nov 2023 16:59:13 +0100 Subject: [PATCH 2/5] Update README Signed-off-by: Matej Gera --- otel-integration/k8s-helm/README.md | 23 ++++++++++++++++++++--- 1 file changed, 20 insertions(+), 3 deletions(-) diff --git a/otel-integration/k8s-helm/README.md b/otel-integration/k8s-helm/README.md index f0cdd4b1..865532d9 100644 --- a/otel-integration/k8s-helm/README.md +++ b/otel-integration/k8s-helm/README.md @@ -48,7 +48,7 @@ This cluster collector provides: ## Kubernetes Dashboard -This chart will also collect, out of the box, all the metrics necessary for [Coralogix Kubernetes Monitoring](https://coralogix.com/docs/apm-kubernetes/), which will allow you to monitor your Kubernetes cluster and applications. To do this, it is necessary to deploy the [Kube-state-metrics](https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-state-metrics) component, which makes it possible to obtain some of these extra metrics. +This chart will also collect, out of the box, all the metrics necessary for [Coralogix Kubernetes Monitoring](https://coralogix.com/docs/apm-kubernetes/), which will allow you to monitor your Kubernetes cluster and applications. **Please be aware** that certain metrics collected by for the dashboard have high cardinality, which means that the number of unique values for a given metric is high and might result in higher costs connected with metrics ingestion and storage. This is applies in particular to the pod related metrics `kube_pod_status_reason`, `kube_pod_status_phase` and `kube_pod_status_qos_class`. @@ -137,7 +137,7 @@ helm upgrade --install otel-coralogix-integration coralogix-charts-virtual/otel- ### Installing the chart on clusters with mixed operating systems (Linux and Windows) -Installing `otel-integration` is also possible on clusters that support running Windows workloads on Windows node alongside Linux nodes (such as [EKS](https://docs.aws.amazon.com/eks/latest/userguide/windows-support.html), [AKS](https://learn.microsoft.com/en-us/azure/aks/windows-faq?tabs=azure-cli) or [GKE](https://cloud.google.com/kubernetes-engine/docs/how-to/creating-a-cluster-windows)). The `kube-state-metrics` and collector will be installed on Linux nodes, as these components are supported only on Linux operating systems. Conversely, the agent will be installed on both Linux and Windows nodes as a daemonset, in order to collect metrics for both operating systems. In order to do so, the chart needs to be installed with few adjustments. +Installing `otel-integration` is also possible on clusters that support running Windows workloads on Windows node alongside Linux nodes (such as [EKS](https://docs.aws.amazon.com/eks/latest/userguide/windows-support.html), [AKS](https://learn.microsoft.com/en-us/azure/aks/windows-faq?tabs=azure-cli) or [GKE](https://cloud.google.com/kubernetes-engine/docs/how-to/creating-a-cluster-windows)). The collector will be installed on Linux nodes, as these components are supported only on Linux operating systems. Conversely, the agent will be installed on both Linux and Windows nodes as a daemonset, in order to collect metrics for both operating systems. In order to do so, the chart needs to be installed with few adjustments. First make sure to add our Helm charts repository to the local repos list with the following command: @@ -189,6 +189,23 @@ env: value: "$(NODE):4317" ``` +### About span metrics +The collector provides a possibility to synthesize R.E.D (Request, Error, Duration) metrics based on the incoming span data. This can be useful to obtain extra metrics about the operations you have instrumented for tracing. For more information, please refer to the [OpenTelemetry Collector documentation](https://github.com/open-telemetry/opentelemetry-collector-contrib/blob/main/connector/spanmetricsconnector/README.md). + +This feature is disabled by default and can be enabled by setting the `spanmetrics.enabled` value to `true` in the `values.yaml` file. + +Beware that enabling the feature will result in creation of additional metrics. Depending on how you instrument your applications, this can result in a significant increase in the number of metrics. This is especially true for cases where the span name includes specific values, such as user IDs or UUIDs. Such instrumentation practice is [strongly discouraged](https://github.com/open-telemetry/opentelemetry-specification/blob/main/specification/trace/api.md#span). + +In such cases, we recommend to either correct your instrumentation or to use the `spanMetrics.spanNameReplacePattern` parameter, to replace the problematic values with a generic placeholder. For example, if your span name corresponds to template `user-1234`, you can use the following pattern to replace the user ID with a generic placeholder. See the following configuration: + +```yaml +spanNameReplacePattern: +- regex: "user-[0-9]+" + replacement: "user-{id}" +``` + +This will result in your spans having generalized name `user-{id}`. + # Performance of the Collector ## Picking the right tracing SDK span processor @@ -348,4 +365,4 @@ Optional settings: # Dependencies -This chart uses [openetelemetry-collector](https://github.com/coralogix/opentelemetry-helm-charts/tree/main/charts/opentelemetry-collector) Helm chart. Also this chart currently depends on the [`kube-state-metrics`](https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-state-metrics) chart to collect extra Kubernetes metrics. +This chart uses [openetelemetry-collector](https://github.com/coralogix/opentelemetry-helm-charts/tree/main/charts/opentelemetry-collector) Helm chart. From 8abb5917b78d291d2dcc461cb01b31f534aefa26 Mon Sep 17 00:00:00 2001 From: Matej Gera Date: Mon, 27 Nov 2023 16:59:32 +0100 Subject: [PATCH 3/5] Bump chart version, add CHANGELOG Signed-off-by: Matej Gera --- otel-integration/CHANGELOG.md | 5 +++++ otel-integration/k8s-helm/Chart.yaml | 8 ++++---- 2 files changed, 9 insertions(+), 4 deletions(-) diff --git a/otel-integration/CHANGELOG.md b/otel-integration/CHANGELOG.md index f7497bac..22e17691 100644 --- a/otel-integration/CHANGELOG.md +++ b/otel-integration/CHANGELOG.md @@ -2,6 +2,11 @@ ## OpenTelemtry-Integration +### v0.0.37 / 2023-11-27 +* [:warning: BREAKING CHANGE] [FEATURE] Add support for span metrics preset. This replaces the deprecated `spanmetricsprocessor` +with `spanmetricsconnector`. The new connector is disabled by default, as opposed the replaces processor. +To enable it, set `presets.spanMetrics.enabled` to `true`. + ### v0.0.36 / 2023-11-15 * [FIX] Change statsd receiver port to 8125 instead of 8127 diff --git a/otel-integration/k8s-helm/Chart.yaml b/otel-integration/k8s-helm/Chart.yaml index a953e437..8b0b2725 100644 --- a/otel-integration/k8s-helm/Chart.yaml +++ b/otel-integration/k8s-helm/Chart.yaml @@ -1,7 +1,7 @@ apiVersion: v2 name: otel-integration description: OpenTelemetry Integration -version: 0.0.36 +version: 0.0.37 keywords: - OpenTelemetry Collector - OpenTelemetry Agent @@ -11,17 +11,17 @@ keywords: dependencies: - name: opentelemetry-collector alias: opentelemetry-agent - version: "0.77.0" + version: "0.77.2" repository: https://cgx.jfrog.io/artifactory/coralogix-charts-virtual condition: opentelemetry-agent.enabled - name: opentelemetry-collector alias: opentelemetry-agent-windows - version: "0.77.0" + version: "0.77.2" repository: https://cgx.jfrog.io/artifactory/coralogix-charts-virtual condition: opentelemetry-agent-windows.enabled - name: opentelemetry-collector alias: opentelemetry-cluster-collector - version: "0.77.0" + version: "0.77.2" repository: https://cgx.jfrog.io/artifactory/coralogix-charts-virtual condition: opentelemetry-cluster-collector.enabled sources: From f41207e716f9794696417cd87cc7a95a1e828bed Mon Sep 17 00:00:00 2001 From: Matej Gera Date: Tue, 28 Nov 2023 12:34:03 +0100 Subject: [PATCH 4/5] Add UPGRADING section Signed-off-by: Matej Gera --- otel-integration/UPGRADING.md | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/otel-integration/UPGRADING.md b/otel-integration/UPGRADING.md index 50506469..52018224 100644 --- a/otel-integration/UPGRADING.md +++ b/otel-integration/UPGRADING.md @@ -4,6 +4,18 @@ These upgrade guidelines only contain instructions for version upgrades which re If the version you want to upgrade to is not listed here, then there is nothing to do for you. Just upgrade and enjoy. +## 0.0.36 to 0.0.37 + +From version `0.0.37`, the deprecated `spanmetricsprocessor` has been removed and replaced by the `spanmetricsconnector` and is **disabled** by default. If you depend on the span metrics, please enable the `spanmetricsconnector` by setting the following in the `presets` section of your configuration: + +```yaml +opentelemetry-agent: + presets: + spanMetrics: + enabled: true +``` +Please beware there are also breaking changes with regards to metric naming and labels. Please see the [this section of documentation](https://github.com/open-telemetry/opentelemetry-collector-contrib/blob/main/connector/spanmetricsconnector/README.md#span-to-metrics-processor-to-span-to-metrics-connector) for more information. + ## 0.0.32 to 0.0.33 Opentelemetry Integration v0.33 has removed Kube State Metrics deployment, as the needed resource attributes and metrics are now part of OpenTelemetry Kubernetes Cluster Receiver. From e8cfeade6cf7de87dc2c3ce34ae70357f2e85246 Mon Sep 17 00:00:00 2001 From: Matej Gera Date: Tue, 28 Nov 2023 12:39:19 +0100 Subject: [PATCH 5/5] Formatting fixes Signed-off-by: Matej Gera --- logs/CHANGELOG.md | 2 +- otel-integration/CHANGELOG.md | 4 ++-- otel-integration/UPGRADING.md | 1 + otel-integration/k8s-helm/README.md | 3 ++- 4 files changed, 6 insertions(+), 4 deletions(-) diff --git a/logs/CHANGELOG.md b/logs/CHANGELOG.md index 1c9eb8ed..65bf0a57 100644 --- a/logs/CHANGELOG.md +++ b/logs/CHANGELOG.md @@ -5,7 +5,7 @@ ### v1.16.3 / 2023-11-23 * [CHANGE] Update the coralogix API - + ### v1.16.3 / 2023-11-22 * [UPGRADE] Upgrade Fluentd version to v1.16.3 diff --git a/otel-integration/CHANGELOG.md b/otel-integration/CHANGELOG.md index 22e17691..780fed73 100644 --- a/otel-integration/CHANGELOG.md +++ b/otel-integration/CHANGELOG.md @@ -4,8 +4,8 @@ ### v0.0.37 / 2023-11-27 * [:warning: BREAKING CHANGE] [FEATURE] Add support for span metrics preset. This replaces the deprecated `spanmetricsprocessor` -with `spanmetricsconnector`. The new connector is disabled by default, as opposed the replaces processor. -To enable it, set `presets.spanMetrics.enabled` to `true`. + with `spanmetricsconnector`. The new connector is disabled by default, as opposed the replaces processor. + To enable it, set `presets.spanMetrics.enabled` to `true`. ### v0.0.36 / 2023-11-15 * [FIX] Change statsd receiver port to 8125 instead of 8127 diff --git a/otel-integration/UPGRADING.md b/otel-integration/UPGRADING.md index 52018224..925cf060 100644 --- a/otel-integration/UPGRADING.md +++ b/otel-integration/UPGRADING.md @@ -14,6 +14,7 @@ opentelemetry-agent: spanMetrics: enabled: true ``` + Please beware there are also breaking changes with regards to metric naming and labels. Please see the [this section of documentation](https://github.com/open-telemetry/opentelemetry-collector-contrib/blob/main/connector/spanmetricsconnector/README.md#span-to-metrics-processor-to-span-to-metrics-connector) for more information. ## 0.0.32 to 0.0.33 diff --git a/otel-integration/k8s-helm/README.md b/otel-integration/k8s-helm/README.md index 865532d9..04d480ab 100644 --- a/otel-integration/k8s-helm/README.md +++ b/otel-integration/k8s-helm/README.md @@ -190,11 +190,12 @@ env: ``` ### About span metrics + The collector provides a possibility to synthesize R.E.D (Request, Error, Duration) metrics based on the incoming span data. This can be useful to obtain extra metrics about the operations you have instrumented for tracing. For more information, please refer to the [OpenTelemetry Collector documentation](https://github.com/open-telemetry/opentelemetry-collector-contrib/blob/main/connector/spanmetricsconnector/README.md). This feature is disabled by default and can be enabled by setting the `spanmetrics.enabled` value to `true` in the `values.yaml` file. -Beware that enabling the feature will result in creation of additional metrics. Depending on how you instrument your applications, this can result in a significant increase in the number of metrics. This is especially true for cases where the span name includes specific values, such as user IDs or UUIDs. Such instrumentation practice is [strongly discouraged](https://github.com/open-telemetry/opentelemetry-specification/blob/main/specification/trace/api.md#span). +Beware that enabling the feature will result in creation of additional metrics. Depending on how you instrument your applications, this can result in a significant increase in the number of metrics. This is especially true for cases where the span name includes specific values, such as user IDs or UUIDs. Such instrumentation practice is [strongly discouraged](https://github.com/open-telemetry/opentelemetry-specification/blob/main/specification/trace/api.md#span). In such cases, we recommend to either correct your instrumentation or to use the `spanMetrics.spanNameReplacePattern` parameter, to replace the problematic values with a generic placeholder. For example, if your span name corresponds to template `user-1234`, you can use the following pattern to replace the user ID with a generic placeholder. See the following configuration: