Skip to content

Commit 7f70c07

Browse files
amotlamotl
committed
OpenTelemetry: OTel Collector tutorial
1 parent 2274821 commit 7f70c07

File tree

8 files changed

+315
-0
lines changed

8 files changed

+315
-0
lines changed

‎docs/integrate/opentelemetry/collector/compose.yaml‎

Lines changed: 43 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,43 @@
1+
# Service composition file for Docker Compose or Podman Compose
2+
3+
services:
4+
5+
cratedb:
6+
image: crate/crate:latest
7+
ports:
8+
- "4200:4200/tcp"
9+
- "5432:5432/tcp"
10+
healthcheck:
11+
test: ["CMD", "curl", "-f", "http://localhost:4200/"]
12+
interval: 5s
13+
timeout: 30s
14+
retries: 5
15+
16+
cratedb-ddl:
17+
image: crate/crate:latest
18+
command: sh -c "crash --hosts http://cratedb:4200/ -c 'SELECT 1'; crash --hosts http://cratedb:4200/ < /var/ddl.sql"
19+
volumes:
20+
- ./ddl.sql:/var/ddl.sql
21+
depends_on:
22+
- cratedb
23+
24+
cratedb-prometheus-adapter:
25+
image: ghcr.io/crate/cratedb-prometheus-adapter:0.5.8
26+
command: -config.file /etc/cratedb-prometheus-adapter.yaml
27+
ports:
28+
- "9268:9268/tcp"
29+
volumes:
30+
- ./cratedb-prometheus-adapter.yaml:/etc/cratedb-prometheus-adapter.yaml
31+
depends_on:
32+
- cratedb-ddl
33+
34+
otelcol:
35+
image: ghcr.io/open-telemetry/opentelemetry-collector-releases/opentelemetry-collector-contrib:0.135.0
36+
ports:
37+
- "2003:2003/tcp" # Carbon plain text protocol based on TCP
38+
- "4317:4317/tcp" # OTLP gRPC
39+
- "4318:4318/tcp" # OTLP HTTP
40+
volumes:
41+
- ./otelcol.yaml:/etc/otelcol-contrib/config.yaml
42+
depends_on:
43+
- cratedb-prometheus-adapter

‎docs/integrate/opentelemetry/collector/cratedb-prometheus-adapter.yaml‎

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,20 @@
1+
cratedb_endpoints:
2+
- host: "cratedb" # Host to connect to (default: "localhost").
3+
port: 5432 # Port to connect to (default: 5432).
4+
user: "crate" # Username to use (default: "crate")
5+
password: "" # Password to use (default: "").
6+
schema: "testdrive" # Schema to use (default: "").
7+
max_connections: 0 # The maximum number of concurrent connections (default: runtime.NumCPU()).
8+
# It will get forwarded to pgx's `pool_max_conns`, and determines
9+
# the maximum number of connections in the connection pool for
10+
# both connection pools (read and write).
11+
read_pool_size_max: 0 # Configure the maximum pool size for read operations individually.
12+
# (default: runtime.NumCPU())
13+
write_pool_size_max: 0 # Configure the maximum pool size for write operations individually.
14+
# (default: runtime.NumCPU())
15+
connect_timeout: 10 # TCP connect timeout (seconds) (default: 10).
16+
# It has the same meaning as libpq's `connect_timeout`.
17+
read_timeout: 5 # Query context timeout for read queries (seconds) (default: 5).
18+
write_timeout: 5 # Query context timeout for write queries (seconds) (default: 5).
19+
enable_tls: false # Whether to connect using TLS (default: false).
20+
allow_insecure_tls: false # Whether to allow insecure / invalid TLS certificates (default: false).

‎docs/integrate/opentelemetry/collector/ddl.sql‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,9 @@
1+
CREATE TABLE IF NOT EXISTS "testdrive"."metrics" (
2+
"timestamp" TIMESTAMP,
3+
"labels_hash" STRING,
4+
"labels" OBJECT(DYNAMIC),
5+
"value" DOUBLE,
6+
"valueRaw" LONG,
7+
"day__generated" TIMESTAMP GENERATED ALWAYS AS date_trunc('day', "timestamp"),
8+
PRIMARY KEY ("timestamp", "labels_hash", "day__generated")
9+
) PARTITIONED BY ("day__generated");

‎docs/integrate/opentelemetry/collector/example.py‎

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,16 @@
1+
# OpenTelemetry demo application.
2+
from opentelemetry import metrics
3+
4+
5+
def main():
6+
7+
meter = metrics.get_meter("testdrive.meter.name")
8+
temperature = meter.create_gauge("temperature")
9+
humidity = meter.create_gauge("humidity")
10+
11+
temperature.set(42.42)
12+
humidity.set(84.84)
13+
14+
15+
if __name__ == "__main__":
16+
main()

‎docs/integrate/opentelemetry/collector/otelcol.yaml‎

Lines changed: 54 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,54 @@
1+
# OpenTelemetry Collector configuration file
2+
#
3+
# https://github.com/open-telemetry/opentelemetry-collector-releases/blob/main/distributions/otelcol-contrib/config.yaml
4+
# https://github.com/open-telemetry/opentelemetry-collector-contrib/tree/main/exporter/prometheusremotewriteexporter
5+
#
6+
# To limit exposure to denial-of-service attacks, change the host in endpoints below from 0.0.0.0 to a specific network interface.
7+
# See https://github.com/open-telemetry/opentelemetry-collector/blob/main/docs/security-best-practices.md#safeguards-against-denial-of-service-attacks
8+
9+
receivers:
10+
otlp:
11+
protocols:
12+
grpc:
13+
endpoint: 0.0.0.0:4317
14+
http:
15+
endpoint: 0.0.0.0:4318
16+
carbon:
17+
endpoint: 0.0.0.0:2003
18+
transport: tcp
19+
parser:
20+
type: plaintext
21+
config:
22+
23+
processors:
24+
batch:
25+
26+
exporters:
27+
debug:
28+
verbosity: detailed
29+
30+
prometheusremotewrite:
31+
endpoint: "http://cratedb-prometheus-adapter:9268/write"
32+
remote_write_queue:
33+
enabled: false
34+
external_labels:
35+
subsystem: "otel-testdrive"
36+
37+
service:
38+
39+
pipelines:
40+
41+
traces:
42+
receivers: [otlp]
43+
processors: [batch]
44+
exporters: [debug]
45+
46+
metrics:
47+
receivers: [otlp, carbon]
48+
processors: [batch]
49+
exporters: [debug, prometheusremotewrite]
50+
51+
logs:
52+
receivers: [otlp]
53+
processors: [batch]
54+
exporters: [debug]

‎docs/integrate/opentelemetry/collector/signal.json‎

Lines changed: 41 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,41 @@
1+
{
2+
"node": {
3+
"identifier": {
4+
"host_name": "hostname",
5+
"pid": 123,
6+
"start_timestamp": "2021-03-10T08:16:43.333Z"
7+
},
8+
"library_info": {
9+
"language": "LANGUAGE_UNSPECIFIED",
10+
"exporter_version": "1.0",
11+
"core_library_version": "1.0"
12+
},
13+
"service_info": {
14+
"name": "app"
15+
},
16+
"attributes": {
17+
}
18+
},
19+
"metrics": [
20+
{
21+
"metric_descriptor": {
22+
"name": "example",
23+
"description": "Description of the metric",
24+
"unit": "float",
25+
"type": "GAUGE_INT64",
26+
"label_keys": [
27+
{
28+
"key": "label1",
29+
"description": "Label one"
30+
}
31+
]
32+
}
33+
}
34+
],
35+
"resource": {
36+
"type": "metric",
37+
"labels": {
38+
"property": "example1"
39+
}
40+
}
41+
}

‎docs/integrate/opentelemetry/collector/tutorial.md‎

Lines changed: 115 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,115 @@
1+
(opentelemetry-tutorial-otelcol)=
2+
# Connect the OpenTelemetry Collector to CrateDB
3+
4+
This tutorial walks you through configuring the [OpenTelemetry Collector],
5+
its built-in [Prometheus Remote Write Exporter], and the
6+
[CrateDB Prometheus Adapter], to receive [OpenTelemetry] [metrics] and
7+
store them into CrateDB.
8+
9+
## Prerequisites
10+
11+
Docker is used for running all components. This approach works consistently
12+
across Linux, macOS, and Windows. Alternatively, you can use Podman.
13+
14+
### Commands
15+
16+
Prepare shortcut for {ref}`crate-crash:index` command.
17+
18+
::::{tab-set}
19+
:sync-group: os
20+
21+
:::{tab-item} Linux and macOS
22+
:sync: unix
23+
Add these settings to your shell profile (`~/.profile`) to make them persistent.
24+
```shell
25+
alias crash="docker run --rm -it --network=host crate/crate:latest crash"
26+
```
27+
:::
28+
:::{tab-item} Windows PowerShell
29+
:sync: powershell
30+
Add these settings to your PowerShell profile (`$PROFILE`) to make them persistent.
31+
```powershell
32+
function crash { docker run --rm -i --network=host crate/crate:latest crash @args }
33+
```
34+
:::
35+
:::{tab-item} Windows Command
36+
:sync: dos
37+
```shell
38+
doskey crash=docker run --rm -i --network=host crate/crate:latest crash $*
39+
```
40+
:::
41+
42+
::::
43+
44+
### Services
45+
46+
Save the files {download}`compose.yaml`,
47+
{download}`cratedb-prometheus-adapter.yaml`,
48+
{download}`otelcol.yaml` and {download}`ddl.sql`
49+
to your machine and start all services using
50+
Docker Compose or Podman Compose.
51+
```shell
52+
docker compose up
53+
```
54+
55+
## Submit data
56+
57+
### Use netcat
58+
59+
Use [netcat] to submit metrics using the [Carbon plaintext protocol].
60+
```shell
61+
printf "temperature;job=app 42.42 1758486061\nhumidity;job=app 84.84 1758486061" \
62+
| nc -c localhost 2003
63+
```
64+
65+
### Use Python
66+
67+
Use the OTel Python language SDK to submit metrics. To do that,
68+
save the Python OTel example file {download}`example.py` to your machine and
69+
use the `opentelemetry-instrument` program to invoke your Python application.
70+
```shell
71+
opentelemetry-instrument --service_name=app python example.py
72+
```
73+
:::{literalinclude} example.py
74+
:::
75+
The [uv] utility can invoke the demo program including dependencies,
76+
otherwise install them using `pip install opentelemetry-distro opentelemetry-exporter-otlp`
77+
or similarly.
78+
```shell
79+
uv run --with=opentelemetry-distro --with=opentelemetry-exporter-otlp \
80+
opentelemetry-instrument --service_name=app python example.py
81+
```
82+
83+
### Use any language
84+
85+
Use any of the available [OpenTelemetry language APIs & SDKs] for C++, C#/.NET,
86+
Erlang/Elixir, Go, Java, JavaScript, PHP, Python, Ruby, Rust, or Swift.
87+
88+
## Explore data
89+
90+
CrateDB stored the metrics in the designated table, ready for inspection and analysis.
91+
```shell
92+
crash --hosts "http://crate:crate@localhost:4200/" \
93+
-c "SELECT * FROM testdrive.metrics ORDER BY timestamp LIMIT 5;"
94+
```
95+
```psql
96+
+---------------+------------------+----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+-------+---------------------+----------------+
97+
| timestamp | labels_hash | labels | value | valueRaw | day__generated |
98+
+---------------+------------------+----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+-------+---------------------+----------------+
99+
| 1758480857158 | 64614d7f1ef80933 | {"__name__": "target_info", "job": "app", "subsystem": "otel-testdrive", "telemetry_auto_version": "0.58b0", "telemetry_sdk_language": "python", "telemetry_sdk_name": "opentelemetry", "telemetry_sdk_version": "1.37.0"} | 1.0 | 4607182418800017408 | 1758412800000 |
100+
| 1758480857158 | 7c6f57205e58af4c | {"__name__": "temperature", "job": "app", "subsystem": "otel-testdrive"} | 42.42 | 4631166901565532406 | 1758412800000 |
101+
| 1758480857158 | 3fce270356467381 | {"__name__": "humidity", "job": "app", "subsystem": "otel-testdrive"} | 84.84 | 4635670501192902902 | 1758412800000 |
102+
+---------------+------------------+----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+-------+---------------------+----------------+
103+
SELECT 3 rows in set (0.005 sec)
104+
```
105+
106+
107+
[Carbon plaintext protocol]: https://graphite.readthedocs.io/en/latest/feeding-carbon.html
108+
[CrateDB Prometheus Adapter]: https://github.com/crate/cratedb-prometheus-adapter
109+
[metrics]: https://opentelemetry.io/docs/concepts/signals/metrics/
110+
[netcat]: https://en.wikipedia.org/wiki/Netcat
111+
[OpenTelemetry]: https://opentelemetry.io/docs/what-is-opentelemetry/
112+
[OpenTelemetry Collector]: https://opentelemetry.io/docs/collector/
113+
[OpenTelemetry language APIs & SDKs]: https://opentelemetry.io/docs/languages/
114+
[Prometheus Remote Write Exporter]: https://github.com/open-telemetry/opentelemetry-collector-contrib/tree/main/exporter/prometheusremotewriteexporter
115+
[uv]: https://docs.astral.sh/uv/

‎docs/integrate/opentelemetry/index.md‎

Lines changed: 17 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -37,6 +37,23 @@ Configure OpenTelemetry Collector to send metrics data to the [CrateDB Prometheu
3737
:::{rubric} Learn
3838
:::
3939

40+
::::{grid}
41+
42+
:::{grid-item-card} Tutorial: Use OpenTelemetry with CrateDB
43+
:link: opentelemetry-tutorial-otelcol
44+
:link-type: ref
45+
How to configure OpenTelemetry Collector to submit metrics to CrateDB.
46+
:::
47+
48+
::::
49+
50+
51+
:::{toctree}
52+
:maxdepth: 1
53+
:hidden:
54+
Collector Tutorial <collector/tutorial>
55+
:::
56+
4057

4158
[CrateDB Prometheus Adapter]: https://github.com/crate/cratedb-prometheus-adapter
4259
[logs]: https://opentelemetry.io/docs/concepts/signals/logs/

0 commit comments

Comments
 (0)