diff --git a/docs/ingest/telemetry/index.md b/docs/ingest/telemetry/index.md index b9967b85..408d1f78 100644 --- a/docs/ingest/telemetry/index.md +++ b/docs/ingest/telemetry/index.md @@ -54,6 +54,12 @@ scenarios. CrateDB offers corresponding integration adapters. :padding: 0 :gutter: 2 +::::{grid-item-card} collectd +:link: collectd +:link-type: ref +Send metrics with collectd, a system and application metrics collection daemon. +:::: + ::::{grid-item-card} Prometheus :link: prometheus :link-type: ref diff --git a/docs/integrate/collectd/Dockerfile b/docs/integrate/collectd/Dockerfile new file mode 100644 index 00000000..4001a460 --- /dev/null +++ b/docs/integrate/collectd/Dockerfile @@ -0,0 +1,10 @@ +FROM docker.io/debian:13-slim + +# Configure system environment. +ENV DEBIAN_FRONTEND=noninteractive + +# Install collectd. +RUN apt update --yes && apt install --yes collectd + +# Run collectd in foreground. +ENTRYPOINT ["collectd", "-f"] diff --git a/docs/integrate/collectd/collectd-cratedb.conf b/docs/integrate/collectd/collectd-cratedb.conf new file mode 100644 index 00000000..ecc314f0 --- /dev/null +++ b/docs/integrate/collectd/collectd-cratedb.conf @@ -0,0 +1,19 @@ +# collectd configuration for storing metrics into CrateDB. +# https://collectd.org/documentation/manpages/collectd.conf.html#plugin-postgresql + +LoadPlugin postgresql + + + + Statement "INSERT INTO doc.collectd_data (p_time, p_host, p_plugin, p_plugin_instance, p_type, p_type_instance, p_value_names, p_type_names, p_values) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9);" + StoreRates true + + + Writer sqlstore + # Commit every 30 seconds (adjust based on write volume) + CommitInterval 30 + Host cratedb + User crate + Password crate + + diff --git a/docs/integrate/collectd/collectd-telegraf.conf b/docs/integrate/collectd/collectd-telegraf.conf new file mode 100644 index 00000000..edabe539 --- /dev/null +++ b/docs/integrate/collectd/collectd-telegraf.conf @@ -0,0 +1,10 @@ +# collectd configuration for sending metrics to Telegraf. +# https://collectd.org/documentation/manpages/collectd.conf.html#plugin-network + +LoadPlugin network + + + Server "telegraf" + # Optional: ServerPort 25826 + # Optional: SecurityLevel "sign" or "encrypt" + diff --git a/docs/integrate/collectd/index.md b/docs/integrate/collectd/index.md new file mode 100644 index 00000000..7b423264 --- /dev/null +++ b/docs/integrate/collectd/index.md @@ -0,0 +1,67 @@ +(collectd)= +# collectd + +```{div} .float-right +[![collectd example graph](https://collectd.org/images/graph-cpu.png){width=180px loading=lazy}][collectd] +``` +```{div} .clearfix +``` + +:::{rubric} About +::: + +[collectd] is a system statistics collection daemon suitable for application +performance metrics periodically and provides mechanisms to store the values +in a variety of ways. + +collectd gathers metrics from various sources, e.g. the operating system, +applications, logfiles and external devices, and stores this information or +makes it available over the network. Those statistics can be used to monitor +systems, find performance bottlenecks (i.e. performance analysis) and predict +future system load (i.e. capacity planning). + +:::{rubric} Synopsis +::: + +Either use the `postgresql` plugin to store metrics into CrateDB, + +:::{literalinclude} collectd-cratedb.conf +::: + +or use the `network` plugin to forward metrics to Telegraf, then +using its built-in [CrateDB Output Plugin for Telegraf]. + +:::{literalinclude} collectd-telegraf.conf +::: + + +:::{rubric} Learn +::: + +::::{grid} + +:::{grid-item-card} Use collectd with CrateDB +:link: collectd-usage-base +:link-type: ref +How to configure collectd to submit metrics to CrateDB. +::: + +:::{grid-item-card} Use collectd with Telegraf and CrateDB +:link: collectd-usage-telegraf +:link-type: ref +How to configure collectd and Telegraf to submit metrics to CrateDB. +::: + +:::: + + +:::{toctree} +:maxdepth: 1 +:hidden: +Usage with collectd +Usage with Telegraf +::: + + +[collectd]: https://collectd.org/ +[CrateDB Output Plugin for Telegraf]: https://github.com/influxdata/telegraf/tree/master/plugins/outputs/cratedb diff --git a/docs/integrate/collectd/telegraf.conf b/docs/integrate/collectd/telegraf.conf new file mode 100644 index 00000000..08cddb27 --- /dev/null +++ b/docs/integrate/collectd/telegraf.conf @@ -0,0 +1,40 @@ +[[inputs.socket_listener]] + service_address = "udp://:25826" + + ## Data format to consume. + ## Each data format has its own unique set of configuration options, read + ## more about them here: + ## https://github.com/influxdata/telegraf/blob/master/docs/DATA_FORMATS_INPUT.md + data_format = "collectd" + + ## Authentication file for cryptographic security levels + # collectd_auth_file = "/etc/collectd/auth_file" + ## One of none (default), sign, or encrypt + # collectd_security_level = "encrypt" + ## Path of to TypesDB specifications + # collectd_typesdb = ["/usr/share/collectd/types.db"] + + ## Multi-value plugins can be handled two ways. + ## "split" will parse and store the multi-value plugin data into separate measurements + ## "join" will parse and store the multi-value plugin as a single multi-value measurement. + ## "split" is the default behavior for backward compatibility with previous versions of InfluxDB. + collectd_parse_multivalue = "split" + +# Configuration for CrateDB to send metrics to. +[[outputs.cratedb]] + ## Connection parameters for accessing the database see + ## https://pkg.go.dev/github.com/jackc/pgx/v4#ParseConfig + ## for available options + url = "postgres://crate:crate@cratedb/doc?sslmode=disable" + + ## Timeout for all CrateDB queries. + # timeout = "5s" + + ## Name of the table to store metrics in. + # table = "metrics" + + ## If true, and the metrics table does not exist, create it automatically. + table_create = true + + ## The character(s) to replace any '.' in an object key with + # key_separator = "_" diff --git a/docs/integrate/collectd/usage-collectd.md b/docs/integrate/collectd/usage-collectd.md new file mode 100644 index 00000000..2ed7a512 --- /dev/null +++ b/docs/integrate/collectd/usage-collectd.md @@ -0,0 +1,151 @@ +(collectd-usage-base)= +# Load data into CrateDB using collectd + +This usage guide shows how to configure and start [collectd] and CrateDB +so that collectd sends system metrics and CrateDB stores them. + +## Prerequisites + +Docker runs all components consistently across Linux, macOS, and Windows. +If you use Podman, substitute podman for docker in the commands. + +### Commands + +Prepare shortcut for the psql command. + +::::{tab-set} +:sync-group: os + +:::{tab-item} Linux and macOS +:sync: unix +To make the settings persistent, add them to your shell profile (`~/.profile`). +```shell +alias psql="docker run --rm -i --network=cratedb-demo docker.io/postgres:16 psql" +``` +::: +:::{tab-item} Windows PowerShell +:sync: powershell +To make the settings persistent, add them to your PowerShell profile (`$PROFILE`). +```powershell +function psql { docker run --rm -i --network=cratedb-demo docker.io/postgres:16 psql @args } +``` +::: +:::{tab-item} Windows Command +:sync: dos +```shell +doskey psql=docker run --rm -i --network=cratedb-demo docker.io/postgres:16 psql $* +``` +::: + +:::: + +### CrateDB + +Create a shared network. +```shell +docker network create cratedb-demo +``` + +Start CrateDB. +```shell +docker run --name=cratedb --rm -it --network=cratedb-demo \ + --publish=4200:4200 --publish=5432:5432 \ + --env=CRATE_HEAP_SIZE=2g docker.io/crate -Cdiscovery.type=single-node +``` + +## Configure + +### Provision database + +Create a database table that stores collected metrics. +```shell +psql "postgresql://crate:crate@cratedb:5432/" <