From c571e7122347850a00daf5e9f74224aeb6fa3b72 Mon Sep 17 00:00:00 2001 From: george-dorin Date: Tue, 2 Sep 2025 16:59:34 +0300 Subject: [PATCH 1/8] Add sd-bridge script --- framework/.changeset/v0.10.17.md | 1 + .../observability/compose/conf/prometheus.yml | 4 + .../observability/compose/docker-compose.yaml | 17 +++ .../observability/compose/scripts/sd-merge.sh | 110 ++++++++++++++++++ 4 files changed, 132 insertions(+) create mode 100644 framework/.changeset/v0.10.17.md create mode 100644 framework/observability/compose/scripts/sd-merge.sh diff --git a/framework/.changeset/v0.10.17.md b/framework/.changeset/v0.10.17.md new file mode 100644 index 000000000..32c9c4040 --- /dev/null +++ b/framework/.changeset/v0.10.17.md @@ -0,0 +1 @@ +- Added SD-bridge script that queries each node’s /discovery endpoint and generates Prometheus targets, enabling automatic collection of LOOPP metrics \ No newline at end of file diff --git a/framework/observability/compose/conf/prometheus.yml b/framework/observability/compose/conf/prometheus.yml index 290aefe2b..005be18d0 100644 --- a/framework/observability/compose/conf/prometheus.yml +++ b/framework/observability/compose/conf/prometheus.yml @@ -35,3 +35,7 @@ scrape_configs: - job_name: 'postgres_exporter_4' static_configs: - targets: ['postgres_exporter_4:9187'] + - job_name: 'node-sd' + file_sd_configs: + - files: ["/etc/prometheus/targets/merged.json"] + refresh_interval: 15s \ No newline at end of file diff --git a/framework/observability/compose/docker-compose.yaml b/framework/observability/compose/docker-compose.yaml index ae915b3cf..1decec8e6 100644 --- a/framework/observability/compose/docker-compose.yaml +++ b/framework/observability/compose/docker-compose.yaml @@ -45,6 +45,7 @@ services: volumes: - /var/run/docker.sock:/var/run/docker.sock - ./conf/prometheus.yml:/etc/prometheus/prometheus.yml + - sd-targets:/etc/prometheus/targets ports: - '9099:9090' @@ -135,6 +136,21 @@ services: - '9304:9187' restart: unless-stopped + sd-merge: + image: alpine:3.20 + command: [ "/bin/sh","-c","apk add --no-cache bash curl jq docker-cli && exec bash scripts/sd-merge.sh" ] + volumes: + - /var/run/docker.sock:/var/run/docker.sock:ro + - sd-targets:/out + - ./scripts:/scripts:ro + environment: + LABEL_MATCH: "framework=ctf" + DISCOVERY_PATH: "/discovery" + DISCOVERY_PORT: "6688" + DISCOVERY_SCHEME: "http" + OUT: "/out/merged.json" + SLEEP: "15" + volumes: loki_data: grafana_data: @@ -142,6 +158,7 @@ volumes: grafana_logs: grafana_plugins: tempo_data: + sd-targets: networks: default: diff --git a/framework/observability/compose/scripts/sd-merge.sh b/framework/observability/compose/scripts/sd-merge.sh new file mode 100644 index 000000000..99a175d9c --- /dev/null +++ b/framework/observability/compose/scripts/sd-merge.sh @@ -0,0 +1,110 @@ +#!/usr/bin/env bash +# sd-merge.sh +# Discover Docker containers by label, fetch each container's /discovery JSON, +# add labels (container_name, scrape_path), merge + dedupe, and write a single file_sd JSON. + +set -Eeuo pipefail + +# -------------------- Configuration (via env) -------------------- +LABEL_MATCH="${LABEL_MATCH:-framework=ctf}" # docker ps --filter "label=$LABEL_MATCH" +DEFAULT_PATH="${DISCOVERY_PATH:-/discovery}" # default discovery path inside each container +DEFAULT_PORT="${DISCOVERY_PORT:-6688}" # default discovery port +DEFAULT_SCHEME="${DISCOVERY_SCHEME:-http}" # http or https +PREFER_NETWORK="${NETWORK_NAME:-}" # prefer IP from this Docker network (optional) +OUT="${OUT:-/out/merged.json}" # file_sd output path +SLEEP="${SLEEP:-15}" # seconds between scans +REQUEST_TIMEOUT="${REQUEST_TIMEOUT:-5}" # curl timeout (s) +REWRITE_TO_IP="${REWRITE_TO_IP:-0}" # 1 = replace host with container IP in targets + +# -------------------- Helpers -------------------- +log(){ printf '[sd-merge] %s\n' "$*" >&2; } + +# Atomic writer: reads stdin, writes to $1.tmp, then mv -> $1 +atomic_write(){ + local path="$1" tmp="$1.tmp" + cat > "$tmp" && mv "$tmp" "$path" +} + +# -------------------- Init -------------------- +mkdir -p "$(dirname "$OUT")" +echo '[]' | atomic_write "$OUT" + +# -------------------- Main loop -------------------- +while true; do + # List container IDs matching the label + mapfile -t cids < <(docker ps -q --filter "label=$LABEL_MATCH" || true) + + if ((${#cids[@]} == 0)); then + echo '[]' | atomic_write "$OUT" + log "no matching containers; wrote empty array" + sleep "$SLEEP" + continue + fi + + # Emit each container's (possibly empty) discovery array, then merge once with jq -s + { + for cid in "${cids[@]}"; do + # Inspect once, reuse for IP, name, and labels + inspect="$(docker inspect "$cid" 2>/dev/null || true)" + [[ -z "$inspect" ]] && { log "skip ${cid:0:12}: inspect failed"; echo '[]'; continue; } + + # Resolve container IP (optionally prefer a specific network) + if [[ -n "$PREFER_NETWORK" ]]; then + ip="$(jq -r --arg n "$PREFER_NETWORK" '.[0].NetworkSettings.Networks[$n].IPAddress // ""' <<<"$inspect")" + else + ip="$(jq -r '.[0].NetworkSettings.Networks | to_entries[0].value.IPAddress // ""' <<<"$inspect")" + fi + [[ -z "$ip" ]] && { log "skip ${cid:0:12}: no IP"; echo '[]'; continue; } + + # Container name and optional per-container overrides + name="$(jq -r '.[0].Name | ltrimstr("/")' <<<"$inspect")" + path="$(jq -r '.[0].Config.Labels.prom_sd_path // empty' <<<"$inspect")"; path="${path:-$DEFAULT_PATH}" + port="$(jq -r '.[0].Config.Labels.prom_sd_port // empty' <<<"$inspect")"; port="${port:-$DEFAULT_PORT}" + scheme="$(jq -r '.[0].Config.Labels.prom_sd_scheme // empty' <<<"$inspect")"; scheme="${scheme:-$DEFAULT_SCHEME}" + + url="${scheme}://${ip}:${port}${path}" + + # Fetch discovery JSON; treat errors as empty array + payload="$(curl -fsSL --max-time "$REQUEST_TIMEOUT" "$url" 2>/dev/null || echo '[]')" + + # Normalize to array, add labels, optional host->IP rewrite while keeping port from targets + if [[ "$REWRITE_TO_IP" == "1" ]]; then + jq --arg ip "$ip" --arg name "$name" ' + (if type=="array" then . else [] end) + | map( + .targets |= map( $ip + ":" + (split(":")[1]) ) | + .labels = ((.labels // {}) + { + container_name: $name, + scrape_path: (.labels.__metrics_path__ // "") + }) + ) + ' <<<"$payload" + else + jq --arg name "$name" ' + (if type=="array" then . else [] end) + | map( + .labels = ((.labels // {}) + { + container_name: $name, + scrape_path: (.labels.__metrics_path__ // "") + }) + ) + ' <<<"$payload" + fi + + log "ok $url" + done + } \ + | jq -s ' + # Merge all arrays, coerce to {targets,labels}, then group by labels and dedupe targets + add // [] + | map({targets: (.targets // []), labels: (.labels // {})}) + | group_by(.labels) + | map({ labels: (.[0].labels) + , targets: ([.[].targets[]] | unique | sort) + }) + ' \ + | atomic_write "$OUT" + + log "wrote $(wc -c < "$OUT") bytes to $OUT" + sleep "$SLEEP" +done From 4b432ff37932d0eace8b3b15d23f215f18c3454c Mon Sep 17 00:00:00 2001 From: george-dorin Date: Tue, 2 Sep 2025 17:01:45 +0300 Subject: [PATCH 2/8] Update changeset --- framework/.changeset/{v0.10.17.md => v0.10.18.md} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename framework/.changeset/{v0.10.17.md => v0.10.18.md} (100%) diff --git a/framework/.changeset/v0.10.17.md b/framework/.changeset/v0.10.18.md similarity index 100% rename from framework/.changeset/v0.10.17.md rename to framework/.changeset/v0.10.18.md From 2959c01019b259ae077acba3b1061392e10d2dc7 Mon Sep 17 00:00:00 2001 From: george-dorin Date: Wed, 3 Sep 2025 12:33:30 +0300 Subject: [PATCH 3/8] Update changeset --- framework/.changeset/v0.10.18.md | 1 - framework/.changeset/v0.10.19.md | 1 + 2 files changed, 1 insertion(+), 1 deletion(-) delete mode 100644 framework/.changeset/v0.10.18.md create mode 100644 framework/.changeset/v0.10.19.md diff --git a/framework/.changeset/v0.10.18.md b/framework/.changeset/v0.10.18.md deleted file mode 100644 index 32c9c4040..000000000 --- a/framework/.changeset/v0.10.18.md +++ /dev/null @@ -1 +0,0 @@ -- Added SD-bridge script that queries each node’s /discovery endpoint and generates Prometheus targets, enabling automatic collection of LOOPP metrics \ No newline at end of file diff --git a/framework/.changeset/v0.10.19.md b/framework/.changeset/v0.10.19.md new file mode 100644 index 000000000..7f52e4353 --- /dev/null +++ b/framework/.changeset/v0.10.19.md @@ -0,0 +1 @@ +- Added SD-bridge script that queries each node’s `/discovery` endpoint and generates Prometheus targets, enabling the automatic collection of LOOPP metrics \ No newline at end of file From 3be7291ac8d0b650be5a70308febad843e45161d Mon Sep 17 00:00:00 2001 From: george-dorin Date: Wed, 3 Sep 2025 13:06:30 +0300 Subject: [PATCH 4/8] Add TODO --- framework/observability/compose/scripts/sd-merge.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/framework/observability/compose/scripts/sd-merge.sh b/framework/observability/compose/scripts/sd-merge.sh index 99a175d9c..3d6cbc466 100644 --- a/framework/observability/compose/scripts/sd-merge.sh +++ b/framework/observability/compose/scripts/sd-merge.sh @@ -2,6 +2,7 @@ # sd-merge.sh # Discover Docker containers by label, fetch each container's /discovery JSON, # add labels (container_name, scrape_path), merge + dedupe, and write a single file_sd JSON. +# TODO: This script should be removed once we convert the prom metrics to OTEL set -Eeuo pipefail From c0d62b27cadb9a8b031fba878fe3309068ecfbb0 Mon Sep 17 00:00:00 2001 From: george-dorin Date: Wed, 3 Sep 2025 13:58:37 +0300 Subject: [PATCH 5/8] Change service_discovery script to go --- .../observability/compose/docker-compose.yaml | 17 +- .../observability/compose/scripts/sd-merge.sh | 111 ------ .../observability/service_discovery/go.mod | 32 ++ .../observability/service_discovery/go.sum | 124 +++++++ .../observability/service_discovery/main.go | 341 ++++++++++++++++++ 5 files changed, 510 insertions(+), 115 deletions(-) delete mode 100644 framework/observability/compose/scripts/sd-merge.sh create mode 100644 framework/observability/service_discovery/go.mod create mode 100644 framework/observability/service_discovery/go.sum create mode 100644 framework/observability/service_discovery/main.go diff --git a/framework/observability/compose/docker-compose.yaml b/framework/observability/compose/docker-compose.yaml index 1decec8e6..36aa3bcbd 100644 --- a/framework/observability/compose/docker-compose.yaml +++ b/framework/observability/compose/docker-compose.yaml @@ -137,19 +137,28 @@ services: restart: unless-stopped sd-merge: - image: alpine:3.20 - command: [ "/bin/sh","-c","apk add --no-cache bash curl jq docker-cli && exec bash scripts/sd-merge.sh" ] + image: golang:1.24-alpine + working_dir: /app + command: ["/bin/sh","-lc", + "apk add --no-cache git ca-certificates && \ + /usr/local/go/bin/go build -trimpath -ldflags='-s -w' -o /usr/local/bin/sd-merge . && \ + exec /usr/local/bin/sd-merge" + ] volumes: - /var/run/docker.sock:/var/run/docker.sock:ro - sd-targets:/out - - ./scripts:/scripts:ro + - ../service_discovery:/app:ro environment: LABEL_MATCH: "framework=ctf" DISCOVERY_PATH: "/discovery" DISCOVERY_PORT: "6688" DISCOVERY_SCHEME: "http" OUT: "/out/merged.json" - SLEEP: "15" + SLEEP: "30" + REQUEST_TIMEOUT: "5" + restart: unless-stopped + + volumes: loki_data: diff --git a/framework/observability/compose/scripts/sd-merge.sh b/framework/observability/compose/scripts/sd-merge.sh deleted file mode 100644 index 3d6cbc466..000000000 --- a/framework/observability/compose/scripts/sd-merge.sh +++ /dev/null @@ -1,111 +0,0 @@ -#!/usr/bin/env bash -# sd-merge.sh -# Discover Docker containers by label, fetch each container's /discovery JSON, -# add labels (container_name, scrape_path), merge + dedupe, and write a single file_sd JSON. -# TODO: This script should be removed once we convert the prom metrics to OTEL - -set -Eeuo pipefail - -# -------------------- Configuration (via env) -------------------- -LABEL_MATCH="${LABEL_MATCH:-framework=ctf}" # docker ps --filter "label=$LABEL_MATCH" -DEFAULT_PATH="${DISCOVERY_PATH:-/discovery}" # default discovery path inside each container -DEFAULT_PORT="${DISCOVERY_PORT:-6688}" # default discovery port -DEFAULT_SCHEME="${DISCOVERY_SCHEME:-http}" # http or https -PREFER_NETWORK="${NETWORK_NAME:-}" # prefer IP from this Docker network (optional) -OUT="${OUT:-/out/merged.json}" # file_sd output path -SLEEP="${SLEEP:-15}" # seconds between scans -REQUEST_TIMEOUT="${REQUEST_TIMEOUT:-5}" # curl timeout (s) -REWRITE_TO_IP="${REWRITE_TO_IP:-0}" # 1 = replace host with container IP in targets - -# -------------------- Helpers -------------------- -log(){ printf '[sd-merge] %s\n' "$*" >&2; } - -# Atomic writer: reads stdin, writes to $1.tmp, then mv -> $1 -atomic_write(){ - local path="$1" tmp="$1.tmp" - cat > "$tmp" && mv "$tmp" "$path" -} - -# -------------------- Init -------------------- -mkdir -p "$(dirname "$OUT")" -echo '[]' | atomic_write "$OUT" - -# -------------------- Main loop -------------------- -while true; do - # List container IDs matching the label - mapfile -t cids < <(docker ps -q --filter "label=$LABEL_MATCH" || true) - - if ((${#cids[@]} == 0)); then - echo '[]' | atomic_write "$OUT" - log "no matching containers; wrote empty array" - sleep "$SLEEP" - continue - fi - - # Emit each container's (possibly empty) discovery array, then merge once with jq -s - { - for cid in "${cids[@]}"; do - # Inspect once, reuse for IP, name, and labels - inspect="$(docker inspect "$cid" 2>/dev/null || true)" - [[ -z "$inspect" ]] && { log "skip ${cid:0:12}: inspect failed"; echo '[]'; continue; } - - # Resolve container IP (optionally prefer a specific network) - if [[ -n "$PREFER_NETWORK" ]]; then - ip="$(jq -r --arg n "$PREFER_NETWORK" '.[0].NetworkSettings.Networks[$n].IPAddress // ""' <<<"$inspect")" - else - ip="$(jq -r '.[0].NetworkSettings.Networks | to_entries[0].value.IPAddress // ""' <<<"$inspect")" - fi - [[ -z "$ip" ]] && { log "skip ${cid:0:12}: no IP"; echo '[]'; continue; } - - # Container name and optional per-container overrides - name="$(jq -r '.[0].Name | ltrimstr("/")' <<<"$inspect")" - path="$(jq -r '.[0].Config.Labels.prom_sd_path // empty' <<<"$inspect")"; path="${path:-$DEFAULT_PATH}" - port="$(jq -r '.[0].Config.Labels.prom_sd_port // empty' <<<"$inspect")"; port="${port:-$DEFAULT_PORT}" - scheme="$(jq -r '.[0].Config.Labels.prom_sd_scheme // empty' <<<"$inspect")"; scheme="${scheme:-$DEFAULT_SCHEME}" - - url="${scheme}://${ip}:${port}${path}" - - # Fetch discovery JSON; treat errors as empty array - payload="$(curl -fsSL --max-time "$REQUEST_TIMEOUT" "$url" 2>/dev/null || echo '[]')" - - # Normalize to array, add labels, optional host->IP rewrite while keeping port from targets - if [[ "$REWRITE_TO_IP" == "1" ]]; then - jq --arg ip "$ip" --arg name "$name" ' - (if type=="array" then . else [] end) - | map( - .targets |= map( $ip + ":" + (split(":")[1]) ) | - .labels = ((.labels // {}) + { - container_name: $name, - scrape_path: (.labels.__metrics_path__ // "") - }) - ) - ' <<<"$payload" - else - jq --arg name "$name" ' - (if type=="array" then . else [] end) - | map( - .labels = ((.labels // {}) + { - container_name: $name, - scrape_path: (.labels.__metrics_path__ // "") - }) - ) - ' <<<"$payload" - fi - - log "ok $url" - done - } \ - | jq -s ' - # Merge all arrays, coerce to {targets,labels}, then group by labels and dedupe targets - add // [] - | map({targets: (.targets // []), labels: (.labels // {})}) - | group_by(.labels) - | map({ labels: (.[0].labels) - , targets: ([.[].targets[]] | unique | sort) - }) - ' \ - | atomic_write "$OUT" - - log "wrote $(wc -c < "$OUT") bytes to $OUT" - sleep "$SLEEP" -done diff --git a/framework/observability/service_discovery/go.mod b/framework/observability/service_discovery/go.mod new file mode 100644 index 000000000..9d23f26de --- /dev/null +++ b/framework/observability/service_discovery/go.mod @@ -0,0 +1,32 @@ +module github.com/smartcontractkit/chainlink-testing-framework/service_discovery + +go 1.24.4 + +require github.com/docker/docker v26.1.1+incompatible + +require ( + github.com/Microsoft/go-winio v0.4.21 // indirect + github.com/containerd/log v0.1.0 // indirect + github.com/distribution/reference v0.6.0 // indirect + github.com/docker/go-connections v0.6.0 // indirect + github.com/docker/go-units v0.5.0 // indirect + github.com/felixge/httpsnoop v1.0.4 // indirect + github.com/go-logr/logr v1.4.3 // indirect + github.com/go-logr/stdr v1.2.2 // indirect + github.com/gogo/protobuf v1.3.2 // indirect + github.com/moby/docker-image-spec v1.3.1 // indirect + github.com/moby/term v0.5.2 // indirect + github.com/morikuni/aec v1.0.0 // indirect + github.com/opencontainers/go-digest v1.0.0 // indirect + github.com/opencontainers/image-spec v1.1.1 // indirect + github.com/pkg/errors v0.9.1 // indirect + go.opentelemetry.io/auto/sdk v1.1.0 // indirect + go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.63.0 // indirect + go.opentelemetry.io/otel v1.38.0 // indirect + go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.38.0 // indirect + go.opentelemetry.io/otel/metric v1.38.0 // indirect + go.opentelemetry.io/otel/trace v1.38.0 // indirect + golang.org/x/sys v0.35.0 // indirect + golang.org/x/time v0.12.0 // indirect + gotest.tools/v3 v3.5.2 // indirect +) diff --git a/framework/observability/service_discovery/go.sum b/framework/observability/service_discovery/go.sum new file mode 100644 index 000000000..e24384319 --- /dev/null +++ b/framework/observability/service_discovery/go.sum @@ -0,0 +1,124 @@ +github.com/Azure/go-ansiterm v0.0.0-20250102033503-faa5f7b0171c h1:udKWzYgxTojEKWjV8V+WSxDXJ4NFATAsZjh8iIbsQIg= +github.com/Azure/go-ansiterm v0.0.0-20250102033503-faa5f7b0171c/go.mod h1:xomTg63KZ2rFqZQzSB4Vz2SUXa1BpHTVz9L5PTmPC4E= +github.com/Microsoft/go-winio v0.4.21 h1:+6mVbXh4wPzUrl1COX9A+ZCvEpYsOBZ6/+kwDnvLyro= +github.com/Microsoft/go-winio v0.4.21/go.mod h1:JPGBdM1cNvN/6ISo+n8V5iA4v8pBzdOpzfwIujj1a84= +github.com/cenkalti/backoff/v5 v5.0.3 h1:ZN+IMa753KfX5hd8vVaMixjnqRZ3y8CuJKRKj1xcsSM= +github.com/cenkalti/backoff/v5 v5.0.3/go.mod h1:rkhZdG3JZukswDf7f0cwqPNk4K0sa+F97BxZthm/crw= +github.com/containerd/log v0.1.0 h1:TCJt7ioM2cr/tfR8GPbGf9/VRAX8D2B4PjzCpfX540I= +github.com/containerd/log v0.1.0/go.mod h1:VRRf09a7mHDIRezVKTRCrOq78v577GXq3bSa3EhrzVo= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/distribution/reference v0.6.0 h1:0IXCQ5g4/QMHHkarYzh5l+u8T3t73zM5QvfrDyIgxBk= +github.com/distribution/reference v0.6.0/go.mod h1:BbU0aIcezP1/5jX/8MP0YiH4SdvB5Y4f/wlDRiLyi3E= +github.com/docker/docker v26.1.1+incompatible h1:oI+4kkAgIwwb54b9OC7Xc3hSgu1RlJA/Lln/DF72djQ= +github.com/docker/docker v26.1.1+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk= +github.com/docker/go-connections v0.6.0 h1:LlMG9azAe1TqfR7sO+NJttz1gy6KO7VJBh+pMmjSD94= +github.com/docker/go-connections v0.6.0/go.mod h1:AahvXYshr6JgfUJGdDCs2b5EZG/vmaMAntpSFH5BFKE= +github.com/docker/go-units v0.5.0 h1:69rxXcBk27SvSaaxTtLh/8llcHD8vYHT7WSdRZ/jvr4= +github.com/docker/go-units v0.5.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk= +github.com/felixge/httpsnoop v1.0.4 h1:NFTV2Zj1bL4mc9sqWACXbQFVBBg2W3GPvqp8/ESS2Wg= +github.com/felixge/httpsnoop v1.0.4/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U= +github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= +github.com/go-logr/logr v1.4.3 h1:CjnDlHq8ikf6E492q6eKboGOC0T8CDaOvkHCIg8idEI= +github.com/go-logr/logr v1.4.3/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= +github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= +github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= +github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= +github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= +github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= +github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU= +github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= +github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/grpc-ecosystem/grpc-gateway/v2 v2.27.2 h1:8Tjv8EJ+pM1xP8mK6egEbD1OgnVTyacbefKhmbLhIhU= +github.com/grpc-ecosystem/grpc-gateway/v2 v2.27.2/go.mod h1:pkJQ2tZHJ0aFOVEEot6oZmaVEZcRme73eIFmhiVuRWs= +github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= +github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= +github.com/moby/docker-image-spec v1.3.1 h1:jMKff3w6PgbfSa69GfNg+zN/XLhfXJGnEx3Nl2EsFP0= +github.com/moby/docker-image-spec v1.3.1/go.mod h1:eKmb5VW8vQEh/BAr2yvVNvuiJuY6UIocYsFu/DxxRpo= +github.com/moby/term v0.5.2 h1:6qk3FJAFDs6i/q3W/pQ97SX192qKfZgGjCQqfCJkgzQ= +github.com/moby/term v0.5.2/go.mod h1:d3djjFCrjnB+fl8NJux+EJzu0msscUP+f8it8hPkFLc= +github.com/morikuni/aec v1.0.0 h1:nP9CBfwrvYnBRgY6qfDQkygYDmYwOilePFkwzv4dU8A= +github.com/morikuni/aec v1.0.0/go.mod h1:BbKIizmSmc5MMPqRYbxO4ZU0S0+P200+tUnFx7PXmsc= +github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8Oi/yOhh5U= +github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM= +github.com/opencontainers/image-spec v1.1.1 h1:y0fUlFfIZhPF1W537XOLg0/fcx6zcHCJwooC2xJA040= +github.com/opencontainers/image-spec v1.1.1/go.mod h1:qpqAh3Dmcf36wStyyWU+kCeDgrGnAve2nCC8+7h8Q0M= +github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= +github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/sirupsen/logrus v1.7.0/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0= +github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ= +github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= +github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= +github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U= +github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U= +github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +go.opentelemetry.io/auto/sdk v1.1.0 h1:cH53jehLUN6UFLY71z+NDOiNJqDdPRaXzTel0sJySYA= +go.opentelemetry.io/auto/sdk v1.1.0/go.mod h1:3wSPjt5PWp2RhlCcmmOial7AvC4DQqZb7a7wCow3W8A= +go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.63.0 h1:RbKq8BG0FI8OiXhBfcRtqqHcZcka+gU3cskNuf05R18= +go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.63.0/go.mod h1:h06DGIukJOevXaj/xrNjhi/2098RZzcLTbc0jDAUbsg= +go.opentelemetry.io/otel v1.38.0 h1:RkfdswUDRimDg0m2Az18RKOsnI8UDzppJAtj01/Ymk8= +go.opentelemetry.io/otel v1.38.0/go.mod h1:zcmtmQ1+YmQM9wrNsTGV/q/uyusom3P8RxwExxkZhjM= +go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.38.0 h1:GqRJVj7UmLjCVyVJ3ZFLdPRmhDUp2zFmQe3RHIOsw24= +go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.38.0/go.mod h1:ri3aaHSmCTVYu2AWv44YMauwAQc0aqI9gHKIcSbI1pU= +go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.38.0 h1:aTL7F04bJHUlztTsNGJ2l+6he8c+y/b//eR0jjjemT4= +go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.38.0/go.mod h1:kldtb7jDTeol0l3ewcmd8SDvx3EmIE7lyvqbasU3QC4= +go.opentelemetry.io/otel/metric v1.38.0 h1:Kl6lzIYGAh5M159u9NgiRkmoMKjvbsKtYRwgfrA6WpA= +go.opentelemetry.io/otel/metric v1.38.0/go.mod h1:kB5n/QoRM8YwmUahxvI3bO34eVtQf2i4utNVLr9gEmI= +go.opentelemetry.io/otel/sdk v1.38.0 h1:l48sr5YbNf2hpCUj/FoGhW9yDkl+Ma+LrVl8qaM5b+E= +go.opentelemetry.io/otel/sdk v1.38.0/go.mod h1:ghmNdGlVemJI3+ZB5iDEuk4bWA3GkTpW+DOoZMYBVVg= +go.opentelemetry.io/otel/sdk/metric v1.38.0 h1:aSH66iL0aZqo//xXzQLYozmWrXxyFkBJ6qT5wthqPoM= +go.opentelemetry.io/otel/sdk/metric v1.38.0/go.mod h1:dg9PBnW9XdQ1Hd6ZnRz689CbtrUp0wMMs9iPcgT9EZA= +go.opentelemetry.io/otel/trace v1.38.0 h1:Fxk5bKrDZJUH+AMyyIXGcFAPah0oRcT+LuNtJrmcNLE= +go.opentelemetry.io/otel/trace v1.38.0/go.mod h1:j1P9ivuFsTceSWe1oY+EeW3sc+Pp42sO++GHkg4wwhs= +go.opentelemetry.io/proto/otlp v1.7.1 h1:gTOMpGDb0WTBOP8JaO72iL3auEZhVmAQg4ipjOVAtj4= +go.opentelemetry.io/proto/otlp v1.7.1/go.mod h1:b2rVh6rfI/s2pHWNlB7ILJcRALpcNDzKhACevjI+ZnE= +golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= +golang.org/x/net v0.43.0 h1:lat02VYK2j4aLzMzecihNvTlJNQUq316m2Mr9rnM6YE= +golang.org/x/net v0.43.0/go.mod h1:vhO1fvI4dGsIjh73sWfUVjj3N7CA9WkKJNQm2svM6Jg= +golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.35.0 h1:vz1N37gP5bs89s7He8XuIYXpyY0+QlsKmzipCbUtyxI= +golang.org/x/sys v0.35.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k= +golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.28.0 h1:rhazDwis8INMIwQ4tpjLDzUhx6RlXqZNPEM0huQojng= +golang.org/x/text v0.28.0/go.mod h1:U8nCwOR8jO/marOQ0QbDiOngZVEBB7MAiitBuMjXiNU= +golang.org/x/time v0.12.0 h1:ScB/8o8olJvc+CQPWrK3fPZNfh7qgwCrY0zJmoEQLSE= +golang.org/x/time v0.12.0/go.mod h1:CDIdPxbZBQxdj6cxyCIdrNogrJKMJ7pr37NYpMcMDSg= +golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +google.golang.org/genproto/googleapis/api v0.0.0-20250825161204-c5933d9347a5 h1:BIRfGDEjiHRrk0QKZe3Xv2ieMhtgRGeLcZQ0mIVn4EY= +google.golang.org/genproto/googleapis/api v0.0.0-20250825161204-c5933d9347a5/go.mod h1:j3QtIyytwqGr1JUDtYXwtMXWPKsEa5LtzIFN1Wn5WvE= +google.golang.org/genproto/googleapis/rpc v0.0.0-20250825161204-c5933d9347a5 h1:eaY8u2EuxbRv7c3NiGK0/NedzVsCcV6hDuU5qPX5EGE= +google.golang.org/genproto/googleapis/rpc v0.0.0-20250825161204-c5933d9347a5/go.mod h1:M4/wBTSeyLxupu3W3tJtOgB14jILAS/XWPSSa3TAlJc= +google.golang.org/grpc v1.75.0 h1:+TW+dqTd2Biwe6KKfhE5JpiYIBWq865PhKGSXiivqt4= +google.golang.org/grpc v1.75.0/go.mod h1:JtPAzKiq4v1xcAB2hydNlWI2RnF85XXcV0mhKXr2ecQ= +google.golang.org/protobuf v1.36.8 h1:xHScyCOEuuwZEc6UtSOvPbAT4zRh0xcNRYekJwfqyMc= +google.golang.org/protobuf v1.36.8/go.mod h1:fuxRtAxBytpl4zzqUh6/eyUujkJdNiuEkXntxiD/uRU= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gotest.tools/v3 v3.5.2 h1:7koQfIKdy+I8UTetycgUqXWSDwpgv193Ka+qRsmBY8Q= +gotest.tools/v3 v3.5.2/go.mod h1:LtdLGcnqToBH83WByAAi/wiwSFCArdFIUV/xxN4pcjA= diff --git a/framework/observability/service_discovery/main.go b/framework/observability/service_discovery/main.go new file mode 100644 index 000000000..8c6c1fc9b --- /dev/null +++ b/framework/observability/service_discovery/main.go @@ -0,0 +1,341 @@ +// sd-merge: discover Docker containers by label, fetch each container's /discovery JSON, +// enrich targets with labels, merge + dedupe, and write a single Prometheus file_sd JSON. +// +// Env (with defaults) +// LABEL_MATCH=framework=ctf # docker ps --filter "label=$LABEL_MATCH" +// DISCOVERY_PATH=/discovery # path inside the container that returns target groups +// DISCOVERY_PORT=6688 # discovery port +// DISCOVERY_SCHEME=http # http or https +// NETWORK_NAME= # prefer IP from this Docker network (optional) +// OUT=/out/merged.json # output path for file_sd +// SLEEP=15 # seconds or Go duration (e.g. 15s, 1m) +// REQUEST_TIMEOUT=5 # seconds or Go duration +// REWRITE_TO_IP=0 # 1 = replace target host with container IP, keep target port + +package main + +import ( + "context" + "encoding/json" + "fmt" + "io" + "log" + "net/http" + "os" + "path/filepath" + "sort" + "strconv" + "strings" + "time" + + "github.com/docker/docker/api/types" + "github.com/docker/docker/api/types/container" + "github.com/docker/docker/api/types/filters" + docker "github.com/docker/docker/client" +) + +// Prometheus file_sd/http_sd target group. +type TargetGroup struct { + Targets []string `json:"targets"` + Labels map[string]string `json:"labels,omitempty"` +} + +type Config struct { + LabelMatch string + DefaultPath string + DefaultPort string + DefaultScheme string + PreferNetwork string + OutPath string + Sleep time.Duration + ReqTimeout time.Duration + RewriteToIP bool +} + +func getenv(key, def string) string { + if v := os.Getenv(key); v != "" { + return v + } + return def +} + +func parseDur(s string, def time.Duration) time.Duration { + if s == "" { + return def + } + if d, err := time.ParseDuration(s); err == nil { + return d + } + if n, err := strconv.Atoi(s); err == nil && n > 0 { + return time.Duration(n) * time.Second + } + return def +} + +func loadConfig() Config { + return Config{ + LabelMatch: getenv("LABEL_MATCH", "framework=ctf"), + DefaultPath: getenv("DISCOVERY_PATH", "/discovery"), + DefaultPort: getenv("DISCOVERY_PORT", "6688"), + DefaultScheme: getenv("DISCOVERY_SCHEME", "http"), + PreferNetwork: getenv("NETWORK_NAME", ""), + OutPath: getenv("OUT", "/out/merged.json"), + Sleep: parseDur(getenv("SLEEP", "15"), 15*time.Second), + ReqTimeout: parseDur(getenv("REQUEST_TIMEOUT", "5"), 5*time.Second), + RewriteToIP: getenv("REWRITE_TO_IP", "0") == "1", + } +} + +func main() { + log.SetFlags(0) + cfg := loadConfig() + log.Printf("[sd-merge] start label=%q out=%s every=%s", cfg.LabelMatch, cfg.OutPath, cfg.Sleep) + + if err := os.MkdirAll(filepath.Dir(cfg.OutPath), 0o755); err != nil { + log.Fatalf("mkdir out: %v", err) + } + _ = atomicWriteJSON(cfg.OutPath, []TargetGroup{}) // ensure file exists + + cli, err := docker.NewClientWithOpts(docker.FromEnv) + if err != nil { + log.Fatalf("docker client: %v", err) + } + defer cli.Close() + + httpClient := &http.Client{Timeout: cfg.ReqTimeout} + ctx := context.Background() + + ticker := time.NewTicker(cfg.Sleep) + defer ticker.Stop() + + for { + if err := runOnce(ctx, cli, httpClient, cfg); err != nil { + log.Printf("[sd-merge] cycle error: %v", err) + } + <-ticker.C + } +} + +// One full discovery+merge cycle. +func runOnce(ctx context.Context, cli *docker.Client, hc *http.Client, cfg Config) error { + ids, err := listContainerIDs(ctx, cli, cfg.LabelMatch) + if err != nil { + return err + } + if len(ids) == 0 { + if err := atomicWriteJSON(cfg.OutPath, []TargetGroup{}); err != nil { + return err + } + log.Printf("[sd-merge] no matching containers -> wrote empty list") + return nil + } + + var all []TargetGroup + + for _, id := range ids { + inspect, err := cli.ContainerInspect(ctx, id) + if err != nil { + log.Printf("[sd-merge] skip %.12s: inspect failed: %v", id, err) + continue + } + ip := pickIP(inspect, cfg.PreferNetwork) + if ip == "" { + log.Printf("[sd-merge] skip %.12s: no IP", id) + continue + } + name := strings.TrimPrefix(inspect.Name, "/") + + // Per-container overrides, with sane defaults. + path := first(inspect.Config.Labels["prom_sd_path"], cfg.DefaultPath) + port := first(inspect.Config.Labels["prom_sd_port"], cfg.DefaultPort) + scheme := first(inspect.Config.Labels["prom_sd_scheme"], cfg.DefaultScheme) + url := fmt.Sprintf("%s://%s:%s%s", scheme, ip, port, path) + + // Fetch discovery JSON (array of target groups). + tgs, err := fetchDiscovery(hc, url) + if err != nil { + log.Printf("[sd-merge] %s fetch failed: %v", url, err) + continue + } + + // Enrich labels and (optionally) rewrite hosts to container IP. + for i := range tgs { + if tgs[i].Labels == nil { + tgs[i].Labels = map[string]string{} + } + if mp := tgs[i].Labels["__metrics_path__"]; mp != "" { + tgs[i].Labels["scrape_path"] = mp + } else { + tgs[i].Labels["scrape_path"] = "" + } + tgs[i].Labels["container_name"] = name + + if cfg.RewriteToIP { + for j, tgt := range tgs[i].Targets { + if p := targetPort(tgt); p != "" { + tgs[i].Targets[j] = ip + ":" + p + } + } + } + } + + all = append(all, tgs...) + log.Printf("[sd-merge] ok %s -> %d groups", url, len(tgs)) + } + + merged := mergeAndDedupe(all) + if err := atomicWriteJSON(cfg.OutPath, merged); err != nil { + return err + } + if fi, err := os.Stat(cfg.OutPath); err == nil { + log.Printf("[sd-merge] wrote %d groups (%d bytes) -> %s", len(merged), fi.Size(), cfg.OutPath) + } + return nil +} + +// listContainerIDs returns IDs of running containers matching a label filter. +func listContainerIDs(ctx context.Context, cli *docker.Client, label string) ([]string, error) { + label = strings.TrimSpace(label) + if label == "" { + return nil, fmt.Errorf("LABEL_MATCH cannot be empty") + } + f := filters.NewArgs() + f.Add("label", label) + cs, err := cli.ContainerList(ctx, container.ListOptions{Filters: f}) + if err != nil { + return nil, err + } + ids := make([]string, 0, len(cs)) + for _, c := range cs { + ids = append(ids, c.ID) + } + return ids, nil +} + +// pickIP chooses the preferred network IP, or the first available. +func pickIP(info types.ContainerJSON, prefer string) string { + if info.NetworkSettings == nil || info.NetworkSettings.Networks == nil { + return "" + } + if prefer != "" { + if es, ok := info.NetworkSettings.Networks[prefer]; ok && es.IPAddress != "" { + return es.IPAddress + } + } + for _, es := range info.NetworkSettings.Networks { + if es.IPAddress != "" { + return es.IPAddress + } + } + return "" +} + +// fetchDiscovery GETs the discovery URL and decodes a JSON array of target groups. +// Any non-200 or non-array payload results in an empty list (not an error). +func fetchDiscovery(hc *http.Client, url string) ([]TargetGroup, error) { + resp, err := hc.Get(url) + if err != nil { + return nil, err + } + defer resp.Body.Close() + + if resp.StatusCode != http.StatusOK { + b, _ := io.ReadAll(io.LimitReader(resp.Body, 4<<10)) + return nil, fmt.Errorf("status %d: %s", resp.StatusCode, string(b)) + } + var out []TargetGroup + if err := json.NewDecoder(resp.Body).Decode(&out); err != nil { + return []TargetGroup{}, nil + } + return out, nil +} + +func targetPort(hostport string) string { + if i := strings.LastIndex(hostport, ":"); i >= 0 && i+1 < len(hostport) { + return hostport[i+1:] + } + return "" +} + +func first(v, def string) string { + if v != "" { + return v + } + return def +} + +// mergeAndDedupe: group by identical label sets and union targets (sorted). +func mergeAndDedupe(in []TargetGroup) []TargetGroup { + byKey := make(map[string]map[string]struct{}) // labelsKey -> targets set + lblOf := make(map[string]map[string]string) + + for _, g := range in { + k := labelsKey(g.Labels) + if byKey[k] == nil { + byKey[k] = make(map[string]struct{}) + lblOf[k] = g.Labels + } + for _, t := range g.Targets { + byKey[k][t] = struct{}{} + } + } + + keys := make([]string, 0, len(byKey)) + for k := range byKey { + keys = append(keys, k) + } + sort.Strings(keys) + + out := make([]TargetGroup, 0, len(keys)) + for _, k := range keys { + ts := make([]string, 0, len(byKey[k])) + for t := range byKey[k] { + ts = append(ts, t) + } + sort.Strings(ts) + out = append(out, TargetGroup{Targets: ts, Labels: lblOf[k]}) + } + return out +} + +// labelsKey builds a canonical string for a label map (sorted key=value lines). +func labelsKey(m map[string]string) string { + if len(m) == 0 { + return "" + } + keys := make([]string, 0, len(m)) + for k := range m { + keys = append(keys, k) + } + sort.Strings(keys) + var b strings.Builder + for _, k := range keys { + b.WriteString(k) + b.WriteString("=") + b.WriteString(m[k]) + b.WriteString("\n") + } + return b.String() +} + +// atomicWriteJSON writes JSON to a temp file and renames it into place. +func atomicWriteJSON(path string, v any) error { + if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil { + return err + } + tmp := path + ".tmp" + f, err := os.Create(tmp) + if err != nil { + return err + } + enc := json.NewEncoder(f) + enc.SetIndent("", " ") + if err := enc.Encode(v); err != nil { + _ = f.Close() + return err + } + if err := f.Close(); err != nil { + return err + } + return os.Rename(tmp, path) +} From ec0371aaafdc0ef9994ec62ed34632be56003f8b Mon Sep 17 00:00:00 2001 From: george-dorin Date: Wed, 3 Sep 2025 14:00:56 +0300 Subject: [PATCH 6/8] Update changeset --- framework/.changeset/v0.10.19.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/framework/.changeset/v0.10.19.md b/framework/.changeset/v0.10.19.md index 7f52e4353..d91622de7 100644 --- a/framework/.changeset/v0.10.19.md +++ b/framework/.changeset/v0.10.19.md @@ -1 +1 @@ -- Added SD-bridge script that queries each node’s `/discovery` endpoint and generates Prometheus targets, enabling the automatic collection of LOOPP metrics \ No newline at end of file +- Added service_discovery script that queries each node’s `/discovery` endpoint and generates Prometheus targets, enabling the automatic collection of LOOPP metrics \ No newline at end of file From 95cbda3e400d5ce4532da4b40584264c94a80f88 Mon Sep 17 00:00:00 2001 From: george-dorin Date: Wed, 3 Sep 2025 14:18:22 +0300 Subject: [PATCH 7/8] Update go.mod --- .../observability/service_discovery/go.mod | 7 +++++-- .../observability/service_discovery/go.sum | 20 +++++++++++-------- 2 files changed, 17 insertions(+), 10 deletions(-) diff --git a/framework/observability/service_discovery/go.mod b/framework/observability/service_discovery/go.mod index 9d23f26de..041862a08 100644 --- a/framework/observability/service_discovery/go.mod +++ b/framework/observability/service_discovery/go.mod @@ -2,10 +2,12 @@ module github.com/smartcontractkit/chainlink-testing-framework/service_discovery go 1.24.4 -require github.com/docker/docker v26.1.1+incompatible +require github.com/docker/docker v28.3.3+incompatible require ( - github.com/Microsoft/go-winio v0.4.21 // indirect + github.com/Microsoft/go-winio v0.6.2 // indirect + github.com/containerd/errdefs v1.0.0 // indirect + github.com/containerd/errdefs/pkg v0.3.0 // indirect github.com/containerd/log v0.1.0 // indirect github.com/distribution/reference v0.6.0 // indirect github.com/docker/go-connections v0.6.0 // indirect @@ -15,6 +17,7 @@ require ( github.com/go-logr/stdr v1.2.2 // indirect github.com/gogo/protobuf v1.3.2 // indirect github.com/moby/docker-image-spec v1.3.1 // indirect + github.com/moby/sys/atomicwriter v0.1.0 // indirect github.com/moby/term v0.5.2 // indirect github.com/morikuni/aec v1.0.0 // indirect github.com/opencontainers/go-digest v1.0.0 // indirect diff --git a/framework/observability/service_discovery/go.sum b/framework/observability/service_discovery/go.sum index e24384319..ab46ae9c9 100644 --- a/framework/observability/service_discovery/go.sum +++ b/framework/observability/service_discovery/go.sum @@ -1,17 +1,21 @@ github.com/Azure/go-ansiterm v0.0.0-20250102033503-faa5f7b0171c h1:udKWzYgxTojEKWjV8V+WSxDXJ4NFATAsZjh8iIbsQIg= github.com/Azure/go-ansiterm v0.0.0-20250102033503-faa5f7b0171c/go.mod h1:xomTg63KZ2rFqZQzSB4Vz2SUXa1BpHTVz9L5PTmPC4E= -github.com/Microsoft/go-winio v0.4.21 h1:+6mVbXh4wPzUrl1COX9A+ZCvEpYsOBZ6/+kwDnvLyro= -github.com/Microsoft/go-winio v0.4.21/go.mod h1:JPGBdM1cNvN/6ISo+n8V5iA4v8pBzdOpzfwIujj1a84= +github.com/Microsoft/go-winio v0.6.2 h1:F2VQgta7ecxGYO8k3ZZz3RS8fVIXVxONVUPlNERoyfY= +github.com/Microsoft/go-winio v0.6.2/go.mod h1:yd8OoFMLzJbo9gZq8j5qaps8bJ9aShtEA8Ipt1oGCvU= github.com/cenkalti/backoff/v5 v5.0.3 h1:ZN+IMa753KfX5hd8vVaMixjnqRZ3y8CuJKRKj1xcsSM= github.com/cenkalti/backoff/v5 v5.0.3/go.mod h1:rkhZdG3JZukswDf7f0cwqPNk4K0sa+F97BxZthm/crw= +github.com/containerd/errdefs v1.0.0 h1:tg5yIfIlQIrxYtu9ajqY42W3lpS19XqdxRQeEwYG8PI= +github.com/containerd/errdefs v1.0.0/go.mod h1:+YBYIdtsnF4Iw6nWZhJcqGSg/dwvV7tyJ/kCkyJ2k+M= +github.com/containerd/errdefs/pkg v0.3.0 h1:9IKJ06FvyNlexW690DXuQNx2KA2cUJXx151Xdx3ZPPE= +github.com/containerd/errdefs/pkg v0.3.0/go.mod h1:NJw6s9HwNuRhnjJhM7pylWwMyAkmCQvQ4GpJHEqRLVk= github.com/containerd/log v0.1.0 h1:TCJt7ioM2cr/tfR8GPbGf9/VRAX8D2B4PjzCpfX540I= github.com/containerd/log v0.1.0/go.mod h1:VRRf09a7mHDIRezVKTRCrOq78v577GXq3bSa3EhrzVo= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/distribution/reference v0.6.0 h1:0IXCQ5g4/QMHHkarYzh5l+u8T3t73zM5QvfrDyIgxBk= github.com/distribution/reference v0.6.0/go.mod h1:BbU0aIcezP1/5jX/8MP0YiH4SdvB5Y4f/wlDRiLyi3E= -github.com/docker/docker v26.1.1+incompatible h1:oI+4kkAgIwwb54b9OC7Xc3hSgu1RlJA/Lln/DF72djQ= -github.com/docker/docker v26.1.1+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk= +github.com/docker/docker v28.3.3+incompatible h1:Dypm25kh4rmk49v1eiVbsAtpAsYURjYkaKubwuBdxEI= +github.com/docker/docker v28.3.3+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk= github.com/docker/go-connections v0.6.0 h1:LlMG9azAe1TqfR7sO+NJttz1gy6KO7VJBh+pMmjSD94= github.com/docker/go-connections v0.6.0/go.mod h1:AahvXYshr6JgfUJGdDCs2b5EZG/vmaMAntpSFH5BFKE= github.com/docker/go-units v0.5.0 h1:69rxXcBk27SvSaaxTtLh/8llcHD8vYHT7WSdRZ/jvr4= @@ -35,6 +39,10 @@ github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= github.com/moby/docker-image-spec v1.3.1 h1:jMKff3w6PgbfSa69GfNg+zN/XLhfXJGnEx3Nl2EsFP0= github.com/moby/docker-image-spec v1.3.1/go.mod h1:eKmb5VW8vQEh/BAr2yvVNvuiJuY6UIocYsFu/DxxRpo= +github.com/moby/sys/atomicwriter v0.1.0 h1:kw5D/EqkBwsBFi0ss9v1VG3wIkVhzGvLklJ+w3A14Sw= +github.com/moby/sys/atomicwriter v0.1.0/go.mod h1:Ul8oqv2ZMNHOceF643P6FKPXeCmYtlQMvpizfsSoaWs= +github.com/moby/sys/sequential v0.6.0 h1:qrx7XFUd/5DxtqcoH1h438hF5TmOvzC/lspjy7zgvCU= +github.com/moby/sys/sequential v0.6.0/go.mod h1:uyv8EUTrca5PnDsdMGXhZe6CCe8U/UiTWd+lL+7b/Ko= github.com/moby/term v0.5.2 h1:6qk3FJAFDs6i/q3W/pQ97SX192qKfZgGjCQqfCJkgzQ= github.com/moby/term v0.5.2/go.mod h1:d3djjFCrjnB+fl8NJux+EJzu0msscUP+f8it8hPkFLc= github.com/morikuni/aec v1.0.0 h1:nP9CBfwrvYnBRgY6qfDQkygYDmYwOilePFkwzv4dU8A= @@ -47,10 +55,8 @@ github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= -github.com/sirupsen/logrus v1.7.0/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0= github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ= github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= -github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U= github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U= github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= @@ -91,9 +97,7 @@ golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.35.0 h1:vz1N37gP5bs89s7He8XuIYXpyY0+QlsKmzipCbUtyxI= golang.org/x/sys v0.35.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= From a8422511660ba1ee5febe75dde2afab5ad84da63 Mon Sep 17 00:00:00 2001 From: george-dorin Date: Wed, 3 Sep 2025 15:22:38 +0300 Subject: [PATCH 8/8] Revert to bash script --- .../observability/compose/docker-compose.yaml | 14 +- .../observability/service_discovery/go.mod | 35 -- .../observability/service_discovery/go.sum | 128 ------- .../observability/service_discovery/main.go | 341 ------------------ .../service_discovery/service_discovery.sh | 110 ++++++ 5 files changed, 114 insertions(+), 514 deletions(-) delete mode 100644 framework/observability/service_discovery/go.mod delete mode 100644 framework/observability/service_discovery/go.sum delete mode 100644 framework/observability/service_discovery/main.go create mode 100644 framework/observability/service_discovery/service_discovery.sh diff --git a/framework/observability/compose/docker-compose.yaml b/framework/observability/compose/docker-compose.yaml index 36aa3bcbd..cdf6d4a9b 100644 --- a/framework/observability/compose/docker-compose.yaml +++ b/framework/observability/compose/docker-compose.yaml @@ -136,14 +136,10 @@ services: - '9304:9187' restart: unless-stopped - sd-merge: - image: golang:1.24-alpine + node_service_discovery: + image: alpine:3.20 working_dir: /app - command: ["/bin/sh","-lc", - "apk add --no-cache git ca-certificates && \ - /usr/local/go/bin/go build -trimpath -ldflags='-s -w' -o /usr/local/bin/sd-merge . && \ - exec /usr/local/bin/sd-merge" - ] + command: [ "/bin/sh","-c","apk add --no-cache bash curl jq docker-cli && exec bash ./service_discovery.sh" ] volumes: - /var/run/docker.sock:/var/run/docker.sock:ro - sd-targets:/out @@ -154,9 +150,7 @@ services: DISCOVERY_PORT: "6688" DISCOVERY_SCHEME: "http" OUT: "/out/merged.json" - SLEEP: "30" - REQUEST_TIMEOUT: "5" - restart: unless-stopped + SLEEP: "15" diff --git a/framework/observability/service_discovery/go.mod b/framework/observability/service_discovery/go.mod deleted file mode 100644 index 041862a08..000000000 --- a/framework/observability/service_discovery/go.mod +++ /dev/null @@ -1,35 +0,0 @@ -module github.com/smartcontractkit/chainlink-testing-framework/service_discovery - -go 1.24.4 - -require github.com/docker/docker v28.3.3+incompatible - -require ( - github.com/Microsoft/go-winio v0.6.2 // indirect - github.com/containerd/errdefs v1.0.0 // indirect - github.com/containerd/errdefs/pkg v0.3.0 // indirect - github.com/containerd/log v0.1.0 // indirect - github.com/distribution/reference v0.6.0 // indirect - github.com/docker/go-connections v0.6.0 // indirect - github.com/docker/go-units v0.5.0 // indirect - github.com/felixge/httpsnoop v1.0.4 // indirect - github.com/go-logr/logr v1.4.3 // indirect - github.com/go-logr/stdr v1.2.2 // indirect - github.com/gogo/protobuf v1.3.2 // indirect - github.com/moby/docker-image-spec v1.3.1 // indirect - github.com/moby/sys/atomicwriter v0.1.0 // indirect - github.com/moby/term v0.5.2 // indirect - github.com/morikuni/aec v1.0.0 // indirect - github.com/opencontainers/go-digest v1.0.0 // indirect - github.com/opencontainers/image-spec v1.1.1 // indirect - github.com/pkg/errors v0.9.1 // indirect - go.opentelemetry.io/auto/sdk v1.1.0 // indirect - go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.63.0 // indirect - go.opentelemetry.io/otel v1.38.0 // indirect - go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.38.0 // indirect - go.opentelemetry.io/otel/metric v1.38.0 // indirect - go.opentelemetry.io/otel/trace v1.38.0 // indirect - golang.org/x/sys v0.35.0 // indirect - golang.org/x/time v0.12.0 // indirect - gotest.tools/v3 v3.5.2 // indirect -) diff --git a/framework/observability/service_discovery/go.sum b/framework/observability/service_discovery/go.sum deleted file mode 100644 index ab46ae9c9..000000000 --- a/framework/observability/service_discovery/go.sum +++ /dev/null @@ -1,128 +0,0 @@ -github.com/Azure/go-ansiterm v0.0.0-20250102033503-faa5f7b0171c h1:udKWzYgxTojEKWjV8V+WSxDXJ4NFATAsZjh8iIbsQIg= -github.com/Azure/go-ansiterm v0.0.0-20250102033503-faa5f7b0171c/go.mod h1:xomTg63KZ2rFqZQzSB4Vz2SUXa1BpHTVz9L5PTmPC4E= -github.com/Microsoft/go-winio v0.6.2 h1:F2VQgta7ecxGYO8k3ZZz3RS8fVIXVxONVUPlNERoyfY= -github.com/Microsoft/go-winio v0.6.2/go.mod h1:yd8OoFMLzJbo9gZq8j5qaps8bJ9aShtEA8Ipt1oGCvU= -github.com/cenkalti/backoff/v5 v5.0.3 h1:ZN+IMa753KfX5hd8vVaMixjnqRZ3y8CuJKRKj1xcsSM= -github.com/cenkalti/backoff/v5 v5.0.3/go.mod h1:rkhZdG3JZukswDf7f0cwqPNk4K0sa+F97BxZthm/crw= -github.com/containerd/errdefs v1.0.0 h1:tg5yIfIlQIrxYtu9ajqY42W3lpS19XqdxRQeEwYG8PI= -github.com/containerd/errdefs v1.0.0/go.mod h1:+YBYIdtsnF4Iw6nWZhJcqGSg/dwvV7tyJ/kCkyJ2k+M= -github.com/containerd/errdefs/pkg v0.3.0 h1:9IKJ06FvyNlexW690DXuQNx2KA2cUJXx151Xdx3ZPPE= -github.com/containerd/errdefs/pkg v0.3.0/go.mod h1:NJw6s9HwNuRhnjJhM7pylWwMyAkmCQvQ4GpJHEqRLVk= -github.com/containerd/log v0.1.0 h1:TCJt7ioM2cr/tfR8GPbGf9/VRAX8D2B4PjzCpfX540I= -github.com/containerd/log v0.1.0/go.mod h1:VRRf09a7mHDIRezVKTRCrOq78v577GXq3bSa3EhrzVo= -github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= -github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/distribution/reference v0.6.0 h1:0IXCQ5g4/QMHHkarYzh5l+u8T3t73zM5QvfrDyIgxBk= -github.com/distribution/reference v0.6.0/go.mod h1:BbU0aIcezP1/5jX/8MP0YiH4SdvB5Y4f/wlDRiLyi3E= -github.com/docker/docker v28.3.3+incompatible h1:Dypm25kh4rmk49v1eiVbsAtpAsYURjYkaKubwuBdxEI= -github.com/docker/docker v28.3.3+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk= -github.com/docker/go-connections v0.6.0 h1:LlMG9azAe1TqfR7sO+NJttz1gy6KO7VJBh+pMmjSD94= -github.com/docker/go-connections v0.6.0/go.mod h1:AahvXYshr6JgfUJGdDCs2b5EZG/vmaMAntpSFH5BFKE= -github.com/docker/go-units v0.5.0 h1:69rxXcBk27SvSaaxTtLh/8llcHD8vYHT7WSdRZ/jvr4= -github.com/docker/go-units v0.5.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk= -github.com/felixge/httpsnoop v1.0.4 h1:NFTV2Zj1bL4mc9sqWACXbQFVBBg2W3GPvqp8/ESS2Wg= -github.com/felixge/httpsnoop v1.0.4/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U= -github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= -github.com/go-logr/logr v1.4.3 h1:CjnDlHq8ikf6E492q6eKboGOC0T8CDaOvkHCIg8idEI= -github.com/go-logr/logr v1.4.3/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= -github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= -github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= -github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= -github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= -github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= -github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU= -github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= -github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= -github.com/grpc-ecosystem/grpc-gateway/v2 v2.27.2 h1:8Tjv8EJ+pM1xP8mK6egEbD1OgnVTyacbefKhmbLhIhU= -github.com/grpc-ecosystem/grpc-gateway/v2 v2.27.2/go.mod h1:pkJQ2tZHJ0aFOVEEot6oZmaVEZcRme73eIFmhiVuRWs= -github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= -github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= -github.com/moby/docker-image-spec v1.3.1 h1:jMKff3w6PgbfSa69GfNg+zN/XLhfXJGnEx3Nl2EsFP0= -github.com/moby/docker-image-spec v1.3.1/go.mod h1:eKmb5VW8vQEh/BAr2yvVNvuiJuY6UIocYsFu/DxxRpo= -github.com/moby/sys/atomicwriter v0.1.0 h1:kw5D/EqkBwsBFi0ss9v1VG3wIkVhzGvLklJ+w3A14Sw= -github.com/moby/sys/atomicwriter v0.1.0/go.mod h1:Ul8oqv2ZMNHOceF643P6FKPXeCmYtlQMvpizfsSoaWs= -github.com/moby/sys/sequential v0.6.0 h1:qrx7XFUd/5DxtqcoH1h438hF5TmOvzC/lspjy7zgvCU= -github.com/moby/sys/sequential v0.6.0/go.mod h1:uyv8EUTrca5PnDsdMGXhZe6CCe8U/UiTWd+lL+7b/Ko= -github.com/moby/term v0.5.2 h1:6qk3FJAFDs6i/q3W/pQ97SX192qKfZgGjCQqfCJkgzQ= -github.com/moby/term v0.5.2/go.mod h1:d3djjFCrjnB+fl8NJux+EJzu0msscUP+f8it8hPkFLc= -github.com/morikuni/aec v1.0.0 h1:nP9CBfwrvYnBRgY6qfDQkygYDmYwOilePFkwzv4dU8A= -github.com/morikuni/aec v1.0.0/go.mod h1:BbKIizmSmc5MMPqRYbxO4ZU0S0+P200+tUnFx7PXmsc= -github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8Oi/yOhh5U= -github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM= -github.com/opencontainers/image-spec v1.1.1 h1:y0fUlFfIZhPF1W537XOLg0/fcx6zcHCJwooC2xJA040= -github.com/opencontainers/image-spec v1.1.1/go.mod h1:qpqAh3Dmcf36wStyyWU+kCeDgrGnAve2nCC8+7h8Q0M= -github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= -github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= -github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= -github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= -github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ= -github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= -github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U= -github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U= -github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= -github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= -go.opentelemetry.io/auto/sdk v1.1.0 h1:cH53jehLUN6UFLY71z+NDOiNJqDdPRaXzTel0sJySYA= -go.opentelemetry.io/auto/sdk v1.1.0/go.mod h1:3wSPjt5PWp2RhlCcmmOial7AvC4DQqZb7a7wCow3W8A= -go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.63.0 h1:RbKq8BG0FI8OiXhBfcRtqqHcZcka+gU3cskNuf05R18= -go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.63.0/go.mod h1:h06DGIukJOevXaj/xrNjhi/2098RZzcLTbc0jDAUbsg= -go.opentelemetry.io/otel v1.38.0 h1:RkfdswUDRimDg0m2Az18RKOsnI8UDzppJAtj01/Ymk8= -go.opentelemetry.io/otel v1.38.0/go.mod h1:zcmtmQ1+YmQM9wrNsTGV/q/uyusom3P8RxwExxkZhjM= -go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.38.0 h1:GqRJVj7UmLjCVyVJ3ZFLdPRmhDUp2zFmQe3RHIOsw24= -go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.38.0/go.mod h1:ri3aaHSmCTVYu2AWv44YMauwAQc0aqI9gHKIcSbI1pU= -go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.38.0 h1:aTL7F04bJHUlztTsNGJ2l+6he8c+y/b//eR0jjjemT4= -go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.38.0/go.mod h1:kldtb7jDTeol0l3ewcmd8SDvx3EmIE7lyvqbasU3QC4= -go.opentelemetry.io/otel/metric v1.38.0 h1:Kl6lzIYGAh5M159u9NgiRkmoMKjvbsKtYRwgfrA6WpA= -go.opentelemetry.io/otel/metric v1.38.0/go.mod h1:kB5n/QoRM8YwmUahxvI3bO34eVtQf2i4utNVLr9gEmI= -go.opentelemetry.io/otel/sdk v1.38.0 h1:l48sr5YbNf2hpCUj/FoGhW9yDkl+Ma+LrVl8qaM5b+E= -go.opentelemetry.io/otel/sdk v1.38.0/go.mod h1:ghmNdGlVemJI3+ZB5iDEuk4bWA3GkTpW+DOoZMYBVVg= -go.opentelemetry.io/otel/sdk/metric v1.38.0 h1:aSH66iL0aZqo//xXzQLYozmWrXxyFkBJ6qT5wthqPoM= -go.opentelemetry.io/otel/sdk/metric v1.38.0/go.mod h1:dg9PBnW9XdQ1Hd6ZnRz689CbtrUp0wMMs9iPcgT9EZA= -go.opentelemetry.io/otel/trace v1.38.0 h1:Fxk5bKrDZJUH+AMyyIXGcFAPah0oRcT+LuNtJrmcNLE= -go.opentelemetry.io/otel/trace v1.38.0/go.mod h1:j1P9ivuFsTceSWe1oY+EeW3sc+Pp42sO++GHkg4wwhs= -go.opentelemetry.io/proto/otlp v1.7.1 h1:gTOMpGDb0WTBOP8JaO72iL3auEZhVmAQg4ipjOVAtj4= -go.opentelemetry.io/proto/otlp v1.7.1/go.mod h1:b2rVh6rfI/s2pHWNlB7ILJcRALpcNDzKhACevjI+ZnE= -golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= -golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= -golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= -golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= -golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= -golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= -golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= -golang.org/x/net v0.43.0 h1:lat02VYK2j4aLzMzecihNvTlJNQUq316m2Mr9rnM6YE= -golang.org/x/net v0.43.0/go.mod h1:vhO1fvI4dGsIjh73sWfUVjj3N7CA9WkKJNQm2svM6Jg= -golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.35.0 h1:vz1N37gP5bs89s7He8XuIYXpyY0+QlsKmzipCbUtyxI= -golang.org/x/sys v0.35.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k= -golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= -golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= -golang.org/x/text v0.28.0 h1:rhazDwis8INMIwQ4tpjLDzUhx6RlXqZNPEM0huQojng= -golang.org/x/text v0.28.0/go.mod h1:U8nCwOR8jO/marOQ0QbDiOngZVEBB7MAiitBuMjXiNU= -golang.org/x/time v0.12.0 h1:ScB/8o8olJvc+CQPWrK3fPZNfh7qgwCrY0zJmoEQLSE= -golang.org/x/time v0.12.0/go.mod h1:CDIdPxbZBQxdj6cxyCIdrNogrJKMJ7pr37NYpMcMDSg= -golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= -golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= -golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= -golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -google.golang.org/genproto/googleapis/api v0.0.0-20250825161204-c5933d9347a5 h1:BIRfGDEjiHRrk0QKZe3Xv2ieMhtgRGeLcZQ0mIVn4EY= -google.golang.org/genproto/googleapis/api v0.0.0-20250825161204-c5933d9347a5/go.mod h1:j3QtIyytwqGr1JUDtYXwtMXWPKsEa5LtzIFN1Wn5WvE= -google.golang.org/genproto/googleapis/rpc v0.0.0-20250825161204-c5933d9347a5 h1:eaY8u2EuxbRv7c3NiGK0/NedzVsCcV6hDuU5qPX5EGE= -google.golang.org/genproto/googleapis/rpc v0.0.0-20250825161204-c5933d9347a5/go.mod h1:M4/wBTSeyLxupu3W3tJtOgB14jILAS/XWPSSa3TAlJc= -google.golang.org/grpc v1.75.0 h1:+TW+dqTd2Biwe6KKfhE5JpiYIBWq865PhKGSXiivqt4= -google.golang.org/grpc v1.75.0/go.mod h1:JtPAzKiq4v1xcAB2hydNlWI2RnF85XXcV0mhKXr2ecQ= -google.golang.org/protobuf v1.36.8 h1:xHScyCOEuuwZEc6UtSOvPbAT4zRh0xcNRYekJwfqyMc= -google.golang.org/protobuf v1.36.8/go.mod h1:fuxRtAxBytpl4zzqUh6/eyUujkJdNiuEkXntxiD/uRU= -gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= -gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= -gotest.tools/v3 v3.5.2 h1:7koQfIKdy+I8UTetycgUqXWSDwpgv193Ka+qRsmBY8Q= -gotest.tools/v3 v3.5.2/go.mod h1:LtdLGcnqToBH83WByAAi/wiwSFCArdFIUV/xxN4pcjA= diff --git a/framework/observability/service_discovery/main.go b/framework/observability/service_discovery/main.go deleted file mode 100644 index 8c6c1fc9b..000000000 --- a/framework/observability/service_discovery/main.go +++ /dev/null @@ -1,341 +0,0 @@ -// sd-merge: discover Docker containers by label, fetch each container's /discovery JSON, -// enrich targets with labels, merge + dedupe, and write a single Prometheus file_sd JSON. -// -// Env (with defaults) -// LABEL_MATCH=framework=ctf # docker ps --filter "label=$LABEL_MATCH" -// DISCOVERY_PATH=/discovery # path inside the container that returns target groups -// DISCOVERY_PORT=6688 # discovery port -// DISCOVERY_SCHEME=http # http or https -// NETWORK_NAME= # prefer IP from this Docker network (optional) -// OUT=/out/merged.json # output path for file_sd -// SLEEP=15 # seconds or Go duration (e.g. 15s, 1m) -// REQUEST_TIMEOUT=5 # seconds or Go duration -// REWRITE_TO_IP=0 # 1 = replace target host with container IP, keep target port - -package main - -import ( - "context" - "encoding/json" - "fmt" - "io" - "log" - "net/http" - "os" - "path/filepath" - "sort" - "strconv" - "strings" - "time" - - "github.com/docker/docker/api/types" - "github.com/docker/docker/api/types/container" - "github.com/docker/docker/api/types/filters" - docker "github.com/docker/docker/client" -) - -// Prometheus file_sd/http_sd target group. -type TargetGroup struct { - Targets []string `json:"targets"` - Labels map[string]string `json:"labels,omitempty"` -} - -type Config struct { - LabelMatch string - DefaultPath string - DefaultPort string - DefaultScheme string - PreferNetwork string - OutPath string - Sleep time.Duration - ReqTimeout time.Duration - RewriteToIP bool -} - -func getenv(key, def string) string { - if v := os.Getenv(key); v != "" { - return v - } - return def -} - -func parseDur(s string, def time.Duration) time.Duration { - if s == "" { - return def - } - if d, err := time.ParseDuration(s); err == nil { - return d - } - if n, err := strconv.Atoi(s); err == nil && n > 0 { - return time.Duration(n) * time.Second - } - return def -} - -func loadConfig() Config { - return Config{ - LabelMatch: getenv("LABEL_MATCH", "framework=ctf"), - DefaultPath: getenv("DISCOVERY_PATH", "/discovery"), - DefaultPort: getenv("DISCOVERY_PORT", "6688"), - DefaultScheme: getenv("DISCOVERY_SCHEME", "http"), - PreferNetwork: getenv("NETWORK_NAME", ""), - OutPath: getenv("OUT", "/out/merged.json"), - Sleep: parseDur(getenv("SLEEP", "15"), 15*time.Second), - ReqTimeout: parseDur(getenv("REQUEST_TIMEOUT", "5"), 5*time.Second), - RewriteToIP: getenv("REWRITE_TO_IP", "0") == "1", - } -} - -func main() { - log.SetFlags(0) - cfg := loadConfig() - log.Printf("[sd-merge] start label=%q out=%s every=%s", cfg.LabelMatch, cfg.OutPath, cfg.Sleep) - - if err := os.MkdirAll(filepath.Dir(cfg.OutPath), 0o755); err != nil { - log.Fatalf("mkdir out: %v", err) - } - _ = atomicWriteJSON(cfg.OutPath, []TargetGroup{}) // ensure file exists - - cli, err := docker.NewClientWithOpts(docker.FromEnv) - if err != nil { - log.Fatalf("docker client: %v", err) - } - defer cli.Close() - - httpClient := &http.Client{Timeout: cfg.ReqTimeout} - ctx := context.Background() - - ticker := time.NewTicker(cfg.Sleep) - defer ticker.Stop() - - for { - if err := runOnce(ctx, cli, httpClient, cfg); err != nil { - log.Printf("[sd-merge] cycle error: %v", err) - } - <-ticker.C - } -} - -// One full discovery+merge cycle. -func runOnce(ctx context.Context, cli *docker.Client, hc *http.Client, cfg Config) error { - ids, err := listContainerIDs(ctx, cli, cfg.LabelMatch) - if err != nil { - return err - } - if len(ids) == 0 { - if err := atomicWriteJSON(cfg.OutPath, []TargetGroup{}); err != nil { - return err - } - log.Printf("[sd-merge] no matching containers -> wrote empty list") - return nil - } - - var all []TargetGroup - - for _, id := range ids { - inspect, err := cli.ContainerInspect(ctx, id) - if err != nil { - log.Printf("[sd-merge] skip %.12s: inspect failed: %v", id, err) - continue - } - ip := pickIP(inspect, cfg.PreferNetwork) - if ip == "" { - log.Printf("[sd-merge] skip %.12s: no IP", id) - continue - } - name := strings.TrimPrefix(inspect.Name, "/") - - // Per-container overrides, with sane defaults. - path := first(inspect.Config.Labels["prom_sd_path"], cfg.DefaultPath) - port := first(inspect.Config.Labels["prom_sd_port"], cfg.DefaultPort) - scheme := first(inspect.Config.Labels["prom_sd_scheme"], cfg.DefaultScheme) - url := fmt.Sprintf("%s://%s:%s%s", scheme, ip, port, path) - - // Fetch discovery JSON (array of target groups). - tgs, err := fetchDiscovery(hc, url) - if err != nil { - log.Printf("[sd-merge] %s fetch failed: %v", url, err) - continue - } - - // Enrich labels and (optionally) rewrite hosts to container IP. - for i := range tgs { - if tgs[i].Labels == nil { - tgs[i].Labels = map[string]string{} - } - if mp := tgs[i].Labels["__metrics_path__"]; mp != "" { - tgs[i].Labels["scrape_path"] = mp - } else { - tgs[i].Labels["scrape_path"] = "" - } - tgs[i].Labels["container_name"] = name - - if cfg.RewriteToIP { - for j, tgt := range tgs[i].Targets { - if p := targetPort(tgt); p != "" { - tgs[i].Targets[j] = ip + ":" + p - } - } - } - } - - all = append(all, tgs...) - log.Printf("[sd-merge] ok %s -> %d groups", url, len(tgs)) - } - - merged := mergeAndDedupe(all) - if err := atomicWriteJSON(cfg.OutPath, merged); err != nil { - return err - } - if fi, err := os.Stat(cfg.OutPath); err == nil { - log.Printf("[sd-merge] wrote %d groups (%d bytes) -> %s", len(merged), fi.Size(), cfg.OutPath) - } - return nil -} - -// listContainerIDs returns IDs of running containers matching a label filter. -func listContainerIDs(ctx context.Context, cli *docker.Client, label string) ([]string, error) { - label = strings.TrimSpace(label) - if label == "" { - return nil, fmt.Errorf("LABEL_MATCH cannot be empty") - } - f := filters.NewArgs() - f.Add("label", label) - cs, err := cli.ContainerList(ctx, container.ListOptions{Filters: f}) - if err != nil { - return nil, err - } - ids := make([]string, 0, len(cs)) - for _, c := range cs { - ids = append(ids, c.ID) - } - return ids, nil -} - -// pickIP chooses the preferred network IP, or the first available. -func pickIP(info types.ContainerJSON, prefer string) string { - if info.NetworkSettings == nil || info.NetworkSettings.Networks == nil { - return "" - } - if prefer != "" { - if es, ok := info.NetworkSettings.Networks[prefer]; ok && es.IPAddress != "" { - return es.IPAddress - } - } - for _, es := range info.NetworkSettings.Networks { - if es.IPAddress != "" { - return es.IPAddress - } - } - return "" -} - -// fetchDiscovery GETs the discovery URL and decodes a JSON array of target groups. -// Any non-200 or non-array payload results in an empty list (not an error). -func fetchDiscovery(hc *http.Client, url string) ([]TargetGroup, error) { - resp, err := hc.Get(url) - if err != nil { - return nil, err - } - defer resp.Body.Close() - - if resp.StatusCode != http.StatusOK { - b, _ := io.ReadAll(io.LimitReader(resp.Body, 4<<10)) - return nil, fmt.Errorf("status %d: %s", resp.StatusCode, string(b)) - } - var out []TargetGroup - if err := json.NewDecoder(resp.Body).Decode(&out); err != nil { - return []TargetGroup{}, nil - } - return out, nil -} - -func targetPort(hostport string) string { - if i := strings.LastIndex(hostport, ":"); i >= 0 && i+1 < len(hostport) { - return hostport[i+1:] - } - return "" -} - -func first(v, def string) string { - if v != "" { - return v - } - return def -} - -// mergeAndDedupe: group by identical label sets and union targets (sorted). -func mergeAndDedupe(in []TargetGroup) []TargetGroup { - byKey := make(map[string]map[string]struct{}) // labelsKey -> targets set - lblOf := make(map[string]map[string]string) - - for _, g := range in { - k := labelsKey(g.Labels) - if byKey[k] == nil { - byKey[k] = make(map[string]struct{}) - lblOf[k] = g.Labels - } - for _, t := range g.Targets { - byKey[k][t] = struct{}{} - } - } - - keys := make([]string, 0, len(byKey)) - for k := range byKey { - keys = append(keys, k) - } - sort.Strings(keys) - - out := make([]TargetGroup, 0, len(keys)) - for _, k := range keys { - ts := make([]string, 0, len(byKey[k])) - for t := range byKey[k] { - ts = append(ts, t) - } - sort.Strings(ts) - out = append(out, TargetGroup{Targets: ts, Labels: lblOf[k]}) - } - return out -} - -// labelsKey builds a canonical string for a label map (sorted key=value lines). -func labelsKey(m map[string]string) string { - if len(m) == 0 { - return "" - } - keys := make([]string, 0, len(m)) - for k := range m { - keys = append(keys, k) - } - sort.Strings(keys) - var b strings.Builder - for _, k := range keys { - b.WriteString(k) - b.WriteString("=") - b.WriteString(m[k]) - b.WriteString("\n") - } - return b.String() -} - -// atomicWriteJSON writes JSON to a temp file and renames it into place. -func atomicWriteJSON(path string, v any) error { - if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil { - return err - } - tmp := path + ".tmp" - f, err := os.Create(tmp) - if err != nil { - return err - } - enc := json.NewEncoder(f) - enc.SetIndent("", " ") - if err := enc.Encode(v); err != nil { - _ = f.Close() - return err - } - if err := f.Close(); err != nil { - return err - } - return os.Rename(tmp, path) -} diff --git a/framework/observability/service_discovery/service_discovery.sh b/framework/observability/service_discovery/service_discovery.sh new file mode 100644 index 000000000..181297abe --- /dev/null +++ b/framework/observability/service_discovery/service_discovery.sh @@ -0,0 +1,110 @@ +#!/usr/bin/env bash +# service_discovery.sh +# Discover Docker containers by label, fetch each container's /discovery JSON, +# add labels (container_name, scrape_path), merge + dedupe, and write a single file_sd JSON. + +set -Eeuo pipefail + +# -------------------- Configuration (via env) -------------------- +LABEL_MATCH="${LABEL_MATCH:-framework=ctf}" # docker ps --filter "label=$LABEL_MATCH" +DEFAULT_PATH="${DISCOVERY_PATH:-/discovery}" # default discovery path inside each container +DEFAULT_PORT="${DISCOVERY_PORT:-6688}" # default discovery port +DEFAULT_SCHEME="${DISCOVERY_SCHEME:-http}" # http or https +PREFER_NETWORK="${NETWORK_NAME:-}" # prefer IP from this Docker network (optional) +OUT="${OUT:-/out/merged.json}" # file_sd output path +SLEEP="${SLEEP:-15}" # seconds between scans +REQUEST_TIMEOUT="${REQUEST_TIMEOUT:-5}" # curl timeout (s) +REWRITE_TO_IP="${REWRITE_TO_IP:-0}" # 1 = replace host with container IP in targets + +# -------------------- Helpers -------------------- +log(){ printf '[sd-merge] %s\n' "$*" >&2; } + +# Atomic writer: reads stdin, writes to $1.tmp, then mv -> $1 +atomic_write(){ + local path="$1" tmp="$1.tmp" + cat > "$tmp" && mv "$tmp" "$path" +} + +# -------------------- Init -------------------- +mkdir -p "$(dirname "$OUT")" +echo '[]' | atomic_write "$OUT" + +# -------------------- Main loop -------------------- +while true; do + # List container IDs matching the label + mapfile -t cids < <(docker ps -q --filter "label=$LABEL_MATCH" || true) + + if ((${#cids[@]} == 0)); then + echo '[]' | atomic_write "$OUT" + log "no matching containers; wrote empty array" + sleep "$SLEEP" + continue + fi + + # Emit each container's (possibly empty) discovery array, then merge once with jq -s + { + for cid in "${cids[@]}"; do + # Inspect once, reuse for IP, name, and labels + inspect="$(docker inspect "$cid" 2>/dev/null || true)" + [[ -z "$inspect" ]] && { log "skip ${cid:0:12}: inspect failed"; echo '[]'; continue; } + + # Resolve container IP (optionally prefer a specific network) + if [[ -n "$PREFER_NETWORK" ]]; then + ip="$(jq -r --arg n "$PREFER_NETWORK" '.[0].NetworkSettings.Networks[$n].IPAddress // ""' <<<"$inspect")" + else + ip="$(jq -r '.[0].NetworkSettings.Networks | to_entries[0].value.IPAddress // ""' <<<"$inspect")" + fi + [[ -z "$ip" ]] && { log "skip ${cid:0:12}: no IP"; echo '[]'; continue; } + + # Container name and optional per-container overrides + name="$(jq -r '.[0].Name | ltrimstr("/")' <<<"$inspect")" + path="$(jq -r '.[0].Config.Labels.prom_sd_path // empty' <<<"$inspect")"; path="${path:-$DEFAULT_PATH}" + port="$(jq -r '.[0].Config.Labels.prom_sd_port // empty' <<<"$inspect")"; port="${port:-$DEFAULT_PORT}" + scheme="$(jq -r '.[0].Config.Labels.prom_sd_scheme // empty' <<<"$inspect")"; scheme="${scheme:-$DEFAULT_SCHEME}" + + url="${scheme}://${ip}:${port}${path}" + + # Fetch discovery JSON; treat errors as empty array + payload="$(curl -fsSL --max-time "$REQUEST_TIMEOUT" "$url" 2>/dev/null || echo '[]')" + + # Normalize to array, add labels, optional host->IP rewrite while keeping port from targets + if [[ "$REWRITE_TO_IP" == "1" ]]; then + jq --arg ip "$ip" --arg name "$name" ' + (if type=="array" then . else [] end) + | map( + .targets |= map( $ip + ":" + (split(":")[1]) ) | + .labels = ((.labels // {}) + { + container_name: $name, + scrape_path: (.labels.__metrics_path__ // "") + }) + ) + ' <<<"$payload" + else + jq --arg name "$name" ' + (if type=="array" then . else [] end) + | map( + .labels = ((.labels // {}) + { + container_name: $name, + scrape_path: (.labels.__metrics_path__ // "") + }) + ) + ' <<<"$payload" + fi + + log "ok $url" + done + } \ + | jq -s ' + # Merge all arrays, coerce to {targets,labels}, then group by labels and dedupe targets + add // [] + | map({targets: (.targets // []), labels: (.labels // {})}) + | group_by(.labels) + | map({ labels: (.[0].labels) + , targets: ([.[].targets[]] | unique | sort) + }) + ' \ + | atomic_write "$OUT" + + log "wrote $(wc -c < "$OUT") bytes to $OUT" + sleep "$SLEEP" +done \ No newline at end of file