diff --git a/.gitignore b/.gitignore index aaadf73..abf0d86 100644 --- a/.gitignore +++ b/.gitignore @@ -27,6 +27,8 @@ go.work.sum # env file .env +# Go artifacts +.cache/go-build/ + # Editor/IDE -# .idea/ -# .vscode/ +.vscode/ diff --git a/Dockerfile b/Dockerfile index 18001df..cf6e228 100644 --- a/Dockerfile +++ b/Dockerfile @@ -14,7 +14,7 @@ RUN CGO_ENABLED=0 go build -o main ./cmd/server/main.go # STAGE 2: Run the app FROM alpine:3.23 -RUN apk --no-cache add ca-certificates +RUN apk --no-cache add ca-certificates curl RUN adduser -D -g '' appuser diff --git a/README.md b/README.md index 0b7db28..8dd71d3 100644 --- a/README.md +++ b/README.md @@ -1,2 +1,66 @@ # example-app A production-ready Go service template featuring a complete observability stack (Prometheus/Grafana), automated CI/CD via GitHub Actions, and Kubernetes deployment. + +# Observability +- Latency (Request latency): a histogram of request durations by endpoint +- Traffic (Request volume): a total request counter by endpoint and response code +- Errors (Status Codes): a total request counter by endpoint and response code +- Saturation: default metrics from Prometheus & Go such as CPU utilization, RAM utilization, Go Goroutines, Go Heap size, file description utilization, etc. + +## Local observability with Docker Compose + +Docker-Compose runs: + +- the Go app on `http://localhost:8080` +- Prometheus on `http://localhost:9090` +- Grafana on `http://localhost:3000` + +### Run the stack + +```bash +docker compose up --build +``` + +Then open: + +- App health endpoint: `http://localhost:8080/health` +- App metrics endpoint: `http://localhost:8080/metrics` +- Prometheus UI: `http://localhost:9090` +- Grafana UI: `http://localhost:3000` using `admin` / `admin` + +### Details about the Configuration + +- Metrics storage uses both a time-based retention limit (`7d`) and a size cap (`512MB`) to keep local disk usage predictable +- Grafana is provisioned automatically so the Prometheus datasource and dashboard are available on first startup +- Docker volumes persist Prometheus and Grafana data across restarts +- The app includes a healthcheck so Prometheus waits until the service is actually ready to scrape + +### How to generate metrics locally + +In another terminal, send a requests in a loop: + +```bash +for i in $(seq 1 40); do curl -s http://localhost:8080/health > /dev/null; done +``` + +After that, you should be able to query these in Prometheus or see them in Grafana: + +- `http_requests_total` +- `http_request_duration_seconds` +- `process_cpu_seconds_total` +- `process_resident_memory_bytes` +- `go_goroutines` +- `go_gc_duration_seconds` +- `go_memstats_heap_sys_bytes` + +### Stop the stack + +```bash +docker compose down +``` + +To also remove persisted Prometheus and Grafana data: + +```bash +docker compose down -v +``` diff --git a/cmd/server/main.go b/cmd/server/main.go index fcf6e3b..6aa4776 100644 --- a/cmd/server/main.go +++ b/cmd/server/main.go @@ -1,12 +1,15 @@ package main import ( + "example-app/internal/debug" "example-app/internal/health" "example-app/internal/middleware" "log/slog" "net/http" "os" "time" + + "github.com/prometheus/client_golang/prometheus/promhttp" ) func main() { @@ -18,7 +21,11 @@ func main() { mux := http.NewServeMux() healthHandler := http.HandlerFunc(health.Handler) - mux.Handle("GET /health", middleware.Logging(healthHandler)) + debugErrorHandler := http.HandlerFunc(debug.ErrorHandler) + + mux.Handle("GET /health", middleware.Logging(middleware.Metrics("/health", healthHandler))) + mux.Handle("GET /debug/error", middleware.Logging(middleware.Metrics("/debug/error", debugErrorHandler))) + mux.Handle("GET /metrics", promhttp.Handler()) port := os.Getenv("PORT") if port == "" { diff --git a/compose.yaml b/compose.yaml new file mode 100644 index 0000000..e8304d9 --- /dev/null +++ b/compose.yaml @@ -0,0 +1,58 @@ +services: + app: + build: + context: . + dockerfile: Dockerfile + container_name: example-app + environment: + PORT: "8080" + ports: + - "8080:8080" + healthcheck: + test: ["CMD-SHELL", "curl -f -s http://localhost:8080/health || exit 1"] + interval: 30s + timeout: 3s + retries: 3 + start_period: 5s + restart: unless-stopped + + prometheus: + image: prom/prometheus:v3.5.0 + container_name: prometheus + command: + - "--config.file=/etc/prometheus/prometheus.yml" + - "--storage.tsdb.path=/prometheus" + - "--storage.tsdb.retention.time=7d" + - "--storage.tsdb.retention.size=512MB" + - "--web.enable-lifecycle" + ports: + - "9090:9090" + volumes: + - ./monitoring/prometheus/prometheus.yml:/etc/prometheus/prometheus.yml:ro + - prometheus-data:/prometheus + depends_on: + app: + condition: service_healthy + restart: unless-stopped + + grafana: + image: grafana/grafana:12.1.1 + container_name: grafana + environment: + GF_SECURITY_ADMIN_USER: ${GRAFANA_USER} + GF_SECURITY_ADMIN_PASSWORD: ${GRAFANA_PASSWORD} + GF_USERS_ALLOW_SIGN_UP: "false" + ports: + - "3000:3000" + volumes: + - ./monitoring/grafana/provisioning:/etc/grafana/provisioning:ro + - ./monitoring/grafana/dashboards:/var/lib/grafana/dashboards:ro + - grafana-data:/var/lib/grafana + depends_on: + prometheus: + condition: service_started + restart: unless-stopped + +volumes: + prometheus-data: + grafana-data: diff --git a/go.mod b/go.mod index bae77e4..1cab513 100644 --- a/go.mod +++ b/go.mod @@ -1,3 +1,20 @@ module example-app go 1.25.6 + +require ( + github.com/prometheus/client_golang v1.23.2 + github.com/prometheus/client_model v0.6.2 +) + +require ( + github.com/beorn7/perks v1.0.1 // indirect + github.com/cespare/xxhash/v2 v2.3.0 // indirect + github.com/kr/text v0.2.0 // indirect + github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect + github.com/prometheus/common v0.66.1 // indirect + github.com/prometheus/procfs v0.16.1 // indirect + go.yaml.in/yaml/v2 v2.4.2 // indirect + golang.org/x/sys v0.35.0 // indirect + google.golang.org/protobuf v1.36.8 // indirect +) diff --git a/go.sum b/go.sum new file mode 100644 index 0000000..d6b8ca9 --- /dev/null +++ b/go.sum @@ -0,0 +1,46 @@ +github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= +github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= +github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= +github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= +github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU= +github.com/klauspost/compress v1.18.0 h1:c/Cqfb0r+Yi+JtIEq73FWXVkRonBlf0CRNYc8Zttxdo= +github.com/klauspost/compress v1.18.0/go.mod h1:2Pp+KzxcywXVXMr50+X0Q/Lsb43OQHYWRCY2AiWywWQ= +github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= +github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= +github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= +github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= +github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc= +github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw= +github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA= +github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/prometheus/client_golang v1.23.2 h1:Je96obch5RDVy3FDMndoUsjAhG5Edi49h0RJWRi/o0o= +github.com/prometheus/client_golang v1.23.2/go.mod h1:Tb1a6LWHB3/SPIzCoaDXI4I8UHKeFTEQ1YCr+0Gyqmg= +github.com/prometheus/client_model v0.6.2 h1:oBsgwpGs7iVziMvrGhE53c/GrLUsZdHnqNwqPLxwZyk= +github.com/prometheus/client_model v0.6.2/go.mod h1:y3m2F6Gdpfy6Ut/GBsUqTWZqCUvMVzSfMLjcu6wAwpE= +github.com/prometheus/common v0.66.1 h1:h5E0h5/Y8niHc5DlaLlWLArTQI7tMrsfQjHV+d9ZoGs= +github.com/prometheus/common v0.66.1/go.mod h1:gcaUsgf3KfRSwHY4dIMXLPV0K/Wg1oZ8+SbZk/HH/dA= +github.com/prometheus/procfs v0.16.1 h1:hZ15bTNuirocR6u0JZ6BAHHmwS1p8B4P6MRqxtzMyRg= +github.com/prometheus/procfs v0.16.1/go.mod h1:teAbpZRB1iIAJYREa1LsoWUXykVXA1KlTmWl8x/U+Is= +github.com/rogpeppe/go-internal v1.10.0 h1:TMyTOH3F/DB16zRVcYyreMH6GnZZrwQVAoYjRBZyWFQ= +github.com/rogpeppe/go-internal v1.10.0/go.mod h1:UQnix2H7Ngw/k4C5ijL5+65zddjncjaFoBhdsK/akog= +github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U= +github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U= +go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= +go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= +go.yaml.in/yaml/v2 v2.4.2 h1:DzmwEr2rDGHl7lsFgAHxmNz/1NlQ7xLIrlN2h5d1eGI= +go.yaml.in/yaml/v2 v2.4.2/go.mod h1:081UH+NErpNdqlCXm3TtEran0rJZGxAYx9hb/ELlsPU= +golang.org/x/sys v0.35.0 h1:vz1N37gP5bs89s7He8XuIYXpyY0+QlsKmzipCbUtyxI= +golang.org/x/sys v0.35.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k= +google.golang.org/protobuf v1.36.8 h1:xHScyCOEuuwZEc6UtSOvPbAT4zRh0xcNRYekJwfqyMc= +google.golang.org/protobuf v1.36.8/go.mod h1:fuxRtAxBytpl4zzqUh6/eyUujkJdNiuEkXntxiD/uRU= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/internal/debug/handler.go b/internal/debug/handler.go new file mode 100644 index 0000000..ea8f69d --- /dev/null +++ b/internal/debug/handler.go @@ -0,0 +1,8 @@ +package debug + +import "net/http" + +// intentional error +func ErrorHandler(w http.ResponseWriter, r *http.Request) { + http.Error(w, "intentional debug error", http.StatusInternalServerError) +} diff --git a/internal/debug/handler_test.go b/internal/debug/handler_test.go new file mode 100644 index 0000000..9bb5504 --- /dev/null +++ b/internal/debug/handler_test.go @@ -0,0 +1,19 @@ +package debug + +import ( + "net/http" + "net/http/httptest" + "testing" +) + +func TestErrorHandler(t *testing.T) { + req := httptest.NewRequest(http.MethodGet, "/debug/error", nil) + recorder := httptest.NewRecorder() + + handler := http.HandlerFunc(ErrorHandler) + handler.ServeHTTP(recorder, req) + + if recorder.Code != http.StatusInternalServerError { + t.Fatalf("unexpected status code: got %d want %d", recorder.Code, http.StatusInternalServerError) + } +} diff --git a/internal/health/handler_test.go b/internal/health/handler_test.go index fe6b7a6..d5f81fa 100644 --- a/internal/health/handler_test.go +++ b/internal/health/handler_test.go @@ -8,7 +8,7 @@ import ( ) func TestHandler(t *testing.T) { - // Create a mock HTTP request to pass to the handler + // mock HTTP request to pass to the handler req, err := http.NewRequest("GET", "/health", nil) if err != nil { t.Fatal(err) @@ -22,24 +22,20 @@ func TestHandler(t *testing.T) { // Call the handler directly handler.ServeHTTP(rr, req) - // Check the status code if status := rr.Code; status != http.StatusOK { t.Errorf("handler returned wrong status code: got %v want %v", status, http.StatusOK) } - // Check the Content-Type header expectedHeader := "application/json" if contentType := rr.Header().Get("Content-Type"); contentType != expectedHeader { t.Errorf("handler returned wrong content type: got %v want %v", contentType, expectedHeader) } - // Check the response body var response HealthResponse if err := json.NewDecoder(rr.Body).Decode(&response); err != nil { t.Fatalf("could not decode JSON response: %v", err) } - // 7. Check the rest of the content in the response ensuring expectations if response.Status != "up" { t.Errorf("handler returned unexpected status: got %v want %v", response.Status, "up") } diff --git a/internal/middleware/metrics.go b/internal/middleware/metrics.go new file mode 100644 index 0000000..b7f251b --- /dev/null +++ b/internal/middleware/metrics.go @@ -0,0 +1,66 @@ +package middleware + +import ( + "net/http" + "strconv" + "time" + + "github.com/prometheus/client_golang/prometheus" +) + +var ( + httpRequestsTotal = prometheus.NewCounterVec( + prometheus.CounterOpts{ + Name: "http_requests_total", + Help: "Total number of HTTP requests processed.", + }, + []string{"route", "code"}, + ) + + httpRequestDurationSeconds = prometheus.NewHistogramVec( + prometheus.HistogramOpts{ + Name: "http_request_duration_seconds", + Help: "Duration of HTTP requests in seconds.", + Buckets: []float64{0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1, 2.5, 5}, + }, + []string{"route"}, + ) +) + +func init() { + prometheus.MustRegister( + httpRequestsTotal, + httpRequestDurationSeconds, + ) +} + +// Request Interceptor +type statusRecorder struct { + http.ResponseWriter + statusCode int +} + +func (recorder *statusRecorder) WriteHeader(statusCode int) { + recorder.statusCode = statusCode + recorder.ResponseWriter.WriteHeader(statusCode) +} + +func Metrics(route string, next http.Handler) http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + start := time.Now() + + recorder := &statusRecorder{ + ResponseWriter: w, + statusCode: http.StatusOK, + } + + next.ServeHTTP(recorder, r) + + durationSeconds := time.Since(start).Seconds() + code := strconv.Itoa(recorder.statusCode) + + httpRequestsTotal.WithLabelValues(route, code).Inc() + + httpRequestDurationSeconds.WithLabelValues(route).Observe(durationSeconds) + }) +} diff --git a/internal/middleware/metrics_test.go b/internal/middleware/metrics_test.go new file mode 100644 index 0000000..6919391 --- /dev/null +++ b/internal/middleware/metrics_test.go @@ -0,0 +1,97 @@ +package middleware + +import ( + "net/http" + "net/http/httptest" + "strconv" + "testing" + + "github.com/prometheus/client_golang/prometheus" + dto "github.com/prometheus/client_model/go" +) + +func TestMetricsRecordsLatencyHistogram(t *testing.T) { + const route = "/test" + + histogramBefore := histogramValue(t, httpRequestDurationSeconds, route) + + handler := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusOK) + }) + + req := httptest.NewRequest(http.MethodGet, route, nil) + recorder := httptest.NewRecorder() + + // Wrap the handler with the metrics middleware and run it + Metrics(route, handler).ServeHTTP(recorder, req) + + histogramAfter := histogramValue(t, httpRequestDurationSeconds, route) + + if histogramAfter.GetSampleCount() != histogramBefore.GetSampleCount()+1 { + t.Fatalf("expected sample count to increase by 1") + } +} + +func TestMetricsRecordsRequestCounterIncrease(t *testing.T) { + const route = "/error" + + statusStr := strconv.Itoa(http.StatusInternalServerError) + totalBefore := counterValue(t, httpRequestsTotal, route, statusStr) + + handler := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + http.Error(w, "service unavailable", http.StatusInternalServerError) + }) + + req := httptest.NewRequest(http.MethodGet, route, nil) + recorder := httptest.NewRecorder() + + // Wrap the handler with the metrics middleware and run it + Metrics(route, handler).ServeHTTP(recorder, req) + + totalAfter := counterValue(t, httpRequestsTotal, route, statusStr) + if totalAfter != totalBefore+1 { + t.Fatalf("unexpected total request count: got %v want %v", totalAfter, totalBefore+1) + } +} + +func counterValue(t *testing.T, collector *prometheus.CounterVec, labels ...string) float64 { + t.Helper() + + metric, err := collector.GetMetricWithLabelValues(labels...) + if err != nil { + t.Fatalf("failed to read counter metric: %v", err) + } + + dtoMetric := &dto.Metric{} + if err := metric.Write(dtoMetric); err != nil { + t.Fatalf("failed to collect counter metric: %v", err) + } + + return dtoMetric.GetCounter().GetValue() +} + +func histogramValue(t *testing.T, collector *prometheus.HistogramVec, labels ...string) *dto.Histogram { + t.Helper() + + observer, err := collector.GetMetricWithLabelValues(labels...) + if err != nil { + t.Fatalf("failed to read histogram metric: %v", err) + } + + exportable, ok := observer.(prometheus.Metric) + if !ok { + t.Fatal("metric does not support data export") + } + + metricDTO := &dto.Metric{} + if err := exportable.Write(metricDTO); err != nil { + t.Fatalf("failed to collect metric state: %v", err) + } + + histogram := metricDTO.GetHistogram() + if histogram == nil { + t.Fatal("collected metric is not a histogram") + } + + return histogram +} diff --git a/monitoring/grafana/dashboards/example-app-overview.json b/monitoring/grafana/dashboards/example-app-overview.json new file mode 100644 index 0000000..aa670db --- /dev/null +++ b/monitoring/grafana/dashboards/example-app-overview.json @@ -0,0 +1,1393 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "grafana", + "uid": "-- Grafana --" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 0, + "id": null, + "links": [], + "panels": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "Average requests per second across all endpoints for the selected dashboard time range.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "orange", + "value": 10 + }, + { + "color": "red", + "value": 50 + } + ] + }, + "unit": "reqps" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 6, + "x": 0, + "y": 0 + }, + "id": 7, + "options": { + "colorMode": "background", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "mean" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "pluginVersion": "12.1.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "sum(rate(http_requests_total[$__rate_interval]))", + "legendFormat": "", + "range": true, + "refId": "A" + } + ], + "title": "Total Request Rate", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "red", + "value": null + }, + { + "color": "yellow", + "value": 0.98 + }, + { + "color": "green", + "value": 0.995 + } + ] + }, + "unit": "percentunit" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 6, + "x": 6, + "y": 0 + }, + "id": 1, + "options": { + "colorMode": "background", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "pluginVersion": "12.1.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "sum(increase(http_requests_total{code=~\"2..\"}[$__range])) / clamp_min(sum(increase(http_requests_total[$__range])), 1)", + "instant": true, + "legendFormat": "", + "range": false, + "refId": "A" + } + ], + "title": "Success Rate", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "Traffic split by the busiest endpoints in the selected time range.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 65, + "gradientMode": "opacity", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "reqps" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 8 + }, + "id": 8, + "options": { + "legend": { + "calcs": [], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "12.1.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "topk(10, sum by (route) (rate(http_requests_total[$__rate_interval])))", + "legendFormat": "{{route}}", + "range": true, + "refId": "A" + } + ], + "title": "Traffic Breakdown by Endpoint", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 70, + "gradientMode": "opacity", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "reqps" + }, + "overrides": [ + { + "matcher": { + "id": "byRegexp", + "options": "^2xx " + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#73BF69", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "^3xx " + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#5794F2", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "^4xx " + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#FF9830", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "^5xx " + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#E02F44", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 0 + }, + "id": 2, + "options": { + "legend": { + "calcs": [], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "12.1.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "sum by (route, status_class) (label_replace(rate(http_requests_total[$__rate_interval]), \"status_class\", \"$1xx\", \"code\", \"([0-9])[0-9][0-9]\"))", + "legendFormat": "{{status_class}} {{route}}", + "range": true, + "refId": "A" + } + ], + "title": "Request Rate by Endpoint and Status Class", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 15, + "gradientMode": "opacity", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 1 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 0, + "y": 16 + }, + "id": 3, + "options": { + "legend": { + "calcs": [], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "12.1.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "sum by (route) (rate(http_request_duration_seconds_sum[$__rate_interval])) / clamp_min(sum by (route) (rate(http_request_duration_seconds_count[$__rate_interval])), 0.0001)", + "legendFormat": "{{route}}", + "range": true, + "refId": "A" + } + ], + "title": "Average Latency by Endpoint", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 15, + "gradientMode": "opacity", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 1 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 12, + "y": 16 + }, + "id": 4, + "options": { + "legend": { + "calcs": [], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "12.1.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.90, sum by (route, le) (rate(http_request_duration_seconds_bucket[$__rate_interval])))", + "legendFormat": "{{route}}", + "range": true, + "refId": "A" + } + ], + "title": "p90 Latency by Endpoint", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 15, + "gradientMode": "opacity", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 1 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 0, + "y": 26 + }, + "id": 5, + "options": { + "legend": { + "calcs": [], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "12.1.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.95, sum by (route, le) (rate(http_request_duration_seconds_bucket[$__rate_interval])))", + "legendFormat": "{{route}}", + "range": true, + "refId": "A" + } + ], + "title": "p95 Latency by Endpoint", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 15, + "gradientMode": "opacity", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 1 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 12, + "y": 26 + }, + "id": 6, + "options": { + "legend": { + "calcs": [], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "12.1.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.99, sum by (route, le) (rate(http_request_duration_seconds_bucket[$__rate_interval])))", + "legendFormat": "{{route}}", + "range": true, + "refId": "A" + } + ], + "title": "p99 Latency by Endpoint", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "CPU used by the Go process as a percentage of a single core.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "opacity", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "percent" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 0, + "y": 36 + }, + "id": 9, + "options": { + "legend": { + "calcs": [], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "12.1.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "rate(process_cpu_seconds_total{job=\"example-app\"}[$__rate_interval]) * 100", + "legendFormat": "cpu used", + "range": true, + "refId": "A" + } + ], + "title": "CPU Utilization", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "Resident memory used by the Go process.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "opacity", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "bytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 8, + "y": 36 + }, + "id": 10, + "options": { + "legend": { + "calcs": [], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "12.1.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "process_resident_memory_bytes{job=\"example-app\"}", + "legendFormat": "rss used", + "range": true, + "refId": "A" + } + ], + "title": "Memory (RAM) Utilization", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "Number of active goroutines in the Go runtime.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "opacity", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 16, + "y": 36 + }, + "id": 11, + "options": { + "legend": { + "calcs": [], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "12.1.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "go_goroutines{job=\"example-app\"}", + "legendFormat": "goroutines", + "range": true, + "refId": "A" + } + ], + "title": "Go Goroutines", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "Go heap reserved from the OS, useful for spotting memory growth before RSS becomes a problem.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "opacity", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "bytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 0, + "y": 44 + }, + "id": 12, + "options": { + "legend": { + "calcs": [], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "12.1.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "go_memstats_heap_sys_bytes{job=\"example-app\"}", + "legendFormat": "heap reserved", + "range": true, + "refId": "A" + } + ], + "title": "Go Heap Reserved", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "GC pause quantiles reported by the Go runtime.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "opacity", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 8, + "y": 44 + }, + "id": 13, + "options": { + "legend": { + "calcs": [], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "12.1.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "go_gc_duration_seconds{job=\"example-app\",quantile=\"0.5\"}", + "legendFormat": "gc pause p50", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "go_gc_duration_seconds{job=\"example-app\",quantile=\"0.99\"}", + "legendFormat": "gc pause p99", + "range": true, + "refId": "B" + } + ], + "title": "GC Pause Duration", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "File descriptor pressure on the process, a useful saturation signal for network services.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "opacity", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "percent" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 16, + "y": 44 + }, + "id": 14, + "options": { + "legend": { + "calcs": [], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "12.1.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "(process_open_fds{job=\"example-app\"} / clamp_min(process_max_fds{job=\"example-app\"}, 1)) * 100", + "legendFormat": "fd used", + "range": true, + "refId": "A" + } + ], + "title": "File Descriptor Utilization", + "type": "timeseries" + } + ], + "refresh": "10s", + "schemaVersion": 41, + "style": "dark", + "tags": [ + "prometheus", + "example-app" + ], + "templating": { + "list": [] + }, + "time": { + "from": "now-15m", + "to": "now" + }, + "timepicker": {}, + "timezone": "browser", + "title": "Example App Overview", + "uid": "example-app-overview", + "version": 4 +} diff --git a/monitoring/grafana/provisioning/dashboards/dashboards.yml b/monitoring/grafana/provisioning/dashboards/dashboards.yml new file mode 100644 index 0000000..0b26595 --- /dev/null +++ b/monitoring/grafana/provisioning/dashboards/dashboards.yml @@ -0,0 +1,11 @@ +apiVersion: 1 + +providers: + - name: example-app + orgId: 1 + folder: Example App + type: file + disableDeletion: false + editable: true + options: + path: /var/lib/grafana/dashboards diff --git a/monitoring/grafana/provisioning/datasources/prometheus.yml b/monitoring/grafana/provisioning/datasources/prometheus.yml new file mode 100644 index 0000000..caa70a8 --- /dev/null +++ b/monitoring/grafana/provisioning/datasources/prometheus.yml @@ -0,0 +1,17 @@ +apiVersion: 1 + +deleteDatasources: + - name: Prometheus + orgId: 1 + +datasources: + - name: Prometheus + uid: prometheus + type: prometheus + access: proxy + url: http://prometheus:9090 + isDefault: true + editable: true + jsonData: + httpMethod: POST + prometheusType: Prometheus diff --git a/monitoring/prometheus/prometheus.yml b/monitoring/prometheus/prometheus.yml new file mode 100644 index 0000000..27f3cff --- /dev/null +++ b/monitoring/prometheus/prometheus.yml @@ -0,0 +1,28 @@ +global: + scrape_interval: 15s + scrape_timeout: 10s + evaluation_interval: 15s + external_labels: + environment: local + service: example-app + +rule_files: [] + +scrape_configs: + - job_name: prometheus + static_configs: + - targets: + - prometheus:9090 + labels: + service: prometheus + + - job_name: example-app + metrics_path: /metrics + scrape_timeout: 5s + honor_timestamps: false + scheme: http + static_configs: + - targets: + - app:8080 + labels: + service: example-app