Added local observability stack with Prometheus, Grafana, and HTTP metrics dashboard

.gitignore

@@ -27,6 +27,8 @@ go.work.sum
 # env file
 .env
 
+# Go artifacts
+.cache/go-build/
+
 # Editor/IDE
-# .idea/
-# .vscode/
+.vscode/

Dockerfile

@@ -14,7 +14,7 @@ RUN CGO_ENABLED=0 go build -o main ./cmd/server/main.go
 # STAGE 2: Run the app
 FROM alpine:3.23
 
-RUN apk --no-cache add ca-certificates
+RUN apk --no-cache add ca-certificates curl
 
 RUN adduser -D -g '' appuser
 

README.md

@@ -1,2 +1,66 @@
 # example-app
 A production-ready Go service template featuring a complete observability stack (Prometheus/Grafana), automated CI/CD via GitHub Actions, and Kubernetes deployment.
+
+# Observability
+- Latency (Request latency): a histogram of request durations by endpoint
+- Traffic (Request volume): a total request counter by endpoint and response code
+- Errors (Status Codes): a total request counter by endpoint and response code
+- Saturation: default metrics from Prometheus & Go such as CPU utilization, RAM utilization, Go Goroutines, Go Heap size, file description utilization, etc.
+
+## Local observability with Docker Compose
+
+Docker-Compose runs:
+
+- the Go app on `http://localhost:8080`
+- Prometheus on `http://localhost:9090`
+- Grafana on `http://localhost:3000`
+
+### Run the stack
+
+```bash
+docker compose up --build
+```
+
+Then open:
+
+- App health endpoint: `http://localhost:8080/health`
+- App metrics endpoint: `http://localhost:8080/metrics`
+- Prometheus UI: `http://localhost:9090`
+- Grafana UI: `http://localhost:3000` using `admin` / `admin`
+
+### Details about the Configuration
+
+- Metrics storage uses both a time-based retention limit (`7d`) and a size cap (`512MB`) to keep local disk usage predictable
+- Grafana is provisioned automatically so the Prometheus datasource and dashboard are available on first startup
+- Docker volumes persist Prometheus and Grafana data across restarts
+- The app includes a healthcheck so Prometheus waits until the service is actually ready to scrape
+
+### How to generate metrics locally
+
+In another terminal, send a requests in a loop:
+
+```bash
+for i in $(seq 1 40); do curl -s http://localhost:8080/health > /dev/null; done
+```
+
+After that, you should be able to query these in Prometheus or see them in Grafana:
+
+- `http_requests_total`
+- `http_request_duration_seconds`
+- `process_cpu_seconds_total`
+- `process_resident_memory_bytes`
+- `go_goroutines`
+- `go_gc_duration_seconds`
+- `go_memstats_heap_sys_bytes`
+
+### Stop the stack
+
+```bash
+docker compose down
+```
+
+To also remove persisted Prometheus and Grafana data:
+
+```bash
+docker compose down -v
+```

cmd/server/main.go

@@ -1,12 +1,15 @@
 package main
 
 import (
+	"example-app/internal/debug"
 	"example-app/internal/health"
 	"example-app/internal/middleware"
 	"log/slog"
 	"net/http"
 	"os"
 	"time"
+
+	"github.com/prometheus/client_golang/prometheus/promhttp"
 )
 
 func main() {
@@ -18,7 +21,11 @@ func main() {
 	mux := http.NewServeMux()
 
 	healthHandler := http.HandlerFunc(health.Handler)
-	mux.Handle("GET /health", middleware.Logging(healthHandler))
+	debugErrorHandler := http.HandlerFunc(debug.ErrorHandler)
+
+	mux.Handle("GET /health", middleware.Logging(middleware.Metrics("/health", healthHandler)))
+	mux.Handle("GET /debug/error", middleware.Logging(middleware.Metrics("/debug/error", debugErrorHandler)))
+	mux.Handle("GET /metrics", promhttp.Handler())
 
 	port := os.Getenv("PORT")
 	if port == "" {

compose.yaml

@@ -0,0 +1,58 @@
+services:
+  app:
+    build:
+      context: .
+      dockerfile: Dockerfile
+    container_name: example-app
+    environment:
+      PORT: "8080"
+    ports:
+      - "8080:8080"
+    healthcheck:
+      test: ["CMD-SHELL", "curl -f -s http://localhost:8080/health || exit 1"]
+      interval: 30s
+      timeout: 3s
+      retries: 3
+      start_period: 5s
+    restart: unless-stopped
+
+  prometheus:
+    image: prom/prometheus:v3.5.0
+    container_name: prometheus
+    command:
+      - "--config.file=/etc/prometheus/prometheus.yml"
+      - "--storage.tsdb.path=/prometheus"
+      - "--storage.tsdb.retention.time=7d"
+      - "--storage.tsdb.retention.size=512MB"
+      - "--web.enable-lifecycle"
+    ports:
+      - "9090:9090"
+    volumes:
+      - ./monitoring/prometheus/prometheus.yml:/etc/prometheus/prometheus.yml:ro
+      - prometheus-data:/prometheus
+    depends_on:
+      app:
+        condition: service_healthy
+    restart: unless-stopped
+
+  grafana:
+    image: grafana/grafana:12.1.1
+    container_name: grafana
+    environment:
+      GF_SECURITY_ADMIN_USER: ${GRAFANA_USER}
+      GF_SECURITY_ADMIN_PASSWORD: ${GRAFANA_PASSWORD}
+      GF_USERS_ALLOW_SIGN_UP: "false"
+    ports:
+      - "3000:3000"
+    volumes:
+      - ./monitoring/grafana/provisioning:/etc/grafana/provisioning:ro
+      - ./monitoring/grafana/dashboards:/var/lib/grafana/dashboards:ro
+      - grafana-data:/var/lib/grafana
+    depends_on:
+      prometheus:
+        condition: service_started
+    restart: unless-stopped
+
+volumes:
+  prometheus-data:
+  grafana-data:

go.mod

@@ -1,3 +1,20 @@
 module example-app
 
 go 1.25.6
+
+require (
+	github.com/prometheus/client_golang v1.23.2
+	github.com/prometheus/client_model v0.6.2
+)
+
+require (
+	github.com/beorn7/perks v1.0.1 // indirect
+	github.com/cespare/xxhash/v2 v2.3.0 // indirect
+	github.com/kr/text v0.2.0 // indirect
+	github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect
+	github.com/prometheus/common v0.66.1 // indirect
+	github.com/prometheus/procfs v0.16.1 // indirect
+	go.yaml.in/yaml/v2 v2.4.2 // indirect
+	golang.org/x/sys v0.35.0 // indirect
+	google.golang.org/protobuf v1.36.8 // indirect
+)

go.sum

@@ -0,0 +1,46 @@
+github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM=
+github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw=
+github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs=
+github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
+github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E=
+github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
+github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8=
+github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU=
+github.com/klauspost/compress v1.18.0 h1:c/Cqfb0r+Yi+JtIEq73FWXVkRonBlf0CRNYc8Zttxdo=
+github.com/klauspost/compress v1.18.0/go.mod h1:2Pp+KzxcywXVXMr50+X0Q/Lsb43OQHYWRCY2AiWywWQ=
+github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE=
+github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk=
+github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
+github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
+github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc=
+github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw=
+github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA=
+github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ=
+github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
+github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
+github.com/prometheus/client_golang v1.23.2 h1:Je96obch5RDVy3FDMndoUsjAhG5Edi49h0RJWRi/o0o=
+github.com/prometheus/client_golang v1.23.2/go.mod h1:Tb1a6LWHB3/SPIzCoaDXI4I8UHKeFTEQ1YCr+0Gyqmg=
+github.com/prometheus/client_model v0.6.2 h1:oBsgwpGs7iVziMvrGhE53c/GrLUsZdHnqNwqPLxwZyk=
+github.com/prometheus/client_model v0.6.2/go.mod h1:y3m2F6Gdpfy6Ut/GBsUqTWZqCUvMVzSfMLjcu6wAwpE=
+github.com/prometheus/common v0.66.1 h1:h5E0h5/Y8niHc5DlaLlWLArTQI7tMrsfQjHV+d9ZoGs=
+github.com/prometheus/common v0.66.1/go.mod h1:gcaUsgf3KfRSwHY4dIMXLPV0K/Wg1oZ8+SbZk/HH/dA=
+github.com/prometheus/procfs v0.16.1 h1:hZ15bTNuirocR6u0JZ6BAHHmwS1p8B4P6MRqxtzMyRg=
+github.com/prometheus/procfs v0.16.1/go.mod h1:teAbpZRB1iIAJYREa1LsoWUXykVXA1KlTmWl8x/U+Is=
+github.com/rogpeppe/go-internal v1.10.0 h1:TMyTOH3F/DB16zRVcYyreMH6GnZZrwQVAoYjRBZyWFQ=
+github.com/rogpeppe/go-internal v1.10.0/go.mod h1:UQnix2H7Ngw/k4C5ijL5+65zddjncjaFoBhdsK/akog=
+github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U=
+github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U=
+go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto=
+go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE=
+go.yaml.in/yaml/v2 v2.4.2 h1:DzmwEr2rDGHl7lsFgAHxmNz/1NlQ7xLIrlN2h5d1eGI=
+go.yaml.in/yaml/v2 v2.4.2/go.mod h1:081UH+NErpNdqlCXm3TtEran0rJZGxAYx9hb/ELlsPU=
+golang.org/x/sys v0.35.0 h1:vz1N37gP5bs89s7He8XuIYXpyY0+QlsKmzipCbUtyxI=
+golang.org/x/sys v0.35.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k=
+google.golang.org/protobuf v1.36.8 h1:xHScyCOEuuwZEc6UtSOvPbAT4zRh0xcNRYekJwfqyMc=
+google.golang.org/protobuf v1.36.8/go.mod h1:fuxRtAxBytpl4zzqUh6/eyUujkJdNiuEkXntxiD/uRU=
+gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk=
+gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q=
+gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
+gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=

internal/debug/handler.go

@@ -0,0 +1,8 @@
+package debug
+
+import "net/http"
+
+// intentional error
+func ErrorHandler(w http.ResponseWriter, r *http.Request) {
+	http.Error(w, "intentional debug error", http.StatusInternalServerError)
+}

internal/debug/handler_test.go

@@ -0,0 +1,19 @@
+package debug
+
+import (
+	"net/http"
+	"net/http/httptest"
+	"testing"
+)
+
+func TestErrorHandler(t *testing.T) {
+	req := httptest.NewRequest(http.MethodGet, "/debug/error", nil)
+	recorder := httptest.NewRecorder()
+
+	handler := http.HandlerFunc(ErrorHandler)
+	handler.ServeHTTP(recorder, req)
+
+	if recorder.Code != http.StatusInternalServerError {
+		t.Fatalf("unexpected status code: got %d want %d", recorder.Code, http.StatusInternalServerError)
+	}
+}

internal/health/handler_test.go

@@ -8,7 +8,7 @@ import (
 )
 
 func TestHandler(t *testing.T) {
-	// Create a mock HTTP request to pass to the handler
+	// mock HTTP request to pass to the handler
 	req, err := http.NewRequest("GET", "/health", nil)
 	if err != nil {
 		t.Fatal(err)
@@ -22,24 +22,20 @@ func TestHandler(t *testing.T) {
 	// Call the handler directly
 	handler.ServeHTTP(rr, req)
 
-	// Check the status code
 	if status := rr.Code; status != http.StatusOK {
 		t.Errorf("handler returned wrong status code: got %v want %v", status, http.StatusOK)
 	}
 
-	// Check the Content-Type header
 	expectedHeader := "application/json"
 	if contentType := rr.Header().Get("Content-Type"); contentType != expectedHeader {
 		t.Errorf("handler returned wrong content type: got %v want %v", contentType, expectedHeader)
 	}
 
-	// Check the response body
 	var response HealthResponse
 	if err := json.NewDecoder(rr.Body).Decode(&response); err != nil {
 		t.Fatalf("could not decode JSON response: %v", err)
 	}
 
-	// 7. Check the rest of the content in the response ensuring expectations
 	if response.Status != "up" {
 		t.Errorf("handler returned unexpected status: got %v want %v", response.Status, "up")
 	}

internal/middleware/metrics.go

@@ -0,0 +1,66 @@
+package middleware
+
+import (
+	"net/http"
+	"strconv"
+	"time"
+
+	"github.com/prometheus/client_golang/prometheus"
+)
+
+var (
+	httpRequestsTotal = prometheus.NewCounterVec(
+		prometheus.CounterOpts{
+			Name: "http_requests_total",
+			Help: "Total number of HTTP requests processed.",
+		},
+		[]string{"route", "code"},
+	)
+
+	httpRequestDurationSeconds = prometheus.NewHistogramVec(
+		prometheus.HistogramOpts{
+			Name:    "http_request_duration_seconds",
+			Help:    "Duration of HTTP requests in seconds.",
+			Buckets: []float64{0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1, 2.5, 5},
+		},
+		[]string{"route"},
+	)
+)
+
+func init() {
+	prometheus.MustRegister(
+		httpRequestsTotal,
+		httpRequestDurationSeconds,
+	)
+}
+
+// Request Interceptor
+type statusRecorder struct {
+	http.ResponseWriter
+	statusCode int
+}
+
+func (recorder *statusRecorder) WriteHeader(statusCode int) {
+	recorder.statusCode = statusCode
+	recorder.ResponseWriter.WriteHeader(statusCode)
+}
+
+func Metrics(route string, next http.Handler) http.Handler {
+	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		start := time.Now()
+
+		recorder := &statusRecorder{
+			ResponseWriter: w,
+			statusCode:     http.StatusOK,
+		}
+
+		next.ServeHTTP(recorder, r)
+
+		durationSeconds := time.Since(start).Seconds()
+		code := strconv.Itoa(recorder.statusCode)
+
+		httpRequestsTotal.WithLabelValues(route, code).Inc()
+
+		httpRequestDurationSeconds.WithLabelValues(route).Observe(durationSeconds)
+	})
+}