diff --git a/alert.rules.yml b/alert.rules.yml index 8b3e115..04633be 100644 --- a/alert.rules.yml +++ b/alert.rules.yml @@ -11,7 +11,7 @@ groups: description: "{{ $labels.job }} has been unreachable for more than 30 seconds." - alert: HighErrorRate - expr: rate(http_requests_total{status=~"5.."}[5m]) > 0.05 + expr: rate(atlas_http_requests_total{status_code=~"5.."}[5m]) > 0.05 for: 1m labels: severity: warning diff --git a/prometheus.yml b/prometheus.yml index 828bd37..2e98c63 100644 --- a/prometheus.yml +++ b/prometheus.yml @@ -79,3 +79,11 @@ scrape_configs: - job_name: 'ai-service' static_configs: - targets: ['ai-service:8065'] + + - job_name: 'live-service' + static_configs: + - targets: ['live-service:8060'] + + - job_name: 'workforce-planning-service' + static_configs: + - targets: ['workforce-planning-service:8017'] diff --git a/services/ai-copilot-service/main.py b/services/ai-copilot-service/main.py index 387c5ec..8d70f20 100644 --- a/services/ai-copilot-service/main.py +++ b/services/ai-copilot-service/main.py @@ -86,7 +86,7 @@ async def lifespan(app: FastAPI): @app.middleware("http") async def internal_auth_middleware(request: Request, call_next): - if request.url.path == "/health": + if request.url.path in ("/health", "/metrics"): return await call_next(request) auth_header = request.headers.get("x-internal-auth") diff --git a/services/ai-service/main.py b/services/ai-service/main.py index dda4206..a3c4884 100644 --- a/services/ai-service/main.py +++ b/services/ai-service/main.py @@ -95,7 +95,7 @@ async def lifespan(app: FastAPI): @app.middleware("http") async def internal_auth_middleware(request: Request, call_next): - if request.url.path == "/health": + if request.url.path in ("/health", "/metrics"): return await call_next(request) auth_header = request.headers.get("x-internal-auth") diff --git a/services/analytics-python-service/main.py b/services/analytics-python-service/main.py index b457e99..48cc439 100644 --- a/services/analytics-python-service/main.py +++ b/services/analytics-python-service/main.py @@ -90,7 +90,7 @@ def custom_openapi(): @app.middleware("http") async def internal_auth_middleware(request: Request, call_next): - if request.url.path == "/health": + if request.url.path in ("/health", "/metrics"): return await call_next(request) auth_header = request.headers.get("x-internal-auth") diff --git a/services/api-gateway-node/index.js b/services/api-gateway-node/index.js index 839057c..ec55bac 100644 --- a/services/api-gateway-node/index.js +++ b/services/api-gateway-node/index.js @@ -13,8 +13,30 @@ const crypto = require('crypto'); const cookieParser = require('cookie-parser'); const dns = require('dns'); const { promisify } = require('util'); +const promClient = require('prom-client'); const resolveDns = promisify(dns.resolve4); +const httpRequestCount = new promClient.Counter({ + name: 'atlas_http_requests_total', + help: 'Total HTTP requests', + labelNames: ['method', 'path', 'status_code'], +}); + +const httpRequestDuration = new promClient.Histogram({ + name: 'atlas_http_request_duration_seconds', + help: 'HTTP request duration in seconds', + labelNames: ['method', 'path', 'status_code'], + buckets: [0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1.0, 2.5, 5.0, 10.0], +}); + +const httpRequestsInProgress = new promClient.Gauge({ + name: 'atlas_http_requests_in_progress', + help: 'Number of HTTP requests in progress', + labelNames: ['method', 'path'], +}); + +promClient.collectDefaultMetrics(); + const REDIS_URL = process.env.REDIS_URL || 'redis://redis:6379'; const redisClient = redis.createClient({ url: REDIS_URL }); @@ -136,6 +158,23 @@ const globalLimiter = rateLimit({ }); app.use(globalLimiter); +function metricsMiddleware(req, res, next) { + if (req.path === '/metrics' || req.path === '/health') { + return next(); + } + const path = req.path.replace(/\/[0-9a-fA-F-]{36}|\/\d+/g, '/:param'); + httpRequestsInProgress.labels({ method: req.method, path }).inc(); + const start = Date.now(); + res.on('finish', () => { + const duration = (Date.now() - start) / 1000; + httpRequestsInProgress.labels({ method: req.method, path }).dec(); + httpRequestCount.labels({ method: req.method, path, status_code: res.statusCode }).inc(); + httpRequestDuration.labels({ method: req.method, path, status_code: res.statusCode }).observe(duration); + }); + next(); +} +app.use(metricsMiddleware); + const authLimiter = rateLimit({ windowMs: 15 * 60 * 1000, max: 30, @@ -298,7 +337,7 @@ const ALLOWED_WS_PATHS = new Set(['/ws', '/notification/ws']); const PUBLIC_AUTH_PATHS = ['/api/auth/login', '/api/auth/register']; function isPublicPath(path) { - if (path === '/health') return true; + if (path === '/health' || path === '/metrics') return true; return PUBLIC_AUTH_PATHS.some((p) => path === p || path.startsWith(p + '?')); } @@ -775,3 +814,8 @@ server.on('upgrade', (req, socket, head) => { app.get('/health', (req, res) => { res.status(200).json({ status: 'API Gateway is running' }); }); + +app.get('/metrics', async (req, res) => { + res.set('Content-Type', promClient.register.contentType); + res.end(await promClient.register.metrics()); +}); diff --git a/services/api-gateway-node/package.json b/services/api-gateway-node/package.json index 17b5abc..83ce812 100644 --- a/services/api-gateway-node/package.json +++ b/services/api-gateway-node/package.json @@ -24,6 +24,7 @@ "http-proxy-middleware": "^3.0.5", "jsonwebtoken": "^9.0.3", "morgan": "^1.10.1", + "prom-client": "^15.1.3", "redis": "^4.6.10" } } diff --git a/services/ats-service/main.py b/services/ats-service/main.py index e7b4371..40f5f70 100644 --- a/services/ats-service/main.py +++ b/services/ats-service/main.py @@ -80,7 +80,7 @@ async def lifespan(app: FastAPI): @app.middleware("http") async def internal_auth_middleware(request: Request, call_next): - if request.url.path == "/health": + if request.url.path in ("/health", "/metrics"): return await call_next(request) auth_header = request.headers.get("x-internal-auth") diff --git a/services/attendance-service/go.mod b/services/attendance-service/go.mod index e2485a1..96311c1 100644 --- a/services/attendance-service/go.mod +++ b/services/attendance-service/go.mod @@ -3,8 +3,10 @@ module atlas/attendance-service go 1.21 require ( + github.com/gofiber/adaptor/v2 v2.2.1 github.com/gofiber/fiber/v2 v2.50.0 github.com/golang-jwt/jwt/v5 v5.2.1 + github.com/prometheus/client_golang v1.19.0 github.com/streadway/amqp v1.1.0 gorm.io/driver/postgres v1.5.2 gorm.io/gorm v1.25.4 diff --git a/services/attendance-service/main.go b/services/attendance-service/main.go index e955ce3..5ff4326 100644 --- a/services/attendance-service/main.go +++ b/services/attendance-service/main.go @@ -5,15 +5,51 @@ import ( "fmt" "log" "os" + "regexp" "time" + "github.com/gofiber/adaptor/v2" "github.com/gofiber/fiber/v2" "github.com/gofiber/fiber/v2/middleware/cors" + "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/client_golang/prometheus/promhttp" "github.com/streadway/amqp" "gorm.io/driver/postgres" "gorm.io/gorm" ) +var ( + httpRequestCount = prometheus.NewCounterVec( + prometheus.CounterOpts{ + Name: "atlas_http_requests_total", + Help: "Total HTTP requests", + }, + []string{"method", "path", "status_code"}, + ) + httpRequestDuration = prometheus.NewHistogramVec( + prometheus.HistogramOpts{ + Name: "atlas_http_request_duration_seconds", + Help: "HTTP request duration in seconds", + Buckets: []float64{0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1.0, 2.5, 5.0, 10.0}, + }, + []string{"method", "path", "status_code"}, + ) + httpRequestsInProgress = prometheus.NewGaugeVec( + prometheus.GaugeOpts{ + Name: "atlas_http_requests_in_progress", + Help: "Number of HTTP requests in progress", + }, + []string{"method", "path"}, + ) + pathParamPattern = regexp.MustCompile(`/[0-9a-fA-F-]{36}|/\d+`) +) + +func init() { + prometheus.MustRegister(httpRequestCount) + prometheus.MustRegister(httpRequestDuration) + prometheus.MustRegister(httpRequestsInProgress) +} + var db *gorm.DB var rabbitChan *amqp.Channel @@ -246,6 +282,28 @@ func main() { AllowHeaders: "Origin,Content-Type,Accept,Authorization,X-Tenant-Id,X-Employee-Id", })) + app.Use(func(c *fiber.Ctx) error { + path := pathParamPattern.ReplaceAllString(c.Path(), "/:param") + httpRequestsInProgress.WithLabelValues(c.Method(), path).Inc() + start := time.Now() + err := c.Next() + duration := time.Since(start).Seconds() + status := fiber.StatusInternalServerError + if err != nil { + if e, ok := err.(*fiber.Error); ok { + status = e.Code + } + } else { + status = c.Response().StatusCode() + } + httpRequestsInProgress.WithLabelValues(c.Method(), path).Dec() + httpRequestCount.WithLabelValues(c.Method(), path, fmt.Sprintf("%d", status)).Inc() + httpRequestDuration.WithLabelValues(c.Method(), path, fmt.Sprintf("%d", status)).Observe(duration) + return err + }) + + app.Get("/metrics", adaptor.HTTPHandler(promhttp.Handler())) + app.Get("/health", func(c *fiber.Ctx) error { return c.JSON(fiber.Map{"status": "Attendance Service is running", "version": "2.0.0"}) }) diff --git a/services/audit-compliance-service/main.py b/services/audit-compliance-service/main.py index 30266fb..954346a 100644 --- a/services/audit-compliance-service/main.py +++ b/services/audit-compliance-service/main.py @@ -150,10 +150,11 @@ async def lifespan(app: FastAPI): ) app.add_middleware(SecurityHeadersMiddleware) +app.add_middleware(AtlasMetricsMiddleware) @app.middleware("http") async def internal_auth_middleware(request: Request, call_next): - if request.url.path == "/health": + if request.url.path in ("/health", "/metrics"): return await call_next(request) auth_header = request.headers.get("x-internal-auth") diff --git a/services/auth-service/index.js b/services/auth-service/index.js index 73fc82f..4fef2b2 100644 --- a/services/auth-service/index.js +++ b/services/auth-service/index.js @@ -21,6 +21,29 @@ const { verifyAuthenticationResponse, } = require('@simplewebauthn/server'); +const promClient = require('prom-client'); + +const httpRequestCount = new promClient.Counter({ + name: 'atlas_http_requests_total', + help: 'Total HTTP requests', + labelNames: ['method', 'path', 'status_code'], +}); + +const httpRequestDuration = new promClient.Histogram({ + name: 'atlas_http_request_duration_seconds', + help: 'HTTP request duration in seconds', + labelNames: ['method', 'path', 'status_code'], + buckets: [0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1.0, 2.5, 5.0, 10.0], +}); + +const httpRequestsInProgress = new promClient.Gauge({ + name: 'atlas_http_requests_in_progress', + help: 'Number of HTTP requests in progress', + labelNames: ['method', 'path'], +}); + +promClient.collectDefaultMetrics(); + const app = express(); app.use(helmet()); app.use(cookieParser()); @@ -44,6 +67,23 @@ app.use( }) ); +function metricsMiddleware(req, res, next) { + if (req.path === '/metrics' || req.path === '/health') { + return next(); + } + const path = req.path.replace(/\/[0-9a-fA-F-]{36}|\/\d+/g, '/:param'); + httpRequestsInProgress.labels({ method: req.method, path }).inc(); + const start = Date.now(); + res.on('finish', () => { + const duration = (Date.now() - start) / 1000; + httpRequestsInProgress.labels({ method: req.method, path }).dec(); + httpRequestCount.labels({ method: req.method, path, status_code: res.statusCode }).inc(); + httpRequestDuration.labels({ method: req.method, path, status_code: res.statusCode }).observe(duration); + }); + next(); +} +app.use(metricsMiddleware); + const PORT = process.env.PORT || 8010; const NODE_ENV = process.env.NODE_ENV || 'development'; const JWT_SECRET = process.env.JWT_SECRET; @@ -2268,6 +2308,11 @@ app.get('/health', (req, res) => { res.status(200).json({ status: 'Auth Service is healthy' }); }); +app.get('/metrics', async (req, res) => { + res.set('Content-Type', promClient.register.contentType); + res.end(await promClient.register.metrics()); +}); + app.listen(PORT, () => { console.log(`Auth service running on port ${PORT}`); }); diff --git a/services/auth-service/package.json b/services/auth-service/package.json index afab0d6..de58ccd 100644 --- a/services/auth-service/package.json +++ b/services/auth-service/package.json @@ -25,6 +25,7 @@ "jsonwebtoken": "^9.0.3", "otplib": "^12.0.1", "pg": "^8.20.0", + "prom-client": "^15.1.3", "uuid": "^9.0.1", "xml-crypto": "^6.0.0" } diff --git a/services/employee-lifecycle-service/main.py b/services/employee-lifecycle-service/main.py index a29f0c7..420f22e 100644 --- a/services/employee-lifecycle-service/main.py +++ b/services/employee-lifecycle-service/main.py @@ -69,7 +69,7 @@ async def lifespan(app: FastAPI): @app.middleware("http") async def internal_auth_middleware(request: Request, call_next): - if request.url.path == "/health": + if request.url.path in ("/health", "/metrics"): return await call_next(request) auth_header = request.headers.get("x-internal-auth") diff --git a/services/integration-service/main.py b/services/integration-service/main.py index b7d5121..cd9b8a8 100644 --- a/services/integration-service/main.py +++ b/services/integration-service/main.py @@ -144,10 +144,11 @@ def _background_loop(): ) app.add_middleware(SecurityHeadersMiddleware) +app.add_middleware(AtlasMetricsMiddleware) @app.middleware("http") async def internal_auth_middleware(request: Request, call_next): - if request.url.path == "/health": + if request.url.path in ("/health", "/metrics"): return await call_next(request) auth_header = request.headers.get("x-internal-auth") diff --git a/services/live-service/main.py b/services/live-service/main.py index 376e991..88b54c4 100644 --- a/services/live-service/main.py +++ b/services/live-service/main.py @@ -344,7 +344,7 @@ def validate_internal_jwt(auth_header: str) -> dict: @app.middleware("http") async def internal_auth_middleware(request: Request, call_next): - if request.url.path == "/health": + if request.url.path in ("/health", "/metrics"): return await call_next(request) auth_header = request.headers.get("x-internal-auth") diff --git a/services/lms-service/go.mod b/services/lms-service/go.mod index 6fea731..0858ba2 100644 --- a/services/lms-service/go.mod +++ b/services/lms-service/go.mod @@ -3,10 +3,12 @@ module github.com/atlas-workforce/lms-service go 1.21 require ( + github.com/gofiber/adaptor/v2 v2.2.1 github.com/gofiber/fiber/v2 v2.52.0 github.com/golang-jwt/jwt/v5 v5.2.1 github.com/google/uuid v1.6.0 github.com/lib/pq v1.10.9 + github.com/prometheus/client_golang v1.19.0 gorm.io/datatypes v1.2.0 gorm.io/driver/postgres v1.5.6 gorm.io/gorm v1.25.7 diff --git a/services/lms-service/main.go b/services/lms-service/main.go index adae355..6896532 100644 --- a/services/lms-service/main.go +++ b/services/lms-service/main.go @@ -4,19 +4,55 @@ import ( "fmt" "log" "os" + "regexp" "time" "github.com/atlas-workforce/lms-service/handlers" "github.com/atlas-workforce/lms-service/middleware" "github.com/atlas-workforce/lms-service/models" + "github.com/gofiber/adaptor/v2" "github.com/gofiber/fiber/v2" "github.com/gofiber/fiber/v2/middleware/cors" "github.com/gofiber/fiber/v2/middleware/logger" "github.com/gofiber/fiber/v2/middleware/recover" + "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/client_golang/prometheus/promhttp" "gorm.io/driver/postgres" "gorm.io/gorm" ) +var ( + httpRequestCount = prometheus.NewCounterVec( + prometheus.CounterOpts{ + Name: "atlas_http_requests_total", + Help: "Total HTTP requests", + }, + []string{"method", "path", "status_code"}, + ) + httpRequestDuration = prometheus.NewHistogramVec( + prometheus.HistogramOpts{ + Name: "atlas_http_request_duration_seconds", + Help: "HTTP request duration in seconds", + Buckets: []float64{0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1.0, 2.5, 5.0, 10.0}, + }, + []string{"method", "path", "status_code"}, + ) + httpRequestsInProgress = prometheus.NewGaugeVec( + prometheus.GaugeOpts{ + Name: "atlas_http_requests_in_progress", + Help: "Number of HTTP requests in progress", + }, + []string{"method", "path"}, + ) + pathParamPattern = regexp.MustCompile(`/[0-9a-fA-F-]{36}|/\d+`) +) + +func init() { + prometheus.MustRegister(httpRequestCount) + prometheus.MustRegister(httpRequestDuration) + prometheus.MustRegister(httpRequestsInProgress) +} + var db *gorm.DB func main() { @@ -76,6 +112,29 @@ func main() { app.Use(recover.New()) app.Use(logger.New()) app.Use(cors.New()) + + app.Use(func(c *fiber.Ctx) error { + path := pathParamPattern.ReplaceAllString(c.Path(), "/:param") + httpRequestsInProgress.WithLabelValues(c.Method(), path).Inc() + start := time.Now() + err := c.Next() + duration := time.Since(start).Seconds() + status := fiber.StatusInternalServerError + if err != nil { + if e, ok := err.(*fiber.Error); ok { + status = e.Code + } + } else { + status = c.Response().StatusCode() + } + httpRequestsInProgress.WithLabelValues(c.Method(), path).Dec() + httpRequestCount.WithLabelValues(c.Method(), path, fmt.Sprintf("%d", status)).Inc() + httpRequestDuration.WithLabelValues(c.Method(), path, fmt.Sprintf("%d", status)).Observe(duration) + return err + }) + + app.Get("/metrics", adaptor.HTTPHandler(promhttp.Handler())) + middleware.InitAuth() app.Use(middleware.AuthMiddleware()) app.Use(middleware.TenantMiddleware()) diff --git a/services/notification-go-service/go.mod b/services/notification-go-service/go.mod index c8d077a..7566da5 100644 --- a/services/notification-go-service/go.mod +++ b/services/notification-go-service/go.mod @@ -6,6 +6,7 @@ require ( github.com/golang-jwt/jwt/v5 v5.3.1 github.com/google/uuid v1.6.0 github.com/gorilla/websocket v1.5.1 + github.com/prometheus/client_golang v1.19.0 github.com/streadway/amqp v1.1.0 ) diff --git a/services/notification-go-service/main.go b/services/notification-go-service/main.go index 029cf5c..59fba8c 100644 --- a/services/notification-go-service/main.go +++ b/services/notification-go-service/main.go @@ -6,6 +6,7 @@ import ( "log" "net/http" "os" + "regexp" "strings" "sync" "time" @@ -13,9 +14,43 @@ import ( "github.com/golang-jwt/jwt/v5" "github.com/google/uuid" "github.com/gorilla/websocket" + "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/client_golang/prometheus/promhttp" "github.com/streadway/amqp" ) +var ( + httpRequestCount = prometheus.NewCounterVec( + prometheus.CounterOpts{ + Name: "atlas_http_requests_total", + Help: "Total HTTP requests", + }, + []string{"method", "path", "status_code"}, + ) + httpRequestDuration = prometheus.NewHistogramVec( + prometheus.HistogramOpts{ + Name: "atlas_http_request_duration_seconds", + Help: "HTTP request duration in seconds", + Buckets: []float64{0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1.0, 2.5, 5.0, 10.0}, + }, + []string{"method", "path", "status_code"}, + ) + httpRequestsInProgress = prometheus.NewGaugeVec( + prometheus.GaugeOpts{ + Name: "atlas_http_requests_in_progress", + Help: "Number of HTTP requests in progress", + }, + []string{"method", "path"}, + ) + pathParamPattern = regexp.MustCompile(`/[0-9a-fA-F-]{36}|/\d+`) +) + +func init() { + prometheus.MustRegister(httpRequestCount) + prometheus.MustRegister(httpRequestDuration) + prometheus.MustRegister(httpRequestsInProgress) +} + var upgrader = websocket.Upgrader{ CheckOrigin: func(r *http.Request) bool { origin := r.Header.Get("Origin") @@ -450,6 +485,34 @@ func processMessage(body []byte) { broadcast <- BroadcastMessage{TenantID: tenantID, Payload: jsonMsg} } +func metricsMiddleware(next http.Handler) http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path == "/metrics" || r.URL.Path == "/health" { + next.ServeHTTP(w, r) + return + } + path := pathParamPattern.ReplaceAllString(r.URL.Path, "/:param") + httpRequestsInProgress.WithLabelValues(r.Method, path).Inc() + start := time.Now() + sw := &statusWriter{ResponseWriter: w, statusCode: http.StatusOK} + next.ServeHTTP(sw, r) + duration := time.Since(start).Seconds() + httpRequestsInProgress.WithLabelValues(r.Method, path).Dec() + httpRequestCount.WithLabelValues(r.Method, path, fmt.Sprintf("%d", sw.statusCode)).Inc() + httpRequestDuration.WithLabelValues(r.Method, path, fmt.Sprintf("%d", sw.statusCode)).Observe(duration) + }) +} + +type statusWriter struct { + http.ResponseWriter + statusCode int +} + +func (sw *statusWriter) WriteHeader(code int) { + sw.statusCode = code + sw.ResponseWriter.WriteHeader(code) +} + func main() { port := os.Getenv("PORT") if port == "" { @@ -461,6 +524,9 @@ func main() { mux := http.NewServeMux() + // Metrics + mux.Handle("/metrics", promhttp.Handler()) + // Health mux.HandleFunc("/health", healthHandler) @@ -489,7 +555,7 @@ func main() { })) log.Printf("Notification Service listening on port %s", port) - if err := http.ListenAndServe(":"+port, mux); err != nil { + if err := http.ListenAndServe(":"+port, metricsMiddleware(mux)); err != nil { log.Fatalf("Failed to start server: %v", err) } } diff --git a/services/security-service/main.py b/services/security-service/main.py index b9a0069..f1e4da8 100644 --- a/services/security-service/main.py +++ b/services/security-service/main.py @@ -107,10 +107,11 @@ async def lifespan(app: FastAPI): ) app.add_middleware(SecurityHeadersMiddleware) +app.add_middleware(AtlasMetricsMiddleware) @app.middleware("http") async def internal_auth_middleware(request: Request, call_next): - if request.url.path == "/health": + if request.url.path in ("/health", "/metrics"): return await call_next(request) auth_header = request.headers.get("x-internal-auth") diff --git a/services/workforce-planning-service/main.py b/services/workforce-planning-service/main.py index 522ce3c..3755b04 100644 --- a/services/workforce-planning-service/main.py +++ b/services/workforce-planning-service/main.py @@ -69,7 +69,7 @@ async def lifespan(app: FastAPI): @app.middleware("http") async def internal_auth_middleware(request: Request, call_next): - if request.url.path == "/health": + if request.url.path in ("/health", "/metrics"): return await call_next(request) auth_header = request.headers.get("x-internal-auth")