fix: expose /metrics endpoint on all services for Prometheus scraping

alert.rules.yml

@@ -11,7 +11,7 @@ groups:
           description: "{{ $labels.job }} has been unreachable for more than 30 seconds."
 
       - alert: HighErrorRate
-        expr: rate(http_requests_total{status=~"5.."}[5m]) > 0.05
+        expr: rate(atlas_http_requests_total{status_code=~"5.."}[5m]) > 0.05
         for: 1m
         labels:
           severity: warning

prometheus.yml

@@ -79,3 +79,11 @@ scrape_configs:
   - job_name: 'ai-service'
     static_configs:
       - targets: ['ai-service:8065']
+
+  - job_name: 'live-service'
+    static_configs:
+      - targets: ['live-service:8060']
+
+  - job_name: 'workforce-planning-service'
+    static_configs:
+      - targets: ['workforce-planning-service:8017']

services/ai-copilot-service/main.py

@@ -86,7 +86,7 @@ async def lifespan(app: FastAPI):
 
 @app.middleware("http")
 async def internal_auth_middleware(request: Request, call_next):
-    if request.url.path == "/health":
+    if request.url.path in ("/health", "/metrics"):
         return await call_next(request)
 
     auth_header = request.headers.get("x-internal-auth")

services/ai-service/main.py

@@ -95,7 +95,7 @@ async def lifespan(app: FastAPI):
 
 @app.middleware("http")
 async def internal_auth_middleware(request: Request, call_next):
-    if request.url.path == "/health":
+    if request.url.path in ("/health", "/metrics"):
         return await call_next(request)
 
     auth_header = request.headers.get("x-internal-auth")

services/analytics-python-service/main.py

@@ -90,7 +90,7 @@ def custom_openapi():
 
 @app.middleware("http")
 async def internal_auth_middleware(request: Request, call_next):
-    if request.url.path == "/health":
+    if request.url.path in ("/health", "/metrics"):
         return await call_next(request)
 
     auth_header = request.headers.get("x-internal-auth")

services/api-gateway-node/index.js

@@ -13,8 +13,30 @@ const crypto = require('crypto');
 const cookieParser = require('cookie-parser');
 const dns = require('dns');
 const { promisify } = require('util');
+const promClient = require('prom-client');
 const resolveDns = promisify(dns.resolve4);
 
+const httpRequestCount = new promClient.Counter({
+  name: 'atlas_http_requests_total',
+  help: 'Total HTTP requests',
+  labelNames: ['method', 'path', 'status_code'],
+});
+
+const httpRequestDuration = new promClient.Histogram({
+  name: 'atlas_http_request_duration_seconds',
+  help: 'HTTP request duration in seconds',
+  labelNames: ['method', 'path', 'status_code'],
+  buckets: [0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1.0, 2.5, 5.0, 10.0],
+});
+
+const httpRequestsInProgress = new promClient.Gauge({
+  name: 'atlas_http_requests_in_progress',
+  help: 'Number of HTTP requests in progress',
+  labelNames: ['method', 'path'],
+});
+
+promClient.collectDefaultMetrics();
+
 const REDIS_URL = process.env.REDIS_URL || 'redis://redis:6379';
 const redisClient = redis.createClient({ url: REDIS_URL });
 
@@ -136,6 +158,23 @@ const globalLimiter = rateLimit({
 });
 app.use(globalLimiter);
 
+function metricsMiddleware(req, res, next) {
+  if (req.path === '/metrics' || req.path === '/health') {
+    return next();
+  }
+  const path = req.path.replace(/\/[0-9a-fA-F-]{36}|\/\d+/g, '/:param');
+  httpRequestsInProgress.labels({ method: req.method, path }).inc();
+  const start = Date.now();
+  res.on('finish', () => {
+    const duration = (Date.now() - start) / 1000;
+    httpRequestsInProgress.labels({ method: req.method, path }).dec();
+    httpRequestCount.labels({ method: req.method, path, status_code: res.statusCode }).inc();
+    httpRequestDuration.labels({ method: req.method, path, status_code: res.statusCode }).observe(duration);
+  });
+  next();
+}
+app.use(metricsMiddleware);
+
 const authLimiter = rateLimit({
   windowMs: 15 * 60 * 1000,
   max: 30,
@@ -298,7 +337,7 @@ const ALLOWED_WS_PATHS = new Set(['/ws', '/notification/ws']);
 const PUBLIC_AUTH_PATHS = ['/api/auth/login', '/api/auth/register'];
 
 function isPublicPath(path) {
-  if (path === '/health') return true;
+  if (path === '/health' || path === '/metrics') return true;
   return PUBLIC_AUTH_PATHS.some((p) => path === p || path.startsWith(p + '?'));
 }
 
@@ -775,3 +814,8 @@ server.on('upgrade', (req, socket, head) => {
 app.get('/health', (req, res) => {
   res.status(200).json({ status: 'API Gateway is running' });
 });
+
+app.get('/metrics', async (req, res) => {
+  res.set('Content-Type', promClient.register.contentType);
+  res.end(await promClient.register.metrics());
+});

services/api-gateway-node/package.json

@@ -24,6 +24,7 @@
     "http-proxy-middleware": "^3.0.5",
     "jsonwebtoken": "^9.0.3",
     "morgan": "^1.10.1",
+    "prom-client": "^15.1.3",
     "redis": "^4.6.10"
   }
 }

services/ats-service/main.py

@@ -80,7 +80,7 @@ async def lifespan(app: FastAPI):
 
 @app.middleware("http")
 async def internal_auth_middleware(request: Request, call_next):
-    if request.url.path == "/health":
+    if request.url.path in ("/health", "/metrics"):
         return await call_next(request)
 
     auth_header = request.headers.get("x-internal-auth")

services/attendance-service/go.mod

@@ -3,8 +3,10 @@ module atlas/attendance-service
 go 1.21
 
 require (
+	github.com/gofiber/adaptor/v2 v2.2.1
 	github.com/gofiber/fiber/v2 v2.50.0
 	github.com/golang-jwt/jwt/v5 v5.2.1
+	github.com/prometheus/client_golang v1.19.0
 	github.com/streadway/amqp v1.1.0
 	gorm.io/driver/postgres v1.5.2
 	gorm.io/gorm v1.25.4

services/attendance-service/main.go

@@ -5,15 +5,51 @@ import (
 	"fmt"
 	"log"
 	"os"
+	"regexp"
 	"time"
 
+	"github.com/gofiber/adaptor/v2"
 	"github.com/gofiber/fiber/v2"
 	"github.com/gofiber/fiber/v2/middleware/cors"
+	"github.com/prometheus/client_golang/prometheus"
+	"github.com/prometheus/client_golang/prometheus/promhttp"
 	"github.com/streadway/amqp"
 	"gorm.io/driver/postgres"
 	"gorm.io/gorm"
 )
 
+var (
+	httpRequestCount = prometheus.NewCounterVec(
+		prometheus.CounterOpts{
+			Name: "atlas_http_requests_total",
+			Help: "Total HTTP requests",
+		},
+		[]string{"method", "path", "status_code"},
+	)
+	httpRequestDuration = prometheus.NewHistogramVec(
+		prometheus.HistogramOpts{
+			Name:    "atlas_http_request_duration_seconds",
+			Help:    "HTTP request duration in seconds",
+			Buckets: []float64{0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1.0, 2.5, 5.0, 10.0},
+		},
+		[]string{"method", "path", "status_code"},
+	)
+	httpRequestsInProgress = prometheus.NewGaugeVec(
+		prometheus.GaugeOpts{
+			Name: "atlas_http_requests_in_progress",
+			Help: "Number of HTTP requests in progress",
+		},
+		[]string{"method", "path"},
+	)
+	pathParamPattern = regexp.MustCompile(`/[0-9a-fA-F-]{36}|/\d+`)
+)
+
+func init() {
+	prometheus.MustRegister(httpRequestCount)
+	prometheus.MustRegister(httpRequestDuration)
+	prometheus.MustRegister(httpRequestsInProgress)
+}
+
 var db *gorm.DB
 var rabbitChan *amqp.Channel
 
@@ -246,6 +282,28 @@ func main() {
 		AllowHeaders: "Origin,Content-Type,Accept,Authorization,X-Tenant-Id,X-Employee-Id",
 	}))
 
+	app.Use(func(c *fiber.Ctx) error {
+		path := pathParamPattern.ReplaceAllString(c.Path(), "/:param")
+		httpRequestsInProgress.WithLabelValues(c.Method(), path).Inc()
+		start := time.Now()
+		err := c.Next()
+		duration := time.Since(start).Seconds()
+		status := fiber.StatusInternalServerError
+		if err != nil {
+			if e, ok := err.(*fiber.Error); ok {
+				status = e.Code
+			}
+		} else {
+			status = c.Response().StatusCode()
+		}
+		httpRequestsInProgress.WithLabelValues(c.Method(), path).Dec()
+		httpRequestCount.WithLabelValues(c.Method(), path, fmt.Sprintf("%d", status)).Inc()
+		httpRequestDuration.WithLabelValues(c.Method(), path, fmt.Sprintf("%d", status)).Observe(duration)
+		return err
+	})
+
+	app.Get("/metrics", adaptor.HTTPHandler(promhttp.Handler()))
+
 	app.Get("/health", func(c *fiber.Ctx) error {
 		return c.JSON(fiber.Map{"status": "Attendance Service is running", "version": "2.0.0"})
 	})

services/audit-compliance-service/main.py

@@ -150,10 +150,11 @@ async def lifespan(app: FastAPI):
 )
 
 app.add_middleware(SecurityHeadersMiddleware)
+app.add_middleware(AtlasMetricsMiddleware)
 
 @app.middleware("http")
 async def internal_auth_middleware(request: Request, call_next):
-    if request.url.path == "/health":
+    if request.url.path in ("/health", "/metrics"):
         return await call_next(request)
 
     auth_header = request.headers.get("x-internal-auth")

services/auth-service/index.js

@@ -21,6 +21,29 @@ const {
   verifyAuthenticationResponse,
 } = require('@simplewebauthn/server');
 
+const promClient = require('prom-client');
+
+const httpRequestCount = new promClient.Counter({
+  name: 'atlas_http_requests_total',
+  help: 'Total HTTP requests',
+  labelNames: ['method', 'path', 'status_code'],
+});
+
+const httpRequestDuration = new promClient.Histogram({
+  name: 'atlas_http_request_duration_seconds',
+  help: 'HTTP request duration in seconds',
+  labelNames: ['method', 'path', 'status_code'],
+  buckets: [0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1.0, 2.5, 5.0, 10.0],
+});
+
+const httpRequestsInProgress = new promClient.Gauge({
+  name: 'atlas_http_requests_in_progress',
+  help: 'Number of HTTP requests in progress',
+  labelNames: ['method', 'path'],
+});
+
+promClient.collectDefaultMetrics();
+
 const app = express();
 app.use(helmet());
 app.use(cookieParser());
@@ -44,6 +67,23 @@ app.use(
   })
 );
 
+function metricsMiddleware(req, res, next) {
+  if (req.path === '/metrics' || req.path === '/health') {
+    return next();
+  }
+  const path = req.path.replace(/\/[0-9a-fA-F-]{36}|\/\d+/g, '/:param');
+  httpRequestsInProgress.labels({ method: req.method, path }).inc();
+  const start = Date.now();
+  res.on('finish', () => {
+    const duration = (Date.now() - start) / 1000;
+    httpRequestsInProgress.labels({ method: req.method, path }).dec();
+    httpRequestCount.labels({ method: req.method, path, status_code: res.statusCode }).inc();
+    httpRequestDuration.labels({ method: req.method, path, status_code: res.statusCode }).observe(duration);
+  });
+  next();
+}
+app.use(metricsMiddleware);
+
 const PORT = process.env.PORT || 8010;
 const NODE_ENV = process.env.NODE_ENV || 'development';
 const JWT_SECRET = process.env.JWT_SECRET;
@@ -2268,6 +2308,11 @@ app.get('/health', (req, res) => {
   res.status(200).json({ status: 'Auth Service is healthy' });
 });
 
+app.get('/metrics', async (req, res) => {
+  res.set('Content-Type', promClient.register.contentType);
+  res.end(await promClient.register.metrics());
+});
+
 app.listen(PORT, () => {
   console.log(`Auth service running on port ${PORT}`);
 });

services/auth-service/package.json

@@ -25,6 +25,7 @@
     "jsonwebtoken": "^9.0.3",
     "otplib": "^12.0.1",
     "pg": "^8.20.0",
+    "prom-client": "^15.1.3",
     "uuid": "^9.0.1",
     "xml-crypto": "^6.0.0"
   }

services/employee-lifecycle-service/main.py

@@ -69,7 +69,7 @@ async def lifespan(app: FastAPI):
 
 @app.middleware("http")
 async def internal_auth_middleware(request: Request, call_next):
-    if request.url.path == "/health":
+    if request.url.path in ("/health", "/metrics"):
         return await call_next(request)
 
     auth_header = request.headers.get("x-internal-auth")

services/integration-service/main.py

@@ -144,10 +144,11 @@ def _background_loop():
 )
 
 app.add_middleware(SecurityHeadersMiddleware)
+app.add_middleware(AtlasMetricsMiddleware)
 
 @app.middleware("http")
 async def internal_auth_middleware(request: Request, call_next):
-    if request.url.path == "/health":
+    if request.url.path in ("/health", "/metrics"):
         return await call_next(request)
 
     auth_header = request.headers.get("x-internal-auth")

services/live-service/main.py

@@ -344,7 +344,7 @@ def validate_internal_jwt(auth_header: str) -> dict:
 
 @app.middleware("http")
 async def internal_auth_middleware(request: Request, call_next):
-    if request.url.path == "/health":
+    if request.url.path in ("/health", "/metrics"):
         return await call_next(request)
 
     auth_header = request.headers.get("x-internal-auth")

services/lms-service/go.mod

@@ -3,10 +3,12 @@ module github.com/atlas-workforce/lms-service
 go 1.21
 
 require (
+	github.com/gofiber/adaptor/v2 v2.2.1
 	github.com/gofiber/fiber/v2 v2.52.0
 	github.com/golang-jwt/jwt/v5 v5.2.1
 	github.com/google/uuid v1.6.0
 	github.com/lib/pq v1.10.9
+	github.com/prometheus/client_golang v1.19.0
 	gorm.io/datatypes v1.2.0
 	gorm.io/driver/postgres v1.5.6
 	gorm.io/gorm v1.25.7