diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml index 244fe26b..7a853dc7 100644 --- a/.github/workflows/deploy.yml +++ b/.github/workflows/deploy.yml @@ -76,6 +76,13 @@ jobs: role-to-assume: ${{ secrets[format('AWS_ROLE_{0}', matrix.environment)] }} aws-region: ${{ env.AWS_REGION }} + - name: Check SLO Compliance (Production Only) + if: matrix.environment == 'prod' + run: | + node performance/scripts/check-slo-compliance.js + env: + PROMETHEUS_URL: ${{ secrets.PROMETHEUS_URL }} + - name: Download plan uses: actions/download-artifact@v3 with: diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 952bc7ec..ac056224 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -14,6 +14,27 @@ env: COVERAGE_FAIL_LINES_PREDICTIQ_API: "17" jobs: + prometheus-rules-test: + name: Prometheus SLO Rules Testing + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + + - name: Install Prometheus tools + run: | + wget https://github.com/prometheus/prometheus/releases/download/v2.45.0/prometheus-2.45.0.linux-amd64.tar.gz + tar xzf prometheus-2.45.0.linux-amd64.tar.gz + sudo mv prometheus-2.45.0.linux-amd64/promtool /usr/local/bin/ + + - name: Check Prometheus rules syntax + run: promtool check rules performance/config/prometheus-slo-rules.yml + + - name: Check alert rules syntax + run: promtool check rules performance/config/alerts.yaml + + - name: Run Prometheus rules unit tests + run: promtool test rules performance/config/prometheus-slo-rules.test.yml + unit-tests: name: Unit Tests runs-on: ubuntu-latest @@ -838,6 +859,7 @@ jobs: all-tests-passed: name: All Tests Passed needs: + - prometheus-rules-test - unit-tests - integration-tests - api-rate-limit-tests diff --git a/contracts/predict-iq/Makefile b/contracts/predict-iq/Makefile index 76944ad1..42fcd600 100644 --- a/contracts/predict-iq/Makefile +++ b/contracts/predict-iq/Makefile @@ -1,6 +1,6 @@ # PredictIQ Test Suite Makefile -.PHONY: help test test-unit test-integration test-all bench coverage clean install-tools format lint audit +.PHONY: help test test-unit test-integration test-all bench coverage clean install-tools format lint audit deploy-testnet deploy-mainnet help: @echo "PredictIQ Test Suite Commands:" @@ -14,6 +14,8 @@ help: @echo " make audit - Run security audit" @echo " make clean - Clean build artifacts" @echo " make install-tools - Install required tools" + @echo " make deploy-testnet - Deploy contract to Stellar testnet" + @echo " make deploy-mainnet - Deploy contract to Stellar mainnet" # Install required testing tools install-tools: @@ -146,3 +148,47 @@ test-report: @echo "Generating test report..." cargo test --workspace -- --format=json | tee test-report.json @echo "Test report saved to test-report.json" + +# Deploy to Stellar testnet +deploy-testnet: + @echo "🚀 Deploying to Stellar Testnet" + @echo "" + @read -p "Are you sure you want to deploy to TESTNET? [y/N] " confirm; \ + if [ "$$confirm" = "y" ] || [ "$$confirm" = "Y" ]; then \ + echo "Proceeding with testnet deployment..."; \ + cargo build --target wasm32-unknown-unknown --release; \ + soroban contract deploy \ + --wasm target/wasm32-unknown-unknown/release/predict_iq.wasm \ + --source-account $(SOROBAN_ACCOUNT) \ + --network testnet; \ + echo "✅ Testnet deployment completed"; \ + else \ + echo "❌ Deployment cancelled"; \ + exit 1; \ + fi + +# Deploy to Stellar mainnet (requires explicit confirmation) +deploy-mainnet: + @echo "🚀 Deploying to Stellar Mainnet" + @echo "" + @echo "⚠️ WARNING: This will deploy to MAINNET" + @echo "⚠️ This action cannot be easily undone" + @echo "" + @read -p "Are you sure you want to deploy to MAINNET? [y/N] " confirm; \ + if [ "$$confirm" = "y" ] || [ "$$confirm" = "Y" ]; then \ + if [ -z "$(CONFIRM_MAINNET)" ] || [ "$(CONFIRM_MAINNET)" != "yes" ]; then \ + echo "❌ Mainnet deployment requires CONFIRM_MAINNET=yes"; \ + echo " Usage: make deploy-mainnet CONFIRM_MAINNET=yes"; \ + exit 1; \ + fi; \ + echo "Proceeding with mainnet deployment..."; \ + cargo build --target wasm32-unknown-unknown --release; \ + soroban contract deploy \ + --wasm target/wasm32-unknown-unknown/release/predict_iq.wasm \ + --source-account $(SOROBAN_ACCOUNT) \ + --network mainnet; \ + echo "✅ Mainnet deployment completed"; \ + else \ + echo "❌ Deployment cancelled"; \ + exit 1; \ + fi diff --git a/docs/runbooks/critical-api-response-time.md b/docs/runbooks/critical-api-response-time.md new file mode 100644 index 00000000..3d098e85 --- /dev/null +++ b/docs/runbooks/critical-api-response-time.md @@ -0,0 +1,50 @@ +# Critical API Response Time Runbook + +## Alert Meaning +The API p99 response time has exceeded 500ms critical threshold for 2 minutes. + +## Impact +- Severe user experience degradation +- Potential request timeouts +- Risk of cascading failures across dependent services + +## Investigation Steps + +1. **Immediate assessment** + ```bash + curl 'http://prometheus:9090/api/v1/query?query=histogram_quantile(0.99,rate(http_request_duration_seconds_bucket[5m]))' + ``` + +2. **Check for errors** + ```bash + curl 'http://prometheus:9090/api/v1/query?query=rate(http_requests_total{status=~"5.."}[5m])' + ``` + +3. **Identify bottlenecks** + - Check database query times + - Review external service calls + - Check cache hit rates + +4. **Check infrastructure** + - CPU and memory on all API instances + - Network latency + - Disk I/O + +## Remediation + +### Immediate Actions +1. **Page on-call engineer** - This is a critical issue +2. **Scale up API instances** if CPU/memory is high +3. **Check for stuck connections** in database +4. **Review recent changes** - rollback if necessary + +### Emergency Actions +1. Enable circuit breakers for external services +2. Reduce cache TTL to force fresh data +3. Temporarily disable non-critical features +4. Redirect traffic to backup region if available + +### Post-Incident +1. Conduct root cause analysis +2. Implement permanent fix +3. Add performance tests to prevent recurrence diff --git a/docs/runbooks/critical-database-queries.md b/docs/runbooks/critical-database-queries.md new file mode 100644 index 00000000..d15d4845 --- /dev/null +++ b/docs/runbooks/critical-database-queries.md @@ -0,0 +1,52 @@ +# Critical Database Queries Runbook + +## Alert Meaning +Database p99 query time has exceeded 100ms critical threshold for 2 minutes. + +## Impact +- Severe API latency +- Risk of connection pool exhaustion +- Potential cascading failures + +## Investigation Steps + +1. **Immediate query analysis** + ```bash + curl 'http://prometheus:9090/api/v1/query?query=histogram_quantile(0.99,rate(db_query_duration_seconds_bucket[5m]))' + ``` + +2. **Check for locks** + ```sql + SHOW PROCESSLIST; + SHOW OPEN TABLES WHERE In_use > 0; + ``` + +3. **Check connection pool** + - Active connections + - Waiting connections + - Connection timeout rate + +4. **Review recent changes** + - Schema changes + - Index changes + - Data volume changes + +## Remediation + +### Immediate Actions +1. **Page on-call DBA** - This is critical +2. Kill long-running queries if safe +3. Check for table locks +4. Review connection pool settings + +### Emergency Actions +1. Increase connection pool size +2. Implement query timeout +3. Redirect traffic if possible +4. Consider read replicas + +### Post-Incident +1. Conduct root cause analysis +2. Optimize identified queries +3. Add query performance tests +4. Implement query monitoring diff --git a/docs/runbooks/critical-db-connection-pool-utilization.md b/docs/runbooks/critical-db-connection-pool-utilization.md new file mode 100644 index 00000000..fee835ba --- /dev/null +++ b/docs/runbooks/critical-db-connection-pool-utilization.md @@ -0,0 +1,47 @@ +# Critical Database Connection Pool Utilization Runbook + +## Alert Meaning +Database connection pool utilization has exceeded 95% critical threshold for 2 minutes. + +## Impact +- Imminent connection pool exhaustion +- New requests will fail +- Service outage risk + +## Investigation Steps + +1. **Immediate assessment** + ```bash + curl 'http://prometheus:9090/api/v1/query?query=db_connections_active / db_connections_max' + ``` + +2. **Check active connections** + ```sql + SHOW PROCESSLIST; + ``` + +3. **Identify blocking queries** + ```sql + SELECT * FROM information_schema.processlist WHERE state != 'Sleep'; + ``` + +## Remediation + +### Immediate Actions +1. **Page on-call DBA** - Critical issue +2. Increase connection pool size immediately +3. Kill idle connections +4. Kill long-running queries if safe +5. Consider read-only mode + +### Emergency Actions +1. Implement connection throttling +2. Redirect traffic to backup database +3. Scale database resources +4. Implement circuit breakers + +### Post-Incident +1. Root cause analysis +2. Optimize connection usage +3. Implement connection monitoring +4. Add connection pool auto-scaling diff --git a/docs/runbooks/high-api-response-time.md b/docs/runbooks/high-api-response-time.md new file mode 100644 index 00000000..21e4e2ae --- /dev/null +++ b/docs/runbooks/high-api-response-time.md @@ -0,0 +1,52 @@ +# High API Response Time Runbook + +## Alert Meaning +The API p95 response time has exceeded 200ms threshold for 5 minutes. + +## Impact +- Degraded user experience with slower API responses +- Potential cascading failures if latency continues to increase +- May indicate resource contention or inefficient queries + +## Investigation Steps + +1. **Check current metrics** + ```bash + curl http://prometheus:9090/api/v1/query?query=histogram_quantile(0.95,rate(http_request_duration_seconds_bucket[5m])) + ``` + +2. **Identify affected endpoints** + ```bash + curl 'http://prometheus:9090/api/v1/query?query=histogram_quantile(0.95,rate(http_request_duration_seconds_bucket{endpoint!=""}[5m])) by (endpoint)' + ``` + +3. **Check database performance** + - Query slow query logs + - Check connection pool utilization + - Monitor active connections + +4. **Check resource utilization** + - CPU usage on API servers + - Memory usage and GC pauses + - Network I/O + +5. **Review recent deployments** + - Check if any code changes were deployed recently + - Review database schema changes + +## Remediation + +### Immediate Actions +1. Check if this is a temporary spike or sustained issue +2. If sustained, consider scaling up API instances +3. Review and optimize slow queries + +### Short-term +1. Implement query caching if not already in place +2. Add database indexes for frequently queried fields +3. Review API endpoint implementations for inefficiencies + +### Long-term +1. Implement performance testing in CI/CD +2. Set up performance regression alerts +3. Establish performance SLOs and budgets diff --git a/docs/runbooks/high-bet-gas-costs.md b/docs/runbooks/high-bet-gas-costs.md new file mode 100644 index 00000000..84885e99 --- /dev/null +++ b/docs/runbooks/high-bet-gas-costs.md @@ -0,0 +1,44 @@ +# High Bet Gas Costs Runbook + +## Alert Meaning +Gas cost for placing bet operations has exceeded 200,000 threshold. + +## Impact +- Increased transaction costs for users +- Reduced user engagement +- Competitive disadvantage + +## Investigation Steps + +1. **Check gas usage** + ```bash + curl 'http://prometheus:9090/api/v1/query?query=contract_gas_used{operation="place_bet"}' + ``` + +2. **Review contract implementation** + - Check for unnecessary operations + - Review data validation logic + - Check for inefficient storage patterns + +3. **Analyze bet patterns** + - Check bet complexity + - Review market state updates + +## Remediation + +### Immediate Actions +1. Analyze recent contract changes +2. Identify gas optimization opportunities +3. Review bet validation logic + +### Short-term +1. Optimize contract operations +2. Reduce validation overhead +3. Implement efficient data structures +4. Batch operations where possible + +### Long-term +1. Implement gas monitoring +2. Add performance benchmarks +3. Establish gas cost targets +4. Regular optimization reviews diff --git a/docs/runbooks/high-contract-gas-costs.md b/docs/runbooks/high-contract-gas-costs.md new file mode 100644 index 00000000..ed127dff --- /dev/null +++ b/docs/runbooks/high-contract-gas-costs.md @@ -0,0 +1,44 @@ +# High Contract Gas Costs Runbook + +## Alert Meaning +Gas cost for market creation operations has exceeded 500,000 threshold. + +## Impact +- Increased transaction costs +- Reduced profitability +- Potential user friction from high fees + +## Investigation Steps + +1. **Check gas usage trends** + ```bash + curl 'http://prometheus:9090/api/v1/query?query=contract_gas_used{operation="create_market"}' + ``` + +2. **Review recent contract changes** + - Check for new features + - Review optimization opportunities + - Check for inefficient operations + +3. **Analyze transaction patterns** + - Check market complexity + - Review data storage patterns + +## Remediation + +### Immediate Actions +1. Review recent contract deployments +2. Analyze gas usage by operation +3. Identify optimization opportunities + +### Short-term +1. Optimize contract code +2. Reduce storage operations +3. Batch operations where possible +4. Implement gas-efficient patterns + +### Long-term +1. Implement gas monitoring +2. Add gas benchmarks to CI/CD +3. Establish gas cost SLOs +4. Regular contract optimization diff --git a/docs/runbooks/high-cpu-usage.md b/docs/runbooks/high-cpu-usage.md new file mode 100644 index 00000000..a87849a2 --- /dev/null +++ b/docs/runbooks/high-cpu-usage.md @@ -0,0 +1,51 @@ +# High CPU Usage Runbook + +## Alert Meaning +CPU usage has exceeded 80% for 10 minutes. + +## Impact +- Reduced system responsiveness +- Potential service degradation +- Risk of timeouts + +## Investigation Steps + +1. **Check CPU usage** + ```bash + top -b -n 1 | head -20 + ``` + +2. **Identify CPU consumers** + ```bash + ps aux --sort=-%cpu | head -20 + ``` + +3. **Check for runaway processes** + - Review application logs + - Check for infinite loops + - Review recent deployments + +4. **Check system load** + ```bash + uptime + ``` + +## Remediation + +### Immediate Actions +1. Identify CPU-intensive processes +2. Scale up CPU if possible +3. Restart services if needed +4. Check for runaway processes + +### Short-term +1. Optimize CPU usage +2. Implement CPU limits +3. Add CPU monitoring +4. Review code for inefficiencies + +### Long-term +1. Implement performance profiling +2. Add performance tests to CI/CD +3. Establish CPU usage SLOs +4. Regular optimization reviews diff --git a/docs/runbooks/high-db-connection-pool-utilization.md b/docs/runbooks/high-db-connection-pool-utilization.md new file mode 100644 index 00000000..5d575d2d --- /dev/null +++ b/docs/runbooks/high-db-connection-pool-utilization.md @@ -0,0 +1,50 @@ +# High Database Connection Pool Utilization Runbook + +## Alert Meaning +Database connection pool utilization has exceeded 80% for 5 minutes. + +## Impact +- Risk of connection exhaustion +- New requests may fail to acquire connections +- Potential service degradation + +## Investigation Steps + +1. **Check connection pool status** + ```bash + curl 'http://prometheus:9090/api/v1/query?query=db_connections_active / db_connections_max' + ``` + +2. **Identify connection consumers** + ```sql + SELECT user, COUNT(*) FROM information_schema.processlist GROUP BY user; + ``` + +3. **Check for idle connections** + ```sql + SELECT * FROM information_schema.processlist WHERE command = 'Sleep'; + ``` + +4. **Review query performance** + - Check for slow queries holding connections + - Look for transaction locks + +## Remediation + +### Immediate Actions +1. Increase connection pool size +2. Kill idle connections +3. Review and optimize slow queries +4. Check for connection leaks + +### Short-term +1. Implement connection pooling +2. Add connection timeout +3. Optimize query performance +4. Implement circuit breakers + +### Long-term +1. Monitor connection pool metrics +2. Establish connection pool SLOs +3. Implement connection pool auto-scaling +4. Regular performance tuning diff --git a/docs/runbooks/high-error-rate.md b/docs/runbooks/high-error-rate.md new file mode 100644 index 00000000..0e75888e --- /dev/null +++ b/docs/runbooks/high-error-rate.md @@ -0,0 +1,52 @@ +# High Error Rate Runbook + +## Alert Meaning +The API error rate (5xx responses) has exceeded 0.1% for 5 minutes. + +## Impact +- Users experiencing failures +- Potential data loss or inconsistency +- Service reliability degradation + +## Investigation Steps + +1. **Check error rate by endpoint** + ```bash + curl 'http://prometheus:9090/api/v1/query?query=rate(http_requests_total{status=~"5.."}[5m]) by (endpoint)' + ``` + +2. **Review error logs** + ```bash + kubectl logs -l app=predictiq-api --tail=1000 | grep ERROR + ``` + +3. **Check specific error types** + - 500: Internal Server Error + - 502: Bad Gateway + - 503: Service Unavailable + - 504: Gateway Timeout + +4. **Check dependencies** + - Database connectivity + - Cache availability + - External service health + +## Remediation + +### Immediate Actions +1. Check application logs for error patterns +2. Verify database connectivity +3. Check cache service status +4. Review recent deployments + +### Short-term +1. Increase logging verbosity if needed +2. Implement circuit breakers +3. Add retry logic with exponential backoff +4. Scale up instances if resource-constrained + +### Long-term +1. Implement comprehensive error tracking +2. Add error budget monitoring +3. Establish error rate SLOs +4. Improve error handling and recovery diff --git a/docs/runbooks/high-memory-usage.md b/docs/runbooks/high-memory-usage.md new file mode 100644 index 00000000..b484f987 --- /dev/null +++ b/docs/runbooks/high-memory-usage.md @@ -0,0 +1,49 @@ +# High Memory Usage Runbook + +## Alert Meaning +Memory usage has exceeded 90% for 5 minutes. + +## Impact +- Risk of OOM (Out of Memory) errors +- Service degradation +- Potential crashes + +## Investigation Steps + +1. **Check memory usage** + ```bash + free -h + ``` + +2. **Identify memory consumers** + ```bash + ps aux --sort=-%mem | head -20 + ``` + +3. **Check for memory leaks** + - Review application logs + - Check for growing memory usage over time + +4. **Check cache size** + - Redis memory usage + - Application cache size + +## Remediation + +### Immediate Actions +1. Identify and kill unnecessary processes +2. Clear caches if safe +3. Scale up memory if possible +4. Restart services if needed + +### Short-term +1. Optimize memory usage +2. Implement memory limits +3. Add memory monitoring +4. Review cache policies + +### Long-term +1. Implement memory profiling +2. Add memory tests to CI/CD +3. Establish memory SLOs +4. Regular optimization reviews diff --git a/docs/runbooks/high-resolution-gas-costs.md b/docs/runbooks/high-resolution-gas-costs.md new file mode 100644 index 00000000..e0308124 --- /dev/null +++ b/docs/runbooks/high-resolution-gas-costs.md @@ -0,0 +1,44 @@ +# High Resolution Gas Costs Runbook + +## Alert Meaning +Gas cost for market resolution operations has exceeded 300,000 threshold. + +## Impact +- Increased operational costs +- Delayed market resolution +- Reduced profitability + +## Investigation Steps + +1. **Check gas usage** + ```bash + curl 'http://prometheus:9090/api/v1/query?query=contract_gas_used{operation="resolve_market"}' + ``` + +2. **Review resolution logic** + - Check for unnecessary state updates + - Review settlement calculations + - Check for inefficient data access + +3. **Analyze market complexity** + - Check number of participants + - Review outcome determination logic + +## Remediation + +### Immediate Actions +1. Analyze resolution operations +2. Identify optimization opportunities +3. Review settlement logic + +### Short-term +1. Optimize resolution code +2. Reduce state updates +3. Implement efficient calculations +4. Batch settlement operations + +### Long-term +1. Implement gas monitoring +2. Add resolution benchmarks +3. Establish gas cost targets +4. Regular optimization reviews diff --git a/docs/runbooks/low-cache-hit-rate.md b/docs/runbooks/low-cache-hit-rate.md new file mode 100644 index 00000000..c9247caa --- /dev/null +++ b/docs/runbooks/low-cache-hit-rate.md @@ -0,0 +1,45 @@ +# Low Cache Hit Rate Runbook + +## Alert Meaning +Cache hit rate has dropped below 80% for 10 minutes. + +## Impact +- Increased database load +- Slower response times +- Higher operational costs + +## Investigation Steps + +1. **Check cache metrics** + ```bash + curl 'http://prometheus:9090/api/v1/query?query=rate(cache_hits_total[5m]) / (rate(cache_hits_total[5m]) + rate(cache_misses_total[5m]))' + ``` + +2. **Check cache size and evictions** + - Monitor Redis memory usage + - Check eviction policy + - Review key expiration rates + +3. **Identify problematic keys** + - Check for cache stampedes + - Review key access patterns + - Check for inefficient caching + +## Remediation + +### Immediate Actions +1. Increase cache size if memory allows +2. Review cache eviction policy +3. Check for cache key collisions + +### Short-term +1. Optimize cache key design +2. Adjust TTL values +3. Implement cache warming for hot keys +4. Add cache statistics monitoring + +### Long-term +1. Implement multi-level caching +2. Use cache-aside pattern more effectively +3. Add cache performance SLOs +4. Implement cache coherence strategies diff --git a/docs/runbooks/low-throughput.md b/docs/runbooks/low-throughput.md new file mode 100644 index 00000000..7a829282 --- /dev/null +++ b/docs/runbooks/low-throughput.md @@ -0,0 +1,51 @@ +# Low Throughput Runbook + +## Alert Meaning +Request throughput has dropped below 1000 req/s for 10 minutes. + +## Impact +- Reduced system capacity +- Potential service degradation +- May indicate upstream issues or traffic routing problems + +## Investigation Steps + +1. **Check current throughput** + ```bash + curl 'http://prometheus:9090/api/v1/query?query=rate(http_requests_total[5m])' + ``` + +2. **Check by endpoint** + ```bash + curl 'http://prometheus:9090/api/v1/query?query=rate(http_requests_total[5m]) by (endpoint)' + ``` + +3. **Check for errors** + - Are requests being rejected? + - Are there connection timeouts? + - Check rate limiting status + +4. **Check infrastructure** + - API instance availability + - Load balancer health + - Network connectivity + +## Remediation + +### Immediate Actions +1. Verify API instances are running +2. Check load balancer configuration +3. Verify DNS resolution +4. Check for rate limiting issues + +### Investigation +1. Review application logs +2. Check for deployment issues +3. Verify database connectivity +4. Check external service dependencies + +### Recovery +1. Restart affected instances if needed +2. Adjust load balancer configuration +3. Scale up if capacity is insufficient +4. Investigate root cause diff --git a/docs/runbooks/performance-degradation.md b/docs/runbooks/performance-degradation.md new file mode 100644 index 00000000..e7c1dac9 --- /dev/null +++ b/docs/runbooks/performance-degradation.md @@ -0,0 +1,50 @@ +# Performance Degradation Runbook + +## Alert Meaning +Response time has degraded by more than 10% compared to 24 hours ago. + +## Impact +- Degraded user experience +- Potential SLO violation +- May indicate resource constraints or code issues + +## Investigation Steps + +1. **Check current performance** + ```bash + curl 'http://prometheus:9090/api/v1/query?query=histogram_quantile(0.95,rate(http_request_duration_seconds_bucket[1h]))' + ``` + +2. **Compare with baseline** + ```bash + curl 'http://prometheus:9090/api/v1/query?query=histogram_quantile(0.95,rate(http_request_duration_seconds_bucket[1h] offset 24h))' + ``` + +3. **Identify affected endpoints** + - Check by endpoint + - Check by method + - Check by status code + +4. **Check for recent changes** + - Review deployments + - Check database changes + - Review infrastructure changes + +## Remediation + +### Immediate Actions +1. Analyze performance trends +2. Identify affected endpoints +3. Check for recent deployments + +### Short-term +1. Optimize identified bottlenecks +2. Review query performance +3. Check resource utilization +4. Consider rollback if recent deployment + +### Long-term +1. Implement performance testing +2. Add performance regression tests +3. Establish performance SLOs +4. Regular optimization reviews diff --git a/docs/runbooks/service-down.md b/docs/runbooks/service-down.md new file mode 100644 index 00000000..edeac5f8 --- /dev/null +++ b/docs/runbooks/service-down.md @@ -0,0 +1,51 @@ +# Service Down Runbook + +## Alert Meaning +A PredictIQ service has been down for more than 1 minute. + +## Impact +- Service unavailable to users +- Potential data loss +- Revenue impact + +## Investigation Steps + +1. **Check service status** + ```bash + kubectl get pods -l app=predictiq-api + ``` + +2. **Check service logs** + ```bash + kubectl logs -l app=predictiq-api --tail=100 + ``` + +3. **Check recent events** + ```bash + kubectl describe pod + ``` + +4. **Check infrastructure** + - Node status + - Resource availability + - Network connectivity + +## Remediation + +### Immediate Actions +1. **Page on-call engineer** - Critical issue +2. Check pod status and restart if needed +3. Check node health +4. Review recent deployments + +### Emergency Actions +1. Rollback recent deployment if applicable +2. Scale up replicas +3. Check for resource constraints +4. Failover to backup if available + +### Post-Incident +1. Root cause analysis +2. Implement monitoring improvements +3. Add health checks +4. Improve deployment process diff --git a/docs/runbooks/significant-performance-degradation.md b/docs/runbooks/significant-performance-degradation.md new file mode 100644 index 00000000..b4c02253 --- /dev/null +++ b/docs/runbooks/significant-performance-degradation.md @@ -0,0 +1,50 @@ +# Significant Performance Degradation Runbook + +## Alert Meaning +Response time has degraded by more than 5% compared to 24 hours ago (alert threshold). + +## Impact +- Noticeable performance degradation +- Potential SLO impact if trend continues +- May indicate emerging issues + +## Investigation Steps + +1. **Check performance trend** + ```bash + curl 'http://prometheus:9090/api/v1/query_range?query=histogram_quantile(0.95,rate(http_request_duration_seconds_bucket[1h]))&start=<24h-ago>&end=&step=1h' + ``` + +2. **Identify trend direction** + - Is degradation continuing? + - Is it stabilizing? + - Are there spikes? + +3. **Check for correlations** + - Database performance + - Cache hit rates + - Resource utilization + +4. **Review recent changes** + - Code deployments + - Infrastructure changes + - Data volume changes + +## Remediation + +### Immediate Actions +1. Monitor trend closely +2. Prepare for escalation if continues +3. Identify potential causes + +### Short-term +1. Optimize identified bottlenecks +2. Review code changes +3. Check resource utilization +4. Consider preventive scaling + +### Long-term +1. Implement continuous performance monitoring +2. Add performance regression tests +3. Establish performance budgets +4. Regular optimization reviews diff --git a/docs/runbooks/slow-database-queries.md b/docs/runbooks/slow-database-queries.md new file mode 100644 index 00000000..223e81d7 --- /dev/null +++ b/docs/runbooks/slow-database-queries.md @@ -0,0 +1,50 @@ +# Slow Database Queries Runbook + +## Alert Meaning +Database p95 query time has exceeded 50ms for 5 minutes. + +## Impact +- Slower API responses +- Increased database load +- Potential connection pool exhaustion + +## Investigation Steps + +1. **Check slow query log** + ```sql + SELECT * FROM mysql.slow_log ORDER BY start_time DESC LIMIT 20; + ``` + +2. **Identify slow queries** + ```bash + curl 'http://prometheus:9090/api/v1/query?query=histogram_quantile(0.95,rate(db_query_duration_seconds_bucket[5m])) by (query)' + ``` + +3. **Check query plans** + - Use EXPLAIN for identified slow queries + - Check for missing indexes + - Review join strategies + +4. **Check database health** + - Connection pool utilization + - Lock contention + - Disk I/O + +## Remediation + +### Immediate Actions +1. Identify and analyze slow queries +2. Check for missing indexes +3. Review query execution plans + +### Short-term +1. Add indexes for frequently queried columns +2. Optimize query logic +3. Implement query result caching +4. Consider query rewriting + +### Long-term +1. Implement query performance monitoring +2. Add database performance tests +3. Establish query performance SLOs +4. Regular index maintenance diff --git a/performance/config/alerts.yaml b/performance/config/alerts.yaml index 093bc7a4..778ff282 100644 --- a/performance/config/alerts.yaml +++ b/performance/config/alerts.yaml @@ -14,6 +14,7 @@ groups: annotations: summary: "High API response time detected" description: "API p95 response time is {{ $value }}s, exceeding 200ms threshold" + runbook_url: "https://docs.predictiq.com/runbooks/high-api-response-time" - alert: CriticalAPIResponseTime expr: histogram_quantile(0.99, rate(http_request_duration_seconds_bucket[5m])) > 0.5 @@ -24,6 +25,7 @@ groups: annotations: summary: "Critical API response time detected" description: "API p99 response time is {{ $value }}s, exceeding 500ms critical threshold" + runbook_url: "https://docs.predictiq.com/runbooks/critical-api-response-time" - alert: HighErrorRate expr: rate(http_requests_total{status=~"5.."}[5m]) / rate(http_requests_total[5m]) > 0.001 @@ -34,6 +36,7 @@ groups: annotations: summary: "High error rate detected" description: "Error rate is {{ $value | humanizePercentage }}, exceeding 0.1% threshold" + runbook_url: "https://docs.predictiq.com/runbooks/high-error-rate" - alert: LowThroughput expr: rate(http_requests_total[5m]) < 1000 @@ -44,6 +47,7 @@ groups: annotations: summary: "Low request throughput" description: "Request throughput is {{ $value }} req/s, below 1000 req/s minimum" + runbook_url: "https://docs.predictiq.com/runbooks/low-throughput" - name: cache_performance interval: 30s @@ -57,6 +61,7 @@ groups: annotations: summary: "Low cache hit rate" description: "Cache hit rate is {{ $value | humanizePercentage }}, below 80% threshold" + runbook_url: "https://docs.predictiq.com/runbooks/low-cache-hit-rate" - name: database_performance interval: 30s @@ -70,6 +75,7 @@ groups: annotations: summary: "Slow database queries detected" description: "Database p95 query time is {{ $value }}s, exceeding 50ms threshold" + runbook_url: "https://docs.predictiq.com/runbooks/slow-database-queries" - alert: CriticalDatabaseQueries expr: histogram_quantile(0.99, rate(db_query_duration_seconds_bucket[5m])) > 0.1 @@ -80,6 +86,7 @@ groups: annotations: summary: "Critical database query performance" description: "Database p99 query time is {{ $value }}s, exceeding 100ms critical threshold" + runbook_url: "https://docs.predictiq.com/runbooks/critical-database-queries" - alert: HighDBConnectionPoolUtilization expr: db_connections_active / db_connections_max > 0.8 @@ -90,6 +97,7 @@ groups: annotations: summary: "High database connection pool utilization" description: "Connection pool utilization is {{ $value | humanizePercentage }}, exceeding 80% threshold" + runbook_url: "https://docs.predictiq.com/runbooks/high-db-connection-pool-utilization" - alert: CriticalDBConnectionPoolUtilization expr: db_connections_active / db_connections_max > 0.95 @@ -100,6 +108,7 @@ groups: annotations: summary: "Critical database connection pool utilization" description: "Connection pool utilization is {{ $value | humanizePercentage }}, exceeding 95% critical threshold" + runbook_url: "https://docs.predictiq.com/runbooks/critical-db-connection-pool-utilization" - name: contract_performance interval: 1m @@ -113,6 +122,7 @@ groups: annotations: summary: "High gas costs for market creation" description: "Create market gas cost is {{ $value }}, exceeding 500000 threshold" + runbook_url: "https://docs.predictiq.com/runbooks/high-contract-gas-costs" - alert: HighBetGasCosts expr: contract_gas_used{operation="place_bet"} > 200000 @@ -123,6 +133,7 @@ groups: annotations: summary: "High gas costs for placing bets" description: "Place bet gas cost is {{ $value }}, exceeding 200000 threshold" + runbook_url: "https://docs.predictiq.com/runbooks/high-bet-gas-costs" - alert: HighResolutionGasCosts expr: contract_gas_used{operation="resolve_market"} > 300000 @@ -133,6 +144,7 @@ groups: annotations: summary: "High gas costs for market resolution" description: "Resolve market gas cost is {{ $value }}, exceeding 300000 threshold" + runbook_url: "https://docs.predictiq.com/runbooks/high-resolution-gas-costs" - name: system_health interval: 30s @@ -146,6 +158,7 @@ groups: annotations: summary: "Service is down" description: "{{ $labels.job }} has been down for more than 1 minute" + runbook_url: "https://docs.predictiq.com/runbooks/service-down" - alert: HighMemoryUsage expr: (node_memory_MemTotal_bytes - node_memory_MemAvailable_bytes) / node_memory_MemTotal_bytes > 0.9 @@ -156,6 +169,7 @@ groups: annotations: summary: "High memory usage" description: "Memory usage is {{ $value | humanizePercentage }}, exceeding 90% threshold" + runbook_url: "https://docs.predictiq.com/runbooks/high-memory-usage" - alert: HighCPUUsage expr: 100 - (avg by (instance) (rate(node_cpu_seconds_total{mode="idle"}[5m])) * 100) > 80 @@ -166,6 +180,7 @@ groups: annotations: summary: "High CPU usage" description: "CPU usage is {{ $value }}%, exceeding 80% threshold" + runbook_url: "https://docs.predictiq.com/runbooks/high-cpu-usage" - name: tts_quota interval: 1m @@ -227,6 +242,7 @@ groups: annotations: summary: "Performance degradation detected" description: "Response time has degraded by more than 10% compared to 24h ago" + runbook_url: "https://docs.predictiq.com/runbooks/performance-degradation" - alert: SignificantPerformanceDegradation expr: | @@ -242,3 +258,4 @@ groups: annotations: summary: "Performance degradation alert threshold reached" description: "Response time has degraded by more than 5% compared to 24h ago (alert threshold)" + runbook_url: "https://docs.predictiq.com/runbooks/significant-performance-degradation" diff --git a/performance/config/prometheus-slo-rules.test.yml b/performance/config/prometheus-slo-rules.test.yml new file mode 100644 index 00000000..17bf5cf7 --- /dev/null +++ b/performance/config/prometheus-slo-rules.test.yml @@ -0,0 +1,112 @@ +# Unit tests for Prometheus SLO rules +tests: + - interval: 1m + input_series: + - series: 'http_requests_total{status="200"}' + values: '100+100x10' + - series: 'http_requests_total{status="500"}' + values: '1+0x10' + - series: 'http_request_duration_seconds_bucket{le="0.2"}' + values: '95+5x10' + - series: 'http_request_duration_seconds_bucket{le="+Inf"}' + values: '100+10x10' + - series: 'db_query_duration_seconds_bucket{le="0.05"}' + values: '90+5x10' + - series: 'db_query_duration_seconds_bucket{le="+Inf"}' + values: '100+10x10' + - series: 'redis_commands_total{status="success"}' + values: '1000+100x10' + - series: 'redis_commands_total{status="failed"}' + values: '1+0x10' + + - interval: 5m + input_series: + - series: 'http_requests_total{status="200"}' + values: '500+100x10' + - series: 'http_requests_total{status="500"}' + values: '5+1x10' + promql_expr_test: + - expr: 'sum(rate(http_requests_total{status=~"2..|3.."}[5m])) / sum(rate(http_requests_total[5m])) * 100' + expected_samples: + - labels: '{}' + value: '99' + + - interval: 1h + input_series: + - series: 'http_requests_total{status="200"}' + values: '3600+100x24' + - series: 'http_requests_total{status="500"}' + values: '36+1x24' + promql_expr_test: + - expr: 'slo:api_availability:success_rate' + expected_samples: + - labels: '{}' + value: '99' + + - interval: 1m + input_series: + - series: 'http_requests_total{status="200"}' + values: '100+0x10' + - series: 'http_requests_total{status="500"}' + values: '100+0x10' + promql_expr_test: + - expr: 'sum(rate(http_requests_total{status=~"2..|3.."}[5m])) / sum(rate(http_requests_total[5m])) * 100' + expected_samples: + - labels: '{}' + value: '50' + + - interval: 1m + input_series: + - series: 'http_request_duration_seconds_bucket{le="0.2"}' + values: '50+0x10' + - series: 'http_request_duration_seconds_bucket{le="+Inf"}' + values: '100+0x10' + promql_expr_test: + - expr: 'histogram_quantile(0.95, sum(rate(http_request_duration_seconds_bucket[5m])) by (le)) <= 0.2' + expected_samples: + - labels: '{}' + value: '1' + + - interval: 1m + input_series: + - series: 'redis_commands_total{status="success"}' + values: '1000+0x10' + - series: 'redis_commands_total{status="failed"}' + values: '0+0x10' + promql_expr_test: + - expr: 'sum(rate(redis_commands_total{status="success"}[5m])) / sum(rate(redis_commands_total[5m])) * 100' + expected_samples: + - labels: '{}' + value: '100' + + - interval: 1m + input_series: + - series: 'http_requests_total{status="200"}' + values: '100+0x10' + - series: 'http_requests_total{status="500"}' + values: '1+0x10' + alert_rule_test: + - alert_rule_name: 'SLOFastBurn' + eval_time: '10m' + expected_alerts: + - exp_labels: + severity: 'critical' + slo: 'api_availability' + exp_annotations: + summary: 'SLO fast burn detected for API availability' + + - interval: 1m + input_series: + - series: 'http_requests_total{status="200"}' + values: '100+0x10' + - series: 'http_requests_total{status="500"}' + values: '100+0x10' + alert_rule_test: + - alert_rule_name: 'SLOErrorBudgetExhausted' + eval_time: '10m' + expected_alerts: + - exp_labels: + severity: 'critical' + slo: 'api_availability' + exp_annotations: + summary: 'SLO error budget exhausted for API availability' diff --git a/performance/scripts/check-slo-compliance.js b/performance/scripts/check-slo-compliance.js new file mode 100755 index 00000000..849780a1 --- /dev/null +++ b/performance/scripts/check-slo-compliance.js @@ -0,0 +1,112 @@ +#!/usr/bin/env node + +/** + * Check SLO compliance before deployment + * Queries Prometheus for current SLO metrics and blocks deployment if error budget exhausted + */ + +const https = require('https'); +const fs = require('fs'); +const path = require('path'); + +const PROMETHEUS_URL = process.env.PROMETHEUS_URL || 'http://prometheus:9090'; +const SLO_CONFIG = path.join(__dirname, '../config/slo.json'); +const ALLOW_OVERRIDE = process.env.CONFIRM_SLO_OVERRIDE === 'yes'; + +async function queryPrometheus(query) { + return new Promise((resolve, reject) => { + const url = new URL(`${PROMETHEUS_URL}/api/v1/query`); + url.searchParams.append('query', query); + + const protocol = url.protocol === 'https:' ? https : require('http'); + + protocol.get(url, (res) => { + let data = ''; + res.on('data', chunk => data += chunk); + res.on('end', () => { + try { + const result = JSON.parse(data); + if (result.status === 'success') { + resolve(result.data.result); + } else { + reject(new Error(`Prometheus error: ${result.error}`)); + } + } catch (e) { + reject(e); + } + }); + }).on('error', reject); + }); +} + +async function checkSLOCompliance() { + try { + const sloConfig = JSON.parse(fs.readFileSync(SLO_CONFIG, 'utf8')); + + console.log('🔍 Checking SLO compliance...\n'); + + let allCompliant = true; + const results = []; + + for (const [sloName, sloTarget] of Object.entries(sloConfig.slos)) { + try { + // Query error budget remaining + const query = `slo:${sloName}:error_budget_remaining`; + const result = await queryPrometheus(query); + + if (result.length === 0) { + console.warn(`⚠️ No data for SLO: ${sloName}`); + continue; + } + + const errorBudgetRemaining = parseFloat(result[0].value[1]); + const compliant = errorBudgetRemaining > 0; + + results.push({ + slo: sloName, + target: sloTarget, + errorBudgetRemaining, + compliant + }); + + const icon = compliant ? '✅' : '❌'; + console.log(`${icon} ${sloName}`); + console.log(` Target: ${sloTarget}%`); + console.log(` Error Budget Remaining: ${errorBudgetRemaining.toFixed(2)}%\n`); + + if (!compliant) { + allCompliant = false; + } + } catch (error) { + console.error(`❌ Error checking ${sloName}: ${error.message}`); + allCompliant = false; + } + } + + // Print summary + console.log('─'.repeat(50)); + if (allCompliant) { + console.log('✅ All SLOs compliant - deployment allowed\n'); + return true; + } else { + console.log('❌ SLO compliance check failed\n'); + + if (ALLOW_OVERRIDE) { + console.log('⚠️ Override enabled via CONFIRM_SLO_OVERRIDE=yes'); + console.log(' Proceeding with deployment (requires approval)\n'); + return true; + } else { + console.log('🛑 Deployment blocked - error budget exhausted'); + console.log(' Set CONFIRM_SLO_OVERRIDE=yes to override (requires approval)\n'); + return false; + } + } + } catch (error) { + console.error(`Fatal error: ${error.message}`); + process.exit(1); + } +} + +checkSLOCompliance().then(success => { + process.exit(success ? 0 : 1); +});