715 716 717 718 observability safety slo (conflict resolved)

.github/workflows/deploy.yml

@@ -76,6 +76,13 @@ jobs:
           role-to-assume: ${{ secrets[format('AWS_ROLE_{0}', matrix.environment)] }}
           aws-region: ${{ env.AWS_REGION }}
 
+      - name: Check SLO Compliance (Production Only)
+        if: matrix.environment == 'prod'
+        run: |
+          node performance/scripts/check-slo-compliance.js
+        env:
+          PROMETHEUS_URL: ${{ secrets.PROMETHEUS_URL }}
+
       - name: Download plan
         uses: actions/download-artifact@v3
         with:

.github/workflows/test.yml

@@ -14,6 +14,27 @@ env:
   COVERAGE_FAIL_LINES_PREDICTIQ_API: "17"
 
 jobs:
+  prometheus-rules-test:
+    name: Prometheus SLO Rules Testing
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Install Prometheus tools
+        run: |
+          wget https://github.com/prometheus/prometheus/releases/download/v2.45.0/prometheus-2.45.0.linux-amd64.tar.gz
+          tar xzf prometheus-2.45.0.linux-amd64.tar.gz
+          sudo mv prometheus-2.45.0.linux-amd64/promtool /usr/local/bin/
+
+      - name: Check Prometheus rules syntax
+        run: promtool check rules performance/config/prometheus-slo-rules.yml
+
+      - name: Check alert rules syntax
+        run: promtool check rules performance/config/alerts.yaml
+
+      - name: Run Prometheus rules unit tests
+        run: promtool test rules performance/config/prometheus-slo-rules.test.yml
+
   unit-tests:
     name: Unit Tests
     runs-on: ubuntu-latest
@@ -838,6 +859,7 @@ jobs:
   all-tests-passed:
     name: All Tests Passed
     needs:
+      - prometheus-rules-test
       - unit-tests
       - integration-tests
       - api-rate-limit-tests

contracts/predict-iq/Makefile

@@ -1,6 +1,6 @@
 # PredictIQ Test Suite Makefile
 
-.PHONY: help test test-unit test-integration test-all bench coverage clean install-tools format lint audit
+.PHONY: help test test-unit test-integration test-all bench coverage clean install-tools format lint audit deploy-testnet deploy-mainnet
 
 help:
 	@echo "PredictIQ Test Suite Commands:"
@@ -14,6 +14,8 @@ help:
 	@echo "  make audit             - Run security audit"
 	@echo "  make clean             - Clean build artifacts"
 	@echo "  make install-tools     - Install required tools"
+	@echo "  make deploy-testnet    - Deploy contract to Stellar testnet"
+	@echo "  make deploy-mainnet    - Deploy contract to Stellar mainnet"
 
 # Install required testing tools
 install-tools:
@@ -146,3 +148,47 @@ test-report:
 	@echo "Generating test report..."
 	cargo test --workspace -- --format=json | tee test-report.json
 	@echo "Test report saved to test-report.json"
+
+# Deploy to Stellar testnet
+deploy-testnet:
+	@echo "🚀 Deploying to Stellar Testnet"
+	@echo ""
+	@read -p "Are you sure you want to deploy to TESTNET? [y/N] " confirm; \
+	if [ "$$confirm" = "y" ] || [ "$$confirm" = "Y" ]; then \
+		echo "Proceeding with testnet deployment..."; \
+		cargo build --target wasm32-unknown-unknown --release; \
+		soroban contract deploy \
+			--wasm target/wasm32-unknown-unknown/release/predict_iq.wasm \
+			--source-account $(SOROBAN_ACCOUNT) \
+			--network testnet; \
+		echo "✅ Testnet deployment completed"; \
+	else \
+		echo "❌ Deployment cancelled"; \
+		exit 1; \
+	fi
+
+# Deploy to Stellar mainnet (requires explicit confirmation)
+deploy-mainnet:
+	@echo "🚀 Deploying to Stellar Mainnet"
+	@echo ""
+	@echo "⚠️  WARNING: This will deploy to MAINNET"
+	@echo "⚠️  This action cannot be easily undone"
+	@echo ""
+	@read -p "Are you sure you want to deploy to MAINNET? [y/N] " confirm; \
+	if [ "$$confirm" = "y" ] || [ "$$confirm" = "Y" ]; then \
+		if [ -z "$(CONFIRM_MAINNET)" ] || [ "$(CONFIRM_MAINNET)" != "yes" ]; then \
+			echo "❌ Mainnet deployment requires CONFIRM_MAINNET=yes"; \
+			echo "   Usage: make deploy-mainnet CONFIRM_MAINNET=yes"; \
+			exit 1; \
+		fi; \
+		echo "Proceeding with mainnet deployment..."; \
+		cargo build --target wasm32-unknown-unknown --release; \
+		soroban contract deploy \
+			--wasm target/wasm32-unknown-unknown/release/predict_iq.wasm \
+			--source-account $(SOROBAN_ACCOUNT) \
+			--network mainnet; \
+		echo "✅ Mainnet deployment completed"; \
+	else \
+		echo "❌ Deployment cancelled"; \
+		exit 1; \
+	fi

docs/runbooks/critical-api-response-time.md

@@ -0,0 +1,50 @@
+# Critical API Response Time Runbook
+
+## Alert Meaning
+The API p99 response time has exceeded 500ms critical threshold for 2 minutes.
+
+## Impact
+- Severe user experience degradation
+- Potential request timeouts
+- Risk of cascading failures across dependent services
+
+## Investigation Steps
+
+1. **Immediate assessment**
+   ```bash
+   curl 'http://prometheus:9090/api/v1/query?query=histogram_quantile(0.99,rate(http_request_duration_seconds_bucket[5m]))'
+   ```
+
+2. **Check for errors**
+   ```bash
+   curl 'http://prometheus:9090/api/v1/query?query=rate(http_requests_total{status=~"5.."}[5m])'
+   ```
+
+3. **Identify bottlenecks**
+   - Check database query times
+   - Review external service calls
+   - Check cache hit rates
+
+4. **Check infrastructure**
+   - CPU and memory on all API instances
+   - Network latency
+   - Disk I/O
+
+## Remediation
+
+### Immediate Actions
+1. **Page on-call engineer** - This is a critical issue
+2. **Scale up API instances** if CPU/memory is high
+3. **Check for stuck connections** in database
+4. **Review recent changes** - rollback if necessary
+
+### Emergency Actions
+1. Enable circuit breakers for external services
+2. Reduce cache TTL to force fresh data
+3. Temporarily disable non-critical features
+4. Redirect traffic to backup region if available
+
+### Post-Incident
+1. Conduct root cause analysis
+2. Implement permanent fix
+3. Add performance tests to prevent recurrence

docs/runbooks/critical-database-queries.md

@@ -0,0 +1,52 @@
+# Critical Database Queries Runbook
+
+## Alert Meaning
+Database p99 query time has exceeded 100ms critical threshold for 2 minutes.
+
+## Impact
+- Severe API latency
+- Risk of connection pool exhaustion
+- Potential cascading failures
+
+## Investigation Steps
+
+1. **Immediate query analysis**
+   ```bash
+   curl 'http://prometheus:9090/api/v1/query?query=histogram_quantile(0.99,rate(db_query_duration_seconds_bucket[5m]))'
+   ```
+
+2. **Check for locks**
+   ```sql
+   SHOW PROCESSLIST;
+   SHOW OPEN TABLES WHERE In_use > 0;
+   ```
+
+3. **Check connection pool**
+   - Active connections
+   - Waiting connections
+   - Connection timeout rate
+
+4. **Review recent changes**
+   - Schema changes
+   - Index changes
+   - Data volume changes
+
+## Remediation
+
+### Immediate Actions
+1. **Page on-call DBA** - This is critical
+2. Kill long-running queries if safe
+3. Check for table locks
+4. Review connection pool settings
+
+### Emergency Actions
+1. Increase connection pool size
+2. Implement query timeout
+3. Redirect traffic if possible
+4. Consider read replicas
+
+### Post-Incident
+1. Conduct root cause analysis
+2. Optimize identified queries
+3. Add query performance tests
+4. Implement query monitoring

docs/runbooks/critical-db-connection-pool-utilization.md

@@ -0,0 +1,47 @@
+# Critical Database Connection Pool Utilization Runbook
+
+## Alert Meaning
+Database connection pool utilization has exceeded 95% critical threshold for 2 minutes.
+
+## Impact
+- Imminent connection pool exhaustion
+- New requests will fail
+- Service outage risk
+
+## Investigation Steps
+
+1. **Immediate assessment**
+   ```bash
+   curl 'http://prometheus:9090/api/v1/query?query=db_connections_active / db_connections_max'
+   ```
+
+2. **Check active connections**
+   ```sql
+   SHOW PROCESSLIST;
+   ```
+
+3. **Identify blocking queries**
+   ```sql
+   SELECT * FROM information_schema.processlist WHERE state != 'Sleep';
+   ```
+
+## Remediation
+
+### Immediate Actions
+1. **Page on-call DBA** - Critical issue
+2. Increase connection pool size immediately
+3. Kill idle connections
+4. Kill long-running queries if safe
+5. Consider read-only mode
+
+### Emergency Actions
+1. Implement connection throttling
+2. Redirect traffic to backup database
+3. Scale database resources
+4. Implement circuit breakers
+
+### Post-Incident
+1. Root cause analysis
+2. Optimize connection usage
+3. Implement connection monitoring
+4. Add connection pool auto-scaling

docs/runbooks/high-api-response-time.md

@@ -0,0 +1,52 @@
+# High API Response Time Runbook
+
+## Alert Meaning
+The API p95 response time has exceeded 200ms threshold for 5 minutes.
+
+## Impact
+- Degraded user experience with slower API responses
+- Potential cascading failures if latency continues to increase
+- May indicate resource contention or inefficient queries
+
+## Investigation Steps
+
+1. **Check current metrics**
+   ```bash
+   curl http://prometheus:9090/api/v1/query?query=histogram_quantile(0.95,rate(http_request_duration_seconds_bucket[5m]))
+   ```
+
+2. **Identify affected endpoints**
+   ```bash
+   curl 'http://prometheus:9090/api/v1/query?query=histogram_quantile(0.95,rate(http_request_duration_seconds_bucket{endpoint!=""}[5m])) by (endpoint)'
+   ```
+
+3. **Check database performance**
+   - Query slow query logs
+   - Check connection pool utilization
+   - Monitor active connections
+
+4. **Check resource utilization**
+   - CPU usage on API servers
+   - Memory usage and GC pauses
+   - Network I/O
+
+5. **Review recent deployments**
+   - Check if any code changes were deployed recently
+   - Review database schema changes
+
+## Remediation
+
+### Immediate Actions
+1. Check if this is a temporary spike or sustained issue
+2. If sustained, consider scaling up API instances
+3. Review and optimize slow queries
+
+### Short-term
+1. Implement query caching if not already in place
+2. Add database indexes for frequently queried fields
+3. Review API endpoint implementations for inefficiencies
+
+### Long-term
+1. Implement performance testing in CI/CD
+2. Set up performance regression alerts
+3. Establish performance SLOs and budgets

docs/runbooks/high-bet-gas-costs.md

@@ -0,0 +1,44 @@
+# High Bet Gas Costs Runbook
+
+## Alert Meaning
+Gas cost for placing bet operations has exceeded 200,000 threshold.
+
+## Impact
+- Increased transaction costs for users
+- Reduced user engagement
+- Competitive disadvantage
+
+## Investigation Steps
+
+1. **Check gas usage**
+   ```bash
+   curl 'http://prometheus:9090/api/v1/query?query=contract_gas_used{operation="place_bet"}'
+   ```
+
+2. **Review contract implementation**
+   - Check for unnecessary operations
+   - Review data validation logic
+   - Check for inefficient storage patterns
+
+3. **Analyze bet patterns**
+   - Check bet complexity
+   - Review market state updates
+
+## Remediation
+
+### Immediate Actions
+1. Analyze recent contract changes
+2. Identify gas optimization opportunities
+3. Review bet validation logic
+
+### Short-term
+1. Optimize contract operations
+2. Reduce validation overhead
+3. Implement efficient data structures
+4. Batch operations where possible
+
+### Long-term
+1. Implement gas monitoring
+2. Add performance benchmarks
+3. Establish gas cost targets
+4. Regular optimization reviews