From 0594bfe5591c52a342683e721757a0bdedf643e7 Mon Sep 17 00:00:00 2001 From: fraware Date: Thu, 16 Jul 2026 12:45:57 -0700 Subject: [PATCH] fix(ci): ungate SaaS/AWS leftovers from inventory gate Make seven workflows that cannot honestly pass without external AWS/SaaS workflow_dispatch-only (lean-offline/art-benchmark pattern). Keep a secret presence guard on dr-cross so manual runs skip cleanly when AWS creds are absent and still fail hard when secrets are present. --- .github/workflows/dr-cross.yaml | 104 +++++++++++------- .github/workflows/edge-load.yaml | 7 +- .github/workflows/loadtest.yaml | 10 +- .github/workflows/perf-proofmeter.yaml | 20 ++-- .github/workflows/pf-cross-repo-consumer.yaml | 9 +- .github/workflows/publish-updates.yaml | 13 +-- .github/workflows/revocation-sync.yaml | 7 +- 7 files changed, 97 insertions(+), 73 deletions(-) diff --git a/.github/workflows/dr-cross.yaml b/.github/workflows/dr-cross.yaml index ee12fe605..e7055bceb 100644 --- a/.github/workflows/dr-cross.yaml +++ b/.github/workflows/dr-cross.yaml @@ -3,10 +3,13 @@ name: Cross-Region Disaster Recovery Test +# Requires live AWS RDS / Route53 / S3 and repo secrets (AWS_ACCESS_KEY_ID, +# AWS_SECRET_ACCESS_KEY, DNS_ZONE_ID, HEALTH_CHECK_ID). Cannot honestly smoke +# on GitHub-hosted runners without those credentials and infra. +# Kept as workflow_dispatch-only so inventory is not gated on an unachievable +# path. DR is not proven in CI until this is run manually with secrets present. +# When secrets are configured, do not continue-on-error on real failures. on: - schedule: - # Run weekly on Sundays at 2 AM UTC - - cron: "0 2 * * 0" workflow_dispatch: inputs: simulate_failover: @@ -36,12 +39,35 @@ jobs: primary_endpoint: ${{ steps.get_endpoints.outputs.primary_endpoint }} secondary_endpoint: ${{ steps.get_endpoints.outputs.secondary_endpoint }} sidecar_url: ${{ steps.get_endpoints.outputs.sidecar_url }} + aws_available: ${{ steps.aws.outputs.available }} steps: - name: Checkout code uses: actions/checkout@v4 + # Guard: empty secrets make configure-aws-credentials fail hard. + # Skip only when secrets are absent; do not continue-on-error when present. + - name: Check AWS credentials availability + id: aws + env: + AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + run: | + if [ -z "$AWS_ACCESS_KEY_ID" ] || [ -z "$AWS_SECRET_ACCESS_KEY" ]; then + echo "available=false" >> "$GITHUB_OUTPUT" + echo "::notice::AWS secrets not configured; DR test not run (DR not proven in CI)" + else + echo "available=true" >> "$GITHUB_OUTPUT" + fi + + - name: Record DR skip (secrets absent) + if: steps.aws.outputs.available != 'true' + run: | + echo "dr-cross: skipped (AWS secrets absent)." + echo "This workflow did not validate cross-region disaster recovery." + - name: Configure AWS credentials + if: steps.aws.outputs.available == 'true' uses: aws-actions/configure-aws-credentials@v4 with: aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} @@ -50,6 +76,7 @@ jobs: - name: Get database endpoints id: get_endpoints + if: steps.aws.outputs.available == 'true' run: | # Get primary database endpoint PRIMARY_ENDPOINT=$(aws rds describe-db-instances \ @@ -79,6 +106,7 @@ jobs: name: Test Region Failover runs-on: ubuntu-latest needs: setup + if: needs.setup.outputs.aws_available == 'true' strategy: matrix: test_scenario: [normal, failover, recovery] @@ -107,21 +135,21 @@ jobs: # Test primary database connectivity echo "Testing primary database connectivity..." if ! pg_isready -h ${{ needs.setup.outputs.primary_endpoint }} -p 5432; then - echo "❌ Primary database is not accessible" + echo "Primary database is not accessible" exit 1 fi - echo "✅ Primary database is accessible" + echo "Primary database is accessible" # Test sidecar heartbeat echo "Testing sidecar heartbeat..." for i in {1..10}; do if curl -f -s "${{ needs.setup.outputs.sidecar_url }}/health" > /dev/null; then - echo "✅ Sidecar heartbeat successful" + echo "Sidecar heartbeat successful" break else - echo "⚠️ Sidecar heartbeat attempt ${i} failed" + echo "Sidecar heartbeat attempt ${i} failed" if [ "${i}" -eq 10 ]; then - echo "❌ All sidecar heartbeat attempts failed" + echo "All sidecar heartbeat attempts failed" exit 1 fi sleep 5 @@ -132,10 +160,10 @@ jobs: echo "Testing DNS resolution..." RESOLVED_IP=$(dig +short ${{ env.DNS_RECORD }} | head -1) if [ -z "$RESOLVED_IP" ]; then - echo "❌ DNS resolution failed" + echo "DNS resolution failed" exit 1 fi - echo "✅ DNS resolution successful: $RESOLVED_IP" + echo "DNS resolution successful: $RESOLVED_IP" - name: Simulate failover if: matrix.test_scenario == 'failover' && github.event.inputs.simulate_failover == 'true' @@ -159,12 +187,12 @@ jobs: SECONDARY_IP=$(dig +short ${{ needs.setup.outputs.secondary_endpoint }} | head -1) if [ "$RESOLVED_IP" = "$SECONDARY_IP" ]; then - echo "✅ DNS failover successful: $RESOLVED_IP" + echo "DNS failover successful: $RESOLVED_IP" break else - echo "⏳ Waiting for DNS failover... (attempt ${i}/30)" + echo "Waiting for DNS failover... (attempt ${i}/30)" if [ "${i}" -eq 30 ]; then - echo "❌ DNS failover timeout" + echo "DNS failover timeout" exit 1 fi sleep 10 @@ -179,10 +207,10 @@ jobs: # Test secondary database connectivity echo "Testing secondary database connectivity..." if ! pg_isready -h ${{ needs.setup.outputs.secondary_endpoint }} -p 5432; then - echo "❌ Secondary database is not accessible" + echo "Secondary database is not accessible" exit 1 fi - echo "✅ Secondary database is accessible" + echo "Secondary database is accessible" # Test sidecar heartbeat during failover echo "Testing sidecar heartbeat during failover..." @@ -196,21 +224,21 @@ jobs: elapsed=$((current_time - start_time)) if [ "${elapsed}" -gt "${max_disruption}" ]; then - echo "❌ Service disruption exceeded ${max_disruption} seconds" + echo "Service disruption exceeded ${max_disruption} seconds" exit 1 fi - echo "✅ Sidecar heartbeat restored after ${elapsed} seconds" + echo "Sidecar heartbeat restored after ${elapsed} seconds" disruption_detected=true break else - echo "⏳ Waiting for service recovery... (attempt $i/20)" + echo "Waiting for service recovery... (attempt $i/20)" sleep 5 fi done if [ "$disruption_detected" = false ]; then - echo "❌ Service did not recover within expected time" + echo "Service did not recover within expected time" exit 1 fi @@ -236,12 +264,12 @@ jobs: PRIMARY_IP=$(dig +short ${{ needs.setup.outputs.primary_endpoint }} | head -1) if [ "$RESOLVED_IP" = "$PRIMARY_IP" ]; then - echo "✅ DNS recovery successful: $RESOLVED_IP" + echo "DNS recovery successful: $RESOLVED_IP" break else - echo "⏳ Waiting for DNS recovery... (attempt ${i}/30)" + echo "Waiting for DNS recovery... (attempt ${i}/30)" if [ "${i}" -eq 30 ]; then - echo "❌ DNS recovery timeout" + echo "DNS recovery timeout" exit 1 fi sleep 10 @@ -251,15 +279,16 @@ jobs: # Final health check echo "Performing final health check..." if ! curl -f -s "${{ needs.setup.outputs.sidecar_url }}/health" > /dev/null; then - echo "❌ Final health check failed" + echo "Final health check failed" exit 1 fi - echo "✅ Final health check passed" + echo "Final health check passed" verify-replication: name: Verify Cross-Region Replication runs-on: ubuntu-latest needs: setup + if: needs.setup.outputs.aws_available == 'true' steps: - name: Checkout code @@ -300,9 +329,9 @@ jobs: # Verify file exists in secondary bucket echo "Verifying replication..." if aws s3 ls "s3://$SECONDARY_BUCKET/test-file.txt" --region ${{ env.SECONDARY_REGION }} > /dev/null; then - echo "✅ S3 replication successful" + echo "S3 replication successful" else - echo "❌ S3 replication failed" + echo "S3 replication failed" exit 1 fi @@ -315,6 +344,7 @@ jobs: name: Test Blue-Green Migration runs-on: ubuntu-latest needs: setup + if: needs.setup.outputs.aws_available == 'true' steps: - name: Checkout code @@ -339,16 +369,16 @@ jobs: --green-db-url "postgresql://test:test@${{ needs.setup.outputs.secondary_endpoint }}:5432/test" \ --dns-zone "${{ env.DNS_ZONE_ID }}" \ --dns-record "${{ env.DNS_RECORD }}"; then - echo "❌ Blue-green migration dry run failed" + echo "Blue-green migration dry run failed" exit 1 fi - echo "✅ Blue-green migration dry run successful" + echo "Blue-green migration dry run successful" generate-report: name: Generate DR Test Report runs-on: ubuntu-latest - needs: [test-failover, verify-replication, test-blue-green-migration] - if: always() + needs: [setup, test-failover, verify-replication, test-blue-green-migration] + if: always() && needs.setup.outputs.aws_available == 'true' steps: - name: Checkout code @@ -382,9 +412,9 @@ jobs: ## Key Metrics - - **Maximum Service Disruption:** < 90 seconds ✅ - - **DNS Failover Time:** < 60 seconds ✅ - - **S3 Replication Latency:** < 60 seconds ✅ + - **Maximum Service Disruption:** < 90 seconds + - **DNS Failover Time:** < 60 seconds + - **S3 Replication Latency:** < 60 seconds ## Recommendations @@ -394,7 +424,7 @@ jobs: EOF - echo "✅ DR test report generated" + echo "DR test report generated" - name: Upload report uses: actions/upload-artifact@v4 @@ -406,13 +436,13 @@ jobs: notify: name: Notify on Failure runs-on: ubuntu-latest - needs: [test-failover, verify-replication, test-blue-green-migration] - if: failure() + needs: [setup, test-failover, verify-replication, test-blue-green-migration] + if: failure() && needs.setup.outputs.aws_available == 'true' steps: - name: Send notification run: | - echo "❌ Cross-region DR test failed!" + echo "Cross-region DR test failed!" echo "Please check the test results and investigate any issues." echo "Critical components that need attention:" echo "- Database connectivity" diff --git a/.github/workflows/edge-load.yaml b/.github/workflows/edge-load.yaml index b951eee27..9a7c5309c 100644 --- a/.github/workflows/edge-load.yaml +++ b/.github/workflows/edge-load.yaml @@ -1,9 +1,10 @@ name: Edge API Load Testing +# Depends on live edge endpoints (api.*.provability-fabric.org), Cloudflare / +# Terraform-deployed infra, and optional InfluxDB. Cannot honestly smoke on +# GitHub-hosted runners without that SaaS/infra. Kept as workflow_dispatch-only +# so inventory is not gated on an unachievable path (same pattern as #194/#196). on: - schedule: - # Run weekly on Mondays at 3 AM UTC - - cron: "0 3 * * 1" workflow_dispatch: inputs: test_duration: diff --git a/.github/workflows/loadtest.yaml b/.github/workflows/loadtest.yaml index adcfe932c..e520f68ac 100644 --- a/.github/workflows/loadtest.yaml +++ b/.github/workflows/loadtest.yaml @@ -1,12 +1,10 @@ name: Load Testing +# Requires docker-compose ledger scale-out, k6 against a live local stack, and +# historically failed as disabled_inactivity. Not maintainable as a gated +# workflow without a dedicated smoke path. Kept as workflow_dispatch-only so +# inventory is not gated on an unachievable path (same pattern as #194/#196). on: - schedule: - # Run daily at 3 AM UTC - - cron: "0 3 * * *" - pull_request: - paths: - - "runtime/**" workflow_dispatch: jobs: diff --git a/.github/workflows/perf-proofmeter.yaml b/.github/workflows/perf-proofmeter.yaml index 79f478930..1203a3ae1 100644 --- a/.github/workflows/perf-proofmeter.yaml +++ b/.github/workflows/perf-proofmeter.yaml @@ -1,19 +1,17 @@ name: ProofMeter Performance +# ProofMeter bench steps assume a running service (docker/service startup is +# still a stub) and historically failed as disabled_inactivity. Cannot honestly +# gate on push/schedule without a real smoke endpoint. Kept as +# workflow_dispatch-only so inventory is not gated on an unachievable path +# (same pattern as #194/#196). on: - push: - branches: [main, develop] - pull_request: - branches: [main] - schedule: - # Run weekly on Mondays at 3 AM UTC - - cron: "0 3 * * 1" + workflow_dispatch: jobs: perf-smoke: - name: Performance Smoke Test (PR) + name: Performance Smoke Test runs-on: ubuntu-latest - if: github.event_name == 'pull_request' timeout-minutes: 10 steps: @@ -69,9 +67,8 @@ jobs: fi perf-full: - name: Performance Full Test (Weekly) + name: Performance Full Test runs-on: ubuntu-latest - if: github.event_name == 'schedule' || github.event_name == 'push' timeout-minutes: 30 steps: @@ -184,7 +181,6 @@ jobs: name: Performance Trends runs-on: ubuntu-latest needs: perf-full - if: github.event_name == 'schedule' steps: - name: Checkout code diff --git a/.github/workflows/pf-cross-repo-consumer.yaml b/.github/workflows/pf-cross-repo-consumer.yaml index 6696a9d11..6ea449093 100644 --- a/.github/workflows/pf-cross-repo-consumer.yaml +++ b/.github/workflows/pf-cross-repo-consumer.yaml @@ -1,9 +1,12 @@ name: Cross-Repo PF CI Consumer +# Calls org/ci-workflows reusable workflow (external SaaS/org wiring). Cannot +# pass without that org repo and historically failed as disabled_inactivity. +# Kept as workflow_dispatch-only so inventory is not gated on an unachievable +# path (same pattern as #194/#196). In-repo PF CI remains pf-reusable-caller / +# pf-ci.yaml. on: - pull_request: - schedule: - - cron: "0 3 * * *" + workflow_dispatch: jobs: pf-ci: diff --git a/.github/workflows/publish-updates.yaml b/.github/workflows/publish-updates.yaml index f96f9bb4e..53b85bca7 100644 --- a/.github/workflows/publish-updates.yaml +++ b/.github/workflows/publish-updates.yaml @@ -1,16 +1,11 @@ name: Publish Updates +# Writes docs/updates.md from live metrics publishers and pushes commits. +# Historically failed as disabled_inactivity; not an honest gated CI path +# without stable external metric sources. Kept as workflow_dispatch-only so +# inventory is not gated on an unachievable path (same pattern as #194/#196). on: - schedule: - # Run nightly at 2 AM UTC - - cron: "0 2 * * *" workflow_dispatch: - # Allow manual triggering - push: - branches: [main] - paths: - - "tools/metrics/**" - - ".github/workflows/publish-updates.yaml" jobs: publish-updates: diff --git a/.github/workflows/revocation-sync.yaml b/.github/workflows/revocation-sync.yaml index 779acd65c..2aba07d13 100644 --- a/.github/workflows/revocation-sync.yaml +++ b/.github/workflows/revocation-sync.yaml @@ -1,9 +1,10 @@ name: Revocation List Sync +# Intended to fetch/sign a canonical external revocation list; current body is +# a mock/demo path and historically failed as disabled_inactivity. Kept as +# workflow_dispatch-only so inventory is not gated on an unachievable path +# (same pattern as #194/#196). on: - schedule: - # Run daily at 4 AM UTC - - cron: "0 4 * * *" workflow_dispatch: jobs: