Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
104 changes: 67 additions & 37 deletions .github/workflows/dr-cross.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -3,10 +3,13 @@

name: Cross-Region Disaster Recovery Test

# Requires live AWS RDS / Route53 / S3 and repo secrets (AWS_ACCESS_KEY_ID,
# AWS_SECRET_ACCESS_KEY, DNS_ZONE_ID, HEALTH_CHECK_ID). Cannot honestly smoke
# on GitHub-hosted runners without those credentials and infra.
# Kept as workflow_dispatch-only so inventory is not gated on an unachievable
# path. DR is not proven in CI until this is run manually with secrets present.
# When secrets are configured, do not continue-on-error on real failures.
on:
schedule:
# Run weekly on Sundays at 2 AM UTC
- cron: "0 2 * * 0"
workflow_dispatch:
inputs:
simulate_failover:
Expand Down Expand Up @@ -36,12 +39,35 @@ jobs:
primary_endpoint: ${{ steps.get_endpoints.outputs.primary_endpoint }}
secondary_endpoint: ${{ steps.get_endpoints.outputs.secondary_endpoint }}
sidecar_url: ${{ steps.get_endpoints.outputs.sidecar_url }}
aws_available: ${{ steps.aws.outputs.available }}

steps:
- name: Checkout code
uses: actions/checkout@v4

# Guard: empty secrets make configure-aws-credentials fail hard.
# Skip only when secrets are absent; do not continue-on-error when present.
- name: Check AWS credentials availability
id: aws
env:
AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
run: |
if [ -z "$AWS_ACCESS_KEY_ID" ] || [ -z "$AWS_SECRET_ACCESS_KEY" ]; then
echo "available=false" >> "$GITHUB_OUTPUT"
echo "::notice::AWS secrets not configured; DR test not run (DR not proven in CI)"
else
echo "available=true" >> "$GITHUB_OUTPUT"
fi

- name: Record DR skip (secrets absent)
if: steps.aws.outputs.available != 'true'
run: |
echo "dr-cross: skipped (AWS secrets absent)."
echo "This workflow did not validate cross-region disaster recovery."

- name: Configure AWS credentials
if: steps.aws.outputs.available == 'true'
uses: aws-actions/configure-aws-credentials@v4
with:
aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
Expand All @@ -50,6 +76,7 @@ jobs:

- name: Get database endpoints
id: get_endpoints
if: steps.aws.outputs.available == 'true'
run: |
# Get primary database endpoint
PRIMARY_ENDPOINT=$(aws rds describe-db-instances \
Expand Down Expand Up @@ -79,6 +106,7 @@ jobs:
name: Test Region Failover
runs-on: ubuntu-latest
needs: setup
if: needs.setup.outputs.aws_available == 'true'
strategy:
matrix:
test_scenario: [normal, failover, recovery]
Expand Down Expand Up @@ -107,21 +135,21 @@ jobs:
# Test primary database connectivity
echo "Testing primary database connectivity..."
if ! pg_isready -h ${{ needs.setup.outputs.primary_endpoint }} -p 5432; then
echo "Primary database is not accessible"
echo "Primary database is not accessible"
exit 1
fi
echo "Primary database is accessible"
echo "Primary database is accessible"

# Test sidecar heartbeat
echo "Testing sidecar heartbeat..."
for i in {1..10}; do
if curl -f -s "${{ needs.setup.outputs.sidecar_url }}/health" > /dev/null; then
echo "Sidecar heartbeat successful"
echo "Sidecar heartbeat successful"
break
else
echo "⚠️ Sidecar heartbeat attempt ${i} failed"
echo "Sidecar heartbeat attempt ${i} failed"
if [ "${i}" -eq 10 ]; then
echo "All sidecar heartbeat attempts failed"
echo "All sidecar heartbeat attempts failed"
exit 1
fi
sleep 5
Expand All @@ -132,10 +160,10 @@ jobs:
echo "Testing DNS resolution..."
RESOLVED_IP=$(dig +short ${{ env.DNS_RECORD }} | head -1)
if [ -z "$RESOLVED_IP" ]; then
echo "DNS resolution failed"
echo "DNS resolution failed"
exit 1
fi
echo "DNS resolution successful: $RESOLVED_IP"
echo "DNS resolution successful: $RESOLVED_IP"

- name: Simulate failover
if: matrix.test_scenario == 'failover' && github.event.inputs.simulate_failover == 'true'
Expand All @@ -159,12 +187,12 @@ jobs:
SECONDARY_IP=$(dig +short ${{ needs.setup.outputs.secondary_endpoint }} | head -1)

if [ "$RESOLVED_IP" = "$SECONDARY_IP" ]; then
echo "DNS failover successful: $RESOLVED_IP"
echo "DNS failover successful: $RESOLVED_IP"
break
else
echo "Waiting for DNS failover... (attempt ${i}/30)"
echo "Waiting for DNS failover... (attempt ${i}/30)"
if [ "${i}" -eq 30 ]; then
echo "DNS failover timeout"
echo "DNS failover timeout"
exit 1
fi
sleep 10
Expand All @@ -179,10 +207,10 @@ jobs:
# Test secondary database connectivity
echo "Testing secondary database connectivity..."
if ! pg_isready -h ${{ needs.setup.outputs.secondary_endpoint }} -p 5432; then
echo "Secondary database is not accessible"
echo "Secondary database is not accessible"
exit 1
fi
echo "Secondary database is accessible"
echo "Secondary database is accessible"

# Test sidecar heartbeat during failover
echo "Testing sidecar heartbeat during failover..."
Expand All @@ -196,21 +224,21 @@ jobs:
elapsed=$((current_time - start_time))

if [ "${elapsed}" -gt "${max_disruption}" ]; then
echo "Service disruption exceeded ${max_disruption} seconds"
echo "Service disruption exceeded ${max_disruption} seconds"
exit 1
fi

echo "Sidecar heartbeat restored after ${elapsed} seconds"
echo "Sidecar heartbeat restored after ${elapsed} seconds"
disruption_detected=true
break
else
echo "Waiting for service recovery... (attempt $i/20)"
echo "Waiting for service recovery... (attempt $i/20)"
sleep 5
fi
done

if [ "$disruption_detected" = false ]; then
echo "Service did not recover within expected time"
echo "Service did not recover within expected time"
exit 1
fi

Expand All @@ -236,12 +264,12 @@ jobs:
PRIMARY_IP=$(dig +short ${{ needs.setup.outputs.primary_endpoint }} | head -1)

if [ "$RESOLVED_IP" = "$PRIMARY_IP" ]; then
echo "DNS recovery successful: $RESOLVED_IP"
echo "DNS recovery successful: $RESOLVED_IP"
break
else
echo "Waiting for DNS recovery... (attempt ${i}/30)"
echo "Waiting for DNS recovery... (attempt ${i}/30)"
if [ "${i}" -eq 30 ]; then
echo "DNS recovery timeout"
echo "DNS recovery timeout"
exit 1
fi
sleep 10
Expand All @@ -251,15 +279,16 @@ jobs:
# Final health check
echo "Performing final health check..."
if ! curl -f -s "${{ needs.setup.outputs.sidecar_url }}/health" > /dev/null; then
echo "Final health check failed"
echo "Final health check failed"
exit 1
fi
echo "Final health check passed"
echo "Final health check passed"

verify-replication:
name: Verify Cross-Region Replication
runs-on: ubuntu-latest
needs: setup
if: needs.setup.outputs.aws_available == 'true'

steps:
- name: Checkout code
Expand Down Expand Up @@ -300,9 +329,9 @@ jobs:
# Verify file exists in secondary bucket
echo "Verifying replication..."
if aws s3 ls "s3://$SECONDARY_BUCKET/test-file.txt" --region ${{ env.SECONDARY_REGION }} > /dev/null; then
echo "S3 replication successful"
echo "S3 replication successful"
else
echo "S3 replication failed"
echo "S3 replication failed"
exit 1
fi

Expand All @@ -315,6 +344,7 @@ jobs:
name: Test Blue-Green Migration
runs-on: ubuntu-latest
needs: setup
if: needs.setup.outputs.aws_available == 'true'

steps:
- name: Checkout code
Expand All @@ -339,16 +369,16 @@ jobs:
--green-db-url "postgresql://test:test@${{ needs.setup.outputs.secondary_endpoint }}:5432/test" \
--dns-zone "${{ env.DNS_ZONE_ID }}" \
--dns-record "${{ env.DNS_RECORD }}"; then
echo "Blue-green migration dry run failed"
echo "Blue-green migration dry run failed"
exit 1
fi
echo "Blue-green migration dry run successful"
echo "Blue-green migration dry run successful"

generate-report:
name: Generate DR Test Report
runs-on: ubuntu-latest
needs: [test-failover, verify-replication, test-blue-green-migration]
if: always()
needs: [setup, test-failover, verify-replication, test-blue-green-migration]
if: always() && needs.setup.outputs.aws_available == 'true'

steps:
- name: Checkout code
Expand Down Expand Up @@ -382,9 +412,9 @@ jobs:

## Key Metrics

- **Maximum Service Disruption:** < 90 seconds
- **DNS Failover Time:** < 60 seconds
- **S3 Replication Latency:** < 60 seconds
- **Maximum Service Disruption:** < 90 seconds
- **DNS Failover Time:** < 60 seconds
- **S3 Replication Latency:** < 60 seconds

## Recommendations

Expand All @@ -394,7 +424,7 @@ jobs:

EOF

echo "DR test report generated"
echo "DR test report generated"

- name: Upload report
uses: actions/upload-artifact@v4
Expand All @@ -406,13 +436,13 @@ jobs:
notify:
name: Notify on Failure
runs-on: ubuntu-latest
needs: [test-failover, verify-replication, test-blue-green-migration]
if: failure()
needs: [setup, test-failover, verify-replication, test-blue-green-migration]
if: failure() && needs.setup.outputs.aws_available == 'true'

steps:
- name: Send notification
run: |
echo "Cross-region DR test failed!"
echo "Cross-region DR test failed!"
echo "Please check the test results and investigate any issues."
echo "Critical components that need attention:"
echo "- Database connectivity"
Expand Down
7 changes: 4 additions & 3 deletions .github/workflows/edge-load.yaml
Original file line number Diff line number Diff line change
@@ -1,9 +1,10 @@
name: Edge API Load Testing

# Depends on live edge endpoints (api.*.provability-fabric.org), Cloudflare /
# Terraform-deployed infra, and optional InfluxDB. Cannot honestly smoke on
# GitHub-hosted runners without that SaaS/infra. Kept as workflow_dispatch-only
# so inventory is not gated on an unachievable path (same pattern as #194/#196).
on:
schedule:
# Run weekly on Mondays at 3 AM UTC
- cron: "0 3 * * 1"
workflow_dispatch:
inputs:
test_duration:
Expand Down
10 changes: 4 additions & 6 deletions .github/workflows/loadtest.yaml
Original file line number Diff line number Diff line change
@@ -1,12 +1,10 @@
name: Load Testing

# Requires docker-compose ledger scale-out, k6 against a live local stack, and
# historically failed as disabled_inactivity. Not maintainable as a gated
# workflow without a dedicated smoke path. Kept as workflow_dispatch-only so
# inventory is not gated on an unachievable path (same pattern as #194/#196).
on:
schedule:
# Run daily at 3 AM UTC
- cron: "0 3 * * *"
pull_request:
paths:
- "runtime/**"
workflow_dispatch:

jobs:
Expand Down
20 changes: 8 additions & 12 deletions .github/workflows/perf-proofmeter.yaml
Original file line number Diff line number Diff line change
@@ -1,19 +1,17 @@
name: ProofMeter Performance

# ProofMeter bench steps assume a running service (docker/service startup is
# still a stub) and historically failed as disabled_inactivity. Cannot honestly
# gate on push/schedule without a real smoke endpoint. Kept as
# workflow_dispatch-only so inventory is not gated on an unachievable path
# (same pattern as #194/#196).
on:
push:
branches: [main, develop]
pull_request:
branches: [main]
schedule:
# Run weekly on Mondays at 3 AM UTC
- cron: "0 3 * * 1"
workflow_dispatch:

jobs:
perf-smoke:
name: Performance Smoke Test (PR)
name: Performance Smoke Test
runs-on: ubuntu-latest
if: github.event_name == 'pull_request'
timeout-minutes: 10

steps:
Expand Down Expand Up @@ -69,9 +67,8 @@ jobs:
fi

perf-full:
name: Performance Full Test (Weekly)
name: Performance Full Test
runs-on: ubuntu-latest
if: github.event_name == 'schedule' || github.event_name == 'push'
timeout-minutes: 30

steps:
Expand Down Expand Up @@ -184,7 +181,6 @@ jobs:
name: Performance Trends
runs-on: ubuntu-latest
needs: perf-full
if: github.event_name == 'schedule'

steps:
- name: Checkout code
Expand Down
9 changes: 6 additions & 3 deletions .github/workflows/pf-cross-repo-consumer.yaml
Original file line number Diff line number Diff line change
@@ -1,9 +1,12 @@
name: Cross-Repo PF CI Consumer

# Calls org/ci-workflows reusable workflow (external SaaS/org wiring). Cannot
# pass without that org repo and historically failed as disabled_inactivity.
# Kept as workflow_dispatch-only so inventory is not gated on an unachievable
# path (same pattern as #194/#196). In-repo PF CI remains pf-reusable-caller /
# pf-ci.yaml.
on:
pull_request:
schedule:
- cron: "0 3 * * *"
workflow_dispatch:

jobs:
pf-ci:
Expand Down
13 changes: 4 additions & 9 deletions .github/workflows/publish-updates.yaml
Original file line number Diff line number Diff line change
@@ -1,16 +1,11 @@
name: Publish Updates

# Writes docs/updates.md from live metrics publishers and pushes commits.
# Historically failed as disabled_inactivity; not an honest gated CI path
# without stable external metric sources. Kept as workflow_dispatch-only so
# inventory is not gated on an unachievable path (same pattern as #194/#196).
on:
schedule:
# Run nightly at 2 AM UTC
- cron: "0 2 * * *"
workflow_dispatch:
# Allow manual triggering
push:
branches: [main]
paths:
- "tools/metrics/**"
- ".github/workflows/publish-updates.yaml"

jobs:
publish-updates:
Expand Down
Loading
Loading