Audit hosted app health #4609
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Audit hosted app health | |
| # Each published Netlify app now warms itself with a one-minute Scheduled | |
| # Function. This GitHub workflow is an independent strict audit, not the | |
| # latency-control mechanism: GitHub Actions schedules can be delayed too long | |
| # to prevent a scale-to-zero database from suspending. | |
| # | |
| # --strict also asks every app to read its own pg_stat_activity, which is how | |
| # the database-pressure signals from the 2026-08-06 analytics outage get | |
| # checked every 15 minutes without putting a single production database credential in | |
| # CI. The app already holds its own. | |
| # | |
| # Scheduled workflows only run on the default branch, so this takes effect once | |
| # merged to main. Use the manual trigger to warm on demand. | |
| on: | |
| schedule: | |
| - cron: "*/15 * * * *" | |
| workflow_dispatch: | |
| permissions: | |
| issues: write | |
| contents: read | |
| concurrency: | |
| group: keep-neon-warm | |
| cancel-in-progress: true | |
| jobs: | |
| warm: | |
| name: Audit app health endpoints | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 5 | |
| env: | |
| GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} | |
| PAGERDUTY_ROUTING_KEY: ${{ secrets.PAGERDUTY_ROUTING_KEY }} | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - uses: actions/setup-node@v4 | |
| with: | |
| node-version: 22 | |
| - name: Ping /_agent-native/health for every app | |
| shell: bash | |
| run: | | |
| set -o pipefail | |
| node scripts/keep-warm.mjs --strict 2>&1 | tee /tmp/keep-warm.log | |
| # One rolling issue, and it means something: open == currently failing. | |
| # | |
| # The previous version posted "Scheduled audit failed: <run url>" on every | |
| # red run. One app's health route hung for a day and buried the issue | |
| # under 14 identical comments, which is exactly how a monitor teaches | |
| # people to stop opening it. Now the comment names what broke, and repeats | |
| # of the SAME breakage stay silent — a new comment means something changed. | |
| - name: Report failure on a rolling issue | |
| if: failure() | |
| shell: bash | |
| run: | | |
| set -euo pipefail | |
| ISSUE_TITLE="Production health check failing" | |
| RUN_URL="${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}" | |
| FAILING="$(grep -E '^\s*(✗|!) ' /tmp/keep-warm.log || true)" | |
| if [ -z "$FAILING" ]; then | |
| FAILING="(no per-app detail — the audit itself failed to run)" | |
| fi | |
| # Digits are stripped before hashing: latencies and counts differ on | |
| # every run, so hashing them made every repeat look like a new | |
| # failure and the de-duplication never engaged. The signature is | |
| # which apps are failing and how, not by how much. | |
| SIGNATURE="$(printf '%s' "$FAILING" | tr -d '0-9' | shasum | cut -c1-12)" | |
| # The backticks are literal Markdown fences in the issue body. | |
| # shellcheck disable=SC2016 | |
| BODY="$(printf '%s\n\n```\n%s\n```\n\n%s\n\n<!-- health-signature: %s -->' \ | |
| "15-minute audit is failing." "$FAILING" "$RUN_URL" "$SIGNATURE")" | |
| page_oncall() { | |
| if [ -z "${PAGERDUTY_ROUTING_KEY:-}" ]; then | |
| echo "::warning::PAGERDUTY_ROUTING_KEY is not configured; GitHub issue fallback is active but no page was sent." | |
| return 0 | |
| fi | |
| # The jq program intentionally uses jq variables, not shell vars. | |
| # shellcheck disable=SC2016 | |
| jq -n \ | |
| --arg routing_key "$PAGERDUTY_ROUTING_KEY" \ | |
| --arg dedup_key "agent-native-production-health" \ | |
| --arg summary "Agent-Native production health audit is failing" \ | |
| --arg source "${{ github.repository }}" \ | |
| --arg run_url "$RUN_URL" \ | |
| --arg failing "$FAILING" \ | |
| --arg signature "$SIGNATURE" \ | |
| '{routing_key: $routing_key, event_action: "trigger", dedup_key: $dedup_key, payload: {summary: $summary, source: $source, severity: "critical", custom_details: {run_url: $run_url, failing_apps: $failing, health_signature: $signature}}}' \ | |
| | curl --fail-with-body --connect-timeout 10 --max-time 20 --silent --show-error \ | |
| --header 'Content-Type: application/json' \ | |
| --data-binary @- \ | |
| https://events.pagerduty.com/v2/enqueue | |
| } | |
| # Page before GitHub Issues bookkeeping. A transient Issues API | |
| # failure must not suppress the independent on-call escalation. | |
| pagerduty_status=0 | |
| page_oncall || pagerduty_status=$? | |
| ISSUE_NUMBER=$(gh issue list --repo "${{ github.repository }}" --state open \ | |
| --search "\"$ISSUE_TITLE\" in:title" --json number,title \ | |
| --jq ".[] | select(.title == \"$ISSUE_TITLE\") | .number" | head -n1) | |
| if [ -z "$ISSUE_NUMBER" ]; then | |
| gh issue create --repo "${{ github.repository }}" \ | |
| --title "$ISSUE_TITLE" --body "$BODY" | |
| else | |
| LAST=$(gh issue view "$ISSUE_NUMBER" --repo "${{ github.repository }}" \ | |
| --json comments,body --jq '[.body] + [.comments[].body] | last') | |
| if printf '%s' "$LAST" | grep -q "health-signature: $SIGNATURE"; then | |
| echo "Same failure as the last report — issue #$ISSUE_NUMBER already open, not commenting again." | |
| else | |
| gh issue comment "$ISSUE_NUMBER" --repo "${{ github.repository }}" --body "$BODY" | |
| fi | |
| fi | |
| if [ "$pagerduty_status" -ne 0 ]; then | |
| echo "::error::PagerDuty delivery failed; GitHub issue reporting completed but the audit remains failed." >&2 | |
| exit "$pagerduty_status" | |
| fi | |
| - name: Close the rolling issue once the fleet is healthy | |
| if: success() | |
| shell: bash | |
| run: | | |
| set -euo pipefail | |
| ISSUE_TITLE="Production health check failing" | |
| pagerduty_status=0 | |
| if [ -n "${PAGERDUTY_ROUTING_KEY:-}" ]; then | |
| # Resolve before GitHub issue bookkeeping so an Issues API failure | |
| # cannot bypass the independent recovery notification. | |
| # The jq program intentionally uses jq variables, not shell vars. | |
| # shellcheck disable=SC2016 | |
| jq -n \ | |
| --arg routing_key "$PAGERDUTY_ROUTING_KEY" \ | |
| --arg dedup_key "agent-native-production-health" \ | |
| --arg source "${{ github.repository }}" \ | |
| '{routing_key: $routing_key, event_action: "resolve", dedup_key: $dedup_key, payload: {summary: "Agent-Native production health audit recovered", source: $source, severity: "critical"}}' \ | |
| | curl --fail-with-body --connect-timeout 10 --max-time 20 --silent --show-error \ | |
| --header 'Content-Type: application/json' \ | |
| --data-binary @- \ | |
| https://events.pagerduty.com/v2/enqueue || pagerduty_status=$? | |
| else | |
| echo "PagerDuty resolve skipped: PAGERDUTY_ROUTING_KEY is not configured." | |
| fi | |
| ISSUE_NUMBER=$(gh issue list --repo "${{ github.repository }}" --state open \ | |
| --search "\"$ISSUE_TITLE\" in:title" --json number,title \ | |
| --jq ".[] | select(.title == \"$ISSUE_TITLE\") | .number" | head -n1) | |
| if [ -n "$ISSUE_NUMBER" ]; then | |
| gh issue close "$ISSUE_NUMBER" --repo "${{ github.repository }}" \ | |
| --comment "Fleet audit is green again: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}" | |
| fi | |
| if [ "$pagerduty_status" -ne 0 ]; then | |
| echo "::error::PagerDuty resolve failed; GitHub issue bookkeeping completed only if available." >&2 | |
| exit "$pagerduty_status" | |
| fi |