deploy: let the probe try every provider it discovers

The first hub deploy with the probe failed at the last step: discovery
returned three providers for the everything service and the probe took
the first, a peer id from a pod the rollout had just replaced. The egress
gate could not reach it to verify its credential and answered 403
"destination is not an enrolled peer". The scheduled runs since show the
same coin toss: whichever of the two stale records lists first fails the
run, the live pod passes it.

A provider record naming a pod that is gone is mesh life, not a fault --
the docs already tell agents that discovery is best-effort per peer -- so
the probe now behaves like an agent should: every discovered provider is
tried, the whole discover-and-call is retried for up to two minutes, and
the run reports how long a live provider took to reach and how many were
tried. The Job deadline and the deploy step's wait grow to match.

Reproduced in kind by rolling the everything canary under the probe:
ready in 2s, live provider reached after 51s, initialize in 9ms.

Author: aojea

.github/k8s/sam-probe-cronjob-template.yaml

@@ -19,7 +19,8 @@ spec:
       # One attempt: a retry would hide exactly the flakiness this exists to
       # measure. A failed Job is the signal.
       backoffLimit: 0
-      activeDeadlineSeconds: 300
+      # 180s to be ready plus 120s to reach a provider, with headroom.
+      activeDeadlineSeconds: 360
       template:
         metadata:
           labels:
@@ -118,35 +119,43 @@ spec:
               ROUTER_MS=$(printf '%s' "$CONN" | grep -oE '"router_latency_ms":[0-9]+' | grep -oE '[0-9]+$')
               PEERS=$(printf '%s' "$CONN" | grep -oE '"connected_peers":[0-9]+' | grep -oE '[0-9]+$')
 
-              # 3. A service someone else advertised is discoverable. A node
-              # that joined seconds ago has a thin routing table, so the lookup
-              # is retried for a while; how long it takes is itself reported.
+              # 3. A service someone else advertised is discoverable, and one
+              # of its providers answers an MCP initialize through the mesh.
+              # Discovery is best-effort per peer: a provider record can name a
+              # pod that a rollout just replaced, and a node that joined
+              # seconds ago has a thin routing table. So every provider is
+              # tried, the whole thing is retried for a while, and how long it
+              # took is itself reported. No trailing slash on the call: the
+              # path maps onto the service's target_url as-is. The body is JSON
+              # or an SSE frame; either names the server.
               PEER=""
-              DISCOVER_START=$(date +%s)
+              TRIED=0
+              LAST_ERR=""
+              REACH_START=$(date +%s)
               while [ -z "$PEER" ]; do
                 if curl -sf --unix-socket "$SOCK" -o /tmp/providers.json \
                   "http://localhost/sam/service/discover?type=mcp&name=${PROBE_SERVICE}&timeout=20s"; then
-                  PEER=$(grep -oE '"peer_id":"[^"]+"' /tmp/providers.json | head -1 | cut -d'"' -f4)
+                  for p in $(grep -oE '"peer_id":"[^"]+"' /tmp/providers.json | cut -d'"' -f4); do
+                    TRIED=$((TRIED + 1))
+                    set -- $(curl -s --unix-socket "$SOCK" -o /tmp/init.out -w '%{http_code} %{time_total}' \
+                      -X POST "http://localhost/sam/${p}/mcp/${PROBE_SERVICE}" \
+                      -H 'Content-Type: application/json' -H 'Accept: application/json, text/event-stream' \
+                      -d '{"jsonrpc":"2.0","id":1,"method":"initialize","params":{"protocolVersion":"2025-06-18","capabilities":{},"clientInfo":{"name":"sam-probe","version":"0"}}}')
+                    if [ "$1" = "200" ] && grep -q '"serverInfo"' /tmp/init.out; then
+                      PEER=$p; CALL_S=$2
+                      break
+                    fi
+                    LAST_ERR="${p}: HTTP $1 $(head -c 120 /tmp/init.out | tr -d '"\n')"
+                  done
                 fi
                 [ -n "$PEER" ] && break
-                [ $(( $(date +%s) - DISCOVER_START )) -ge 90 ] && fail discover "no provider advertises ${PROBE_SERVICE} after 90s"
-                sleep 3
+                [ $(( $(date +%s) - REACH_START )) -ge 120 ] && fail call "no provider of ${PROBE_SERVICE} answered initialize in 120s (${TRIED} attempts; last: ${LAST_ERR:-none discovered})"
+                sleep 5
               done
-              DISCOVER_S=$(( $(date +%s) - DISCOVER_START ))
+              REACH_S=$(( $(date +%s) - REACH_START ))
 
-              # 4. And answers an MCP initialize through the mesh, end to end.
-              # No trailing slash: the path maps onto the service's target_url
-              # as-is. The body is JSON or an SSE frame; either names the server.
-              set -- $(curl -s --unix-socket "$SOCK" -o /tmp/init.out -w '%{http_code} %{time_total}' \
-                -X POST "http://localhost/sam/${PEER}/mcp/${PROBE_SERVICE}" \
-                -H 'Content-Type: application/json' -H 'Accept: application/json, text/event-stream' \
-                -d '{"jsonrpc":"2.0","id":1,"method":"initialize","params":{"protocolVersion":"2025-06-18","capabilities":{},"clientInfo":{"name":"sam-probe","version":"0"}}}')
-              CALL_CODE=$1; CALL_S=$2
-              [ "$CALL_CODE" = "200" ] || fail call "initialize on ${PEER} returned HTTP ${CALL_CODE}: $(head -c 200 /tmp/init.out)"
-              grep -q '"serverInfo"' /tmp/init.out || fail call "initialize on ${PEER} returned no serverInfo: $(head -c 200 /tmp/init.out)"
-
-              printf '{"probe":"sam-cold-path","ok":true,"ready_s":%d,"router_latency_ms":%s,"connected_peers":%s,"discover_s":%d,"call_s":%s,"provider":"%s","elapsed_s":%d}\n' \
-                "$READY_S" "${ROUTER_MS:-null}" "${PEERS:-null}" "$DISCOVER_S" "$CALL_S" "$PEER" "$(( $(date +%s) - START ))"
+              printf '{"probe":"sam-cold-path","ok":true,"ready_s":%d,"router_latency_ms":%s,"connected_peers":%s,"reach_s":%d,"providers_tried":%d,"call_s":%s,"provider":"%s","elapsed_s":%d}\n' \
+                "$READY_S" "${ROUTER_MS:-null}" "${PEERS:-null}" "$REACH_S" "$TRIED" "$CALL_S" "$PEER" "$(( $(date +%s) - START ))"
           volumes:
           - name: config-volume
             configMap:

.github/workflows/deploy.yaml

@@ -809,7 +809,7 @@ jobs:
 
           # `kubectl wait` can only wait for one condition; a failed Job would
           # otherwise sit out the whole timeout.
-          for _ in $(seq 1 60); do
+          for _ in $(seq 1 72); do
             status=$(kubectl get job "${JOB}" -n "${CANARY_NAMESPACE}" -o jsonpath='{range .status.conditions[*]}{.type}={.status}{"\n"}{end}')
             case "${status}" in
               *Complete=True*) break ;;