fix(vast): bound SSH readiness observations and waits

CHANGELOG.md

@@ -11,6 +11,7 @@
 
 ### Fixes
 
+- Bound Vast SSH-endpoint readiness requests and retry waits by the startup deadline, preserving cancellation causes without exposing redacted transport secrets. [PR 2448](https://github.com/openclaw/crabbox/pull/2448).
 - Preserve RunPod SSH-readiness cancellation causes, distinguish startup deadlines from caller cancellation, and retain completed provider errors. [PR 2449](https://github.com/openclaw/crabbox/pull/2449).
 
 - Use EC2 instance metadata for AWS vCPU quota admission and readiness, including bare-metal types, and keep unknown instance costs out of capacity recommendations. [PR 2302](https://github.com/openclaw/crabbox/pull/2302). Thanks @vincentkoc.

docs/providers/vast.md

@@ -177,6 +177,11 @@ URLs.
 10. Wait for Crabbox SSH bootstrap readiness and write a local lease claim.
 11. Run normal Crabbox SSH sync, command execution, status, list, and cleanup.
 
+The initial native SSH-endpoint wait has a ten-minute elapsed-time budget that
+bounds both API requests and the gaps between observations. Caller cancellation
+stops the wait without extending that budget; lifecycle timestamps do not control
+it. A completed ready response or terminal provider error retains precedence.
+
 The provider requires Linux. It does not advertise desktop, browser, code-server,
 Tailscale, coordinator, or provider-managed sync support in this release.
 Actions hydration works only as normal command execution on the resulting Linux

internal/providers/vast/backend.go

@@ -49,16 +49,7 @@ func newBackend(spec core.ProviderSpec, cfg core.Config, rt core.Runtime) *backe
 		return core.WaitForSSHReady(ctx, target, b.stderr(), phase, timeout)
 	}
 	b.runSSH = core.RunSSHQuiet
-	b.sleep = func(ctx context.Context, d time.Duration) error {
-		timer := time.NewTimer(d)
-		defer timer.Stop()
-		select {
-		case <-ctx.Done():
-			return ctx.Err()
-		case <-timer.C:
-			return nil
-		}
-	}
+	b.sleep = shared.SleepContext
 	return b
 }
 
@@ -377,26 +368,42 @@ func vastMatchingSSHKeyID(keys []vastInstanceSSHKey, publicKey string) string {
 }
 
 func (b *backend) waitForInstanceReady(ctx context.Context, client vastAPI, id int) (vastInstance, error) {
-	deadline := b.now().Add(b.pollTimeout)
-	result, err := shared.Poll(context.WithoutCancel(ctx), 0, vastPollInterval,
-		func(context.Context, time.Duration) error { return b.sleep(ctx, vastPollInterval) },
-		func(context.Context) (vastInstance, error) { return client.GetInstance(ctx, id) },
+	budgetExpired := errors.New("Vast SSH readiness deadline exceeded")
+	waitCtx, cancel := context.WithTimeoutCause(ctx, b.pollTimeout, budgetExpired)
+	defer cancel()
+	var observationError error
+	result, err := shared.Poll(waitCtx, 0, vastPollInterval, b.sleep,
+		func(ctx context.Context) (vastInstance, error) { return client.GetInstance(ctx, id) },
 		func(_ context.Context, instance vastInstance, fetchErr error) (bool, error) {
 			if fetchErr != nil {
+				var apiErr *vastAPIError
+				if cause := context.Cause(waitCtx); cause != nil && !errors.As(fetchErr, &apiErr) &&
+					(errors.Is(fetchErr, cause) || errors.Is(fetchErr, waitCtx.Err())) {
+					return false, errors.Join(cause, fetchErr)
+				}
+				observationError = fetchErr
 				return false, fetchErr
 			}
 			if isVastInstanceRunning(instance) && strings.TrimSpace(instance.SSHHost) != "" && instance.SSHPort > 0 {
 				return true, nil
 			}
 			if isTerminalVastStatus(instance.Status) {
-				return false, core.Exit(5, "vast instance %d reached terminal status %s", id, instance.Status)
-			}
-			if b.now().After(deadline) {
-				return false, core.Exit(5, "timed out waiting for Vast instance %d to expose SSH", id)
+				observationError = core.Exit(5, "vast instance %d reached terminal status %s", id, instance.Status)
+				return false, observationError
 			}
 			return false, nil
 		}, nil)
 	if err != nil {
+		// A completed provider response retains precedence over later cancellation.
+		if observationError != nil {
+			return vastInstance{}, observationError
+		}
+		if errors.Is(err, budgetExpired) {
+			return vastInstance{}, shared.PollTerminationError(waitCtx, err, core.Exit(5, "timed out waiting for Vast instance %d to expose SSH", id))
+		}
+		if cause := context.Cause(waitCtx); cause != nil && errors.Is(err, cause) {
+			return vastInstance{}, shared.PollTerminationError(waitCtx, err, waitCtx.Err())
+		}
 		return vastInstance{}, err
 	}
 	return result.Value, nil

internal/providers/vast/backend_test.go

@@ -27,6 +27,7 @@ type fakeVastAPI struct {
 	listErr                 error
 	createErr               error
 	getErr                  error
+	getFn                   func(context.Context, int) (vastInstance, error)
 	manageErr               error
 	destroyErr              error
 	destroyFn               func()
@@ -109,7 +110,10 @@ func (f *fakeVastAPI) CreateInstance(_ context.Context, offerID int, input vastC
 	return vastCreateInstanceResponse{Success: true, NewContract: item.ID, Instance: item}, nil
 }
 
-func (f *fakeVastAPI) GetInstance(_ context.Context, id int) (vastInstance, error) {
+func (f *fakeVastAPI) GetInstance(ctx context.Context, id int) (vastInstance, error) {
+	if f.getFn != nil {
+		return f.getFn(ctx, id)
+	}
 	if f.getErr != nil {
 		return vastInstance{}, f.getErr
 	}
@@ -206,7 +210,7 @@ func (f *fakeVastAPI) DetachInstanceSSHKey(_ context.Context, id int, keyID stri
 	return f.detachErr
 }
 
-func newTestBackend(t *testing.T, api *fakeVastAPI) *backend {
+func newTestBackend(t *testing.T, api vastAPI) *backend {
 	t.Helper()
 	testutil.IsolateUserDirs(t)
 	cfg := core.BaseConfig()
@@ -232,6 +236,129 @@ func newTestBackend(t *testing.T, api *fakeVastAPI) *backend {
 	return b
 }
 
+func TestWaitForInstanceReadyStopsBeforeCanceledRead(t *testing.T) {
+	cause := errors.New("caller stopped")
+	ctx, cancel := context.WithCancelCause(t.Context())
+	cancel(cause)
+	api := &fakeVastAPI{getFn: func(context.Context, int) (vastInstance, error) {
+		t.Fatal("canceled readiness initiated a read")
+		return vastInstance{}, nil
+	}}
+	b := newTestBackend(t, api)
+	_, err := b.waitForInstanceReady(ctx, api, 100)
+	if !errors.Is(err, cause) || !errors.Is(err, context.Canceled) {
+		t.Fatalf("err=%v, want caller cause and cancellation", err)
+	}
+}
+
+func TestWaitForInstanceReadyBoundsObservationsAndSleep(t *testing.T) {
+	for _, phase := range []string{"read Err", "read Cause", "sleep"} {
+		t.Run(phase, func(t *testing.T) {
+			ctx, cancel := context.WithTimeout(t.Context(), time.Second)
+			defer cancel()
+			api := &fakeVastAPI{getFn: func(ctx context.Context, _ int) (vastInstance, error) {
+				if phase == "sleep" {
+					return vastInstance{Status: "loading"}, nil
+				}
+				<-ctx.Done()
+				if phase == "read Cause" {
+					return vastInstance{}, context.Cause(ctx)
+				}
+				return vastInstance{}, ctx.Err()
+			}}
+			b := newTestBackend(t, api)
+			b.pollTimeout = 20 * time.Millisecond
+			b.rt.Clock = &lifecycleClock{current: time.Date(2026, 1, 1, 0, 0, 0, 0, time.UTC)}
+			b.sleep = func(ctx context.Context, _ time.Duration) error {
+				<-ctx.Done()
+				return ctx.Err()
+			}
+			_, err := b.waitForInstanceReady(ctx, api, 100)
+			var exit core.ExitError
+			if !core.AsExitError(err, &exit) || exit.Code != 5 || !strings.Contains(err.Error(), "timed out waiting for Vast instance 100 to expose SSH") || !errors.Is(err, context.DeadlineExceeded) {
+				t.Fatalf("err=%v, want bounded readiness timeout with deadline identity", err)
+			}
+			if ctx.Err() != nil {
+				t.Fatalf("parent guard expired before readiness budget: %v", ctx.Err())
+			}
+		})
+	}
+}
+
+func TestWaitForInstanceReadyPreservesCallerCause(t *testing.T) {
+	for _, phase := range []string{"read", "sleep"} {
+		t.Run(phase, func(t *testing.T) {
+			cause := errors.New("private caller cancellation")
+			ctx, cancel := context.WithCancelCause(t.Context())
+			defer cancel(nil)
+			api := &fakeVastAPI{getFn: func(ctx context.Context, _ int) (vastInstance, error) {
+				if phase == "read" {
+					cancel(cause)
+					return vastInstance{}, ctx.Err()
+				}
+				return vastInstance{Status: "loading"}, nil
+			}}
+			b := newTestBackend(t, api)
+			b.sleep = func(ctx context.Context, _ time.Duration) error {
+				cancel(cause)
+				return ctx.Err()
+			}
+			_, err := b.waitForInstanceReady(ctx, api, 100)
+			if !errors.Is(err, cause) || !errors.Is(err, context.Canceled) {
+				t.Fatalf("err=%v, want custom cause and context cancellation", err)
+			}
+			if strings.Contains(err.Error(), cause.Error()) {
+				t.Fatalf("diagnostic exposed private context cause: %v", err)
+			}
+		})
+	}
+}
+
+func TestWaitForInstanceReadyPreservesCompletedObservation(t *testing.T) {
+	for _, outcome := range []string{"ready", "terminal", "API failure", "client deadline"} {
+		t.Run(outcome, func(t *testing.T) {
+			ctx, cancel := context.WithCancel(t.Context())
+			defer cancel()
+			apiErr := &vastAPIError{StatusCode: 403, Status: "403 Forbidden"}
+			api := &fakeVastAPI{getFn: func(context.Context, int) (vastInstance, error) {
+				if outcome != "client deadline" {
+					cancel()
+				}
+				switch outcome {
+				case "ready":
+					return vastInstance{ID: 100, Status: "running", SSHHost: "203.0.113.1", SSHPort: 2222}, nil
+				case "terminal":
+					return vastInstance{ID: 100, Status: "exited"}, nil
+				case "API failure":
+					return vastInstance{}, apiErr
+				default:
+					return vastInstance{}, context.DeadlineExceeded
+				}
+			}}
+			b := newTestBackend(t, api)
+			got, err := b.waitForInstanceReady(ctx, api, 100)
+			switch outcome {
+			case "ready":
+				if err != nil || got.ID != 100 {
+					t.Fatalf("got=%+v err=%v", got, err)
+				}
+			case "terminal":
+				if err == nil || !strings.Contains(err.Error(), "reached terminal status exited") {
+					t.Fatalf("err=%v", err)
+				}
+			case "API failure":
+				if err != apiErr {
+					t.Fatalf("err=%v, want original API response", err)
+				}
+			case "client deadline":
+				if err != context.DeadlineExceeded {
+					t.Fatalf("err=%v, want independent client deadline", err)
+				}
+			}
+		})
+	}
+}
+
 func TestNewBackendPreservesExplicitGenericSSHUser(t *testing.T) {
 	cfg := core.BaseConfig()
 	cfg.Provider = providerName

internal/providers/vast/client.go

@@ -169,7 +169,7 @@ func (c *vastClient) do(ctx context.Context, method, path string, body any, out
 	}
 	resp, err := c.httpClient.Do(req)
 	if err != nil {
-		return redactVastString(err.Error(), c.apiKey)
+		return shared.ExitErrorWithCause(1, redactVastText(err.Error(), c.apiKey), err)
 	}
 	defer resp.Body.Close()
 	data, readErr := io.ReadAll(io.LimitReader(resp.Body, vastMaxResponseBytes+1))
@@ -632,10 +632,6 @@ func isLoopbackHTTPURL(parsed *url.URL) bool {
 	return host == "localhost" || host == "127.0.0.1" || host == "::1" || (ip != nil && ip.IsLoopback())
 }
 
-func redactVastString(value, apiKey string) error {
-	return errors.New(redactVastText(value, apiKey))
-}
-
 func redactVastText(value, apiKey string) string {
 	out := value
 	if apiKey != "" {

internal/providers/vast/client_test.go

@@ -4,10 +4,12 @@ import (
 	"context"
 	"encoding/json"
 	"errors"
+	"fmt"
 	"net/http"
 	"net/http/httptest"
 	"strings"
 	"testing"
+	"time"
 
 	core "github.com/openclaw/crabbox/internal/cli"
 	"github.com/openclaw/crabbox/internal/testutil"
@@ -79,6 +81,46 @@ func TestRedactVastAPIErrorSecrets(t *testing.T) {
 	}
 }
 
+func TestTransportErrorPreservesCauseWithoutDisplayingSecrets(t *testing.T) {
+	cause := fmt.Errorf("failed with vast-secret: %w", context.DeadlineExceeded)
+	client, err := newVastClient(core.VastConfig{APIKey: "vast-secret", APIURL: "https://example.test"}, core.Runtime{HTTP: &http.Client{Transport: testutil.RoundTripFunc(func(*http.Request) (*http.Response, error) {
+		return nil, cause
+	})}})
+	if err != nil {
+		t.Fatal(err)
+	}
+	_, err = client.GetInstance(t.Context(), 100)
+	if !errors.Is(err, cause) || !errors.Is(err, context.DeadlineExceeded) {
+		t.Fatalf("err=%v, want original transport cause", err)
+	}
+	if strings.Contains(err.Error(), "vast-secret") || !strings.Contains(err.Error(), "<redacted>") {
+		t.Fatalf("transport diagnostic not redacted: %v", err)
+	}
+}
+
+func TestReadinessDeadlineCancelsNativeHTTPClient(t *testing.T) {
+	server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		select {
+		case <-r.Context().Done():
+		case <-t.Context().Done():
+		}
+	}))
+	defer server.Close()
+	client, err := newVastClient(core.VastConfig{APIKey: "fixture-key", APIURL: server.URL}, core.Runtime{HTTP: server.Client()})
+	if err != nil {
+		t.Fatal(err)
+	}
+	b := newTestBackend(t, client)
+	b.pollTimeout = 50 * time.Millisecond
+	ctx, cancel := context.WithTimeout(t.Context(), time.Second)
+	defer cancel()
+	_, err = b.waitForInstanceReady(ctx, client, 100)
+	var exit core.ExitError
+	if !core.AsExitError(err, &exit) || exit.Code != 5 || !errors.Is(err, context.DeadlineExceeded) || ctx.Err() != nil {
+		t.Fatalf("err=%v parent=%v, want own readiness deadline from real HTTP request", err, ctx.Err())
+	}
+}
+
 func TestOfferSearchPayloadAndDecode(t *testing.T) {
 	server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
 		if r.Method != http.MethodPost || r.URL.Path != "/api/v0/bundles/" {