|
| 1 | +package slack |
| 2 | + |
| 3 | +import ( |
| 4 | + "database/sql" |
| 5 | + "fmt" |
| 6 | + "math" |
| 7 | + "testing" |
| 8 | + "time" |
| 9 | + |
| 10 | + "github.com/anthropics/usage-dashboard/internal/store" |
| 11 | +) |
| 12 | + |
| 13 | +func newCalc(t *testing.T) (*Calculator, *store.Store) { |
| 14 | + t.Helper() |
| 15 | + s, err := store.Open(":memory:") |
| 16 | + if err != nil { |
| 17 | + t.Fatalf("open store: %v", err) |
| 18 | + } |
| 19 | + cfg := Config{ |
| 20 | + HeadroomThreshold: 0.10, |
| 21 | + QuietPeriodSeconds: 300, |
| 22 | + FreshnessThresholdMs: 48 * 3600 * 1000, // 48 hours |
| 23 | + } |
| 24 | + return NewCalculator(s.DB(), cfg), s |
| 25 | +} |
| 26 | + |
| 27 | +func insertWindow(t *testing.T, db *sql.DB, kind string, startedAt, endsAt time.Time, baselineTotal float64, baselineSource string) int64 { |
| 28 | + t.Helper() |
| 29 | + res, err := db.Exec( |
| 30 | + `INSERT INTO windows (kind, started_at, ends_at, baseline_total, baseline_source, closed) |
| 31 | + VALUES (?, ?, ?, ?, ?, 0)`, |
| 32 | + kind, startedAt, endsAt, baselineTotal, baselineSource, |
| 33 | + ) |
| 34 | + if err != nil { |
| 35 | + t.Fatalf("insert window: %v", err) |
| 36 | + } |
| 37 | + id, err := res.LastInsertId() |
| 38 | + if err != nil { |
| 39 | + t.Fatalf("LastInsertId: %v", err) |
| 40 | + } |
| 41 | + return id |
| 42 | +} |
| 43 | + |
| 44 | +func fptr(v float64) *float64 { return &v } |
| 45 | + |
| 46 | +// (a) combineSlackFractions returns min(a, b), propagates nil correctly, |
| 47 | +// and returns nil if either input is nil. |
| 48 | +func TestCombineSlackFractions(t *testing.T) { |
| 49 | + c := &Calculator{} |
| 50 | + |
| 51 | + tests := []struct { |
| 52 | + name string |
| 53 | + a, b *WindowMetrics |
| 54 | + want *float64 |
| 55 | + }{ |
| 56 | + {"both present, a smaller", &WindowMetrics{SlackFraction: fptr(0.10)}, &WindowMetrics{SlackFraction: fptr(0.50)}, fptr(0.10)}, |
| 57 | + {"both present, b smaller", &WindowMetrics{SlackFraction: fptr(0.50)}, &WindowMetrics{SlackFraction: fptr(0.10)}, fptr(0.10)}, |
| 58 | + {"both present, equal", &WindowMetrics{SlackFraction: fptr(0.30)}, &WindowMetrics{SlackFraction: fptr(0.30)}, fptr(0.30)}, |
| 59 | + {"both present, negative wins", &WindowMetrics{SlackFraction: fptr(0.20)}, &WindowMetrics{SlackFraction: fptr(-0.05)}, fptr(-0.05)}, |
| 60 | + {"both metrics nil", nil, nil, nil}, |
| 61 | + {"a metric nil", nil, &WindowMetrics{SlackFraction: fptr(0.5)}, nil}, |
| 62 | + {"b metric nil", &WindowMetrics{SlackFraction: fptr(0.5)}, nil, nil}, |
| 63 | + {"a SlackFraction nil", &WindowMetrics{SlackFraction: nil}, &WindowMetrics{SlackFraction: fptr(0.5)}, nil}, |
| 64 | + {"b SlackFraction nil", &WindowMetrics{SlackFraction: fptr(0.5)}, &WindowMetrics{SlackFraction: nil}, nil}, |
| 65 | + {"both SlackFraction nil", &WindowMetrics{SlackFraction: nil}, &WindowMetrics{SlackFraction: nil}, nil}, |
| 66 | + } |
| 67 | + |
| 68 | + fmtFrac := func(p *float64) string { |
| 69 | + if p == nil { |
| 70 | + return "<nil>" |
| 71 | + } |
| 72 | + return fmt.Sprintf("%v", *p) |
| 73 | + } |
| 74 | + for _, tt := range tests { |
| 75 | + t.Run(tt.name, func(t *testing.T) { |
| 76 | + got := c.combineSlackFractions(tt.a, tt.b) |
| 77 | + if (got == nil) != (tt.want == nil) { |
| 78 | + t.Fatalf("nil-ness mismatch: got=%s want=%s", fmtFrac(got), fmtFrac(tt.want)) |
| 79 | + } |
| 80 | + if got != nil && *got != *tt.want { |
| 81 | + t.Errorf("got %v, want %v", *got, *tt.want) |
| 82 | + } |
| 83 | + }) |
| 84 | + } |
| 85 | +} |
| 86 | + |
| 87 | +// (b) GetSlack returns null slack_combined_fraction when the 5-hour window |
| 88 | +// has no events yet (per docs/slack-indicator.md: window is undefined until |
| 89 | +// first use). Two sub-cases: the simple "no windows at all" case and the |
| 90 | +// more discriminating case where a weekly window exists with events but no |
| 91 | +// 5-hour window does — combined must still be nil. |
| 92 | +func TestGetSlack_NullCombinedFractionWhenNoEvents(t *testing.T) { |
| 93 | + t.Run("empty database", func(t *testing.T) { |
| 94 | + c, s := newCalc(t) |
| 95 | + defer s.Close() |
| 96 | + |
| 97 | + resp, err := c.GetSlack() |
| 98 | + if err != nil { |
| 99 | + t.Fatalf("GetSlack: %v", err) |
| 100 | + } |
| 101 | + if resp.SlackFraction != nil { |
| 102 | + t.Errorf("expected nil slack_combined_fraction, got %v", *resp.SlackFraction) |
| 103 | + } |
| 104 | + if resp.ReleaseRecommended { |
| 105 | + t.Error("expected release_recommended=false") |
| 106 | + } |
| 107 | + }) |
| 108 | + |
| 109 | + t.Run("weekly window present, no five_hour window", func(t *testing.T) { |
| 110 | + c, s := newCalc(t) |
| 111 | + defer s.Close() |
| 112 | + |
| 113 | + now := time.Now().UTC() |
| 114 | + insertWindow(t, s.DB(), "weekly", now.Add(-24*time.Hour), now.Add(6*24*time.Hour), 5000.0, "snapshot:1") |
| 115 | + |
| 116 | + // Event inside the weekly window — gives weekly a non-nil |
| 117 | + // SlackFraction. The 5-hour window does not exist. |
| 118 | + cost := 5.0 |
| 119 | + if _, err := s.InsertUsageEvent( |
| 120 | + now.Add(-1*time.Hour), "api", |
| 121 | + "sess-1", "msg-1", "", "claude-3-5-sonnet-20241022", |
| 122 | + 1000, 500, 0, 0, |
| 123 | + &cost, "reported", "{}", |
| 124 | + ); err != nil { |
| 125 | + t.Fatalf("insert event: %v", err) |
| 126 | + } |
| 127 | + |
| 128 | + resp, err := c.GetSlack() |
| 129 | + if err != nil { |
| 130 | + t.Fatalf("GetSlack: %v", err) |
| 131 | + } |
| 132 | + if resp.SlackFraction != nil { |
| 133 | + t.Errorf("expected nil slack_combined_fraction when 5-hour absent, got %v", *resp.SlackFraction) |
| 134 | + } |
| 135 | + if resp.ReleaseRecommended { |
| 136 | + t.Error("expected release_recommended=false when 5-hour window absent") |
| 137 | + } |
| 138 | + }) |
| 139 | +} |
| 140 | + |
| 141 | +// (c) RecordRelease writes a row whose window_id resolves to the 5-hour |
| 142 | +// window containing released_at. |
| 143 | +func TestRecordRelease_ResolvesWindowID(t *testing.T) { |
| 144 | + c, s := newCalc(t) |
| 145 | + defer s.Close() |
| 146 | + |
| 147 | + now := time.Now().UTC() |
| 148 | + startedAt := now.Add(-1 * time.Hour) |
| 149 | + endsAt := now.Add(4 * time.Hour) |
| 150 | + wantID := insertWindow(t, s.DB(), "five_hour", startedAt, endsAt, 1000.0, "snapshot:1") |
| 151 | + |
| 152 | + // Also insert a non-overlapping older five_hour window to ensure the |
| 153 | + // resolver picks the one bracketing released_at, not just the latest. |
| 154 | + insertWindow(t, s.DB(), "five_hour", now.Add(-10*time.Hour), now.Add(-5*time.Hour), 800.0, "snapshot:0") |
| 155 | + |
| 156 | + cost := 1.20 |
| 157 | + slackVal := 8.40 |
| 158 | + releaseID, err := c.RecordRelease(now, "nightly-lint", &cost, &slackVal) |
| 159 | + if err != nil { |
| 160 | + t.Fatalf("RecordRelease: %v", err) |
| 161 | + } |
| 162 | + |
| 163 | + var gotWindowID int64 |
| 164 | + err = s.DB().QueryRow(`SELECT window_id FROM slack_releases WHERE id = ?`, releaseID).Scan(&gotWindowID) |
| 165 | + if err != nil { |
| 166 | + t.Fatalf("query slack_releases: %v", err) |
| 167 | + } |
| 168 | + if gotWindowID != wantID { |
| 169 | + t.Errorf("window_id: got %d, want %d", gotWindowID, wantID) |
| 170 | + } |
| 171 | +} |
| 172 | + |
| 173 | +// (d) RecordRelease returns an error when no matching 5-hour window |
| 174 | +// contains released_at. |
| 175 | +func TestRecordRelease_ErrorWhenNoWindow(t *testing.T) { |
| 176 | + c, s := newCalc(t) |
| 177 | + defer s.Close() |
| 178 | + |
| 179 | + now := time.Now().UTC() |
| 180 | + // Insert a weekly window that contains released_at, but no five_hour |
| 181 | + // window — RecordRelease must still error. |
| 182 | + insertWindow(t, s.DB(), "weekly", now.Add(-24*time.Hour), now.Add(6*24*time.Hour), 5000.0, "snapshot:1") |
| 183 | + |
| 184 | + if _, err := c.RecordRelease(now, "nightly-lint", nil, nil); err == nil { |
| 185 | + t.Error("expected error when no five_hour window matches released_at") |
| 186 | + } |
| 187 | +} |
| 188 | + |
| 189 | +// (e) SetPaused(true) forces release_recommended=false even when slack is |
| 190 | +// positive. Asserted via the typed SlackResponse.ReleaseRecommended field |
| 191 | +// rather than any gate-map key. |
| 192 | +func TestSetPaused_ForcesReleaseRecommendedFalse(t *testing.T) { |
| 193 | + c, s := newCalc(t) |
| 194 | + defer s.Close() |
| 195 | + |
| 196 | + now := time.Now().UTC() |
| 197 | + insertWindow(t, s.DB(), "five_hour", now.Add(-1*time.Hour), now.Add(4*time.Hour), 1000.0, "snapshot:1") |
| 198 | + insertWindow(t, s.DB(), "weekly", now.Add(-24*time.Hour), now.Add(6*24*time.Hour), 5000.0, "snapshot:1") |
| 199 | + |
| 200 | + // Small consumption keeps slack positive on both windows. |
| 201 | + cost := 5.0 |
| 202 | + if _, err := s.InsertUsageEvent( |
| 203 | + now.Add(-30*time.Minute), "api", |
| 204 | + "sess-1", "msg-1", "", "claude-3-5-sonnet-20241022", |
| 205 | + 1000, 500, 0, 0, |
| 206 | + &cost, "reported", "{}", |
| 207 | + ); err != nil { |
| 208 | + t.Fatalf("insert event: %v", err) |
| 209 | + } |
| 210 | + |
| 211 | + // Fresh quota snapshot satisfies any freshness gate. |
| 212 | + rem, total := 950.0, 1000.0 |
| 213 | + if _, err := s.InsertQuotaSnapshot( |
| 214 | + now, now, "userscript", |
| 215 | + &rem, &total, nil, |
| 216 | + &rem, &total, nil, |
| 217 | + "{}", |
| 218 | + ); err != nil { |
| 219 | + t.Fatalf("insert snapshot: %v", err) |
| 220 | + } |
| 221 | + |
| 222 | + c.SetPaused(true) |
| 223 | + resp, err := c.GetSlack() |
| 224 | + if err != nil { |
| 225 | + t.Fatalf("GetSlack: %v", err) |
| 226 | + } |
| 227 | + |
| 228 | + if !resp.Paused { |
| 229 | + t.Error("expected Paused=true") |
| 230 | + } |
| 231 | + if resp.SlackFraction == nil { |
| 232 | + t.Fatal("test setup invalid: expected positive slack fraction, got nil") |
| 233 | + } |
| 234 | + if *resp.SlackFraction <= 0 { |
| 235 | + t.Fatalf("test setup invalid: expected positive slack fraction, got %v", *resp.SlackFraction) |
| 236 | + } |
| 237 | + if resp.ReleaseRecommended { |
| 238 | + t.Error("expected ReleaseRecommended=false when paused, regardless of slack") |
| 239 | + } |
| 240 | +} |
| 241 | + |
| 242 | +// (f) Computed Progress/Expected/Slack match the formulas from |
| 243 | +// docs/slack-indicator.md: |
| 244 | +// |
| 245 | +// progress(t) = clamp((t - t0) / (t1 - t0), 0, 1) |
| 246 | +// E(t) = Q * progress(t) |
| 247 | +// slack(t) = E(t) - U(t) |
| 248 | +// slack_fraction(t) = slack(t) / Q |
| 249 | +// |
| 250 | +// Window bounds bracket time.Now() so the test does not depend on |
| 251 | +// wall-clock alignment. |
| 252 | +func TestComputeMetrics_FormulasMatchDocs(t *testing.T) { |
| 253 | + c, s := newCalc(t) |
| 254 | + defer s.Close() |
| 255 | + |
| 256 | + const baseline = 1000.0 |
| 257 | + const consumed = 50.0 |
| 258 | + |
| 259 | + now := time.Now().UTC() |
| 260 | + startedAt := now.Add(-1 * time.Hour) |
| 261 | + endsAt := now.Add(4 * time.Hour) // 5-hour window total |
| 262 | + insertWindow(t, s.DB(), "five_hour", startedAt, endsAt, baseline, "snapshot:1") |
| 263 | + |
| 264 | + cost := consumed |
| 265 | + if _, err := s.InsertUsageEvent( |
| 266 | + now.Add(-30*time.Minute), "api", |
| 267 | + "sess-1", "msg-1", "", "claude-3-5-sonnet-20241022", |
| 268 | + 1000, 500, 0, 0, |
| 269 | + &cost, "reported", "{}", |
| 270 | + ); err != nil { |
| 271 | + t.Fatalf("insert event: %v", err) |
| 272 | + } |
| 273 | + |
| 274 | + resp, err := c.GetSlack() |
| 275 | + if err != nil { |
| 276 | + t.Fatalf("GetSlack: %v", err) |
| 277 | + } |
| 278 | + m := resp.FiveHourWindow |
| 279 | + if m == nil { |
| 280 | + t.Fatal("expected FiveHourWindow metrics, got nil") |
| 281 | + } |
| 282 | + |
| 283 | + // Recompute the expected values relative to a "now" sampled inside |
| 284 | + // the test, using the same formulas the docs prescribe. The window is |
| 285 | + // 5 hours; sub-second drift between inserting and reading is |
| 286 | + // negligible relative to the tolerances below (0.5 of $1000). |
| 287 | + windowDur := endsAt.Sub(startedAt).Seconds() |
| 288 | + elapsed := time.Since(startedAt).Seconds() |
| 289 | + if elapsed < 0 { |
| 290 | + elapsed = 0 |
| 291 | + } |
| 292 | + if elapsed > windowDur { |
| 293 | + elapsed = windowDur |
| 294 | + } |
| 295 | + progress := elapsed / windowDur |
| 296 | + expectedE := baseline * progress |
| 297 | + expectedSlack := expectedE - consumed |
| 298 | + expectedSlackFrac := expectedSlack / baseline |
| 299 | + |
| 300 | + // Progress (cumulative consumed) must equal the inserted cost exactly. |
| 301 | + if math.Abs(m.Progress-consumed) > 1e-6 { |
| 302 | + t.Errorf("Progress (consumed): got %v, want %v", m.Progress, consumed) |
| 303 | + } |
| 304 | + |
| 305 | + // Allow a small tolerance for time-since-insertion drift. |
| 306 | + const tol = 0.5 |
| 307 | + if math.Abs(m.Expected-expectedE) > tol { |
| 308 | + t.Errorf("Expected (E): got %v, want ~%v (tol %v)", m.Expected, expectedE, tol) |
| 309 | + } |
| 310 | + if math.Abs(m.Slack-expectedSlack) > tol { |
| 311 | + t.Errorf("Slack (E - U): got %v, want ~%v (tol %v)", m.Slack, expectedSlack, tol) |
| 312 | + } |
| 313 | + if m.SlackFraction == nil { |
| 314 | + t.Fatal("expected non-nil SlackFraction") |
| 315 | + } |
| 316 | + if math.Abs(*m.SlackFraction-expectedSlackFrac) > tol/baseline { |
| 317 | + t.Errorf("SlackFraction: got %v, want ~%v", *m.SlackFraction, expectedSlackFrac) |
| 318 | + } |
| 319 | +} |
0 commit comments