diff --git a/TEMPLATES.md b/TEMPLATES.md index 6ec75e3b5..3e1bd1348 100644 --- a/TEMPLATES.md +++ b/TEMPLATES.md @@ -3016,7 +3016,7 @@ Verify the database name, or run the command against the SchemaBot instance that **Status**: In Progress -**`users`**: ๐ŸŸฆ๐ŸŸฆ๐ŸŸฆ๐ŸŸฆ๐ŸŸฆ๐ŸŸฆ๐ŸŸฆ๐ŸŸฆ๐ŸŸฆโฌœโฌœโฌœโฌœโฌœโฌœโฌœโฌœโฌœโฌœโฌœ 48% +**`users`**: ๐ŸŸฆ๐ŸŸฆ๐ŸŸฆ๐ŸŸฆ๐ŸŸฆ๐ŸŸฆ๐ŸŸฆ๐ŸŸฆ๐ŸŸฆโฌœโฌœโฌœโฌœโฌœโฌœโฌœโฌœโฌœโฌœโฌœ 48.61% ```sql ALTER TABLE `users` ADD INDEX `idx_email_created`(`email`, `created_at`); @@ -3099,7 +3099,7 @@ schemabot apply -e staging **Status**: Stopped -**`users`**: ๐ŸŸง๐ŸŸง๐ŸŸง๐ŸŸง๐ŸŸง๐ŸŸง๐ŸŸงโฌœโฌœโฌœโฌœโฌœโฌœโฌœโฌœโฌœโฌœโฌœโฌœโฌœ โน๏ธ Stopped at 39% +**`users`**: ๐ŸŸง๐ŸŸง๐ŸŸง๐ŸŸง๐ŸŸง๐ŸŸง๐ŸŸงโฌœโฌœโฌœโฌœโฌœโฌœโฌœโฌœโฌœโฌœโฌœโฌœโฌœ โน๏ธ Stopped at 39.34% ```sql ALTER TABLE `users` ADD INDEX `idx_email_created`(`email`, `created_at`); @@ -3174,11 +3174,11 @@ _Last updated: 2026-01-01 00:00:0 **Status**: In Progress -๐Ÿ“Š 1 running (22%) ยท 2 queued +๐Ÿ“Š 1 running (21.92%) ยท 2 queued **Schema `testapp`** -**`orders`**: ๐ŸŸฆ๐ŸŸฆ๐ŸŸฆ๐ŸŸฆโฌœโฌœโฌœโฌœโฌœโฌœโฌœโฌœโฌœโฌœโฌœโฌœโฌœโฌœโฌœโฌœ 22% +**`orders`**: ๐ŸŸฆ๐ŸŸฆ๐ŸŸฆ๐ŸŸฆโฌœโฌœโฌœโฌœโฌœโฌœโฌœโฌœโฌœโฌœโฌœโฌœโฌœโฌœโฌœโฌœ 21.92% ```sql ALTER TABLE `orders` ADD INDEX `idx_user_id`(`user_id`); @@ -3221,11 +3221,11 @@ _Last updated: 2026-01-01 00:00:0 **Status**: In Progress -๐Ÿ“Š 1/3 complete ยท 1 running (62%) ยท 1 queued +๐Ÿ“Š 1/3 complete ยท 1 running (62.38%) ยท 1 queued **Schema `testapp`** -**`users`**: ๐ŸŸฆ๐ŸŸฆ๐ŸŸฆ๐ŸŸฆ๐ŸŸฆ๐ŸŸฆ๐ŸŸฆ๐ŸŸฆ๐ŸŸฆ๐ŸŸฆ๐ŸŸฆ๐ŸŸฆโฌœโฌœโฌœโฌœโฌœโฌœโฌœโฌœ 62% +**`users`**: ๐ŸŸฆ๐ŸŸฆ๐ŸŸฆ๐ŸŸฆ๐ŸŸฆ๐ŸŸฆ๐ŸŸฆ๐ŸŸฆ๐ŸŸฆ๐ŸŸฆ๐ŸŸฆ๐ŸŸฆโฌœโฌœโฌœโฌœโฌœโฌœโฌœโฌœ 62.38% ```sql ALTER TABLE `users` ADD INDEX `idx_email`(`email`); @@ -3316,11 +3316,11 @@ _Last updated: 2026-01-01 00:00:0 **Status**: In Progress -๐Ÿ“Š 1/3 complete ยท 1 running (62%) ยท 1 queued +๐Ÿ“Š 1/3 complete ยท 1 running (62.38%) ยท 1 queued **Schema `testapp`** -**`users`**: ๐ŸŸฆ๐ŸŸฆ๐ŸŸฆ๐ŸŸฆ๐ŸŸฆ๐ŸŸฆ๐ŸŸฆ๐ŸŸฆ๐ŸŸฆ๐ŸŸฆ๐ŸŸฆ๐ŸŸฆโฌœโฌœโฌœโฌœโฌœโฌœโฌœโฌœ 62% (throttled) +**`users`**: ๐ŸŸฆ๐ŸŸฆ๐ŸŸฆ๐ŸŸฆ๐ŸŸฆ๐ŸŸฆ๐ŸŸฆ๐ŸŸฆ๐ŸŸฆ๐ŸŸฆ๐ŸŸฆ๐ŸŸฆโฌœโฌœโฌœโฌœโฌœโฌœโฌœโฌœ 62.38% (throttled) ```sql ALTER TABLE `users` ADD INDEX `idx_email`(`email`); @@ -3415,7 +3415,7 @@ _Last updated: 2026-01-01 00:00:0 **Schema `testapp`** -**`users`**: ๐ŸŸฆ๐ŸŸฆ๐ŸŸฆ๐ŸŸฆโฌœโฌœโฌœโฌœโฌœโฌœโฌœโฌœโฌœโฌœโฌœโฌœโฌœโฌœโฌœโฌœ ๐Ÿ” Checksumming to verify data (21%) +**`users`**: ๐ŸŸฆ๐ŸŸฆ๐ŸŸฆ๐ŸŸฆโฌœโฌœโฌœโฌœโฌœโฌœโฌœโฌœโฌœโฌœโฌœโฌœโฌœโฌœโฌœโฌœ ๐Ÿ” Checksumming to verify data (21.92%) ```sql ALTER TABLE `users` ADD INDEX `idx_email`(`email`); @@ -3505,11 +3505,11 @@ _Last updated: 2026-01-01 00:00:0 **Status**: In Progress -๐Ÿ“Š 2/3 complete ยท 1 running (17%) +๐Ÿ“Š 2/3 complete ยท 1 running (16.67%) **Schema `testapp`** -**`products`**: ๐ŸŸฆ๐ŸŸฆ๐ŸŸฆโฌœโฌœโฌœโฌœโฌœโฌœโฌœโฌœโฌœโฌœโฌœโฌœโฌœโฌœโฌœโฌœโฌœ 17% +**`products`**: ๐ŸŸฆ๐ŸŸฆ๐ŸŸฆโฌœโฌœโฌœโฌœโฌœโฌœโฌœโฌœโฌœโฌœโฌœโฌœโฌœโฌœโฌœโฌœโฌœ 16.67% ```sql ALTER TABLE `products` ADD INDEX `idx_price`(`price_cents`); @@ -3554,7 +3554,7 @@ _Last updated: 2026-01-01 00:00:0 **Schema `commerce`** -**`users`**: ๐ŸŸฆ๐ŸŸฆ๐ŸŸฆ๐ŸŸฆ๐ŸŸฆ๐ŸŸฆ๐ŸŸฆ๐ŸŸฆ๐ŸŸฆ๐ŸŸฆ๐ŸŸฆ๐ŸŸฆโฌœโฌœโฌœโฌœโฌœโฌœโฌœโฌœ 62% +**`users`**: ๐ŸŸฆ๐ŸŸฆ๐ŸŸฆ๐ŸŸฆ๐ŸŸฆ๐ŸŸฆ๐ŸŸฆ๐ŸŸฆ๐ŸŸฆ๐ŸŸฆ๐ŸŸฆ๐ŸŸฆโฌœโฌœโฌœโฌœโฌœโฌœโฌœโฌœ 62.38% ```sql ALTER TABLE `users` ADD INDEX `idx_email`(`email`); @@ -3588,7 +3588,7 @@ _Last updated: 2026-01-01 00:00:0 **Schema `commerce`** -**`orders`**: ๐ŸŸฆ๐ŸŸฆ๐ŸŸฆ๐ŸŸฆ๐ŸŸฆ๐ŸŸฆ๐ŸŸฆ๐ŸŸฆ๐ŸŸฆ๐ŸŸฆ๐ŸŸฆ๐ŸŸฆ๐ŸŸฆ๐ŸŸฆโฌœโฌœโฌœโฌœโฌœโฌœ 70% +**`orders`**: ๐ŸŸฆ๐ŸŸฆ๐ŸŸฆ๐ŸŸฆ๐ŸŸฆ๐ŸŸฆ๐ŸŸฆ๐ŸŸฆ๐ŸŸฆ๐ŸŸฆ๐ŸŸฆ๐ŸŸฆ๐ŸŸฆ๐ŸŸฆโฌœโฌœโฌœโฌœโฌœโฌœ 70.00% ```sql ALTER TABLE `orders` ADD COLUMN `region` varchar(32); @@ -4008,7 +4008,7 @@ schemabot apply -e staging **Schema `testapp`** -**`users`**: ๐ŸŸง๐ŸŸง๐ŸŸง๐ŸŸง๐ŸŸง๐ŸŸง๐ŸŸง๐ŸŸง๐ŸŸง๐ŸŸง๐ŸŸง๐ŸŸง๐ŸŸง๐ŸŸงโฌœโฌœโฌœโฌœโฌœโฌœ โน๏ธ Stopped at 72% +**`users`**: ๐ŸŸง๐ŸŸง๐ŸŸง๐ŸŸง๐ŸŸง๐ŸŸง๐ŸŸง๐ŸŸง๐ŸŸง๐ŸŸง๐ŸŸง๐ŸŸง๐ŸŸง๐ŸŸงโฌœโฌœโฌœโฌœโฌœโฌœ โน๏ธ Stopped at 72.00% ```sql ALTER TABLE `users` ADD INDEX `idx_email`(`email`); @@ -4449,7 +4449,7 @@ Cutover is already in progress. SchemaBot will keep reporting progress from the **Status**: Stopped -**`users`**: ๐ŸŸง๐ŸŸง๐ŸŸง๐ŸŸง๐ŸŸง๐ŸŸงโฌœโฌœโฌœโฌœโฌœโฌœโฌœโฌœโฌœโฌœโฌœโฌœโฌœโฌœ โน๏ธ Stopped at 32% +**`users`**: ๐ŸŸง๐ŸŸง๐ŸŸง๐ŸŸง๐ŸŸง๐ŸŸงโฌœโฌœโฌœโฌœโฌœโฌœโฌœโฌœโฌœโฌœโฌœโฌœโฌœโฌœ โน๏ธ Stopped at 31.94% ```sql ALTER TABLE `users` ADD INDEX `idx_email_created`(`email`, `created_at`); @@ -4609,7 +4609,7 @@ _Last updated: 2026-01-01 00:00:0 **Status**: Stopped -**`users`**: ๐ŸŸง๐ŸŸง๐ŸŸง๐ŸŸง๐ŸŸง๐ŸŸงโฌœโฌœโฌœโฌœโฌœโฌœโฌœโฌœโฌœโฌœโฌœโฌœโฌœโฌœ โน๏ธ Stopped at 32% +**`users`**: ๐ŸŸง๐ŸŸง๐ŸŸง๐ŸŸง๐ŸŸง๐ŸŸงโฌœโฌœโฌœโฌœโฌœโฌœโฌœโฌœโฌœโฌœโฌœโฌœโฌœโฌœ โน๏ธ Stopped at 31.94% ```sql ALTER TABLE `users` ADD INDEX `idx_email_created`(`email`, `created_at`); @@ -4679,7 +4679,7 @@ ALTER TABLE `products` ADD INDEX `idx_price`(`price_cents`); 1 of 3 tables completed before failure. -**`users`** โ€” Failed at 30% +**`users`** โ€” Failed at 30.00% ```sql ALTER TABLE `users` DROP COLUMN `full_name`, @@ -4737,7 +4737,7 @@ schemabot apply -e staging 1 of 2 tables completed before stop. -**`users`** โ€” Stopped at 72% +**`users`** โ€” Stopped at 72.00% ```sql ALTER TABLE `users` ADD INDEX `idx_email`(`email`); ``` @@ -5135,7 +5135,7 @@ ALTER TABLE `events` ADD INDEX `idx_created_at`(`created_at`); **Schema `testapp`** -**`users`**: ๐ŸŸฆ๐ŸŸฆ๐ŸŸฆ๐ŸŸฆ๐ŸŸฆ๐ŸŸฆ๐ŸŸฆ๐ŸŸฆ๐ŸŸฆโฌœโฌœโฌœโฌœโฌœโฌœโฌœโฌœโฌœโฌœโฌœ 45% +**`users`**: ๐ŸŸฆ๐ŸŸฆ๐ŸŸฆ๐ŸŸฆ๐ŸŸฆ๐ŸŸฆ๐ŸŸฆ๐ŸŸฆ๐ŸŸฆโฌœโฌœโฌœโฌœโฌœโฌœโฌœโฌœโฌœโฌœโฌœ 45.00% ```sql ALTER TABLE `users` DROP INDEX `idx_email`; @@ -5197,7 +5197,7 @@ Single table progress (default): โ”€โ”€ testapp โ”€โ”€ - ~ users: ๐ŸŸฆ๐ŸŸฆ๐ŸŸฆ๐ŸŸฆ๐ŸŸฆ๐ŸŸฆ๐ŸŸฆ๐ŸŸฆ๐ŸŸฆโฌœโฌœโฌœโฌœโฌœโฌœโฌœโฌœโฌœโฌœโฌœ 48% + ~ users: ๐ŸŸฆ๐ŸŸฆ๐ŸŸฆ๐ŸŸฆ๐ŸŸฆ๐ŸŸฆ๐ŸŸฆ๐ŸŸฆ๐ŸŸฆโฌœโฌœโฌœโฌœโฌœโฌœโฌœโฌœโฌœโฌœโฌœ 48.61% ALTER TABLE `users` ADD INDEX `idx_email_created`(`email`, `created_at`); โ€ข Rows: 3,500,000 / 7,200,000 ยท ETA: 5m 30s @@ -5273,7 +5273,7 @@ The new apply will only process tables that haven't completed. โ”€โ”€ testapp โ”€โ”€ - ~ users: ๐ŸŸง๐ŸŸง๐ŸŸง๐ŸŸง๐ŸŸง๐ŸŸง๐ŸŸงโฌœโฌœโฌœโฌœโฌœโฌœโฌœโฌœโฌœโฌœโฌœโฌœโฌœ โน๏ธ Stopped at 39% + ~ users: ๐ŸŸง๐ŸŸง๐ŸŸง๐ŸŸง๐ŸŸง๐ŸŸง๐ŸŸงโฌœโฌœโฌœโฌœโฌœโฌœโฌœโฌœโฌœโฌœโฌœโฌœโฌœ โน๏ธ Stopped at 39.34% ALTER TABLE `users` ADD INDEX `idx_email_created`(`email`, `created_at`); โ€ข Rows: 156,342 / 397,453 @@ -5387,7 +5387,7 @@ Sequential mode: First table running, others queued โ””โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”˜ - ~ users: ๐ŸŸฆ๐ŸŸฆ๐ŸŸฆ๐ŸŸฆ๐ŸŸฆ๐ŸŸฆ๐ŸŸฆโฌœโฌœโฌœโฌœโฌœโฌœโฌœโฌœโฌœโฌœโฌœโฌœโฌœ 35% + ~ users: ๐ŸŸฆ๐ŸŸฆ๐ŸŸฆ๐ŸŸฆ๐ŸŸฆ๐ŸŸฆ๐ŸŸฆโฌœโฌœโฌœโฌœโฌœโฌœโฌœโฌœโฌœโฌœโฌœโฌœโฌœ 35.00% ALTER TABLE `users` ADD INDEX `idx_email_created`(`email`, `created_at`); โ€ข Rows: 875,000 / 2,500,000 ยท ETA: 8m 30s @@ -5416,7 +5416,7 @@ Sequential mode: First complete, second running โ””โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”˜ - ~ orders: ๐ŸŸฆ๐ŸŸฆ๐ŸŸฆ๐ŸŸฆ๐ŸŸฆ๐ŸŸฆ๐ŸŸฆ๐ŸŸฆ๐ŸŸฆ๐ŸŸฆ๐ŸŸฆ๐ŸŸฆโฌœโฌœโฌœโฌœโฌœโฌœโฌœโฌœ 60% + ~ orders: ๐ŸŸฆ๐ŸŸฆ๐ŸŸฆ๐ŸŸฆ๐ŸŸฆ๐ŸŸฆ๐ŸŸฆ๐ŸŸฆ๐ŸŸฆ๐ŸŸฆ๐ŸŸฆ๐ŸŸฆโฌœโฌœโฌœโฌœโฌœโฌœโฌœโฌœ 60.00% ALTER TABLE `orders` ADD INDEX `idx_user_status`(`user_id`, `status`); โ€ข Rows: 3,000,000 / 5,000,000 ยท ETA: 12m 15s @@ -5474,7 +5474,7 @@ Sequential mode: First complete, second paused by the engine's throttler โ””โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”˜ - ~ orders: ๐ŸŸฆ๐ŸŸฆ๐ŸŸฆ๐ŸŸฆ๐ŸŸฆ๐ŸŸฆ๐ŸŸฆ๐ŸŸฆ๐ŸŸฆ๐ŸŸฆ๐ŸŸฆ๐ŸŸฆโฌœโฌœโฌœโฌœโฌœโฌœโฌœโฌœ 62% (throttled) + ~ orders: ๐ŸŸฆ๐ŸŸฆ๐ŸŸฆ๐ŸŸฆ๐ŸŸฆ๐ŸŸฆ๐ŸŸฆ๐ŸŸฆ๐ŸŸฆ๐ŸŸฆ๐ŸŸฆ๐ŸŸฆโฌœโฌœโฌœโฌœโฌœโฌœโฌœโฌœ 62.00% (throttled) ALTER TABLE `orders` ADD INDEX `idx_user_status`(`user_id`, `status`); โ€ข Rows: 3,100,000 / 5,000,000 โ€ข โ„น๏ธ Throttled: commit-latency 112.4ms >= 100ms ยท backing off while database writes commit slowly @@ -5504,7 +5504,7 @@ Sequential mode: First complete, second checksumming โ””โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”˜ - ~ orders: ๐ŸŸฆ๐ŸŸฆ๐ŸŸฆ๐ŸŸฆ๐ŸŸฆ๐ŸŸฆ๐ŸŸฆ๐ŸŸฆ๐ŸŸฆ๐ŸŸฆ๐ŸŸฆ๐ŸŸฆโฌœโฌœโฌœโฌœโฌœโฌœโฌœโฌœ ๐Ÿ” Checksumming to verify data (60%) + ~ orders: ๐ŸŸฆ๐ŸŸฆ๐ŸŸฆ๐ŸŸฆ๐ŸŸฆ๐ŸŸฆ๐ŸŸฆ๐ŸŸฆ๐ŸŸฆ๐ŸŸฆ๐ŸŸฆ๐ŸŸฆโฌœโฌœโฌœโฌœโฌœโฌœโฌœโฌœ ๐Ÿ” Checksumming to verify data (60.00%) ALTER TABLE `orders` ADD INDEX `idx_user_status`(`user_id`, `status`); โ€ข Rows verified: 3,000,000 / 5,000,000 @@ -5562,7 +5562,7 @@ Sequential mode: First two complete, third running โ””โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”˜ - ~ products: ๐ŸŸฆ๐ŸŸฆ๐ŸŸฆ๐ŸŸฆ๐ŸŸฆ๐ŸŸฆ๐ŸŸฆ๐ŸŸฆ๐ŸŸฆ๐ŸŸฆ๐ŸŸฆ๐ŸŸฆ๐ŸŸฆ๐ŸŸฆ๐ŸŸฆ๐ŸŸฆโฌœโฌœโฌœโฌœ 80% + ~ products: ๐ŸŸฆ๐ŸŸฆ๐ŸŸฆ๐ŸŸฆ๐ŸŸฆ๐ŸŸฆ๐ŸŸฆ๐ŸŸฆ๐ŸŸฆ๐ŸŸฆ๐ŸŸฆ๐ŸŸฆ๐ŸŸฆ๐ŸŸฆ๐ŸŸฆ๐ŸŸฆโฌœโฌœโฌœโฌœ 80.00% ALTER TABLE `products` ADD COLUMN `weight_grams` int DEFAULT 0; โ€ข Rows: 160,000 / 200,000 ยท ETA: 2m 45s @@ -5686,7 +5686,7 @@ Sequential mode: User stopped mid-apply โ””โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”˜ - ~ orders: ๐ŸŸง๐ŸŸง๐ŸŸง๐ŸŸง๐ŸŸง๐ŸŸง๐ŸŸง๐ŸŸงโฌœโฌœโฌœโฌœโฌœโฌœโฌœโฌœโฌœโฌœโฌœโฌœ โน๏ธ Stopped at 42% + ~ orders: ๐ŸŸง๐ŸŸง๐ŸŸง๐ŸŸง๐ŸŸง๐ŸŸง๐ŸŸง๐ŸŸงโฌœโฌœโฌœโฌœโฌœโฌœโฌœโฌœโฌœโฌœโฌœโฌœ โน๏ธ Stopped at 42.06% ALTER TABLE `orders` ADD INDEX `idx_user_status`(`user_id`, `status`); โ€ข Rows: 112,045 / 266,383 @@ -6039,7 +6039,7 @@ Press Enter to deploy or proceed via the PlanetScale console (ESC to detach) โ”€โ”€ myapp_sharded โ”€โ”€ - ~ orders: ๐ŸŸง๐ŸŸง๐ŸŸง๐ŸŸง๐ŸŸง๐ŸŸงโฌœโฌœโฌœโฌœโฌœโฌœโฌœโฌœโฌœโฌœโฌœโฌœโฌœโฌœ ๐Ÿšซ Cancelled at 30% + ~ orders: ๐ŸŸง๐ŸŸง๐ŸŸง๐ŸŸง๐ŸŸง๐ŸŸงโฌœโฌœโฌœโฌœโฌœโฌœโฌœโฌœโฌœโฌœโฌœโฌœโฌœโฌœ ๐Ÿšซ Cancelled at 30.00% ALTER TABLE `orders` ADD INDEX `idx_total`(`total_cents`); โ€ข Shards: 2 (2 cancelled) @@ -6067,7 +6067,7 @@ Press Enter to deploy or proceed via the PlanetScale console (ESC to detach) โ”€โ”€ commerce_sharded โ”€โ”€ - ~ transactions: ๐ŸŸฆ๐ŸŸฆ๐ŸŸฆ๐ŸŸฆ๐ŸŸฆ๐ŸŸฆ๐ŸŸฆ๐ŸŸฆ๐ŸŸฆ๐ŸŸฆ๐ŸŸฆโฌœโฌœโฌœโฌœโฌœโฌœโฌœโฌœโฌœ 57% + ~ transactions: ๐ŸŸฆ๐ŸŸฆ๐ŸŸฆ๐ŸŸฆ๐ŸŸฆ๐ŸŸฆ๐ŸŸฆ๐ŸŸฆ๐ŸŸฆ๐ŸŸฆ๐ŸŸฆโฌœโฌœโฌœโฌœโฌœโฌœโฌœโฌœโฌœ 57.21% ALTER TABLE `transactions` ADD COLUMN `region_id` int; โ€ข Rows: 110,573,340 / 193,280,000 ยท ETA: 4m 40s โ€ข Shards: 256 (30 waiting for cutover, 226 copying) @@ -6075,11 +6075,11 @@ Press Enter to deploy or proceed via the PlanetScale console (ESC to detach) โ— 01-02: waiting for cutover โ— 02-03: waiting for cutover ... 27 more waiting for cutover - โ—‰ ff-: 10% (101,000/1,010,000 rows) ETA 4m 40s - โ—‰ fe-ff: 10% (100,800/1,008,000 rows) ETA 4m 40s - โ—‰ fd-fe: 11% (110,660/1,006,000 rows) ETA 4m 38s - โ—‰ fc-fd: 11% (110,440/1,004,000 rows) ETA 4m 38s - โ—‰ fb-fc: 12% (120,240/1,002,000 rows) ETA 4m 36s + โ—‰ ff-: 10.00% (101,000/1,010,000 rows) ETA 4m 40s + โ—‰ fe-ff: 10.00% (100,800/1,008,000 rows) ETA 4m 40s + โ—‰ fd-fe: 11.00% (110,660/1,006,000 rows) ETA 4m 38s + โ—‰ fc-fd: 11.00% (110,440/1,004,000 rows) ETA 4m 38s + โ—‰ fb-fc: 12.00% (120,240/1,002,000 rows) ETA 4m 36s ... 221 more copying shards ``` @@ -6103,7 +6103,7 @@ Press Enter to deploy or proceed via the PlanetScale console (ESC to detach) โ”€โ”€ commerce_sharded โ”€โ”€ - ~ transactions: ๐ŸŸฆ๐ŸŸฆ๐ŸŸฆ๐ŸŸฆ๐ŸŸฆ๐ŸŸฆ๐ŸŸฆ๐ŸŸฆ๐ŸŸฆ๐ŸŸฆ๐ŸŸฆ๐ŸŸฆ๐ŸŸฆโฌœโฌœโฌœโฌœโฌœโฌœโฌœ 67% + ~ transactions: ๐ŸŸฆ๐ŸŸฆ๐ŸŸฆ๐ŸŸฆ๐ŸŸฆ๐ŸŸฆ๐ŸŸฆ๐ŸŸฆ๐ŸŸฆ๐ŸŸฆ๐ŸŸฆ๐ŸŸฆ๐ŸŸฆโฌœโฌœโฌœโฌœโฌœโฌœโฌœ 67.89% ALTER TABLE `transactions` ADD COLUMN `region_id` int; โ€ข Rows: 29,435,000 / 43,360,000 ยท ETA: 2m 37s โ€ข Shards: 32 (12 waiting for cutover, 20 copying) @@ -6111,11 +6111,11 @@ Press Enter to deploy or proceed via the PlanetScale console (ESC to detach) โ— 08-10: waiting for cutover โ— 10-18: waiting for cutover ... 9 more waiting for cutover - โ—‰ f8-: 23% (347,300/1,510,000 rows) ETA 2m 37s - โ—‰ f0-f8: 26% (390,000/1,500,000 rows) ETA 2m 34s - โ—‰ e8-f0: 29% (432,100/1,490,000 rows) ETA 2m 31s - โ—‰ e0-e8: 32% (473,600/1,480,000 rows) ETA 2m 28s - โ—‰ d8-e0: 35% (514,500/1,470,000 rows) ETA 2m 25s + โ—‰ f8-: 23.00% (347,300/1,510,000 rows) ETA 2m 37s + โ—‰ f0-f8: 26.00% (390,000/1,500,000 rows) ETA 2m 34s + โ—‰ e8-f0: 29.00% (432,100/1,490,000 rows) ETA 2m 31s + โ—‰ e0-e8: 32.00% (473,600/1,480,000 rows) ETA 2m 28s + โ—‰ d8-e0: 35.00% (514,500/1,470,000 rows) ETA 2m 25s ... 15 more copying shards โ”€โ”€ commerce_001 โ”€โ”€ @@ -6281,12 +6281,12 @@ Press Enter to deploy or proceed via the PlanetScale console (ESC to detach) โ”€โ”€ myapp_sharded โ”€โ”€ - ~ orders: ๐ŸŸฆ๐ŸŸฆ๐ŸŸฆ๐ŸŸฆ๐ŸŸฆ๐ŸŸฆ๐ŸŸฆ๐ŸŸฆ๐ŸŸฆ๐ŸŸฆ๐ŸŸฆ๐ŸŸฆ๐ŸŸฆ๐ŸŸฆโฌœโฌœโฌœโฌœโฌœโฌœ 70% + ~ orders: ๐ŸŸฆ๐ŸŸฆ๐ŸŸฆ๐ŸŸฆ๐ŸŸฆ๐ŸŸฆ๐ŸŸฆ๐ŸŸฆ๐ŸŸฆ๐ŸŸฆ๐ŸŸฆ๐ŸŸฆ๐ŸŸฆ๐ŸŸฆโฌœโฌœโฌœโฌœโฌœโฌœ 70.00% ALTER TABLE `orders` ADD INDEX `idx_total`(`total_cents`); โ€ข Rows: 2,800,000 / 4,000,000 ยท ETA: 2m 0s โ€ข Shards: 2 (2 copying) - โ—‰ -80: 95% (2,000,000/2,100,000 rows) ETA 10s - โ—‰ 80-: 42% (800,000/1,900,000 rows) ETA 2m 0s + โ—‰ -80: 95.24% (2,000,000/2,100,000 rows) ETA 10s + โ—‰ 80-: 42.11% (800,000/1,900,000 rows) ETA 2m 0s ``` @@ -6538,7 +6538,7 @@ Apply watch mode: Running with footer controls โ”€โ”€ testapp โ”€โ”€ - ~ users: ๐ŸŸฆ๐ŸŸฆ๐ŸŸฆ๐ŸŸฆ๐ŸŸฆ๐ŸŸฆ๐ŸŸฆ๐ŸŸฆ๐ŸŸฆ๐ŸŸฆ๐ŸŸฆ๐ŸŸฆโฌœโฌœโฌœโฌœโฌœโฌœโฌœโฌœ 62% + ~ users: ๐ŸŸฆ๐ŸŸฆ๐ŸŸฆ๐ŸŸฆ๐ŸŸฆ๐ŸŸฆ๐ŸŸฆ๐ŸŸฆ๐ŸŸฆ๐ŸŸฆ๐ŸŸฆ๐ŸŸฆโฌœโฌœโฌœโฌœโฌœโฌœโฌœโฌœ 62.38% ALTER TABLE `users` ADD INDEX `idx_email_created`(`email`, `created_at`); โ€ข Rows: 914,707 / 1,466,232 ยท ETA: 3m 15s @@ -6616,7 +6616,7 @@ Checkpoint saved. Use 'schemabot start -e staging apply-a1b2c3d4e5f67890' to res โ”€โ”€ testapp โ”€โ”€ - ~ users: ๐ŸŸง๐ŸŸง๐ŸŸง๐ŸŸง๐ŸŸง๐ŸŸง๐ŸŸงโฌœโฌœโฌœโฌœโฌœโฌœโฌœโฌœโฌœโฌœโฌœโฌœโฌœ โน๏ธ Stopped at 39% + ~ users: ๐ŸŸง๐ŸŸง๐ŸŸง๐ŸŸง๐ŸŸง๐ŸŸง๐ŸŸงโฌœโฌœโฌœโฌœโฌœโฌœโฌœโฌœโฌœโฌœโฌœโฌœโฌœ โน๏ธ Stopped at 39.34% ALTER TABLE `users` ADD INDEX `idx_email_created`(`email`, `created_at`); โ€ข Rows: 156,342 / 397,453 @@ -6664,7 +6664,7 @@ Resuming from checkpoint... โ”€โ”€ testapp โ”€โ”€ - ~ users: ๐ŸŸฆ๐ŸŸฆ๐ŸŸฆ๐ŸŸฆ๐ŸŸฆ๐ŸŸฆ๐ŸŸฆ๐ŸŸฆโฌœโฌœโฌœโฌœโฌœโฌœโฌœโฌœโฌœโฌœโฌœโฌœ 40% + ~ users: ๐ŸŸฆ๐ŸŸฆ๐ŸŸฆ๐ŸŸฆ๐ŸŸฆ๐ŸŸฆ๐ŸŸฆ๐ŸŸฆโฌœโฌœโฌœโฌœโฌœโฌœโฌœโฌœโฌœโฌœโฌœโฌœ 39.75% ALTER TABLE `users` ADD INDEX `idx_email_created`(`email`, `created_at`); โ€ข Rows: 158,000 / 397,453 ยท ETA: 8m 0s @@ -6832,11 +6832,11 @@ SchemaBot triggers cutover automatically โ€” no action needed. **Status**: In Progress -๐Ÿ“Š 1/3 complete ยท 1 running (62%) ยท 1 queued +๐Ÿ“Š 1/3 complete ยท 1 running (62.38%) ยท 1 queued **Schema `testapp`** -**`users`**: ๐ŸŸฆ๐ŸŸฆ๐ŸŸฆ๐ŸŸฆ๐ŸŸฆ๐ŸŸฆ๐ŸŸฆ๐ŸŸฆ๐ŸŸฆ๐ŸŸฆ๐ŸŸฆ๐ŸŸฆโฌœโฌœโฌœโฌœโฌœโฌœโฌœโฌœ 62% +**`users`**: ๐ŸŸฆ๐ŸŸฆ๐ŸŸฆ๐ŸŸฆ๐ŸŸฆ๐ŸŸฆ๐ŸŸฆ๐ŸŸฆ๐ŸŸฆ๐ŸŸฆ๐ŸŸฆ๐ŸŸฆโฌœโฌœโฌœโฌœโฌœโฌœโฌœโฌœ 62.38% ```sql ALTER TABLE `users` ADD INDEX `idx_email`(`email`); @@ -7783,7 +7783,7 @@ This schema change was cancelled and cannot be resumed. Open a new schema change ๐Ÿ”„ eu-west โ€” running table copy (orders-eu-west) - ~ orders: ๐ŸŸฆ๐ŸŸฆ๐ŸŸฆ๐ŸŸฆ๐ŸŸฆ๐ŸŸฆ๐ŸŸฆโฌœโฌœโฌœโฌœโฌœโฌœโฌœโฌœโฌœโฌœโฌœโฌœโฌœ 35% + ~ orders: ๐ŸŸฆ๐ŸŸฆ๐ŸŸฆ๐ŸŸฆ๐ŸŸฆ๐ŸŸฆ๐ŸŸฆโฌœโฌœโฌœโฌœโฌœโฌœโฌœโฌœโฌœโฌœโฌœโฌœโฌœ 35.00% ALTER TABLE `orders` ADD COLUMN `source` varchar(32); โ€ข Rows: 42,000 / 120,000 ยท ETA: 4m 0s @@ -7914,7 +7914,7 @@ Sequential mode: First table running, others queued โ””โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”˜ - ~ users: ๐ŸŸฆ๐ŸŸฆ๐ŸŸฆ๐ŸŸฆ๐ŸŸฆ๐ŸŸฆ๐ŸŸฆโฌœโฌœโฌœโฌœโฌœโฌœโฌœโฌœโฌœโฌœโฌœโฌœโฌœ 35% + ~ users: ๐ŸŸฆ๐ŸŸฆ๐ŸŸฆ๐ŸŸฆ๐ŸŸฆ๐ŸŸฆ๐ŸŸฆโฌœโฌœโฌœโฌœโฌœโฌœโฌœโฌœโฌœโฌœโฌœโฌœโฌœ 35.00% ALTER TABLE `users` ADD INDEX `idx_email_created`(`email`, `created_at`); โ€ข Rows: 875,000 / 2,500,000 ยท ETA: 8m 30s @@ -7943,7 +7943,7 @@ Sequential mode: First complete, second running โ””โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”˜ - ~ orders: ๐ŸŸฆ๐ŸŸฆ๐ŸŸฆ๐ŸŸฆ๐ŸŸฆ๐ŸŸฆ๐ŸŸฆ๐ŸŸฆ๐ŸŸฆ๐ŸŸฆ๐ŸŸฆ๐ŸŸฆโฌœโฌœโฌœโฌœโฌœโฌœโฌœโฌœ 60% + ~ orders: ๐ŸŸฆ๐ŸŸฆ๐ŸŸฆ๐ŸŸฆ๐ŸŸฆ๐ŸŸฆ๐ŸŸฆ๐ŸŸฆ๐ŸŸฆ๐ŸŸฆ๐ŸŸฆ๐ŸŸฆโฌœโฌœโฌœโฌœโฌœโฌœโฌœโฌœ 60.00% ALTER TABLE `orders` ADD INDEX `idx_user_status`(`user_id`, `status`); โ€ข Rows: 3,000,000 / 5,000,000 ยท ETA: 12m 15s @@ -7972,7 +7972,7 @@ Sequential mode: First two complete, third running โ””โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”˜ - ~ products: ๐ŸŸฆ๐ŸŸฆ๐ŸŸฆ๐ŸŸฆ๐ŸŸฆ๐ŸŸฆ๐ŸŸฆ๐ŸŸฆ๐ŸŸฆ๐ŸŸฆ๐ŸŸฆ๐ŸŸฆ๐ŸŸฆ๐ŸŸฆ๐ŸŸฆ๐ŸŸฆโฌœโฌœโฌœโฌœ 80% + ~ products: ๐ŸŸฆ๐ŸŸฆ๐ŸŸฆ๐ŸŸฆ๐ŸŸฆ๐ŸŸฆ๐ŸŸฆ๐ŸŸฆ๐ŸŸฆ๐ŸŸฆ๐ŸŸฆ๐ŸŸฆ๐ŸŸฆ๐ŸŸฆ๐ŸŸฆ๐ŸŸฆโฌœโฌœโฌœโฌœ 80.00% ALTER TABLE `products` ADD COLUMN `weight_grams` int DEFAULT 0; โ€ข Rows: 160,000 / 200,000 ยท ETA: 2m 45s @@ -8096,7 +8096,7 @@ Sequential mode: User stopped mid-apply โ””โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”˜ - ~ orders: ๐ŸŸง๐ŸŸง๐ŸŸง๐ŸŸง๐ŸŸง๐ŸŸง๐ŸŸง๐ŸŸงโฌœโฌœโฌœโฌœโฌœโฌœโฌœโฌœโฌœโฌœโฌœโฌœ โน๏ธ Stopped at 42% + ~ orders: ๐ŸŸง๐ŸŸง๐ŸŸง๐ŸŸง๐ŸŸง๐ŸŸง๐ŸŸง๐ŸŸงโฌœโฌœโฌœโฌœโฌœโฌœโฌœโฌœโฌœโฌœโฌœโฌœ โน๏ธ Stopped at 42.06% ALTER TABLE `orders` ADD INDEX `idx_user_status`(`user_id`, `status`); โ€ข Rows: 112,045 / 266,383 @@ -8128,15 +8128,15 @@ Atomic mode (--defer-cutover): All tables copy rows, then cutover together โ””โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”˜ - ~ orders: ๐ŸŸฆ๐ŸŸฆ๐ŸŸฆ๐ŸŸฆ๐ŸŸฆ๐ŸŸฆ๐ŸŸฆ๐ŸŸฆ๐ŸŸฆ๐ŸŸฆ๐ŸŸฆ๐ŸŸฆ๐ŸŸฆ๐ŸŸฆโฌœโฌœโฌœโฌœโฌœโฌœ 72% + ~ orders: ๐ŸŸฆ๐ŸŸฆ๐ŸŸฆ๐ŸŸฆ๐ŸŸฆ๐ŸŸฆ๐ŸŸฆ๐ŸŸฆ๐ŸŸฆ๐ŸŸฆ๐ŸŸฆ๐ŸŸฆ๐ŸŸฆ๐ŸŸฆโฌœโฌœโฌœโฌœโฌœโฌœ 72.00% ALTER TABLE `orders` ADD INDEX `idx_user_status`(`user_id`, `status`); โ€ข Rows: 1,800,000 / 2,500,000 ยท ETA: 3m 15s - ~ products: ๐ŸŸฆ๐ŸŸฆ๐ŸŸฆ๐ŸŸฆ๐ŸŸฆ๐ŸŸฆ๐ŸŸฆ๐ŸŸฆ๐ŸŸฆโฌœโฌœโฌœโฌœโฌœโฌœโฌœโฌœโฌœโฌœโฌœ 45% + ~ products: ๐ŸŸฆ๐ŸŸฆ๐ŸŸฆ๐ŸŸฆ๐ŸŸฆ๐ŸŸฆ๐ŸŸฆ๐ŸŸฆ๐ŸŸฆโฌœโฌœโฌœโฌœโฌœโฌœโฌœโฌœโฌœโฌœโฌœ 45.00% ALTER TABLE `products` ADD INDEX `idx_category`(`category`); โ€ข Rows: 450,000 / 1,000,000 ยท ETA: 6m 20s - ~ users: ๐ŸŸฆ๐ŸŸฆ๐ŸŸฆ๐ŸŸฆ๐ŸŸฆ๐ŸŸฆ๐ŸŸฆ๐ŸŸฆ๐ŸŸฆ๐ŸŸฆ๐ŸŸฆ๐ŸŸฆ๐ŸŸฆ๐ŸŸฆ๐ŸŸฆ๐ŸŸฆ๐ŸŸฆโฌœโฌœโฌœ 89% + ~ users: ๐ŸŸฆ๐ŸŸฆ๐ŸŸฆ๐ŸŸฆ๐ŸŸฆ๐ŸŸฆ๐ŸŸฆ๐ŸŸฆ๐ŸŸฆ๐ŸŸฆ๐ŸŸฆ๐ŸŸฆ๐ŸŸฆ๐ŸŸฆ๐ŸŸฆ๐ŸŸฆ๐ŸŸฆโฌœโฌœโฌœ 88.89% ALTER TABLE `users` ADD INDEX `idx_email_created`(`email`, `created_at`); โ€ข Rows: 6,400,000 / 7,200,000 ยท ETA: 1m 45s @@ -8252,15 +8252,15 @@ Defer cutover: Stopped by user (s) โ””โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”˜ - ~ orders: ๐ŸŸง๐ŸŸง๐ŸŸง๐ŸŸง๐ŸŸง๐ŸŸง๐ŸŸง๐ŸŸง๐ŸŸง๐ŸŸง๐ŸŸง๐ŸŸง๐ŸŸง๐ŸŸงโฌœโฌœโฌœโฌœโฌœโฌœ โน๏ธ Stopped at 72% + ~ orders: ๐ŸŸง๐ŸŸง๐ŸŸง๐ŸŸง๐ŸŸง๐ŸŸง๐ŸŸง๐ŸŸง๐ŸŸง๐ŸŸง๐ŸŸง๐ŸŸง๐ŸŸง๐ŸŸงโฌœโฌœโฌœโฌœโฌœโฌœ โน๏ธ Stopped at 72.00% ALTER TABLE `orders` ADD INDEX `idx_user_status`(`user_id`, `status`); โ€ข Rows: 1,800,000 / 2,500,000 - ~ products: ๐ŸŸง๐ŸŸง๐ŸŸง๐ŸŸง๐ŸŸง๐ŸŸง๐ŸŸง๐ŸŸง๐ŸŸงโฌœโฌœโฌœโฌœโฌœโฌœโฌœโฌœโฌœโฌœโฌœ โน๏ธ Stopped at 45% + ~ products: ๐ŸŸง๐ŸŸง๐ŸŸง๐ŸŸง๐ŸŸง๐ŸŸง๐ŸŸง๐ŸŸง๐ŸŸงโฌœโฌœโฌœโฌœโฌœโฌœโฌœโฌœโฌœโฌœโฌœ โน๏ธ Stopped at 45.00% ALTER TABLE `products` ADD INDEX `idx_category`(`category`); โ€ข Rows: 450,000 / 1,000,000 - ~ users: ๐ŸŸง๐ŸŸง๐ŸŸง๐ŸŸง๐ŸŸง๐ŸŸง๐ŸŸง๐ŸŸง๐ŸŸง๐ŸŸง๐ŸŸง๐ŸŸง๐ŸŸง๐ŸŸง๐ŸŸง๐ŸŸง๐ŸŸงโฌœโฌœโฌœ โน๏ธ Stopped at 89% + ~ users: ๐ŸŸง๐ŸŸง๐ŸŸง๐ŸŸง๐ŸŸง๐ŸŸง๐ŸŸง๐ŸŸง๐ŸŸง๐ŸŸง๐ŸŸง๐ŸŸง๐ŸŸง๐ŸŸง๐ŸŸง๐ŸŸง๐ŸŸงโฌœโฌœโฌœ โน๏ธ Stopped at 88.89% ALTER TABLE `users` ADD INDEX `idx_email_created`(`email`, `created_at`); โ€ข Rows: 6,400,000 / 7,200,000 @@ -8415,7 +8415,7 @@ Environment: production External operation ID: remote-op-eu-west-001 External apply ID: remote-apply-eu-west-001 - ~ orders: ๐ŸŸฆ๐ŸŸฆ๐ŸŸฆ๐ŸŸฆ๐ŸŸฆ๐ŸŸฆ๐ŸŸฆโฌœโฌœโฌœโฌœโฌœโฌœโฌœโฌœโฌœโฌœโฌœโฌœโฌœ 35% + ~ orders: ๐ŸŸฆ๐ŸŸฆ๐ŸŸฆ๐ŸŸฆ๐ŸŸฆ๐ŸŸฆ๐ŸŸฆโฌœโฌœโฌœโฌœโฌœโฌœโฌœโฌœโฌœโฌœโฌœโฌœโฌœ 35.42% ALTER TABLE `orders` ADD COLUMN `source` varchar(32); โ€ข Rows: 42,500 / 120,000 ยท ETA: 4m 0s diff --git a/pkg/cmd/commands/apply.go b/pkg/cmd/commands/apply.go index f67c9cda5..9ea078b14 100644 --- a/pkg/cmd/commands/apply.go +++ b/pkg/cmd/commands/apply.go @@ -827,7 +827,7 @@ func (e *logEmitter) emitTableStateChange(tbl *apitypes.TableProgressResponse, t kvs := tableKVs(recoveringLogMessage(tbl), tbl, ts) if tbl.RowsTotal > 0 && tbl.PercentComplete < 100 { kvs = append(kvs, - "progress", fmt.Sprintf("%d%%", min(int(tbl.PercentComplete), 100)), + "progress", ui.FormatRowCopyPercent(int(tbl.PercentComplete), tbl.RowsCopied, tbl.RowsTotal), "rows", fmt.Sprintf("%s/%s", ui.FormatNumber(ui.ClampRows(tbl.RowsCopied, tbl.RowsTotal)), ui.FormatNumber(tbl.RowsTotal)), ) if tbl.ETASeconds > 0 { @@ -842,8 +842,8 @@ func (e *logEmitter) emitTableStateChange(tbl *apitypes.TableProgressResponse, t e.emit(kvs...) case state.Apply.Stopped: kvs := tableKVs("Table stopped", tbl, ts) - if tbl.PercentComplete > 0 { - kvs = append(kvs, "progress", fmt.Sprintf("%d%%", min(int(tbl.PercentComplete), 100))) + if tbl.PercentComplete > 0 || tbl.RowsCopied > 0 { + kvs = append(kvs, "progress", ui.FormatRowCopyPercent(int(tbl.PercentComplete), tbl.RowsCopied, tbl.RowsTotal)) } e.emit(kvs...) default: @@ -874,9 +874,8 @@ func (e *logEmitter) emitProgressHeartbeat(tbl *apitypes.TableProgressResponse, "rows_copied", fmt.Sprintf("%s so far", ui.FormatNumber(tbl.RowsCopied)), ) } else { - pct := ui.ClampPercent(int(tbl.PercentComplete)) kvs = append(kvs, - "progress", fmt.Sprintf("%d%%", pct), + "progress", ui.FormatRowCopyPercent(int(tbl.PercentComplete), tbl.RowsCopied, tbl.RowsTotal), "rows", fmt.Sprintf("%s/%s", ui.FormatNumber(ui.ClampRows(tbl.RowsCopied, tbl.RowsTotal)), ui.FormatNumber(tbl.RowsTotal)), ) if tbl.ETASeconds > 0 { diff --git a/pkg/cmd/commands/apply_log_test.go b/pkg/cmd/commands/apply_log_test.go index 0077d631a..c6e7269ec 100644 --- a/pkg/cmd/commands/apply_log_test.go +++ b/pkg/cmd/commands/apply_log_test.go @@ -161,7 +161,7 @@ func TestLogEmitter_EmitTableStateChange(t *testing.T) { status: state.Apply.Recovering, pct: 42, wantMsg: "Row copy in progress during restart recovery", - wantFields: []string{"table=users", "progress=42%", "rows=420/1,000", "eta=\"2m 0s\""}, + wantFields: []string{"table=users", "progress=42.00%", "rows=420/1,000", "eta=\"2m 0s\""}, }, { name: "cutting over", @@ -234,7 +234,7 @@ func TestLogEmitter_EmitProgressHeartbeat(t *testing.T) { assert.Contains(t, plain, "Copying rows") assert.Contains(t, plain, "table=orders") - assert.Contains(t, plain, "progress=45%") + assert.Contains(t, plain, "progress=45.00%") assert.Contains(t, plain, "rows=99,450/221,000") assert.Contains(t, plain, "5m 30s") }) @@ -257,12 +257,12 @@ func TestLogEmitter_EmitProgressHeartbeat(t *testing.T) { plain := stripANSI(output) assert.Contains(t, plain, "table=products") - assert.Contains(t, plain, "progress=20%") + assert.Contains(t, plain, "progress=20.00%") assert.Contains(t, plain, "rows=10,000/50,000") assert.Contains(t, plain, "eta=") }) - t.Run("clamps percent to 100", func(t *testing.T) { + t.Run("rows override an overshooting whole-number percent", func(t *testing.T) { e := &logEmitter{} ts := &tableLogState{} tbl := &apitypes.TableProgressResponse{ @@ -277,7 +277,7 @@ func TestLogEmitter_EmitProgressHeartbeat(t *testing.T) { }) plain := stripANSI(output) - assert.Contains(t, plain, "progress=100%") + assert.Contains(t, plain, "progress=96.09%") }) t.Run("estimate exceeded shows finalizing copy progress", func(t *testing.T) { diff --git a/pkg/cmd/commands/watch_tui_test.go b/pkg/cmd/commands/watch_tui_test.go index a734d8417..de6a96282 100644 --- a/pkg/cmd/commands/watch_tui_test.go +++ b/pkg/cmd/commands/watch_tui_test.go @@ -248,7 +248,7 @@ func TestWatchModel_ChecksummingShowsVerifyProgress(t *testing.T) { model := updated.(WatchModel) view := model.View() - assert.Contains(t, view, "Checksumming to verify data (9%)") + assert.Contains(t, view, "Checksumming to verify data (8.78%)") assert.Contains(t, view, "Rows verified: 73,075,845 / 832,771,089") assert.Contains(t, view, "(throttled)") } @@ -270,9 +270,9 @@ func TestWatchModel_RecoveringShowsCopyingRows(t *testing.T) { model := updated.(WatchModel) view := model.View() - assert.Contains(t, view, "Row copy in progress (42%)") + assert.Contains(t, view, "Row copy in progress (42.00%)") assert.Contains(t, view, "Rows: 420 / 1,000") - assert.Contains(t, view, "Row copy is in progress (42%)") + assert.Contains(t, view, "Row copy is in progress (42.00%)") assert.Contains(t, view, "progress returns to the normal row-copy view") assert.Contains(t, view, "SchemaBot is recovering after restart") assert.NotContains(t, view, "Cutover will be available once recovery completes") diff --git a/pkg/cmd/commands/watch_tui_view.go b/pkg/cmd/commands/watch_tui_view.go index 27c832c7b..5b9e53675 100644 --- a/pkg/cmd/commands/watch_tui_view.go +++ b/pkg/cmd/commands/watch_tui_view.go @@ -240,7 +240,7 @@ func (m WatchModel) progressView() string { case state.IsState(m.state, state.Apply.Recovering): b.WriteString("\n\n") if pct, ok := recoveringCopyPercent(m.tables); ok { - fmt.Fprintf(&b, "SchemaBot is recovering after restart.\nRow copy is in progress (%d%%); once recovery completes, progress returns to the normal row-copy view. (ESC to detach)\n", pct) + fmt.Fprintf(&b, "SchemaBot is recovering after restart.\nRow copy is in progress (%s); once recovery completes, progress returns to the normal row-copy view. (ESC to detach)\n", pct) } else { b.WriteString("SchemaBot is recovering after restart.\n") b.WriteString("Cutover will be available once recovery completes. (ESC to detach)\n") @@ -277,17 +277,24 @@ func (m WatchModel) fetchErrorLine() string { return errStyle.Render(label+": "+m.errorMsg) + "\n" } -func recoveringCopyPercent(tables []templates.TableProgress) (int, bool) { - percent := 100 +// recoveringCopyPercent returns the least-progressed recovering table's copy +// percent as display text, so the recovery footer never overstates how far +// the slowest table has come. +func recoveringCopyPercent(tables []templates.TableProgress) (string, bool) { + fraction := 0.0 + text := "" found := false for _, table := range tables { if state.NormalizeTaskStatus(table.Status) != state.Task.Recovering || table.RowsTotal <= 0 || table.PercentComplete >= 100 { continue } - percent = min(percent, ui.ClampPercent(table.PercentComplete)) + if frac := ui.RowCopyFraction(table.PercentComplete, table.RowsCopied, table.RowsTotal); !found || frac < fraction { + fraction = frac + text = ui.FormatRowCopyPercent(table.PercentComplete, table.RowsCopied, table.RowsTotal) + } found = true } - return percent, found + return text, found } // renderTables renders tables with the shared FormatNamespacedTables / diff --git a/pkg/cmd/internal/templates/progress.go b/pkg/cmd/internal/templates/progress.go index e9740da99..88024438d 100644 --- a/pkg/cmd/internal/templates/progress.go +++ b/pkg/cmd/internal/templates/progress.go @@ -492,8 +492,10 @@ func FormatTableProgressWithActivity(t TableProgress, activityBar, activityLabel // source. On a large table this can run for hours, so show how far the // verify has progressed once Spirit has reported a total. if t.ChecksumRowsTotal > 0 { - pct := ui.RowCopyDisplayPercent(int(math.Round(float64(t.ChecksumRowsChecked)*100/float64(t.ChecksumRowsTotal))), t.ChecksumRowsChecked) - fmt.Fprintf(&b, indentTable+progressSymbol(t.ChangeType)+"%s: %s ๐Ÿ” Checksumming to verify data (%d%%)%s\n", t.TableName, ui.ProgressBarRowCopy(pct), pct, throttledSuffix(t)) + checksumPct := int(math.Round(float64(t.ChecksumRowsChecked) * 100 / float64(t.ChecksumRowsTotal))) + pct := ui.RowCopyDisplayPercent(checksumPct, t.ChecksumRowsChecked) + fmt.Fprintf(&b, indentTable+progressSymbol(t.ChangeType)+"%s: %s ๐Ÿ” Checksumming to verify data (%s)%s\n", t.TableName, ui.ProgressBarRowCopy(pct), + ui.FormatRowCopyPercent(checksumPct, t.ChecksumRowsChecked, t.ChecksumRowsTotal), throttledSuffix(t)) if t.DDL != "" { b.WriteString(formatProgressDDLForDialect(t.Dialect, t.DDL)) } @@ -536,7 +538,8 @@ func FormatTableProgressWithActivity(t TableProgress, activityBar, activityLabel if recoveringIsCopyingRows(t) { pct := ui.RowCopyDisplayPercent(t.PercentComplete, t.RowsCopied) bar := ui.ProgressBarRowCopy(pct) - fmt.Fprintf(&b, indentTable+progressSymbol(t.ChangeType)+"%s: %s Row copy in progress (%d%%)\n", t.TableName, bar, pct) + fmt.Fprintf(&b, indentTable+progressSymbol(t.ChangeType)+"%s: %s Row copy in progress (%s)\n", t.TableName, bar, + ui.FormatRowCopyPercent(t.PercentComplete, t.RowsCopied, t.RowsTotal)) if t.DDL != "" { b.WriteString(formatProgressDDLForDialect(t.Dialect, t.DDL)) } @@ -633,7 +636,8 @@ func FormatTableProgressWithActivity(t TableProgress, activityBar, activityLabel if t.PercentComplete > 0 || t.RowsCopied > 0 { cancelledPercent := ui.RowCopyDisplayPercent(t.PercentComplete, t.RowsCopied) bar := ui.ProgressBar(cancelledPercent, ui.ColorOrange) - fmt.Fprintf(&b, indentTable+progressSymbol(t.ChangeType)+"%s: %s ๐Ÿšซ Cancelled at %d%%\n", t.TableName, bar, cancelledPercent) + fmt.Fprintf(&b, indentTable+progressSymbol(t.ChangeType)+"%s: %s ๐Ÿšซ Cancelled at %s\n", t.TableName, bar, + ui.FormatRowCopyPercent(t.PercentComplete, t.RowsCopied, t.RowsTotal)) } else { fmt.Fprintf(&b, indentTable+progressSymbol(t.ChangeType)+"%s: ๐Ÿšซ Cancelled (not started)\n", t.TableName) } @@ -652,7 +656,8 @@ func FormatTableProgressWithActivity(t TableProgress, activityBar, activityLabel // At 100% = was waiting for cutover when stopped fmt.Fprintf(&b, indentTable+progressSymbol(t.ChangeType)+"%s: %s โน๏ธ Stopped (was waiting for cutover)\n", t.TableName, bar) case t.PercentComplete > 0 || t.RowsCopied > 0: - fmt.Fprintf(&b, indentTable+progressSymbol(t.ChangeType)+"%s: %s โน๏ธ Stopped at %d%%\n", t.TableName, bar, stoppedPercent) + fmt.Fprintf(&b, indentTable+progressSymbol(t.ChangeType)+"%s: %s โน๏ธ Stopped at %s\n", t.TableName, bar, + ui.FormatRowCopyPercent(t.PercentComplete, t.RowsCopied, t.RowsTotal)) default: fmt.Fprintf(&b, indentTable+progressSymbol(t.ChangeType)+"%s: โน๏ธ Stopped (not started)\n", t.TableName) } @@ -679,7 +684,8 @@ func FormatTableProgressWithActivity(t TableProgress, activityBar, activityLabel // Parsed successfully - show emoji progress bar with structured data displayPercent := ui.RowCopyDisplayPercent(info.Percent, info.RowsCopied) bar := ui.ProgressBarRowCopy(displayPercent) - fmt.Fprintf(&b, indentTable+progressSymbol(t.ChangeType)+"%s: %s %d%%%s\n", t.TableName, bar, displayPercent, throttledSuffix(t)) + fmt.Fprintf(&b, indentTable+progressSymbol(t.ChangeType)+"%s: %s %s%s\n", t.TableName, bar, + ui.FormatRowCopyPercent(info.Percent, info.RowsCopied, info.RowsTotal), throttledSuffix(t)) if t.DDL != "" { b.WriteString(formatProgressDDLForDialect(t.Dialect, t.DDL)) } @@ -716,7 +722,8 @@ func FormatTableProgressWithActivity(t TableProgress, activityBar, activityLabel // Row copy in progress โ€” show progress bar with structured fields displayPercent := ui.RowCopyDisplayPercent(t.PercentComplete, t.RowsCopied) bar := ui.ProgressBarRowCopy(displayPercent) - fmt.Fprintf(&b, indentTable+progressSymbol(t.ChangeType)+"%s: %s %d%%%s\n", t.TableName, bar, displayPercent, throttledSuffix(t)) + fmt.Fprintf(&b, indentTable+progressSymbol(t.ChangeType)+"%s: %s %s%s\n", t.TableName, bar, + ui.FormatRowCopyPercent(t.PercentComplete, t.RowsCopied, t.RowsTotal), throttledSuffix(t)) if t.DDL != "" { b.WriteString(formatProgressDDLForDialect(t.Dialect, t.DDL)) diff --git a/pkg/cmd/internal/templates/progress_shard.go b/pkg/cmd/internal/templates/progress_shard.go index 96a152892..c13bd752a 100644 --- a/pkg/cmd/internal/templates/progress_shard.go +++ b/pkg/cmd/internal/templates/progress_shard.go @@ -121,12 +121,8 @@ func formatShardLine(s ShardProgress) string { case state.Task.Completed: return fmt.Sprintf(indentShardLine+"%sโœ“ %s%s: %s rows\n", ANSIGreen, s.Shard, ANSIReset, ui.FormatNumber(s.RowsTotal)) case state.Task.Running: - pct := s.PercentComplete - if pct == 0 && s.RowsTotal > 0 { - pct = int(s.RowsCopied * 100 / s.RowsTotal) - } - pct = ui.RowCopyDisplayPercent(pct, s.RowsCopied) - detail := fmt.Sprintf("%d%% (%s/%s rows)", pct, ui.FormatNumber(ui.ClampRows(s.RowsCopied, s.RowsTotal)), ui.FormatNumber(s.RowsTotal)) + detail := fmt.Sprintf("%s (%s/%s rows)", ui.FormatRowCopyPercent(s.PercentComplete, s.RowsCopied, s.RowsTotal), + ui.FormatNumber(ui.ClampRows(s.RowsCopied, s.RowsTotal)), ui.FormatNumber(s.RowsTotal)) if s.ETASeconds > 0 { detail += fmt.Sprintf(" ETA %s", FormatDurationSeconds(s.ETASeconds)) } diff --git a/pkg/cmd/internal/templates/progress_shard_test.go b/pkg/cmd/internal/templates/progress_shard_test.go index fe6833e58..6986ab8f0 100644 --- a/pkg/cmd/internal/templates/progress_shard_test.go +++ b/pkg/cmd/internal/templates/progress_shard_test.go @@ -118,7 +118,10 @@ func TestFormatDurationSeconds(t *testing.T) { } } -func TestFormatShardLineDisplaysOnePercentAfterCopyStarts(t *testing.T) { +// A copying shard that hasn't reached 1% shows the true fraction computed +// from its row counts, so a shard's early progress on a huge table reads as +// the small fraction it is instead of a rounded-up 1%. +func TestFormatShardLineShowsSubPercentFraction(t *testing.T) { line := formatShardLine(ShardProgress{ Shard: "-80", Status: state.Task.Running, @@ -127,8 +130,8 @@ func TestFormatShardLineDisplaysOnePercentAfterCopyStarts(t *testing.T) { PercentComplete: 0, }) - assert.Contains(t, line, "1% (3,000/1,604,159 rows)") - assert.NotContains(t, line, "0%") + assert.Contains(t, line, "0.19% (3,000/1,604,159 rows)") + assert.NotContains(t, line, " 0%") } func TestIsPlanetScaleEngine(t *testing.T) { diff --git a/pkg/cmd/internal/templates/progress_states_test.go b/pkg/cmd/internal/templates/progress_states_test.go index 65766c31b..7ef7d461d 100644 --- a/pkg/cmd/internal/templates/progress_states_test.go +++ b/pkg/cmd/internal/templates/progress_states_test.go @@ -635,17 +635,18 @@ func TestFormatTableProgress_Checksumming(t *testing.T) { TableName: "orders", ChangeType: "alter", Status: state.Task.Checksumming, ChecksumRowsChecked: 321450, ChecksumRowsTotal: 1466232, }) - assert.Contains(t, withProgress, "๐Ÿ” Checksumming to verify data (22%)") + assert.Contains(t, withProgress, "๐Ÿ” Checksumming to verify data (21.92%)") assert.Contains(t, withProgress, "Rows verified: 321,450 / 1,466,232") - // A verify that has only just begun still renders as visibly started: a - // non-zero checked count floors the display at 1% instead of showing an - // empty "not started" bar while verification is actively running. + // A verify that has only just begun still renders as visibly started: the + // bar shows a first segment, and the percent shows the true sub-1% fraction + // (floored away from 0.00%) instead of an empty "not started" display while + // verification is actively running. justStarted := FormatTableProgress(TableProgress{ TableName: "orders", ChangeType: "alter", Status: state.Task.Checksumming, ChecksumRowsChecked: 1, ChecksumRowsTotal: 1000000, }) - assert.Contains(t, justStarted, "๐Ÿ” Checksumming to verify data (1%)") + assert.Contains(t, justStarted, "๐Ÿ” Checksumming to verify data (0.01%)") assert.Contains(t, justStarted, "๐ŸŸฆ") } @@ -661,7 +662,7 @@ func TestFormatTableProgress_Throttled(t *testing.T) { RowsCopied: 45000, RowsTotal: 100000, PercentComplete: 45, Throttled: true, ThrottleReason: "redo-aware 4 > 3", }) - assert.Contains(t, copying, "45% (throttled)", + assert.Contains(t, copying, "45.00% (throttled)", "the annotation lands on the header line next to the percent") assert.Contains(t, copying, "โ„น๏ธ Throttled: redo-aware 4 > 3 ยท backing off while the database's active threads exceed its budget") @@ -670,7 +671,7 @@ func TestFormatTableProgress_Throttled(t *testing.T) { RowsCopied: 45000, RowsTotal: 100000, PercentComplete: 45, Throttled: true, }) - assert.Contains(t, noReason, "45% (throttled)") + assert.Contains(t, noReason, "45.00% (throttled)") assert.NotContains(t, noReason, "โ„น๏ธ Throttled", "no tooltip without a reason") unknownSignal := FormatTableProgress(TableProgress{ @@ -687,7 +688,7 @@ func TestFormatTableProgress_Throttled(t *testing.T) { ChecksumRowsChecked: 321450, ChecksumRowsTotal: 1466232, Throttled: true, ThrottleReason: "threads-running 21 > 18", }) - assert.Contains(t, checksumming, "๐Ÿ” Checksumming to verify data (22%) (throttled)") + assert.Contains(t, checksumming, "๐Ÿ” Checksumming to verify data (21.92%) (throttled)") assert.Contains(t, checksumming, "โ„น๏ธ Throttled: threads-running 21 > 18 ยท backing off while the database's active threads exceed its budget") notThrottled := FormatTableProgress(TableProgress{ @@ -789,7 +790,7 @@ func TestFormatTableProgress_InstantAlterRendering(t *testing.T) { PercentComplete: 25, } output := FormatTableProgress(copying) - assert.Contains(t, output, "25%", "a copying instant-flagged ALTER shows its real percent") + assert.Contains(t, output, "25.00%", "a copying instant-flagged ALTER shows its real percent") assert.NotContains(t, output, "Applying instantly", "the instant label must not mask copy progress") instant := TableProgress{ @@ -834,16 +835,22 @@ func TestFormatTableProgress_CreateDropLabels(t *testing.T) { assert.Contains(t, output, ui.ProgressBarRowCopy(45)) assert.NotContains(t, output, ui.ProgressBarRowCopy(100)) + // Once row counts arrive, the displayed percent is computed from them + // rather than the engine's stale whole-number percent. tp.RowsCopied = 420 tp.RowsTotal = 1000 tp.ETASeconds = 120 output = FormatTableProgress(tp) - assert.Contains(t, output, "Row copy in progress (45%)") + assert.Contains(t, output, "Row copy in progress (42.00%)") assert.Contains(t, output, "Rows: 420 / 1,000 ยท ETA: 2m") assert.NotContains(t, output, "Recovering state...") } -func TestFormatTableProgress_RowCopyDisplaysOnePercentAfterCopyStarts(t *testing.T) { +// A copy that has begun but not yet reached 1% shows the true fraction +// computed from the row counts, with the bar's first segment lit so the +// operator sees both that copying started and how little of a huge table has +// actually copied. +func TestFormatTableProgress_SubPercentRowCopyShowsFraction(t *testing.T) { tp := TableProgress{ TableName: "orders", ChangeType: "alter", @@ -855,7 +862,7 @@ func TestFormatTableProgress_RowCopyDisplaysOnePercentAfterCopyStarts(t *testing output := FormatTableProgress(tp) - assert.Contains(t, output, "orders: "+ui.ProgressBarRowCopy(1)+" 1%") + assert.Contains(t, output, "orders: "+ui.ProgressBarRowCopy(1)+" 0.19%") assert.Contains(t, output, "Rows: 3,000 / 1,604,159") assert.NotContains(t, output, " 0%") } @@ -1053,7 +1060,7 @@ func TestFormatTableProgressOperatorHaltedBars(t *testing.T) { RowsCopied: 300, RowsTotal: 1000, }) - assert.Contains(t, cancelled, "๐Ÿšซ Cancelled at 30%") + assert.Contains(t, cancelled, "๐Ÿšซ Cancelled at 30.00%") assert.Contains(t, cancelled, ui.ColorOrange) assert.NotContains(t, cancelled, ui.ColorRed) diff --git a/pkg/ui/format.go b/pkg/ui/format.go index 380eb62c7..9580fb1db 100644 --- a/pkg/ui/format.go +++ b/pkg/ui/format.go @@ -3,6 +3,7 @@ package ui import ( "fmt" + "math" "regexp" "strings" "time" @@ -176,9 +177,11 @@ func ClampPercent(pct int) int { return pct } -// RowCopyDisplayPercent returns the percentage to show for row-copy progress. -// A non-zero copied row count means copying has begun, so display at least 1% -// even when integer progress rounds down to 0%. +// RowCopyDisplayPercent returns the whole-number percentage for row-copy +// progress bars and threshold comparisons. A non-zero copied row count means +// copying has begun, so it reports at least 1% even when integer progress +// rounds down to 0%. Textual percents render through FormatRowCopyPercent, +// which shows the exact row-derived fraction instead of the bump. func RowCopyDisplayPercent(pct int, rowsCopied int64) int { displayPercent := ClampPercent(pct) if displayPercent == 0 && rowsCopied > 0 { @@ -187,6 +190,45 @@ func RowCopyDisplayPercent(pct int, rowsCopied int64) int { return displayPercent } +// RowCopyFraction returns row-copy progress as a float percent in [0, 100], +// preferring the exact fraction computed from the row counts over the +// engine's whole-number percent, which has already lost the precision. With +// no row counts to compute from, it falls back to the clamped whole-number +// percent. Renderers comparing tables or shards by progress should compare +// this value so the selection agrees with what FormatRowCopyPercent displays. +func RowCopyFraction(pct int, rowsCopied, rowsTotal int64) float64 { + if rowsCopied > 0 && rowsTotal > 0 { + return math.Min(float64(rowsCopied)/float64(rowsTotal)*100, 100) + } + return float64(ClampPercent(pct)) +} + +// FormatRowCopyPercent renders row-copy progress as a percent string at its +// true precision. When the row counts are known the percent is recomputed +// from them and rendered with two decimals (e.g. "0.03%", "45.37%"), so +// progress on a huge table reads as the fraction it is instead of a rounded +// whole number. The rendering is bounded on both ends by what the counts say: +// floored at 0.01% so an in-flight copy never reads as 0.00%, and capped at +// 99.99% while copied rows still trail the total, so a copy never reads as +// finished before it is โ€” an operator deciding whether to keep waiting takes +// "100.00%" as done. Without row counts it falls back to the engine's +// whole-number percent, or "<1%" when copying has begun but there is no +// total to compute a fraction from. +func FormatRowCopyPercent(pct int, rowsCopied, rowsTotal int64) string { + if rowsCopied > 0 && rowsTotal > 0 { + frac := math.Max(RowCopyFraction(pct, rowsCopied, rowsTotal), 0.01) + if rowsCopied < rowsTotal { + frac = math.Min(frac, 99.99) + } + return fmt.Sprintf("%.2f%%", frac) + } + display := ClampPercent(pct) + if display == 0 && rowsCopied > 0 { + return "<1%" + } + return fmt.Sprintf("%d%%", display) +} + // NowFunc returns the current time. Override in previews for deterministic output. var NowFunc = time.Now diff --git a/pkg/ui/format_test.go b/pkg/ui/format_test.go index 6c391c6d6..362ea7064 100644 --- a/pkg/ui/format_test.go +++ b/pkg/ui/format_test.go @@ -136,6 +136,41 @@ func TestRowCopyDisplayPercent(t *testing.T) { assert.Equal(t, 100, RowCopyDisplayPercent(145, 42)) } +func TestRowCopyFraction(t *testing.T) { + assert.InDelta(t, 45.37, RowCopyFraction(45, 453_700, 1_000_000), 0.0001) + assert.InDelta(t, 100, RowCopyFraction(99, 1_100_000, 1_000_000), 0.0001) + assert.InDelta(t, 45, RowCopyFraction(45, 0, 0), 0.0001) + assert.InDelta(t, 100, RowCopyFraction(145, 0, 0), 0.0001) +} + +func TestFormatRowCopyPercent(t *testing.T) { + // Row counts are the source of truth: the percent is recomputed from them + // at two-decimal precision, whatever the whole-number percent says. + assert.Equal(t, "45.00%", FormatRowCopyPercent(45, 450_000, 1_000_000)) + assert.Equal(t, "45.37%", FormatRowCopyPercent(45, 453_700, 1_000_000)) + assert.Equal(t, "0.03%", FormatRowCopyPercent(0, 13_186_540, 43_234_523_345)) + assert.Equal(t, "0.19%", FormatRowCopyPercent(0, 3_000, 1_604_159)) + assert.Equal(t, "100.00%", FormatRowCopyPercent(100, 1_000_000, 1_000_000)) + + // Floored at 0.01% once copying has begun, and capped at 100% when the + // copied count overshoots the estimated total. + assert.Equal(t, "0.01%", FormatRowCopyPercent(0, 1, 1_000_000)) + assert.Equal(t, "100.00%", FormatRowCopyPercent(99, 1_100_000, 1_000_000)) + + // Capped at 99.99% while copied rows still trail the total: "100.00%" + // reads as done, and a copy in its final stretch is not. + assert.Equal(t, "99.99%", FormatRowCopyPercent(99, 999_999_999, 1_000_000_000)) + assert.Equal(t, "99.99%", FormatRowCopyPercent(100, 43_234_523_344, 43_234_523_345)) + assert.Equal(t, "99.99%", FormatRowCopyPercent(99, 99_999, 100_000)) + + // Without row counts, the engine's whole-number percent renders clamped; + // a copy that has begun with no total falls back to "<1%". + assert.Equal(t, "0%", FormatRowCopyPercent(0, 0, 1_000_000)) + assert.Equal(t, "45%", FormatRowCopyPercent(45, 0, 0)) + assert.Equal(t, "100%", FormatRowCopyPercent(145, 0, 0)) + assert.Equal(t, "<1%", FormatRowCopyPercent(0, 42, 0)) +} + func TestLintReasons(t *testing.T) { t.Run("splits engine-joined violations into cleaned messages", func(t *testing.T) { assert.Equal(t, diff --git a/pkg/webhook/apply.go b/pkg/webhook/apply.go index 86f0ec5ca..4e8db0dba 100644 --- a/pkg/webhook/apply.go +++ b/pkg/webhook/apply.go @@ -158,6 +158,8 @@ func shardProgressForTable(shardsByTable map[string][]*storage.Task, applyOperat Shard: r.Shard, Status: string(r.State), PercentComplete: r.ProgressPercent, + RowsCopied: r.RowsCopied, + RowsTotal: r.RowsTotal, }) } sort.Slice(out, func(i, j int) bool { return out[i].Shard < out[j].Shard }) diff --git a/pkg/webhook/apply_comment_integration_test.go b/pkg/webhook/apply_comment_integration_test.go index 2f13fcb40..e22bfa2de 100644 --- a/pkg/webhook/apply_comment_integration_test.go +++ b/pkg/webhook/apply_comment_integration_test.go @@ -440,7 +440,7 @@ func TestE2EResumeRotatesProgressComment(t *testing.T) { select { case edited := <-capture.edits: assert.Equal(t, newProgressID, edited.CommentID, "later edits land on the new comment") - assert.Contains(t, edited.Body, "70%", "once running, the new comment shows real row-copy progress") + assert.Contains(t, edited.Body, "70.00%", "once running, the new comment shows real row-copy progress") case created := <-capture.creates: t.Fatalf("resume must rotate exactly once; got another new comment %d", created.ID) case <-time.After(5 * time.Second): diff --git a/pkg/webhook/apply_test.go b/pkg/webhook/apply_test.go index 84de38a83..2836c444e 100644 --- a/pkg/webhook/apply_test.go +++ b/pkg/webhook/apply_test.go @@ -42,7 +42,7 @@ func TestFormatProgressComment(t *testing.T) { assert.Contains(t, body, "staging") assert.Contains(t, body, "`users`") assert.Contains(t, body, "`orders`") - assert.Contains(t, body, "45%") + assert.Contains(t, body, "45.00%") assert.Contains(t, body, "Schema Change Status") // The running state is communicated on the always-present Status line. assert.Contains(t, body, "**Status**: In Progress") diff --git a/pkg/webhook/templates/apply.go b/pkg/webhook/templates/apply.go index 2ef908c96..f942d5499 100644 --- a/pkg/webhook/templates/apply.go +++ b/pkg/webhook/templates/apply.go @@ -81,6 +81,8 @@ type ShardProgressData struct { Shard string Status string // canonical lowercase shard/task state PercentComplete int + RowsCopied int64 + RowsTotal int64 } // ApplyStatusCommentData contains all data needed to render an apply status PR comment. @@ -460,6 +462,7 @@ func writeProgressSummary(sb *strings.Builder, tables []TableProgressData) { var completed, running, catchingUp, checksumming, queued, waiting, failed, retrying, stopped, recovering, cutting, cancelled, other int var runningPct int + var runningPctText string var runningEstimateExceeded bool for _, t := range tables { @@ -479,6 +482,7 @@ func writeProgressSummary(sb *strings.Builder, tables []TableProgressData) { runningEstimateExceeded = true } else { runningPct = ui.RowCopyDisplayPercent(t.PercentComplete, t.RowsCopied) + runningPctText = ui.FormatRowCopyPercent(t.PercentComplete, t.RowsCopied, t.RowsTotal) } case state.Task.CatchingUp, state.Task.PostChecksum: // Both binlog drains (before and after the verify) read as @@ -525,7 +529,7 @@ func writeProgressSummary(sb *strings.Builder, tables []TableProgressData) { if runningEstimateExceeded { label += " (finalizing copy)" } else if runningPct > 0 { - label += fmt.Sprintf(" (%d%%)", runningPct) + label += fmt.Sprintf(" (%s)", runningPctText) } parts = append(parts, label) } @@ -777,7 +781,8 @@ func renderTableProgress(sb *strings.Builder, dialect schema.Dialect, table Tabl // show how far the verify has progressed once Spirit reports a total. if table.ChecksumRowsTotal > 0 { pct := ui.ClampPercent(int(table.ChecksumRowsChecked * 100 / table.ChecksumRowsTotal)) - fmt.Fprintf(sb, "**`%s`**: %s \U0001f50d Checksumming to verify data (%d%%)%s\n", table.TableName, ui.ProgressBarRowCopy(pct), pct, throttledSuffix(table)) + fmt.Fprintf(sb, "**`%s`**: %s \U0001f50d Checksumming to verify data (%s)%s\n", table.TableName, ui.ProgressBarRowCopy(pct), + ui.FormatRowCopyPercent(pct, table.ChecksumRowsChecked, table.ChecksumRowsTotal), throttledSuffix(table)) writeDDLLine(sb, dialect, table.DDL) fmt.Fprintf(sb, "- Rows verified: %s / %s\n", ui.FormatNumber(ui.ClampRows(table.ChecksumRowsChecked, table.ChecksumRowsTotal)), ui.FormatNumber(table.ChecksumRowsTotal)) @@ -805,7 +810,8 @@ func renderTableProgress(sb *strings.Builder, dialect schema.Dialect, table Tabl if recoveringIsCopyingRows(table) { pct := ui.RowCopyDisplayPercent(table.PercentComplete, table.RowsCopied) bar := ui.ProgressBarRowCopy(pct) - fmt.Fprintf(sb, "**`%s`**: %s Row copy in progress (%d%%)\n", table.TableName, bar, pct) + fmt.Fprintf(sb, "**`%s`**: %s Row copy in progress (%s)\n", table.TableName, bar, + ui.FormatRowCopyPercent(table.PercentComplete, table.RowsCopied, table.RowsTotal)) writeDDLLine(sb, dialect, table.DDL) writeRowsAndETA(sb, table) break @@ -917,8 +923,9 @@ func renderShardSummary(sb *strings.Builder, table TableProgressData) { if len(table.Shards) <= shardNamesInlineLimit { parts := make([]string, 0, len(table.Shards)) for _, sh := range table.Shards { - if isCopyingShardStatus(sh.Status) && sh.PercentComplete > 0 { - parts = append(parts, fmt.Sprintf("%s %s %d%%", shardGlyph(sh.Status), sh.Shard, sh.PercentComplete)) + if isCopyingShardStatus(sh.Status) && (sh.PercentComplete > 0 || sh.RowsCopied > 0) { + parts = append(parts, fmt.Sprintf("%s %s %s", shardGlyph(sh.Status), sh.Shard, + ui.FormatRowCopyPercent(sh.PercentComplete, sh.RowsCopied, sh.RowsTotal))) continue } part := fmt.Sprintf("%s %s", shardGlyph(sh.Status), sh.Shard) @@ -934,8 +941,8 @@ func renderShardSummary(sb *strings.Builder, table TableProgressData) { } var complete, copying, ready, failed, queued, other int - slowestShard := "" - slowestPct := -1 + slowestShard, slowestText := "", "" + slowestFraction := -1.0 for _, sh := range table.Shards { // Shards parked at the cutover barrier count through the shared // readiness predicate, keeping the shard buckets consistent with the @@ -954,9 +961,10 @@ func renderShardSummary(sb *strings.Builder, table TableProgressData) { default: if isCopyingShardStatus(sh.Status) { copying++ - if slowestPct < 0 || sh.PercentComplete < slowestPct { - slowestPct = sh.PercentComplete + if frac := ui.RowCopyFraction(sh.PercentComplete, sh.RowsCopied, sh.RowsTotal); slowestFraction < 0 || frac < slowestFraction { + slowestFraction = frac slowestShard = sh.Shard + slowestText = ui.FormatRowCopyPercent(sh.PercentComplete, sh.RowsCopied, sh.RowsTotal) } } else { other++ @@ -983,8 +991,8 @@ func renderShardSummary(sb *strings.Builder, table TableProgressData) { buckets = append(buckets, fmt.Sprintf("%d โ€ฆ", other)) } line := fmt.Sprintf(" โ”” %d shards: %s", len(table.Shards), strings.Join(buckets, " ยท ")) - if slowestShard != "" && slowestPct >= 0 { - line += fmt.Sprintf(" ยท slowest %s %d%%", slowestShard, slowestPct) + if slowestShard != "" && slowestFraction >= 0 { + line += fmt.Sprintf(" ยท slowest %s %s", slowestShard, slowestText) } sb.WriteString(line + "\n") } @@ -1060,7 +1068,8 @@ func renderRunningTable(sb *strings.Builder, dialect schema.Dialect, table Table return } bar := ui.ProgressBarRowCopy(pct) - fmt.Fprintf(sb, "**`%s`**: %s %d%%%s\n", table.TableName, bar, pct, throttledSuffix(table)) + fmt.Fprintf(sb, "**`%s`**: %s %s%s\n", table.TableName, bar, + ui.FormatRowCopyPercent(table.PercentComplete, table.RowsCopied, table.RowsTotal), throttledSuffix(table)) writeDDLLine(sb, dialect, table.DDL) writeRowsAndETA(sb, table) } else { @@ -1106,17 +1115,24 @@ func recoveringIsCopyingRows(table TableProgressData) bool { return table.RowsTotal > 0 && table.PercentComplete < 100 } -func recoveringCopyPercent(tables []TableProgressData) (int, bool) { - percent := 100 +// recoveringCopyPercent returns the least-progressed recovering table's copy +// percent as display text, so the recovery summary never overstates how far +// the slowest table has come. +func recoveringCopyPercent(tables []TableProgressData) (string, bool) { + fraction := 0.0 + text := "" found := false for _, table := range tables { if state.NormalizeTaskStatus(table.Status) != state.Task.Recovering || !recoveringIsCopyingRows(table) { continue } - percent = min(percent, ui.RowCopyDisplayPercent(table.PercentComplete, table.RowsCopied)) + if frac := ui.RowCopyFraction(table.PercentComplete, table.RowsCopied, table.RowsTotal); !found || frac < fraction { + fraction = frac + text = ui.FormatRowCopyPercent(table.PercentComplete, table.RowsCopied, table.RowsTotal) + } found = true } - return percent, found + return text, found } // renderStoppedTable renders a table in the stopped state. @@ -1128,7 +1144,8 @@ func renderStoppedTable(sb *strings.Builder, dialect schema.Dialect, table Table case table.PercentComplete > 0 || table.RowsCopied > 0: pct := ui.RowCopyDisplayPercent(table.PercentComplete, table.RowsCopied) bar := ui.ProgressBarStopped(pct) - fmt.Fprintf(sb, "**`%s`**: %s \u23f9\ufe0f Stopped at %d%%\n", table.TableName, bar, pct) + fmt.Fprintf(sb, "**`%s`**: %s \u23f9\ufe0f Stopped at %s\n", table.TableName, bar, + ui.FormatRowCopyPercent(table.PercentComplete, table.RowsCopied, table.RowsTotal)) default: fmt.Fprintf(sb, "**`%s`**: \u23f9\ufe0f Stopped (not started)\n", table.TableName) } @@ -1211,7 +1228,7 @@ func writeApplyFooter(sb *strings.Builder, data ApplyStatusCommentData) { case state.Apply.Recovering: sb.WriteString("\n---\n\n") if pct, ok := recoveringCopyPercent(data.Tables); ok { - fmt.Fprintf(sb, "Recovering after restart. Row copy is in progress (%d%%); once recovery completes, progress returns to the normal row-copy view.\n", pct) + fmt.Fprintf(sb, "Recovering after restart. Row copy is in progress (%s); once recovery completes, progress returns to the normal row-copy view.\n", pct) } else { sb.WriteString("Recovering after restart. Cutover will be available once recovery completes.\n") } @@ -1710,13 +1727,13 @@ func writeSummaryTableEntry(sb *strings.Builder, dialect schema.Dialect, t Table case state.Task.Failed: label := "Failed" if t.PercentComplete > 0 || t.RowsCopied > 0 { - label = fmt.Sprintf("Failed at %d%%", ui.RowCopyDisplayPercent(t.PercentComplete, t.RowsCopied)) + label = fmt.Sprintf("Failed at %s", ui.FormatRowCopyPercent(t.PercentComplete, t.RowsCopied, t.RowsTotal)) } fmt.Fprintf(sb, "**`%s`** โ€” %s\n", t.TableName, label) case state.Task.Stopped: label := "Stopped" if t.PercentComplete > 0 || t.RowsCopied > 0 { - label = fmt.Sprintf("Stopped at %d%%", ui.RowCopyDisplayPercent(t.PercentComplete, t.RowsCopied)) + label = fmt.Sprintf("Stopped at %s", ui.FormatRowCopyPercent(t.PercentComplete, t.RowsCopied, t.RowsTotal)) } fmt.Fprintf(sb, "**`%s`** โ€” %s\n", t.TableName, label) case "reverted": diff --git a/pkg/webhook/templates/apply_test.go b/pkg/webhook/templates/apply_test.go index f6724037b..3dee16baf 100644 --- a/pkg/webhook/templates/apply_test.go +++ b/pkg/webhook/templates/apply_test.go @@ -255,7 +255,7 @@ func TestRenderApplyStatusComment_Checksumming(t *testing.T) { assert.Contains(t, result, "## Schema Change Status", "apply stays in progress; checksumming is table-level") assert.Contains(t, result, "**`orders`**") - assert.Contains(t, result, "๐Ÿ” Checksumming to verify data (21%)") + assert.Contains(t, result, "๐Ÿ” Checksumming to verify data (21.92%)") assert.Contains(t, result, "Rows verified: 321,450 / 1,466,232") assert.Contains(t, result, "1 checksumming") } @@ -284,7 +284,7 @@ func TestRenderApplyStatusComment_Throttled(t *testing.T) { result := RenderApplyStatusComment(data) - assert.Contains(t, result, "45% (throttled)", + assert.Contains(t, result, "45.00% (throttled)", "the annotation lands on the header line next to the percent") assert.Contains(t, result, "- โ„น๏ธ _Throttled: redo-aware 4 > 3 ยท backing off while the database's active threads exceed its budget ([docs](https://github.com/block/schemabot/blob/main/docs/throttle.md))_", "the reason renders as a tooltip bullet with its tip and the doc link") @@ -301,7 +301,7 @@ func TestRenderApplyStatusComment_Throttled(t *testing.T) { data.Tables[0].ThrottleReason = "" noReason := RenderApplyStatusComment(data) - assert.Contains(t, noReason, "45% (throttled)") + assert.Contains(t, noReason, "45.00% (throttled)") assert.NotContains(t, noReason, "โ„น๏ธ Throttled", "no tooltip without a reason") data.Tables[0].Throttled = false @@ -335,7 +335,7 @@ func TestRenderApplyStatusComment_ThrottledChecksumming(t *testing.T) { result := RenderApplyStatusComment(data) - assert.Contains(t, result, "๐Ÿ” Checksumming to verify data (21%) (throttled)") + assert.Contains(t, result, "๐Ÿ” Checksumming to verify data (21.92%) (throttled)") assert.Contains(t, result, "- โ„น๏ธ _Throttled: threads-running 21 > 18 ยท backing off while the database's active threads exceed its budget ([docs](https://github.com/block/schemabot/blob/main/docs/throttle.md))_") } @@ -498,7 +498,7 @@ func TestRenderApplyStatusComment_Running(t *testing.T) { assert.NotContains(t, result, "**Last updated**") // Progress summary assert.Contains(t, result, "๐Ÿ“Š 1/3 complete") - assert.Contains(t, result, "1 running (45%)") + assert.Contains(t, result, "1 running (45.00%)") assert.Contains(t, result, "1 queued") assert.Contains(t, result, "**`users`**") @@ -508,7 +508,7 @@ func TestRenderApplyStatusComment_Running(t *testing.T) { assert.Contains(t, result, "๐ŸŸฉ") // green bar for completed assert.Contains(t, result, "**`users`**") - assert.Contains(t, result, "45%") + assert.Contains(t, result, "45.00%") assert.Contains(t, result, "๐ŸŸฆ") // blue bar for running assert.Contains(t, result, "45,000 / 100,000") assert.Contains(t, result, "ETA: 3m 15s") @@ -757,7 +757,12 @@ func TestRenderApplyStatusComment_DeployRequestLink(t *testing.T) { assert.NotContains(t, RenderApplyStatusComment(base), "Deploy request:") } -func TestRenderApplyStatusComment_RowCopyDisplaysOnePercentAfterCopyStarts(t *testing.T) { +// A copy that has begun but not yet reached 1% shows the true fraction +// computed from the row counts โ€” in the running summary and on the table's +// progress line โ€” so early progress on a huge table reads as the small +// fraction it is instead of a rounded-up 1%. The bar still lights its first +// segment so the copy reads as started. +func TestRenderApplyStatusComment_SubPercentRowCopyShowsFraction(t *testing.T) { data := ApplyStatusCommentData{ Database: "testapp", Environment: "staging", @@ -772,8 +777,8 @@ func TestRenderApplyStatusComment_RowCopyDisplaysOnePercentAfterCopyStarts(t *te result := RenderApplyStatusComment(data) - assert.Contains(t, result, "1 running (1%)") - assert.Contains(t, result, "**`orders`**: "+ui.ProgressBarRowCopy(1)+" 1%") + assert.Contains(t, result, "1 running (0.19%)") + assert.Contains(t, result, "**`orders`**: "+ui.ProgressBarRowCopy(1)+" 0.19%") assert.Contains(t, result, "Rows: 3,000 / 1,604,159") assert.NotContains(t, result, " 0%") } @@ -1303,7 +1308,7 @@ func TestRenderApplyStatusComment_Stopped(t *testing.T) { assert.Contains(t, result, "## Schema Change Status โ€” Staging") assert.Contains(t, result, "**Status**: Stopped") assert.Contains(t, result, "๐ŸŸง") // orange bar for stopped - assert.Contains(t, result, "โน๏ธ Stopped at 72%") + assert.Contains(t, result, "โน๏ธ Stopped at 72.00%") assert.Contains(t, result, "72,000 / 100,000") // Progress summary assert.Contains(t, result, "๐Ÿ“Š 1/2 complete") @@ -1449,9 +1454,9 @@ func TestRenderApplyStatusComment_RecoveringCopyingRows(t *testing.T) { result := RenderApplyStatusComment(data) - assert.Contains(t, result, "Row copy in progress (42%)") + assert.Contains(t, result, "Row copy in progress (42.00%)") assert.Contains(t, result, "Rows: 420 / 1,000 ยท ETA: 2m") - assert.Contains(t, result, "Row copy is in progress (42%)") + assert.Contains(t, result, "Row copy is in progress (42.00%)") assert.Contains(t, result, "progress returns to the normal row-copy view") assert.Contains(t, result, "Recovering after restart") assert.NotContains(t, result, "Cutover will be available once recovery completes") @@ -1665,7 +1670,7 @@ func TestPreviewCommentApplyProgress(t *testing.T) { assert.Contains(t, result, "**`orders`**") assert.Contains(t, result, "**`users`**") assert.Contains(t, result, "**`products`**") - assert.Contains(t, result, "62%") + assert.Contains(t, result, "62.38%") assert.Contains(t, result, "Queued") } @@ -1702,7 +1707,7 @@ func TestPreviewCommentApplyStopped(t *testing.T) { assert.Contains(t, result, "Schema Change Status") assert.Contains(t, result, "**Status**: Stopped") - assert.Contains(t, result, "Stopped at 72%") + assert.Contains(t, result, "Stopped at 72.00%") assert.Contains(t, result, "schemabot start") } @@ -1768,7 +1773,7 @@ func TestPreviewCommentSummaryFailed(t *testing.T) { assert.Contains(t, result, "1 of 3 tables completed before failure.") // Single namespace โ€” no header, but table entries present assert.NotContains(t, result, "### ") - assert.Contains(t, result, "**`users`** โ€” Failed at 30%") + assert.Contains(t, result, "**`users`** โ€” Failed at 30.00%") assert.Contains(t, result, "**`orders`** โ€” Completed", "on a failed apply, the reader must be able to tell which tables made it") assert.Contains(t, result, "**`products`** โ€” Cancelled") } @@ -1780,7 +1785,7 @@ func TestPreviewCommentSummaryStopped(t *testing.T) { assert.Contains(t, result, "1 of 2 tables completed before stop.") // Single namespace โ€” no header assert.NotContains(t, result, "### ") - assert.Contains(t, result, "**`users`** โ€” Stopped at 72%") + assert.Contains(t, result, "**`users`** โ€” Stopped at 72.00%") assert.Contains(t, result, "**`orders`** โ€” Completed", "on a stopped apply, the reader must be able to tell which tables made it") // A stopped change is resumable. assert.Contains(t, result, "schemabot start") diff --git a/pkg/webhook/templates/sharded_apply.go b/pkg/webhook/templates/sharded_apply.go index 9a0fa5e7a..867d7c47a 100644 --- a/pkg/webhook/templates/sharded_apply.go +++ b/pkg/webhook/templates/sharded_apply.go @@ -437,7 +437,7 @@ func shardedTableStatusPhrase(status string) string { case state.Task.Recovering: return "๐Ÿ”„ Recovering state..." case state.Task.Failed: - return "โŒ Failed" + return glyph.Failed + " Failed" case state.Task.FailedRetryable: return "๐Ÿ”„ Interrupted โ€” retrying automatically" case state.Task.Stopped: