Send the sweep strip as a bitmap rather than a list of positions

report_client.html

@@ -405,7 +405,7 @@
   <script>
   (() => {
     "use strict";
-    const SCHEMA_VERSION=3, DEFAULT_POLL=2000, STUCK_REQUEST_MILLIS=60000;
+    const SCHEMA_VERSION=4, DEFAULT_POLL=2000, STUCK_REQUEST_MILLIS=60000;
     const branchStatusNames=["unqueued","queued","evaluating","finalizing","done"];
     const branchWorkerStatusNames=["active","waiting"];
     // The overview answers what the swarm is doing right now, so it offers
@@ -1030,7 +1030,7 @@
       card.append(title);
       if(row.spine?.length)card.append(renderSpine(Array.isArray(row.spine)?row.spine:spineFromText(row.spine)));
       const parts=[];
-      if(row.candidate_count){const line=element("div","progress-line");const fill=Array.isArray(row.completed_candidate_indexes)?sweepStrip(row.candidate_count,row.completed_candidate_indexes,branchWorkers,previousWorkersById):progress(row.completed_candidate_count,row.candidate_count);line.append(fill);card.append(line);parts.push(numText(row.completed_candidate_count)+"/"+numText(row.candidate_count)+" cands");}
+      if(row.candidate_count){const line=element("div","progress-line");const fill=typeof row.completed_candidate_bitmap==="string"?sweepStrip(row.candidate_count,row.completed_candidate_bitmap,branchWorkers,previousWorkersById):progress(row.completed_candidate_count,row.candidate_count);line.append(fill);card.append(line);parts.push(numText(row.completed_candidate_count)+"/"+numText(row.candidate_count)+" cands");}
       parts.push(numText(row.answer_count)+(row.answer_count===1?" answer":" answers"));
       const best=bestText(row);if(best)parts.push(best);
       if(row.worker_count)parts.push(numText(row.worker_count)+(row.worker_count===1?" worker":" workers"));
@@ -1099,13 +1099,28 @@
       if(worker.candidate_index===null||worker.candidate_index===undefined)return null;
       return {candidate_index:worker.candidate_index,state:"working"};
     }
-    function sweepStrip(candidateCount,completedCandidateIndexes,workers,previousWorkersById){
+    // One bit per candidate, little-endian within each byte, matching
+    // report_model.encode_candidate_bitmap.  A branch carries up to the whole
+    // vocabulary of candidates, and the overview draws one strip per active
+    // branch on a page that polls every two seconds; as a list of positions
+    // that reached 91,643 bytes for a single production branch against 2,478
+    // for the same branch as bits.  Testing bits directly also avoids building
+    // a Set of every completed position just to ask about ranges of it.
+    function decodeCandidateBitmap(bitmap){
+      if(typeof bitmap!=="string"||!bitmap)return new Uint8Array(0);
+      let binary;
+      try{binary=atob(bitmap);}catch(error){return new Uint8Array(0);}
+      const bytes=new Uint8Array(binary.length);
+      for(let index=0;index<binary.length;index++)bytes[index]=binary.charCodeAt(index);
+      return bytes;
+    }
+    function sweepStrip(candidateCount,completedCandidateBitmap,workers,previousWorkersById){
       const cellCount=Math.max(1,Math.min(50,candidateCount));
-      const strip=element("div","sweep"),done=new Set(completedCandidateIndexes||[]);
+      const strip=element("div","sweep"),done=decodeCandidateBitmap(completedCandidateBitmap);
       strip.dataset.cellCount=cellCount;
       for(let cell=0;cell<cellCount;cell++){
         const start=Math.floor(cell*candidateCount/cellCount),end=Math.max(start+1,Math.floor((cell+1)*candidateCount/cellCount));
-        let doneCount=0;for(let index=start;index<end;index++)if(done.has(index))doneCount++;
+        let doneCount=0;for(let index=start;index<end;index++)if(done[index>>3]&(1<<(index&7)))doneCount++;
         const cellNode=element("span","sweep-cell");
         // A 100% fill is reserved for a cell whose every candidate is done;
         // partial completion is capped below full (and lifted off zero) so a
@@ -1714,7 +1729,7 @@
       const queueFacts=data.queue?[["status",data.queue.branch_status],["worker",data.queue.branch_worker_status],["priority",data.queue.priority],["budget",data.queue.budget],["best",bestGuessCell(data.queue,branch.answer_count)||"none yet"],["ERD ceiling",erdValue(data.queue.ceiling,branch.answer_count,{ceiling:true})],["nodes",data.queue.search_node_count],...(bundleSummary.wall_millis?[['wall time',formatDuration(bundleSummary.wall_millis/1000)]]:[])]:null;
       fragment.append(section("Queue",queueFacts?labeledFacts(queueFacts):element("p","empty","No current queue state")));
       const claimSummary=data.claim_summary||{};
-      if(data.queue&&data.queue.candidate_count){const liveWorkers=(data.workers||[]).filter(worker=>worker.is_live);const candidateFacts=[["candidates",formatInteger(data.queue.completed_candidate_count)+"/"+formatInteger(data.queue.candidate_count)+" ="],["evaluated",claimSummary.evaluated_count||0],["+ one-level ERD prunes",data.queue.one_level_erd_pruned_candidate_count??data.queue.bulk_completed_candidate_count],["+ two-level ERD prunes",data.queue.two_level_erd_pruned_candidate_count??0],...(claimSummary.provenance_unknown_count?[["+ unattributed",claimSummary.provenance_unknown_count]]:[])];const candidateStatusFacts=[...(claimSummary.in_flight_count?[["in flight",claimSummary.in_flight_count]]:[])],candidateETAFacts=[];const candidateETA=data.candidate_eta;if(candidateETA?.state==="ready"||candidateETA?.state==="rough"){const etaDataMinutes=Math.round(candidateETA.sample_duration_seconds/60),etaWorkers=candidateETA.worker_count_changed?"; scaling "+candidateETA.sample_worker_count+"→"+candidateETA.current_worker_count+" workers":"";candidateETAFacts.push([(candidateETA.state==="rough"?"Rough ETA":"ETA")+" ("+etaDataMinutes+" min data"+etaWorkers+")","~"+formatDuration(candidateETA.estimated_seconds)]);candidateETAFacts.push(["ETA work remaining","checks "+numText(candidateETA.remaining_inspection_count)+" · full evals ~"+numText(candidateETA.expected_full_evaluation_count)]);}else if(candidateETA?.state==="learning")candidateETAFacts.push(["ETA","learning: first 3 min of current work sample"]);const republishedList=data.republished_candidates||[];if(republishedList.length)candidateStatusFacts.push(["candidates re-queued",numText(republishedList.length)+" (up to "+Math.max(...republishedList.map(row=>row.republish_count))+"× each)"]);if(claimSummary.worker_contributions?.length)candidateStatusFacts.push(["worker evals",claimSummary.worker_contributions.map(row=>workerName({worker_id:row.worker_id})+":"+numText(row.done_count)).join(" ")]);const candidates=element("div");candidates.append(sweepStrip(data.queue.candidate_count,data.completed_candidate_indexes,liveWorkers,previousBy(previous,"workers","worker_id")),labeledFacts(candidateFacts));if(candidateStatusFacts.length)candidates.append(labeledFacts(candidateStatusFacts));for(const fact of candidateETAFacts)candidates.append(labeledFacts([fact]));fragment.append(section("Candidates",candidates));}
+      if(data.queue&&data.queue.candidate_count){const liveWorkers=(data.workers||[]).filter(worker=>worker.is_live);const candidateFacts=[["candidates",formatInteger(data.queue.completed_candidate_count)+"/"+formatInteger(data.queue.candidate_count)+" ="],["evaluated",claimSummary.evaluated_count||0],["+ one-level ERD prunes",data.queue.one_level_erd_pruned_candidate_count??data.queue.bulk_completed_candidate_count],["+ two-level ERD prunes",data.queue.two_level_erd_pruned_candidate_count??0],...(claimSummary.provenance_unknown_count?[["+ unattributed",claimSummary.provenance_unknown_count]]:[])];const candidateStatusFacts=[...(claimSummary.in_flight_count?[["in flight",claimSummary.in_flight_count]]:[])],candidateETAFacts=[];const candidateETA=data.candidate_eta;if(candidateETA?.state==="ready"||candidateETA?.state==="rough"){const etaDataMinutes=Math.round(candidateETA.sample_duration_seconds/60),etaWorkers=candidateETA.worker_count_changed?"; scaling "+candidateETA.sample_worker_count+"→"+candidateETA.current_worker_count+" workers":"";candidateETAFacts.push([(candidateETA.state==="rough"?"Rough ETA":"ETA")+" ("+etaDataMinutes+" min data"+etaWorkers+")","~"+formatDuration(candidateETA.estimated_seconds)]);candidateETAFacts.push(["ETA work remaining","checks "+numText(candidateETA.remaining_inspection_count)+" · full evals ~"+numText(candidateETA.expected_full_evaluation_count)]);}else if(candidateETA?.state==="learning")candidateETAFacts.push(["ETA","learning: first 3 min of current work sample"]);const republishedList=data.republished_candidates||[];if(republishedList.length)candidateStatusFacts.push(["candidates re-queued",numText(republishedList.length)+" (up to "+Math.max(...republishedList.map(row=>row.republish_count))+"× each)"]);if(claimSummary.worker_contributions?.length)candidateStatusFacts.push(["worker evals",claimSummary.worker_contributions.map(row=>workerName({worker_id:row.worker_id})+":"+numText(row.done_count)).join(" ")]);const candidates=element("div");candidates.append(sweepStrip(data.queue.candidate_count,data.completed_candidate_bitmap,liveWorkers,previousBy(previous,"workers","worker_id")),labeledFacts(candidateFacts));if(candidateStatusFacts.length)candidates.append(labeledFacts(candidateStatusFacts));for(const fact of candidateETAFacts)candidates.append(labeledFacts([fact]));fragment.append(section("Candidates",candidates));}
       // Live workers sits directly under Candidates: they are the two panels
       // that change on every poll, and keeping them adjacent gives them a
       // chance to share one scrolled page.

report_model.py

@@ -3,6 +3,7 @@
 from __future__ import annotations
 
 from dataclasses import dataclass, field, replace
+import base64
 import collections
 import datetime
 import os
@@ -34,7 +35,7 @@
 from wordle_ui import fmt_pattern, parse_pattern
 
 
-SCHEMA_VERSION = 3
+SCHEMA_VERSION = 4
 WORKER_STALE_SECONDS = 20
 DEFAULT_TREE_PAGE_SIZE = 10
 # A tree page groups sibling nodes by the guess word on their spine, and a page
@@ -774,6 +775,56 @@ def _worker_process_is_running(pid):
         return False
 
 
+def encode_candidate_bitmap(indexes, candidate_count):
+    """Pack completed candidate positions into a base64 little-endian bitset.
+
+    The sweep strip needs to know which candidates a branch has finished, and
+    a branch carries up to the whole vocabulary of them.  Sent as a list of
+    positions that is tens of thousands of numbers -- 91,643 bytes for one
+    production branch, and the overview draws one strip per active branch on a
+    page the client polls every two seconds.
+
+    A bitset is one bit per candidate whatever the pattern: 2,478 base64 bytes
+    for a 14,855-candidate branch, against 6,890 to 91,643 for the same
+    branches as positions.  Run-length encoding was measured first and rejected
+    on the same data: it is smaller on a contiguous sweep and barely smaller on
+    a fragmented one (64.7% of the raw list at worst), and a swarm of six
+    workers claiming scattered ranges produces exactly the fragmented case.
+    The bitset has no worst case to avoid.
+
+    Returns None when there is nothing to draw, so a branch with no candidate
+    count carries no field rather than an empty one.
+    """
+    if not candidate_count or candidate_count < 1:
+        return None
+    bits = bytearray((candidate_count + 7) // 8)
+    for index in indexes or ():
+        if index is None or not 0 <= index < candidate_count:
+            continue
+        bits[index >> 3] |= 1 << (index & 7)
+    return base64.b64encode(bytes(bits)).decode("ascii")
+
+
+def decode_candidate_bitmap(bitmap, candidate_count):
+    """Positions set in a bitmap `encode_candidate_bitmap` produced.
+
+    The renderers count completions per cell at their own resolutions -- the
+    browser draws up to 50 cells, the terminal between 10 and 40 -- so the wire
+    format keeps every position rather than a histogram either of them would
+    have to resample.
+    """
+    if not bitmap or not candidate_count:
+        return []
+    try:
+        bits = base64.b64decode(bitmap, validate=True)
+    except (ValueError, TypeError):
+        return []
+    return [
+        index for index in range(min(candidate_count, len(bits) * 8))
+        if bits[index >> 3] & (1 << (index & 7))
+    ]
+
+
 def _normalize_worker(row, generated_at, answer_set):
     branch_key_value = _row_value(row, "current_branch_key")
     branch_key = bytes(branch_key_value) if branch_key_value is not None else None
@@ -1044,8 +1095,9 @@ def _queue_overview(sources, generated_at, answer_set, report):
             # displays can draw the candidate sweep. Other statuses have no
             # live claim rows.
             if normalized["branch_worker_status"] == "active":
-                normalized["completed_candidate_indexes"] = (
-                    completed_candidate_indexes[bytes(row["branch_key"])]
+                normalized["completed_candidate_bitmap"] = encode_candidate_bitmap(
+                    completed_candidate_indexes[bytes(row["branch_key"])],
+                    normalized.get("candidate_count"),
                 )
             normalized_rows.append(normalized)
 
@@ -2806,8 +2858,9 @@ def collect_branch_report(sources: ReportOpeners, request: ReportRequest) -> dic
             {"candidate_index": row["idx"], "republish_count": row["count"]}
             for row in republish_rows
         ],
-        "completed_candidate_indexes": sorted(
-            row["idx"] for row in claim_rows if row["done"]
+        "completed_candidate_bitmap": encode_candidate_bitmap(
+            (row["idx"] for row in claim_rows if row["done"]),
+            (queue_payload or {}).get("candidate_count"),
         ),
         "claims": normalized_claims if request.include_claims else None,
         "claim_summary": _summarize_claims(normalized_claims),

report_terminal.py

@@ -13,6 +13,7 @@
 import time
 
 from report_model import (
+    decode_candidate_bitmap,
     ROOT_PROGRESS_DISPLAY_STATES,
     ROOT_PROGRESS_GROUP_STATES,
     ReportRequest,
@@ -1253,7 +1254,8 @@ def _render_branch_sections(report, previous_report, color, width, display_order
         ]
         sweep = candidate_sweep_bar(
             candidate_count,
-            data.get("completed_candidate_indexes") or (),
+            decode_candidate_bitmap(
+                data.get("completed_candidate_bitmap"), candidate_count),
             worker_positions,
             width=max(10, min(40, width - 4)),
         )