From bc4c6303ec4e64debb7e9f4bb43944834b526315 Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 5 Oct 2026 07:59:01 +0000 Subject: [PATCH] Report a failed coverage check as unknown, and treat the iBabs timeout as one A coverage check that failed stored 0/0/0 and /api/status published them as a ratio of 1, so Texel read as complete while its supplier timed out. With `error` set, supplierDocuments, heldDocuments, missingDocuments and ratio are now null. iBabs reports its own database timeout as Status=ERR in an HTTP 200, worded "Execution Timeout Expired". isSoapTimeout only matched "timed out", so the adaptive window halving never applied to it. --- API.md | 2 +- src/ibabs/extractor.ts | 10 +++++++++- src/types.ts | 10 ++++++---- tests/coverage_check.test.ts | 33 +++++++++++++++++++++++++++++++++ tests/ibabs.test.ts | 14 ++++++++++++++ web/status_api.ts | 19 ++++++++++++++++++- 6 files changed, 81 insertions(+), 7 deletions(-) diff --git a/API.md b/API.md index 07528207..d15c4354 100644 --- a/API.md +++ b/API.md @@ -407,7 +407,7 @@ at all — i.e. how long the outage has lasted. | `latestContentDate` | Newest meeting date held for this organization. Often in the future — an agenda is published before the meeting happens | | `lastIndexedAt` | When anything was last written to the search index for this organization | | `indexedDocuments` | Documents in the search index for this organization, exact at the moment of the request: one per document, pages not counted. The number to reconcile against the source system's own list. Absent when the index did not answer | -| `coverage` | Present once the weekly coverage check has covered this organization. What the source system's own API listed for a date window, against what the index holds: `supplierDocuments`, `heldDocuments`, `missingDocuments` (the first two partition the third), `ratio` (held over supplier; 1 means complete), `windowFrom`/`windowTo`, `checkedAt`, `missingSample` (a few missing document ids), `lowerBound` (true when some supplier requests failed, so the gap may be larger), and `error` when the check itself failed. `state: "ok"` says the last import ran; `coverage` says whether it asked for everything | +| `coverage` | Present once the weekly coverage check has covered this organization. What the source system's own API listed for a date window, against what the index holds: `supplierDocuments`, `heldDocuments`, `missingDocuments` (the first two partition the third), `ratio` (held over supplier; 1 means complete), `windowFrom`/`windowTo`, `checkedAt`, `missingSample` (a few missing document ids), `lowerBound` (true when some supplier requests failed, so the gap may be larger), and `error` when the check itself failed. When `error` is set the four numbers `supplierDocuments`, `heldDocuments`, `missingDocuments` and `ratio` are `null`: the check measured nothing, which is not the same as measuring zero. `state: "ok"` says the last import ran; `coverage` says whether it asked for everything | | `discontinuedAt` | The date the organization ceased to exist. Only on `discontinued` | | `succeededBy` | `{ cbsId, label, sourceKey }` of the organization that took over. `sourceKey` is absent when we do not import the successor. Only on `discontinued` | diff --git a/src/ibabs/extractor.ts b/src/ibabs/extractor.ts index 12c0b8e7..d34f2f12 100644 --- a/src/ibabs/extractor.ts +++ b/src/ibabs/extractor.ts @@ -63,7 +63,15 @@ function isSoapTimeout(error: unknown): boolean { if (!(error instanceof Error)) return false; if (error.name === "TimeoutError" || error.name === "AbortError") return true; const msg = error.message.toLowerCase(); - return msg.includes("signal timed out") || msg.includes("timed out"); + // iBabs reports its own database timeout as `Status=ERR` inside an HTTP 200, + // worded "Execution Timeout Expired", which contains neither "timed out" nor + // arrives as a TimeoutError. It is the same condition, a window too heavy to + // answer in time, so halving the window applies to it as well (#309). + return ( + msg.includes("signal timed out") || + msg.includes("timed out") || + msg.includes("execution timeout expired") + ); } async function listMeetingsAdaptive( diff --git a/src/types.ts b/src/types.ts index f68a959c..0419ba7a 100644 --- a/src/types.ts +++ b/src/types.ts @@ -936,10 +936,12 @@ export interface SourceCoverage { checkedAt: string; windowFrom: string; windowTo: string; - supplierDocuments: number; - heldDocuments: number; - missingDocuments: number; - ratio: number; + /** `null` in this and the next three fields when `error` is set: the check + * measured nothing, which is not the same as measuring zero. */ + supplierDocuments: number | null; + heldDocuments: number | null; + missingDocuments: number | null; + ratio: number | null; lowerBound: boolean; /** A few of the missing document ids, for a human to verify. */ missingSample: string[]; diff --git a/tests/coverage_check.test.ts b/tests/coverage_check.test.ts index da2ef38d..dd6ac54c 100644 --- a/tests/coverage_check.test.ts +++ b/tests/coverage_check.test.ts @@ -134,3 +134,36 @@ Deno.test("a recorded check surfaces on the status response as coverage", async const other = response.sources.find((row) => row.sourceKey === "dongen"); assertEquals(other?.coverage, undefined, "no check, no field"); }); + +Deno.test("a failed check publishes no numbers instead of zeros", async () => { + // The shape of #309: the supplier timed out, the check stored 0/0/0 and the + // status read as a ratio of 1 for a source holding 5,890 documents. + await recordCoverageCheck({ + source_key: "texel", + checked_at: "2026-09-12T09:25:50.606Z", + window_from: "2025-09-11", + window_to: "2026-09-11", + supplier_documents: 0, + held_documents: 0, + missing_documents: 0, + missing_sample: [], + supplier_meetings: 0, + register_entries: 0, + warnings: 0, + error: "Execution Timeout Expired.", + }); + const response = statusTest.buildStatusResponse({ + runStatus: { sources: [], supplierWindows: [] }, + indexActivity: null, + coverageChecks: await latestCoverageChecks(), + now: Date.parse("2026-09-14T12:00:00Z"), + windowHours: 36, + }); + const coverage = response.sources.find((row) => row.sourceKey === "texel")?.coverage; + assertEquals(coverage?.error, "Execution Timeout Expired."); + assertEquals(coverage?.supplierDocuments, null); + assertEquals(coverage?.heldDocuments, null); + assertEquals(coverage?.missingDocuments, null); + assertEquals(coverage?.ratio, null); + assertEquals(coverage?.lowerBound, true); +}); diff --git a/tests/ibabs.test.ts b/tests/ibabs.test.ts index 4dd255a7..da8f7bfc 100644 --- a/tests/ibabs.test.ts +++ b/tests/ibabs.test.ts @@ -222,6 +222,20 @@ Deno.test("listMeetingsAdaptive halves the chunk on SOAP timeout", async () => { assert(calls.length === 3, "one failed full-range call + two half-range calls"); }); +Deno.test("isSoapTimeout also recognises the database timeout iBabs returns as ERR", () => { + const { isSoapTimeout } = ibabsExtractorTest; + // Worded as in #309; it arrives as an ordinary Error, not a TimeoutError. + assert( + isSoapTimeout( + new Error( + "Execution Timeout Expired. The timeout period elapsed prior to completion of the operation or the server is not responding.", + ), + ), + "the iBabs database timeout is a timeout", + ); + assert(!isSoapTimeout(new Error("No public account!")), "other errors are not"); +}); + Deno.test("listMeetingsAdaptive stops splitting below the floor and rethrows", async () => { const { listMeetingsAdaptive } = ibabsExtractorTest; const source = getIbabsSource("amstelveen"); diff --git a/web/status_api.ts b/web/status_api.ts index 3bb7f261..d631d429 100644 --- a/web/status_api.ts +++ b/web/status_api.ts @@ -196,6 +196,24 @@ let cached: { value: StatusResponse; expiresAt: number } | null = null; let inFlight: Promise | null = null; function toSourceCoverage(check: CoverageCheckRecord): SourceCoverage { + // A check that failed measured nothing. The stored zeros are the absence of an + // answer, and published as numbers they read as "the supplier lists nothing" + // with a ratio of 1, which is how Texel came to be reported as complete while + // its supplier timed out (#309). `null` is what "unknown" looks like. + if (check.error) { + return { + checkedAt: check.checked_at, + windowFrom: check.window_from, + windowTo: check.window_to, + supplierDocuments: null, + heldDocuments: null, + missingDocuments: null, + ratio: null, + lowerBound: true, + missingSample: [], + error: check.error, + }; + } return { checkedAt: check.checked_at, windowFrom: check.window_from, @@ -209,7 +227,6 @@ function toSourceCoverage(check: CoverageCheckRecord): SourceCoverage { : 1, lowerBound: check.warnings > 0, missingSample: check.missing_sample, - error: check.error, }; }