Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion API.md
Original file line number Diff line number Diff line change
Expand Up @@ -407,7 +407,7 @@ at all — i.e. how long the outage has lasted.
| `latestContentDate` | Newest meeting date held for this organization. Often in the future — an agenda is published before the meeting happens |
| `lastIndexedAt` | When anything was last written to the search index for this organization |
| `indexedDocuments` | Documents in the search index for this organization, exact at the moment of the request: one per document, pages not counted. The number to reconcile against the source system's own list. Absent when the index did not answer |
| `coverage` | Present once the weekly coverage check has covered this organization. What the source system's own API listed for a date window, against what the index holds: `supplierDocuments`, `heldDocuments`, `missingDocuments` (the first two partition the third), `ratio` (held over supplier; 1 means complete), `windowFrom`/`windowTo`, `checkedAt`, `missingSample` (a few missing document ids), `lowerBound` (true when some supplier requests failed, so the gap may be larger), and `error` when the check itself failed. `state: "ok"` says the last import ran; `coverage` says whether it asked for everything |
| `coverage` | Present once the weekly coverage check has covered this organization. What the source system's own API listed for a date window, against what the index holds: `supplierDocuments`, `heldDocuments`, `missingDocuments` (the first two partition the third), `ratio` (held over supplier; 1 means complete), `windowFrom`/`windowTo`, `checkedAt`, `missingSample` (a few missing document ids), `lowerBound` (true when some supplier requests failed, so the gap may be larger), and `error` when the check itself failed. When `error` is set the four numbers `supplierDocuments`, `heldDocuments`, `missingDocuments` and `ratio` are `null`: the check measured nothing, which is not the same as measuring zero. `state: "ok"` says the last import ran; `coverage` says whether it asked for everything |
| `discontinuedAt` | The date the organization ceased to exist. Only on `discontinued` |
| `succeededBy` | `{ cbsId, label, sourceKey }` of the organization that took over. `sourceKey` is absent when we do not import the successor. Only on `discontinued` |

Expand Down
10 changes: 9 additions & 1 deletion src/ibabs/extractor.ts
Original file line number Diff line number Diff line change
Expand Up @@ -63,7 +63,15 @@ function isSoapTimeout(error: unknown): boolean {
if (!(error instanceof Error)) return false;
if (error.name === "TimeoutError" || error.name === "AbortError") return true;
const msg = error.message.toLowerCase();
return msg.includes("signal timed out") || msg.includes("timed out");
// iBabs reports its own database timeout as `Status=ERR` inside an HTTP 200,
// worded "Execution Timeout Expired", which contains neither "timed out" nor
// arrives as a TimeoutError. It is the same condition, a window too heavy to
// answer in time, so halving the window applies to it as well (#309).
return (
msg.includes("signal timed out") ||
msg.includes("timed out") ||
msg.includes("execution timeout expired")
);
}

async function listMeetingsAdaptive(
Expand Down
10 changes: 6 additions & 4 deletions src/types.ts
Original file line number Diff line number Diff line change
Expand Up @@ -936,10 +936,12 @@ export interface SourceCoverage {
checkedAt: string;
windowFrom: string;
windowTo: string;
supplierDocuments: number;
heldDocuments: number;
missingDocuments: number;
ratio: number;
/** `null` in this and the next three fields when `error` is set: the check
* measured nothing, which is not the same as measuring zero. */
supplierDocuments: number | null;
heldDocuments: number | null;
missingDocuments: number | null;
ratio: number | null;
lowerBound: boolean;
/** A few of the missing document ids, for a human to verify. */
missingSample: string[];
Expand Down
33 changes: 33 additions & 0 deletions tests/coverage_check.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -134,3 +134,36 @@ Deno.test("a recorded check surfaces on the status response as coverage", async
const other = response.sources.find((row) => row.sourceKey === "dongen");
assertEquals(other?.coverage, undefined, "no check, no field");
});

Deno.test("a failed check publishes no numbers instead of zeros", async () => {
// The shape of #309: the supplier timed out, the check stored 0/0/0 and the
// status read as a ratio of 1 for a source holding 5,890 documents.
await recordCoverageCheck({
source_key: "texel",
checked_at: "2026-09-12T09:25:50.606Z",
window_from: "2025-09-11",
window_to: "2026-09-11",
supplier_documents: 0,
held_documents: 0,
missing_documents: 0,
missing_sample: [],
supplier_meetings: 0,
register_entries: 0,
warnings: 0,
error: "Execution Timeout Expired.",
});
const response = statusTest.buildStatusResponse({
runStatus: { sources: [], supplierWindows: [] },
indexActivity: null,
coverageChecks: await latestCoverageChecks(),
now: Date.parse("2026-09-14T12:00:00Z"),
windowHours: 36,
});
const coverage = response.sources.find((row) => row.sourceKey === "texel")?.coverage;
assertEquals(coverage?.error, "Execution Timeout Expired.");
assertEquals(coverage?.supplierDocuments, null);
assertEquals(coverage?.heldDocuments, null);
assertEquals(coverage?.missingDocuments, null);
assertEquals(coverage?.ratio, null);
assertEquals(coverage?.lowerBound, true);
});
14 changes: 14 additions & 0 deletions tests/ibabs.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -222,6 +222,20 @@ Deno.test("listMeetingsAdaptive halves the chunk on SOAP timeout", async () => {
assert(calls.length === 3, "one failed full-range call + two half-range calls");
});

Deno.test("isSoapTimeout also recognises the database timeout iBabs returns as ERR", () => {
const { isSoapTimeout } = ibabsExtractorTest;
// Worded as in #309; it arrives as an ordinary Error, not a TimeoutError.
assert(
isSoapTimeout(
new Error(
"Execution Timeout Expired. The timeout period elapsed prior to completion of the operation or the server is not responding.",
),
),
"the iBabs database timeout is a timeout",
);
assert(!isSoapTimeout(new Error("No public account!")), "other errors are not");
});

Deno.test("listMeetingsAdaptive stops splitting below the floor and rethrows", async () => {
const { listMeetingsAdaptive } = ibabsExtractorTest;
const source = getIbabsSource("amstelveen");
Expand Down
19 changes: 18 additions & 1 deletion web/status_api.ts
Original file line number Diff line number Diff line change
Expand Up @@ -196,6 +196,24 @@ let cached: { value: StatusResponse; expiresAt: number } | null = null;
let inFlight: Promise<StatusResponse> | null = null;

function toSourceCoverage(check: CoverageCheckRecord): SourceCoverage {
// A check that failed measured nothing. The stored zeros are the absence of an
// answer, and published as numbers they read as "the supplier lists nothing"
// with a ratio of 1, which is how Texel came to be reported as complete while
// its supplier timed out (#309). `null` is what "unknown" looks like.
if (check.error) {
return {
checkedAt: check.checked_at,
windowFrom: check.window_from,
windowTo: check.window_to,
supplierDocuments: null,
heldDocuments: null,
missingDocuments: null,
ratio: null,
lowerBound: true,
missingSample: [],
error: check.error,
};
}
return {
checkedAt: check.checked_at,
windowFrom: check.window_from,
Expand All @@ -209,7 +227,6 @@ function toSourceCoverage(check: CoverageCheckRecord): SourceCoverage {
: 1,
lowerBound: check.warnings > 0,
missingSample: check.missing_sample,
error: check.error,
};
}

Expand Down
Loading