fix: final source-rot repairs + bot-block allowlist for health check

problem-packs/air-quality/wildfire-smoke-health-global/evidence.json

@@ -46,10 +46,10 @@
     "evidence_type": "dataset",
     "source": {
       "title": "GEOS-Chem Global Chemical Transport Model",
-      "url": "https://geos-chem.org/"
+      "url": "https://geoschem.github.io"
     },
     "source_date": "2025-01-01",
-    "access_date": "2026-06-07",
+    "access_date": "2026-06-18",
     "method": "Reviewed GEOS-Chem model documentation for PM2.5 species resolution, fire-emissions integration (GFED, GFAS), spatial resolution, and validation studies.",
     "limitations": [
       "GEOS-Chem PM2.5 estimates depend on fire-emissions inventories (GFED, GFAS) that have their own burned-area and emission-factor uncertainties.",

problem-packs/energy-access/productive-use-energy-global/evidence.json

@@ -2,22 +2,21 @@
   {
     "id": "world-bank-productive-use-2023",
     "problem_id": "energy-access/productive-use-energy-global",
-    "claim": "World Bank estimates that electrification for productive use in agriculture can increase rural incomes by 20-30 percent, but fewer than 10 percent of mini-grids in Sub-Saharan Africa serve productive-use customers.",
+    "claim": "ESMAP's mini-grid market analysis identifies productive use of electricity (agriculture, commerce, light industry) as central to mini-grid commercial viability, while noting it remains underdeveloped in most Sub-Saharan African mini-grids where residential demand dominates.",
     "evidence_type": "primary-source",
     "source": {
-      "title": "Productive Uses of Energy for Rural Development",
-      "url": "https://www.worldbank.org/en/topic/energy/publication/productive-uses-of-energy"
+      "title": "ESMAP Global Facility on Mini Grids — Scaling Up Mini Grid Markets (World Bank)",
+      "url": "https://www.worldbank.org/en/results/2020/11/11/esmap-global-facility-on-mini-grids-scaling-up-mini-grid-markets-to-provide-electricity-to-half-a-billion-people-by-2030"
     },
     "source_date": "2023-06-01",
-    "access_date": "2026-06-07",
+    "access_date": "2026-06-18",
     "method": "Reviewed World Bank productive-use framework, income-impact estimates, and mini-grid survey data for Sub-Saharan Africa.",
     "limitations": [
-      "The 20-30 percent income increase is from controlled studies in specific contexts — generalization across agricultural value chains and market environments is uncertain.",
-      "The 'fewer than 10 percent' figure is from a limited survey of mini-grid operators — actual productive-use rates may differ across operator types and countries.",
-      "Income-impact estimates depend on baseline income measurement methodology and confound controls that vary across studies.",
-      "The assessment does not capture productive use that emerges years after electrification — adoption timelines for agricultural equipment may be longer than study follow-up periods."
+      "Re-sourced June 2026 after the original World Bank topic page returned HTTP 404; the specific '20-30% income increase' and 'fewer than 10% of mini-grids serve productive use' figures were not carried over and need a dedicated source.",
+      "This page describes program direction, not a peer-reviewed impact estimate.",
+      "Productive-use adoption varies widely by country and tariff design."
     ],
-    "confidence": "high"
+    "confidence": "low"
   },
   {
     "id": "agricultural-value-chain-electrification",
@@ -42,22 +41,21 @@
   {
     "id": "mini-grid-productive-use-demand",
     "problem_id": "energy-access/productive-use-energy-global",
-    "claim": "Mini-grid operators in SSA report that residential lighting connections dominate their customer base, with productive-use loads typically representing less than 20 percent of total demand — the business model for rural mini-grids depends on productive-use growth that rarely materializes at projected rates.",
+    "claim": "ESMAP's Mini Grids market outlook reports that demand on Sub-Saharan African mini-grids is dominated by residential connections, with productive-use loads a minority of demand, and that productive-use growth — a key assumption in mini-grid business models — frequently lags projections.",
     "evidence_type": "field-report",
     "source": {
-      "title": "Mini-grid operator survey: productive-use demand in Sub-Saharan Africa",
-      "url": "https://www.minigridfinancingfacility.org/"
+      "title": "Mini Grids for Half a Billion People: Market Outlook and Handbook for Decision-Makers (ESMAP, World Bank)",
+      "url": "https://documents1.worldbank.org/curated/en/099122223180539643/pdf/P17515118ea44a020192941b81f7e8498e1.pdf"
     },
     "source_date": "2023-03-01",
-    "access_date": "2026-06-07",
+    "access_date": "2026-06-18",
     "method": "Reviewed mini-grid operator survey methodology, productive-use load data, and business-model assessment across SSA countries.",
     "limitations": [
-      "The survey covers a limited number of mini-grid operators — results may not represent the full diversity of mini-grid business models and contexts.",
-      "Productive-use load data is self-reported by operators without independent verification.",
-      "The 'less than 20 percent' figure is an aggregate — some mini-grids serving market-connected areas may have higher productive-use shares.",
-      "The survey does not capture why productive-use demand remains low — affordability, technical capacity, and market barriers are inferred rather than directly measured."
+      "Re-sourced June 2026 after the original operator-survey URL (minigridfinancingfacility.org) became unreachable; the specific 'less than 20 percent of demand' figure needs a dedicated operator-survey source.",
+      "Market-outlook figures are aggregate and vary substantially by country, operator, and tariff design.",
+      "Demand composition evolves as connections mature, so a single snapshot understates later productive-use uptake."
     ],
-    "confidence": "high"
+    "confidence": "low"
   },
   {
     "id": "enterprise-formation-electrification",

problem-packs/energy-access/productive-use-energy-global/evidence.md

@@ -8,15 +8,15 @@ The machine-readable ledger is `evidence.json`.
 
 ### World Bank Productive Use 2023
 
-Use this source for the income-impact baseline. The 20-30 percent figure is from controlled studies in specific contexts — generalization across value chains and market environments is uncertain. The 'fewer than 10 percent' figure is from a limited operator survey. Income-impact estimates depend on baseline methodology and confound controls. Always document context-dependency and measurement limitations.
+Use this source (ESMAP Global Facility on Mini Grids) for program-level context on productive use as a driver of mini-grid viability. Re-sourced June 2026 after the original World Bank topic page rotted (HTTP 404); the prior '20-30 percent income increase' and 'fewer than 10 percent of mini-grids serve productive use' figures need a dedicated peer-reviewed source before reuse.
 
 ### Agricultural Value Chain Electrification
 
 Use this source for the strongest evidence on value-chain-specific productive use. Equipment adoption depends on affordability, technical capacity, and market access — electricity alone does not guarantee productive use. Most studies measure ownership rather than actual usage frequency. Publication bias toward positive findings cannot be excluded. Always present with value-chain specificity and adoption-barrier documentation.
 
 ### Mini-Grid Productive Use Demand
 
-Use this source to document the demand-side constraint: residential lighting dominates mini-grid customer bases with productive-use loads below 20 percent. The survey covers a limited number of operators. Business-model viability depends on productive-use growth that rarely materializes. Always document operator-survey limitations and the demand-gap context.
+Use this source (ESMAP Mini Grids market outlook) to document that residential connections dominate Sub-Saharan African mini-grid demand and productive-use growth often lags projections. Re-sourced June 2026 after the operator-survey URL became unreachable; the specific 'below 20 percent of demand' figure needs a dedicated operator-survey source.
 
 ### Enterprise Formation Electrification
 

scripts/source-check-allowlist.json

@@ -0,0 +1,20 @@
+{
+  "description": "URLs confirmed live by manual/browser check but which refuse automated requests (connection reset/timeout) from CI runners. For these, the scheduled source health check downgrades CONNECTION-LEVEL failures to warnings. A definite HTTP error status (e.g. 404, 410) still fails the check even for an allowlisted URL, so genuine rot is never masked. Re-verify on the 'verified' date cadence.",
+  "allow": [
+    {
+      "url": "https://cpcb.nic.in/national-air-quality-monitoring-programme/",
+      "reason": "India Central Pollution Control Board NAMP page; live in browser but blocks automated access.",
+      "verified": "2026-06-18"
+    },
+    {
+      "url": "https://cpcb.nic.in/monitoring-network-3/",
+      "reason": "CPCB monitoring network page; live in browser but blocks automated access.",
+      "verified": "2026-06-18"
+    },
+    {
+      "url": "https://caqm.nic.in/",
+      "reason": "India Commission for Air Quality Management; live in browser but blocks automated access.",
+      "verified": "2026-06-18"
+    }
+  ]
+}

scripts/verify-sources.mjs

@@ -42,7 +42,26 @@ const checkUrl = async (url) => {
   throw new Error(`${url} returned HTTP ${get.status}`);
 };
 
+const loadAllowlist = async () => {
+  try {
+    const raw = await fs.readFile(
+      path.join(root, "scripts", "source-check-allowlist.json"),
+      "utf8"
+    );
+    const parsed = JSON.parse(raw);
+    return new Set((parsed.allow || []).map((entry) => entry.url));
+  } catch {
+    return new Set();
+  }
+};
+
+// A connection-level failure (DNS/TLS/timeout/reset) is reported by checkUrl as
+// "<url> failed: ...". A definite HTTP error status is "<url> returned HTTP <n>".
+// Only the former is suppressible via the allowlist; real HTTP errors always fail.
+const isConnectionFailure = (message) => !/ returned HTTP /.test(message);
+
 const main = async () => {
+  const allowlist = await loadAllowlist();
   const evidenceFiles = await walkFiles(
     path.join(root, "problem-packs"),
     (file) => path.basename(file) === "evidence.json"
@@ -61,17 +80,32 @@ const main = async () => {
   }
 
   const failures = [];
+  const warnings = [];
   for (const record of records) {
     try {
       const result = await checkUrl(record.source.url);
       const suffix = result.note ? ` (${result.note})` : "";
       console.log(`${record.id}: ${result.method} ${result.status}${suffix}`);
     } catch (err) {
+      if (allowlist.has(record.source.url) && isConnectionFailure(err.message)) {
+        console.warn(
+          `WARN (allowlisted, unreachable by CI): ${record.id} — ${err.message}`
+        );
+        warnings.push({ id: record.id, url: record.source.url, error: err.message });
+        continue;
+      }
       console.error(`FAIL: ${record.id} — ${err.message}`);
       failures.push({ id: record.id, url: record.source.url, error: err.message });
     }
   }
 
+  if (warnings.length > 0) {
+    console.warn(
+      `\n${warnings.length} allowlisted URL(s) were unreachable by CI (verify manually):`
+    );
+    for (const w of warnings) console.warn(`  ${w.id}: ${w.url}`);
+  }
+
   if (failures.length > 0) {
     console.error(`\n${failures.length} URL(s) failed verification:`);
     for (const f of failures) {