Skip to content

Commit f372681

Browse files
authored
fix: repair last rotted source URLs + allowlist for bot-blocked hosts (#45)
- geos-chem-smoke-composition: geos-chem.org -> geoschem.github.io (the model's canonical, CI-reachable home) - world-bank-productive-use-2023 + mini-grid-productive-use-demand: original pages dead/unreachable; re-sourced to verifiable ESMAP/World Bank mini-grid resources and softened the unverifiable specific percentages (confidence low; figures flagged for dedicated re-sourcing) - verify-sources.mjs: load scripts/source-check-allowlist.json. For URLs confirmed live by manual check but that refuse automated CI requests (cpcb.nic.in, caqm.nic.in), CONNECTION-level failures are downgraded to warnings. A definite HTTP error (404/410) still fails even when allowlisted, so genuine rot is never masked. This clears the scheduled source-health check of false-positives while keeping it a true rot detector. evidence.md ledgers updated to match.
1 parent ada9f84 commit f372681

5 files changed

Lines changed: 74 additions & 22 deletions

File tree

‎problem-packs/air-quality/wildfire-smoke-health-global/evidence.json‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -46,10 +46,10 @@
4646
"evidence_type": "dataset",
4747
"source": {
4848
"title": "GEOS-Chem Global Chemical Transport Model",
49-
"url": "https://geos-chem.org/"
49+
"url": "https://geoschem.github.io"
5050
},
5151
"source_date": "2025-01-01",
52-
"access_date": "2026-06-07",
52+
"access_date": "2026-06-18",
5353
"method": "Reviewed GEOS-Chem model documentation for PM2.5 species resolution, fire-emissions integration (GFED, GFAS), spatial resolution, and validation studies.",
5454
"limitations": [
5555
"GEOS-Chem PM2.5 estimates depend on fire-emissions inventories (GFED, GFAS) that have their own burned-area and emission-factor uncertainties.",

‎problem-packs/energy-access/productive-use-energy-global/evidence.json‎

Lines changed: 16 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -2,22 +2,21 @@
22
{
33
"id": "world-bank-productive-use-2023",
44
"problem_id": "energy-access/productive-use-energy-global",
5-
"claim": "World Bank estimates that electrification for productive use in agriculture can increase rural incomes by 20-30 percent, but fewer than 10 percent of mini-grids in Sub-Saharan Africa serve productive-use customers.",
5+
"claim": "ESMAP's mini-grid market analysis identifies productive use of electricity (agriculture, commerce, light industry) as central to mini-grid commercial viability, while noting it remains underdeveloped in most Sub-Saharan African mini-grids where residential demand dominates.",
66
"evidence_type": "primary-source",
77
"source": {
8-
"title": "Productive Uses of Energy for Rural Development",
9-
"url": "https://www.worldbank.org/en/topic/energy/publication/productive-uses-of-energy"
8+
"title": "ESMAP Global Facility on Mini Grids — Scaling Up Mini Grid Markets (World Bank)",
9+
"url": "https://www.worldbank.org/en/results/2020/11/11/esmap-global-facility-on-mini-grids-scaling-up-mini-grid-markets-to-provide-electricity-to-half-a-billion-people-by-2030"
1010
},
1111
"source_date": "2023-06-01",
12-
"access_date": "2026-06-07",
12+
"access_date": "2026-06-18",
1313
"method": "Reviewed World Bank productive-use framework, income-impact estimates, and mini-grid survey data for Sub-Saharan Africa.",
1414
"limitations": [
15-
"The 20-30 percent income increase is from controlled studies in specific contexts — generalization across agricultural value chains and market environments is uncertain.",
16-
"The 'fewer than 10 percent' figure is from a limited survey of mini-grid operators — actual productive-use rates may differ across operator types and countries.",
17-
"Income-impact estimates depend on baseline income measurement methodology and confound controls that vary across studies.",
18-
"The assessment does not capture productive use that emerges years after electrification — adoption timelines for agricultural equipment may be longer than study follow-up periods."
15+
"Re-sourced June 2026 after the original World Bank topic page returned HTTP 404; the specific '20-30% income increase' and 'fewer than 10% of mini-grids serve productive use' figures were not carried over and need a dedicated source.",
16+
"This page describes program direction, not a peer-reviewed impact estimate.",
17+
"Productive-use adoption varies widely by country and tariff design."
1918
],
20-
"confidence": "high"
19+
"confidence": "low"
2120
},
2221
{
2322
"id": "agricultural-value-chain-electrification",
@@ -42,22 +41,21 @@
4241
{
4342
"id": "mini-grid-productive-use-demand",
4443
"problem_id": "energy-access/productive-use-energy-global",
45-
"claim": "Mini-grid operators in SSA report that residential lighting connections dominate their customer base, with productive-use loads typically representing less than 20 percent of total demand — the business model for rural mini-grids depends on productive-use growth that rarely materializes at projected rates.",
44+
"claim": "ESMAP's Mini Grids market outlook reports that demand on Sub-Saharan African mini-grids is dominated by residential connections, with productive-use loads a minority of demand, and that productive-use growth — a key assumption in mini-grid business models — frequently lags projections.",
4645
"evidence_type": "field-report",
4746
"source": {
48-
"title": "Mini-grid operator survey: productive-use demand in Sub-Saharan Africa",
49-
"url": "https://www.minigridfinancingfacility.org/"
47+
"title": "Mini Grids for Half a Billion People: Market Outlook and Handbook for Decision-Makers (ESMAP, World Bank)",
48+
"url": "https://documents1.worldbank.org/curated/en/099122223180539643/pdf/P17515118ea44a020192941b81f7e8498e1.pdf"
5049
},
5150
"source_date": "2023-03-01",
52-
"access_date": "2026-06-07",
51+
"access_date": "2026-06-18",
5352
"method": "Reviewed mini-grid operator survey methodology, productive-use load data, and business-model assessment across SSA countries.",
5453
"limitations": [
55-
"The survey covers a limited number of mini-grid operators — results may not represent the full diversity of mini-grid business models and contexts.",
56-
"Productive-use load data is self-reported by operators without independent verification.",
57-
"The 'less than 20 percent' figure is an aggregate — some mini-grids serving market-connected areas may have higher productive-use shares.",
58-
"The survey does not capture why productive-use demand remains low — affordability, technical capacity, and market barriers are inferred rather than directly measured."
54+
"Re-sourced June 2026 after the original operator-survey URL (minigridfinancingfacility.org) became unreachable; the specific 'less than 20 percent of demand' figure needs a dedicated operator-survey source.",
55+
"Market-outlook figures are aggregate and vary substantially by country, operator, and tariff design.",
56+
"Demand composition evolves as connections mature, so a single snapshot understates later productive-use uptake."
5957
],
60-
"confidence": "high"
58+
"confidence": "low"
6159
},
6260
{
6361
"id": "enterprise-formation-electrification",

‎problem-packs/energy-access/productive-use-energy-global/evidence.md‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -8,15 +8,15 @@ The machine-readable ledger is `evidence.json`.
88

99
### World Bank Productive Use 2023
1010

11-
Use this source for the income-impact baseline. The 20-30 percent figure is from controlled studies in specific contexts — generalization across value chains and market environments is uncertain. The 'fewer than 10 percent' figure is from a limited operator survey. Income-impact estimates depend on baseline methodology and confound controls. Always document context-dependency and measurement limitations.
11+
Use this source (ESMAP Global Facility on Mini Grids) for program-level context on productive use as a driver of mini-grid viability. Re-sourced June 2026 after the original World Bank topic page rotted (HTTP 404); the prior '20-30 percent income increase' and 'fewer than 10 percent of mini-grids serve productive use' figures need a dedicated peer-reviewed source before reuse.
1212

1313
### Agricultural Value Chain Electrification
1414

1515
Use this source for the strongest evidence on value-chain-specific productive use. Equipment adoption depends on affordability, technical capacity, and market access — electricity alone does not guarantee productive use. Most studies measure ownership rather than actual usage frequency. Publication bias toward positive findings cannot be excluded. Always present with value-chain specificity and adoption-barrier documentation.
1616

1717
### Mini-Grid Productive Use Demand
1818

19-
Use this source to document the demand-side constraint: residential lighting dominates mini-grid customer bases with productive-use loads below 20 percent. The survey covers a limited number of operators. Business-model viability depends on productive-use growth that rarely materializes. Always document operator-survey limitations and the demand-gap context.
19+
Use this source (ESMAP Mini Grids market outlook) to document that residential connections dominate Sub-Saharan African mini-grid demand and productive-use growth often lags projections. Re-sourced June 2026 after the operator-survey URL became unreachable; the specific 'below 20 percent of demand' figure needs a dedicated operator-survey source.
2020

2121
### Enterprise Formation Electrification
2222

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,20 @@
1+
{
2+
"description": "URLs confirmed live by manual/browser check but which refuse automated requests (connection reset/timeout) from CI runners. For these, the scheduled source health check downgrades CONNECTION-LEVEL failures to warnings. A definite HTTP error status (e.g. 404, 410) still fails the check even for an allowlisted URL, so genuine rot is never masked. Re-verify on the 'verified' date cadence.",
3+
"allow": [
4+
{
5+
"url": "https://cpcb.nic.in/national-air-quality-monitoring-programme/",
6+
"reason": "India Central Pollution Control Board NAMP page; live in browser but blocks automated access.",
7+
"verified": "2026-06-18"
8+
},
9+
{
10+
"url": "https://cpcb.nic.in/monitoring-network-3/",
11+
"reason": "CPCB monitoring network page; live in browser but blocks automated access.",
12+
"verified": "2026-06-18"
13+
},
14+
{
15+
"url": "https://caqm.nic.in/",
16+
"reason": "India Commission for Air Quality Management; live in browser but blocks automated access.",
17+
"verified": "2026-06-18"
18+
}
19+
]
20+
}

‎scripts/verify-sources.mjs‎

Lines changed: 34 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -42,7 +42,26 @@ const checkUrl = async (url) => {
4242
throw new Error(`${url} returned HTTP ${get.status}`);
4343
};
4444

45+
const loadAllowlist = async () => {
46+
try {
47+
const raw = await fs.readFile(
48+
path.join(root, "scripts", "source-check-allowlist.json"),
49+
"utf8"
50+
);
51+
const parsed = JSON.parse(raw);
52+
return new Set((parsed.allow || []).map((entry) => entry.url));
53+
} catch {
54+
return new Set();
55+
}
56+
};
57+
58+
// A connection-level failure (DNS/TLS/timeout/reset) is reported by checkUrl as
59+
// "<url> failed: ...". A definite HTTP error status is "<url> returned HTTP <n>".
60+
// Only the former is suppressible via the allowlist; real HTTP errors always fail.
61+
const isConnectionFailure = (message) => !/ returned HTTP /.test(message);
62+
4563
const main = async () => {
64+
const allowlist = await loadAllowlist();
4665
const evidenceFiles = await walkFiles(
4766
path.join(root, "problem-packs"),
4867
(file) => path.basename(file) === "evidence.json"
@@ -61,17 +80,32 @@ const main = async () => {
6180
}
6281

6382
const failures = [];
83+
const warnings = [];
6484
for (const record of records) {
6585
try {
6686
const result = await checkUrl(record.source.url);
6787
const suffix = result.note ? ` (${result.note})` : "";
6888
console.log(`${record.id}: ${result.method} ${result.status}${suffix}`);
6989
} catch (err) {
90+
if (allowlist.has(record.source.url) && isConnectionFailure(err.message)) {
91+
console.warn(
92+
`WARN (allowlisted, unreachable by CI): ${record.id} — ${err.message}`
93+
);
94+
warnings.push({ id: record.id, url: record.source.url, error: err.message });
95+
continue;
96+
}
7097
console.error(`FAIL: ${record.id} — ${err.message}`);
7198
failures.push({ id: record.id, url: record.source.url, error: err.message });
7299
}
73100
}
74101

102+
if (warnings.length > 0) {
103+
console.warn(
104+
`\n${warnings.length} allowlisted URL(s) were unreachable by CI (verify manually):`
105+
);
106+
for (const w of warnings) console.warn(` ${w.id}: ${w.url}`);
107+
}
108+
75109
if (failures.length > 0) {
76110
console.error(`\n${failures.length} URL(s) failed verification:`);
77111
for (const f of failures) {

0 commit comments

Comments
 (0)