Skip to content

Commit d04468b

Browse files
Merge pull request #562 from appdevforall/feat/K2GO-394-maps-resilient-download
K2GO-394 feat(maps): resilient base-map download via dash-node
2 parents 8b1f058 + ecad64b commit d04468b

54 files changed

Lines changed: 1044 additions & 114 deletions

File tree

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

‎.github/workflows/publish-catalogs.yml‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -38,6 +38,7 @@ jobs:
3838
python3 tools/build_kolibri_catalog.py || echo "::warning::kolibri generator failed; its upload will be skipped"
3939
python3 tools/build_kolibri_catalog.py --tree topics && gzip -9 -n -f src/main/assets/kolibri_tree.jsonl || echo "::warning::kolibri tree generator failed; its upload will be skipped"
4040
python3 tools/build_kiwix_catalog.py || echo "::warning::kiwix generator failed; its upload will be skipped"
41+
python3 tools/build_maps_catalog.py || echo "::warning::maps generator failed; its upload will be skipped"
4142
4243
- name: Build manifests and upload to Cloudflare R2
4344
env:
@@ -86,3 +87,4 @@ jobs:
8687
publish kolibri src/main/assets/kolibri_catalog.jsonl "application/x-ndjson" jsonl
8788
publish kolibri-tree src/main/assets/kolibri_tree.jsonl.gz "application/gzip" none
8889
publish kiwix src/main/assets/kiwix_catalog.csv "text/csv" none
90+
publish maps src/main/assets/maps_catalog.csv "text/csv" none

‎controller/app/build.gradle‎

Lines changed: 22 additions & 80 deletions
Original file line numberDiff line numberDiff line change
@@ -637,91 +637,33 @@ task refreshModuleSizes {
637637

638638
preBuild.dependsOn refreshModuleSizes
639639

640-
// ---- maps layer size auto-capture (offline fallback) --------------------------
641-
// Fetches the <size> from each map layer's .meta4 on the maps mirror and writes
642-
// src/main/assets/maps_sizes.csv, so the Choose screen's OFFLINE fallback (MapsCatalog)
643-
// is captured at package time instead of hand-maintained — same pattern as rootfs above.
644-
// The file names follow roles/maps/defaults/main.yml; update the three data dates below
645-
// when the role bumps them. Non-fatal: an incomplete fetch keeps the committed CSV.
646-
task refreshMapsSizes {
640+
// ---- maps catalog refresh (release / manual) ----------------------------------
641+
// Regenerates src/main/assets/maps_catalog.csv (group,level,file,bytes,date) from the
642+
// maps mirror's .meta4 pointers via tools/build_maps_catalog.py -- the ONE builder for
643+
// this catalog (it replaced the old inline refreshMapsSizes fetch; no double build path).
644+
// Like the Kiwix catalog it does NOT run on every build: it runs on a RELEASE and on
645+
// demand (./gradlew refreshMapsCatalog), and the weekly publish-catalogs workflow runs
646+
// the same script to publish the CSV + manifest to Cloudflare. The committed CSV is the
647+
// offline floor -- MapsCatalog reads it for the Choose size AND the download file id.
648+
// Never fails the build: if python3 is missing or the fetch is blocked, the committed
649+
// CSV (last known) is kept.
650+
task refreshMapsCatalog {
647651
group = 'build setup'
648-
description = 'Capture map layer sizes (.meta4) into assets/maps_sizes.csv for the offline fallback'
649-
def csv = file('src/main/assets/maps_sizes.csv')
652+
description = 'Regenerate assets/maps_catalog.csv from the maps mirror .meta4 (release / manual)'
650653
doLast {
651-
def base = 'https://iiab.switnet.org/maps/2/'
652-
// Data dates from roles/maps/defaults/main.yml (maps_vector_data_date / maps_slow_data_date
653-
// / maps_search_data_date). Keep in sync with the role.
654-
def vectorDate = '2026-07-01'
655-
def slowDate = '2025-12-10'
656-
def searchDate = '2026-04-22'
657-
// "group|level" -> mirror file name (whole-world pmtiles; search is a tarball).
658-
def files = [
659-
'base|nat-z8' : "naturalearth-openmaptiles.${slowDate}.z00-z08.pmtiles",
660-
'base|11' : "openstreetmap-openmaptiles.${vectorDate}.z00-z11.pmtiles",
661-
'base|14' : "openstreetmap-openmaptiles.${vectorDate}.z00-z14.pmtiles",
662-
'satellite|7' : "s2maps-sentinel2-2023.${slowDate}.z00-z07.pmtiles",
663-
'satellite|9' : "s2maps-sentinel2-2023.${slowDate}.z00-z09.pmtiles",
664-
'satellite|11' : "s2maps-sentinel2-2023.${slowDate}.z00-z11.pmtiles",
665-
'satellite|13' : "s2maps-sentinel2-2023.${slowDate}.z00-z13.pmtiles",
666-
'terrain|7' : "terrarium.${slowDate}.z00-z07.pmtiles",
667-
'terrain|8' : "terrarium.${slowDate}.z00-z08.pmtiles",
668-
'terrain|9' : "terrarium.${slowDate}.z00-z09.pmtiles",
669-
'terrain|10' : "terrarium.${slowDate}.z00-z10.pmtiles",
670-
'search|pop-1k-cities' : "static-search.${searchDate}.pop-1k-cities.tar.gz",
671-
]
672-
def sizePat = ~/<size>(\d+)<\/size>/
673-
def fetched = [:]
674-
files.each { k, fname ->
675-
def url = "${base}${fname}.meta4"
676-
try {
677-
def conn = (HttpURLConnection) new URL(url).openConnection()
678-
conn.setRequestProperty('User-Agent', 'iiab-android-build')
679-
conn.connectTimeout = 6000
680-
conn.readTimeout = 6000
681-
if (conn.responseCode != 200) {
682-
logger.warn(">> [maps sizes] HTTP ${conn.responseCode} for ${url}")
683-
conn.disconnect(); return
684-
}
685-
def body = conn.inputStream.getText('UTF-8')
686-
conn.disconnect()
687-
def m = sizePat.matcher(body)
688-
if (m.find()) { fetched[k] = m.group(1) }
689-
else { logger.warn(">> [maps sizes] no <size> element in ${url}") }
690-
} catch (Exception e) {
691-
logger.warn(">> [maps sizes] fetch failed for ${url}: ${e.message}")
692-
}
693-
}
694-
if (fetched.size() != files.size()) {
695-
logger.warn(">> [maps sizes] incomplete fetch (${fetched.size()}/${files.size()}); keeping existing ${csv.name} (last known).")
696-
return
697-
}
698-
def existing = [:]
699-
if (csv.exists()) {
700-
csv.eachLine { line ->
701-
line = line.trim()
702-
if (line.isEmpty() || line.startsWith('#')) return
703-
def p = line.split(',')
704-
if (p.length >= 3) existing["${p[0].trim()}|${p[1].trim()}".toString()] = p[2].trim()
705-
}
706-
}
707-
if (existing == fetched) {
708-
println ">> [maps sizes] up to date (${fetched.size()} entries unchanged)."
709-
return
710-
}
711-
def today = new Date().format('yyyy-MM-dd')
712-
def rows = files.keySet().collect { k ->
713-
def gl = k.split('\\|', 2)
714-
"${gl[0]},${gl[1]},${fetched[k]},${today}".toString()
654+
try {
655+
def proc = ['python3', 'tools/build_maps_catalog.py'].execute(null, projectDir)
656+
proc.consumeProcessOutput(System.out, System.err)
657+
proc.waitFor()
658+
if (proc.exitValue() != 0) logger.warn(">> [maps catalog] generator exit ${proc.exitValue()}; kept existing CSV.")
659+
} catch (Exception e) {
660+
logger.warn(">> [maps catalog] skipped (${e.message}); kept existing CSV.")
715661
}
716-
if (!csv.parentFile.exists()) csv.parentFile.mkdirs()
717-
csv.text = '# group,level,bytes,fetched_date -- AUTO-GENERATED by the refreshMapsSizes Gradle task; do not edit by hand\n' +
718-
'# Whole-world pmtiles from https://iiab.switnet.org/maps/2 (roles/maps/defaults dates).\n' +
719-
rows.join('\n') + '\n'
720-
println ">> [maps sizes] updated ${rows.size()} entries in ${csv.name} (${today})."
721662
}
722663
}
723-
724-
preBuild.dependsOn refreshMapsSizes
664+
tasks.matching { it.name == 'assembleRelease' || it.name == 'bundleRelease' }.configureEach {
665+
dependsOn refreshMapsCatalog
666+
}
725667

726668
// ---- Kiwix ZIM catalog refresh (release-only) ---------------------------------
727669
// Regenerates src/main/assets/kiwix_catalog.csv from the Kiwix directory listings via
Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,14 @@
1+
# group,level,file,bytes,fetched_date -- AUTO-GENERATED by tools/build_maps_catalog.py; do not edit by hand
2+
# Whole-world pmtiles from https://iiab.switnet.org/maps/2 (roles/maps/defaults dates). The app sends 'file' as the download id; the box composes the URL.
3+
base,nat-z8,naturalearth-openmaptiles.2025-12-10.z00-z08.pmtiles,89435511,2026-09-07
4+
base,11,openstreetmap-openmaptiles.2026-07-01.z00-z11.pmtiles,9010265257,2026-09-07
5+
base,14,openstreetmap-openmaptiles.2026-07-01.z00-z14.pmtiles,87411164966,2026-09-07
6+
satellite,7,s2maps-sentinel2-2023.2025-12-10.z00-z07.pmtiles,88772638,2026-09-07
7+
satellite,9,s2maps-sentinel2-2023.2025-12-10.z00-z09.pmtiles,1247283362,2026-09-07
8+
satellite,11,s2maps-sentinel2-2023.2025-12-10.z00-z11.pmtiles,22312696177,2026-09-07
9+
satellite,13,s2maps-sentinel2-2023.2025-12-10.z00-z13.pmtiles,290318099827,2026-09-07
10+
terrain,7,terrarium.2025-12-10.z00-z07.pmtiles,978219778,2026-09-07
11+
terrain,8,terrarium.2025-12-10.z00-z08.pmtiles,6427331113,2026-09-07
12+
terrain,9,terrarium.2025-12-10.z00-z09.pmtiles,28621213143,2026-09-07
13+
terrain,10,terrarium.2025-12-10.z00-z10.pmtiles,106207946492,2026-09-07
14+
search,pop-1k-cities,static-search.2026-04-22.pop-1k-cities.tar.gz,16111581,2026-09-07

‎controller/app/src/main/assets/maps_sizes.csv‎

Lines changed: 0 additions & 14 deletions
This file was deleted.

‎controller/app/src/main/java/org/appdevforall/k2go/install/domain/MapsRunroleCommand.java‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -14,6 +14,10 @@
1414
* RUNTIME from the completion marker in iiab_state.yml -- --reinstall over a completed or
1515
* base-seeded install, plain runrole to recover a half-done one (a bare --reinstall errors
1616
* when the marker was already deleted by a prior failed --reinstall run).
17+
* K2GO-394: base-map pmtiles are downloaded by dash-node (the durable job engine)
18+
* BEFORE this runs, so the role's is_proot download task only asserts the files are
19+
* present and skips via creates:. This command carries no download handshake -- it is
20+
* the plain post-processing runrole.
1721
* ============================================================================
1822
*/
1923
package org.appdevforall.k2go.install.domain;

0 commit comments

Comments
 (0)