Spiral workspace ergo

lasagna/fit.py

@@ -1290,6 +1290,7 @@ def _run_flatten_mode(
 	opt_cfg: cli_opt.OptConfig,
 	progress_enabled: bool,
 	out_dir: str | None,
+	lifecycle_fn=None,
 ) -> int:
 	ext_surfaces_cfg = cfg.get("external_surfaces", None)
 	if not isinstance(ext_surfaces_cfg, list) or len(ext_surfaces_cfg) != 1:
@@ -1414,6 +1415,8 @@ def _progress(*, step: int, total: int, loss: float, **_kw: object) -> None:
 		seed_xyz=None,
 		out_dir=out_dir,
 	)
+	if lifecycle_fn is not None:
+		lifecycle_fn("saving", "Saving optimized flatten model")
 
 	if device.type == "cuda":
 		peak_gb = torch.cuda.max_memory_allocated(device) / 2**30
@@ -1441,7 +1444,7 @@ def _progress(*, step: int, total: int, loss: float, **_kw: object) -> None:
 	return 0
 
 
-def main(argv: list[str] | None = None) -> int:
+def main(argv: list[str] | None = None, *, lifecycle_fn=None) -> int:
 	if argv is None:
 		argv = sys.argv[1:]
 
@@ -1490,6 +1493,7 @@ def main(argv: list[str] | None = None) -> int:
 			opt_cfg=opt_cfg,
 			progress_enabled=progress_enabled,
 			out_dir=_out_dir,
+			lifecycle_fn=lifecycle_fn,
 		)
 
 	data_cfg = cli_data.from_args(args)
@@ -2420,6 +2424,8 @@ def _progress(*, step: int, total: int, loss: float, **_kw: object) -> None:
 		require_snap_surf_map_state=(model_init == "model" and self_map_init != "off"),
 	)
 	_stage_done("optimizer", _t)
+	if lifecycle_fn is not None:
+		lifecycle_fn("saving", "Saving optimized Lasagna model")
 
 	if device.type == "cuda":
 		peak_gb = torch.cuda.max_memory_allocated(device) / 2**30

lasagna/fit2tifxyz.py

@@ -30,6 +30,8 @@ class ExportConfig:
 	voxel_size_um: float | None = None
 	target_volume_shape_zyx: tuple[int, int, int] | None = None
 	flow_gate_channels: str = "auto"
+	omit_model: bool = False
+	flatten_map_output: str | None = None
 
 
 def _valid_xyz_mask(x: np.ndarray, y: np.ndarray, z: np.ndarray) -> np.ndarray:
@@ -52,6 +54,10 @@ def _build_parser() -> argparse.ArgumentParser:
 		help="Export all windings into a single tifxyz")
 	g.add_argument("--copy-model", action="store_true", default=False,
 		help="Deprecated no-op; model checkpoints are always copied")
+	g.add_argument("--omit-model", action="store_true", default=False,
+		help="Do not copy the model checkpoint into exported tifxyz directories")
+	g.add_argument("--flatten-map-output", default=None,
+		help="For flatten checkpoints, write the output-to-source map as a .npy sidecar")
 	g.add_argument("--output-name", default=None, help="Override tifxyz directory name")
 	g.add_argument("--voxel-size-um", type=float, default=None,
 		help="Voxel size in micrometers (for area calculation)")
@@ -1001,6 +1007,10 @@ def _export_flatten_checkpoint(
 	map_yx = _as_numpy_float32(st["flatten_map_flat"], name="flatten_map_flat")
 	if map_yx.ndim != 3 or map_yx.shape[-1] != 2:
 		raise ValueError("flatten_map_flat must have shape (H, W, 2)")
+	if cfg.flatten_map_output:
+		map_output = Path(cfg.flatten_map_output)
+		map_output.parent.mkdir(parents=True, exist_ok=True)
+		np.save(str(map_output), map_yx, allow_pickle=False)
 
 	mesh = st.get("mesh_flat")
 	if mesh is None:
@@ -1068,7 +1078,7 @@ def _export_flatten_checkpoint(
 		z=z,
 		d=d,
 		scale=meta_scale,
-		model_source=Path(cfg.input),
+		model_source=None if cfg.omit_model else Path(cfg.input),
 		copy_model=cfg.copy_model,
 		fit_config=fit_config,
 		job_spec=job_spec,
@@ -1106,6 +1116,11 @@ def _check_cancel() -> None:
 			else tuple(int(v) for v in args.target_volume_shape_zyx)
 		),
 		flow_gate_channels=str(args.flow_gate_channels),
+		omit_model=bool(args.omit_model),
+		flatten_map_output=(
+			None if args.flatten_map_output in (None, "")
+			else str(args.flatten_map_output)
+		),
 	)
 
 	dev = torch.device(cfg.device)
@@ -1271,7 +1286,8 @@ def _check_cancel() -> None:
 			single_segment_border=BORDER_W,
 		)
 		_write_tifxyz(out_dir=out_dir, x=x_all, y=y_all, z=z_all, d=d_all, scale=meta_scale,
-					  model_source=Path(cfg.input), copy_model=cfg.copy_model, fit_config=fit_config,
+					  model_source=None if cfg.omit_model else Path(cfg.input),
+					  copy_model=cfg.copy_model, fit_config=fit_config,
 					  job_spec=job_spec,
 					  object_refs=object_refs,
 					  area=area, components=components if D > 1 else None,
@@ -1343,7 +1359,8 @@ def _check_cancel() -> None:
 				layer_index=d,
 			)
 			_write_tifxyz(out_dir=out_dir, x=x, y=y, z=z, d=d_layer, scale=meta_scale,
-						  model_source=Path(cfg.input), copy_model=cfg.copy_model, fit_config=fit_config,
+						  model_source=None if cfg.omit_model else Path(cfg.input),
+						  copy_model=cfg.copy_model, fit_config=fit_config,
 						  job_spec=job_spec,
 						  object_refs=object_refs,
 						  area=area,

lasagna/fit_service.py

@@ -1909,8 +1909,19 @@ def _wrapped_progress(*, step: int, total: int, loss: float, **kw: Any) -> None:
             try:
                 import fit as fit_mod
                 job.set_running("loading", 0, 0, 0.0)
-                fit_mod.main([cfg_path])
-                if isinstance(job_spec, dict) and Path(model_output).is_file():
+
+                def _fit_lifecycle(stage: str, stage_name: str) -> None:
+                    job.set_running(
+                        stage, 0, 0, 0.0, stage_name=stage_name)
+                    _check_cancel()
+
+                fit_mod.main([cfg_path], lifecycle_fn=_fit_lifecycle)
+                if (body.get("embed_job_metadata", True)
+                        and isinstance(job_spec, dict)
+                        and Path(model_output).is_file()):
+                    job.set_running(
+                        "saving", 0, 0, 0.0,
+                        stage_name="Embedding Lasagna job metadata")
                     import torch
                     st = torch.load(str(model_output), map_location="cpu", weights_only=False)
                     if isinstance(st, dict):
@@ -1927,14 +1938,23 @@ def _wrapped_progress(*, step: int, total: int, loss: float, **kw: Any) -> None:
                 return
 
             save_t0 = time.perf_counter()
-            job.set_running("exporting", 0, 0, 0.0)
+            job.set_running(
+                "exporting", 0, 0, 0.0,
+                stage_name="Exporting flattened preview surface")
             import fit2tifxyz
             export_argv = ["--input", str(model_output), "--output", str(output_dir)]
             if body.get("single_segment"):
                 export_argv.append("--single-segment")
             output_name = body.get("output_name")
             if output_name:
                 export_argv.extend(["--output-name", str(output_name)])
+            if body.get("omit_model", False):
+                export_argv.append("--omit-model")
+            if body.get("export_flatten_map", False):
+                export_argv.extend([
+                    "--flatten-map-output",
+                    str(Path(output_dir) / ".flatten-map.npy"),
+                ])
             print(
                 f"[fit-service] exporting tifxyz: output_name={str(output_name or '').strip()!r} "
                 f"output_dir={output_dir}",
@@ -1952,6 +1972,9 @@ def _wrapped_progress(*, step: int, total: int, loss: float, **kw: Any) -> None:
                 ])
             _check_cancel()
             fit2tifxyz.main(export_argv, cancel_fn=_check_cancel)
+            job.set_running(
+                "finalizing", 0, 0, 0.0,
+                stage_name="Finalizing Lasagna preview export")
             _normalize_single_tifxyz_output(Path(output_dir), str(output_name or ""))
             _check_cancel()
             save_s = time.perf_counter() - save_t0

lasagna/model.py

@@ -3254,11 +3254,11 @@ def _flatten_invert_forward_uv_map(
 
 		total = int(out_h) * int(out_w)
 		chunk = max(1, int(chunk_points))
-		if use_cuda and chunk < (1 << 20):
-			# Peak GPU memory scales with chunk * k through the candidate edge and
-			# barycentric tensors. If k_candidates grows large or OOMs appear,
-			# lower this cap.
-			chunk = 1 << 20
+		# if use_cuda and chunk < (1 << 20):
+		# 	# Peak GPU memory scales with chunk * k through the candidate edge and
+		# 	# barycentric tensors. If k_candidates grows large or OOMs appear,
+		# 	# lower this cap.
+		# 	chunk = 1 << 20
 		for start in range(0, total, chunk):
 			stop = min(total, start + chunk)
 			flat_idx = np.arange(start, stop, dtype=np.int64)

lasagna/tests/test_opt_loss_flatten.py

@@ -8,6 +8,7 @@
 from pathlib import Path
 from unittest import mock
 
+import numpy as np
 import torch
 import tifffile
 
@@ -1052,6 +1053,38 @@ def test_fit2tifxyz_exports_flatten_checkpoint(self) -> None:
 			y = tifffile.imread(str(out / "vc3d_name.tifxyz" / "y.tif"))
 			z = tifffile.imread(str(out / "vc3d_name.tifxyz" / "z.tif"))
 			self.assertTrue(bool(((x == -1.0) & (y == -1.0) & (z == -1.0)).any()))
+			self.assertTrue((out / "vc3d_name.tifxyz" / "model.pt").is_file())
+
+	def test_fit2tifxyz_can_export_ephemeral_flatten_without_model(self) -> None:
+		xyz = _flat_grid(4, 4)
+		valid = torch.ones(4, 4, dtype=torch.bool)
+		mdl = _make_flatten_model(xyz, valid, mesh_step=1)
+		with tempfile.TemporaryDirectory() as td:
+			root = Path(td)
+			model_path = root / "flatten_model.pt"
+			map_path = root / "out" / ".flatten-map.npy"
+			out = root / "out"
+			fit._save_flatten_model(
+				str(model_path),
+				mdl=mdl,
+				data=fit._dummy_flatten_data(),
+				fit_config={"args": {"model-init": "flatten"}},
+			)
+
+			fit2tifxyz.main([
+				"--input", str(model_path),
+				"--output", str(out),
+				"--output-name", "preview.tifxyz",
+				"--omit-model",
+				"--flatten-map-output", str(map_path),
+			])
+
+			self.assertFalse((out / "preview.tifxyz" / "model.pt").exists())
+			expected = torch.load(
+				model_path, map_location="cpu", weights_only=False
+			)["flatten_map_flat"].numpy()
+			np.testing.assert_array_equal(
+				np.load(map_path, allow_pickle=False), expected)
 
 	def test_forward_fit_mode_writes_normal_flatten_outputs(self) -> None:
 		with tempfile.TemporaryDirectory() as td:

scrollprize.org/docs/02_data_datasets.md

@@ -72,7 +72,7 @@ This dataset contains manual annotations of the scroll wraps, recording which pa
 Spiral annotations for scroll PHercParis4 (Scroll 1). Contents include ~27,000 verified and ~204,000 unverified surface patches, traced tracks, and point collections, together with `same_windings` / `relative_windings` / `abs_winding` graphs, the fitted `umbilicus`, fiber and outer-shell geometry, and the volume inputs used by the fitting pipeline (~49.6 GB total).
 
 - [README](pathname:///data/datasets/spiral-input-PHercParis4-README.md)
-- [Browse on Hugging Face](https://huggingface.co/buckets/scrollprize/datasets/tree/spiral/PHercParis4)
+- [Browse on the data server](https://dl.ash2txt.org/datasets/spiral_datasets/PHercParis4/)
 - [Tutorial: Spiral Fitting](tutorial_spiral) — how to fit a whole-scroll surface to these annotations
 
 {/* TODO: add more scroll subsections here later, e.g. "### Scroll5", each following the Paris4 pattern above. */}

scrollprize.org/docs/38_tutorial_spiral.md

@@ -106,12 +106,12 @@ The spiral scripts declare their dependencies in their own [`pyproject.toml`](ht
 
 #### Get the dataset
 
-Ready-made inputs for PHerc. Paris 4 (Scroll 1) are published in the [`spiral-input` dataset](data_datasets#spiral-input-2026-07), which lives in the [`scrollprize/datasets` storage bucket](https://huggingface.co/buckets/scrollprize/datasets/tree/spiral) on Hugging Face (~50 GB):
+Ready-made inputs are published in the [`spiral-input` dataset](data_datasets#spiral-input-2026-07), which lives on the dl.ash2txt.org data server : [Spiral Datasets](https://dl.ash2txt.org/datasets/spiral_datasets/PHercParis4/) (~90 GB):
 
 ```bash
-uvx --from huggingface_hub hf buckets sync \
-  hf://buckets/scrollprize/datasets/spiral/PHercParis4 \
-  ./spiral-dataset/PHercParis4
+rclone copy :http: ./spiral_datasets/phercparis4 \
+    --http-url https://dl.ash2txt.org/datasets/spiral_datasets/PHercParis4/ \
+    --transfers 32 -P
 ```
 
 `hf buckets sync` works like `rsync`: re-running it resumes interrupted downloads. The dataset contains verified and unverified patches, tracks, fibers, the outer shell, winding annotation JSONs, the umbilicus, and the volume inputs — see the [dataset README](pathname:///data/datasets/spiral-input-PHercParis4-README.md) for the exact layout.

volume-cartographer/apps/VC3D/CMakeLists.txt

@@ -406,5 +406,6 @@ install(DIRECTORY ${CMAKE_SOURCE_DIR}/scripts/spiral/
     DESTINATION share/volume-cartographer/spiral
     FILES_MATCHING
         PATTERN "*.py"
+        PATTERN "*.json"
         PATTERN "tests" EXCLUDE
         PATTERN "__pycache__" EXCLUDE)