Build with wasm exception handling (#81)

For now, we need to use a custom rust toolchain for this. Apparently all
alternatives to this don't work. -Zbuild-std doesn't work with
panic=abort (rust-lang/cargo#7359)
and my attempts to use a custom sysroot with either
https://github.com/RalfJung/rustc-build-sysroot/ or
https://github.com/DianaNites/cargo-sysroot/ seem to hit the same
problem as with `-Zbuild-std`.

Thus, I think the only reasonable way to go is to build the sysroot from
the rust source directory. This is done by
https://github.com/pyodide/rust-emscripten-wasm-eh-sysroot/

Author: hoodmane

pyodide_build/common.py

@@ -451,7 +451,9 @@ def to_bool(value: str) -> bool:
     return value.lower() not in {"", "0", "false", "no", "off"}
 
 
-def download_and_unpack_archive(url: str, path: Path, descr: str) -> None:
+def download_and_unpack_archive(
+    url: str, path: Path, descr: str, *, exists_ok: bool = False
+) -> None:
     """
     Download the cross-build environment from the given URL and extract it to the given path.
 
@@ -465,7 +467,7 @@ def download_and_unpack_archive(url: str, path: Path, descr: str) -> None:
     """
     logger.info("Downloading %s from %s", descr, url)
 
-    if path.exists():
+    if not exists_ok and path.exists():
         raise FileExistsError(f"Path {path} already exists")
 
     try:

pyodide_build/config.py

@@ -179,6 +179,7 @@ def _get_make_environment_vars(self) -> Mapping[str, str]:
     "cpythoninstall": "CPYTHONINSTALL",
     "rustflags": "RUSTFLAGS",
     "rust_toolchain": "RUST_TOOLCHAIN",
+    "rust_emscripten_target_url": "RUST_EMSCRIPTEN_TARGET_URL",
     "cflags": "SIDE_MODULE_CFLAGS",
     "cxxflags": "SIDE_MODULE_CXXFLAGS",
     "ldflags": "SIDE_MODULE_LDFLAGS",
@@ -210,6 +211,7 @@ def _get_make_environment_vars(self) -> Mapping[str, str]:
     "cxxflags",
     "ldflags",
     "rust_toolchain",
+    "rust_emscripten_target_url",
     "meson_cross_file",
     "skip_emscripten_version_check",
     "build_dependency_index_url",
@@ -229,6 +231,7 @@ def _get_make_environment_vars(self) -> Mapping[str, str]:
     "cargo_build_target": "wasm32-unknown-emscripten",
     "cargo_target_wasm32_unknown_emscripten_linker": "emcc",
     "rust_toolchain": "nightly-2025-02-01",
+    "rust_emscripten_target_url": "",
     # Other configuration
     "pyodide_jobs": "1",
     "skip_emscripten_version_check": "0",

pyodide_build/pypabuild.py

@@ -295,6 +295,7 @@ def get_build_env(
         args["orig__name__"] = __name__
         args["pythoninclude"] = get_build_flag("PYTHONINCLUDE")
         args["PATH"] = env["PATH"]
+        args["abi"] = get_build_flag("PYODIDE_ABI_VERSION")
 
         pywasmcross_env = json.dumps(args)
         # Store into environment variable and to disk. In most cases we will

pyodide_build/pywasmcross.py

@@ -77,6 +77,9 @@ class CrossCompileArgs(NamedTuple):
     target_install_dir: str = ""  # The path to the target Python installation
     pythoninclude: str = ""  # path to the cross-compiled Python include directory
     exports: Literal["whole_archive", "requested", "pyinit"] | list[str] = "pyinit"
+    # Pyodide abi, e.g., 2025_0
+    # Sometimes we have to inject compile flags only for certain abis.
+    abi: str = ""
 
 
 def is_link_cmd(line: list[str]) -> bool:
@@ -93,7 +96,7 @@ def is_link_cmd(line: list[str]) -> bool:
     return False
 
 
-def replay_genargs_handle_dashl(arg: str, used_libs: set[str]) -> str | None:
+def replay_genargs_handle_dashl(arg: str, used_libs: set[str], abi: str) -> str | None:
     """
     Figure out how to replace a `-lsomelib` argument.
 
@@ -118,6 +121,13 @@ def replay_genargs_handle_dashl(arg: str, used_libs: set[str]) -> str | None:
     if arg == "-lgfortran":
         return None
 
+    # Some Emscripten libraries that use setjmp/longjmp.
+    # The Emscripten linker should automatically know to use these variants so
+    # this shouldn't be necessary.
+    # This suffix will need to change soon to `-legacysjlj`.
+    if abi > "2025" and arg in ["-lfreetype", "-lpng"]:
+        arg += "-wasm-sjlj"
+
     # WASM link doesn't like libraries being included twice
     # skip second one
     if arg in used_libs:
@@ -555,7 +565,7 @@ def handle_command_generate_args(  # noqa: C901
             continue
 
         if arg.startswith("-l"):
-            result = replay_genargs_handle_dashl(arg, used_libs)
+            result = replay_genargs_handle_dashl(arg, used_libs, build_args.abi)
         elif arg.startswith("-I"):
             result = replay_genargs_handle_dashI(arg, build_args.target_install_dir)
         elif arg.startswith("-Wl"):
@@ -638,6 +648,7 @@ def compiler_main():
         target_install_dir=PYWASMCROSS_ARGS["target_install_dir"],
         pythoninclude=PYWASMCROSS_ARGS["pythoninclude"],
         exports=PYWASMCROSS_ARGS["exports"],
+        abi=PYWASMCROSS_ARGS["abi"],
     )
     basename = Path(sys.argv[0]).name
     args = list(sys.argv)

pyodide_build/recipe/graph_builder.py

@@ -30,6 +30,7 @@
 from pyodide_build import build_env
 from pyodide_build.build_env import BuildArgs
 from pyodide_build.common import (
+    download_and_unpack_archive,
     exit_with_stdio,
     extract_wheel_metadata_file,
     find_matching_wheels,
@@ -700,26 +701,54 @@ def run(self, n_jobs: int, already_built: set[str]) -> None:
                         self.build_queue.put((job_priority(dependent), dependent))
 
 
+def _run(cmd, *args, check=False, **kwargs):
+    result = subprocess.run(cmd, *args, **kwargs, check=check)
+    if result.returncode != 0:
+        logger.error("ERROR: command failed %s", " ".join(cmd))
+        exit_with_stdio(result)
+    return result
+
+
 def _ensure_rust_toolchain():
     rust_toolchain = build_env.get_build_flag("RUST_TOOLCHAIN")
-    result = subprocess.run(
-        ["rustup", "toolchain", "install", rust_toolchain], check=False
-    )
-    if result.returncode == 0:
-        result = subprocess.run(
+    _run(["rustup", "toolchain", "install", rust_toolchain])
+    _run(["rustup", "default", rust_toolchain])
+
+    url = build_env.get_build_flag("RUST_EMSCRIPTEN_TARGET_URL")
+    if not url:
+        # Install target with rustup target add
+        _run(
             [
                 "rustup",
                 "target",
                 "add",
                 "wasm32-unknown-emscripten",
                 "--toolchain",
                 rust_toolchain,
-            ],
-            check=False,
+            ]
         )
-    if result.returncode != 0:
-        logger.error("ERROR: rustup toolchain install failed")
-        exit_with_stdio(result)
+        return
+
+    # Now we are going to delete the normal wasm32-unknown-emscripten sysroot
+    # and replace it with our wasm-eh version.
+    # We place the "install_token" to indicate that our custom sysroot has been
+    # installed and which URL we got it from.
+    result = _run(
+        ["rustup", "which", "--toolchain", rust_toolchain, "rustc"],
+        capture_output=True,
+        text=True,
+    )
+
+    toolchain_root = Path(result.stdout).parents[1]
+    rustlib = toolchain_root / "lib/rustlib"
+    install_token = rustlib / "wasm32-unknown-emscripten_install-url.txt"
+    if install_token.exists() and install_token.read_text() == url:
+        return
+    shutil.rmtree(rustlib / "wasm32-unknown-emscripten", ignore_errors=True)
+    download_and_unpack_archive(
+        url, rustlib, "wasm32-unknown-emscripten target", exists_ok=True
+    )
+    install_token.write_text(url)
 
 
 def build_from_graph(

pyproject.toml

@@ -133,7 +133,7 @@ lint.select = [
 
 lint.logger-objects = ["pyodide_build.logger.logger"]
 
-lint.ignore = ["E402", "E501", "E731", "E741", "PERF401", "PLW2901", "UP038"]
+lint.ignore = ["E402", "E501", "E731", "E741", "PERF401", "PLW2901", "UP038", "PLR0912"]
 # line-length = 219  # E501: Recommended goal is 88 to match black
 target-version = "py312"