diff --git a/.github/workflows/hyperlight-e2e.yml b/.github/workflows/hyperlight-e2e.yml index 59c3e74c..8532c52e 100644 --- a/.github/workflows/hyperlight-e2e.yml +++ b/.github/workflows/hyperlight-e2e.yml @@ -67,7 +67,7 @@ jobs: $pyhlHome = Join-Path $env:LOCALAPPDATA "pyhl" New-Item -ItemType Directory -Force -Path $pyhlHome | Out-Null - $tag = "v0.10.0" + $tag = "v0.12.0" .\crane.exe export ghcr.io/hyperlight-dev/hyperlight-unikraft/python-agent-driver-kernel:$tag kernel.tar tar -xf kernel.tar -C $pyhlHome kernel diff --git a/src/Cargo.lock b/src/Cargo.lock index 0c693c46..c117762c 100644 --- a/src/Cargo.lock +++ b/src/Cargo.lock @@ -185,6 +185,15 @@ dependencies = [ "generic-array", ] +[[package]] +name = "block-buffer" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d2f6c7dbe95a6ed67ad9f18e57daf93a2f034c524b99fd2b76d18fdfeb6660aa" +dependencies = [ + "hybrid-array", +] + [[package]] name = "bstr" version = "1.12.1" @@ -326,7 +335,7 @@ version = "0.4.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "773f3b9af64447d2ce9850330c473515014aa235e6a783b02db81ff39e4a3dad" dependencies = [ - "crypto-common", + "crypto-common 0.1.7", "inout", ] @@ -376,6 +385,33 @@ version = "1.0.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1d07550c9036bf2ae0c684c4297d503f838287c83c53686d05370d0e139ae570" +[[package]] +name = "const-oid" +version = "0.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a6ef517f0926dd24a1582492c791b6a4818a4d94e789a334894aa15b0d12f55c" + +[[package]] +name = "const_format" +version = "0.2.36" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4481a617ad9a412be3b97c5d403fef8ed023103368908b9c50af598ff467cc1e" +dependencies = [ + "const_format_proc_macros", + "konst", +] + +[[package]] +name = "const_format_proc_macros" +version = "0.2.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d57c2eccfb16dbac1f4e61e206105db5820c9d26c3c472bc17c774259ef7744" +dependencies = [ + "proc-macro2", + "quote", + "unicode-xid", +] + [[package]] name = "constant_time_eq" version = "0.3.1" @@ -461,6 +497,50 @@ dependencies = [ "typenum", ] +[[package]] +name = "crypto-common" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ce6e4c961d6cd6c9a86db418387425e8bdeaf05b3c8bc1411e6dca4c252f1453" +dependencies = [ + "hybrid-array", +] + +[[package]] +name = "darling" +version = "0.20.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fc7f46116c46ff9ab3eb1597a45688b6715c6e628b5c133e288e709a29bcb4ee" +dependencies = [ + "darling_core", + "darling_macro", +] + +[[package]] +name = "darling_core" +version = "0.20.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0d00b9596d185e565c2207a0b01f8bd1a135483d02d9b7b0a54b11da8d53412e" +dependencies = [ + "fnv", + "ident_case", + "proc-macro2", + "quote", + "strsim", + "syn", +] + +[[package]] +name = "darling_macro" +version = "0.20.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fc34b93ccb385b40dc71c6fceac4b2ad23662c7eeb248cf10d529b7e055b6ead" +dependencies = [ + "darling_core", + "quote", + "syn", +] + [[package]] name = "deflate64" version = "0.1.12" @@ -484,17 +564,59 @@ dependencies = [ "syn", ] +[[package]] +name = "derive_builder" +version = "0.20.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "507dfb09ea8b7fa618fcf76e953f4f5e192547945816d5358edffe39f6f94947" +dependencies = [ + "derive_builder_macro", +] + +[[package]] +name = "derive_builder_core" +version = "0.20.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2d5bcf7b024d6835cfb3d473887cd966994907effbe9227e8c8219824d06c4e8" +dependencies = [ + "darling", + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "derive_builder_macro" +version = "0.20.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ab63b0e2bf4d5928aff72e83a7dace85d7bba5fe12dcc3c5a572d78caffd3f3c" +dependencies = [ + "derive_builder_core", + "syn", +] + [[package]] name = "digest" version = "0.10.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292" dependencies = [ - "block-buffer", - "crypto-common", + "block-buffer 0.10.4", + "crypto-common 0.1.7", "subtle", ] +[[package]] +name = "digest" +version = "0.11.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f1dd6dbb5841937940781866fa1281a1ff7bd3bf827091440879f9994983d5c2" +dependencies = [ + "block-buffer 0.12.1", + "const-oid", + "crypto-common 0.2.2", +] + [[package]] name = "dirs" version = "6.0.0" @@ -577,6 +699,12 @@ version = "0.1.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5baebc0774151f905a1a2cc41989300b1e6fbb29aff0ceffa1064fdd3088d582" +[[package]] +name = "fixedbitset" +version = "0.5.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d674e81391d1e1ab681a28d99df07927c6d4aa5b027d7da16ba32d1d21ecd99" + [[package]] name = "flatbuffers" version = "25.12.19" @@ -597,6 +725,12 @@ dependencies = [ "miniz_oxide", ] +[[package]] +name = "fnv" +version = "1.0.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" + [[package]] name = "foldhash" version = "0.1.5" @@ -694,6 +828,17 @@ dependencies = [ "wasip3", ] +[[package]] +name = "getset" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6cf442baaabe4213ce7d1239afc26c039180b6456da2cededa316ae2c8a77a77" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "git2" version = "0.21.0" @@ -751,13 +896,19 @@ version = "0.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea" +[[package]] +name = "hex" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7f24254aa9a54b5c858eaee2f5bccdb46aaf0e486a595ed5fd8f86ba55232a70" + [[package]] name = "hmac" version = "0.12.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6c49c37c09c17a53d937dfbb742eb3a961d65a994e6bcdcf37e7399d0cc8ab5e" dependencies = [ - "digest", + "digest 0.10.7", ] [[package]] @@ -805,6 +956,15 @@ version = "1.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "df3b46402a9d5adb4c86a0cf463f42e19994e3ee891101b1841f30a545cb49a9" +[[package]] +name = "hybrid-array" +version = "0.4.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9155a582abd142abc056962c29e3ce5ff2ad5469f4246b537ed42c5deba857da" +dependencies = [ + "typenum", +] + [[package]] name = "hyper" version = "1.10.1" @@ -842,12 +1002,18 @@ dependencies = [ [[package]] name = "hyperlight-common" -version = "0.15.0" -source = "git+https://github.com/danbugs/hyperlight?rev=5cf37d92#5cf37d92262c918e7d633577a6cf946fb1f069bd" +version = "0.16.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "436dac5cc08da3db27de38b174a9b22498ab6443aaffd83b97f66a5d2f1d74cc" dependencies = [ "anyhow", + "bitflags 2.13.0", + "bytemuck", + "bytes", + "fixedbitset", "flatbuffers", "log", + "smallvec", "spin", "thiserror", "tracing", @@ -856,8 +1022,9 @@ dependencies = [ [[package]] name = "hyperlight-host" -version = "0.15.0" -source = "git+https://github.com/danbugs/hyperlight?rev=5cf37d92#5cf37d92262c918e7d633577a6cf946fb1f069bd" +version = "0.16.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ef3edc7875e70a3b5cc291a17f78cfc352e24c9a665a5a1bbdbc46ada3df43b5" dependencies = [ "anyhow", "bitflags 2.13.0", @@ -869,6 +1036,7 @@ dependencies = [ "crossbeam-channel", "flatbuffers", "goblin", + "hex", "hyperlight-common", "kvm-bindings", "kvm-ioctls", @@ -878,15 +1046,18 @@ dependencies = [ "metrics", "mshv-bindings", "mshv-ioctls", + "oci-spec", "page_size", "rand", "rust-embed", + "serde", "serde_json", + "sha2 0.11.0", + "tempfile", "termcolor", "thiserror", "tracing", "tracing-core", - "tracing-log", "uuid", "vmm-sys-util", "windows", @@ -896,9 +1067,10 @@ dependencies = [ ] [[package]] -name = "hyperlight-unikraft-host" -version = "0.10.0" -source = "git+https://github.com/hyperlight-dev/hyperlight-unikraft?tag=v0.10.0#c2092b3a060658575402a1bfc2febffcf27ef462" +name = "hyperlight-unikraft" +version = "0.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d46311f291f462b4a525748a1b1bd50068a0a8b33dda82791d90063d1baa8aa1" dependencies = [ "anyhow", "base64", @@ -906,7 +1078,6 @@ dependencies = [ "flate2", "hyperlight-host", "libc", - "memmap2", "nix", "serde_json", "socket2 0.5.10", @@ -919,7 +1090,7 @@ dependencies = [ name = "hyperlight_common" version = "0.7.0" dependencies = [ - "hyperlight-unikraft-host", + "hyperlight-unikraft", "wxc_common", ] @@ -1035,6 +1206,12 @@ version = "2.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3d3067d79b975e8844ca9eb072e16b31c3c1c36928edf9c6789548c524d0d954" +[[package]] +name = "ident_case" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b9e0384b61958566e926dc50660321d12159025e767c18e043daf26b70104c39" + [[package]] name = "idna" version = "1.1.0" @@ -1134,6 +1311,21 @@ dependencies = [ "wasm-bindgen", ] +[[package]] +name = "konst" +version = "0.2.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "128133ed7824fcd73d6e7b17957c5eb7bacb885649bd8c69708b2331a10bcefb" +dependencies = [ + "konst_macro_rules", +] + +[[package]] +name = "konst_macro_rules" +version = "0.2.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a4933f3f57a8e9d9da04db23fb153356ecaf00cbd14aee46279c33dc80925c37" + [[package]] name = "kvm-bindings" version = "0.14.1" @@ -1145,9 +1337,9 @@ dependencies = [ [[package]] name = "kvm-ioctls" -version = "0.24.0" +version = "0.25.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "333f77a20344a448f3f70664918135fddeb804e938f28a99d685bd92926e0b19" +checksum = "06ac372c120eb893b086d1a12027669cf2b478d1f71204021ffa7adf57948d63" dependencies = [ "bitflags 2.13.0", "kvm-bindings", @@ -1312,15 +1504,6 @@ version = "2.8.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "88904434abc2901f197fe8cc55f0445e7ded921dba5911dad2e2b39b48e663c4" -[[package]] -name = "memmap2" -version = "0.9.10" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "714098028fe011992e1c3962653c96b2d578c4b4bce9036e15ff220319b1e0e3" -dependencies = [ - "libc", -] - [[package]] name = "memoffset" version = "0.9.1" @@ -1526,6 +1709,23 @@ dependencies = [ "syn", ] +[[package]] +name = "oci-spec" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3df6f876ad774d6a676f7e968f5c3edacc32f90e65fe680a8b686235396556fb" +dependencies = [ + "const_format", + "derive_builder", + "getset", + "regex", + "serde", + "serde_json", + "strum", + "strum_macros", + "thiserror", +] + [[package]] name = "once_cell" version = "1.21.4" @@ -1583,7 +1783,7 @@ version = "0.12.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f8ed6a7761f76e3b9f92dfb0a60a6a6477c61024b775147ff0973a02653abaf2" dependencies = [ - "digest", + "digest 0.10.7", "hmac", ] @@ -1718,6 +1918,18 @@ dependencies = [ "thiserror", ] +[[package]] +name = "regex" +version = "1.12.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f1292b7759ae1cb9ec195452d1390a074f0cd8541ab7a5a8c31cd6db45d4a6ba" +dependencies = [ + "aho-corasick", + "memchr", + "regex-automata", + "regex-syntax", +] + [[package]] name = "regex-automata" version = "0.4.14" @@ -1781,7 +1993,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5bcdef0be6fe7f6fa333b1073c949729274b05f123a0ad7efcb8efd878e5c3b1" dependencies = [ "globset", - "sha2", + "sha2 0.10.9", "walkdir", ] @@ -1990,7 +2202,7 @@ checksum = "e3bf829a2d51ab4a5ddf1352d8470c140cadc8301b2ae1789db023f01cedd6ba" dependencies = [ "cfg-if", "cpufeatures 0.2.17", - "digest", + "digest 0.10.7", ] [[package]] @@ -2001,7 +2213,18 @@ checksum = "a7507d819769d01a365ab707794a4084392c824f54a7a6a7862f8c3d0892b283" dependencies = [ "cfg-if", "cpufeatures 0.2.17", - "digest", + "digest 0.10.7", +] + +[[package]] +name = "sha2" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "446ba717509524cb3f22f17ecc096f10f4822d76ab5c0b9822c5f9c284e825f4" +dependencies = [ + "cfg-if", + "cpufeatures 0.3.0", + "digest 0.11.3", ] [[package]] @@ -2069,9 +2292,9 @@ dependencies = [ [[package]] name = "spin" -version = "0.10.0" +version = "0.12.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d5fe4ccb98d9c292d56fec89a5e07da7fc4cf0dc11e156b41793132775d3e591" +checksum = "bd5231412d905519dca6a5deb0327d407be68d6c941feec004533401d3a0a715" dependencies = [ "lock_api", ] @@ -2088,6 +2311,24 @@ version = "0.11.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7da8b5736845d9f2fcb837ea5d9e2628564b3b043a70948a3f0b778838c5fb4f" +[[package]] +name = "strum" +version = "0.27.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "af23d6f6c1a224baef9d3f61e287d2761385a5b88fdab4eb4c6f11aeb54c4bcf" + +[[package]] +name = "strum_macros" +version = "0.27.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7695ce3845ea4b33927c055a39dc438a45b059f7c1b3d91d38d10355fb8cbca7" +dependencies = [ + "heck", + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "subtle" version = "2.6.1" @@ -2259,17 +2500,6 @@ dependencies = [ "valuable", ] -[[package]] -name = "tracing-log" -version = "0.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ee855f1f400bd0e5c02d150ae5de3840039a3f54b025156404e34c23c03f47c3" -dependencies = [ - "log", - "once_cell", - "tracing-core", -] - [[package]] name = "try-lock" version = "0.2.5" diff --git a/src/backends/hyperlight/common/Cargo.toml b/src/backends/hyperlight/common/Cargo.toml index 943ac436..1c86c589 100644 --- a/src/backends/hyperlight/common/Cargo.toml +++ b/src/backends/hyperlight/common/Cargo.toml @@ -8,8 +8,8 @@ license.workspace = true wxc_common = { workspace = true } [target.'cfg(target_arch = "x86_64")'.dependencies] -hyperlight-unikraft-host = { git = "https://github.com/hyperlight-dev/hyperlight-unikraft", tag = "v0.10.0", optional = true } +hyperlight-unikraft = { version = "0.12.0", optional = true } [features] default = [] -hyperlight = ["dep:hyperlight-unikraft-host"] +hyperlight = ["dep:hyperlight-unikraft"] diff --git a/src/backends/hyperlight/common/src/lib.rs b/src/backends/hyperlight/common/src/lib.rs index 98f98308..82e483cc 100644 --- a/src/backends/hyperlight/common/src/lib.rs +++ b/src/backends/hyperlight/common/src/lib.rs @@ -12,7 +12,7 @@ #![cfg(all(feature = "hyperlight", target_arch = "x86_64"))] //! `HyperlightScriptRunner` — executes Python code inside a Hyperlight + Unikraft -//! micro-VM, driven by the `hyperlight-unikraft-host::pyhl` library. +//! micro-VM, driven by the `hyperlight-unikraft::pyhl` library. //! //! | Property | Value | //! |---------------------|-----------------------------------------------------------| @@ -135,7 +135,7 @@ const DEFAULT_HOME_LEAF: &str = "pyhl"; // compile-time dep on internal path constants. const KERNEL_FILE: &str = "kernel"; const INITRD_FILE: &str = "initrd.cpio"; -const SNAPSHOT_FILE: &str = "snapshot.hls"; +const SNAPSHOT_DIR: &str = "snapshot"; const ERR_PROXY_POLICY: &str = "network proxy is not supported by the hyperlight backend"; const ERR_WORKDIR: &str = @@ -200,9 +200,9 @@ pub fn setup(force: bool, logger: &mut Logger) -> Result { logger.log_line(&format!( "hyperlight: snapshot already present at {:?}; nothing to do \ (pass --force to rebuild)", - home.join(SNAPSHOT_FILE) + home.join(SNAPSHOT_DIR) )); - return Ok(home.join(SNAPSHOT_FILE)); + return Ok(home.join(SNAPSHOT_DIR)); } std::fs::create_dir_all(&home).map_err(|e| format!("create image home {home:?}: {e}"))?; @@ -211,11 +211,12 @@ pub fn setup(force: bool, logger: &mut Logger) -> Result { let opts = pyhl::InstallOptions { home: &home, source: pyhl::InstallSource::Ghcr { - tag: Some("v0.10.0"), + tag: Some("v0.12.0"), }, mounts: &[], network: None, listen_ports: None, + max_surrogates: None, force, }; let report = pyhl::install(&opts).map_err(|e| format!("hyperlight install: {e:#}"))?; @@ -464,7 +465,7 @@ impl HyperlightScriptRunner { } logger.log_line(&format!( "hyperlight: no snapshot at {:?}; auto-installing from kernel + initrd", - home.join(SNAPSHOT_FILE) + home.join(SNAPSHOT_DIR) )); let kernel = home.join(KERNEL_FILE); let initrd = home.join(INITRD_FILE); @@ -477,6 +478,7 @@ impl HyperlightScriptRunner { mounts: &preopens, network: network.as_ref(), listen_ports: None, + max_surrogates: None, force: false, }; let report = pyhl::install(&opts) @@ -488,7 +490,7 @@ impl HyperlightScriptRunner { } logger.log_line(&format!("hyperlight: using image home {home:?}")); - let rt = pyhl::Runtime::new(home, &preopens, network.as_ref(), None) + let rt = pyhl::Runtime::new(home, &preopens, network.as_ref(), None, None) .map_err(|e| PyhlError::Runtime(format!("open hyperlight runtime: {e:#}")))?; self.runtime = Some(rt); self.active_home = Some(home.to_path_buf()); @@ -584,7 +586,7 @@ impl ScriptRunner for HyperlightScriptRunner { fn is_installed(home: &Path) -> bool { home.join(KERNEL_FILE).is_file() && home.join(INITRD_FILE).is_file() - && home.join(SNAPSHOT_FILE).is_file() + && home.join(SNAPSHOT_DIR).join("index.json").is_file() } /// A home has the raw inputs we need to auto-install a snapshot. diff --git a/src/testing/wxc_e2e_tests/src/lib.rs b/src/testing/wxc_e2e_tests/src/lib.rs index 0f390b24..a58b8000 100644 --- a/src/testing/wxc_e2e_tests/src/lib.rs +++ b/src/testing/wxc_e2e_tests/src/lib.rs @@ -234,7 +234,7 @@ pub fn run_lxc_config(config_file: &str, extra_args: &[&str]) -> CommandResult { } /// Return whether the Hyperlight snapshot is installed at the default -/// location (`%LOCALAPPDATA%\pyhl\snapshot.hls`). +/// location (`%LOCALAPPDATA%\pyhl\snapshot\index.json`). pub fn has_hyperlight_snapshot() -> bool { let home = std::env::var_os("LOCALAPPDATA") .map(PathBuf::from) @@ -243,7 +243,7 @@ pub fn has_hyperlight_snapshot() -> bool { .map(|v| PathBuf::from(v).join("AppData").join("Local")) .unwrap_or_default() }); - let snapshot = home.join("pyhl").join("snapshot.hls"); + let snapshot = home.join("pyhl").join("snapshot").join("index.json"); if snapshot.is_file() { println!("Using Hyperlight snapshot at {}", snapshot.display()); true