#804: hardware video decoding (NVDEC / VA-API / VideoToolbox / Media Foundation) via filmcraft-platform

Cargo.toml

@@ -72,26 +72,31 @@ image = { version = "0.25", default-features = false, features = ["png", "jpeg"]
 
 # media codecs + containers: FilmCraft's pure-Rust crates, pinned to a revision (plan/adr/0001).
 # Only `crates/media`, `crates/export` and `crates/interchange` may depend on these.
-filmcraft-codecs = { git = "https://github.com/storytold/filmcraft", rev = "2c5ba7a72619935f0481f99d59409e0d1844b234" }
-filmcraft-media = { git = "https://github.com/storytold/filmcraft", rev = "2c5ba7a72619935f0481f99d59409e0d1844b234" }
-filmcraft-frame = { git = "https://github.com/storytold/filmcraft", rev = "2c5ba7a72619935f0481f99d59409e0d1844b234" }
-filmcraft-color = { git = "https://github.com/storytold/filmcraft", rev = "2c5ba7a72619935f0481f99d59409e0d1844b234" }
-filmcraft-time = { git = "https://github.com/storytold/filmcraft", rev = "2c5ba7a72619935f0481f99d59409e0d1844b234" }
+# The exception is `filmcraft-platform` (hardware video decoding through the operating system:
+# VideoToolbox, Media Foundation, NVDEC, VA-API). It is FilmCraft's one crate with OS FFI (`unsafe`
+# confined to its FFI modules), so only the desktop app binary (`apps/effectcraft`, L6) depends on
+# it and registers its decoders at startup; our own crates stay `unsafe`-free and wasm-clean.
+filmcraft-platform = { git = "https://github.com/storytold/filmcraft", rev = "4e2d0bc2d3646f9cf27ef7aa8b2095c6858c7efe" }
+filmcraft-codecs = { git = "https://github.com/storytold/filmcraft", rev = "4e2d0bc2d3646f9cf27ef7aa8b2095c6858c7efe" }
+filmcraft-media = { git = "https://github.com/storytold/filmcraft", rev = "4e2d0bc2d3646f9cf27ef7aa8b2095c6858c7efe" }
+filmcraft-frame = { git = "https://github.com/storytold/filmcraft", rev = "4e2d0bc2d3646f9cf27ef7aa8b2095c6858c7efe" }
+filmcraft-color = { git = "https://github.com/storytold/filmcraft", rev = "4e2d0bc2d3646f9cf27ef7aa8b2095c6858c7efe" }
+filmcraft-time = { git = "https://github.com/storytold/filmcraft", rev = "4e2d0bc2d3646f9cf27ef7aa8b2095c6858c7efe" }
 # encoders + muxer (crates/export)
-filmcraft-h264enc = { git = "https://github.com/storytold/filmcraft", rev = "2c5ba7a72619935f0481f99d59409e0d1844b234" }
-filmcraft-prores = { git = "https://github.com/storytold/filmcraft", rev = "2c5ba7a72619935f0481f99d59409e0d1844b234" }
-filmcraft-aac = { git = "https://github.com/storytold/filmcraft", rev = "2c5ba7a72619935f0481f99d59409e0d1844b234" }
-filmcraft-isobmff = { git = "https://github.com/storytold/filmcraft", rev = "2c5ba7a72619935f0481f99d59409e0d1844b234" }
+filmcraft-h264enc = { git = "https://github.com/storytold/filmcraft", rev = "4e2d0bc2d3646f9cf27ef7aa8b2095c6858c7efe" }
+filmcraft-prores = { git = "https://github.com/storytold/filmcraft", rev = "4e2d0bc2d3646f9cf27ef7aa8b2095c6858c7efe" }
+filmcraft-aac = { git = "https://github.com/storytold/filmcraft", rev = "4e2d0bc2d3646f9cf27ef7aa8b2095c6858c7efe" }
+filmcraft-isobmff = { git = "https://github.com/storytold/filmcraft", rev = "4e2d0bc2d3646f9cf27ef7aa8b2095c6858c7efe" }
 # decoder used as the round-trip oracle of effectcraft-opusenc (crates/export dev-dependency)
-filmcraft-opus = { git = "https://github.com/storytold/filmcraft", rev = "2c5ba7a72619935f0481f99d59409e0d1844b234" }
-filmcraft-vp9 = { git = "https://github.com/storytold/filmcraft", rev = "2c5ba7a72619935f0481f99d59409e0d1844b234" }
+filmcraft-opus = { git = "https://github.com/storytold/filmcraft", rev = "4e2d0bc2d3646f9cf27ef7aa8b2095c6858c7efe" }
+filmcraft-vp9 = { git = "https://github.com/storytold/filmcraft", rev = "4e2d0bc2d3646f9cf27ef7aa8b2095c6858c7efe" }
 # decoders used as the round-trip oracles of effectcraft-hevcenc / effectcraft-av1enc (dev-dependencies only)
-filmcraft-hevc = { git = "https://github.com/storytold/filmcraft", rev = "2c5ba7a72619935f0481f99d59409e0d1844b234" }
-filmcraft-av1 = { git = "https://github.com/storytold/filmcraft", rev = "2c5ba7a72619935f0481f99d59409e0d1844b234" }
+filmcraft-hevc = { git = "https://github.com/storytold/filmcraft", rev = "4e2d0bc2d3646f9cf27ef7aa8b2095c6858c7efe" }
+filmcraft-av1 = { git = "https://github.com/storytold/filmcraft", rev = "4e2d0bc2d3646f9cf27ef7aa8b2095c6858c7efe" }
 # timeline interchange (EDL, FCP7 XML, FCPXML, OTIO) for Premiere Pro interop (crates/interchange only)
-filmcraft-interchange = { git = "https://github.com/storytold/filmcraft", rev = "2c5ba7a72619935f0481f99d59409e0d1844b234" }
-filmcraft-project = { git = "https://github.com/storytold/filmcraft", rev = "2c5ba7a72619935f0481f99d59409e0d1844b234" }
-filmcraft-geom = { git = "https://github.com/storytold/filmcraft", rev = "2c5ba7a72619935f0481f99d59409e0d1844b234" }
+filmcraft-interchange = { git = "https://github.com/storytold/filmcraft", rev = "4e2d0bc2d3646f9cf27ef7aa8b2095c6858c7efe" }
+filmcraft-project = { git = "https://github.com/storytold/filmcraft", rev = "4e2d0bc2d3646f9cf27ef7aa8b2095c6858c7efe" }
+filmcraft-geom = { git = "https://github.com/storytold/filmcraft", rev = "4e2d0bc2d3646f9cf27ef7aa8b2095c6858c7efe" }
 # The symphonia filmcraft-codecs already builds, named only to turn on its AIFF reader: the pinned
 # rev enables wav alone (FilmCraft main enables aiff too). Drop it when the pin moves past that.
 symphonia = { version = "0.5.5", default-features = false, features = ["aiff"] }

apps/effectcraft/Cargo.toml

@@ -26,11 +26,25 @@ pollster = { workspace = true }
 cpal = { workspace = true }
 log = { workspace = true }
 
+# Settings ▸ Import ▸ Enable Hardware Accelerated Decoding: the operating system's H.264 / HEVC
+# decoders (VideoToolbox, Media Foundation / DXVA, NVDEC, VA-API), registered at startup in front of
+# FilmCraft's software decoders. All OS FFI lives in this FilmCraft crate; libva and the NVIDIA
+# driver are loaded at run time, so building needs no SDK and a system without them decodes in
+# software.
+[target.'cfg(not(target_arch = "wasm32"))'.dependencies]
+filmcraft-platform = { workspace = true }
+
 [target.'cfg(target_os = "linux")'.dependencies]
 # Settings ▸ Appearance ▸ Sync with System: the XDG desktop portal's colour scheme. Already in
 # Cargo.lock (accesskit, through eframe), so it adds no new crates.
 zbus = "5.19"
 
+[dev-dependencies]
+# tests/hardware_decoding.rs: footage decoded with the registered hardware decoders
+effectcraft-media = { workspace = true }
+effectcraft-raster = { workspace = true }
+effectcraft-render = { workspace = true }
+
 [build-dependencies]
 # Icon + VERSIONINFO for effectcraft.exe; build.rs only uses it when targeting Windows.
 winresource = "0.1"

apps/effectcraft/src/hw_decode.rs

@@ -0,0 +1,22 @@
+//! Hardware video decoding (Settings ▸ Import ▸ Enable Hardware Accelerated Decoding).
+//!
+//! FilmCraft's `filmcraft-platform` puts the operating system's H.264 / HEVC decoders
+//! (VideoToolbox on macOS, Media Foundation / DXVA on Windows, NVDEC and VA-API on Linux) in front
+//! of FilmCraft's software decoders. All OS FFI stays in that crate; EffectCraft only registers it
+//! here and flips the setting through `effectcraft-media`. Registering is harmless: while the
+//! setting is off, for streams the hardware can't take, and when no driver is present, the software
+//! decoders run, and a hardware decoder that fails mid-stream switches to software.
+
+/// Register the hardware decoders on a background thread: probing the drivers (loading libva or
+/// the NVIDIA driver, opening a render node) must not hold up the window. Footage opened before it
+/// finishes decodes in software until it is reopened.
+pub fn register_in_background() {
+    let spawned = std::thread::Builder::new().name("ec-hw-decode".into()).spawn(|| {
+        let t0 = std::time::Instant::now();
+        let available = filmcraft_platform::register();
+        log::info!("hardware video decoding: {available:?} ({:.0} ms)", t0.elapsed().as_secs_f64() * 1e3);
+    });
+    if let Err(e) = spawned {
+        log::warn!("hardware video decoding not registered: {e}");
+    }
+}

apps/effectcraft/src/main.rs

@@ -20,6 +20,7 @@ mod appimage;
 mod apple_events;
 mod audio_out;
 mod control_server;
+mod hw_decode;
 mod launch_guard;
 #[cfg(target_os = "macos")]
 mod native_menu;
@@ -151,6 +152,9 @@ fn main() -> eframe::Result {
     }
     // The font menus list the installed fonts: read their names while the window opens.
     effectcraft_engine::text::fonts::scan_system_in_background();
+    // The operating system's H.264 / HEVC decoders, in front of the software ones (Settings ▸
+    // Import ▸ Enable Hardware Accelerated Decoding).
+    hw_decode::register_in_background();
     // Wayland shows the window's icon from its desktop entry: an AppImage brings its own.
     #[cfg(target_os = "linux")]
     appimage::integrate_from_env(ICON_PNG);

apps/effectcraft/tests/hardware_decoding.rs

@@ -0,0 +1,161 @@
+//! Settings ▸ Import ▸ Enable Hardware Accelerated Decoding with the decoders the app registers
+//! (`filmcraft-platform`): off decodes in software, codecs the hardware does not take keep the
+//! software decoders, and where this machine has a hardware decoder its frames match software's.
+//!
+//! Fixtures come from ffmpeg (an external generator, never linked) into `target/fixtures/hw-decode`;
+//! the tests skip when ffmpeg is missing, and the hardware smoke test skips without a hardware
+//! decoder. The switch is process-wide, so the tests take turns (`SERIAL`) and live in their own
+//! test binary.
+
+use std::path::{Path, PathBuf};
+use std::process::Command;
+use std::sync::{Mutex, PoisonError};
+
+use effectcraft_media::{MediaPool, probe};
+use effectcraft_raster::Image;
+use effectcraft_render::FootageSource;
+
+static SERIAL: Mutex<()> = Mutex::new(());
+
+fn fixture(name: &str, args: &[&str]) -> Option<PathBuf> {
+    if !Command::new("ffmpeg").arg("-version").output().is_ok_and(|o| o.status.success()) {
+        eprintln!("SKIPPED: ffmpeg not found");
+        return None;
+    }
+    let out = Path::new(env!("CARGO_MANIFEST_DIR")).join("../../target/fixtures/hw-decode").join(name);
+    if out.exists() {
+        return Some(out);
+    }
+    std::fs::create_dir_all(out.parent()?).ok()?;
+    let tmp = out.with_file_name(format!("tmp-{}-{name}", std::process::id()));
+    let ok = Command::new("ffmpeg").args(["-hide_banner", "-loglevel", "error", "-y"]).args(args).arg(&tmp).status().is_ok_and(|s| s.success());
+    if !ok {
+        eprintln!("SKIPPED: ffmpeg could not make {name}");
+        return None;
+    }
+    std::fs::rename(&tmp, &out).ok()?;
+    Some(out)
+}
+
+/// H.264 High with B-frames and two GOPs (reordering and a seek).
+fn h264() -> Option<PathBuf> {
+    fixture(
+        "h264_high.mp4",
+        &[
+            "-f",
+            "lavfi",
+            "-i",
+            "testsrc2=s=640x360:r=24:d=2,noise=alls=12:allf=t",
+            "-c:v",
+            "libx264",
+            "-profile:v",
+            "high",
+            "-x264-params",
+            "bframes=3:keyint=24:min-keyint=24:scenecut=0",
+            "-pix_fmt",
+            "yuv420p",
+        ],
+    )
+}
+
+/// VP9: no hardware decoder registered on Linux takes it.
+fn vp9() -> Option<PathBuf> {
+    fixture("vp9.webm", &["-f", "lavfi", "-i", "testsrc2=s=320x240:r=24:d=1", "-c:v", "libvpx-vp9", "-b:v", "500k", "-pix_fmt", "yuv420p"])
+}
+
+/// A pool without read-ahead: no decode may outlive its test (the hardware counters are
+/// process-wide).
+fn quiet_pool() -> MediaPool {
+    let p = MediaPool::new();
+    p.set_read_ahead(false);
+    p
+}
+
+fn frames(pool: &MediaPool, path: &Path, n: i64) -> Vec<std::sync::Arc<Image>> {
+    let f = probe(path.to_str().unwrap()).unwrap();
+    (0..n).map(|i| pool.frame_at(&f, f.frame_rate.tick_of(i)).unwrap()).collect()
+}
+
+fn psnr(a: &Image, b: &Image) -> f64 {
+    assert_eq!((a.width, a.height), (b.width, b.height));
+    let se: f64 = a.data.iter().zip(&b.data).flat_map(|(p, q)| (0..4).map(move |k| ((p[k] - q[k]) as f64).powi(2))).sum();
+    let mse = se / (a.data.len() * 4) as f64;
+    if mse == 0.0 { f64::INFINITY } else { 10.0 * (1.0 / mse).log10() }
+}
+
+fn hw_frames(pool: &MediaPool) -> u64 {
+    pool.hardware_decoding().unwrap().frames
+}
+
+#[test]
+fn off_decodes_everything_in_software() {
+    let _serial = SERIAL.lock().unwrap_or_else(PoisonError::into_inner);
+    let Some(path) = h264() else { return };
+    let pool = quiet_pool();
+    pool.set_hardware_decoding(false);
+    let available = filmcraft_platform::register();
+    assert_eq!(filmcraft_platform::register(), available, "registering twice is harmless");
+    let before = hw_frames(&pool);
+    assert_eq!(frames(&pool, &path, 30).len(), 30);
+    let status = pool.hardware_decoding().unwrap();
+    assert!(!status.enabled);
+    assert_eq!(status.frames, before, "off: no hardware frames ({available:?})");
+    assert!(status.summary().starts_with("software"), "{}", status.summary());
+    pool.set_hardware_decoding(true);
+}
+
+#[test]
+fn codecs_the_hardware_does_not_take_stay_in_software() {
+    let _serial = SERIAL.lock().unwrap_or_else(PoisonError::into_inner);
+    let Some(path) = vp9() else { return };
+    filmcraft_platform::register();
+    let (hw, sw) = (quiet_pool(), quiet_pool());
+    hw.set_hardware_decoding(true);
+    let before = hw_frames(&hw);
+    let a = frames(&hw, &path, 12);
+    assert_eq!(hw_frames(&hw), before, "VP9 is not decoded in hardware");
+    sw.set_hardware_decoding(false);
+    let b = frames(&sw, &path, 12);
+    for (x, y) in a.iter().zip(&b) {
+        assert_eq!(x.data, y.data, "same software decoder, same pixels");
+    }
+    sw.set_hardware_decoding(true);
+}
+
+/// Where this machine has a hardware H.264 decoder: it takes the stream, its frames match the
+/// software decoder's, and switching the setting off reopens the footage in software.
+#[test]
+fn hardware_decode_smoke() {
+    let _serial = SERIAL.lock().unwrap_or_else(PoisonError::into_inner);
+    let Some(path) = h264() else { return };
+    let available = filmcraft_platform::register();
+    if let filmcraft_platform::Availability::Unavailable(why) = available {
+        eprintln!("SKIPPED: no hardware decoder ({why})");
+        return;
+    }
+    let pool = quiet_pool();
+    pool.set_hardware_decoding(true);
+    let before = pool.hardware_decoding().unwrap();
+    let hw = frames(&pool, &path, 48);
+    let after = pool.hardware_decoding().unwrap();
+    if after.sessions == before.sessions {
+        eprintln!("SKIPPED: {available:?} declined this H.264 stream (declined {} → {})", before.declined, after.declined);
+        return;
+    }
+    assert!(after.frames >= before.frames + 48, "{before:?} → {after:?}");
+    assert!(after.summary().contains("on"), "{}", after.summary());
+
+    let sw_pool = quiet_pool();
+    pool.set_hardware_decoding(false);
+    let sw = frames(&sw_pool, &path, 48);
+    for (i, (a, b)) in hw.iter().zip(&sw).enumerate() {
+        let db = psnr(a, b);
+        assert!(db > 60.0, "frame {i}: hardware vs software {db:.1} dB");
+    }
+    // Off closed the pool's decoders: what it decodes from now on comes from software.
+    pool.clear_frames();
+    let n0 = hw_frames(&pool);
+    assert_eq!(frames(&pool, &path, 12).len(), 12);
+    assert_eq!(hw_frames(&pool), n0, "off: no hardware frames after the switch");
+    pool.set_hardware_decoding(true);
+}

crates/engine/src/commands/app_more.rs

@@ -425,6 +425,7 @@ pub fn system_report(s: &Session) -> Value {
         "gpu": gpu,
         "gpuNote": if gpu.is_none() { s.accel_note.clone() } else { None },
         "windowGraphics": s.window_adapter,
+        "videoDecoding": s.footage.hardware_decoding().map(|h| json!({"summary": h.summary(), "detail": h})),
         "caches": {"layer": gb(budgets.layer as u64), "footage": gb(budgets.media as u64), "preview": gb(budgets.preview as u64)},
         "import": IMPORT_FORMATS,
         "export": export,
@@ -458,6 +459,9 @@ pub fn report_text(r: &Value) -> String {
     if let Some(w) = r.get("windowGraphics").and_then(Value::as_str) {
         t.push_str(&format!("Window graphics: {w}\n"));
     }
+    if let Some(v) = r.get("videoDecoding").and_then(|v| v.get("summary")).and_then(Value::as_str) {
+        t.push_str(&format!("Video decoding: {v}\n"));
+    }
     let c = &r["caches"];
     t.push_str(&format!(
         "Cache budgets: layer {}, footage {}, preview {}\n",

crates/engine/src/prefs.rs

@@ -226,6 +226,10 @@ page!(Import {
     unlabeled_alpha: String = "ask".into(),
     /// Default drag import as: `footage`, `comp`, `compLayerSizes`.
     drag_import_as: String = "footage".into(),
+    /// Enable Hardware Accelerated Decoding: H.264 / HEVC footage decodes on the GPU's video
+    /// decoder where the desktop app found one (VideoToolbox, Media Foundation, NVDEC, VA-API);
+    /// everything else, and everything while off, decodes in software.
+    hardware_decoding: bool = true,
 });
 
 page!(Export {
@@ -1307,6 +1311,8 @@ pub fn pages() -> Vec<Page> {
                     Kind::Choice(&[("Footage", "footage"), ("Composition", "comp"), ("Composition - Retain Layer Sizes", "compLayerSizes")]),
                     true,
                 ),
+                Section("Video Footage"),
+                s("import.hardwareDecoding", "Enable Hardware Accelerated Decoding", B, true),
             ],
         },
         Page {

crates/engine/src/session_settings.rs

@@ -30,6 +30,9 @@ impl Session {
         self.apply_cache_budgets();
         self.configure_disk_cache();
         self.footage.set_conform_folder(self.conformed_audio_folder());
+        // Settings ▸ Import ▸ Enable Hardware Accelerated Decoding (closes open decoders when it
+        // changes, so footage already open follows it).
+        self.footage.set_hardware_decoding(self.prefs.import.hardware_decoding);
         // Switches Affect Nested Comps changes what precomp layers render: drop cached pixels.
         let nested = self.prefs.general.switches_affect_nested_comps;
         if self.applied_nested_switches.replace(nested).is_some_and(|was| was != nested) {

crates/engine/src/sysinfo.rs

@@ -206,6 +206,8 @@ pub fn gpu_summary(s: &crate::Session) -> String {
         Some(adapter) => text.push_str(&format!("Window graphics: {adapter}\n")),
         None => text.push_str("Window graphics: not known yet (the window has not drawn its first frame).\n"),
     }
+    let decoding = s.footage.hardware_decoding().map_or_else(|| "software".to_string(), |h| h.summary());
+    text.push_str(&format!("Video decoding: {decoding}.\n"));
     text
 }
 

crates/export/tests/vp9_roundtrip.rs

@@ -41,7 +41,7 @@ fn source(w: usize, h: usize, kind: u32) -> (Vec<u8>, Vec<u8>, Vec<u8>) {
 }
 
 fn decode(frame: &[u8]) -> filmcraft_vp9::Picture {
-    let mut d = Decoder::new();
+    let mut d = Decoder::with_threads(1);
     let mut pics = d.decode(frame, 0).expect("decode");
     assert_eq!(pics.len(), 1);
     pics.remove(0)
@@ -189,7 +189,7 @@ fn planes_of(p: &filmcraft_vp9::Picture, w: usize, h: usize) -> [Vec<u8>; 3] {
 fn run_sequence(cfg: EncoderConfig, n: usize, kind: u32) -> (usize, f64) {
     let (w, h) = (cfg.width as usize, cfg.height as usize);
     let mut e = Vp9Encoder::new(cfg);
-    let mut d = Decoder::new();
+    let mut d = Decoder::with_threads(1);
     let (mut bytes, mut db) = (0usize, 0.0);
     for t in 0..n {
         let (y, u, v) = moving(w, h, t, kind);
@@ -344,7 +344,7 @@ fn frames_never_end_in_a_superframe_marker() {
     }
     let Some((w, h, q, frames, recon)) = padded else { panic!("no frame needed the padding: widen the search") };
     // The padded frames decode exactly, with FilmCraft's decoder and with ffmpeg.
-    let mut d = Decoder::new();
+    let mut d = Decoder::with_threads(1);
     for (t, (f, r)) in frames.iter().zip(&recon).enumerate() {
         let mut pics = d.decode(f, t as i64).unwrap_or_else(|e| panic!("{w}x{h} q{q} frame {t}: {e:?}"));
         assert_eq!(planes_of(&pics.remove(0), w, h), *r, "{w}x{h} q{q} frame {t}");

crates/host/tests/timeline_interop.rs

@@ -56,8 +56,9 @@ fn still(p: &mut fp::Project, path: &str) -> fp::ItemId {
             color: Default::default(),
             has_alpha: false,
             bitrate: None,
+            hdr: None,
         }),
-        audio: None,
+        audio_streams: Vec::new(),
         container: String::new(),
         start_timecode: None,
         file_size: None,