From 85a34554e8fd1e5f27410df07040011eafc43bd9 Mon Sep 17 00:00:00 2001 From: vangel76 Date: Sun, 11 Oct 2026 14:38:55 +0200 Subject: [PATCH 1/3] Bump FilmCraft to 4e2d0bc2 (#804) All filmcraft-* git dependencies move from 2c5ba7a (2026-10-03) to FilmCraft main 4e2d0bc2 (2026-10-10), the first revision with filmcraft-platform. Fixes for what changed: - MediaInfo.audio became audio_streams (audio() / has_audio()); VideoStreamInfo gained hdr. - FilmCraft now rounds a frame request to the track's nearest timestamp unit (halves up), so the media pool's mid-frame request landed on the next frame when the timescale equals the frame rate (our own MP4 / MOV exports at 10 fps): it asks a quarter into the frame instead. - filmcraft_vp9::Decoder::new() decodes on frame threads and may return a picture from a later call; the VP9 round-trip tests that expect one picture per call use with_threads(1). Co-Authored-By: Claude Opus 5.5 --- Cargo.lock | 115 ++++++++++++++------------ Cargo.toml | 32 +++---- crates/export/tests/vp9_roundtrip.rs | 6 +- crates/host/tests/timeline_interop.rs | 3 +- crates/interchange/src/export.rs | 6 +- crates/interchange/src/import.rs | 4 +- crates/interchange/src/tests.rs | 6 +- crates/media/src/pool.rs | 9 +- crates/media/src/probe.rs | 4 +- 9 files changed, 104 insertions(+), 81 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index ba84c75f5..9e990db30 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2019,8 +2019,8 @@ checksum = "b97b65636e5b9ef369943878ac74335ba1c55c1cb6adbf1e2c293c624248d693" [[package]] name = "filmcraft-aac" -version = "0.1.1" -source = "git+https://github.com/storytold/filmcraft?rev=2c5ba7a72619935f0481f99d59409e0d1844b234#2c5ba7a72619935f0481f99d59409e0d1844b234" +version = "0.6.0" +source = "git+https://github.com/storytold/filmcraft?rev=4e2d0bc2d3646f9cf27ef7aa8b2095c6858c7efe#4e2d0bc2d3646f9cf27ef7aa8b2095c6858c7efe" dependencies = [ "filmcraft-bitstream", "thiserror 2.0.21", @@ -2028,16 +2028,26 @@ dependencies = [ [[package]] name = "filmcraft-ac3" -version = "0.1.1" -source = "git+https://github.com/storytold/filmcraft?rev=2c5ba7a72619935f0481f99d59409e0d1844b234#2c5ba7a72619935f0481f99d59409e0d1844b234" +version = "0.6.0" +source = "git+https://github.com/storytold/filmcraft?rev=4e2d0bc2d3646f9cf27ef7aa8b2095c6858c7efe#4e2d0bc2d3646f9cf27ef7aa8b2095c6858c7efe" dependencies = [ "thiserror 2.0.21", ] +[[package]] +name = "filmcraft-apv" +version = "0.6.0" +source = "git+https://github.com/storytold/filmcraft?rev=4e2d0bc2d3646f9cf27ef7aa8b2095c6858c7efe#4e2d0bc2d3646f9cf27ef7aa8b2095c6858c7efe" +dependencies = [ + "filmcraft-bitstream", + "rayon", + "thiserror 2.0.21", +] + [[package]] name = "filmcraft-av1" -version = "0.1.1" -source = "git+https://github.com/storytold/filmcraft?rev=2c5ba7a72619935f0481f99d59409e0d1844b234#2c5ba7a72619935f0481f99d59409e0d1844b234" +version = "0.6.0" +source = "git+https://github.com/storytold/filmcraft?rev=4e2d0bc2d3646f9cf27ef7aa8b2095c6858c7efe#4e2d0bc2d3646f9cf27ef7aa8b2095c6858c7efe" dependencies = [ "filmcraft-bitstream", "rayon", @@ -2046,21 +2056,22 @@ dependencies = [ [[package]] name = "filmcraft-bitstream" -version = "0.1.1" -source = "git+https://github.com/storytold/filmcraft?rev=2c5ba7a72619935f0481f99d59409e0d1844b234#2c5ba7a72619935f0481f99d59409e0d1844b234" +version = "0.6.0" +source = "git+https://github.com/storytold/filmcraft?rev=4e2d0bc2d3646f9cf27ef7aa8b2095c6858c7efe#4e2d0bc2d3646f9cf27ef7aa8b2095c6858c7efe" [[package]] name = "filmcraft-cfb" -version = "0.1.1" -source = "git+https://github.com/storytold/filmcraft?rev=2c5ba7a72619935f0481f99d59409e0d1844b234#2c5ba7a72619935f0481f99d59409e0d1844b234" +version = "0.6.0" +source = "git+https://github.com/storytold/filmcraft?rev=4e2d0bc2d3646f9cf27ef7aa8b2095c6858c7efe#4e2d0bc2d3646f9cf27ef7aa8b2095c6858c7efe" [[package]] name = "filmcraft-codecs" -version = "0.1.1" -source = "git+https://github.com/storytold/filmcraft?rev=2c5ba7a72619935f0481f99d59409e0d1844b234#2c5ba7a72619935f0481f99d59409e0d1844b234" +version = "0.6.0" +source = "git+https://github.com/storytold/filmcraft?rev=4e2d0bc2d3646f9cf27ef7aa8b2095c6858c7efe#4e2d0bc2d3646f9cf27ef7aa8b2095c6858c7efe" dependencies = [ "filmcraft-aac", "filmcraft-ac3", + "filmcraft-apv", "filmcraft-av1", "filmcraft-bitstream", "filmcraft-color", @@ -2088,16 +2099,16 @@ dependencies = [ [[package]] name = "filmcraft-color" -version = "0.1.1" -source = "git+https://github.com/storytold/filmcraft?rev=2c5ba7a72619935f0481f99d59409e0d1844b234#2c5ba7a72619935f0481f99d59409e0d1844b234" +version = "0.6.0" +source = "git+https://github.com/storytold/filmcraft?rev=4e2d0bc2d3646f9cf27ef7aa8b2095c6858c7efe#4e2d0bc2d3646f9cf27ef7aa8b2095c6858c7efe" dependencies = [ "serde", ] [[package]] name = "filmcraft-dnx" -version = "0.1.1" -source = "git+https://github.com/storytold/filmcraft?rev=2c5ba7a72619935f0481f99d59409e0d1844b234#2c5ba7a72619935f0481f99d59409e0d1844b234" +version = "0.6.0" +source = "git+https://github.com/storytold/filmcraft?rev=4e2d0bc2d3646f9cf27ef7aa8b2095c6858c7efe#4e2d0bc2d3646f9cf27ef7aa8b2095c6858c7efe" dependencies = [ "filmcraft-bitstream", "rayon", @@ -2106,8 +2117,8 @@ dependencies = [ [[package]] name = "filmcraft-frame" -version = "0.1.1" -source = "git+https://github.com/storytold/filmcraft?rev=2c5ba7a72619935f0481f99d59409e0d1844b234#2c5ba7a72619935f0481f99d59409e0d1844b234" +version = "0.6.0" +source = "git+https://github.com/storytold/filmcraft?rev=4e2d0bc2d3646f9cf27ef7aa8b2095c6858c7efe#4e2d0bc2d3646f9cf27ef7aa8b2095c6858c7efe" dependencies = [ "filmcraft-color", "filmcraft-time", @@ -2117,16 +2128,16 @@ dependencies = [ [[package]] name = "filmcraft-geom" -version = "0.1.1" -source = "git+https://github.com/storytold/filmcraft?rev=2c5ba7a72619935f0481f99d59409e0d1844b234#2c5ba7a72619935f0481f99d59409e0d1844b234" +version = "0.6.0" +source = "git+https://github.com/storytold/filmcraft?rev=4e2d0bc2d3646f9cf27ef7aa8b2095c6858c7efe#4e2d0bc2d3646f9cf27ef7aa8b2095c6858c7efe" dependencies = [ "serde", ] [[package]] name = "filmcraft-h264" -version = "0.1.1" -source = "git+https://github.com/storytold/filmcraft?rev=2c5ba7a72619935f0481f99d59409e0d1844b234#2c5ba7a72619935f0481f99d59409e0d1844b234" +version = "0.6.0" +source = "git+https://github.com/storytold/filmcraft?rev=4e2d0bc2d3646f9cf27ef7aa8b2095c6858c7efe#4e2d0bc2d3646f9cf27ef7aa8b2095c6858c7efe" dependencies = [ "filmcraft-bitstream", "rayon", @@ -2135,8 +2146,8 @@ dependencies = [ [[package]] name = "filmcraft-h264enc" -version = "0.1.1" -source = "git+https://github.com/storytold/filmcraft?rev=2c5ba7a72619935f0481f99d59409e0d1844b234#2c5ba7a72619935f0481f99d59409e0d1844b234" +version = "0.6.0" +source = "git+https://github.com/storytold/filmcraft?rev=4e2d0bc2d3646f9cf27ef7aa8b2095c6858c7efe#4e2d0bc2d3646f9cf27ef7aa8b2095c6858c7efe" dependencies = [ "filmcraft-bitstream", "rayon", @@ -2145,8 +2156,8 @@ dependencies = [ [[package]] name = "filmcraft-hevc" -version = "0.1.1" -source = "git+https://github.com/storytold/filmcraft?rev=2c5ba7a72619935f0481f99d59409e0d1844b234#2c5ba7a72619935f0481f99d59409e0d1844b234" +version = "0.6.0" +source = "git+https://github.com/storytold/filmcraft?rev=4e2d0bc2d3646f9cf27ef7aa8b2095c6858c7efe#4e2d0bc2d3646f9cf27ef7aa8b2095c6858c7efe" dependencies = [ "filmcraft-bitstream", "rayon", @@ -2155,8 +2166,8 @@ dependencies = [ [[package]] name = "filmcraft-interchange" -version = "0.1.1" -source = "git+https://github.com/storytold/filmcraft?rev=2c5ba7a72619935f0481f99d59409e0d1844b234#2c5ba7a72619935f0481f99d59409e0d1844b234" +version = "0.6.0" +source = "git+https://github.com/storytold/filmcraft?rev=4e2d0bc2d3646f9cf27ef7aa8b2095c6858c7efe#4e2d0bc2d3646f9cf27ef7aa8b2095c6858c7efe" dependencies = [ "filmcraft-cfb", "filmcraft-color", @@ -2172,8 +2183,8 @@ dependencies = [ [[package]] name = "filmcraft-isobmff" -version = "0.1.1" -source = "git+https://github.com/storytold/filmcraft?rev=2c5ba7a72619935f0481f99d59409e0d1844b234#2c5ba7a72619935f0481f99d59409e0d1844b234" +version = "0.6.0" +source = "git+https://github.com/storytold/filmcraft?rev=4e2d0bc2d3646f9cf27ef7aa8b2095c6858c7efe#4e2d0bc2d3646f9cf27ef7aa8b2095c6858c7efe" dependencies = [ "filmcraft-bitstream", "thiserror 2.0.21", @@ -2181,13 +2192,13 @@ dependencies = [ [[package]] name = "filmcraft-matroska" -version = "0.1.1" -source = "git+https://github.com/storytold/filmcraft?rev=2c5ba7a72619935f0481f99d59409e0d1844b234#2c5ba7a72619935f0481f99d59409e0d1844b234" +version = "0.6.0" +source = "git+https://github.com/storytold/filmcraft?rev=4e2d0bc2d3646f9cf27ef7aa8b2095c6858c7efe#4e2d0bc2d3646f9cf27ef7aa8b2095c6858c7efe" [[package]] name = "filmcraft-media" -version = "0.1.1" -source = "git+https://github.com/storytold/filmcraft?rev=2c5ba7a72619935f0481f99d59409e0d1844b234#2c5ba7a72619935f0481f99d59409e0d1844b234" +version = "0.6.0" +source = "git+https://github.com/storytold/filmcraft?rev=4e2d0bc2d3646f9cf27ef7aa8b2095c6858c7efe#4e2d0bc2d3646f9cf27ef7aa8b2095c6858c7efe" dependencies = [ "filmcraft-color", "filmcraft-frame", @@ -2200,8 +2211,8 @@ dependencies = [ [[package]] name = "filmcraft-mpeg2v" -version = "0.1.1" -source = "git+https://github.com/storytold/filmcraft?rev=2c5ba7a72619935f0481f99d59409e0d1844b234#2c5ba7a72619935f0481f99d59409e0d1844b234" +version = "0.6.0" +source = "git+https://github.com/storytold/filmcraft?rev=4e2d0bc2d3646f9cf27ef7aa8b2095c6858c7efe#4e2d0bc2d3646f9cf27ef7aa8b2095c6858c7efe" dependencies = [ "rayon", "thiserror 2.0.21", @@ -2209,31 +2220,31 @@ dependencies = [ [[package]] name = "filmcraft-mpegts" -version = "0.1.1" -source = "git+https://github.com/storytold/filmcraft?rev=2c5ba7a72619935f0481f99d59409e0d1844b234#2c5ba7a72619935f0481f99d59409e0d1844b234" +version = "0.6.0" +source = "git+https://github.com/storytold/filmcraft?rev=4e2d0bc2d3646f9cf27ef7aa8b2095c6858c7efe#4e2d0bc2d3646f9cf27ef7aa8b2095c6858c7efe" dependencies = [ "thiserror 2.0.21", ] [[package]] name = "filmcraft-mxf" -version = "0.1.1" -source = "git+https://github.com/storytold/filmcraft?rev=2c5ba7a72619935f0481f99d59409e0d1844b234#2c5ba7a72619935f0481f99d59409e0d1844b234" +version = "0.6.0" +source = "git+https://github.com/storytold/filmcraft?rev=4e2d0bc2d3646f9cf27ef7aa8b2095c6858c7efe#4e2d0bc2d3646f9cf27ef7aa8b2095c6858c7efe" [[package]] name = "filmcraft-ogg" -version = "0.1.1" -source = "git+https://github.com/storytold/filmcraft?rev=2c5ba7a72619935f0481f99d59409e0d1844b234#2c5ba7a72619935f0481f99d59409e0d1844b234" +version = "0.6.0" +source = "git+https://github.com/storytold/filmcraft?rev=4e2d0bc2d3646f9cf27ef7aa8b2095c6858c7efe#4e2d0bc2d3646f9cf27ef7aa8b2095c6858c7efe" [[package]] name = "filmcraft-opus" -version = "0.1.1" -source = "git+https://github.com/storytold/filmcraft?rev=2c5ba7a72619935f0481f99d59409e0d1844b234#2c5ba7a72619935f0481f99d59409e0d1844b234" +version = "0.6.0" +source = "git+https://github.com/storytold/filmcraft?rev=4e2d0bc2d3646f9cf27ef7aa8b2095c6858c7efe#4e2d0bc2d3646f9cf27ef7aa8b2095c6858c7efe" [[package]] name = "filmcraft-project" -version = "0.1.1" -source = "git+https://github.com/storytold/filmcraft?rev=2c5ba7a72619935f0481f99d59409e0d1844b234#2c5ba7a72619935f0481f99d59409e0d1844b234" +version = "0.6.0" +source = "git+https://github.com/storytold/filmcraft?rev=4e2d0bc2d3646f9cf27ef7aa8b2095c6858c7efe#4e2d0bc2d3646f9cf27ef7aa8b2095c6858c7efe" dependencies = [ "filmcraft-color", "filmcraft-geom", @@ -2246,8 +2257,8 @@ dependencies = [ [[package]] name = "filmcraft-prores" -version = "0.1.1" -source = "git+https://github.com/storytold/filmcraft?rev=2c5ba7a72619935f0481f99d59409e0d1844b234#2c5ba7a72619935f0481f99d59409e0d1844b234" +version = "0.6.0" +source = "git+https://github.com/storytold/filmcraft?rev=4e2d0bc2d3646f9cf27ef7aa8b2095c6858c7efe#4e2d0bc2d3646f9cf27ef7aa8b2095c6858c7efe" dependencies = [ "filmcraft-bitstream", "rayon", @@ -2256,16 +2267,16 @@ dependencies = [ [[package]] name = "filmcraft-time" -version = "0.1.1" -source = "git+https://github.com/storytold/filmcraft?rev=2c5ba7a72619935f0481f99d59409e0d1844b234#2c5ba7a72619935f0481f99d59409e0d1844b234" +version = "0.6.0" +source = "git+https://github.com/storytold/filmcraft?rev=4e2d0bc2d3646f9cf27ef7aa8b2095c6858c7efe#4e2d0bc2d3646f9cf27ef7aa8b2095c6858c7efe" dependencies = [ "serde", ] [[package]] name = "filmcraft-vp9" -version = "0.1.1" -source = "git+https://github.com/storytold/filmcraft?rev=2c5ba7a72619935f0481f99d59409e0d1844b234#2c5ba7a72619935f0481f99d59409e0d1844b234" +version = "0.6.0" +source = "git+https://github.com/storytold/filmcraft?rev=4e2d0bc2d3646f9cf27ef7aa8b2095c6858c7efe#4e2d0bc2d3646f9cf27ef7aa8b2095c6858c7efe" dependencies = [ "filmcraft-bitstream", "rayon", diff --git a/Cargo.toml b/Cargo.toml index 38ba10a0e..ced9393fc 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -71,26 +71,26 @@ image = { version = "0.25", default-features = false, features = ["png", "jpeg"] # media codecs + containers: FilmCraft's pure-Rust crates, pinned to a revision (plan/adr/0001). # Only `crates/media`, `crates/export` and `crates/interchange` may depend on these. -filmcraft-codecs = { git = "https://github.com/storytold/filmcraft", rev = "2c5ba7a72619935f0481f99d59409e0d1844b234" } -filmcraft-media = { git = "https://github.com/storytold/filmcraft", rev = "2c5ba7a72619935f0481f99d59409e0d1844b234" } -filmcraft-frame = { git = "https://github.com/storytold/filmcraft", rev = "2c5ba7a72619935f0481f99d59409e0d1844b234" } -filmcraft-color = { git = "https://github.com/storytold/filmcraft", rev = "2c5ba7a72619935f0481f99d59409e0d1844b234" } -filmcraft-time = { git = "https://github.com/storytold/filmcraft", rev = "2c5ba7a72619935f0481f99d59409e0d1844b234" } +filmcraft-codecs = { git = "https://github.com/storytold/filmcraft", rev = "4e2d0bc2d3646f9cf27ef7aa8b2095c6858c7efe" } +filmcraft-media = { git = "https://github.com/storytold/filmcraft", rev = "4e2d0bc2d3646f9cf27ef7aa8b2095c6858c7efe" } +filmcraft-frame = { git = "https://github.com/storytold/filmcraft", rev = "4e2d0bc2d3646f9cf27ef7aa8b2095c6858c7efe" } +filmcraft-color = { git = "https://github.com/storytold/filmcraft", rev = "4e2d0bc2d3646f9cf27ef7aa8b2095c6858c7efe" } +filmcraft-time = { git = "https://github.com/storytold/filmcraft", rev = "4e2d0bc2d3646f9cf27ef7aa8b2095c6858c7efe" } # encoders + muxer (crates/export) -filmcraft-h264enc = { git = "https://github.com/storytold/filmcraft", rev = "2c5ba7a72619935f0481f99d59409e0d1844b234" } -filmcraft-prores = { git = "https://github.com/storytold/filmcraft", rev = "2c5ba7a72619935f0481f99d59409e0d1844b234" } -filmcraft-aac = { git = "https://github.com/storytold/filmcraft", rev = "2c5ba7a72619935f0481f99d59409e0d1844b234" } -filmcraft-isobmff = { git = "https://github.com/storytold/filmcraft", rev = "2c5ba7a72619935f0481f99d59409e0d1844b234" } +filmcraft-h264enc = { git = "https://github.com/storytold/filmcraft", rev = "4e2d0bc2d3646f9cf27ef7aa8b2095c6858c7efe" } +filmcraft-prores = { git = "https://github.com/storytold/filmcraft", rev = "4e2d0bc2d3646f9cf27ef7aa8b2095c6858c7efe" } +filmcraft-aac = { git = "https://github.com/storytold/filmcraft", rev = "4e2d0bc2d3646f9cf27ef7aa8b2095c6858c7efe" } +filmcraft-isobmff = { git = "https://github.com/storytold/filmcraft", rev = "4e2d0bc2d3646f9cf27ef7aa8b2095c6858c7efe" } # decoder used as the round-trip oracle of effectcraft-opusenc (crates/export dev-dependency) -filmcraft-opus = { git = "https://github.com/storytold/filmcraft", rev = "2c5ba7a72619935f0481f99d59409e0d1844b234" } -filmcraft-vp9 = { git = "https://github.com/storytold/filmcraft", rev = "2c5ba7a72619935f0481f99d59409e0d1844b234" } +filmcraft-opus = { git = "https://github.com/storytold/filmcraft", rev = "4e2d0bc2d3646f9cf27ef7aa8b2095c6858c7efe" } +filmcraft-vp9 = { git = "https://github.com/storytold/filmcraft", rev = "4e2d0bc2d3646f9cf27ef7aa8b2095c6858c7efe" } # decoders used as the round-trip oracles of effectcraft-hevcenc / effectcraft-av1enc (dev-dependencies only) -filmcraft-hevc = { git = "https://github.com/storytold/filmcraft", rev = "2c5ba7a72619935f0481f99d59409e0d1844b234" } -filmcraft-av1 = { git = "https://github.com/storytold/filmcraft", rev = "2c5ba7a72619935f0481f99d59409e0d1844b234" } +filmcraft-hevc = { git = "https://github.com/storytold/filmcraft", rev = "4e2d0bc2d3646f9cf27ef7aa8b2095c6858c7efe" } +filmcraft-av1 = { git = "https://github.com/storytold/filmcraft", rev = "4e2d0bc2d3646f9cf27ef7aa8b2095c6858c7efe" } # timeline interchange (EDL, FCP7 XML, FCPXML, OTIO) for Premiere Pro interop (crates/interchange only) -filmcraft-interchange = { git = "https://github.com/storytold/filmcraft", rev = "2c5ba7a72619935f0481f99d59409e0d1844b234" } -filmcraft-project = { git = "https://github.com/storytold/filmcraft", rev = "2c5ba7a72619935f0481f99d59409e0d1844b234" } -filmcraft-geom = { git = "https://github.com/storytold/filmcraft", rev = "2c5ba7a72619935f0481f99d59409e0d1844b234" } +filmcraft-interchange = { git = "https://github.com/storytold/filmcraft", rev = "4e2d0bc2d3646f9cf27ef7aa8b2095c6858c7efe" } +filmcraft-project = { git = "https://github.com/storytold/filmcraft", rev = "4e2d0bc2d3646f9cf27ef7aa8b2095c6858c7efe" } +filmcraft-geom = { git = "https://github.com/storytold/filmcraft", rev = "4e2d0bc2d3646f9cf27ef7aa8b2095c6858c7efe" } # gpu (the same wgpu eframe / egui-wgpu use) wgpu = { version = "30.0.1", features = ["fragile-send-sync-non-atomic-wasm"] } diff --git a/crates/export/tests/vp9_roundtrip.rs b/crates/export/tests/vp9_roundtrip.rs index e7c2b0346..1432e3892 100644 --- a/crates/export/tests/vp9_roundtrip.rs +++ b/crates/export/tests/vp9_roundtrip.rs @@ -41,7 +41,7 @@ fn source(w: usize, h: usize, kind: u32) -> (Vec, Vec, Vec) { } fn decode(frame: &[u8]) -> filmcraft_vp9::Picture { - let mut d = Decoder::new(); + let mut d = Decoder::with_threads(1); let mut pics = d.decode(frame, 0).expect("decode"); assert_eq!(pics.len(), 1); pics.remove(0) @@ -189,7 +189,7 @@ fn planes_of(p: &filmcraft_vp9::Picture, w: usize, h: usize) -> [Vec; 3] { fn run_sequence(cfg: EncoderConfig, n: usize, kind: u32) -> (usize, f64) { let (w, h) = (cfg.width as usize, cfg.height as usize); let mut e = Vp9Encoder::new(cfg); - let mut d = Decoder::new(); + let mut d = Decoder::with_threads(1); let (mut bytes, mut db) = (0usize, 0.0); for t in 0..n { let (y, u, v) = moving(w, h, t, kind); @@ -344,7 +344,7 @@ fn frames_never_end_in_a_superframe_marker() { } let Some((w, h, q, frames, recon)) = padded else { panic!("no frame needed the padding: widen the search") }; // The padded frames decode exactly, with FilmCraft's decoder and with ffmpeg. - let mut d = Decoder::new(); + let mut d = Decoder::with_threads(1); for (t, (f, r)) in frames.iter().zip(&recon).enumerate() { let mut pics = d.decode(f, t as i64).unwrap_or_else(|e| panic!("{w}x{h} q{q} frame {t}: {e:?}")); assert_eq!(planes_of(&pics.remove(0), w, h), *r, "{w}x{h} q{q} frame {t}"); diff --git a/crates/host/tests/timeline_interop.rs b/crates/host/tests/timeline_interop.rs index 5123dc721..9688a409f 100644 --- a/crates/host/tests/timeline_interop.rs +++ b/crates/host/tests/timeline_interop.rs @@ -56,8 +56,9 @@ fn still(p: &mut fp::Project, path: &str) -> fp::ItemId { color: Default::default(), has_alpha: false, bitrate: None, + hdr: None, }), - audio: None, + audio_streams: Vec::new(), container: String::new(), start_timecode: None, file_size: None, diff --git a/crates/interchange/src/export.rs b/crates/interchange/src/export.rs index 7da18913a..da6ef1382 100644 --- a/crates/interchange/src/export.rs +++ b/crates/interchange/src/export.rs @@ -684,8 +684,12 @@ fn media_info(name: &str, kind: MediaKind, duration: Tick, video: Option { MediaKind::AudioOnly => FootageKind::Audio, MediaKind::Still => FootageKind::Still, MediaKind::ImageSequence => FootageKind::Sequence, - _ if v.is_none() && info.audio.is_some() => FootageKind::Audio, + _ if v.is_none() && info.has_audio() => FootageKind::Audio, _ => FootageKind::Video, }; let _ = name; @@ -284,7 +284,7 @@ impl Importer<'_> { frame_rate: rate_in(info.frame_rate()), duration: tick_in(info.duration), has_video: kind != FootageKind::Audio, - has_audio: info.audio.is_some() || kind == FootageKind::Audio, + has_audio: info.has_audio() || kind == FootageKind::Audio, alpha: if v.is_some_and(|v| v.has_alpha) { AlphaMode::Straight } else { AlphaMode::Ignore }, codec: v.map(|v| v.codec.clone()).unwrap_or_default(), missing: true, diff --git a/crates/interchange/src/tests.rs b/crates/interchange/src/tests.rs index 3ad7fcaae..6e29c79d2 100644 --- a/crates/interchange/src/tests.rs +++ b/crates/interchange/src/tests.rs @@ -36,8 +36,12 @@ fn info(kind: filmcraft_media::MediaKind, video: bool, audio: bool) -> filmcraft color: Default::default(), has_alpha: false, bitrate: None, + hdr: None, }), - audio: audio.then(|| filmcraft_media::AudioStreamInfo { sample_rate: 48_000, channels: 2, codec: "pcm".into(), bits_per_sample: Some(16) }), + audio_streams: audio + .then(|| filmcraft_media::AudioStreamInfo { sample_rate: 48_000, channels: 2, codec: "pcm".into(), bits_per_sample: Some(16) }) + .into_iter() + .collect(), container: String::new(), start_timecode: None, file_size: None, diff --git a/crates/media/src/pool.rs b/crates/media/src/pool.rs index ce980c563..f0426c1b0 100644 --- a/crates/media/src/pool.rs +++ b/crates/media/src/pool.rs @@ -494,7 +494,7 @@ impl MediaPool { let path: Arc = footage.path.as_str().into(); let src = self.inner.source(&path).ok_or("the file can't be opened")?; let s0 = start.to_units_floor(rate as i64); - let native = src.info().audio.as_ref().map_or(rate, |a| a.sample_rate); + let native = src.info().audio().map_or(rate, |a| a.sample_rate); if native == 0 || rate == 0 || native == rate { return read_stereo(&src, s0, frames, rate); } @@ -648,9 +648,12 @@ impl Inner { let rate = footage.frame_rate; let n = Self::frame_count(rate, footage.duration); let i = Self::frame_index(rate, t, n, footage.loop_count); - // Middle of the frame in the file's own timing (robust to timestamp rounding). + // A quarter into the frame in the file's own timing: FilmCraft rounds a request to + // the track's nearest timestamp unit (halves up), so the middle of a frame would + // land on the next one when the timescale is the frame rate; a quarter still + // tolerates timestamps rounded either way. let native = footage.native_rate.unwrap_or(rate); - let mt = native.tick_of(i) + Tick(native.frame_duration().0 / 2); + let mt = native.tick_of(i) + Tick(native.frame_duration().0 / 4); Loc { key: key(&footage.path, i), media_t: Some(mt) } } } diff --git a/crates/media/src/probe.rs b/crates/media/src/probe.rs index 6e8718fdf..17f285dda 100644 --- a/crates/media/src/probe.rs +++ b/crates/media/src/probe.rs @@ -86,7 +86,7 @@ pub(crate) fn footage_from_info(path: &str, info: &filmcraft_media::MediaInfo) - (_, None) => FootageKind::Audio, }; let rate = v.map_or(FrameRate::FPS_30, |v| rate_from_fc(v.frame_rate)); - let codec = match (v, info.audio.as_ref()) { + let codec = match (v, info.audio()) { (Some(v), _) => v.codec.clone(), (None, Some(a)) => a.codec.clone(), _ => String::new(), @@ -101,7 +101,7 @@ pub(crate) fn footage_from_info(path: &str, info: &filmcraft_media::MediaInfo) - native_rate: None, duration: if kind == FootageKind::Still { Tick::ZERO } else { Tick(info.duration.0) }, has_video: v.is_some(), - has_audio: info.audio.is_some(), + has_audio: info.has_audio(), alpha: if v.is_some_and(|v| v.has_alpha) { AlphaMode::Straight } else { AlphaMode::Ignore }, premul_color: [0.0; 3], loop_count: 1, From 8e069d023c8693c78c7c41ff17c4a51927ae2032 Mon Sep 17 00:00:00 2001 From: vangel76 Date: Sun, 11 Oct 2026 15:00:04 +0200 Subject: [PATCH 2/3] #804: hardware video decoding through filmcraft-platform The desktop app registers FilmCraft's hardware decoders (VideoToolbox on macOS, Media Foundation / DXVA on Windows, NVDEC and VA-API on Linux) in front of the software decoders, on a background thread (probing the drivers takes 0.3-0.6 s). filmcraft-platform is a dependency of the app binary only: all OS FFI stays in that FilmCraft crate, our crates stay unsafe-free and the L0-L4 / wasm32 builds are unchanged. - Settings > Import > Enable Hardware Accelerated Decoding (import.hardwareDecoding, on by default; automation id settings.import.hardwareDecoding; prefs.set): flips FilmCraft's switch through FootageSource::set_hardware_decoding; a change closes the media pool's open decoders, so footage already open follows it. - GPU Information and the System Compatibility Report show the backend in use and its counters. - Streams the hardware does not take, decoders that fail mid-stream, other codecs, the CLI and the web app decode in software. - tests/hardware_decoding.rs: off decodes in software, VP9 stays in software, and a hardware smoke test (skips without a device) matches the software frames and follows the switch. Co-Authored-By: Claude Opus 5.5 --- Cargo.lock | 156 +++++++++++++++++++ Cargo.toml | 5 + apps/effectcraft/Cargo.toml | 14 ++ apps/effectcraft/src/hw_decode.rs | 22 +++ apps/effectcraft/src/main.rs | 4 + apps/effectcraft/tests/hardware_decoding.rs | 161 ++++++++++++++++++++ crates/engine/src/commands/app_more.rs | 4 + crates/engine/src/prefs.rs | 6 + crates/engine/src/session_settings.rs | 3 + crates/media/src/hw.rs | 31 ++++ crates/media/src/lib.rs | 1 + crates/media/src/pool.rs | 14 ++ crates/render/src/lib.rs | 48 ++++++ crates/ui-egui/src/i18n/ui.rs | 8 + crates/ui-egui/src/menus.rs | 3 +- docs/architecture.md | 2 +- docs/gaps.md | 2 +- docs/hardware-parity.md | 5 +- docs/preferences.md | 7 + 19 files changed, 491 insertions(+), 5 deletions(-) create mode 100644 apps/effectcraft/src/hw_decode.rs create mode 100644 apps/effectcraft/tests/hardware_decoding.rs create mode 100644 crates/media/src/hw.rs diff --git a/Cargo.lock b/Cargo.lock index 9e990db30..00310085f 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1148,9 +1148,13 @@ dependencies = [ "cpal", "effectcraft-engine", "effectcraft-host", + "effectcraft-media", + "effectcraft-raster", + "effectcraft-render", "effectcraft-ui-egui", "eframe", "egui", + "filmcraft-platform", "fmv-macos-events", "log", "muda", @@ -2044,6 +2048,11 @@ dependencies = [ "thiserror 2.0.21", ] +[[package]] +name = "filmcraft-audio-dsp" +version = "0.6.0" +source = "git+https://github.com/storytold/filmcraft?rev=4e2d0bc2d3646f9cf27ef7aa8b2095c6858c7efe#4e2d0bc2d3646f9cf27ef7aa8b2095c6858c7efe" + [[package]] name = "filmcraft-av1" version = "0.6.0" @@ -2059,6 +2068,19 @@ name = "filmcraft-bitstream" version = "0.6.0" source = "git+https://github.com/storytold/filmcraft?rev=4e2d0bc2d3646f9cf27ef7aa8b2095c6858c7efe#4e2d0bc2d3646f9cf27ef7aa8b2095c6858c7efe" +[[package]] +name = "filmcraft-captions" +version = "0.6.0" +source = "git+https://github.com/storytold/filmcraft?rev=4e2d0bc2d3646f9cf27ef7aa8b2095c6858c7efe#4e2d0bc2d3646f9cf27ef7aa8b2095c6858c7efe" +dependencies = [ + "filmcraft-color", + "filmcraft-project", + "filmcraft-text", + "filmcraft-time", + "roxmltree", + "thiserror 2.0.21", +] + [[package]] name = "filmcraft-cfb" version = "0.6.0" @@ -2115,6 +2137,35 @@ dependencies = [ "thiserror 2.0.21", ] +[[package]] +name = "filmcraft-export" +version = "0.6.0" +source = "git+https://github.com/storytold/filmcraft?rev=4e2d0bc2d3646f9cf27ef7aa8b2095c6858c7efe#4e2d0bc2d3646f9cf27ef7aa8b2095c6858c7efe" +dependencies = [ + "filmcraft-aac", + "filmcraft-apv", + "filmcraft-audio-dsp", + "filmcraft-color", + "filmcraft-dnx", + "filmcraft-frame", + "filmcraft-geom", + "filmcraft-h264enc", + "filmcraft-isobmff", + "filmcraft-media", + "filmcraft-mxf", + "filmcraft-project", + "filmcraft-prores", + "filmcraft-render", + "filmcraft-text", + "filmcraft-time", + "image", + "rayon", + "serde", + "serde_json", + "thiserror 2.0.21", + "web-time", +] + [[package]] name = "filmcraft-frame" version = "0.6.0" @@ -2241,6 +2292,29 @@ name = "filmcraft-opus" version = "0.6.0" source = "git+https://github.com/storytold/filmcraft?rev=4e2d0bc2d3646f9cf27ef7aa8b2095c6858c7efe#4e2d0bc2d3646f9cf27ef7aa8b2095c6858c7efe" +[[package]] +name = "filmcraft-platform" +version = "0.6.0" +source = "git+https://github.com/storytold/filmcraft?rev=4e2d0bc2d3646f9cf27ef7aa8b2095c6858c7efe#4e2d0bc2d3646f9cf27ef7aa8b2095c6858c7efe" +dependencies = [ + "filmcraft-bitstream", + "filmcraft-codecs", + "filmcraft-color", + "filmcraft-export", + "filmcraft-frame", + "filmcraft-h264", + "filmcraft-hevc", + "filmcraft-isobmff", + "filmcraft-time", + "libloading", + "log", + "objc2-core-foundation", + "objc2-core-media", + "objc2-core-video", + "objc2-video-toolbox", + "windows 0.62.2", +] + [[package]] name = "filmcraft-project" version = "0.6.0" @@ -2265,6 +2339,38 @@ dependencies = [ "thiserror 2.0.21", ] +[[package]] +name = "filmcraft-render" +version = "0.6.0" +source = "git+https://github.com/storytold/filmcraft?rev=4e2d0bc2d3646f9cf27ef7aa8b2095c6858c7efe#4e2d0bc2d3646f9cf27ef7aa8b2095c6858c7efe" +dependencies = [ + "filmcraft-audio-dsp", + "filmcraft-captions", + "filmcraft-color", + "filmcraft-frame", + "filmcraft-geom", + "filmcraft-media", + "filmcraft-project", + "filmcraft-text", + "filmcraft-time", + "rayon", + "serde", + "serde_json", +] + +[[package]] +name = "filmcraft-text" +version = "0.6.0" +source = "git+https://github.com/storytold/filmcraft?rev=4e2d0bc2d3646f9cf27ef7aa8b2095c6858c7efe#4e2d0bc2d3646f9cf27ef7aa8b2095c6858c7efe" +dependencies = [ + "filmcraft-color", + "harfrust", + "skrifa", + "unicode-bidi", + "unicode-linebreak", + "unicode-segmentation", +] + [[package]] name = "filmcraft-time" version = "0.6.0" @@ -3539,7 +3645,9 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2a180dd8642fa45cdb7dd721cd4c11b1cadd4929ce112ebd8b9f5803cc79d536" dependencies = [ "bitflags 2.13.2", + "block2 0.6.2", "dispatch2", + "libc", "objc2 0.6.4", ] @@ -3580,6 +3688,37 @@ dependencies = [ "objc2-foundation 0.2.2", ] +[[package]] +name = "objc2-core-media" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "05ec576860167a15dd9fce7fbee7512beb4e31f532159d3482d1f9c6caedf31d" +dependencies = [ + "bitflags 2.13.2", + "block2 0.6.2", + "dispatch2", + "objc2 0.6.4", + "objc2-core-audio", + "objc2-core-audio-types", + "objc2-core-foundation", + "objc2-core-video", +] + +[[package]] +name = "objc2-core-video" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d425caf1df73233f29fd8a5c3e5edbc30d2d4307870f802d18f00d83dc5141a6" +dependencies = [ + "bitflags 2.13.2", + "block2 0.6.2", + "objc2 0.6.4", + "objc2-core-foundation", + "objc2-core-graphics", + "objc2-io-surface", + "objc2-metal 0.3.2", +] + [[package]] name = "objc2-encode" version = "4.1.0" @@ -3752,6 +3891,23 @@ dependencies = [ "objc2-foundation 0.2.2", ] +[[package]] +name = "objc2-video-toolbox" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "05bf9a3c14831a7d9641b0d81d87dd913ee238a012b2fde27db5a84b56f5df3e" +dependencies = [ + "bitflags 2.13.2", + "block2 0.6.2", + "objc2 0.6.4", + "objc2-core-foundation", + "objc2-core-graphics", + "objc2-core-media", + "objc2-core-video", + "objc2-foundation 0.3.2", + "objc2-metal 0.3.2", +] + [[package]] name = "once_cell" version = "1.21.4" diff --git a/Cargo.toml b/Cargo.toml index ced9393fc..a15263338 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -71,6 +71,11 @@ image = { version = "0.25", default-features = false, features = ["png", "jpeg"] # media codecs + containers: FilmCraft's pure-Rust crates, pinned to a revision (plan/adr/0001). # Only `crates/media`, `crates/export` and `crates/interchange` may depend on these. +# The exception is `filmcraft-platform` (hardware video decoding through the operating system: +# VideoToolbox, Media Foundation, NVDEC, VA-API). It is FilmCraft's one crate with OS FFI (`unsafe` +# confined to its FFI modules), so only the desktop app binary (`apps/effectcraft`, L6) depends on +# it and registers its decoders at startup; our own crates stay `unsafe`-free and wasm-clean. +filmcraft-platform = { git = "https://github.com/storytold/filmcraft", rev = "4e2d0bc2d3646f9cf27ef7aa8b2095c6858c7efe" } filmcraft-codecs = { git = "https://github.com/storytold/filmcraft", rev = "4e2d0bc2d3646f9cf27ef7aa8b2095c6858c7efe" } filmcraft-media = { git = "https://github.com/storytold/filmcraft", rev = "4e2d0bc2d3646f9cf27ef7aa8b2095c6858c7efe" } filmcraft-frame = { git = "https://github.com/storytold/filmcraft", rev = "4e2d0bc2d3646f9cf27ef7aa8b2095c6858c7efe" } diff --git a/apps/effectcraft/Cargo.toml b/apps/effectcraft/Cargo.toml index 5fabc10cb..3a92cdbd6 100644 --- a/apps/effectcraft/Cargo.toml +++ b/apps/effectcraft/Cargo.toml @@ -22,11 +22,25 @@ pollster = { workspace = true } cpal = { workspace = true } log = { workspace = true } +# Settings ▸ Import ▸ Enable Hardware Accelerated Decoding: the operating system's H.264 / HEVC +# decoders (VideoToolbox, Media Foundation / DXVA, NVDEC, VA-API), registered at startup in front of +# FilmCraft's software decoders. All OS FFI lives in this FilmCraft crate; libva and the NVIDIA +# driver are loaded at run time, so building needs no SDK and a system without them decodes in +# software. +[target.'cfg(not(target_arch = "wasm32"))'.dependencies] +filmcraft-platform = { workspace = true } + [target.'cfg(target_os = "linux")'.dependencies] # Settings ▸ Appearance ▸ Sync with System: the XDG desktop portal's colour scheme. Already in # Cargo.lock (accesskit, through eframe), so it adds no new crates. zbus = "5.19" +[dev-dependencies] +# tests/hardware_decoding.rs: footage decoded with the registered hardware decoders +effectcraft-media = { workspace = true } +effectcraft-raster = { workspace = true } +effectcraft-render = { workspace = true } + [build-dependencies] # Icon + VERSIONINFO for effectcraft.exe; build.rs only uses it when targeting Windows. winresource = "0.1" diff --git a/apps/effectcraft/src/hw_decode.rs b/apps/effectcraft/src/hw_decode.rs new file mode 100644 index 000000000..ee88149c6 --- /dev/null +++ b/apps/effectcraft/src/hw_decode.rs @@ -0,0 +1,22 @@ +//! Hardware video decoding (Settings ▸ Import ▸ Enable Hardware Accelerated Decoding). +//! +//! FilmCraft's `filmcraft-platform` puts the operating system's H.264 / HEVC decoders +//! (VideoToolbox on macOS, Media Foundation / DXVA on Windows, NVDEC and VA-API on Linux) in front +//! of FilmCraft's software decoders. All OS FFI stays in that crate; EffectCraft only registers it +//! here and flips the setting through `effectcraft-media`. Registering is harmless: while the +//! setting is off, for streams the hardware can't take, and when no driver is present, the software +//! decoders run, and a hardware decoder that fails mid-stream switches to software. + +/// Register the hardware decoders on a background thread: probing the drivers (loading libva or +/// the NVIDIA driver, opening a render node) must not hold up the window. Footage opened before it +/// finishes decodes in software until it is reopened. +pub fn register_in_background() { + let spawned = std::thread::Builder::new().name("ec-hw-decode".into()).spawn(|| { + let t0 = std::time::Instant::now(); + let available = filmcraft_platform::register(); + log::info!("hardware video decoding: {available:?} ({:.0} ms)", t0.elapsed().as_secs_f64() * 1e3); + }); + if let Err(e) = spawned { + log::warn!("hardware video decoding not registered: {e}"); + } +} diff --git a/apps/effectcraft/src/main.rs b/apps/effectcraft/src/main.rs index 1b5f74f39..2c7ee04ff 100644 --- a/apps/effectcraft/src/main.rs +++ b/apps/effectcraft/src/main.rs @@ -18,6 +18,7 @@ mod appimage; mod apple_events; mod audio_out; mod control_server; +mod hw_decode; mod launch_guard; #[cfg(target_os = "macos")] mod native_menu; @@ -97,6 +98,9 @@ fn main() -> eframe::Result { }; // The font menus list the installed fonts: read their names while the window opens. effectcraft_engine::text::fonts::scan_system_in_background(); + // The operating system's H.264 / HEVC decoders, in front of the software ones (Settings ▸ + // Import ▸ Enable Hardware Accelerated Decoding). + hw_decode::register_in_background(); // Wayland shows the window's icon from its desktop entry: an AppImage brings its own. #[cfg(target_os = "linux")] appimage::integrate_from_env(ICON_PNG); diff --git a/apps/effectcraft/tests/hardware_decoding.rs b/apps/effectcraft/tests/hardware_decoding.rs new file mode 100644 index 000000000..d7a50aca2 --- /dev/null +++ b/apps/effectcraft/tests/hardware_decoding.rs @@ -0,0 +1,161 @@ +//! Settings ▸ Import ▸ Enable Hardware Accelerated Decoding with the decoders the app registers +//! (`filmcraft-platform`): off decodes in software, codecs the hardware does not take keep the +//! software decoders, and where this machine has a hardware decoder its frames match software's. +//! +//! Fixtures come from ffmpeg (an external generator, never linked) into `target/fixtures/hw-decode`; +//! the tests skip when ffmpeg is missing, and the hardware smoke test skips without a hardware +//! decoder. The switch is process-wide, so the tests take turns (`SERIAL`) and live in their own +//! test binary. + +use std::path::{Path, PathBuf}; +use std::process::Command; +use std::sync::{Mutex, PoisonError}; + +use effectcraft_media::{MediaPool, probe}; +use effectcraft_raster::Image; +use effectcraft_render::FootageSource; + +static SERIAL: Mutex<()> = Mutex::new(()); + +fn fixture(name: &str, args: &[&str]) -> Option { + if !Command::new("ffmpeg").arg("-version").output().is_ok_and(|o| o.status.success()) { + eprintln!("SKIPPED: ffmpeg not found"); + return None; + } + let out = Path::new(env!("CARGO_MANIFEST_DIR")).join("../../target/fixtures/hw-decode").join(name); + if out.exists() { + return Some(out); + } + std::fs::create_dir_all(out.parent()?).ok()?; + let tmp = out.with_file_name(format!("tmp-{}-{name}", std::process::id())); + let ok = Command::new("ffmpeg").args(["-hide_banner", "-loglevel", "error", "-y"]).args(args).arg(&tmp).status().is_ok_and(|s| s.success()); + if !ok { + eprintln!("SKIPPED: ffmpeg could not make {name}"); + return None; + } + std::fs::rename(&tmp, &out).ok()?; + Some(out) +} + +/// H.264 High with B-frames and two GOPs (reordering and a seek). +fn h264() -> Option { + fixture( + "h264_high.mp4", + &[ + "-f", + "lavfi", + "-i", + "testsrc2=s=640x360:r=24:d=2,noise=alls=12:allf=t", + "-c:v", + "libx264", + "-profile:v", + "high", + "-x264-params", + "bframes=3:keyint=24:min-keyint=24:scenecut=0", + "-pix_fmt", + "yuv420p", + ], + ) +} + +/// VP9: no hardware decoder registered on Linux takes it. +fn vp9() -> Option { + fixture("vp9.webm", &["-f", "lavfi", "-i", "testsrc2=s=320x240:r=24:d=1", "-c:v", "libvpx-vp9", "-b:v", "500k", "-pix_fmt", "yuv420p"]) +} + +/// A pool without read-ahead: no decode may outlive its test (the hardware counters are +/// process-wide). +fn quiet_pool() -> MediaPool { + let p = MediaPool::new(); + p.set_read_ahead(false); + p +} + +fn frames(pool: &MediaPool, path: &Path, n: i64) -> Vec> { + let f = probe(path.to_str().unwrap()).unwrap(); + (0..n).map(|i| pool.frame_at(&f, f.frame_rate.tick_of(i)).unwrap()).collect() +} + +fn psnr(a: &Image, b: &Image) -> f64 { + assert_eq!((a.width, a.height), (b.width, b.height)); + let se: f64 = a.data.iter().zip(&b.data).flat_map(|(p, q)| (0..4).map(move |k| ((p[k] - q[k]) as f64).powi(2))).sum(); + let mse = se / (a.data.len() * 4) as f64; + if mse == 0.0 { f64::INFINITY } else { 10.0 * (1.0 / mse).log10() } +} + +fn hw_frames(pool: &MediaPool) -> u64 { + pool.hardware_decoding().unwrap().frames +} + +#[test] +fn off_decodes_everything_in_software() { + let _serial = SERIAL.lock().unwrap_or_else(PoisonError::into_inner); + let Some(path) = h264() else { return }; + let pool = quiet_pool(); + pool.set_hardware_decoding(false); + let available = filmcraft_platform::register(); + assert_eq!(filmcraft_platform::register(), available, "registering twice is harmless"); + let before = hw_frames(&pool); + assert_eq!(frames(&pool, &path, 30).len(), 30); + let status = pool.hardware_decoding().unwrap(); + assert!(!status.enabled); + assert_eq!(status.frames, before, "off: no hardware frames ({available:?})"); + assert!(status.summary().starts_with("software"), "{}", status.summary()); + pool.set_hardware_decoding(true); +} + +#[test] +fn codecs_the_hardware_does_not_take_stay_in_software() { + let _serial = SERIAL.lock().unwrap_or_else(PoisonError::into_inner); + let Some(path) = vp9() else { return }; + filmcraft_platform::register(); + let (hw, sw) = (quiet_pool(), quiet_pool()); + hw.set_hardware_decoding(true); + let before = hw_frames(&hw); + let a = frames(&hw, &path, 12); + assert_eq!(hw_frames(&hw), before, "VP9 is not decoded in hardware"); + sw.set_hardware_decoding(false); + let b = frames(&sw, &path, 12); + for (x, y) in a.iter().zip(&b) { + assert_eq!(x.data, y.data, "same software decoder, same pixels"); + } + sw.set_hardware_decoding(true); +} + +/// Where this machine has a hardware H.264 decoder: it takes the stream, its frames match the +/// software decoder's, and switching the setting off reopens the footage in software. +#[test] +fn hardware_decode_smoke() { + let _serial = SERIAL.lock().unwrap_or_else(PoisonError::into_inner); + let Some(path) = h264() else { return }; + let available = filmcraft_platform::register(); + if let filmcraft_platform::Availability::Unavailable(why) = available { + eprintln!("SKIPPED: no hardware decoder ({why})"); + return; + } + let pool = quiet_pool(); + pool.set_hardware_decoding(true); + let before = pool.hardware_decoding().unwrap(); + let hw = frames(&pool, &path, 48); + let after = pool.hardware_decoding().unwrap(); + if after.sessions == before.sessions { + eprintln!("SKIPPED: {available:?} declined this H.264 stream (declined {} → {})", before.declined, after.declined); + return; + } + assert!(after.frames >= before.frames + 48, "{before:?} → {after:?}"); + assert!(after.summary().contains("on"), "{}", after.summary()); + + let sw_pool = quiet_pool(); + pool.set_hardware_decoding(false); + let sw = frames(&sw_pool, &path, 48); + for (i, (a, b)) in hw.iter().zip(&sw).enumerate() { + let db = psnr(a, b); + assert!(db > 60.0, "frame {i}: hardware vs software {db:.1} dB"); + } + // Off closed the pool's decoders: what it decodes from now on comes from software. + pool.clear_frames(); + let n0 = hw_frames(&pool); + assert_eq!(frames(&pool, &path, 12).len(), 12); + assert_eq!(hw_frames(&pool), n0, "off: no hardware frames after the switch"); + pool.set_hardware_decoding(true); +} diff --git a/crates/engine/src/commands/app_more.rs b/crates/engine/src/commands/app_more.rs index b6101c5a9..bb1d1e01d 100644 --- a/crates/engine/src/commands/app_more.rs +++ b/crates/engine/src/commands/app_more.rs @@ -349,6 +349,7 @@ pub fn system_report(s: &Session) -> Value { "gpu": gpu, "gpuNote": if gpu.is_none() { s.accel_note.clone() } else { None }, "windowGraphics": s.window_adapter, + "videoDecoding": s.footage.hardware_decoding().map(|h| json!({"summary": h.summary(), "detail": h})), "caches": {"layer": gb(budgets.layer as u64), "footage": gb(budgets.media as u64), "preview": gb(budgets.preview as u64)}, "import": IMPORT_FORMATS, "export": export, @@ -382,6 +383,9 @@ pub fn report_text(r: &Value) -> String { if let Some(w) = r.get("windowGraphics").and_then(Value::as_str) { t.push_str(&format!("Window graphics: {w}\n")); } + if let Some(v) = r.get("videoDecoding").and_then(|v| v.get("summary")).and_then(Value::as_str) { + t.push_str(&format!("Video decoding: {v}\n")); + } let c = &r["caches"]; t.push_str(&format!( "Cache budgets: layer {}, footage {}, preview {}\n", diff --git a/crates/engine/src/prefs.rs b/crates/engine/src/prefs.rs index 8f6af9c2c..16bb45964 100644 --- a/crates/engine/src/prefs.rs +++ b/crates/engine/src/prefs.rs @@ -223,6 +223,10 @@ page!(Import { unlabeled_alpha: String = "ask".into(), /// Default drag import as: `footage`, `comp`, `compLayerSizes`. drag_import_as: String = "footage".into(), + /// Enable Hardware Accelerated Decoding: H.264 / HEVC footage decodes on the GPU's video + /// decoder where the desktop app found one (VideoToolbox, Media Foundation, NVDEC, VA-API); + /// everything else, and everything while off, decodes in software. + hardware_decoding: bool = true, }); page!(Export { @@ -1243,6 +1247,8 @@ pub fn pages() -> Vec { Kind::Choice(&[("Footage", "footage"), ("Composition", "comp"), ("Composition - Retain Layer Sizes", "compLayerSizes")]), true, ), + Section("Video Footage"), + s("import.hardwareDecoding", "Enable Hardware Accelerated Decoding", B, true), ], }, Page { diff --git a/crates/engine/src/session_settings.rs b/crates/engine/src/session_settings.rs index 34f24786a..7d4488ce9 100644 --- a/crates/engine/src/session_settings.rs +++ b/crates/engine/src/session_settings.rs @@ -30,6 +30,9 @@ impl Session { self.apply_cache_budgets(); self.configure_disk_cache(); self.footage.set_conform_folder(self.conformed_audio_folder()); + // Settings ▸ Import ▸ Enable Hardware Accelerated Decoding (closes open decoders when it + // changes, so footage already open follows it). + self.footage.set_hardware_decoding(self.prefs.import.hardware_decoding); // Switches Affect Nested Comps changes what precomp layers render: drop cached pixels. let nested = self.prefs.general.switches_affect_nested_comps; if self.applied_nested_switches.replace(nested).is_some_and(|was| was != nested) { diff --git a/crates/media/src/hw.rs b/crates/media/src/hw.rs new file mode 100644 index 000000000..a875013e4 --- /dev/null +++ b/crates/media/src/hw.rs @@ -0,0 +1,31 @@ +//! Hardware video decoding (Settings ▸ Import ▸ Enable Hardware Accelerated Decoding). +//! +//! The decoders themselves are not here: the desktop app registers the operating system's +//! decoders (VideoToolbox, Media Foundation, NVDEC, VA-API) in front of FilmCraft's own through +//! `filmcraft-platform`, the one FilmCraft crate with OS FFI. This crate only flips FilmCraft's +//! process-wide switch and reads its counters, so it stays pure Rust, free of `unsafe` and builds +//! for wasm32 (where nothing is registered and everything decodes in software). Streams a hardware +//! decoder declines, and decoders that fail mid-stream, decode in software. + +use effectcraft_render::HardwareDecoding; + +/// Let registered hardware decoders take the video decoders created from now on (`true`, the +/// default), or decode everything in software. Returns whether the setting changed. +pub fn set_hardware_decoding(on: bool) -> bool { + let changed = filmcraft_codecs::hw::hardware_decoding() != on; + filmcraft_codecs::hw::set_hardware_decoding(on); + changed +} + +/// The setting and the counters so far. +pub fn hardware_decoding() -> HardwareDecoding { + let s = filmcraft_codecs::hw::hw_stats(); + HardwareDecoding { + enabled: filmcraft_codecs::hw::hardware_decoding(), + backend: filmcraft_codecs::hw::hw_backend().map(str::to_string), + frames: s.frames, + sessions: s.sessions, + declined: s.declined, + fallbacks: s.fallbacks, + } +} diff --git a/crates/media/src/lib.rs b/crates/media/src/lib.rs index 000c16245..56c9bb317 100644 --- a/crates/media/src/lib.rs +++ b/crates/media/src/lib.rs @@ -18,6 +18,7 @@ mod convert; pub mod exr_channels; mod gif_anim; +pub mod hw; mod layered; pub use layered::vector_doc; mod pool; diff --git a/crates/media/src/pool.rs b/crates/media/src/pool.rs index f0426c1b0..957620eae 100644 --- a/crates/media/src/pool.rs +++ b/crates/media/src/pool.rs @@ -322,6 +322,12 @@ impl MediaPool { lock(&self.inner.sources).clear(); } + /// Close every open decoder (decoded frames stay cached): the next reads open new ones, under + /// the current Hardware Accelerated Decoding setting. + pub fn close_decoders(&self) { + lock(&self.inner.sources).clear(); + } + /// Forget a file (after it changed on disk, or to retry a file that failed to open). pub fn forget(&self, path: &str) { lock(&self.inner.sources).remove(path); @@ -791,6 +797,14 @@ impl FootageSource for MediaPool { fn forget(&self, path: &str) { MediaPool::forget(self, path); } + fn set_hardware_decoding(&self, on: bool) { + if crate::hw::set_hardware_decoding(on) { + self.close_decoders(); + } + } + fn hardware_decoding(&self) -> Option { + Some(crate::hw::hardware_decoding()) + } fn cache_budget(&self) -> Option { Some(self.budget()) } diff --git a/crates/render/src/lib.rs b/crates/render/src/lib.rs index f4a5a98a3..622bcc367 100644 --- a/crates/render/src/lib.rs +++ b/crates/render/src/lib.rs @@ -85,6 +85,54 @@ pub trait FootageSource: Send + Sync { fn vector_frame(&self, _item: ItemId, _footage: &Footage, _scale: f64) -> Option> { None } + /// Settings ▸ Import ▸ Enable Hardware Accelerated Decoding: whether video decoders opened from + /// now on may be the operating system's hardware decoders (when the app registered any). A + /// change closes the open decoders, so the next reads use it; decoded frames stay valid. + /// Sources without video decoders ignore it. + fn set_hardware_decoding(&self, _on: bool) {} + /// Hardware video decoding as this source uses it (GPU Information, System Compatibility + /// Report); `None` for sources without video decoders. + fn hardware_decoding(&self) -> Option { + None + } +} + +/// Hardware video decoding in this process ([`FootageSource::hardware_decoding`]). +#[derive(Clone, Debug, Default, PartialEq, Eq, serde::Serialize)] +pub struct HardwareDecoding { + /// Settings ▸ Import ▸ Enable Hardware Accelerated Decoding. + pub enabled: bool, + /// The operating system decoder the app registered ("NVDEC", "VA-API", "VideoToolbox", + /// "Media Foundation"); `None`: none on this system or in this build, everything decodes in + /// software. + pub backend: Option, + /// Pictures the hardware decoded so far. + pub frames: u64, + /// Hardware decoders created. + pub sessions: u64, + /// Streams the hardware declined (decoded in software from the start). + pub declined: u64, + /// Hardware decoders that switched to software mid-stream. + pub fallbacks: u64, +} + +impl HardwareDecoding { + /// One line for the reports: "VA-API (on; 1200 frames decoded in hardware)", "software + /// (hardware decoding off; NVDEC available)". + pub fn summary(&self) -> String { + match (&self.backend, self.enabled) { + (None, _) => "software (no hardware decoder on this system)".into(), + (Some(b), false) => format!("software (hardware decoding off; {b} available)"), + (Some(b), true) => { + let mut s = format!("{b} (on; {} frames decoded in hardware", self.frames); + if self.fallbacks > 0 { + s.push_str(&format!(", {} switched to software", self.fallbacks)); + } + s.push(')'); + s + } + } + } } /// Footage that can be rasterised at any scale (SVG). diff --git a/crates/ui-egui/src/i18n/ui.rs b/crates/ui-egui/src/i18n/ui.rs index f7c779053..323183612 100644 --- a/crates/ui-egui/src/i18n/ui.rs +++ b/crates/ui-egui/src/i18n/ui.rs @@ -508,6 +508,8 @@ const JAPANESE: &[(&str, &str)] = &[ ("⌘/Ctrl-drag a pin to record its motion", "⌘/Ctrl+ドラッグでピンの動きを記録"), ("▣ {}", "▣ {}"), ("⚠ {} is used by:", "⚠ {}は次のコマンドで使用中:"), + ("Video Footage", "ビデオフッテージ"), + ("Enable Hardware Accelerated Decoding", "ハードウェアアクセラレーションによるデコードを有効にする"), ]; const SIMPLIFIED_CHINESE: &[(&str, &str)] = &[ @@ -1033,6 +1035,8 @@ const SIMPLIFIED_CHINESE: &[(&str, &str)] = &[ ("After Effects project", "After Effects 项目"), ("Lottie animation", "Lottie 动画"), ("Project", "项目"), + ("Video Footage", "视频素材"), + ("Enable Hardware Accelerated Decoding", "启用硬件加速解码"), ]; const TRADITIONAL_CHINESE: &[(&str, &str)] = &[ @@ -1553,6 +1557,8 @@ const TRADITIONAL_CHINESE: &[(&str, &str)] = &[ ("After Effects project", "After Effects 專案"), ("Lottie animation", "Lottie 動畫"), ("Project", "專案"), + ("Video Footage", "視訊素材"), + ("Enable Hardware Accelerated Decoding", "啟用硬體加速解碼"), ]; /// Ukrainian (uk): original translations of the same interface rows as the Chinese catalogs. @@ -1988,6 +1994,8 @@ const UKRAINIAN: &[(&str, &str)] = &[ ("⌘/Ctrl-drag a pin to record its motion", "Перетягніть опорну точку з ⌘/Ctrl, щоб записати її рух"), ("▣ {}", "▣ {}"), ("⚠ {} is used by:", "⚠ {} використовується в:"), + ("Video Footage", "Відеоматеріал"), + ("Enable Hardware Accelerated Decoding", "Увімкнути апаратно прискорене декодування"), ]; #[cfg(test)] diff --git a/crates/ui-egui/src/menus.rs b/crates/ui-egui/src/menus.rs index 5c61b4977..cd3e9eddd 100644 --- a/crates/ui-egui/src/menus.rs +++ b/crates/ui-egui/src/menus.rs @@ -774,11 +774,12 @@ pub fn frontend(app: &mut EffectcraftApp, ctx: &egui::Context, id: &str, p: Valu } "app.gpuInfo" => { let threads = std::thread::available_parallelism().map(|n| n.get()).unwrap_or(1); + let decoding = app.session.footage.hardware_decoding().map_or_else(|| "software".to_string(), |h| h.summary()); crate::panels::dialogs::info( app, "GPU Information", &format!( - "Compositing: CPU, {threads} threads (pure Rust, rayon).\nDisplay: egui on wgpu.\nLayer cache: {} MB • Preview cache: {} MB.", + "Compositing: CPU, {threads} threads (pure Rust, rayon).\nDisplay: egui on wgpu.\nVideo decoding: {decoding}.\nLayer cache: {} MB • Preview cache: {} MB.", app.session.layer_cache.budget() >> 20, app.frames.budget() >> 20 ), diff --git a/docs/architecture.md b/docs/architecture.md index 9e6fc0e7c..28a3e381d 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -47,7 +47,7 @@ cpal or muda. Everything in L0 to L4, the egui UI and the web app also build for | L4 | `host` | A fully wired `Session` (media, expressions, scripting, WebAssembly plug-ins, exporter) for the frontends | | L5 | `ui-egui` | The desktop interface: docking, panels, viewer, timeline, graph editor, dialogs, control channel | | L5 | `automation` | The MCP server, headless or bridged to the running app | -| L6 | apps `effectcraft`, `effectcraft-cli`, `effectcraft-web` | Desktop app; command-line tool (render, exec, get/set, MCP); the browser app (wasm32, [web.md](web.md)) | +| L6 | apps `effectcraft`, `effectcraft-cli`, `effectcraft-web` | Desktop app (it alone registers FilmCraft's `filmcraft-platform` hardware video decoders, the one dependency with OS FFI); command-line tool (render, exec, get/set, MCP); the browser app (wasm32, [web.md](web.md)) | Allowed same-layer edges: `path → keyframe, raster`, `text → path`, `pdf → svg, text`, `effects → project, text, path, track`, `media / expr / export / gpu → render`, `export → media`, `lottie → format`, `host → engine, script`, diff --git a/docs/gaps.md b/docs/gaps.md index acb22a630..bda4fabde 100644 --- a/docs/gaps.md +++ b/docs/gaps.md @@ -71,7 +71,7 @@ are grouped by area; the issue tracker has the rest. | 12 | **3D interaction and depth** | No 3D transform gizmos (#392, #347); Z position (#489); camera aperture (#260); reflections (#261); bevel options (#315); 3D issue reopened (#307) | Advanced 3D is After Effects' growth area since 2023 | 20–35 h | G2 · [ui-parity.md](ui-parity.md) | | 13 | **3D model formats and materials** | After Effects 26 imports FBX, USD/USDZ, STL besides glTF and OBJ (`usd_plugins`), and Substance 3D materials with mesh displacement (26.0–26.2); we read glTF and OBJ | Product shots and 3D scenes come as FBX/USDZ | 25–45 h | — · [file-format-parity.md](file-format-parity.md) | | 14 | **Object Matte** (26.2, disk cache in 26.5) | Not present; MobileSAM is already in `effectcraft-segment` and could back it | The new one-click roto; Roto Brush covers part of it | 15–30 h | G7 · [target-app-parity.md](target-app-parity.md) | -| 15 | **Hardware video decode and encode** | No VideoToolbox, NVENC, Quick Sync or VA-API; #498 asks for it; FilmCraft has VideoToolbox / NVENC / VA-API paths to borrow | 4K playback and export speed | 25–45 h | G5/G6 · [hardware-parity.md](hardware-parity.md) | +| 15 | **Hardware video decode and encode** | Decode: H.264 / HEVC through FilmCraft's `filmcraft-platform` (VideoToolbox, Media Foundation, NVDEC, VA-API; #804). Encode: no VideoToolbox, NVENC or Quick Sync; #498 asks for it; FilmCraft has VideoToolbox / NVENC paths to borrow | 4K playback and export speed | 25–45 h | G5/G6 · [hardware-parity.md](hardware-parity.md) | | 16 | **Localization depth** | Menus in 6 languages; panels in 4; effect names and ~1,650 effect parameter names untranslated; French, German, Korean, Italian, Russian (After Effects ships them) absent; #599 Hangul boxes on macOS | After Effects is used heavily in Japan, China, Korea, Brazil, Europe | 60–110 h + native review | G9 · [localization-parity.md](localization-parity.md) | | 17 | **Media Browser lists files import refuses** | `media_browser.rs` lists TGA, DPX, HDR, AVI, MXF; `media::is_importable` has none of them; #431 any image imports as a sequence; #324 MP3 goes silent | Confusing failures on common VFX formats (DPX, TGA) | 15–30 h | G6 · [file-format-parity.md](file-format-parity.md) | | 18 | **Relative paths and relinking** | Projects store absolute footage paths; no relink when a project moves | Moving or sharing a project breaks it | 6–12 h | G4 · [file-format-parity.md](file-format-parity.md) | diff --git a/docs/hardware-parity.md b/docs/hardware-parity.md index 922f75e65..edca461d0 100644 --- a/docs/hardware-parity.md +++ b/docs/hardware-parity.md @@ -1,6 +1,6 @@ # Hardware parity -> **Last reviewed:** 2026-10-10 · **Last updated:** 2026-10-10 · **Change:** major (first hardware parity document, from the code and open issues against After Effects 2026 26.5) · **Target:** Adobe After Effects 2026 +> **Last reviewed:** 2026-10-10 · **Last updated:** 2026-10-11 · **Change:** minor (hardware video decode, #804) · **Target:** Adobe After Effects 2026 Each piece of hardware After Effects uses, per platform, against EffectCraft. Ours comes from the code at `origin/main` 1446e27 (`crates/gpu`, `crates/render`, `crates/ui-egui`) and the issue @@ -17,7 +17,7 @@ monitors are missing, and two GPU start-up failures are open. | CPU fallback and device-loss recovery | yes | yes | yes; Windows ARM64 Adreno panics after device loss (#591) | yes | yes (#225 fixed) | Out of video memory falls back to the CPU (#106) | | GPU detection and choice | GPU info, renderer choice | Backend Auto per comp | "GPU isn't being detected" (#596); Intel HD 5500 fix unverified (#198) | yes | yes | No settings page listing adapters yet | | Multi-core rendering | Multi-Frame Rendering | rayon pools; preview scheduler sized to the render pool | same | same | Web Workers, one engine per worker | No benchmark against Multi-Frame Rendering | -| Hardware video decode | H.264 / HEVC / ProRes (Apple silicon) | none (software, FilmCraft) | none | none | none | FilmCraft has VideoToolbox, Media Foundation/DXVA and VA-API paths to borrow (gap 15) | +| Hardware video decode | H.264 / HEVC / ProRes (Apple silicon) | H.264 / HEVC via VideoToolbox (untested here) | H.264 / HEVC via Media Foundation / DXVA (untested here) | H.264 / HEVC via NVDEC and VA-API | none (software) | `filmcraft-platform` registered by the desktop app; Settings ▸ Import ▸ Enable Hardware Accelerated Decoding; software fallback; bit-exact with software on Linux (#804); no ProRes (gap 15) | | Hardware video encode | H.264 / HEVC via VideoToolbox, NVENC, Quick Sync (render queue and Media Encoder) | none | none | none | none | #498 asks for NVENC; our software encoders are slower and lack B-frames | | Video output devices | Mercury Transmit to AJA, Blackmagic, second display | second preview window only (Video Preview) | same | same | — | No SDI/HDMI device output | | HDR display | HDR monitoring on EDR / HDR displays | tone-mapped SDR view; no EDR output | no HDR output | no | no | Colour engine handles HDR maths (Rec. 2100 PQ/HLG output files) | @@ -40,3 +40,4 @@ the Adreno and Intel cases. | Date | Change | Summary | |---|---|---| | 2026-10-10 | major | First hardware parity document | +| 2026-10-11 | minor | Hardware video decode through FilmCraft's `filmcraft-platform` (#804) | diff --git a/docs/preferences.md b/docs/preferences.md index 157eca99f..13b153b23 100644 --- a/docs/preferences.md +++ b/docs/preferences.md @@ -123,6 +123,13 @@ Commands (CLI, MCP, control channel): - `import.unlabeledAlpha`: alpha of TGA / TIFF / movie footage (Ask opens Interpret Footage; Guess takes premultiplied for movies, straight for stills) - `import.dragImportAs`: layered files dropped on the window import as footage or a comp +- `import.hardwareDecoding`: Enable Hardware Accelerated Decoding (on by default, as in After + Effects). The desktop app decodes H.264 and HEVC footage with the operating system's hardware + decoder where it finds one (VideoToolbox on macOS, Media Foundation / DXVA on Windows, NVDEC and + VA-API on Linux, through FilmCraft's `filmcraft-platform`); streams the hardware can't take, + decoders that fail mid-stream, every other codec, the CLI and the web app decode in software. + Footage already open switches with the setting (its decoders reopen); no restart needed. Help ▸ + GPU Information and the System Compatibility Report show the backend in use - `export.defaultOutputFolder`: where relative Render Queue outputs go - `export.segmentSequences`, `export.segmentSequenceFiles`: sequences split into numbered folders of N files; `export.segmentMovies`, `export.segmentMovieMb`: movies split into From 01049719cf33fd3e50a6b8cf505d115701e5bab9 Mon Sep 17 00:00:00 2001 From: vangel76 Date: Mon, 12 Oct 2026 19:22:05 +0200 Subject: [PATCH 3/3] Adapt rotation and AAC downmix to the FilmCraft bump FilmCraft 4e2d0bc turns pure-rotation MP4/MOV frames itself and hands AAC 3.0-5.1 over in WAV order; keep only our own turn for headers it leaves as stored, and the matching downmix gains. Co-Authored-By: Claude Sonnet 5.5 --- crates/media/src/downmix.rs | 39 +++++++++++++++++++++++++++++++++++-- crates/media/src/pool.rs | 2 +- crates/media/src/probe.rs | 11 +++++++++-- 3 files changed, 47 insertions(+), 5 deletions(-) diff --git a/crates/media/src/downmix.rs b/crates/media/src/downmix.rs index 2ef417dd7..e8a9237e0 100644 --- a/crates/media/src/downmix.rs +++ b/crates/media/src/downmix.rs @@ -64,11 +64,30 @@ pub fn stereo_gains(count: usize, aac: bool) -> Vec { out } +/// The gains for AAC as FilmCraft's decoder hands it over: channel configurations 3 to 6 (3 to 6 +/// channels) are already in WAV/SMPTE order (L R C [Cs] / [LFE] Ls Rs), everything else (mono, +/// stereo, 7 and 8 channels, program-config layouts) in AAC's element order. +pub fn decoded_aac_gains(count: usize) -> Vec { + match count { + // C L R Cs, reordered to L R C Cs: the same gains, permuted. + 4 => { + let g = stereo_gains(4, true); + [1, 2, 0, 3].iter().filter_map(|&i| g.get(i).copied()).collect() + } + 3 | 5 | 6 => stereo_gains(count, false), + _ => stereo_gains(count, true), + } +} + /// Downmix planar `channels` to interleaved stereo in `out` (`out.len() / 2` frames); frames a /// channel is too short for count as silence, extra frames in `out` stay as they are. pub fn downmix_into(channels: &[Vec], aac: bool, out: &mut [f32]) { - let gains = stereo_gains(channels.len(), aac); - for (ch, g) in channels.iter().zip(&gains) { + let gains = if aac { decoded_aac_gains(channels.len()) } else { stereo_gains(channels.len(), false) }; + mix(channels, &gains, out); +} + +fn mix(channels: &[Vec], gains: &[Pair], out: &mut [f32]) { + for (ch, g) in channels.iter().zip(gains) { if g[0] == 0.0 && g[1] == 0.0 { continue; } @@ -98,6 +117,22 @@ mod tests { } } + #[test] + fn decoded_aac_is_in_wav_order_for_three_to_six_channels() { + // FilmCraft hands AAC 3.0 to 5.1 over as L R C [LFE] Ls Rs, the order of the plain layouts. + for n in [3, 5, 6] { + assert_eq!(decoded_aac_gains(n), stereo_gains(n, false), "{n} channels"); + } + let g = decoded_aac_gains(4); + assert_eq!(g.len(), 4); + assert!(g[0][0] > 0.0 && g[0][1] == 0.0 && g[1][1] > 0.0 && g[1][0] == 0.0, "L R first"); + assert!(g[2][0] > 0.0 && g[2][0] == g[2][1], "the centre is third and feeds both sides"); + assert!(g[3][0] > 0.0 && g[3][0] < g[2][0], "the rear centre is last, below the centre"); + for n in [1, 2, 7, 8, 9] { + assert_eq!(decoded_aac_gains(n), stereo_gains(n, true), "{n} channels"); + } + } + #[test] fn five_one_matches_the_bs775_matrix() { let k = 1.0 / (1.0 + 2.0 * FRAC_1_SQRT_2); diff --git a/crates/media/src/pool.rs b/crates/media/src/pool.rs index c2b47fa87..362217db8 100644 --- a/crates/media/src/pool.rs +++ b/crates/media/src/pool.rs @@ -932,7 +932,7 @@ fn read_stereo(src: &SharedSource, s0: i64, frames: usize, rate: u32) -> std::re } let buf = src.audio(s0.max(0), frames - skip, rate).map_err(|e| e.to_string())?; let Some(dst) = skip.checked_mul(2).and_then(|i| out.get_mut(i..)) else { return Ok(out) }; - let aac = src.info().audio.as_ref().is_some_and(|a| a.codec.starts_with("AAC")); + let aac = src.info().audio().is_some_and(|a| a.codec.starts_with("AAC")); crate::downmix::downmix_into(&buf.channels, aac, dst); Ok(out) } diff --git a/crates/media/src/probe.rs b/crates/media/src/probe.rs index dcf42d6d8..7c5d1c491 100644 --- a/crates/media/src/probe.rs +++ b/crates/media/src/probe.rs @@ -239,12 +239,19 @@ pub(crate) fn load_oriented(bytes: &[u8], fmt: Option) -> im /// The rotation an MP4/MOV movie is shown with (a phone held upright records sideways frames and /// says so in its video track's header): the quarter turn nearest the track's rotation. Only the /// rotation counts; any mirroring, scaling or offset in the header is ignored, and other -/// containers have none. +/// containers have none. FilmCraft turns the frames of a pure 0/90/180/270° header itself, so this +/// is only the quarter turn nearest to a header it does not turn. pub(crate) fn video_turn(bytes: &[u8]) -> Orientation { let Ok(file) = filmcraft_isobmff::open(bytes) else { return Orientation::NoTransforms }; - let Some(m) = file.tracks.iter().find(|t| t.kind == filmcraft_isobmff::TrackKind::Video).map(|t| t.matrix) else { + let Some(track) = file.tracks.iter().find(|t| t.kind == filmcraft_isobmff::TrackKind::Video) else { return Orientation::NoTransforms; }; + // A pure rotation is already applied by FilmCraft's decoder (frames and size come out + // upright); only the headers it leaves as stored (mirrored, scaled, sheared) are turned here. + if track.display_rotation().is_some() { + return Orientation::NoTransforms; + } + let m = track.matrix; // [a b u; c d v; x y w]: (a, b) is where the picture's x axis points. let (a, b) = (m[0] as i64, m[1] as i64); match (a.abs() >= b.abs(), a >= 0, b >= 0) {