diff --git a/.github/workflows/full-validation.yml b/.github/workflows/full-validation.yml index 9f7538e7..ef02f89a 100644 --- a/.github/workflows/full-validation.yml +++ b/.github/workflows/full-validation.yml @@ -227,22 +227,13 @@ jobs: - run: cargo xtask machete repo-lint: - runs-on: macos-latest + runs-on: ubuntu-latest steps: - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 - uses: dtolnay/rust-toolchain@3c5f7ea28cd621ae0bf5283f0e981fb97b8a7af9 with: toolchain: stable - - uses: dtolnay/rust-toolchain@3c5f7ea28cd621ae0bf5283f0e981fb97b8a7af9 - with: - toolchain: nightly-2026-06-28 - targets: aarch64-apple-darwin - - uses: taiki-e/install-action@7a79fe8c3a13344501c80d99cae481c1c9085912 - with: - tool: cargo-public-api@0.52.0 - - run: cargo public-api --version || cargo install cargo-public-api --version 0.52.0 --locked - - name: Run normal and strict repository policy - run: cargo xtask repo-lint --strict + - run: cargo xtask repo-lint release-integrity: runs-on: ubuntu-latest diff --git a/CHANGELOG.md b/CHANGELOG.md index 18f89943..54bfe2d6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,8 +3,49 @@ This changelog tracks the current release line. Historical phase notes and stale roadmap entries have been removed from the public documentation set. +## [Unreleased] + +Staged workspace version: `0.8.0`. + +- Breaking: decoding is strict by default in `j2k` and `j2k-native`. Callers + that intentionally accept the documented JP2/JPH optional-metadata + recoveries must select `DecodeSettings::lenient()` per job; raw codestream, + entropy, bounds, overflow, allocation, and resource-safety checks remain + strict in both modes. +- Breaking: lenient decoding now warns only when a permitted recovery was + actually used. `J2kDecodeWarning::LenientDecodeMode` is renamed to + `J2kDecodeWarning::LenientMetadataRecovery`. New + `J2kView::parse_with_settings` and `J2kDecoder::new_with_settings` + constructors keep this policy local to one input. +- Adds `CudaLosslessEncoder`, a reusable, exclusively borrowed lossless-encode + façade with explicit `Auto`, CPU-only, and strict-CUDA routing. Its opaque + result reports requested and actual execution plus any unavailable-device or + incomplete-route fallback; execution errors are returned without a hidden + full CPU retry, and the encoder can be reused after failure. +- Routes 25–38-bit `Auto` lossless encode requests to the supported CPU path + while preserving `RequireDevice` failure semantics. +- Defines stable, experimental, implementation, and binary API tiers in the + ordered release manifest, and makes the `0.7.5`→`0.8.0` semver transition + fail closed with an exact source- and behavior-break ledger. A later `0.8.x` + candidate must use a published `v0.8.0` baseline. +- Fixes the published `j2k-ml 0.7.5` `cuda` and `metal` clean-consumer + failures. The adapters now use accelerator codec decode, explicit + decoded-pixel readback, and ordinary Burn tensor upload through released + dependency APIs. +- Adds `CudaUploadBurnDecoder` and `MetalUploadBurnDecoder` as explicit aliases + for staged host-memory upload behavior while preserving `CudaBurnDecoder`, + `MetalBurnDecoder`, and their inherent method paths unchanged. +- Removes the private CubeCL and wgpu patch sources from the release graph. + This is not a direct-destination or zero-copy accelerator contract. + ## [0.7.5] - 2026-07-22 +- Known release defect: the published `j2k-ml 0.7.5` `cuda` and `metal` + features depend on accelerator interop methods that are not present in the + registry CubeCL and wgpu releases selected by clean consumers. The `cpu` + feature is unaffected. Version `0.8.0` replaces those routes with explicitly + staged accelerator decode, decoded-pixel readback, and ordinary Burn tensor + upload using released public APIs. - Compatibility exception: this release is an explicitly source-incompatible `0.7.x` patch against the published `0.7.3` API. The reviewed compatibility evidence and migration notes cover the complete contracted surface. diff --git a/Cargo.lock b/Cargo.lock index ccd90def..3d49a31b 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2554,7 +2554,7 @@ checksum = "8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682" [[package]] name = "j2k" -version = "0.7.5" +version = "0.8.0" dependencies = [ "criterion", "j2k-codec-math", @@ -2566,9 +2566,19 @@ dependencies = [ "thiserror 2.0.18", ] +[[package]] +name = "j2k-alloc-probe" +version = "0.8.0" +dependencies = [ + "j2k-native", + "j2k-profile", + "proptest", + "rayon", +] + [[package]] name = "j2k-cli" -version = "0.7.5" +version = "0.8.0" dependencies = [ "j2k", "j2k-jpeg", @@ -2578,7 +2588,7 @@ dependencies = [ [[package]] name = "j2k-codec-math" -version = "0.7.5" +version = "0.8.0" [[package]] name = "j2k-compare" @@ -2595,14 +2605,14 @@ dependencies = [ [[package]] name = "j2k-core" -version = "0.7.5" +version = "0.8.0" dependencies = [ "thiserror 2.0.18", ] [[package]] name = "j2k-cuda" -version = "0.7.5" +version = "0.8.0" dependencies = [ "criterion", "j2k", @@ -2616,7 +2626,7 @@ dependencies = [ [[package]] name = "j2k-cuda-runtime" -version = "0.7.5" +version = "0.8.0" dependencies = [ "j2k-codec-math", "j2k-core", @@ -2627,7 +2637,7 @@ dependencies = [ [[package]] name = "j2k-jpeg" -version = "0.7.5" +version = "0.8.0" dependencies = [ "criterion", "j2k-codec-math", @@ -2645,7 +2655,7 @@ dependencies = [ [[package]] name = "j2k-jpeg-cuda" -version = "0.7.5" +version = "0.8.0" dependencies = [ "criterion", "j2k-core", @@ -2658,7 +2668,7 @@ dependencies = [ [[package]] name = "j2k-jpeg-metal" -version = "0.7.5" +version = "0.8.0" dependencies = [ "criterion", "j2k-core", @@ -2674,7 +2684,7 @@ dependencies = [ [[package]] name = "j2k-metal" -version = "0.7.5" +version = "0.8.0" dependencies = [ "cc", "j2k", @@ -2693,7 +2703,7 @@ dependencies = [ [[package]] name = "j2k-metal-support" -version = "0.7.5" +version = "0.8.0" dependencies = [ "j2k-core", "j2k-test-support", @@ -2703,7 +2713,7 @@ dependencies = [ [[package]] name = "j2k-ml" -version = "0.7.5" +version = "0.8.0" dependencies = [ "burn-core", "burn-cuda", @@ -2714,7 +2724,6 @@ dependencies = [ "j2k", "j2k-core", "j2k-cuda", - "j2k-cuda-runtime", "j2k-metal", "j2k-metal-support", "j2k-native", @@ -2724,7 +2733,7 @@ dependencies = [ [[package]] name = "j2k-native" -version = "0.7.5" +version = "0.8.0" dependencies = [ "criterion", "fearless_simd", @@ -2738,18 +2747,18 @@ dependencies = [ [[package]] name = "j2k-profile" -version = "0.7.5" +version = "0.8.0" [[package]] name = "j2k-test-support" -version = "0.7.5" +version = "0.8.0" dependencies = [ "j2k-native", ] [[package]] name = "j2k-tilecodec" -version = "0.7.5" +version = "0.8.0" dependencies = [ "criterion", "flate2", @@ -2761,7 +2770,7 @@ dependencies = [ [[package]] name = "j2k-transcode" -version = "0.7.5" +version = "0.8.0" dependencies = [ "criterion", "j2k", @@ -2778,7 +2787,7 @@ dependencies = [ [[package]] name = "j2k-transcode-cuda" -version = "0.7.5" +version = "0.8.0" dependencies = [ "j2k-core", "j2k-cuda-runtime", @@ -2790,7 +2799,7 @@ dependencies = [ [[package]] name = "j2k-transcode-metal" -version = "0.7.5" +version = "0.8.0" dependencies = [ "criterion", "j2k-codec-math", @@ -2809,7 +2818,7 @@ dependencies = [ [[package]] name = "j2k-transcode-test-support" -version = "0.7.5" +version = "0.8.0" dependencies = [ "j2k-native", "j2k-transcode", @@ -2818,7 +2827,7 @@ dependencies = [ [[package]] name = "j2k-types" -version = "0.7.5" +version = "0.8.0" [[package]] name = "jni-sys" @@ -5672,6 +5681,7 @@ dependencies = [ "j2k-profile", "j2k-test-support", "proc-macro2", + "serde", "serde_json", "serde_yaml_ng", "syn", diff --git a/Cargo.toml b/Cargo.toml index c2f2a445..95a80d92 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -2,6 +2,7 @@ resolver = "2" members = [ "crates/j2k", + "crates/j2k-alloc-probe", "crates/j2k-codec-math", "crates/j2k-core", "crates/j2k-compare", @@ -30,7 +31,7 @@ exclude = [ ] [workspace.package] -version = "0.7.5" +version = "0.8.0" edition = "2021" rust-version = "1.96" license = "MIT OR Apache-2.0" @@ -63,6 +64,7 @@ fearless_simd = "0.4" libm = { version = "0.2", default-features = false } openjpeg-sys = "1.0.12" proc-macro2 = "1.0.106" +serde = { version = "1", features = ["derive"] } serde_json = "1" serde_yaml_ng = "0.10" syn = "2.0.117" diff --git a/crates/j2k-alloc-probe/Cargo.toml b/crates/j2k-alloc-probe/Cargo.toml new file mode 100644 index 00000000..2ce3880c --- /dev/null +++ b/crates/j2k-alloc-probe/Cargo.toml @@ -0,0 +1,34 @@ +[package] +name = "j2k-alloc-probe" +description = "Dev-only process-wide allocation probes for j2k codec boundaries" +version.workspace = true +edition.workspace = true +rust-version.workspace = true +license.workspace = true +publish = false + +[lib] +name = "j2k_alloc_probe" +path = "src/lib.rs" + +[dependencies] + +[dev-dependencies] +j2k-native = { path = "../j2k-native" } +j2k-profile = { path = "../j2k-profile" } +proptest = { workspace = true } +rayon = { workspace = true } + +[[test]] +name = "allocation_probe" +path = "tests/main.rs" +harness = false + +[lints.rust] +unsafe_code = "allow" +unsafe_op_in_unsafe_fn = "deny" +unreachable_pub = "warn" + +[lints.clippy] +all = { level = "warn", priority = -1 } +pedantic = { level = "warn", priority = -1 } diff --git a/crates/j2k-alloc-probe/src/lib.rs b/crates/j2k-alloc-probe/src/lib.rs new file mode 100644 index 00000000..2104f49d --- /dev/null +++ b/crates/j2k-alloc-probe/src/lib.rs @@ -0,0 +1,444 @@ +// SPDX-License-Identifier: MIT OR Apache-2.0 + +//! Process-wide allocation measurement for serial codec regression probes. +//! +//! The meter is process-global because codec work may allocate on Rayon worker +//! threads. A measurement must join all work it starts before returning. +//! Measurements are serialized and cannot be nested. Epoch checks prevent a +//! delayed allocator call from reporting into a later measurement, and each +//! measurement waits for already-admitted reporters before taking its +//! snapshot. + +use std::alloc::{GlobalAlloc, Layout, System}; +use std::sync::atomic::{AtomicU64, AtomicU8, Ordering}; + +#[cfg(test)] +use std::sync::atomic::AtomicBool; + +const IDLE: u8 = 0; +const ARMING: u8 = 1; +const ACTIVE: u8 = 2; + +#[cfg(test)] +const TEST_HOOK_IDLE: u8 = 0; +#[cfg(test)] +const TEST_HOOK_ARMED: u8 = 1; +#[cfg(test)] +const TEST_HOOK_PAUSED: u8 = 2; + +static STATE: AtomicU8 = AtomicU8::new(IDLE); +static EPOCH: AtomicU64 = AtomicU64::new(0); +static ACTIVE_REPORTERS: AtomicU64 = AtomicU64::new(0); +static ALLOCATIONS: AtomicU64 = AtomicU64::new(0); +static REALLOCATIONS: AtomicU64 = AtomicU64::new(0); +static DEALLOCATIONS: AtomicU64 = AtomicU64::new(0); +static REQUESTED_BYTES: AtomicU64 = AtomicU64::new(0); + +#[cfg(test)] +static TEST_HOOK: AtomicU8 = AtomicU8::new(TEST_HOOK_IDLE); +#[cfg(test)] +static TEST_HOOK_RELEASE: AtomicBool = AtomicBool::new(false); +#[cfg(test)] +static TEST_OBSERVED_EPOCH: AtomicU64 = AtomicU64::new(u64::MAX); +#[cfg(test)] +static TEST_RECHECKED_EPOCH: AtomicU64 = AtomicU64::new(u64::MAX); +#[cfg(test)] +static TEST_HOOK_RECORDED: AtomicBool = AtomicBool::new(false); + +struct CountingAllocator; + +#[global_allocator] +static GLOBAL_ALLOCATOR: CountingAllocator = CountingAllocator; + +// SAFETY: every operation delegates to `System` with the exact pointer, layout, +// and replacement size supplied by the caller. Atomic bookkeeping neither +// dereferences nor changes allocation pointers. +unsafe impl GlobalAlloc for CountingAllocator { + unsafe fn alloc(&self, layout: Layout) -> *mut u8 { + let reporter = Reporter::begin(); + // SAFETY: forwards the caller's valid allocation layout unchanged. + let pointer = unsafe { System.alloc(layout) }; + if !pointer.is_null() { + reporter.record_allocation(layout.size()); + } + pointer + } + + unsafe fn alloc_zeroed(&self, layout: Layout) -> *mut u8 { + let reporter = Reporter::begin(); + // SAFETY: forwards the caller's valid allocation layout unchanged. + let pointer = unsafe { System.alloc_zeroed(layout) }; + if !pointer.is_null() { + reporter.record_allocation(layout.size()); + } + pointer + } + + unsafe fn dealloc(&self, pointer: *mut u8, layout: Layout) { + let reporter = Reporter::begin(); + reporter.record_deallocation(); + // SAFETY: forwards the pointer and its original layout unchanged. + unsafe { System.dealloc(pointer, layout) }; + } + + unsafe fn realloc(&self, pointer: *mut u8, layout: Layout, new_size: usize) -> *mut u8 { + let reporter = Reporter::begin(); + // SAFETY: forwards the pointer, original layout, and requested + // replacement size unchanged. + let replacement = unsafe { System.realloc(pointer, layout, new_size) }; + if !replacement.is_null() { + reporter.record_reallocation(new_size); + } + replacement + } +} + +/// Successful allocator activity observed during one measurement window. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct AllocationStats { + /// Successful allocation calls, including zeroed allocations. + pub allocations: u64, + /// Successful reallocation calls. + pub reallocations: u64, + /// Deallocation calls observed during the window. + /// + /// This count is diagnostic only: it may include allocations created + /// before the window and never reduces a budget. + pub deallocations: u64, + /// Sum of requested bytes for successful allocations and reallocations. + /// + /// Reallocations contribute their full new requested size. Deallocations + /// never reduce this value, so freeing pre-existing storage cannot conceal + /// later allocation work. + pub requested_bytes: u64, +} + +impl AllocationStats { + /// Sum of successful allocation and reallocation calls. + #[must_use] + pub const fn allocation_calls(self) -> u64 { + self.allocations.saturating_add(self.reallocations) + } +} + +/// Upper bounds for successful allocation work in one measured operation. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct Budget { + allocation_calls: u64, + requested_bytes: u64, +} + +impl Budget { + /// Require no successful allocation or reallocation calls. + #[must_use] + pub const fn zero() -> Self { + Self { + allocation_calls: 0, + requested_bytes: 0, + } + } + + /// Bound total requested bytes while allowing any number of calls. + #[must_use] + pub const fn total_bytes(max_requested_bytes: u64) -> Self { + Self { + allocation_calls: u64::MAX, + requested_bytes: max_requested_bytes, + } + } + + /// Also bound the sum of successful allocation and reallocation calls. + #[must_use] + pub const fn with_max_calls(mut self, max_allocation_calls: u64) -> Self { + self.allocation_calls = max_allocation_calls; + self + } + + const fn accepts(self, stats: AllocationStats) -> bool { + stats.allocation_calls() <= self.allocation_calls + && stats.requested_bytes <= self.requested_bytes + } +} + +/// Measure one operation with process-global counters. +/// +/// The operation must join every worker task it starts before returning. +/// Activity from unrelated threads in the same process is intentionally part +/// of the measurement. +/// +/// # Panics +/// +/// Panics if another measurement is active or being armed. +pub fn measure(operation: impl FnOnce() -> R) -> (R, AllocationStats) { + let guard = MeasurementGuard::begin(); + let result = operation(); + let stats = guard.finish(); + (result, stats) +} + +/// Run an operation and panic after metering is disabled if it exceeds a +/// budget. +/// +/// # Panics +/// +/// Panics if the budget is exceeded or another measurement is active. +pub fn assert_allocations(label: &str, budget: Budget, operation: impl FnOnce() -> R) -> R { + let (result, stats) = measure(operation); + assert!( + budget.accepts(stats), + "{label} exceeded allocation budget: stats={stats:?}, budget={budget:?}" + ); + result +} + +struct MeasurementGuard { + finished: bool, +} + +impl MeasurementGuard { + fn begin() -> Self { + if STATE + .compare_exchange(IDLE, ARMING, Ordering::AcqRel, Ordering::Acquire) + .is_err() + { + panic!("allocation measurements must be serial and non-nested"); + } + reset_counters(); + EPOCH.fetch_add(1, Ordering::AcqRel); + STATE.store(ACTIVE, Ordering::Release); + Self { finished: false } + } + + fn finish(mut self) -> AllocationStats { + STATE.store(ARMING, Ordering::Release); + wait_for_active_reporters(); + let stats = snapshot(); + STATE.store(IDLE, Ordering::Release); + self.finished = true; + stats + } +} + +impl Drop for MeasurementGuard { + fn drop(&mut self) { + if !self.finished { + STATE.store(ARMING, Ordering::Release); + wait_for_active_reporters(); + STATE.store(IDLE, Ordering::Release); + } + } +} + +struct Reporter { + active: bool, + #[cfg(test)] + test_hooked: bool, +} + +impl Reporter { + fn begin() -> Self { + let observed_epoch = EPOCH.load(Ordering::Acquire); + let observed_active = STATE.load(Ordering::Acquire) == ACTIVE; + + let test_hooked = pause_test_reporter_after_observation(observed_active); + #[cfg(not(test))] + let _ = test_hooked; + + if !observed_active { + return Self { + active: false, + #[cfg(test)] + test_hooked: false, + }; + } + + ACTIVE_REPORTERS.fetch_add(1, Ordering::AcqRel); + let rechecked_epoch = EPOCH.load(Ordering::Acquire); + let active = STATE.load(Ordering::Acquire) == ACTIVE && rechecked_epoch == observed_epoch; + #[cfg(test)] + if test_hooked { + TEST_OBSERVED_EPOCH.store(observed_epoch, Ordering::Release); + TEST_RECHECKED_EPOCH.store(rechecked_epoch, Ordering::Release); + } + if !active { + ACTIVE_REPORTERS.fetch_sub(1, Ordering::Release); + } + Self { + active, + #[cfg(test)] + test_hooked, + } + } + + fn record_allocation(&self, size: usize) { + if self.active { + #[cfg(test)] + self.mark_test_hook_recorded(); + saturating_increment(&ALLOCATIONS); + saturating_add(&REQUESTED_BYTES, size_as_u64(size)); + } + } + + fn record_reallocation(&self, new_size: usize) { + if self.active { + #[cfg(test)] + self.mark_test_hook_recorded(); + saturating_increment(&REALLOCATIONS); + saturating_add(&REQUESTED_BYTES, size_as_u64(new_size)); + } + } + + fn record_deallocation(&self) { + if self.active { + #[cfg(test)] + self.mark_test_hook_recorded(); + saturating_increment(&DEALLOCATIONS); + } + } + + #[cfg(test)] + fn mark_test_hook_recorded(&self) { + if self.test_hooked { + TEST_HOOK_RECORDED.store(true, Ordering::Release); + } + } +} + +impl Drop for Reporter { + fn drop(&mut self) { + if self.active { + ACTIVE_REPORTERS.fetch_sub(1, Ordering::Release); + } + } +} + +fn pause_test_reporter_after_observation(observed_active: bool) -> bool { + #[cfg(test)] + if observed_active + && TEST_HOOK + .compare_exchange( + TEST_HOOK_ARMED, + TEST_HOOK_PAUSED, + Ordering::AcqRel, + Ordering::Acquire, + ) + .is_ok() + { + while !TEST_HOOK_RELEASE.load(Ordering::Acquire) { + std::hint::spin_loop(); + } + return true; + } + #[cfg(not(test))] + let _ = observed_active; + false +} + +fn wait_for_active_reporters() { + while ACTIVE_REPORTERS.load(Ordering::Acquire) != 0 { + std::hint::spin_loop(); + } +} + +fn reset_counters() { + ALLOCATIONS.store(0, Ordering::Relaxed); + REALLOCATIONS.store(0, Ordering::Relaxed); + DEALLOCATIONS.store(0, Ordering::Relaxed); + REQUESTED_BYTES.store(0, Ordering::Relaxed); +} + +fn snapshot() -> AllocationStats { + AllocationStats { + allocations: ALLOCATIONS.load(Ordering::Relaxed), + reallocations: REALLOCATIONS.load(Ordering::Relaxed), + deallocations: DEALLOCATIONS.load(Ordering::Relaxed), + requested_bytes: REQUESTED_BYTES.load(Ordering::Relaxed), + } +} + +fn saturating_increment(counter: &AtomicU64) { + saturating_add(counter, 1); +} + +fn saturating_add(counter: &AtomicU64, value: u64) { + let _ = counter.fetch_update(Ordering::Relaxed, Ordering::Relaxed, |current| { + Some(current.saturating_add(value)) + }); +} + +fn size_as_u64(size: usize) -> u64 { + u64::try_from(size).unwrap_or(u64::MAX) +} + +#[cfg(test)] +mod tests { + use std::alloc::{alloc, dealloc, Layout}; + use std::sync::atomic::{AtomicBool, Ordering}; + + use super::{ + measure, TEST_HOOK, TEST_HOOK_ARMED, TEST_HOOK_IDLE, TEST_HOOK_PAUSED, TEST_HOOK_RECORDED, + TEST_HOOK_RELEASE, TEST_OBSERVED_EPOCH, TEST_RECHECKED_EPOCH, + }; + + static WORKER_START: AtomicBool = AtomicBool::new(false); + static WORKER_ALLOCATED: AtomicBool = AtomicBool::new(false); + static WORKER_DROP: AtomicBool = AtomicBool::new(false); + + #[test] + fn delayed_allocator_observation_cannot_leak_into_the_next_measurement() { + TEST_HOOK.store(TEST_HOOK_IDLE, Ordering::Release); + TEST_HOOK_RELEASE.store(false, Ordering::Release); + TEST_OBSERVED_EPOCH.store(u64::MAX, Ordering::Release); + TEST_RECHECKED_EPOCH.store(u64::MAX, Ordering::Release); + TEST_HOOK_RECORDED.store(false, Ordering::Release); + WORKER_START.store(false, Ordering::Release); + WORKER_ALLOCATED.store(false, Ordering::Release); + WORKER_DROP.store(false, Ordering::Release); + + let worker = std::thread::spawn(|| { + while !WORKER_START.load(Ordering::Acquire) { + std::hint::spin_loop(); + } + let layout = Layout::from_size_align(4096, 1).expect("valid test allocation layout"); + // SAFETY: `layout` is nonzero and valid. The returned pointer is + // checked and later freed exactly once with the same layout. + let allocation = unsafe { alloc(layout) }; + assert!(!allocation.is_null(), "test allocation must succeed"); + WORKER_ALLOCATED.store(true, Ordering::Release); + while !WORKER_DROP.load(Ordering::Acquire) { + std::hint::spin_loop(); + } + // SAFETY: `allocation` came from `alloc(layout)`, is non-null, and + // has not been freed or reallocated. + unsafe { dealloc(allocation, layout) }; + }); + + let _ = measure(|| { + TEST_HOOK.store(TEST_HOOK_ARMED, Ordering::Release); + WORKER_START.store(true, Ordering::Release); + while TEST_HOOK.load(Ordering::Acquire) != TEST_HOOK_PAUSED { + std::hint::spin_loop(); + } + }); + + let _ = measure(|| { + TEST_HOOK_RELEASE.store(true, Ordering::Release); + while !WORKER_ALLOCATED.load(Ordering::Acquire) { + std::hint::spin_loop(); + } + }); + + WORKER_DROP.store(true, Ordering::Release); + worker.join().expect("allocation worker"); + + assert_ne!( + TEST_OBSERVED_EPOCH.load(Ordering::Acquire), + TEST_RECHECKED_EPOCH.load(Ordering::Acquire), + "the planted allocator call must span two measurement epochs" + ); + assert!( + !TEST_HOOK_RECORDED.load(Ordering::Acquire), + "an allocation observed in the prior epoch leaked into the next measurement" + ); + } +} diff --git a/crates/j2k-alloc-probe/tests/main.rs b/crates/j2k-alloc-probe/tests/main.rs new file mode 100644 index 00000000..ca21de8a --- /dev/null +++ b/crates/j2k-alloc-probe/tests/main.rs @@ -0,0 +1,431 @@ +// SPDX-License-Identifier: MIT OR Apache-2.0 + +use std::hint::black_box; +use std::panic::{catch_unwind, AssertUnwindSafe}; +use std::sync::{mpsc, OnceLock}; + +use j2k_alloc_probe::{assert_allocations, measure, Budget}; +use j2k_native::{ + decode_ht_code_block_scalar_with_workspace, encode, + encode_precomputed_htj2k_97_with_accelerator_and_max_host_bytes, + CpuOnlyJ2kEncodeStageAccelerator, DecodeSettings, DecoderContext, EncodeError, EncodeOptions, + HtCodeBlockDecodeJob, HtCodeBlockDecodeWorkspace, Image, J2kForwardDwt97Output, + PrecomputedHtj2k97Component, PrecomputedHtj2k97Image, +}; +use proptest::{ + collection::vec, + prop_assert, prop_assert_eq, + test_runner::{Config, TestRunner}, +}; +use rayon::{ThreadPool, ThreadPoolBuilder}; + +const KIB: u64 = 1024; +static PROBE_POOL: OnceLock = OnceLock::new(); + +fn main() { + probe_pool().broadcast(|_| {}); + + let cases: &[(&str, fn())] = &[ + ( + "harness_counts_allocation_and_deallocation", + harness_counts_allocation_and_deallocation, + ), + ("harness_counts_reallocation", harness_counts_reallocation), + ( + "preexisting_deallocation_cannot_credit_budget", + preexisting_deallocation_cannot_credit_budget, + ), + ( + "preexisting_frees_never_hide_requested_bytes", + preexisting_frees_never_hide_requested_bytes, + ), + ( + "concurrent_measurement_is_rejected_without_resetting_counters", + concurrent_measurement_is_rejected_without_resetting_counters, + ), + ( + "panicking_measurement_releases_global_meter", + panicking_measurement_releases_global_meter, + ), + ( + "warmed_scalar_decode_workspace_reuses_without_allocating", + warmed_scalar_decode_workspace_reuses_without_allocating, + ), + ( + "warmed_decoder_context_has_bounded_transients", + warmed_decoder_context_has_bounded_transients, + ), + ( + "profile_row_formatting_is_single_allocation", + profile_row_formatting_is_single_allocation, + ), + ( + "precomputed_encode_obeys_ledger_and_allocator_budgets", + precomputed_encode_obeys_ledger_and_allocator_budgets, + ), + ]; + + let mut failures = 0_u32; + for (name, case) in cases { + print!("test {name} ... "); + let result = catch_unwind(AssertUnwindSafe(case)); + if result.is_ok() { + println!("ok"); + } else { + failures += 1; + println!("FAILED"); + } + } + + println!( + "test result: {}. {} passed; {} failed", + if failures == 0 { "ok" } else { "FAILED" }, + cases.len() - usize::try_from(failures).expect("failure count fits usize"), + failures + ); + if failures != 0 { + std::process::exit(1); + } +} + +fn harness_counts_allocation_and_deallocation() { + let (allocation, retained) = measure(|| black_box(Box::new([0_u8; 4096]))); + assert_eq!(retained.allocations, 1); + assert_eq!(retained.reallocations, 0); + assert_eq!(retained.deallocations, 0); + assert!(retained.requested_bytes >= 4096); + drop(allocation); + + let ((), released) = measure(|| { + drop(black_box(Box::new([0_u8; 4096]))); + }); + assert_eq!(released.allocations, 1); + assert_eq!(released.reallocations, 0); + assert_eq!(released.deallocations, 1); + assert!(released.requested_bytes >= 4096); +} + +fn harness_counts_reallocation() { + let ((), stats) = measure(|| { + let mut bytes = Vec::new(); + bytes + .try_reserve_exact(16) + .expect("small allocation must succeed"); + bytes.resize(16, 1_u8); + bytes + .try_reserve_exact(4096) + .expect("growth allocation must succeed"); + black_box(&bytes); + }); + + assert_eq!(stats.allocations, 1); + assert!(stats.reallocations >= 1); + assert!(stats.deallocations >= 1); + assert!(stats.requested_bytes >= 4096); +} + +fn preexisting_deallocation_cannot_credit_budget() { + let preexisting = black_box(Box::new([0_u8; 8192])); + let violation = without_panic_output(|| { + catch_unwind(|| { + let replacement = assert_allocations( + "replacement after pre-existing free", + Budget::total_bytes(2047), + || { + drop(preexisting); + black_box(Box::new([1_u8; 2048])) + }, + ); + drop(replacement); + }) + }); + assert!( + violation.is_err(), + "a pre-existing free incorrectly credited the byte budget" + ); +} + +fn preexisting_frees_never_hide_requested_bytes() { + let mut runner = TestRunner::new(Config { + cases: 128, + failure_persistence: None, + ..Config::default() + }); + runner + .run( + &(1_usize..8192, vec(1_usize..8192, 1..48)), + |(preexisting_size, requested_sizes)| { + let preexisting = vec![0_u8; preexisting_size].into_boxed_slice(); + let expected_bytes = requested_sizes.iter().try_fold(0_u64, |total, &size| { + total.checked_add(u64::try_from(size).unwrap_or(u64::MAX)) + }); + let expected_bytes = expected_bytes.unwrap_or(u64::MAX); + + let ((), stats) = measure(|| { + drop(preexisting); + for size in &requested_sizes { + let allocation = vec![0_u8; *size].into_boxed_slice(); + black_box(&allocation); + drop(allocation); + } + }); + + prop_assert!(stats.requested_bytes >= expected_bytes); + prop_assert!(stats.allocations >= requested_sizes.len() as u64); + prop_assert_eq!(stats.reallocations, 0); + Ok(()) + }, + ) + .expect("allocation trace property"); +} + +fn concurrent_measurement_is_rejected_without_resetting_counters() { + let (entered_sender, entered_receiver) = mpsc::sync_channel(0); + let (release_sender, release_receiver) = mpsc::sync_channel(0); + let first = std::thread::spawn(move || { + measure(|| { + let marker = black_box(Box::new([3_u8; 16 * 1024])); + entered_sender + .send(()) + .expect("announce active measurement"); + release_receiver.recv().expect("release active measurement"); + marker + }) + }); + entered_receiver + .recv() + .expect("first measurement became active"); + + let second = without_panic_output(|| catch_unwind(|| measure(|| ()))); + assert!(second.is_err(), "concurrent measurement must be rejected"); + + release_sender + .send(()) + .expect("release first measurement after rejected contender"); + let (allocation, stats) = first.join().expect("first measurement thread"); + assert!(stats.allocations >= 1); + assert!( + stats.requested_bytes >= 16 * KIB, + "rejected contender reset the active measurement: {stats:?}" + ); + drop(allocation); +} + +fn panicking_measurement_releases_global_meter() { + let panic = without_panic_output(|| catch_unwind(|| measure(|| panic!("planted panic")))); + assert!(panic.is_err(), "measured panic must propagate"); + + let ((), stats) = measure(|| ()); + assert_eq!(stats.allocations, 0); + assert_eq!(stats.reallocations, 0); + assert_eq!(stats.requested_bytes, 0); +} + +fn warmed_scalar_decode_workspace_reuses_without_allocating() { + const SIGPROP_BLOCK: [u8; 12] = [ + 0x0E, 0xB2, 0x3E, 0x30, 0xFD, 0x6B, 0x5C, 0x7A, 0xF7, 0x56, 0x00, 0x02, + ]; + let job = HtCodeBlockDecodeJob { + data: &SIGPROP_BLOCK, + cleanup_length: 11, + refinement_length: 1, + width: 4, + height: 4, + output_stride: 4, + missing_bit_planes: 4, + number_of_coding_passes: 2, + num_bitplanes: 6, + roi_shift: 0, + stripe_causal: false, + strict: true, + dequantization_step: 1.0, + }; + let mut output = [0.0_f32; 16]; + let mut workspace = HtCodeBlockDecodeWorkspace::default(); + + decode_ht_code_block_scalar_with_workspace(job, &mut output, &mut workspace) + .expect("warm scalar HT workspace"); + assert_allocations("warm scalar HT workspace", Budget::zero(), || { + decode_ht_code_block_scalar_with_workspace(job, &mut output, &mut workspace) + }) + .expect("decode with warm scalar HT workspace"); + assert!( + output.iter().any(|coefficient| *coefficient != 0.0), + "fixture must exercise nonzero decode work" + ); +} + +fn warmed_decoder_context_has_bounded_transients() { + let pool = probe_pool(); + let pixels = (0_u8..64).collect::>(); + let encoded = pool + .install(|| { + encode( + &pixels, + 8, + 8, + 1, + 8, + false, + &EncodeOptions { + reversible: true, + num_decomposition_levels: 1, + ..EncodeOptions::default() + }, + ) + }) + .expect("encode decoder fixture"); + let image = Image::new(&encoded, &DecodeSettings::default()).expect("parse decoder fixture"); + let mut context = DecoderContext::default(); + let mut output = vec![0_u8; 64]; + + pool.install(|| image.decode_into(&mut output, &mut context)) + .expect("warm decoder context"); + assert_allocations( + "warm DecoderContext reuse", + Budget::total_bytes(64 * KIB).with_max_calls(64), + || pool.install(|| image.decode_into(&mut output, &mut context)), + ) + .expect("decode with warm context"); + assert_eq!(output, pixels); +} + +fn profile_row_formatting_is_single_allocation() { + let fields = [("route", "scalar"), ("result", "success")]; + drop( + j2k_profile::format_profile_key_value_fields(&fields).expect("warm profile row formatting"), + ); + + let row = assert_allocations( + "profile row formatting", + Budget::total_bytes(KIB).with_max_calls(1), + || j2k_profile::format_profile_key_value_fields(&fields), + ) + .expect("format profile fields"); + assert_eq!(row, " route=scalar result=success"); +} + +fn precomputed_encode_obeys_ledger_and_allocator_budgets() { + let image = precomputed_image(); + let options = precomputed_options(); + let pool = probe_pool(); + let exact_cap = minimum_successful_cap(pool, &image, &options); + assert!( + exact_cap > 0, + "ledger boundary must charge at least one byte" + ); + + let encoded = assert_allocations( + "precomputed HTJ2K encode", + Budget::total_bytes(512 * KIB).with_max_calls(256), + || encode_precomputed_at_cap(pool, &image, &options, exact_cap), + ) + .expect("exact ledger cap must encode"); + assert!(!encoded.is_empty()); + + let error = encode_precomputed_at_cap(pool, &image, &options, exact_cap - 1) + .expect_err("one byte below the exact ledger cap must fail"); + assert!(matches!( + error, + EncodeError::AllocationTooLarge { cap, requested, .. } + if cap == exact_cap - 1 && requested > cap + )); +} + +fn minimum_successful_cap( + pool: &ThreadPool, + image: &PrecomputedHtj2k97Image, + options: &EncodeOptions, +) -> usize { + let mut high = 1_usize; + loop { + match encode_precomputed_at_cap(pool, image, options, high) { + Ok(output) => { + drop(output); + break; + } + Err(EncodeError::AllocationTooLarge { .. }) => { + high = high.checked_mul(2).expect("ledger boundary must fit usize"); + } + Err(error) => panic!("unexpected encode error while finding cap: {error}"), + } + } + + let mut low = 0_usize; + while low < high { + let midpoint = low + (high - low) / 2; + match encode_precomputed_at_cap(pool, image, options, midpoint) { + Ok(output) => { + drop(output); + high = midpoint; + } + Err(EncodeError::AllocationTooLarge { .. }) => low = midpoint + 1, + Err(error) => panic!("unexpected encode error while narrowing cap: {error}"), + } + } + low +} + +fn encode_precomputed_at_cap( + pool: &ThreadPool, + image: &PrecomputedHtj2k97Image, + options: &EncodeOptions, + cap: usize, +) -> Result, EncodeError> { + pool.install(|| { + encode_precomputed_htj2k_97_with_accelerator_and_max_host_bytes( + image, + options, + &mut CpuOnlyJ2kEncodeStageAccelerator, + cap, + ) + }) +} + +fn precomputed_options() -> EncodeOptions { + EncodeOptions { + num_decomposition_levels: 0, + reversible: false, + guard_bits: 2, + use_ht_block_coding: true, + code_block_width_exp: 2, + code_block_height_exp: 2, + ..EncodeOptions::default() + } +} + +fn precomputed_image() -> PrecomputedHtj2k97Image { + PrecomputedHtj2k97Image { + width: 1, + height: 1, + bit_depth: 8, + signed: false, + components: vec![PrecomputedHtj2k97Component { + x_rsiz: 1, + y_rsiz: 1, + dwt: J2kForwardDwt97Output { + ll: vec![1.0], + ll_width: 1, + ll_height: 1, + levels: Vec::new(), + }, + }], + } +} + +fn probe_pool() -> &'static ThreadPool { + PROBE_POOL.get_or_init(|| { + ThreadPoolBuilder::new() + .num_threads(2) + .build() + .expect("build fixed two-thread probe pool") + }) +} + +fn without_panic_output(operation: impl FnOnce() -> R) -> R { + let original_hook = std::panic::take_hook(); + std::panic::set_hook(Box::new(|_| {})); + let result = operation(); + std::panic::set_hook(original_hook); + result +} diff --git a/crates/j2k-cli/Cargo.toml b/crates/j2k-cli/Cargo.toml index 548d947a..a330ee12 100644 --- a/crates/j2k-cli/Cargo.toml +++ b/crates/j2k-cli/Cargo.toml @@ -18,9 +18,9 @@ path = "src/main.rs" doc = false [dependencies] -j2k = { path = "../j2k", version = "=0.7.5" } -j2k-jpeg = { path = "../j2k-jpeg", version = "=0.7.5" } -j2k-transcode = { path = "../j2k-transcode", version = "=0.7.5" } +j2k = { path = "../j2k", version = "=0.8.0" } +j2k-jpeg = { path = "../j2k-jpeg", version = "=0.8.0" } +j2k-transcode = { path = "../j2k-transcode", version = "=0.8.0" } [dev-dependencies] j2k-test-support = { path = "../j2k-test-support" } diff --git a/crates/j2k-compare/Cargo.toml b/crates/j2k-compare/Cargo.toml index b08b83f4..ad6d0368 100644 --- a/crates/j2k-compare/Cargo.toml +++ b/crates/j2k-compare/Cargo.toml @@ -13,10 +13,10 @@ publish = false ignored = ["cc"] [dependencies] -j2k-core = { path = "../j2k-core", version = "=0.7.5" } -j2k-native = { path = "../j2k-native", version = "=0.7.5" } -j2k = { path = "../j2k", version = "=0.7.5" } -j2k-test-support = { path = "../j2k-test-support", version = "=0.7.5" } +j2k-core = { path = "../j2k-core", version = "=0.8.0" } +j2k-native = { path = "../j2k-native", version = "=0.8.0" } +j2k = { path = "../j2k", version = "=0.8.0" } +j2k-test-support = { path = "../j2k-test-support", version = "=0.8.0" } image = { workspace = true } openjpeg-sys = { workspace = true } diff --git a/crates/j2k-cuda-runtime/Cargo.toml b/crates/j2k-cuda-runtime/Cargo.toml index 3735f262..fbc7c492 100644 --- a/crates/j2k-cuda-runtime/Cargo.toml +++ b/crates/j2k-cuda-runtime/Cargo.toml @@ -48,13 +48,13 @@ cuda-oxide-jpeg-decode = [] cuda-oxide-jpeg-encode = [] [dependencies] -j2k-codec-math = { path = "../j2k-codec-math", version = "=0.7.5" } -j2k-core = { path = "../j2k-core", version = "=0.7.5" } +j2k-codec-math = { path = "../j2k-codec-math", version = "=0.8.0" } +j2k-core = { path = "../j2k-core", version = "=0.8.0" } libloading = { workspace = true } thiserror = { workspace = true } [build-dependencies] -j2k-codec-math = { path = "../j2k-codec-math", version = "=0.7.5" } +j2k-codec-math = { path = "../j2k-codec-math", version = "=0.8.0" } [dev-dependencies] j2k-test-support = { path = "../j2k-test-support" } diff --git a/crates/j2k-cuda/Cargo.toml b/crates/j2k-cuda/Cargo.toml index 3fe47d72..54cc204b 100644 --- a/crates/j2k-cuda/Cargo.toml +++ b/crates/j2k-cuda/Cargo.toml @@ -34,11 +34,11 @@ cuda-runtime = [ cuda-profiling = ["cuda-runtime", "j2k-cuda-runtime/cuda-profiling"] [dependencies] -j2k-core = { path = "../j2k-core", version = "=0.7.5" } -j2k-cuda-runtime = { path = "../j2k-cuda-runtime", version = "=0.7.5", optional = true } -j2k = { path = "../j2k", version = "=0.7.5" } -j2k-native = { path = "../j2k-native", version = "=0.7.5" } -j2k-profile = { path = "../j2k-profile", version = "=0.7.5" } +j2k-core = { path = "../j2k-core", version = "=0.8.0" } +j2k-cuda-runtime = { path = "../j2k-cuda-runtime", version = "=0.8.0", optional = true } +j2k = { path = "../j2k", version = "=0.8.0" } +j2k-native = { path = "../j2k-native", version = "=0.8.0" } +j2k-profile = { path = "../j2k-profile", version = "=0.8.0" } thiserror = { workspace = true } [dev-dependencies] diff --git a/crates/j2k-cuda/README.md b/crates/j2k-cuda/README.md index 91df6807..b36df1cf 100644 --- a/crates/j2k-cuda/README.md +++ b/crates/j2k-cuda/README.md @@ -16,6 +16,45 @@ CUDA-resident surface and buffer types, and the CUDA-buffer encode APIs. Without that feature, strict CUDA requests cannot dispatch and return `CudaUnavailable` or the corresponding structured unsupported-request error. +## Host-input lossless encode routing + +`CudaLosslessEncoder::encode` honors the `EncodeBackendPreference` stored in +each job's options. `CpuOnly` never probes CUDA. `Auto` may use supported CUDA +stages and returns a CPU result when the runtime/device is unavailable or CUDA +does not cover every required stage. `RequireDevice` fails unless CUDA satisfies +the complete route. A CUDA execution error is never hidden by retrying the job +on the CPU. + +The opaque result reports the requested preference, the backend that satisfied +the complete encode contract, any Auto fallback reason, and per-stage CUDA +dispatches. A CPU backend with nonzero CUDA dispatches means CUDA completed some +stages but the CPU satisfied the overall contract. + +```rust +use j2k::{ + EncodeBackendPreference, J2kBlockCodingMode, J2kLosslessEncodeOptions, + J2kLosslessSamples, +}; +use j2k_cuda::CudaLosslessEncoder; + +let pixels = [0_u8; 16 * 16]; +let samples = J2kLosslessSamples::new(&pixels, 16, 16, 1, 8, false)?; +let options = J2kLosslessEncodeOptions::default() + .with_backend(EncodeBackendPreference::Auto) + .with_block_coding_mode(J2kBlockCodingMode::HighThroughput); +let mut encoder = CudaLosslessEncoder::new(); +let result = encoder.encode(samples, &options)?; + +assert_eq!(result.requested_backend(), EncodeBackendPreference::Auto); +# Ok::<(), Box>(()) +``` + +`CudaLosslessEncoder::encode_strict_cuda` and the compatibility +`encode_j2k_lossless_with_cuda` free function deliberately override the stored +preference with `RequireDevice`. The encoder can be moved between threads, but +is not shareable by reference: each call takes exclusive mutable access. +Input/route/runtime errors clear cached accelerator state before the next job. + `cuda-runtime` is not proof that every CUDA Oxide kernel was built on the local host. Product PTX is generated only on supported Linux cuda-oxide build hosts; other builds may embed placeholder PTX. Set `J2K_REQUIRE_CUDA_OXIDE_BUILD=1` diff --git a/crates/j2k-cuda/src/batch/decoder.rs b/crates/j2k-cuda/src/batch/decoder.rs index 1d2b5eaa..32035f03 100644 --- a/crates/j2k-cuda/src/batch/decoder.rs +++ b/crates/j2k-cuda/src/batch/decoder.rs @@ -278,7 +278,7 @@ impl CudaBatchDecoder { .iter() .map(|image| image.plan().output_rect()) .collect(), - warnings: decode_warnings(group.options(), group.images().len()), + warnings: decode_warnings(group.images()), ranges: pending.ranges().to_vec(), }; Ok(SubmittedCudaExternalBatch { diff --git a/crates/j2k-cuda/src/batch/input.rs b/crates/j2k-cuda/src/batch/input.rs index 41edba47..022bcd06 100644 --- a/crates/j2k-cuda/src/batch/input.rs +++ b/crates/j2k-cuda/src/batch/input.rs @@ -8,8 +8,7 @@ use super::{ PreparedBatchGroup, Surface, SurfaceResidency, }; use super::{ - BatchColor, BatchDecodeOptions, BatchGroupInfo, BatchLayout, Error, J2kDecodeWarning, - PixelFormat, + BatchColor, BatchGroupInfo, BatchLayout, Error, J2kDecodeWarning, PixelFormat, PreparedImage, }; pub(super) fn group_pixel_format(info: &BatchGroupInfo) -> Result { @@ -162,14 +161,12 @@ pub(super) fn native_color_group_storage( ) } -pub(super) fn decode_warnings( - options: BatchDecodeOptions, - count: usize, -) -> Vec> { - (0..count) - .map(|_| { - if options.settings.lenient_tolerance_enabled() { - vec![J2kDecodeWarning::LenientDecodeMode] +pub(super) fn decode_warnings(images: &[PreparedImage]) -> Vec> { + images + .iter() + .map(|image| { + if image.used_lenient_metadata_recovery() { + vec![J2kDecodeWarning::LenientMetadataRecovery] } else { Vec::new() } diff --git a/crates/j2k-cuda/src/batch/resident_submission.rs b/crates/j2k-cuda/src/batch/resident_submission.rs index 61889263..745560c6 100644 --- a/crates/j2k-cuda/src/batch/resident_submission.rs +++ b/crates/j2k-cuda/src/batch/resident_submission.rs @@ -29,7 +29,7 @@ impl ResidentGroupMetadata { .iter() .map(|image| image.plan().output_rect()) .collect(), - warnings: decode_warnings(group.options(), group.images().len()), + warnings: decode_warnings(group.images()), } } diff --git a/crates/j2k-cuda/src/encode.rs b/crates/j2k-cuda/src/encode.rs index f3c90b46..fdc542f0 100644 --- a/crates/j2k-cuda/src/encode.rs +++ b/crates/j2k-cuda/src/encode.rs @@ -24,8 +24,10 @@ use crate::{runtime::cuda_error, session::CudaSession}; mod api; pub use self::api::{encode_j2k_lossless_with_cuda, encode_j2k_lossless_with_cuda_and_profile}; +mod facade; #[cfg(feature = "cuda-runtime")] use self::api::{reject_non_cuda_encode_backend, strict_cuda_encode_options}; +pub use self::facade::{CudaEncodeFallbackReason, CudaLosslessEncodeResult, CudaLosslessEncoder}; #[cfg(feature = "cuda-runtime")] mod resident; #[cfg(feature = "cuda-runtime")] diff --git a/crates/j2k-cuda/src/encode/facade.rs b/crates/j2k-cuda/src/encode/facade.rs new file mode 100644 index 00000000..68617240 --- /dev/null +++ b/crates/j2k-cuda/src/encode/facade.rs @@ -0,0 +1,197 @@ +// SPDX-License-Identifier: MIT OR Apache-2.0 + +use core::{cell::Cell, marker::PhantomData}; + +use j2k::{EncodeBackendPreference, EncodedJ2k, J2kLosslessEncodeOptions, J2kLosslessSamples}; +use j2k_core::BackendKind; + +use super::CudaEncodeStageAccelerator; + +/// Reason an `Auto` lossless encode completed on the CPU. +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +#[non_exhaustive] +pub enum CudaEncodeFallbackReason { + /// CUDA support was not compiled or no usable CUDA runtime/device was found. + DeviceUnavailable, + /// CUDA was available, but it did not implement every stage required by the request. + DeviceRouteIncomplete, +} + +/// Opaque result from [`CudaLosslessEncoder`]. +/// +/// The requested backend is retained separately from the backend that actually +/// satisfied the request. A CPU result is a fallback only when the request used +/// [`EncodeBackendPreference::Auto`]. +#[derive(Clone, Debug, Eq, PartialEq)] +pub struct CudaLosslessEncodeResult { + requested_backend: EncodeBackendPreference, + fallback_reason: Option, + encoded: EncodedJ2k, +} + +impl CudaLosslessEncodeResult { + fn new( + requested_backend: EncodeBackendPreference, + device_unavailable: bool, + encoded: EncodedJ2k, + ) -> Self { + let fallback_reason = if requested_backend == EncodeBackendPreference::Auto + && encoded.backend == BackendKind::Cpu + { + Some(if device_unavailable { + CudaEncodeFallbackReason::DeviceUnavailable + } else { + CudaEncodeFallbackReason::DeviceRouteIncomplete + }) + } else { + None + }; + Self { + requested_backend, + fallback_reason, + encoded, + } + } + + /// Backend preference supplied for this encode job. + #[must_use] + pub const fn requested_backend(&self) -> EncodeBackendPreference { + self.requested_backend + } + + /// Backend that satisfied the encode contract. + #[must_use] + pub const fn actual_backend(&self) -> BackendKind { + self.encoded.backend + } + + /// Why an `Auto` request completed on the CPU, if it did. + #[must_use] + pub const fn fallback_reason(&self) -> Option { + self.fallback_reason + } + + /// Encode-stage dispatches observed while producing the codestream. + #[must_use] + pub const fn dispatch_report(&self) -> j2k::J2kEncodeDispatchReport { + self.encoded.dispatch_report + } + + /// Borrow the encoded codestream and its image metadata. + #[must_use] + pub const fn encoded(&self) -> &EncodedJ2k { + &self.encoded + } + + /// Consume the route report and return the encoded codestream and metadata. + #[must_use] + pub fn into_encoded(self) -> EncodedJ2k { + self.encoded + } +} + +/// Reusable CUDA-aware lossless JPEG 2000 encoder. +/// +/// [`Self::encode`] honors each job's [`EncodeBackendPreference`]: +/// +/// - `CpuOnly` does not initialize or submit CUDA work. +/// - `Auto` may use CUDA stages and returns a CPU result when CUDA is unavailable +/// or the device route does not cover every required stage. +/// - `RequireDevice` returns an error unless CUDA satisfies every required stage. +/// +/// `Auto` does not retry on the CPU after a CUDA execution error. Such errors +/// can indicate uncertain device state and are returned to the caller. The +/// encoder discards its cached accelerator state after any error, so a later +/// job can safely retry initialization or select `CpuOnly`. +/// +/// The encoder is `Send` but intentionally not `Sync`; encoding requires +/// exclusive `&mut self` access. Move one encoder to each worker, or put it +/// behind a mutex when jobs must share it. +/// +/// ```compile_fail +/// fn require_sync() {} +/// require_sync::(); +/// ``` +#[derive(Debug)] +pub struct CudaLosslessEncoder { + accelerator: CudaEncodeStageAccelerator, + not_sync: PhantomData>, +} + +impl Default for CudaLosslessEncoder { + fn default() -> Self { + Self::new() + } +} + +impl CudaLosslessEncoder { + /// Create a reusable encoder with lazily initialized CUDA state. + #[must_use] + pub fn new() -> Self { + Self { + accelerator: CudaEncodeStageAccelerator::default(), + not_sync: PhantomData, + } + } + + /// Encode one job according to its backend preference. + /// + /// Device unavailability and incomplete device-stage coverage are recoverable + /// only for `Auto`. Input, allocation, validation, and CUDA execution errors + /// are returned without a second CPU encode attempt. + pub fn encode( + &mut self, + samples: J2kLosslessSamples<'_>, + options: &J2kLosslessEncodeOptions, + ) -> Result { + self.encode_with_options(samples, *options) + } + + /// Encode one job with a strict CUDA contract. + /// + /// This method ignores the job's stored backend preference and behaves as + /// [`EncodeBackendPreference::RequireDevice`]. It exists alongside + /// [`Self::encode`] so CUDA-specific callers can opt into fail-closed routing + /// without changing reusable per-job option templates. + pub fn encode_strict_cuda( + &mut self, + samples: J2kLosslessSamples<'_>, + options: &J2kLosslessEncodeOptions, + ) -> Result { + self.encode_with_options( + samples, + options.with_backend(EncodeBackendPreference::RequireDevice), + ) + } + + fn encode_with_options( + &mut self, + samples: J2kLosslessSamples<'_>, + options: J2kLosslessEncodeOptions, + ) -> Result { + self.accelerator.begin_encode_attempt(); + let requested_backend = options.backend; + let encoded = if requested_backend == EncodeBackendPreference::CpuOnly { + j2k::encode_j2k_lossless(samples, &options) + } else { + j2k::encode_j2k_lossless_with_accelerator( + samples, + &options, + BackendKind::Cuda, + &mut self.accelerator, + ) + }; + + match encoded { + Ok(encoded) => Ok(CudaLosslessEncodeResult::new( + requested_backend, + self.accelerator.device_unavailable_observed(), + encoded, + )), + Err(error) => { + self.accelerator = CudaEncodeStageAccelerator::default(); + Err(error.into()) + } + } + } +} diff --git a/crates/j2k-cuda/src/encode/stage.rs b/crates/j2k-cuda/src/encode/stage.rs index e846af17..f69b7f66 100644 --- a/crates/j2k-cuda/src/encode/stage.rs +++ b/crates/j2k-cuda/src/encode/stage.rs @@ -88,6 +88,8 @@ pub struct CudaEncodeStageAccelerator { context: Option, #[cfg(feature = "cuda-runtime")] encode_resources: Option>, + #[cfg(feature = "cuda-runtime")] + device_unavailable_observed: bool, #[cfg_attr( not(feature = "cuda-runtime"), expect(dead_code, reason = "profiling state is used only by the CUDA runtime") @@ -130,6 +132,24 @@ pub struct CudaEncodeStageAccelerator { } impl CudaEncodeStageAccelerator { + pub(super) fn begin_encode_attempt(&mut self) { + #[cfg(feature = "cuda-runtime")] + { + self.device_unavailable_observed = false; + } + } + + pub(super) const fn device_unavailable_observed(&self) -> bool { + #[cfg(feature = "cuda-runtime")] + { + self.device_unavailable_observed + } + #[cfg(not(feature = "cuda-runtime"))] + { + true + } + } + /// Create an encode-stage accelerator with optional CUDA stage timing collection. #[must_use] #[doc(hidden)] @@ -221,6 +241,7 @@ impl CudaEncodeStageAccelerator { match CudaContext::system_default() { Ok(context) => self.context = Some(context), Err(error) if !cuda_runtime_required() && error.is_unavailable() => { + self.device_unavailable_observed = true; return Ok(None); } Err(error) => { diff --git a/crates/j2k-cuda/src/encode/tests/facade.rs b/crates/j2k-cuda/src/encode/tests/facade.rs new file mode 100644 index 00000000..a07ea09d --- /dev/null +++ b/crates/j2k-cuda/src/encode/tests/facade.rs @@ -0,0 +1,98 @@ +// SPDX-License-Identifier: MIT OR Apache-2.0 + +#[cfg(not(feature = "cuda-runtime"))] +use super::CudaEncodeFallbackReason; +use super::{ + assert_strict_cuda_classic_tier1_error, BackendKind, CudaLosslessEncoder, + EncodeBackendPreference, J2kBlockCodingMode, J2kEncodeValidation, J2kLosslessEncodeOptions, + J2kLosslessSamples, +}; + +#[cfg(not(feature = "cuda-runtime"))] +#[test] +fn cuda_encoder_auto_reports_unavailable_fallback_and_reuses_after_success() { + let pixels = vec![17u8; 16 * 16]; + let samples = + J2kLosslessSamples::new(&pixels, 16, 16, 1, 8, false).expect("valid gray8 samples"); + let options = J2kLosslessEncodeOptions::default() + .with_backend(EncodeBackendPreference::Auto) + .with_block_coding_mode(J2kBlockCodingMode::HighThroughput) + .with_max_decomposition_levels(Some(0)) + .with_validation(J2kEncodeValidation::CpuRoundTrip); + let mut encoder = CudaLosslessEncoder::new(); + + for _ in 0..2 { + let result = encoder + .encode(samples, &options) + .expect("Auto must fall back when CUDA support is not compiled"); + + assert_eq!(result.requested_backend(), EncodeBackendPreference::Auto); + assert_eq!(result.actual_backend(), BackendKind::Cpu); + assert_eq!( + result.fallback_reason(), + Some(CudaEncodeFallbackReason::DeviceUnavailable) + ); + assert_eq!( + result.dispatch_report(), + j2k::J2kEncodeDispatchReport::default() + ); + assert!(!result.encoded().codestream.is_empty()); + } +} + +#[test] +fn cuda_encoder_cpu_only_reports_no_fallback() { + let pixels = vec![23u8; 16 * 16]; + let samples = + J2kLosslessSamples::new(&pixels, 16, 16, 1, 8, false).expect("valid gray8 samples"); + let options = J2kLosslessEncodeOptions::default() + .with_backend(EncodeBackendPreference::CpuOnly) + .with_max_decomposition_levels(Some(0)) + .with_validation(J2kEncodeValidation::CpuRoundTrip); + let mut encoder = CudaLosslessEncoder::new(); + + let result = encoder + .encode(samples, &options) + .expect("CPU-only encode must not require CUDA"); + + assert_eq!(result.requested_backend(), EncodeBackendPreference::CpuOnly); + assert_eq!(result.actual_backend(), BackendKind::Cpu); + assert_eq!(result.fallback_reason(), None); + assert_eq!( + result.dispatch_report(), + j2k::J2kEncodeDispatchReport::default() + ); +} + +#[test] +fn cuda_encoder_reuses_after_strict_route_failure() { + let pixels = vec![31u8; 16 * 16]; + let samples = + J2kLosslessSamples::new(&pixels, 16, 16, 1, 8, false).expect("valid gray8 samples"); + let cpu_template = J2kLosslessEncodeOptions::default() + .with_backend(EncodeBackendPreference::CpuOnly) + .with_block_coding_mode(J2kBlockCodingMode::Classic) + .with_max_decomposition_levels(Some(0)) + .with_validation(J2kEncodeValidation::External); + let cpu_options = cpu_template.with_validation(J2kEncodeValidation::CpuRoundTrip); + let mut encoder = CudaLosslessEncoder::new(); + + let error = encoder + .encode_strict_cuda(samples, &cpu_template) + .expect_err("strict method must override the CPU-only template"); + assert_strict_cuda_classic_tier1_error(&error, "reusable strict CUDA encoder"); + + let result = encoder + .encode(samples, &cpu_options) + .expect("a route failure must not poison later jobs"); + assert_eq!(result.requested_backend(), EncodeBackendPreference::CpuOnly); + assert_eq!(result.actual_backend(), BackendKind::Cpu); + assert_eq!(result.fallback_reason(), None); +} + +#[test] +fn cuda_lossless_encoder_is_send() { + fn assert_send() {} + + assert_send::(); +} diff --git a/crates/j2k-cuda/src/encode/tests/mod.rs b/crates/j2k-cuda/src/encode/tests/mod.rs index b71574a0..77d96765 100644 --- a/crates/j2k-cuda/src/encode/tests/mod.rs +++ b/crates/j2k-cuda/src/encode/tests/mod.rs @@ -1,5 +1,6 @@ // SPDX-License-Identifier: MIT OR Apache-2.0 +mod facade; mod htj2k; mod packetization; mod resident; @@ -17,6 +18,8 @@ use super::packetization::{ #[cfg(feature = "cuda-runtime")] use super::stage::cuda_dwt53_output_to_j2k; use super::stage::cuda_packetization_plan_fallback_reason; +#[cfg(not(feature = "cuda-runtime"))] +use super::CudaEncodeFallbackReason; #[cfg(feature = "cuda-runtime")] use super::{ cuda_resident_input_error, encode_lossless_from_cuda_buffer_to_cuda_buffer_with_report, @@ -24,7 +27,7 @@ use super::{ }; use super::{ encode_j2k_lossless_with_cuda, encode_j2k_lossless_with_cuda_and_profile, - CudaEncodeStageAccelerator, + CudaEncodeStageAccelerator, CudaLosslessEncoder, }; #[cfg(feature = "cuda-runtime")] use crate::CudaSession; @@ -42,9 +45,9 @@ use j2k::{ J2kPacketizationPacketDescriptor, J2kPacketizationProgressionOrder, J2kPacketizationResolution, J2kPacketizationSubband, J2kQuantizeSubbandJob, }; -use j2k_core::CodecError; #[cfg(feature = "cuda-runtime")] -use j2k_core::{BackendKind, PixelFormat}; +use j2k_core::PixelFormat; +use j2k_core::{BackendKind, CodecError}; #[cfg(feature = "cuda-runtime")] use j2k_cuda_runtime::{ CudaContext, CudaHtj2kEncodeCodeBlockJob, CudaHtj2kEncodeCodeBlockRegionJob, CudaJ2kQuantizeJob, diff --git a/crates/j2k-cuda/src/lib.rs b/crates/j2k-cuda/src/lib.rs index 15f1c556..9003fd90 100644 --- a/crates/j2k-cuda/src/lib.rs +++ b/crates/j2k-cuda/src/lib.rs @@ -41,7 +41,8 @@ pub(crate) use direct_plan::{ }; pub use encode::{ encode_j2k_lossless_with_cuda, encode_j2k_lossless_with_cuda_and_profile, - CudaEncodeStageAccelerator, CudaEncodeStageTimings, + CudaEncodeFallbackReason, CudaEncodeStageAccelerator, CudaEncodeStageTimings, + CudaLosslessEncodeResult, CudaLosslessEncoder, }; #[cfg(feature = "cuda-runtime")] pub use encode::{ diff --git a/crates/j2k-cuda/tests/lossless_encoder_api.rs b/crates/j2k-cuda/tests/lossless_encoder_api.rs new file mode 100644 index 00000000..7c5632e5 --- /dev/null +++ b/crates/j2k-cuda/tests/lossless_encoder_api.rs @@ -0,0 +1,34 @@ +// SPDX-License-Identifier: MIT OR Apache-2.0 + +use j2k::{ + EncodeBackendPreference, J2kEncodeDispatchReport, J2kEncodeValidation, + J2kLosslessEncodeOptions, J2kLosslessSamples, +}; +use j2k_core::BackendKind; +use j2k_cuda::CudaLosslessEncoder; + +#[test] +fn external_consumer_can_use_the_preference_honoring_encoder() { + fn assert_send() {} + assert_send::(); + + let pixels = [41_u8; 16 * 16]; + let samples = + J2kLosslessSamples::new(&pixels, 16, 16, 1, 8, false).expect("valid gray8 samples"); + let options = J2kLosslessEncodeOptions::default() + .with_backend(EncodeBackendPreference::CpuOnly) + .with_max_decomposition_levels(Some(0)) + .with_validation(J2kEncodeValidation::CpuRoundTrip); + let mut encoder = CudaLosslessEncoder::new(); + + let result = encoder + .encode(samples, &options) + .expect("CPU-only route must not require CUDA"); + + assert_eq!(result.requested_backend(), EncodeBackendPreference::CpuOnly); + assert_eq!(result.actual_backend(), BackendKind::Cpu); + assert_eq!(result.fallback_reason(), None); + assert_eq!(result.dispatch_report(), J2kEncodeDispatchReport::default()); + assert!(result.encoded().codestream.starts_with(&[0xff, 0x4f])); + assert!(!result.into_encoded().codestream.is_empty()); +} diff --git a/crates/j2k-jpeg-cuda/Cargo.toml b/crates/j2k-jpeg-cuda/Cargo.toml index 3931f171..768ccc5b 100644 --- a/crates/j2k-jpeg-cuda/Cargo.toml +++ b/crates/j2k-jpeg-cuda/Cargo.toml @@ -27,10 +27,10 @@ cuda-runtime = [ ] [dependencies] -j2k-core = { path = "../j2k-core", version = "=0.7.5" } -j2k-cuda-runtime = { path = "../j2k-cuda-runtime", version = "=0.7.5", optional = true } -j2k-jpeg = { path = "../j2k-jpeg", version = "=0.7.5" } -j2k-profile = { path = "../j2k-profile", version = "=0.7.5" } +j2k-core = { path = "../j2k-core", version = "=0.8.0" } +j2k-cuda-runtime = { path = "../j2k-cuda-runtime", version = "=0.8.0", optional = true } +j2k-jpeg = { path = "../j2k-jpeg", version = "=0.8.0" } +j2k-profile = { path = "../j2k-profile", version = "=0.8.0" } thiserror = { workspace = true } [dev-dependencies] diff --git a/crates/j2k-jpeg-metal/Cargo.toml b/crates/j2k-jpeg-metal/Cargo.toml index c1673f0d..605caa97 100644 --- a/crates/j2k-jpeg-metal/Cargo.toml +++ b/crates/j2k-jpeg-metal/Cargo.toml @@ -19,10 +19,10 @@ name = "j2k_jpeg_metal" path = "src/lib.rs" [dependencies] -j2k-core = { path = "../j2k-core", version = "=0.7.5" } -j2k-jpeg = { path = "../j2k-jpeg", version = "=0.7.5" } -j2k-metal-support = { path = "../j2k-metal-support", version = "=0.7.5" } -j2k-profile = { path = "../j2k-profile", version = "=0.7.5" } +j2k-core = { path = "../j2k-core", version = "=0.8.0" } +j2k-jpeg = { path = "../j2k-jpeg", version = "=0.8.0" } +j2k-metal-support = { path = "../j2k-metal-support", version = "=0.8.0" } +j2k-profile = { path = "../j2k-profile", version = "=0.8.0" } thiserror = { workspace = true } [target.'cfg(target_os = "macos")'.dependencies] diff --git a/crates/j2k-jpeg/Cargo.toml b/crates/j2k-jpeg/Cargo.toml index 4ec9f11d..3f389277 100644 --- a/crates/j2k-jpeg/Cargo.toml +++ b/crates/j2k-jpeg/Cargo.toml @@ -27,9 +27,9 @@ bench-libjpeg-turbo = [] bench-internals = [] [dependencies] -j2k-codec-math = { path = "../j2k-codec-math", version = "=0.7.5" } -j2k-core = { path = "../j2k-core", version = "=0.7.5" } -j2k-profile = { path = "../j2k-profile", version = "=0.7.5" } +j2k-codec-math = { path = "../j2k-codec-math", version = "=0.8.0" } +j2k-core = { path = "../j2k-core", version = "=0.8.0" } +j2k-profile = { path = "../j2k-profile", version = "=0.8.0" } thiserror = { workspace = true } memchr = { workspace = true } rayon = { workspace = true } diff --git a/crates/j2k-jpeg/fuzz/Cargo.lock b/crates/j2k-jpeg/fuzz/Cargo.lock index 393cfc1d..a97aeec2 100644 --- a/crates/j2k-jpeg/fuzz/Cargo.lock +++ b/crates/j2k-jpeg/fuzz/Cargo.lock @@ -77,18 +77,18 @@ dependencies = [ [[package]] name = "j2k-codec-math" -version = "0.7.5" +version = "0.8.0" [[package]] name = "j2k-core" -version = "0.7.5" +version = "0.8.0" dependencies = [ "thiserror", ] [[package]] name = "j2k-jpeg" -version = "0.7.5" +version = "0.8.0" dependencies = [ "j2k-codec-math", "j2k-core", @@ -108,7 +108,7 @@ dependencies = [ [[package]] name = "j2k-profile" -version = "0.7.5" +version = "0.8.0" [[package]] name = "jobserver" diff --git a/crates/j2k-metal-support/Cargo.toml b/crates/j2k-metal-support/Cargo.toml index 813a74b7..ef27d9a1 100644 --- a/crates/j2k-metal-support/Cargo.toml +++ b/crates/j2k-metal-support/Cargo.toml @@ -19,7 +19,7 @@ name = "j2k_metal_support" path = "src/lib.rs" [dependencies] -j2k-core = { path = "../j2k-core", version = "=0.7.5" } +j2k-core = { path = "../j2k-core", version = "=0.8.0" } log = { workspace = true } [target.'cfg(target_os = "macos")'.dependencies] diff --git a/crates/j2k-metal/Cargo.toml b/crates/j2k-metal/Cargo.toml index 16cdda3a..157c4db7 100644 --- a/crates/j2k-metal/Cargo.toml +++ b/crates/j2k-metal/Cargo.toml @@ -24,12 +24,12 @@ name = "j2k_metal" path = "src/lib.rs" [dependencies] -j2k-codec-math = { path = "../j2k-codec-math", version = "=0.7.5" } -j2k-core = { path = "../j2k-core", version = "=0.7.5" } -j2k = { path = "../j2k", version = "=0.7.5" } -j2k-native = { path = "../j2k-native", version = "=0.7.5" } -j2k-metal-support = { path = "../j2k-metal-support", version = "=0.7.5" } -j2k-profile = { path = "../j2k-profile", version = "=0.7.5" } +j2k-codec-math = { path = "../j2k-codec-math", version = "=0.8.0" } +j2k-core = { path = "../j2k-core", version = "=0.8.0" } +j2k = { path = "../j2k", version = "=0.8.0" } +j2k-native = { path = "../j2k-native", version = "=0.8.0" } +j2k-metal-support = { path = "../j2k-metal-support", version = "=0.8.0" } +j2k-profile = { path = "../j2k-profile", version = "=0.8.0" } thiserror = { workspace = true } [target.'cfg(target_os = "macos")'.dependencies] diff --git a/crates/j2k-metal/src/batch_decoder.rs b/crates/j2k-metal/src/batch_decoder.rs index c04b570e..63f80f13 100644 --- a/crates/j2k-metal/src/batch_decoder.rs +++ b/crates/j2k-metal/src/batch_decoder.rs @@ -195,13 +195,7 @@ mod batch_contract_tests { .map(|image| image.plan().output_rect()) .collect::>() ); - assert_eq!( - completion.warnings(), - [ - vec![J2kDecodeWarning::LenientDecodeMode], - vec![J2kDecodeWarning::LenientDecodeMode], - ] - ); + assert_eq!(completion.warnings(), [Vec::new(), Vec::new()]); // The synchronous convenience API must use the same coalesced graph, // not the legacy per-image destination loop. diff --git a/crates/j2k-metal/src/batch_decoder/contracts.rs b/crates/j2k-metal/src/batch_decoder/contracts.rs index 21570411..7862ce4e 100644 --- a/crates/j2k-metal/src/batch_decoder/contracts.rs +++ b/crates/j2k-metal/src/batch_decoder/contracts.rs @@ -2,13 +2,13 @@ //! Shared Metal batch result and metadata contracts. -#[cfg(target_os = "macos")] -use super::{BatchDecodeOptions, Buffer, ResidentMetalImage}; use super::{ BatchGroupInfo, Error, IndexedBatchError, J2kDecodeWarning, PreparedBatchGroup, Rect, Surface, }; #[cfg(any(test, target_os = "macos"))] use super::{BatchLayout, PixelFormat}; +#[cfg(target_os = "macos")] +use super::{Buffer, ResidentMetalImage}; #[cfg(any(test, target_os = "macos"))] pub(super) fn validate_group_contract(info: &BatchGroupInfo) -> Result { @@ -33,7 +33,7 @@ pub struct MetalBatchGroupCompletion { impl MetalBatchGroupCompletion { #[cfg(target_os = "macos")] - pub(super) fn from_prepared(group: &PreparedBatchGroup, options: BatchDecodeOptions) -> Self { + pub(super) fn from_prepared(group: &PreparedBatchGroup) -> Self { let decoded_rects = group .images() .iter() @@ -42,9 +42,9 @@ impl MetalBatchGroupCompletion { let warnings = group .images() .iter() - .map(|_| { - if options.settings.lenient_tolerance_enabled() { - vec![J2kDecodeWarning::LenientDecodeMode] + .map(|image| { + if image.used_lenient_metadata_recovery() { + vec![J2kDecodeWarning::LenientMetadataRecovery] } else { Vec::new() } diff --git a/crates/j2k-metal/src/batch_decoder/external.rs b/crates/j2k-metal/src/batch_decoder/external.rs index 5b8bd9b0..221cfea9 100644 --- a/crates/j2k-metal/src/batch_decoder/external.rs +++ b/crates/j2k-metal/src/batch_decoder/external.rs @@ -187,7 +187,7 @@ impl MetalBatchDecoder { Ok(SubmittedMetalGroupDecodeInto { submission, destination, - completion: MetalBatchGroupCompletion::from_prepared(group, group.options()), + completion: MetalBatchGroupCompletion::from_prepared(group), }) } } diff --git a/crates/j2k-metal/src/batch_decoder/submission.rs b/crates/j2k-metal/src/batch_decoder/submission.rs index 1c85e89b..101aa10d 100644 --- a/crates/j2k-metal/src/batch_decoder/submission.rs +++ b/crates/j2k-metal/src/batch_decoder/submission.rs @@ -176,11 +176,11 @@ pub(super) struct MetalResidentGroupMetadata { #[cfg(target_os = "macos")] impl MetalResidentGroupMetadata { - pub(super) fn from_prepared(group: &PreparedBatchGroup, options: BatchDecodeOptions) -> Self { + pub(super) fn from_prepared(group: &PreparedBatchGroup, _options: BatchDecodeOptions) -> Self { let MetalBatchGroupCompletion { decoded_rects, warnings, - } = MetalBatchGroupCompletion::from_prepared(group, options); + } = MetalBatchGroupCompletion::from_prepared(group); Self { info: group.info().clone(), source_indices: group.source_indices().to_vec(), diff --git a/crates/j2k-metal/src/encode/encoded.rs b/crates/j2k-metal/src/encode/encoded.rs index 26eaff22..f65a0d71 100644 --- a/crates/j2k-metal/src/encode/encoded.rs +++ b/crates/j2k-metal/src/encode/encoded.rs @@ -92,8 +92,12 @@ impl MetalEncodedJ2k { /// Construct an encoded codestream from a caller-owned Metal allocation. /// - /// `codestream_range` identifies the valid bytes and `capacity` starts at - /// the same offset. Both ranges are validated against the allocation. + /// `codestream_range` is a half-open byte range + /// `start..end` indexing `codestream_buffer`; it identifies exactly the valid + /// codestream bytes. `capacity` also begins at `start`. Construction checks + /// `start <= end`, `end - start <= capacity`, that `start + capacity` does + /// not overflow, and that the complete capacity range is within the Metal + /// allocation. /// /// # Safety /// diff --git a/crates/j2k-metal/src/encode_bitstream_classic_core.metal b/crates/j2k-metal/src/encode_bitstream_classic_core.metal index 87c963cb..9e7a96d0 100644 --- a/crates/j2k-metal/src/encode_bitstream_classic_core.metal +++ b/crates/j2k-metal/src/encode_bitstream_classic_core.metal @@ -247,6 +247,9 @@ inline void j2k_raw_writer_write_bit(thread J2kRawBitWriter &writer, uint bit) { inline void j2k_raw_writer_finish(thread J2kRawBitWriter &writer) { if (writer.bits_in_buffer == 0u) { + if (writer.last_byte_was_ff != 0u) { + j2k_raw_writer_push(writer, uchar(0u)); + } return; } const uint limit = writer.last_byte_was_ff != 0u ? 7u : 8u; diff --git a/crates/j2k-metal/src/ht.rs b/crates/j2k-metal/src/ht.rs index 5d49e32d..3d54c500 100644 --- a/crates/j2k-metal/src/ht.rs +++ b/crates/j2k-metal/src/ht.rs @@ -340,6 +340,10 @@ mod tests { .expect("capture cleanup job"); let mut job = capture.first.expect("captured cleanup job"); + job.missing_bit_planes = job + .missing_bit_planes + .checked_sub(1) + .expect("synthetic refinement job needs a second actual bitplane"); job.data.push(0); job.refinement_length = 1; job.number_of_coding_passes = number_of_coding_passes; diff --git a/crates/j2k-ml/Cargo.toml b/crates/j2k-ml/Cargo.toml index 3cf76320..d5aeb179 100644 --- a/crates/j2k-ml/Cargo.toml +++ b/crates/j2k-ml/Cargo.toml @@ -25,7 +25,6 @@ cpu = [] cuda = [ "dep:burn-cuda", "dep:j2k-cuda", - "dep:j2k-cuda-runtime", ] metal = [ "dep:burn-wgpu", @@ -37,17 +36,16 @@ metal = [ burn-core = { workspace = true } burn-cuda = { workspace = true, optional = true } burn-wgpu = { workspace = true, optional = true } -j2k = { path = "../j2k", version = "=0.7.5" } -j2k-cuda = { path = "../j2k-cuda", version = "=0.7.5", features = ["cuda-runtime"], optional = true } -j2k-cuda-runtime = { path = "../j2k-cuda-runtime", version = "=0.7.5", optional = true } -j2k-metal = { path = "../j2k-metal", version = "=0.7.5", optional = true } -j2k-metal-support = { path = "../j2k-metal-support", version = "=0.7.5", optional = true } +j2k = { path = "../j2k", version = "=0.8.0" } +j2k-cuda = { path = "../j2k-cuda", version = "=0.8.0", features = ["cuda-runtime"], optional = true } +j2k-metal = { path = "../j2k-metal", version = "=0.8.0", optional = true } +j2k-metal-support = { path = "../j2k-metal-support", version = "=0.8.0", optional = true } thiserror = { workspace = true } [dev-dependencies] criterion = { workspace = true } -j2k-core = { path = "../j2k-core" } -j2k-native = { path = "../j2k-native" } +j2k-core = { path = "../j2k-core", version = "=0.8.0" } +j2k-native = { path = "../j2k-native", version = "=0.8.0" } j2k-test-support = { path = "../j2k-test-support", features = ["j2k-native-fixtures"] } [target.'cfg(all(target_arch = "aarch64", target_os = "linux"))'.dev-dependencies] diff --git a/crates/j2k-ml/src/completion.rs b/crates/j2k-ml/src/completion.rs index a055dcac..d6a58499 100644 --- a/crates/j2k-ml/src/completion.rs +++ b/crates/j2k-ml/src/completion.rs @@ -45,16 +45,11 @@ pub(crate) fn burn_group_error_is_fatal(error: &BurnDecodeError) -> bool { BurnDecodeError::Infrastructure(_) | BurnDecodeError::AcceleratorInterop { .. } => true, #[cfg(feature = "cuda")] BurnDecodeError::Cuda(source) => source.session_is_unusable(), - #[cfg(feature = "cuda")] - BurnDecodeError::CudaTransfer(_) => true, - #[cfg(feature = "cuda")] - BurnDecodeError::CudaCodec(source) => source.session_is_unusable(), #[cfg(feature = "metal")] BurnDecodeError::Metal(source) => source.session_is_unusable(), BurnDecodeError::UnsupportedDType { .. } | BurnDecodeError::SampleTypeMismatch | BurnDecodeError::SizeOverflow - | BurnDecodeError::StagingSizeMismatch { .. } | BurnDecodeError::UnsupportedCodecContract => false, } } diff --git a/crates/j2k-ml/src/cuda.rs b/crates/j2k-ml/src/cuda.rs index 4750b06b..6b4f7c3f 100644 --- a/crates/j2k-ml/src/cuda.rs +++ b/crates/j2k-ml/src/cuda.rs @@ -4,21 +4,13 @@ mod batch; -pub use batch::{CudaUploadBurnDecoder, SubmittedCudaUploadBurnBatch}; +pub use batch::{CudaBurnDecoder, SubmittedCudaBurnBatch}; -/// Compatibility alias for [`CudaUploadBurnDecoder`]. +/// Explicit name for [`CudaBurnDecoder`]'s staged host-memory upload behavior. /// /// This alias has the same explicitly staged behavior; it does not restore the /// former direct-destination implementation. -#[deprecated( - since = "0.7.6", - note = "use CudaUploadBurnDecoder to make staging explicit" -)] -pub type CudaBurnDecoder = CudaUploadBurnDecoder; +pub type CudaUploadBurnDecoder = CudaBurnDecoder; -/// Compatibility alias for [`SubmittedCudaUploadBurnBatch`]. -#[deprecated( - since = "0.7.6", - note = "use SubmittedCudaUploadBurnBatch to make staging explicit" -)] -pub type SubmittedCudaBurnBatch = SubmittedCudaUploadBurnBatch; +/// Explicit name for the staged upload represented by [`SubmittedCudaBurnBatch`]. +pub type SubmittedCudaUploadBurnBatch = SubmittedCudaBurnBatch; diff --git a/crates/j2k-ml/src/cuda/batch.rs b/crates/j2k-ml/src/cuda/batch.rs index 9b6ba677..4cd30bab 100644 --- a/crates/j2k-ml/src/cuda/batch.rs +++ b/crates/j2k-ml/src/cuda/batch.rs @@ -12,21 +12,21 @@ use crate::{BurnBatchDecode, BurnBatchGroupError, BurnDecodeError}; /// Pending CUDA codec decode whose completed pixels will be staged through /// host memory and uploaded with Burn's ordinary tensor API. #[must_use = "submitted CUDA upload batches must be waited or dropped"] -pub struct SubmittedCudaUploadBurnBatch { +pub struct SubmittedCudaBurnBatch { pending: SubmittedCudaResidentBatch, device: CudaDevice, } -impl core::fmt::Debug for SubmittedCudaUploadBurnBatch { +impl core::fmt::Debug for SubmittedCudaBurnBatch { fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { - f.debug_struct("SubmittedCudaUploadBurnBatch") + f.debug_struct("SubmittedCudaBurnBatch") .field("pending", &self.pending) .field("device", &self.device) .finish() } } -impl SubmittedCudaUploadBurnBatch { +impl SubmittedCudaBurnBatch { /// Number of successfully submitted homogeneous codec groups. #[must_use] pub fn len(&self) -> usize { @@ -52,12 +52,12 @@ impl SubmittedCudaUploadBurnBatch { /// copied to host staging and then uploaded through [`burn_core::tensor::Tensor::from_data`]. /// This type does not provide direct-destination or zero-copy behavior. #[derive(Debug)] -pub struct CudaUploadBurnDecoder { +pub struct CudaBurnDecoder { codec: CodecDecoder, device: CudaDevice, } -impl CudaUploadBurnDecoder { +impl CudaBurnDecoder { /// Create a staged decoder for one Burn CUDA device. #[must_use] pub fn new(device: CudaDevice, options: BatchDecodeOptions) -> Self { @@ -117,12 +117,12 @@ impl CudaUploadBurnDecoder { self.submit_prepared(prepared)?.wait() } - /// Submit CUDA codec work. The later [`SubmittedCudaUploadBurnBatch::wait`] + /// Submit CUDA codec work. The later [`SubmittedCudaBurnBatch::wait`] /// performs a synchronous device-to-host copy and ordinary Burn upload. pub fn submit( &mut self, inputs: Vec, - ) -> Result { + ) -> Result { let prepared = self.prepare(inputs)?; self.submit_prepared(&prepared) } @@ -131,9 +131,9 @@ impl CudaUploadBurnDecoder { pub fn submit_prepared( &mut self, prepared: &PreparedBatch, - ) -> Result { + ) -> Result { ensure_dtypes::(prepared, &self.device)?; - Ok(SubmittedCudaUploadBurnBatch { + Ok(SubmittedCudaBurnBatch { pending: self.codec.submit_prepared(prepared)?, device: self.device.clone(), }) @@ -165,7 +165,12 @@ fn materialize( for group in codec_groups { let (info, source_indices, decoded_rects, warnings, _surfaces, dense) = group.into_parts(); let mut bytes = vec![0; crate::staging::byte_len(source_indices.len(), &info)?]; - dense.buffer().copy_to_host(&mut bytes)?; + dense.buffer().copy_to_host(&mut bytes).map_err(|source| { + BurnDecodeError::AcceleratorInterop { + backend: "CUDA staged readback", + message: source.to_string(), + } + })?; groups.push(crate::staging::materialize( info, source_indices, @@ -179,7 +184,7 @@ fn materialize( .into_iter() .map(|error| { let (source_indices, source) = error.into_parts(); - BurnBatchGroupError::new(source_indices, BurnDecodeError::CudaCodec(source)) + cuda_group_error(source_indices, source) }) .collect(); Ok(BurnBatchDecode { @@ -188,3 +193,44 @@ fn materialize( group_errors, }) } + +fn cuda_group_error(source_indices: Vec, source: j2k_cuda::Error) -> BurnBatchGroupError { + let adapter_source = BurnDecodeError::Cuda(j2k_cuda::CudaBatchError::GroupExecution { + source_indices: source_indices.clone(), + source: Box::new(source), + }); + BurnBatchGroupError::new(source_indices, adapter_source) +} + +#[cfg(test)] +mod tests { + use super::cuda_group_error; + use crate::BurnDecodeError; + + #[test] + fn cuda_group_failure_preserves_the_existing_typed_error_chain() { + let error = cuda_group_error( + vec![2, 4], + j2k_cuda::Error::UnsupportedCudaRequest { + reason: "test rejection", + }, + ); + + assert_eq!(error.source_indices(), [2, 4]); + let BurnDecodeError::Cuda(j2k_cuda::CudaBatchError::GroupExecution { + source_indices, + source, + .. + }) = error.source() + else { + panic!("CUDA codec failure must use the patch-compatible batch error variant"); + }; + assert_eq!(source_indices, &[2, 4]); + assert!(matches!( + source.as_ref(), + j2k_cuda::Error::UnsupportedCudaRequest { + reason: "test rejection" + } + )); + } +} diff --git a/crates/j2k-ml/src/error.rs b/crates/j2k-ml/src/error.rs index 5dc6ebd8..345a5c94 100644 --- a/crates/j2k-ml/src/error.rs +++ b/crates/j2k-ml/src/error.rs @@ -21,26 +21,10 @@ pub enum BurnDecodeError { /// A newer codec contract cannot be represented by this adapter version. #[error("unsupported codec batch layout or sample type")] UnsupportedCodecContract, - /// Completed codec output did not contain the exact dense byte count. - #[error("staged codec output has {actual} bytes; expected {expected}")] - StagingSizeMismatch { - /// Exact byte count implied by the group metadata. - expected: usize, - /// Byte count returned by the accelerator codec. - actual: usize, - }, /// CUDA rejected or could not complete one homogeneous codec group. #[cfg(feature = "cuda")] #[error(transparent)] Cuda(#[from] j2k_cuda::CudaBatchError), - /// A completed CUDA allocation could not be copied to host staging. - #[cfg(feature = "cuda")] - #[error(transparent)] - CudaTransfer(#[from] j2k_cuda_runtime::CudaError), - /// One CUDA group failed after other groups remained usable. - #[cfg(feature = "cuda")] - #[error(transparent)] - CudaCodec(#[from] j2k_cuda::Error), /// Metal rejected or could not complete one homogeneous codec group. #[cfg(feature = "metal")] #[error(transparent)] diff --git a/crates/j2k-ml/src/lib.rs b/crates/j2k-ml/src/lib.rs index 45ceb58e..ff08158f 100644 --- a/crates/j2k-ml/src/lib.rs +++ b/crates/j2k-ml/src/lib.rs @@ -34,42 +34,15 @@ pub mod metal; #[cfg(feature = "cpu")] pub use cpu::CpuBurnDecoder; #[cfg(feature = "cuda")] -pub use cuda::{CudaUploadBurnDecoder, SubmittedCudaUploadBurnBatch}; +pub use cuda::{ + CudaBurnDecoder, CudaUploadBurnDecoder, SubmittedCudaBurnBatch, SubmittedCudaUploadBurnBatch, +}; pub use error::BurnDecodeError; #[cfg(feature = "metal")] -pub use metal::{MetalUploadBurnDecoder, SubmittedMetalUploadBurnBatch}; - -/// Compatibility alias for [`CudaUploadBurnDecoder`]. -#[cfg(feature = "cuda")] -#[deprecated( - since = "0.7.6", - note = "use CudaUploadBurnDecoder to make staging explicit" -)] -pub type CudaBurnDecoder = CudaUploadBurnDecoder; - -/// Compatibility alias for [`SubmittedCudaUploadBurnBatch`]. -#[cfg(feature = "cuda")] -#[deprecated( - since = "0.7.6", - note = "use SubmittedCudaUploadBurnBatch to make staging explicit" -)] -pub type SubmittedCudaBurnBatch = SubmittedCudaUploadBurnBatch; - -/// Compatibility alias for [`MetalUploadBurnDecoder`]. -#[cfg(feature = "metal")] -#[deprecated( - since = "0.7.6", - note = "use MetalUploadBurnDecoder to make staging explicit" -)] -pub type MetalBurnDecoder = MetalUploadBurnDecoder; - -/// Compatibility alias for [`SubmittedMetalUploadBurnBatch`]. -#[cfg(feature = "metal")] -#[deprecated( - since = "0.7.6", - note = "use SubmittedMetalUploadBurnBatch to make staging explicit" -)] -pub type SubmittedMetalBurnBatch = SubmittedMetalUploadBurnBatch; +pub use metal::{ + MetalBurnDecoder, MetalUploadBurnDecoder, SubmittedMetalBurnBatch, + SubmittedMetalUploadBurnBatch, +}; /// Ordinary rank-4 Burn integer tensor tagged with its exact codec sample type. #[derive(Debug)] diff --git a/crates/j2k-ml/src/metal.rs b/crates/j2k-ml/src/metal.rs index 9e52e4e6..418eacde 100644 --- a/crates/j2k-ml/src/metal.rs +++ b/crates/j2k-ml/src/metal.rs @@ -4,21 +4,13 @@ mod batch; -pub use batch::{MetalUploadBurnDecoder, SubmittedMetalUploadBurnBatch}; +pub use batch::{MetalBurnDecoder, SubmittedMetalBurnBatch}; -/// Compatibility alias for [`MetalUploadBurnDecoder`]. +/// Explicit name for [`MetalBurnDecoder`]'s staged host-memory upload behavior. /// /// This alias has the same explicitly staged behavior; it does not restore the /// former direct-destination implementation. -#[deprecated( - since = "0.7.6", - note = "use MetalUploadBurnDecoder to make staging explicit" -)] -pub type MetalBurnDecoder = MetalUploadBurnDecoder; +pub type MetalUploadBurnDecoder = MetalBurnDecoder; -/// Compatibility alias for [`SubmittedMetalUploadBurnBatch`]. -#[deprecated( - since = "0.7.6", - note = "use SubmittedMetalUploadBurnBatch to make staging explicit" -)] -pub type SubmittedMetalBurnBatch = SubmittedMetalUploadBurnBatch; +/// Explicit name for the staged upload represented by [`SubmittedMetalBurnBatch`]. +pub type SubmittedMetalUploadBurnBatch = SubmittedMetalBurnBatch; diff --git a/crates/j2k-ml/src/metal/batch.rs b/crates/j2k-ml/src/metal/batch.rs index 950d5661..67204a15 100644 --- a/crates/j2k-ml/src/metal/batch.rs +++ b/crates/j2k-ml/src/metal/batch.rs @@ -16,15 +16,15 @@ use crate::{BurnBatchDecode, BurnDecodeError}; /// host memory and uploaded with Burn's ordinary tensor API. #[cfg(target_os = "macos")] #[must_use = "submitted Metal upload batches must be waited or dropped"] -pub struct SubmittedMetalUploadBurnBatch { +pub struct SubmittedMetalBurnBatch { pending: SubmittedMetalPreparedBatch, device: WgpuDevice, } #[cfg(target_os = "macos")] -impl core::fmt::Debug for SubmittedMetalUploadBurnBatch { +impl core::fmt::Debug for SubmittedMetalBurnBatch { fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { - f.debug_struct("SubmittedMetalUploadBurnBatch") + f.debug_struct("SubmittedMetalBurnBatch") .field("pending", &self.pending) .field("device", &self.device) .finish() @@ -32,7 +32,7 @@ impl core::fmt::Debug for SubmittedMetalUploadBurnBatch { } #[cfg(target_os = "macos")] -impl SubmittedMetalUploadBurnBatch { +impl SubmittedMetalBurnBatch { /// Number of successfully submitted homogeneous codec groups. #[must_use] pub fn len(&self) -> usize { @@ -55,12 +55,12 @@ impl SubmittedMetalUploadBurnBatch { /// Uninhabited pending Metal upload returned only for cross-platform API compatibility. #[cfg(not(target_os = "macos"))] #[derive(Debug)] -pub struct SubmittedMetalUploadBurnBatch { +pub struct SubmittedMetalBurnBatch { _private: (), } #[cfg(not(target_os = "macos"))] -impl SubmittedMetalUploadBurnBatch { +impl SubmittedMetalBurnBatch { /// No Metal groups can be submitted on this host. #[must_use] pub const fn len(&self) -> usize { @@ -84,14 +84,14 @@ impl SubmittedMetalUploadBurnBatch { /// JPEG 2000 decoding executes on Metal. Completed codec-owned device output is /// copied to host staging and then uploaded through [`burn_core::tensor::Tensor::from_data`]. /// This type does not provide direct-destination or zero-copy behavior. -pub struct MetalUploadBurnDecoder { +pub struct MetalBurnDecoder { codec: CodecDecoder, device: WgpuDevice, } -impl core::fmt::Debug for MetalUploadBurnDecoder { +impl core::fmt::Debug for MetalBurnDecoder { fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { - f.debug_struct("MetalUploadBurnDecoder") + f.debug_struct("MetalBurnDecoder") .field("codec", &self.codec) .field("device", &self.device) .field("options", &self.codec.options()) @@ -99,7 +99,7 @@ impl core::fmt::Debug for MetalUploadBurnDecoder { } } -impl MetalUploadBurnDecoder { +impl MetalBurnDecoder { /// Create a Metal codec session and target Burn's default wgpu device. #[cfg(target_os = "macos")] pub fn system_default(options: BatchDecodeOptions) -> Result { @@ -165,13 +165,13 @@ impl MetalUploadBurnDecoder { self.submit_prepared(prepared)?.wait() } - /// Submit Metal codec work. The later [`SubmittedMetalUploadBurnBatch::wait`] + /// Submit Metal codec work. The later [`SubmittedMetalBurnBatch::wait`] /// performs a synchronous device-to-host copy and ordinary Burn upload. #[cfg(target_os = "macos")] pub fn submit( &mut self, inputs: Vec, - ) -> Result { + ) -> Result { let prepared = self.prepare(inputs)?; self.submit_prepared(&prepared) } @@ -181,7 +181,7 @@ impl MetalUploadBurnDecoder { pub fn submit( &mut self, _inputs: Vec, - ) -> Result { + ) -> Result { Err(unavailable()) } @@ -190,9 +190,9 @@ impl MetalUploadBurnDecoder { pub fn submit_prepared( &mut self, prepared: &PreparedBatch, - ) -> Result { + ) -> Result { ensure_dtypes::(prepared, &self.device)?; - Ok(SubmittedMetalUploadBurnBatch { + Ok(SubmittedMetalBurnBatch { pending: self.codec.submit_prepared(prepared)?, device: self.device.clone(), }) @@ -203,7 +203,7 @@ impl MetalUploadBurnDecoder { pub fn submit_prepared( &mut self, _prepared: &PreparedBatch, - ) -> Result { + ) -> Result { Err(unavailable()) } } diff --git a/crates/j2k-ml/src/staging.rs b/crates/j2k-ml/src/staging.rs index 24b030e6..e1bed842 100644 --- a/crates/j2k-ml/src/staging.rs +++ b/crates/j2k-ml/src/staging.rs @@ -34,9 +34,12 @@ pub(crate) fn materialize( let shape = tensor_shape(source_indices.len(), &info)?; let expected = byte_len(source_indices.len(), &info)?; if bytes.len() != expected { - return Err(BurnDecodeError::StagingSizeMismatch { - expected, - actual: bytes.len(), + return Err(BurnDecodeError::AcceleratorInterop { + backend: "codec staging", + message: format!( + "staged codec output has {} bytes; expected {expected}", + bytes.len() + ), }); } let tensor = match info.sample_type { @@ -77,17 +80,24 @@ pub(crate) fn materialize( #[cfg(test)] mod tests { + #[cfg(not(all(target_arch = "aarch64", target_os = "linux")))] + use burn_flex::{Flex, FlexDevice}; + #[cfg(all(target_arch = "aarch64", target_os = "linux"))] + use burn_ndarray::NdArrayDevice::Cpu as FlexDevice; use j2k::{ BatchAlpha, BatchCodecRoute, BatchColor, BatchLayout, BatchWaveletTransform, CompressedPayloadKind, CompressedTransferSyntax, }; use j2k_core::Colorspace; - use super::byte_len; + use super::{byte_len, materialize}; + use crate::BurnDecodeError; - #[test] - fn staged_byte_length_tracks_native_width_and_layout() { - let info = j2k::BatchGroupInfo { + #[cfg(all(target_arch = "aarch64", target_os = "linux"))] + type Flex = burn_ndarray::NdArray; + + fn test_info() -> j2k::BatchGroupInfo { + j2k::BatchGroupInfo { dimensions: (3, 2), sample_type: j2k::NativeSampleType::U16, precision: 12, @@ -100,7 +110,29 @@ mod tests { colorspace: Colorspace::SRgb, transfer_syntax: CompressedTransferSyntax::HtJpeg2000Lossless, payload_kind: CompressedPayloadKind::Jpeg2000Codestream, + } + } + + #[test] + fn staged_byte_length_tracks_native_width_and_layout() { + assert_eq!(byte_len(4, &test_info()).unwrap(), 4 * 3 * 2 * 3 * 2); + } + + #[test] + fn staged_size_mismatch_uses_the_existing_interop_error_contract() { + let result = materialize::( + test_info(), + vec![0], + Vec::new(), + vec![Vec::new()], + vec![0; 35], + &FlexDevice, + ); + + let Err(BurnDecodeError::AcceleratorInterop { backend, message }) = result else { + panic!("size mismatch must use the patch-compatible interop variant"); }; - assert_eq!(byte_len(4, &info).unwrap(), 4 * 3 * 2 * 3 * 2); + assert_eq!(backend, "codec staging"); + assert_eq!(message, "staged codec output has 35 bytes; expected 36"); } } diff --git a/crates/j2k-ml/tests/cuda.rs b/crates/j2k-ml/tests/cuda.rs index 19a51469..d4236c94 100644 --- a/crates/j2k-ml/tests/cuda.rs +++ b/crates/j2k-ml/tests/cuda.rs @@ -54,9 +54,17 @@ fn staged_cuda_preserves_native_u16_and_i16_samples() { } let unsigned = [0_u16, 1, 2048, 4095]; let signed = [-2048_i16, -1, 0, 2047]; + let unsigned_bytes = unsigned + .into_iter() + .flat_map(u16::to_ne_bytes) + .collect::>(); + let signed_bytes = signed + .into_iter() + .flat_map(i16::to_ne_bytes) + .collect::>(); let cases = [ ( - encode_gray(&unsigned, 12, false), + encode_gray(&unsigned_bytes, 12, false), DType::U16, unsigned .iter() @@ -64,7 +72,7 @@ fn staged_cuda_preserves_native_u16_and_i16_samples() { .collect::>(), ), ( - encode_gray(&signed, 12, true), + encode_gray(&signed_bytes, 12, true), DType::I16, signed .iter() @@ -150,9 +158,14 @@ fn staged_cuda_rgb_preserves_subnative_codes_and_burn_layout() { let rgb12 = (0_u32..4 * 4 * 3) .map(|value| ((value * 977 + 31) & 0x0fff) as u16) .collect::>(); + let rgb12_bytes = rgb12 + .iter() + .copied() + .flat_map(u16::to_ne_bytes) + .collect::>(); let cases = [ (encode_rgb(&rgb7, 7), DType::U8), - (encode_rgb(&rgb12, 12), DType::U16), + (encode_rgb(&rgb12_bytes, 12), DType::U16), ]; for layout in [BatchLayout::Nhwc, BatchLayout::Nchw] { @@ -329,11 +342,7 @@ fn openjph_i16_oracle(fixture: &OpenJphBatchFixture) -> Vec { } } -fn encode_gray(samples: &[T], precision: u8, signed: bool) -> Vec { - let byte_len = std::mem::size_of_val(samples); - // SAFETY: plain integer fixtures are copied immediately into the encoder; - // their native little-endian representation is the codec's input format. - let bytes = unsafe { std::slice::from_raw_parts(samples.as_ptr().cast::(), byte_len) }; +fn encode_gray(bytes: &[u8], precision: u8, signed: bool) -> Vec { let samples = J2kLosslessSamples::new(bytes, 2, 2, 1, precision, signed) .expect("valid native integer samples"); encode_j2k_lossless( @@ -346,11 +355,7 @@ fn encode_gray(samples: &[T], precision: u8, signed: bool) -> Vec { .codestream } -fn encode_rgb(samples: &[T], precision: u8) -> Vec { - let byte_len = std::mem::size_of_val(samples); - // SAFETY: plain integer fixtures are copied immediately into the encoder; - // their native little-endian representation is the codec's input format. - let bytes = unsafe { std::slice::from_raw_parts(samples.as_ptr().cast::(), byte_len) }; +fn encode_rgb(bytes: &[u8], precision: u8) -> Vec { let samples = J2kLosslessSamples::new(bytes, 4, 4, 3, precision, false) .expect("valid RGB native integer samples"); encode_j2k_lossless( diff --git a/crates/j2k-ml/tests/cuda_batch_sessions.rs b/crates/j2k-ml/tests/cuda_batch_sessions.rs index 473e5cf8..739bbde5 100644 --- a/crates/j2k-ml/tests/cuda_batch_sessions.rs +++ b/crates/j2k-ml/tests/cuda_batch_sessions.rs @@ -231,8 +231,17 @@ fn cuda_burn_batch_continues_after_one_group_submit_failure() { assert_eq!(output.groups[0].source_indices, [1]); assert_eq!(output.group_errors.len(), 1); assert_eq!(output.group_errors[0].source_indices(), &[0]); + let BurnDecodeError::Cuda(j2k_cuda::CudaBatchError::GroupExecution { + source_indices, + source, + .. + }) = output.group_errors[0].source() + else { + panic!("group-local CUDA failure must use the published batch error contract"); + }; + assert_eq!(source_indices, &[0]); assert!(matches!( - output.group_errors[0].source(), - BurnDecodeError::CudaCodec(j2k_cuda::Error::UnsupportedCudaRequest { .. }) + source.as_ref(), + j2k_cuda::Error::UnsupportedCudaRequest { .. } )); } diff --git a/crates/j2k-ml/tests/metal/sessions.rs b/crates/j2k-ml/tests/metal/sessions.rs index 3eb019eb..d89d7abc 100644 --- a/crates/j2k-ml/tests/metal/sessions.rs +++ b/crates/j2k-ml/tests/metal/sessions.rs @@ -2,6 +2,36 @@ use super::*; +#[test] +fn compatibility_metal_decoder_names_keep_debug_and_submit_contract() { + if !metal_runtime_gate("j2k-ml 0.7 Metal decoder compatibility names") { + return; + } + + let mut decoder = j2k_ml::MetalBurnDecoder::system_default(BatchDecodeOptions::default()) + .expect("0.7-compatible J2K/Burn Metal session"); + assert!( + format!("{decoder:?}").starts_with("MetalBurnDecoder"), + "the concrete 0.7 decoder name must remain visible in Debug output" + ); + + let submitted = decoder + .submit(vec![EncodedImage::full(Arc::from( + openhtj2k_refinement_fixture(), + ))]) + .expect("submit through the 0.7-compatible decoder name"); + assert!( + format!("{submitted:?}").starts_with("SubmittedMetalBurnBatch"), + "the concrete 0.7 submitted-batch name must remain visible in Debug output" + ); + assert_eq!(submitted.len(), 1); + + let output = submitted.wait().expect("finish compatibility-name batch"); + assert!(output.errors.is_empty()); + assert!(output.group_errors.is_empty()); + assert_eq!(output.groups.len(), 1); +} + #[test] fn repeated_default_decoders_reuse_one_initialized_burn_device() { if !metal_runtime_gate("j2k-ml repeated paired Metal device") { diff --git a/crates/j2k-native/Cargo.toml b/crates/j2k-native/Cargo.toml index 5598e9b7..429c92c0 100644 --- a/crates/j2k-native/Cargo.toml +++ b/crates/j2k-native/Cargo.toml @@ -28,9 +28,9 @@ fearless_simd = { workspace = true, optional = true } libm = { workspace = true } log = { workspace = true, optional = true } rayon = { workspace = true, optional = true } -j2k-profile = { path = "../j2k-profile", version = "=0.7.5", default-features = false } -j2k-codec-math = { path = "../j2k-codec-math", version = "=0.7.5" } -j2k-types = { path = "../j2k-types", version = "=0.7.5" } +j2k-profile = { path = "../j2k-profile", version = "=0.8.0", default-features = false } +j2k-codec-math = { path = "../j2k-codec-math", version = "=0.8.0" } +j2k-types = { path = "../j2k-types", version = "=0.8.0" } [dev-dependencies] criterion = { workspace = true } diff --git a/crates/j2k-native/src/color.rs b/crates/j2k-native/src/color.rs index 6e72e381..f3ff6817 100644 --- a/crates/j2k-native/src/color.rs +++ b/crates/j2k-native/src/color.rs @@ -25,7 +25,7 @@ pub(crate) fn resolve_alpha_and_color_space( header: &Header<'_>, settings: &DecodeSettings, retained_baseline_bytes: usize, -) -> Result<(ColorSpace, bool)> { +) -> Result<(ColorSpace, bool, bool)> { let mut num_components = header.component_infos.len(); // Override number of components with what is actually in the palette box @@ -64,6 +64,7 @@ pub(crate) fn resolve_alpha_and_color_space( }; let actual_num_components = header.component_infos.len(); + let mut used_lenient_metadata_recovery = false; // Validate the number of channels. if boxes.palette.is_none() @@ -76,6 +77,7 @@ pub(crate) fn resolve_alpha_and_color_space( // See OPENJPEG test case orb-blue10-lin-j2k. Assume that we have an // alpha channel in this case. has_alpha = true; + used_lenient_metadata_recovery = true; } else { // Color space is invalid, attempt to repair. if actual_num_components == 1 || (actual_num_components == 2 && has_alpha) { @@ -97,7 +99,7 @@ pub(crate) fn resolve_alpha_and_color_space( } } - Ok((color_space, has_alpha)) + Ok((color_space, has_alpha, used_lenient_metadata_recovery)) } /// The color space of the image. diff --git a/crates/j2k-native/src/image.rs b/crates/j2k-native/src/image.rs index 889133d3..7d71bce5 100644 --- a/crates/j2k-native/src/image.rs +++ b/crates/j2k-native/src/image.rs @@ -42,20 +42,31 @@ pub struct DecodeSettings { pub resolve_palette_indices: bool, /// Whether strict mode should be enabled when decoding. /// - /// The default is lenient for compatibility with older releases. Lenient - /// mode may tolerate malformed optional container metadata that strict mode - /// rejects. Use [`DecodeSettings::strict`] for fail-closed validation of - /// public or adversarial inputs. + /// The default is strict. Lenient mode is limited to the recoveries listed + /// on [`DecodeSettings::lenient`]; it never relaxes codestream, bounds, + /// overflow, allocation, or resource-limit validation. pub strict: bool, /// A hint for the target resolution that the image should be decoded at. pub target_resolution: Option<(u32, u32)>, } impl DecodeSettings { - /// Compatibility decode settings. - /// - /// Lenient mode keeps the historical behavior of accepting recoverable - /// optional metadata problems where possible. + /// Compatibility settings for explicitly recoverable JP2/JPH metadata. + /// + /// Lenient mode may: + /// + /// - ignore a malformed trailing top-level box after the required image + /// header and codestream boxes were parsed; + /// - ignore a malformed trailing child box after the required `ihdr` and + /// `colr` boxes were parsed; + /// - ignore a malformed optional `cdef` or `pclr` box, preserving any + /// earlier complete value; and + /// - infer an undeclared alpha channel when an otherwise consistent JP2 + /// color declaration has exactly one extra codestream component. + /// + /// Raw codestream validation and entropy decoding remain strict. Bounds, + /// integer-overflow, allocation, and resource-limit checks are identical + /// in both modes. #[must_use] pub const fn lenient() -> Self { Self { @@ -85,7 +96,7 @@ impl DecodeSettings { impl Default for DecodeSettings { fn default() -> Self { - Self::lenient() + Self::strict() } } @@ -103,6 +114,9 @@ pub struct Image<'a> { pub(crate) boxes: ImageBoxes, /// Settings that should be applied during decoding. pub(crate) settings: DecodeSettings, + /// Whether parsing used one of the explicitly documented lenient + /// container-metadata recoveries. + pub(crate) used_lenient_metadata_recovery: bool, /// Whether the image has an alpha channel. pub(crate) has_alpha: bool, /// The color space of the image. @@ -124,35 +138,54 @@ impl<'a> ImageSource<'a> { } } -impl<'a> Image<'a> { - pub(crate) fn from_parsed_parts( - source: ImageSource<'a>, - header: Header<'a>, +pub(crate) struct ImageProperties { + boxes: ImageBoxes, + settings: DecodeSettings, + color_space: ColorSpace, + has_alpha: bool, + used_lenient_metadata_recovery: bool, +} + +impl ImageProperties { + pub(crate) const fn new( boxes: ImageBoxes, settings: DecodeSettings, color_space: ColorSpace, has_alpha: bool, - ) -> Result { - Self::from_parsed_parts_with_retained_baseline( - source, - header, + used_lenient_metadata_recovery: bool, + ) -> Self { + Self { boxes, settings, color_space, has_alpha, - 0, - ) + used_lenient_metadata_recovery, + } + } +} + +impl<'a> Image<'a> { + pub(crate) fn from_parsed_parts( + source: ImageSource<'a>, + header: Header<'a>, + properties: ImageProperties, + ) -> Result { + Self::from_parsed_parts_with_retained_baseline(source, header, properties, 0) } pub(crate) fn from_parsed_parts_with_retained_baseline( source: ImageSource<'a>, header: Header<'a>, - boxes: ImageBoxes, - settings: DecodeSettings, - color_space: ColorSpace, - has_alpha: bool, + properties: ImageProperties, retained_baseline_bytes: usize, ) -> Result { + let ImageProperties { + boxes, + settings, + color_space, + has_alpha, + used_lenient_metadata_recovery, + } = properties; let metadata_bytes = retained_metadata_bytes(&header, &boxes, &color_space)?; allocation::combine_retained_bytes(retained_baseline_bytes, metadata_bytes)?; Ok(Self { @@ -161,6 +194,7 @@ impl<'a> Image<'a> { header, boxes, settings, + used_lenient_metadata_recovery, has_alpha, color_space, }) @@ -181,6 +215,13 @@ impl<'a> Image<'a> { self.retained_metadata_bytes() } + /// Whether parsing used an explicitly documented lenient metadata recovery. + #[doc(hidden)] + #[must_use] + pub const fn used_lenient_metadata_recovery(&self) -> bool { + self.used_lenient_metadata_recovery + } + /// Try to create a new JPEG2000 image from the given data. /// /// # Errors diff --git a/crates/j2k-native/src/j2c/bitplane/tests.rs b/crates/j2k-native/src/j2c/bitplane/tests.rs index 00d12743..3a83591e 100644 --- a/crates/j2k-native/src/j2c/bitplane/tests.rs +++ b/crates/j2k-native/src/j2c/bitplane/tests.rs @@ -416,6 +416,65 @@ fn scan_unit_masks_track_significance_and_current_bitplane_zero_coding() { assert_eq!(ctx.zero_coding_scan_masks[scan_unit], 0); } +#[test] +fn classic_selective_bypass_round_trips_padded_rgb8_cb_block() { + let pixels = (0..7 * 5 * 3) + .map(|index| u8::try_from((index * 41) & 0xff).expect("masked pixel fits u8")) + .collect::>(); + let mut coefficients = vec![0i32; 8 * 8]; + for y in 0..5usize { + for x in 0..7usize { + let src = (y * 7 + x) * 3; + coefficients[y * 8 + x] = i32::from(pixels[src + 2]) - i32::from(pixels[src + 1]); + } + } + let style = CodeBlockStyle { + selective_arithmetic_coding_bypass: true, + ..CodeBlockStyle::default() + }; + let encoded = bitplane_encode::encode_code_block_segments_with_style( + &coefficients, + 8, + 8, + SubBandType::LowLow, + 9, + style, + ); + let segments = encoded + .segments + .iter() + .map(|segment| J2kCodeBlockSegment { + data_offset: segment.data_offset, + data_length: segment.data_length, + start_coding_pass: segment.start_coding_pass, + end_coding_pass: segment.end_coding_pass, + use_arithmetic: segment.use_arithmetic, + }) + .collect::>(); + let mut ctx = BitPlaneDecodeContext::default(); + + decode_code_block_segments_validated( + &encoded.data, + &segments, + 8, + 8, + encoded.num_zero_bitplanes, + encoded.num_coding_passes, + 9, + SubBandType::LowLow, + &style, + true, + &mut ctx, + ) + .expect("decode selective-bypass Cb code block"); + + let decoded = ctx + .coefficient_rows() + .flat_map(|row| row.iter().map(Coefficient::get)) + .collect::>(); + assert_eq!(decoded, coefficients); +} + #[test] fn cleanup_candidate_mask_excludes_significant_and_zero_coded_coefficients() { let mut ctx = BitPlaneDecodeContext::default(); diff --git a/crates/j2k-native/src/j2c/codestream/model.rs b/crates/j2k-native/src/j2c/codestream/model.rs index 7125fe89..eec6b421 100644 --- a/crates/j2k-native/src/j2c/codestream/model.rs +++ b/crates/j2k-native/src/j2c/codestream/model.rs @@ -441,3 +441,13 @@ impl SizeData { self.reference_grid_height - self.image_area_y_offset } } + +crate::move_only::assert_move_only!( + PpmMarkerData<'static>, + PacketLengthMarker, + ComponentInfo, + QuantizationInfo, + CodingStyleDefault, + CodingStyleComponent, + CodingStyleParameters, +); diff --git a/crates/j2k-native/src/j2c/codestream_write.rs b/crates/j2k-native/src/j2c/codestream_write.rs index 4471cfe8..4a00ea3b 100644 --- a/crates/j2k-native/src/j2c/codestream_write.rs +++ b/crates/j2k-native/src/j2c/codestream_write.rs @@ -78,6 +78,8 @@ pub(crate) struct EncodeParams { pub(crate) precinct_exponents: Vec<(u8, u8)>, } +crate::move_only::assert_move_only!(EncodeParams); + impl Default for EncodeParams { fn default() -> Self { Self { diff --git a/crates/j2k-native/src/j2c/encode/roi_plan.rs b/crates/j2k-native/src/j2c/encode/roi_plan.rs index 1c5a49d0..0b77a21a 100644 --- a/crates/j2k-native/src/j2c/encode/roi_plan.rs +++ b/crates/j2k-native/src/j2c/encode/roi_plan.rs @@ -15,6 +15,8 @@ pub(super) struct ComponentRoiEncodePlan { pub(super) regions: Vec, } +crate::move_only::assert_move_only!(ComponentRoiEncodePlan); + #[derive(Clone, Copy, Debug)] pub(super) struct ComponentRoiEncodeRegion { pub(super) x: u32, diff --git a/crates/j2k-native/src/j2c/mod.rs b/crates/j2k-native/src/j2c/mod.rs index b9d75e47..6ec13d00 100644 --- a/crates/j2k-native/src/j2c/mod.rs +++ b/crates/j2k-native/src/j2c/mod.rs @@ -33,7 +33,7 @@ use alloc::vec::Vec; use super::jp2::colr::{ColorSpace, ColorSpecificationBox, EnumeratedColorspace}; use super::jp2::ImageBoxes; use crate::error::{bail, FormatError, MarkerError, Result}; -use crate::image::ImageSource; +use crate::image::{ImageProperties, ImageSource}; use crate::j2c::codestream::markers; use crate::reader::BitReader; use crate::{resolve_alpha_and_color_space, DecodeSettings, Image}; @@ -65,6 +65,8 @@ pub(crate) struct ComponentData { pub(crate) signed: bool, } +crate::move_only::assert_move_only!(ComponentData); + pub(crate) fn parse<'a>(stream: &'a [u8], settings: &DecodeSettings) -> Result> { parse_with_retained_baseline(stream, settings, 0) } @@ -74,8 +76,10 @@ pub(crate) fn parse_with_retained_baseline<'a>( settings: &DecodeSettings, retained_baseline_bytes: usize, ) -> Result> { + let mut strict_settings = *settings; + strict_settings.strict = true; let parsed_codestream = - parse_raw_with_retained_baseline(stream, settings, retained_baseline_bytes)?; + parse_raw_with_retained_baseline(stream, &strict_settings, retained_baseline_bytes)?; let header = &parsed_codestream.header; // Raw codestreams do not carry JP2 channel definitions. Keep the // conventional grayscale/RGB assumptions for 1- and 3-component images, @@ -101,29 +105,24 @@ pub(crate) fn parse_with_retained_baseline<'a>( retained_baseline_bytes, )?; - let (color_space, has_alpha) = resolve_alpha_and_color_space( + let (color_space, has_alpha, _) = resolve_alpha_and_color_space( &boxes, &parsed_codestream.header, - settings, + &strict_settings, retained_baseline_bytes, )?; + let properties = ImageProperties::new(boxes, *settings, color_space, has_alpha, false); if retained_baseline_bytes == 0 { Image::from_parsed_parts( ImageSource::new(stream, parsed_codestream.data), parsed_codestream.header, - boxes, - *settings, - color_space, - has_alpha, + properties, ) } else { Image::from_parsed_parts_with_retained_baseline( ImageSource::new(stream, parsed_codestream.data), parsed_codestream.header, - boxes, - *settings, - color_space, - has_alpha, + properties, retained_baseline_bytes, ) } diff --git a/crates/j2k-native/src/j2c/roi.rs b/crates/j2k-native/src/j2c/roi.rs index 33103aca..770e4d50 100644 --- a/crates/j2k-native/src/j2c/roi.rs +++ b/crates/j2k-native/src/j2c/roi.rs @@ -20,6 +20,8 @@ pub(crate) struct RoiPlan { final_windows: Vec>, } +crate::move_only::assert_move_only!(RoiPlan); + impl RoiPlan { #[expect( clippy::similar_names, diff --git a/crates/j2k-native/src/j2c/tile.rs b/crates/j2k-native/src/j2c/tile.rs index 1b94d176..aa45498b 100644 --- a/crates/j2k-native/src/j2c/tile.rs +++ b/crates/j2k-native/src/j2c/tile.rs @@ -92,6 +92,14 @@ struct PacketLengthMetadata { lengths: Vec, } +crate::move_only::assert_move_only!( + Tile<'static>, + MergedTilePart<'static>, + SeparatedTilePart<'static>, + TilePart<'static>, + PacketLengthMetadata, +); + impl PacketLengthMetadata { fn new(present: bool, lengths: Vec) -> Self { Self { present, lengths } diff --git a/crates/j2k-native/src/j2c/tile/cursor.rs b/crates/j2k-native/src/j2c/tile/cursor.rs index 62b52c7e..b2d49e11 100644 --- a/crates/j2k-native/src/j2c/tile/cursor.rs +++ b/crates/j2k-native/src/j2c/tile/cursor.rs @@ -116,10 +116,14 @@ impl<'data> TilePartCursor<'_, 'data> { let PacketLengthExpectation::Length(expected) = expected else { return Some(()); }; - let packet_start = packet_start?; let actual = match self { - Self::Merged { data, .. } => data.offset().checked_sub(packet_start)?, - Self::Separated { .. } => return Some(()), + Self::Merged { data, .. } => data.offset().checked_sub(packet_start?)?, + Self::Separated { .. } => { + // PLT entries still advance one-for-one when packet headers + // live in PPM/PPT, but there is no single interleaved range + // from which to validate the combined header-and-body length. + return Some(()); + } }; (actual == expected as usize).then_some(()) } diff --git a/crates/j2k-native/src/j2c/tile/cursor/tests.rs b/crates/j2k-native/src/j2c/tile/cursor/tests.rs index 4b2b4152..d1a1e5e6 100644 --- a/crates/j2k-native/src/j2c/tile/cursor/tests.rs +++ b/crates/j2k-native/src/j2c/tile/cursor/tests.rs @@ -68,3 +68,24 @@ fn plt_length_cursor_validates_each_packet_and_resets_for_reuse() { assert_eq!(mismatched.body().read_bytes(2), Some(data.as_slice())); assert_eq!(mismatched.validate_packet_length(packet_start), None); } + +#[test] +fn separated_packet_headers_consume_plt_entries_without_merged_offsets() { + let header = [0_u8]; + let body = [0x11_u8, 0x22]; + let tile_part = TilePart::Separated(SeparatedTilePart { + headers: vec![BitReader::new(&header)], + body: BitReader::new(&body), + packet_lengths: PacketLengthMetadata::new(true, vec![2]), + }); + let mut cursor = tile_part.cursor().expect("separated cursor"); + + assert_eq!(cursor.packet_start_offset(), None); + assert_eq!(cursor.body().read_bytes(2), Some(body.as_slice())); + cursor + .validate_packet_length(None) + .expect("separated headers do not expose one merged packet offset"); + cursor + .validate_all_packet_lengths_consumed() + .expect("the separated PLT entry was consumed"); +} diff --git a/crates/j2k-native/src/jp2/container.rs b/crates/j2k-native/src/jp2/container.rs index 14427caf..1e70f965 100644 --- a/crates/j2k-native/src/jp2/container.rs +++ b/crates/j2k-native/src/jp2/container.rs @@ -2,21 +2,22 @@ //! JP2/JPH container traversal and native decode parse orchestration. +mod header; + use crate::error::{bail, FormatError, Result}; -use crate::image::ImageSource; +use crate::image::{ImageProperties, ImageSource}; use crate::reader::BitReader; use crate::{resolve_alpha_and_color_space, DecodeSettings, Image}; +#[cfg(test)] +pub(super) use self::header::parse_jp2_header_box; +use self::header::parse_jp2_header_box_tracked; use super::allocation; -use super::cdef; -use super::cmap::{self, ComponentMappingBox, ComponentMappingEntry, ComponentMappingType}; -use super::colr; -use super::image_header::{parse_bits_per_component, parse_image_header}; +use super::cmap::{ComponentMappingBox, ComponentMappingEntry, ComponentMappingType}; use super::metadata::{ public_image_header, public_metadata_from_boxes, ImageBoxes, Jp2FileMetadata, Jp2ImageHeaderMetadata, }; -use super::pclr; use super::r#box::{self, FILE_TYPE, JP2_SIGNATURE}; use super::validation::{ validate_codestream_file_kind, validate_component_precision_metadata, @@ -52,6 +53,7 @@ struct ParsedJp2Container<'a> { codestream_offset: usize, codestream: &'a [u8], boxes: ImageBoxes, + used_lenient_metadata_recovery: bool, } const JP2_SIGNATURE_PAYLOAD: [u8; 4] = [0x0D, 0x0A, 0x87, 0x0A]; @@ -136,9 +138,11 @@ pub(crate) fn parse_with_retained_baseline( retained_baseline_bytes, "retained JP2 parse owners", )?; + let mut codestream_settings = settings; + codestream_settings.strict = true; let parsed_codestream = crate::j2c::parse_raw_with_retained_baseline( container.codestream, - &settings, + &codestream_settings, retained_box_bytes, )?; validate_codestream_file_kind(container.file_kind, &parsed_codestream.header)?; @@ -171,29 +175,32 @@ pub(crate) fn parse_with_retained_baseline( image_boxes.component_mapping = Some(ComponentMappingBox { entries: mappings }); } - let (color_space, has_alpha) = resolve_alpha_and_color_space( + let (color_space, has_alpha, recovered_alpha_metadata) = resolve_alpha_and_color_space( &image_boxes, &parsed_codestream.header, &settings, retained_baseline_bytes, )?; + let used_lenient_metadata_recovery = + container.used_lenient_metadata_recovery || recovered_alpha_metadata; + let properties = ImageProperties::new( + image_boxes, + settings, + color_space, + has_alpha, + used_lenient_metadata_recovery, + ); if retained_baseline_bytes == 0 { Image::from_parsed_parts( ImageSource::new(data, parsed_codestream.data), parsed_codestream.header, - image_boxes, - settings, - color_space, - has_alpha, + properties, ) } else { Image::from_parsed_parts_with_retained_baseline( ImageSource::new(data, parsed_codestream.data), parsed_codestream.header, - image_boxes, - settings, - color_space, - has_alpha, + properties, retained_baseline_bytes, ) } @@ -235,11 +242,15 @@ fn parse_jp2_container_with_strict_and_retained_baseline( let mut image_boxes = None; let mut codestream = None; + let mut used_lenient_metadata_recovery = false; while !reader.at_end() { - let current_box = match r#box::read_checked(&mut reader) { - Ok(current_box) => current_box, - Err(error) if strict => return Err(error), - Err(_) => break, + let Some(current_box) = read_recoverable_metadata_box( + &mut reader, + strict, + &mut used_lenient_metadata_recovery, + )? + else { + break; }; match current_box.box_type { @@ -247,11 +258,13 @@ fn parse_jp2_container_with_strict_and_retained_baseline( if image_boxes.is_some() || codestream.is_some() { bail!(FormatError::InvalidBox); } - image_boxes = Some(parse_jp2_header_box( + let parsed_header = parse_jp2_header_box_tracked( current_box.data, strict, retained_baseline_bytes, - )?); + )?; + used_lenient_metadata_recovery |= parsed_header.used_lenient_metadata_recovery; + image_boxes = Some(parsed_header.boxes); } r#box::CONTIGUOUS_CODESTREAM => { if image_boxes.is_none() || codestream.is_some() { @@ -271,99 +284,23 @@ fn parse_jp2_container_with_strict_and_retained_baseline( codestream_offset, codestream, boxes, + used_lenient_metadata_recovery, }) } -pub(super) fn parse_jp2_header_box( - data: &[u8], +fn read_recoverable_metadata_box<'a>( + reader: &mut BitReader<'a>, strict: bool, - retained_baseline_bytes: usize, -) -> Result { - let color_spec_count = count_color_specification_boxes(data, strict)?; - let mut budget = allocation::Jp2AllocationBudget::from_live_bytes(retained_baseline_bytes)?; - let mut boxes = ImageBoxes { - color_specifications: budget.try_vec(color_spec_count, "JP2 COLR metadata")?, - ..ImageBoxes::default() - }; - let mut saw_image_header = false; - let mut reader = BitReader::new(data); - - while !reader.at_end() { - let child_box = match r#box::read_checked(&mut reader) { - Ok(child_box) => child_box, - Err(error) if strict => return Err(error), - Err(_) => break, - }; - match child_box.box_type { - r#box::IMAGE_HEADER => { - if saw_image_header { - bail!(FormatError::InvalidBox); - } - boxes.image_header = Some(parse_image_header(child_box.data)?); - saw_image_header = true; - } - r#box::BITS_PER_COMPONENT => { - let parsed = parse_bits_per_component(child_box.data, &mut budget)?; - let replaced = core::mem::replace(&mut boxes.bits_per_component, parsed); - budget.release_vec(&replaced)?; - } - r#box::CHANNEL_DEFINITION => { - let mut attempt = budget; - match cdef::parse(&mut boxes, child_box.data, &mut attempt) { - Ok(()) => budget = attempt, - Err(crate::DecodeError::Format(_)) if !strict => {} - Err(error) => return Err(error), - } - } - r#box::COLOUR_SPECIFICATION => { - colr::parse(&mut boxes, child_box.data, &mut budget)?; - } - r#box::PALETTE => { - let mut attempt = budget; - match pclr::parse(&mut boxes, child_box.data, &mut attempt) { - Ok(()) => budget = attempt, - Err(crate::DecodeError::Format(_)) if !strict => {} - Err(error) => return Err(error), - } - } - r#box::COMPONENT_MAPPING => { - cmap::parse(&mut boxes, child_box.data, &mut budget)?; - } - _ => { - ldebug!("ignoring header box 0x{:08X}", child_box.box_type); - } - } - } - - if !saw_image_header { - bail!(FormatError::MissingRequiredBox("ihdr")); - } - if boxes.primary_color_specification().is_none() { - bail!(FormatError::MissingRequiredBox("colr")); - } - Ok(boxes) -} - -fn count_color_specification_boxes(data: &[u8], strict: bool) -> Result { - let mut count = 0_usize; - let mut reader = BitReader::new(data); - while !reader.at_end() { - let child_box = match r#box::read_checked(&mut reader) { - Ok(child_box) => child_box, - Err(error) if strict => return Err(error), - Err(_) => break, - }; - if child_box.box_type == r#box::COLOUR_SPECIFICATION { - count = count - .checked_add(1) - .ok_or(crate::DecodeError::AllocationTooLarge { - what: "JP2 COLR metadata", - requested: usize::MAX, - cap: crate::DEFAULT_MAX_DECODE_BYTES, - })?; + used_lenient_metadata_recovery: &mut bool, +) -> Result>> { + match r#box::read_checked(reader) { + Ok(parsed) => Ok(Some(parsed)), + Err(error) if strict => Err(error), + Err(_) => { + *used_lenient_metadata_recovery = true; + Ok(None) } } - Ok(count) } fn classify_file_type(payload: &[u8]) -> Result { diff --git a/crates/j2k-native/src/jp2/container/header.rs b/crates/j2k-native/src/jp2/container/header.rs new file mode 100644 index 00000000..17530ff8 --- /dev/null +++ b/crates/j2k-native/src/jp2/container/header.rs @@ -0,0 +1,137 @@ +// SPDX-License-Identifier: MIT OR Apache-2.0 + +//! JP2 header child traversal and recoverable optional-metadata parsing. + +use crate::error::{bail, FormatError, Result}; +use crate::reader::BitReader; + +use super::read_recoverable_metadata_box; +use crate::jp2::allocation; +use crate::jp2::cdef; +use crate::jp2::cmap; +use crate::jp2::colr; +use crate::jp2::image_header::{parse_bits_per_component, parse_image_header}; +use crate::jp2::metadata::ImageBoxes; +use crate::jp2::pclr; +use crate::jp2::r#box; + +pub(super) struct ParsedJp2Header { + pub(super) boxes: ImageBoxes, + pub(super) used_lenient_metadata_recovery: bool, +} + +#[cfg(test)] +pub(in crate::jp2) fn parse_jp2_header_box( + data: &[u8], + strict: bool, + retained_baseline_bytes: usize, +) -> Result { + Ok(parse_jp2_header_box_tracked(data, strict, retained_baseline_bytes)?.boxes) +} + +pub(super) fn parse_jp2_header_box_tracked( + data: &[u8], + strict: bool, + retained_baseline_bytes: usize, +) -> Result { + let (color_spec_count, mut used_lenient_metadata_recovery) = + count_color_specification_boxes(data, strict)?; + let mut budget = allocation::Jp2AllocationBudget::from_live_bytes(retained_baseline_bytes)?; + let mut boxes = ImageBoxes { + color_specifications: budget.try_vec(color_spec_count, "JP2 COLR metadata")?, + ..ImageBoxes::default() + }; + let mut saw_image_header = false; + let mut reader = BitReader::new(data); + + while !reader.at_end() { + let Some(child_box) = read_recoverable_metadata_box( + &mut reader, + strict, + &mut used_lenient_metadata_recovery, + )? + else { + break; + }; + match child_box.box_type { + r#box::IMAGE_HEADER => { + if saw_image_header { + bail!(FormatError::InvalidBox); + } + boxes.image_header = Some(parse_image_header(child_box.data)?); + saw_image_header = true; + } + r#box::BITS_PER_COMPONENT => { + let parsed = parse_bits_per_component(child_box.data, &mut budget)?; + let replaced = core::mem::replace(&mut boxes.bits_per_component, parsed); + budget.release_vec(&replaced)?; + } + r#box::CHANNEL_DEFINITION => { + let mut attempt = budget; + match cdef::parse(&mut boxes, child_box.data, &mut attempt) { + Ok(()) => budget = attempt, + Err(crate::DecodeError::Format(_)) if !strict => { + used_lenient_metadata_recovery = true; + } + Err(error) => return Err(error), + } + } + r#box::COLOUR_SPECIFICATION => { + colr::parse(&mut boxes, child_box.data, &mut budget)?; + } + r#box::PALETTE => { + let mut attempt = budget; + match pclr::parse(&mut boxes, child_box.data, &mut attempt) { + Ok(()) => budget = attempt, + Err(crate::DecodeError::Format(_)) if !strict => { + used_lenient_metadata_recovery = true; + } + Err(error) => return Err(error), + } + } + r#box::COMPONENT_MAPPING => { + cmap::parse(&mut boxes, child_box.data, &mut budget)?; + } + _ => { + ldebug!("ignoring header box 0x{:08X}", child_box.box_type); + } + } + } + + if !saw_image_header { + bail!(FormatError::MissingRequiredBox("ihdr")); + } + if boxes.primary_color_specification().is_none() { + bail!(FormatError::MissingRequiredBox("colr")); + } + Ok(ParsedJp2Header { + boxes, + used_lenient_metadata_recovery, + }) +} + +fn count_color_specification_boxes(data: &[u8], strict: bool) -> Result<(usize, bool)> { + let mut count = 0_usize; + let mut used_lenient_metadata_recovery = false; + let mut reader = BitReader::new(data); + while !reader.at_end() { + let Some(child_box) = read_recoverable_metadata_box( + &mut reader, + strict, + &mut used_lenient_metadata_recovery, + )? + else { + break; + }; + if child_box.box_type == r#box::COLOUR_SPECIFICATION { + count = count + .checked_add(1) + .ok_or(crate::DecodeError::AllocationTooLarge { + what: "JP2 COLR metadata", + requested: usize::MAX, + cap: crate::DEFAULT_MAX_DECODE_BYTES, + })?; + } + } + Ok((count, used_lenient_metadata_recovery)) +} diff --git a/crates/j2k-native/src/lib.rs b/crates/j2k-native/src/lib.rs index 570382e8..4f5c66ad 100644 --- a/crates/j2k-native/src/lib.rs +++ b/crates/j2k-native/src/lib.rs @@ -156,6 +156,7 @@ mod direct_cpu; mod direct_plan; mod direct_roi; pub(crate) mod math; +mod move_only; #[doc(hidden)] pub mod packet_math; pub(crate) mod profile; diff --git a/crates/j2k-native/src/math.rs b/crates/j2k-native/src/math.rs index 3f9bca91..742c4536 100644 --- a/crates/j2k-native/src/math.rs +++ b/crates/j2k-native/src/math.rs @@ -781,6 +781,8 @@ pub(crate) struct SimdBuffer { original_len: usize, } +crate::move_only::assert_move_only!(SimdBuffer<8>); + #[derive(Debug, Clone, Copy)] pub(crate) struct SimdBufferAllocationError; diff --git a/crates/j2k-native/src/move_only.rs b/crates/j2k-native/src/move_only.rs new file mode 100644 index 00000000..31695717 --- /dev/null +++ b/crates/j2k-native/src/move_only.rs @@ -0,0 +1,27 @@ +// SPDX-License-Identifier: MIT OR Apache-2.0 + +//! Compile-time assertions for owner types that must not implement `Clone`. + +pub(crate) trait AmbiguousIfClone {} + +impl AmbiguousIfClone<()> for T {} + +pub(crate) struct CloneMarker; + +impl AmbiguousIfClone for T {} + +pub(crate) struct RequireUnambiguous, Marker>( + core::marker::PhantomData<(*const T, Marker)>, +); + +macro_rules! assert_move_only { + ($($ty:ty),* $(,)?) => { + $( + const _: () = { + let _: Option<$crate::move_only::RequireUnambiguous<$ty, _>> = None; + }; + )* + }; +} + +pub(crate) use assert_move_only; diff --git a/crates/j2k-native/src/tests.rs b/crates/j2k-native/src/tests.rs index 7b3ac5c1..b8230237 100644 --- a/crates/j2k-native/src/tests.rs +++ b/crates/j2k-native/src/tests.rs @@ -2324,7 +2324,8 @@ fn try_deinterleave_reference_rejects_invalid_geometry() { #[test] fn decode_settings_constructors_make_strictness_explicit() { - assert!(DecodeSettings::default().lenient_tolerance_enabled()); + assert!(DecodeSettings::default().strict); + assert!(!DecodeSettings::default().lenient_tolerance_enabled()); assert!(DecodeSettings::lenient().lenient_tolerance_enabled()); assert!(!DecodeSettings::strict().lenient_tolerance_enabled()); assert!(DecodeSettings::strict().strict); diff --git a/crates/j2k-native/src/writer.rs b/crates/j2k-native/src/writer.rs index 9abf781b..06633a1d 100644 --- a/crates/j2k-native/src/writer.rs +++ b/crates/j2k-native/src/writer.rs @@ -51,6 +51,10 @@ impl BitAccumulator { )] fn flush(&mut self) -> Option { if self.bits_in_buffer == 0 { + if self.last_byte_was_ff { + self.last_byte_was_ff = false; + return Some(0); + } return None; } let limit = if self.last_byte_was_ff { 7 } else { 8 }; @@ -307,6 +311,14 @@ mod tests { assert_eq!(data[1], 0b101_0101); } + #[test] + fn test_trailing_ff_is_followed_by_stuffed_zero_byte() { + let mut w = BitWriter::new(); + w.write_bits(0xFF, 8); + + assert_eq!(w.finish(), vec![0xFF, 0x00]); + } + #[test] fn test_marker_write() { let mut w = BitWriter::new(); diff --git a/crates/j2k-test-support/Cargo.toml b/crates/j2k-test-support/Cargo.toml index 91ac0606..5771be7c 100644 --- a/crates/j2k-test-support/Cargo.toml +++ b/crates/j2k-test-support/Cargo.toml @@ -16,7 +16,7 @@ path = "src/lib.rs" j2k-native-fixtures = ["dep:j2k-native"] [dependencies] -j2k-native = { path = "../j2k-native", version = "=0.7.5", optional = true } +j2k-native = { path = "../j2k-native", version = "=0.8.0", optional = true } [lints.rust] unsafe_code = "forbid" diff --git a/crates/j2k-tilecodec/Cargo.toml b/crates/j2k-tilecodec/Cargo.toml index 9c14ed32..c8231057 100644 --- a/crates/j2k-tilecodec/Cargo.toml +++ b/crates/j2k-tilecodec/Cargo.toml @@ -19,7 +19,7 @@ name = "j2k_tilecodec" path = "src/lib.rs" [dependencies] -j2k-core = { path = "../j2k-core", version = "=0.7.5" } +j2k-core = { path = "../j2k-core", version = "=0.8.0" } thiserror = { workspace = true } flate2 = { workspace = true } zstd = { workspace = true } diff --git a/crates/j2k-tilecodec/fuzz/Cargo.lock b/crates/j2k-tilecodec/fuzz/Cargo.lock index 19edae94..d6af2315 100644 --- a/crates/j2k-tilecodec/fuzz/Cargo.lock +++ b/crates/j2k-tilecodec/fuzz/Cargo.lock @@ -72,14 +72,14 @@ dependencies = [ [[package]] name = "j2k-core" -version = "0.7.5" +version = "0.8.0" dependencies = [ "thiserror", ] [[package]] name = "j2k-tilecodec" -version = "0.7.5" +version = "0.8.0" dependencies = [ "flate2", "j2k-core", diff --git a/crates/j2k-transcode-cuda/Cargo.toml b/crates/j2k-transcode-cuda/Cargo.toml index b5c76aec..37b3ed29 100644 --- a/crates/j2k-transcode-cuda/Cargo.toml +++ b/crates/j2k-transcode-cuda/Cargo.toml @@ -29,10 +29,10 @@ cuda-runtime = [ cuda-profiling = ["cuda-runtime", "j2k-cuda-runtime/cuda-profiling"] [dependencies] -j2k-core = { path = "../j2k-core", version = "=0.7.5" } -j2k-transcode = { path = "../j2k-transcode", version = "=0.7.5" } -j2k-cuda-runtime = { path = "../j2k-cuda-runtime", version = "=0.7.5", optional = true } -j2k-native = { path = "../j2k-native", version = "=0.7.5" } +j2k-core = { path = "../j2k-core", version = "=0.8.0" } +j2k-transcode = { path = "../j2k-transcode", version = "=0.8.0" } +j2k-cuda-runtime = { path = "../j2k-cuda-runtime", version = "=0.8.0", optional = true } +j2k-native = { path = "../j2k-native", version = "=0.8.0" } [dev-dependencies] j2k-test-support = { path = "../j2k-test-support" } diff --git a/crates/j2k-transcode-metal/Cargo.toml b/crates/j2k-transcode-metal/Cargo.toml index c3aac98e..bacf54ab 100644 --- a/crates/j2k-transcode-metal/Cargo.toml +++ b/crates/j2k-transcode-metal/Cargo.toml @@ -26,21 +26,21 @@ default = [] bench-internals = ["j2k-transcode/dev-support"] [dependencies] -j2k-codec-math = { path = "../j2k-codec-math", version = "=0.7.5" } -j2k-core = { path = "../j2k-core", version = "=0.7.5" } -j2k-metal-support = { path = "../j2k-metal-support", version = "=0.7.5" } -j2k-transcode = { path = "../j2k-transcode", version = "=0.7.5" } +j2k-codec-math = { path = "../j2k-codec-math", version = "=0.8.0" } +j2k-core = { path = "../j2k-core", version = "=0.8.0" } +j2k-metal-support = { path = "../j2k-metal-support", version = "=0.8.0" } +j2k-transcode = { path = "../j2k-transcode", version = "=0.8.0" } [target.'cfg(target_os = "macos")'.dependencies] -j2k-metal = { path = "../j2k-metal", version = "=0.7.5" } +j2k-metal = { path = "../j2k-metal", version = "=0.8.0" } metal = { workspace = true } [dev-dependencies] criterion = { workspace = true } rayon = { workspace = true } -j2k-native = { path = "../j2k-native", version = "=0.7.5" } -j2k-jpeg = { path = "../j2k-jpeg", version = "=0.7.5" } -j2k-profile = { path = "../j2k-profile", version = "=0.7.5" } +j2k-native = { path = "../j2k-native", version = "=0.8.0" } +j2k-jpeg = { path = "../j2k-jpeg", version = "=0.8.0" } +j2k-profile = { path = "../j2k-profile", version = "=0.8.0" } j2k-test-support = { path = "../j2k-test-support" } j2k-transcode-test-support = { path = "../j2k-transcode-test-support" } diff --git a/crates/j2k-transcode-test-support/Cargo.toml b/crates/j2k-transcode-test-support/Cargo.toml index 11b45eef..55ba23e5 100644 --- a/crates/j2k-transcode-test-support/Cargo.toml +++ b/crates/j2k-transcode-test-support/Cargo.toml @@ -13,11 +13,11 @@ name = "j2k_transcode_test_support" path = "src/lib.rs" [dependencies] -j2k-transcode = { path = "../j2k-transcode", version = "=0.7.5", features = ["dev-support"] } -j2k-types = { path = "../j2k-types", version = "=0.7.5" } +j2k-transcode = { path = "../j2k-transcode", version = "=0.8.0", features = ["dev-support"] } +j2k-types = { path = "../j2k-types", version = "=0.8.0" } [dev-dependencies] -j2k-native = { path = "../j2k-native", version = "=0.7.5" } +j2k-native = { path = "../j2k-native", version = "=0.8.0" } [lints.rust] unsafe_code = "forbid" diff --git a/crates/j2k-transcode-test-support/src/dct53_multilevel.rs b/crates/j2k-transcode-test-support/src/dct53_multilevel.rs index f13af2eb..067df63a 100644 --- a/crates/j2k-transcode-test-support/src/dct53_multilevel.rs +++ b/crates/j2k-transcode-test-support/src/dct53_multilevel.rs @@ -28,6 +28,8 @@ pub struct Dwt53MultiLevel { pub final_ll_height: usize, } +crate::move_only::assert_move_only!(Dwt53MultiLevel); + impl Dwt53MultiLevel { /// Maximum absolute coefficient difference across matching levels. #[must_use] diff --git a/crates/j2k-transcode-test-support/src/lib.rs b/crates/j2k-transcode-test-support/src/lib.rs index e936a6d5..76eacb7f 100644 --- a/crates/j2k-transcode-test-support/src/lib.rs +++ b/crates/j2k-transcode-test-support/src/lib.rs @@ -8,6 +8,7 @@ pub mod corpus_validation; pub mod dct53_1d; pub mod dct53_multilevel; mod dwt_diff; +mod move_only; use j2k_transcode::{ htj2k97_subband_delta, htj2k97_subband_total_bitplanes, Dwt97TwoDimensional, diff --git a/crates/j2k-transcode-test-support/src/move_only.rs b/crates/j2k-transcode-test-support/src/move_only.rs new file mode 100644 index 00000000..31695717 --- /dev/null +++ b/crates/j2k-transcode-test-support/src/move_only.rs @@ -0,0 +1,27 @@ +// SPDX-License-Identifier: MIT OR Apache-2.0 + +//! Compile-time assertions for owner types that must not implement `Clone`. + +pub(crate) trait AmbiguousIfClone {} + +impl AmbiguousIfClone<()> for T {} + +pub(crate) struct CloneMarker; + +impl AmbiguousIfClone for T {} + +pub(crate) struct RequireUnambiguous, Marker>( + core::marker::PhantomData<(*const T, Marker)>, +); + +macro_rules! assert_move_only { + ($($ty:ty),* $(,)?) => { + $( + const _: () = { + let _: Option<$crate::move_only::RequireUnambiguous<$ty, _>> = None; + }; + )* + }; +} + +pub(crate) use assert_move_only; diff --git a/crates/j2k-transcode/Cargo.toml b/crates/j2k-transcode/Cargo.toml index 160c7f2f..58a39b5a 100644 --- a/crates/j2k-transcode/Cargo.toml +++ b/crates/j2k-transcode/Cargo.toml @@ -22,12 +22,12 @@ dev-support = [] [dependencies] rayon = { workspace = true } -j2k-core = { path = "../j2k-core", version = "=0.7.5" } -j2k-codec-math = { path = "../j2k-codec-math", version = "=0.7.5" } -j2k-jpeg = { path = "../j2k-jpeg", version = "=0.7.5" } -j2k = { path = "../j2k", version = "=0.7.5" } -j2k-native = { path = "../j2k-native", version = "=0.7.5" } -j2k-profile = { path = "../j2k-profile", version = "=0.7.5", default-features = false } +j2k-core = { path = "../j2k-core", version = "=0.8.0" } +j2k-codec-math = { path = "../j2k-codec-math", version = "=0.8.0" } +j2k-jpeg = { path = "../j2k-jpeg", version = "=0.8.0" } +j2k = { path = "../j2k", version = "=0.8.0" } +j2k-native = { path = "../j2k-native", version = "=0.8.0" } +j2k-profile = { path = "../j2k-profile", version = "=0.8.0", default-features = false } [dev-dependencies] criterion = { workspace = true } diff --git a/crates/j2k-transcode/fuzz/Cargo.lock b/crates/j2k-transcode/fuzz/Cargo.lock index e822ea8d..8eb46a0c 100644 --- a/crates/j2k-transcode/fuzz/Cargo.lock +++ b/crates/j2k-transcode/fuzz/Cargo.lock @@ -83,7 +83,7 @@ dependencies = [ [[package]] name = "j2k" -version = "0.7.5" +version = "0.8.0" dependencies = [ "j2k-codec-math", "j2k-core", @@ -94,18 +94,18 @@ dependencies = [ [[package]] name = "j2k-codec-math" -version = "0.7.5" +version = "0.8.0" [[package]] name = "j2k-core" -version = "0.7.5" +version = "0.8.0" dependencies = [ "thiserror", ] [[package]] name = "j2k-jpeg" -version = "0.7.5" +version = "0.8.0" dependencies = [ "j2k-codec-math", "j2k-core", @@ -117,7 +117,7 @@ dependencies = [ [[package]] name = "j2k-native" -version = "0.7.5" +version = "0.8.0" dependencies = [ "fearless_simd", "j2k-codec-math", @@ -129,11 +129,11 @@ dependencies = [ [[package]] name = "j2k-profile" -version = "0.7.5" +version = "0.8.0" [[package]] name = "j2k-transcode" -version = "0.7.5" +version = "0.8.0" dependencies = [ "j2k", "j2k-codec-math", @@ -155,7 +155,7 @@ dependencies = [ [[package]] name = "j2k-types" -version = "0.7.5" +version = "0.8.0" [[package]] name = "jobserver" diff --git a/crates/j2k-transcode/src/accelerator_contracts.rs b/crates/j2k-transcode/src/accelerator_contracts.rs index a5de5d07..1c3cd2d2 100644 --- a/crates/j2k-transcode/src/accelerator_contracts.rs +++ b/crates/j2k-transcode/src/accelerator_contracts.rs @@ -132,6 +132,11 @@ pub struct PreencodedHtj2k97CompactBatchGroups { pub groups: Vec>, } +crate::move_only::assert_move_only!( + PreencodedHtj2k97CompactBatch, + PreencodedHtj2k97CompactBatchGroups, +); + /// Encode parameters needed to quantize 9/7 output directly into HTJ2K /// code-block coefficient layout. #[derive(Debug, Clone, Copy, PartialEq)] diff --git a/crates/j2k-transcode/src/jpeg_to_htj2k/output.rs b/crates/j2k-transcode/src/jpeg_to_htj2k/output.rs index 0fcaa9ee..fec04e6f 100644 --- a/crates/j2k-transcode/src/jpeg_to_htj2k/output.rs +++ b/crates/j2k-transcode/src/jpeg_to_htj2k/output.rs @@ -11,6 +11,8 @@ pub struct EncodedTranscode { pub report: TranscodeReport, } +crate::move_only::assert_move_only!(EncodedTranscode); + /// One JPEG tile input for batch transcode. #[derive(Debug, Clone, Copy)] pub struct JpegTileBatchInput<'a> { diff --git a/crates/j2k-transcode/src/lib.rs b/crates/j2k-transcode/src/lib.rs index f190d3df..d2d6d0af 100644 --- a/crates/j2k-transcode/src/lib.rs +++ b/crates/j2k-transcode/src/lib.rs @@ -140,6 +140,7 @@ mod htj2k97_codeblock_oracle; mod jpeg_to_htj2k; #[doc(hidden)] pub mod metrics; +mod move_only; mod pipeline_map; mod resident; mod reversible53; @@ -189,6 +190,12 @@ pub struct Dwt97TwoDimensional { pub high_height: usize, } +move_only::assert_move_only!( + ReversibleDwt53FirstLevel, + Dwt53TwoDimensional, + Dwt97TwoDimensional, +); + pub use dct53_2d::{dct8x8_blocks_then_dwt53_float, dct8x8_blocks_to_dwt53_float_linear}; pub use dct97_2d::dct8x8_blocks_then_dwt97_float; pub use dct97_2d::{dct8x8_blocks_then_dwt97_float_with_scratch, Dct97GridScratch}; diff --git a/crates/j2k-transcode/src/move_only.rs b/crates/j2k-transcode/src/move_only.rs new file mode 100644 index 00000000..31695717 --- /dev/null +++ b/crates/j2k-transcode/src/move_only.rs @@ -0,0 +1,27 @@ +// SPDX-License-Identifier: MIT OR Apache-2.0 + +//! Compile-time assertions for owner types that must not implement `Clone`. + +pub(crate) trait AmbiguousIfClone {} + +impl AmbiguousIfClone<()> for T {} + +pub(crate) struct CloneMarker; + +impl AmbiguousIfClone for T {} + +pub(crate) struct RequireUnambiguous, Marker>( + core::marker::PhantomData<(*const T, Marker)>, +); + +macro_rules! assert_move_only { + ($($ty:ty),* $(,)?) => { + $( + const _: () = { + let _: Option<$crate::move_only::RequireUnambiguous<$ty, _>> = None; + }; + )* + }; +} + +pub(crate) use assert_move_only; diff --git a/crates/j2k-types/src/lib.rs b/crates/j2k-types/src/lib.rs index 512bd3ce..269badb9 100644 --- a/crates/j2k-types/src/lib.rs +++ b/crates/j2k-types/src/lib.rs @@ -22,6 +22,7 @@ pub use decode_payload::{ mod limits; #[doc(hidden)] pub use limits::{MAX_JPEG2000_PART1_COMPONENTS, MAX_JPEG2000_PART1_SAMPLE_BIT_DEPTH}; +mod move_only; mod resident; #[doc(hidden)] pub use resident::{ @@ -1113,3 +1114,33 @@ pub struct PreencodedHtj2k97CompactCodeBlock { /// Number of missing most-significant bitplanes. pub num_zero_bitplanes: u8, } + +move_only::assert_move_only!( + EncodedJ2kCodeBlock, + EncodedHtJ2kCodeBlock, + J2kPacketizationSubband<'static>, + J2kPacketizationResolution<'static>, + J2kForwardDwt53Output, + J2kForwardDwt53Level, + J2kForwardDwt97Output, + J2kForwardDwt97Level, + PrecomputedHtj2k53Component, + PrecomputedHtj2k53Image, + PrecomputedHtj2k97Component, + PrecomputedHtj2k97Image, + PrequantizedHtj2k97Image, + PrequantizedHtj2k97Component, + PrequantizedHtj2k97Resolution, + PrequantizedHtj2k97Subband, + PrequantizedHtj2k97CodeBlock, + PreencodedHtj2k97Image, + PreencodedHtj2k97Component, + PreencodedHtj2k97Resolution, + PreencodedHtj2k97Subband, + PreencodedHtj2k97CodeBlock, + PreencodedHtj2k97CompactImage, + PreencodedHtj2k97CompactComponent, + PreencodedHtj2k97CompactResolution, + PreencodedHtj2k97CompactSubband, + PreencodedHtj2k97CompactCodeBlock, +); diff --git a/crates/j2k-types/src/move_only.rs b/crates/j2k-types/src/move_only.rs new file mode 100644 index 00000000..31695717 --- /dev/null +++ b/crates/j2k-types/src/move_only.rs @@ -0,0 +1,27 @@ +// SPDX-License-Identifier: MIT OR Apache-2.0 + +//! Compile-time assertions for owner types that must not implement `Clone`. + +pub(crate) trait AmbiguousIfClone {} + +impl AmbiguousIfClone<()> for T {} + +pub(crate) struct CloneMarker; + +impl AmbiguousIfClone for T {} + +pub(crate) struct RequireUnambiguous, Marker>( + core::marker::PhantomData<(*const T, Marker)>, +); + +macro_rules! assert_move_only { + ($($ty:ty),* $(,)?) => { + $( + const _: () = { + let _: Option<$crate::move_only::RequireUnambiguous<$ty, _>> = None; + }; + )* + }; +} + +pub(crate) use assert_move_only; diff --git a/crates/j2k/Cargo.toml b/crates/j2k/Cargo.toml index 939fc654..b010a989 100644 --- a/crates/j2k/Cargo.toml +++ b/crates/j2k/Cargo.toml @@ -20,16 +20,16 @@ name = "j2k" path = "src/lib.rs" [dependencies] -j2k-codec-math = { path = "../j2k-codec-math", version = "=0.7.5" } -j2k-core = { path = "../j2k-core", version = "=0.7.5" } -j2k-native = { path = "../j2k-native", version = "=0.7.5" } -j2k-types = { path = "../j2k-types", version = "=0.7.5" } +j2k-codec-math = { path = "../j2k-codec-math", version = "=0.8.0" } +j2k-core = { path = "../j2k-core", version = "=0.8.0" } +j2k-native = { path = "../j2k-native", version = "=0.8.0" } +j2k-types = { path = "../j2k-types", version = "=0.8.0" } thiserror = { workspace = true } [dev-dependencies] proptest = { workspace = true } criterion = { workspace = true } -j2k-native = { path = "../j2k-native", version = "=0.7.5" } +j2k-native = { path = "../j2k-native", version = "=0.8.0" } j2k-test-support = { path = "../j2k-test-support", features = ["j2k-native-fixtures"] } [[bench]] diff --git a/crates/j2k/README.md b/crates/j2k/README.md index 297b02f1..566a0eed 100644 --- a/crates/j2k/README.md +++ b/crates/j2k/README.md @@ -28,12 +28,22 @@ backend. ## Decode strictness -`j2k_native::DecodeSettings::default()` remains lenient for compatibility. -Lenient mode may tolerate recoverable optional container metadata problems that -`DecodeSettings::strict()` rejects. Public `j2k` decode outcomes report -`J2kDecodeWarning::LenientDecodeMode` when the retained lenient default is used; -callers that need fail-closed validation should construct native images with -`DecodeSettings::strict()` or treat that warning as nonpublishable input. +`j2k::DecodeSettings::default()` and +`j2k_native::DecodeSettings::default()` are strict. The one-shot +`J2kView::parse` and `J2kDecoder::new` paths use that default. + +Compatibility callers may opt one image into `DecodeSettings::lenient()` with +`J2kView::parse_with_settings` or `J2kDecoder::new_with_settings`. Lenient mode +is deliberately narrow. It may ignore a malformed trailing top-level box after +the required JP2/JPH header and codestream, ignore a malformed trailing child +box after `ihdr` and `colr`, ignore malformed optional `cdef` or `pclr` +metadata while preserving an earlier complete value, or infer one undeclared +alpha component. Raw codestream and entropy validation, bounds and overflow +checks, allocation limits, and all resource-safety checks remain strict. + +Selecting lenient mode is configuration, not a warning. A successful decode +reports `J2kDecodeWarning::LenientMetadataRecovery` only when parsing actually +used one of those recoveries. ## Links diff --git a/crates/j2k/fuzz/Cargo.lock b/crates/j2k/fuzz/Cargo.lock index 40610fa9..f63510d1 100644 --- a/crates/j2k/fuzz/Cargo.lock +++ b/crates/j2k/fuzz/Cargo.lock @@ -83,7 +83,7 @@ dependencies = [ [[package]] name = "j2k" -version = "0.7.5" +version = "0.8.0" dependencies = [ "j2k-codec-math", "j2k-core", @@ -94,11 +94,11 @@ dependencies = [ [[package]] name = "j2k-codec-math" -version = "0.7.5" +version = "0.8.0" [[package]] name = "j2k-core" -version = "0.7.5" +version = "0.8.0" dependencies = [ "thiserror", ] @@ -113,7 +113,7 @@ dependencies = [ [[package]] name = "j2k-native" -version = "0.7.5" +version = "0.8.0" dependencies = [ "fearless_simd", "j2k-codec-math", @@ -125,11 +125,11 @@ dependencies = [ [[package]] name = "j2k-profile" -version = "0.7.5" +version = "0.8.0" [[package]] name = "j2k-types" -version = "0.7.5" +version = "0.8.0" [[package]] name = "jobserver" diff --git a/crates/j2k/src/backend.rs b/crates/j2k/src/backend.rs index 684d27f3..aa9c3406 100644 --- a/crates/j2k/src/backend.rs +++ b/crates/j2k/src/backend.rs @@ -15,7 +15,7 @@ pub(crate) fn image( } pub(crate) fn inspect_info(bytes: &[u8]) -> Result { - let image = image(bytes, DecodeSettings::default(), None)?; + let image = image(bytes, DecodeSettings::strict(), None)?; Ok(inspect_info_from_image(&image)) } diff --git a/crates/j2k/src/batch/direct.rs b/crates/j2k/src/batch/direct.rs index db10f091..fdbcc8f2 100644 --- a/crates/j2k/src/batch/direct.rs +++ b/crates/j2k/src/batch/direct.rs @@ -8,8 +8,8 @@ use j2k_native::{ J2kDirectCpuScratch, J2kRect, }; -use crate::decode::{decode_warnings_for_settings, validate_buffer}; -use crate::{DecodeSettings, J2kError, TileRegionScaledDecodeJob}; +use crate::decode::{decode_warnings_for_recovery, validate_buffer}; +use crate::{J2kError, TileRegionScaledDecodeJob}; use super::admission::{BatchAllocationBudget, BatchAllocationClaim}; use super::allocation::GENERIC_WORKER_CLAIM_BYTES; @@ -269,10 +269,7 @@ impl DirectColorRegionCache { } fn success_outcome(decoded: Rect) -> BatchOutcome { - BatchOutcome::new( - decoded, - decode_warnings_for_settings(DecodeSettings::default()), - ) + BatchOutcome::new(decoded, decode_warnings_for_recovery(false)) } fn is_direct_color_u8_format(fmt: PixelFormat) -> bool { diff --git a/crates/j2k/src/decode.rs b/crates/j2k/src/decode.rs index e5209a23..3eed2a51 100644 --- a/crates/j2k/src/decode.rs +++ b/crates/j2k/src/decode.rs @@ -22,9 +22,9 @@ pub use settings::DecodeSettings; #[derive(Debug, Clone, Copy, PartialEq, Eq)] #[non_exhaustive] pub enum J2kDecodeWarning { - /// Decode used lenient settings, so recoverable malformed optional metadata - /// may be tolerated instead of rejected as it would be in strict mode. - LenientDecodeMode, + /// Parsing used one of [`DecodeSettings::lenient`]'s documented JP2/JPH + /// metadata recoveries. + LenientMetadataRecovery, } // A successful batch can produce at most one warning. Keep that owner @@ -35,23 +35,29 @@ const _: [(); 0] = [(); core::mem::size_of::()]; impl fmt::Display for J2kDecodeWarning { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { match self { - Self::LenientDecodeMode => f.write_str("lenient decode mode enabled"), + Self::LenientMetadataRecovery => f.write_str("lenient JP2/JPH metadata recovery used"), } } } pub(crate) type J2kDecodeOutcome = DecodeOutcome; -pub(crate) fn decode_warnings_for_settings(settings: DecodeSettings) -> Vec { +pub(crate) fn decode_warnings_for_recovery( + used_lenient_metadata_recovery: bool, +) -> Vec { let mut warnings = Vec::new(); - if settings.lenient_tolerance_enabled() { + if used_lenient_metadata_recovery { // `J2kDecodeWarning` is statically constrained to a ZST above, so this // push cannot enter the allocator. - warnings.push(J2kDecodeWarning::LenientDecodeMode); + warnings.push(J2kDecodeWarning::LenientMetadataRecovery); } warnings } +pub(crate) fn decode_warnings_for_image(image: &Image<'_>) -> Vec { + decode_warnings_for_recovery(image.used_lenient_metadata_recovery()) +} + pub(crate) fn decode_image_into_with_native_context<'a>( image: &Image<'a>, native_context: &mut j2k_native::DecoderContext<'a>, @@ -161,22 +167,21 @@ pub(crate) fn validate_region(roi: Rect, dims: (u32, u32)) -> Result<(), J2kErro #[cfg(test)] mod tests { - use super::DecodeSettings; - use super::{decode_warnings_for_settings, J2kDecodeWarning}; + use super::{decode_warnings_for_recovery, J2kDecodeWarning}; #[test] - fn decode_warnings_report_lenient_decode_mode() { + fn decode_warnings_only_report_an_actual_lenient_recovery() { + assert!(decode_warnings_for_recovery(false).is_empty()); assert_eq!( - decode_warnings_for_settings(DecodeSettings::default()), - vec![J2kDecodeWarning::LenientDecodeMode] + decode_warnings_for_recovery(true), + vec![J2kDecodeWarning::LenientMetadataRecovery] ); - assert!(decode_warnings_for_settings(DecodeSettings::strict()).is_empty()); } #[test] fn decode_warning_owner_is_statically_allocation_free() { assert_eq!(core::mem::size_of::(), 0); - let warnings = decode_warnings_for_settings(DecodeSettings::default()); + let warnings = decode_warnings_for_recovery(true); assert_eq!(warnings.len(), 1); assert_eq!( warnings diff --git a/crates/j2k/src/decode/settings.rs b/crates/j2k/src/decode/settings.rs index 8041447b..ef35e001 100644 --- a/crates/j2k/src/decode/settings.rs +++ b/crates/j2k/src/decode/settings.rs @@ -14,7 +14,13 @@ pub struct DecodeSettings { } impl DecodeSettings { - /// Compatibility policy that permits recoverable optional-metadata errors. + /// Compatibility policy for explicitly recoverable JP2/JPH metadata. + /// + /// Lenient mode may ignore malformed trailing boxes after required + /// metadata, ignore malformed optional `cdef` or `pclr` boxes, and infer + /// one undeclared alpha component. Raw codestream and entropy validation, + /// bounds and overflow checks, allocation limits, and all resource-safety + /// checks remain strict. #[must_use] pub const fn lenient() -> Self { Self { strict: false } @@ -52,7 +58,7 @@ impl DecodeSettings { impl Default for DecodeSettings { fn default() -> Self { - Self::lenient() + Self::strict() } } @@ -62,7 +68,8 @@ mod tests { #[test] fn facade_policy_preserves_strictness_and_internal_geometry() { - assert!(DecodeSettings::default().lenient_tolerance_enabled()); + assert!(DecodeSettings::default().is_strict()); + assert!(!DecodeSettings::default().lenient_tolerance_enabled()); assert!(!DecodeSettings::lenient().is_strict()); assert!(DecodeSettings::strict().is_strict()); diff --git a/crates/j2k/src/encode.rs b/crates/j2k/src/encode.rs index 428c7fcd..84408718 100644 --- a/crates/j2k/src/encode.rs +++ b/crates/j2k/src/encode.rs @@ -210,8 +210,10 @@ pub fn encode_j2k_lossless_typed_components( /// Encode interleaved samples with an optional device encode-stage accelerator. /// /// Accelerators return CPU fallback by reporting no dispatch. `Auto` accepts -/// that fallback; `RequireDevice` requires at least one dispatch. Any -/// accelerator error or codestream validation error is returned to the caller. +/// that fallback and routes CPU-only sample precisions directly to the native +/// encoder; `RequireDevice` requires every stage needed by the request. Any +/// accelerator execution error or codestream validation error is returned to +/// the caller without a second encode attempt. pub fn encode_j2k_lossless_with_accelerator( samples: J2kLosslessSamples<'_>, options: &J2kLosslessEncodeOptions, @@ -219,14 +221,17 @@ pub fn encode_j2k_lossless_with_accelerator( accelerator: &mut impl J2kEncodeStageAccelerator, ) -> Result { validate_lossless_high_bit_options(samples, options)?; + if options.backend == EncodeBackendPreference::CpuOnly { + return encode_j2k_lossless(samples, options); + } if samples.bit_depth > MAX_RAW_PIXEL_ENCODE_BIT_DEPTH { + if options.backend == EncodeBackendPreference::Auto { + return encode_j2k_lossless(samples, options); + } return Err(J2kError::Unsupported(Unsupported { what: "25-38 bit lossless encode currently uses the CPU classic reversible path only", })); } - if options.backend == EncodeBackendPreference::CpuOnly { - return encode_j2k_lossless(samples, options); - } let before = accelerator.dispatch_report(); let required_stages = required_encode_stages(samples, *options, accelerated_backend); diff --git a/crates/j2k/src/encode/contracts.rs b/crates/j2k/src/encode/contracts.rs index ad1a2787..8b586f59 100644 --- a/crates/j2k/src/encode/contracts.rs +++ b/crates/j2k/src/encode/contracts.rs @@ -51,7 +51,11 @@ macro_rules! define_encoded_j2k { /// Backend preference for JPEG 2000 lossless encoding. #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Default)] pub enum EncodeBackendPreference { - /// Pick the fastest safe backend exposed by the caller, falling back to CPU. + /// Pick the fastest safe backend exposed by the caller. + /// + /// An accelerator may decline an unavailable or unsupported stage and let + /// the CPU complete the request. Accelerator execution errors are returned + /// instead of silently retrying the entire encode on the CPU. #[default] Auto, /// Require the pure Rust CPU encoder. diff --git a/crates/j2k/src/owned_batch.rs b/crates/j2k/src/owned_batch.rs index 5b874a15..ffc9f1e4 100644 --- a/crates/j2k/src/owned_batch.rs +++ b/crates/j2k/src/owned_batch.rs @@ -17,10 +17,10 @@ use crate::{ scheduler::run_retained_chunks, worker::BatchWorker, }, - decode::decode_warnings_for_settings, + decode::decode_warnings_for_recovery, parse::{extract_j2k_codestream_payload, parse_image_info}, - CpuDecodeParallelism, DecodeSettings, DeviceDecodePlan, J2kDecodeWarning, J2kError, - J2kSupportInfo, TileBatchOptions, + CpuDecodeParallelism, DeviceDecodePlan, J2kDecodeWarning, J2kError, J2kSupportInfo, + TileBatchOptions, }; mod contracts; diff --git a/crates/j2k/src/owned_batch/cpu_execute.rs b/crates/j2k/src/owned_batch/cpu_execute.rs index 3acd017b..5dd93b61 100644 --- a/crates/j2k/src/owned_batch/cpu_execute.rs +++ b/crates/j2k/src/owned_batch/cpu_execute.rs @@ -8,7 +8,7 @@ use super::cpu_materialize::{ }; use super::cpu_staged_execute::run_staged_typed_group; use super::{ - decode_warnings_for_settings, run_retained_chunks, Arc, BatchDecodeOptions, BatchErrorStage, + decode_warnings_for_recovery, run_retained_chunks, Arc, BatchDecodeOptions, BatchErrorStage, BatchGroupInfo, BatchInfrastructureError, BatchItemError, BatchWorker, CpuBatchGroup, CpuBatchSamples, CpuDecodeParallelism, CpuGroupFastWorkspace, CpuStagedWorkspace, IndexedBatchError, J2kError, NativeSampleType, NonZeroUsize, PreparedBatchGroup, PreparedImage, @@ -133,7 +133,9 @@ pub(super) fn decode_cpu_group( .collect(); let warnings = successful_slots .iter() - .map(|_| decode_warnings_for_settings(options.settings)) + .map(|&slot| { + decode_warnings_for_recovery(group.images[slot].used_lenient_metadata_recovery()) + }) .collect(); Ok(Some(CpuBatchGroup::new( group.info.clone(), diff --git a/crates/j2k/src/owned_batch/prepare/image.rs b/crates/j2k/src/owned_batch/prepare/image.rs index 2edb9e5e..d99aec33 100644 --- a/crates/j2k/src/owned_batch/prepare/image.rs +++ b/crates/j2k/src/owned_batch/prepare/image.rs @@ -6,8 +6,8 @@ use super::super::{ extract_j2k_codestream_payload, parse_image_info, Arc, BatchAlpha, BatchCodecRoute, BatchColor, BatchDecodeOptions, BatchErrorStage, BatchExecutionShape, BatchGroupInfo, BatchItemError, BatchLayout, BatchWaveletTransform, BatchWorker, Colorspace, CompressedTransferSyntax, - DecodeSettings, DeviceDecodePlan, Downscale, EncodedImage, J2kCodestreamRange, J2kError, - J2kSupportInfo, NativeSampleType, NonRepresentableReason, PreparationDepth, PrepareImageResult, + DeviceDecodePlan, Downscale, EncodedImage, J2kCodestreamRange, J2kError, J2kSupportInfo, + NativeSampleType, NonRepresentableReason, PreparationDepth, PrepareImageResult, PreparedClassicPlan, PreparedCodecPlan, PreparedHtj2kPlan, PreparedImage, PreparedImageInner, }; @@ -37,15 +37,26 @@ pub(super) fn prepare_image( offset: codestream.codestream_offset(), length: codestream.codestream().len(), }; + let target_resolution = (plan.scale() != Downscale::None).then_some(( + plan.source_dims().0.div_ceil(plan.scale().denominator()), + plan.source_dims().1.div_ceil(plan.scale().denominator()), + )); + let native_settings = options.settings.to_native(target_resolution); + let native_image = + j2k_native::Image::new(&input.bytes, &native_settings).map_err(|source| { + BatchItemError::Codec { + stage: BatchErrorStage::Prepare, + source: Arc::new(J2kError::from_native_decode_error(source)), + } + })?; + let used_lenient_metadata_recovery = native_image.used_lenient_metadata_recovery(); let codec_plan = match info.route { BatchCodecRoute::Classic => { - prepare_classic_offset_plan(&input, &info, plan, options.settings, worker)? + prepare_classic_offset_plan(&native_image, &info, plan, worker)? .map_or(PreparedCodecPlan::MetadataOnly, PreparedCodecPlan::Classic) } - BatchCodecRoute::Htj2k => { - prepare_htj2k_offset_plan(&input, &info, plan, options.settings, worker)? - .map_or(PreparedCodecPlan::MetadataOnly, PreparedCodecPlan::Htj2k) - } + BatchCodecRoute::Htj2k => prepare_htj2k_offset_plan(&native_image, &info, plan, worker)? + .map_or(PreparedCodecPlan::MetadataOnly, PreparedCodecPlan::Htj2k), }; reconcile_codec_plan_metadata(&mut info, &codec_plan)?; let preparation_depth = codec_plan.preparation_depth(); @@ -56,6 +67,7 @@ pub(super) fn prepare_image( request: input.request, source_index, decode_settings: options.settings, + used_lenient_metadata_recovery, support: Arc::new(support), plan, codestream_range, @@ -82,10 +94,9 @@ pub(super) fn batch_execution_shape( } fn prepare_classic_offset_plan( - input: &EncodedImage, + image: &j2k_native::Image<'_>, info: &BatchGroupInfo, plan: DeviceDecodePlan, - decode_settings: DecodeSettings, worker: &mut BatchWorker, ) -> Result, BatchItemError> { if info.route != BatchCodecRoute::Classic @@ -96,19 +107,8 @@ fn prepare_classic_offset_plan( { return Ok(None); } - let target_resolution = (plan.scale() != Downscale::None).then_some(( - plan.source_dims().0.div_ceil(plan.scale().denominator()), - plan.source_dims().1.div_ceil(plan.scale().denominator()), - )); - let native_settings = decode_settings.to_native(target_resolution); - let image = j2k_native::Image::new(&input.bytes, &native_settings).map_err(|source| { - BatchItemError::Codec { - stage: BatchErrorStage::Prepare, - source: Arc::new(J2kError::from_native_decode_error(source)), - } - })?; let output = plan.output_rect(); - match worker.build_prepared_classic_plan(&image, (output.x, output.y, output.w, output.h)) { + match worker.build_prepared_classic_plan(image, (output.x, output.y, output.w, output.h)) { Ok(plan) => Ok(Some(PreparedClassicPlan::from_native(plan))), Err(j2k_native::DecodeError::Decoding( j2k_native::DecodingError::DirectPlanUnsupported(_) @@ -122,10 +122,9 @@ fn prepare_classic_offset_plan( } fn prepare_htj2k_offset_plan( - input: &EncodedImage, + image: &j2k_native::Image<'_>, info: &BatchGroupInfo, plan: DeviceDecodePlan, - decode_settings: DecodeSettings, worker: &mut BatchWorker, ) -> Result, BatchItemError> { if info.route != BatchCodecRoute::Htj2k @@ -136,19 +135,8 @@ fn prepare_htj2k_offset_plan( { return Ok(None); } - let target_resolution = (plan.scale() != Downscale::None).then_some(( - plan.source_dims().0.div_ceil(plan.scale().denominator()), - plan.source_dims().1.div_ceil(plan.scale().denominator()), - )); - let native_settings = decode_settings.to_native(target_resolution); - let image = j2k_native::Image::new(&input.bytes, &native_settings).map_err(|source| { - BatchItemError::Codec { - stage: BatchErrorStage::Prepare, - source: Arc::new(J2kError::from_native_decode_error(source)), - } - })?; let output = plan.output_rect(); - match worker.build_prepared_htj2k_plan(&image, (output.x, output.y, output.w, output.h)) { + match worker.build_prepared_htj2k_plan(image, (output.x, output.y, output.w, output.h)) { Ok(plan) => Ok(Some(PreparedHtj2kPlan::from_native(plan))), Err(j2k_native::DecodeError::Decoding( j2k_native::DecodingError::DirectPlanUnsupported(_), diff --git a/crates/j2k/src/owned_batch/prepared.rs b/crates/j2k/src/owned_batch/prepared.rs index 6bfa1cd2..89391885 100644 --- a/crates/j2k/src/owned_batch/prepared.rs +++ b/crates/j2k/src/owned_batch/prepared.rs @@ -40,6 +40,7 @@ pub(super) struct PreparedImageInner { pub(super) request: DecodeRequest, pub(super) source_index: usize, pub(super) decode_settings: DecodeSettings, + pub(super) used_lenient_metadata_recovery: bool, pub(super) support: Arc, pub(super) plan: DeviceDecodePlan, pub(super) codestream_range: J2kCodestreamRange, @@ -71,6 +72,13 @@ impl PreparedImage { self.inner.decode_settings } + /// Whether preparation used an explicitly documented lenient metadata recovery. + #[doc(hidden)] + #[must_use] + pub fn used_lenient_metadata_recovery(&self) -> bool { + self.inner.used_lenient_metadata_recovery + } + /// Parsed codestream and wrapper metadata. #[must_use] pub fn support(&self) -> &J2kSupportInfo { diff --git a/crates/j2k/src/view.rs b/crates/j2k/src/view.rs index d7deb922..db1733c5 100644 --- a/crates/j2k/src/view.rs +++ b/crates/j2k/src/view.rs @@ -4,7 +4,7 @@ use crate::{ backend::{image as backend_image, inspect_info, inspect_info_from_image, Image}, decode::{ decode_image_into_with_native_context, decode_image_region_into_with_native_context, - decode_warnings_for_settings, validate_buffer, validate_region, J2kDecodeOutcome, + decode_warnings_for_image, validate_buffer, validate_region, J2kDecodeOutcome, J2kDecodedComponents, J2kDecodedNativeComponents, }, parse::{parse_image_info, parse_info}, @@ -28,6 +28,7 @@ pub struct J2kView<'a> { info: Info, support_info: Option, image: Option>, + settings: DecodeSettings, passthrough: Option<(CompressedTransferSyntax, CompressedPayloadKind)>, } @@ -38,21 +39,45 @@ impl<'a> J2kView<'a> { /// Returns [`J2kError`] when the input is not a supported JP2/J2C/HTJ2K /// stream or when backend inspection rejects the codestream. pub fn parse(input: &'a [u8]) -> Result { + Self::parse_with_settings(input, DecodeSettings::default()) + } + + /// Parse container/codestream metadata with a per-image validation policy. + /// + /// [`DecodeSettings::default`] is strict. Explicit lenient mode is limited + /// to the JP2/JPH metadata recoveries documented by + /// [`DecodeSettings::lenient`]. + /// + /// # Errors + /// Returns [`J2kError`] when the input is unsupported or malformed outside + /// the selected policy's documented recovery boundary. + pub fn parse_with_settings( + input: &'a [u8], + settings: DecodeSettings, + ) -> Result { + let image = backend_image(input, settings, None)?; let (info, support_info, passthrough) = match parse_image_info(input) { Ok(parsed) => { let support_info = parsed.into_support_info(); let passthrough = Some((support_info.transfer_syntax, support_info.payload_kind)); (support_info.info.clone(), Some(support_info), passthrough) } - Err(error) if should_retry_with_backend(&error) => (inspect_info(input)?, None, None), + Err(error) + if should_fallback_to_backend_after_parse_error( + &error, + image.used_lenient_metadata_recovery(), + ) => + { + (inspect_info_from_image(&image), None, None) + } Err(error) => return Err(error), }; - let image = Some(backend_image(input, DecodeSettings::default(), None)?); Ok(Self { bytes: input, info, support_info, - image, + image: Some(image), + settings, passthrough, }) } @@ -90,6 +115,7 @@ pub struct J2kDecoder<'a> { bytes: &'a [u8], info: Info, image: Option>, + settings: DecodeSettings, native_context: j2k_native::DecoderContext<'a>, } @@ -135,6 +161,18 @@ impl<'a> J2kDecoder<'a> { Self::from_view(J2kView::parse(input)?) } + /// Create a decoder with a per-image validation policy. + /// + /// The policy is retained by this decoder and reused by full-resolution, + /// region, and scaled decode jobs. It does not mutate [`crate::J2kContext`] + /// or leak into another decoder. + /// + /// # Errors + /// Returns [`J2kError`] for unsupported or malformed input. + pub fn new_with_settings(input: &'a [u8], settings: DecodeSettings) -> Result { + Self::from_view(J2kView::parse_with_settings(input, settings)?) + } + /// Create a decoder from a previously parsed [`J2kView`]. /// /// # Errors @@ -144,6 +182,7 @@ impl<'a> J2kDecoder<'a> { bytes: view.bytes, info: view.info, image: view.image, + settings: view.settings, native_context: j2k_native::DecoderContext::default(), }) } @@ -296,7 +335,7 @@ impl<'a> J2kDecoder<'a> { decode_image_into_with_native_context(image, native_context, out, stride, fmt)?; Ok(j2k_core::DecodeOutcome::new( Rect::full(self.info.dimensions), - decode_warnings_for_settings(DecodeSettings::default()), + decode_warnings_for_image(image), )) } @@ -336,7 +375,7 @@ impl<'a> J2kDecoder<'a> { decode_image_region_into_with_native_context(image, native_context, out, stride, fmt, roi)?; Ok(j2k_core::DecodeOutcome::new( roi, - decode_warnings_for_settings(DecodeSettings::default()), + decode_warnings_for_image(image), )) } @@ -359,9 +398,12 @@ impl<'a> J2kDecoder<'a> { if scale == Downscale::None { return self.decode_into_with_scratch(pool, out, stride, fmt); } - let settings = DecodeSettings::default(); - let warnings = decode_warnings_for_settings(settings); - let image = backend_image(self.bytes, settings, Some(self.scaled_target_dims(scale)))?; + let image = backend_image( + self.bytes, + self.settings, + Some(self.scaled_target_dims(scale)), + )?; + let warnings = decode_warnings_for_image(&image); let image_dims = (image.width(), image.height()); validate_buffer(image_dims, out.len(), stride, fmt)?; let mut native_context = self.scaled_decode_native_context(); @@ -396,9 +438,12 @@ impl<'a> J2kDecoder<'a> { validate_region(roi, self.info.dimensions)?; let scaled_roi = roi.scaled_covering(scale); validate_buffer((scaled_roi.w, scaled_roi.h), out.len(), stride, fmt)?; - let settings = DecodeSettings::default(); - let warnings = decode_warnings_for_settings(settings); - let image = backend_image(self.bytes, settings, Some(self.scaled_target_dims(scale)))?; + let image = backend_image( + self.bytes, + self.settings, + Some(self.scaled_target_dims(scale)), + )?; + let warnings = decode_warnings_for_image(&image); let image_dims = (image.width(), image.height()); validate_region(scaled_roi, image_dims)?; let mut native_context = self.scaled_decode_native_context(); @@ -415,7 +460,7 @@ impl<'a> J2kDecoder<'a> { fn ensure_image(&mut self) -> Result<(), J2kError> { if self.image.is_none() { - self.image = Some(backend_image(self.bytes, DecodeSettings::default(), None)?); + self.image = Some(backend_image(self.bytes, self.settings, None)?); if self.info.tile_layout.is_none() { self.info = inspect_info_from_image(self.cached_image()?); } @@ -474,33 +519,12 @@ fn should_retry_with_backend(error: &J2kError) -> bool { ) } -#[cfg(test)] -mod tests { - use super::*; - #[test] - fn scaled_decode_native_context_preserves_configured_parallelism() { - let mut decoder = J2kDecoder { - bytes: &[], - info: Info { - dimensions: (1, 1), - components: 1, - colorspace: j2k_core::Colorspace::SGray, - bit_depth: 8, - tile_layout: None, - coded_unit_layout: None, - restart_interval: None, - resolution_levels: 1, - }, - image: None, - native_context: j2k_native::DecoderContext::default(), - }; - decoder.set_cpu_decode_parallelism(CpuDecodeParallelism::Serial); - - let native_context = decoder.scaled_decode_native_context(); - - assert_eq!( - native_context.cpu_decode_parallelism(), - CpuDecodeParallelism::Serial.to_native() - ); - } +fn should_fallback_to_backend_after_parse_error( + error: &J2kError, + used_lenient_metadata_recovery: bool, +) -> bool { + should_retry_with_backend(error) || used_lenient_metadata_recovery } + +#[cfg(test)] +mod tests; diff --git a/crates/j2k/src/view/tests.rs b/crates/j2k/src/view/tests.rs new file mode 100644 index 00000000..e023347d --- /dev/null +++ b/crates/j2k/src/view/tests.rs @@ -0,0 +1,41 @@ +// SPDX-License-Identifier: MIT OR Apache-2.0 + +use super::*; + +#[test] +fn parse_fallback_requires_an_actual_lenient_recovery() { + let error = J2kError::Unsupported(j2k_core::Unsupported { + what: "test metadata rejection", + }); + + assert!(!should_fallback_to_backend_after_parse_error(&error, false)); + assert!(should_fallback_to_backend_after_parse_error(&error, true)); +} + +#[test] +fn scaled_decode_native_context_preserves_configured_parallelism() { + let mut decoder = J2kDecoder { + bytes: &[], + info: Info { + dimensions: (1, 1), + components: 1, + colorspace: j2k_core::Colorspace::SGray, + bit_depth: 8, + tile_layout: None, + coded_unit_layout: None, + restart_interval: None, + resolution_levels: 1, + }, + image: None, + settings: DecodeSettings::strict(), + native_context: j2k_native::DecoderContext::default(), + }; + decoder.set_cpu_decode_parallelism(CpuDecodeParallelism::Serial); + + let native_context = decoder.scaled_decode_native_context(); + + assert_eq!( + native_context.cpu_decode_parallelism(), + CpuDecodeParallelism::Serial.to_native() + ); +} diff --git a/crates/j2k/tests/decode.rs b/crates/j2k/tests/decode.rs index cbd6e282..84f3040c 100644 --- a/crates/j2k/tests/decode.rs +++ b/crates/j2k/tests/decode.rs @@ -1,10 +1,10 @@ // SPDX-License-Identifier: MIT OR Apache-2.0 use j2k::{ - encode_j2k_lossless_components, EncodeBackendPreference, J2kBlockCodingMode, J2kCodec, - J2kComponentPlane, J2kContext, J2kDecoder, J2kError, J2kLosslessComponentPlane, - J2kLosslessComponentSamples, J2kLosslessEncodeOptions, J2kRowDecodeOptions, - ReversibleTransform, + encode_j2k_lossless_components, DecodeSettings as FacadeDecodeSettings, + EncodeBackendPreference, J2kBlockCodingMode, J2kCodec, J2kComponentPlane, J2kContext, + J2kDecodeWarning, J2kDecoder, J2kError, J2kLosslessComponentPlane, J2kLosslessComponentSamples, + J2kLosslessEncodeOptions, J2kRowDecodeOptions, J2kView, ReversibleTransform, }; use j2k_core::{ BufferError, CodecContext, Downscale, ImageDecodeRows, PixelFormat, Rect, RowSink, @@ -320,6 +320,81 @@ fn decode_gray8_jp2_roundtrips_reversible_pixels() { assert_eq!(out, pixels); } +#[test] +fn explicit_lenient_mode_does_not_warn_for_a_valid_stream() { + let pixels = [3, 9, 27, 81]; + let codestream = encode_codestream(&pixels, 2, 2, 1, 8, true); + let native = Image::new(&codestream, &DecodeSettings::lenient()).expect("native image"); + assert!(!native.used_lenient_metadata_recovery()); + let mut decoder = J2kDecoder::new_with_settings(&codestream, FacadeDecodeSettings::lenient()) + .expect("lenient decoder"); + let mut out = [0_u8; 4]; + let outcome = decoder + .decode_into(&mut out, 2, PixelFormat::Gray8) + .expect("decode"); + + assert_eq!(out, pixels); + assert!(outcome.warnings.is_empty()); +} + +#[test] +fn strict_default_rejects_but_explicit_lenient_recovers_trailing_jp2_metadata() { + let pixels = [3, 9, 27, 81]; + let codestream = encode_codestream(&pixels, 2, 2, 1, 8, true); + let mut jp2 = wrap_jp2_codestream(&codestream, 2, 2, 1, 8, 17); + jp2.extend_from_slice(&[0, 0, 0, 16, b'x', b'm', b'l', b' ']); + + assert!(FacadeDecodeSettings::default().is_strict()); + assert!(J2kView::parse(&jp2).is_err()); + assert!(J2kDecoder::new(&jp2).is_err()); + let native = Image::new(&jp2, &DecodeSettings::lenient()).expect("lenient native image"); + assert!(native.used_lenient_metadata_recovery()); + + let view = + J2kView::parse_with_settings(&jp2, FacadeDecodeSettings::lenient()).expect("lenient view"); + let mut decoder = J2kDecoder::from_view(view).expect("decoder from lenient view"); + let mut out = [0_u8; 4]; + let outcome = decoder + .decode_into(&mut out, 2, PixelFormat::Gray8) + .expect("lenient recovery decode"); + + assert_eq!(out, pixels); + assert_eq!( + outcome.warnings, + vec![J2kDecodeWarning::LenientMetadataRecovery] + ); + + let mut scaled = [0_u8; 1]; + let scaled_outcome = decoder + .decode_scaled_into( + &mut j2k::J2kScratchPool::new(), + &mut scaled, + 1, + PixelFormat::Gray8, + Downscale::Half, + ) + .expect("scaled lenient recovery decode"); + assert_eq!( + scaled_outcome.warnings, + vec![J2kDecodeWarning::LenientMetadataRecovery] + ); +} + +#[test] +fn lenient_container_policy_does_not_relax_truncated_codestream_validation() { + let pixels = [3, 9, 27, 81]; + let mut codestream = encode_codestream(&pixels, 2, 2, 1, 8, true); + assert_eq!(codestream.pop(), Some(0xd9)); + assert_eq!(codestream.pop(), Some(0xff)); + + let mut decoder = J2kDecoder::new_with_settings(&codestream, FacadeDecodeSettings::lenient()) + .expect("main header remains inspectable"); + let mut out = [0_u8; 4]; + assert!(decoder + .decode_into(&mut out, 2, PixelFormat::Gray8) + .is_err()); +} + #[test] fn decode_components_exposes_signed_gray8_public_samples() { let pixels = [-10_i8, -1, 0, 12].map(|sample| sample.to_le_bytes()[0]); diff --git a/crates/j2k/tests/encode_lossless.rs b/crates/j2k/tests/encode_lossless.rs index b38e5fb2..80abd8bb 100644 --- a/crates/j2k/tests/encode_lossless.rs +++ b/crates/j2k/tests/encode_lossless.rs @@ -2165,6 +2165,46 @@ fn accelerator_facade_auto_falls_back_when_no_stage_dispatches() { assert_eq!(decode_native(&encoded.codestream).data, pixels); } +#[test] +fn accelerator_facade_auto_uses_cpu_for_cpu_only_sample_precision() { + #[derive(Default)] + struct NoDispatchAccelerator; + + impl J2kEncodeStageAccelerator for NoDispatchAccelerator {} + + let values = [ + 0_u32, + 1, + 255, + 65_535, + 16_777_216, + (1_u32 << 28) + 17, + (1_u32 << 28) - 1, + 1_u32 << 28, + (1_u32 << 29) - 1, + ]; + let pixels = values + .iter() + .flat_map(|sample| unsigned_31_bytes(*sample)) + .collect::>(); + let samples = J2kLosslessSamples::new(&pixels, 3, 3, 1, 29, false).unwrap(); + let mut accelerator = NoDispatchAccelerator; + + let encoded = encode_j2k_lossless_with_accelerator( + samples, + &auto_options() + .with_block_coding_mode(J2kBlockCodingMode::Classic) + .with_max_decomposition_levels(Some(0)), + BackendKind::Cuda, + &mut accelerator, + ) + .expect("Auto must use the supported CPU path for 29-bit samples"); + + assert_eq!(encoded.backend, BackendKind::Cpu); + assert_eq!(encoded.dispatch_report, J2kEncodeDispatchReport::default()); + assert_eq!(decode_native(&encoded.codestream).data, pixels); +} + #[test] fn accelerator_facade_preserves_native_stage_error() { struct FailingDeinterleaveAccelerator; diff --git a/crates/j2k/tests/owned_batch/reuse.rs b/crates/j2k/tests/owned_batch/reuse.rs index e3899b3c..0dd4d8d4 100644 --- a/crates/j2k/tests/owned_batch/reuse.rs +++ b/crates/j2k/tests/owned_batch/reuse.rs @@ -180,6 +180,28 @@ fn regrouping_prepared_images_reports_settings_mismatches_at_submission_indices( assert_eq!(batch_output.groups()[0].source_indices(), [1]); } +#[test] +fn valid_lenient_batch_does_not_report_a_recovery_warning() { + let options = BatchDecodeOptions { + settings: DecodeSettings::lenient(), + ..BatchDecodeOptions::default() + }; + let prepared = prepare_batch( + vec![EncodedImage::full(Arc::from(htj2k_gray8_fixture(3, 2)))], + options, + ) + .expect("prepare valid lenient image"); + let mut decoder = CpuBatchDecoder::new(options); + + let output = decoder + .decode_prepared(&prepared) + .expect("decode valid lenient image"); + + assert!(output.errors().is_empty()); + assert_eq!(output.groups().len(), 1); + assert_eq!(output.groups()[0].warnings(), &[Vec::new()]); +} + #[test] fn cpu_session_reuses_multitile_classic_prepared_workspace_across_decodes() { let options = BatchDecodeOptions { diff --git a/docs/architecture.md b/docs/architecture.md index b6985939..c591d7d2 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -2,7 +2,16 @@ This document records current workspace boundaries. It is not a roadmap. -The public crate release centers on `j2k`. Runtime backend selection defaults to `Auto`: CPU remains the portable baseline, and explicit CUDA or Metal requests are strict. Native decode settings retain a lenient default for compatibility, but `DecodeSettings::strict()` is the fail-closed constructor and public `j2k` decode outcomes surface `J2kDecodeWarning::LenientDecodeMode` when lenient tolerance is enabled. The living support boundary is maintained in [`docs/public-support.md`](public-support.md). +The public crate release centers on `j2k`. Runtime backend selection defaults to `Auto`: +CPU remains the portable baseline, and explicit CUDA or Metal +requests are strict. Decode settings are strict by default. Explicit lenient +settings are retained per image, never on shared `J2kContext`, and are limited +to the JP2/JPH metadata recoveries documented by `DecodeSettings::lenient`. +Codestream, bounds, overflow, allocation, and resource-limit validation remain +strict in both modes. Decode outcomes surface +`J2kDecodeWarning::LenientMetadataRecovery` only when a recovery actually +occurred. The living support boundary is maintained in +[`docs/public-support.md`](public-support.md). The codec support boundary is JPEG 2000 Part 1 codestreams, JP2 still-image files, HTJ2K Part 15 codestreams, and JPH still-image files. JPX / JPEG 2000 @@ -28,6 +37,7 @@ still-image correctness. Keep row-level status synchronized with | `j2k-transcode` | transcode | JPEG-to-HTJ2K coefficient-domain transcode algorithms and shared contracts. | | `j2k-cli` | CLI | Command-line inspection and JPEG-to-HTJ2K smoke transcode entry point. | | `j2k-test-support`, `j2k-transcode-test-support` | dev helper | Shared fixture, benchmark input, and transcode oracle helpers for tests, benches, and examples. | +| `j2k-alloc-probe` | dev helper | Serial process-wide measurement of successful allocation calls and gross requested bytes at real codec boundaries. | | `j2k-compare` | tooling | Comparator tooling. | | `xtask` | workspace tool | Repository automation under `xtask/`. | @@ -64,7 +74,7 @@ j2k-compare -> j2k-core, j2k, j2k-native, j2k-test-support j2k-transcode -> j2k-codec-math, j2k-core, j2k, j2k-native, j2k-jpeg, j2k-profile j2k-metal-support -> j2k-core j2k-cuda-runtime -> j2k-codec-math, j2k-core -j2k-ml -> j2k, j2k-cuda, j2k-cuda-runtime, j2k-metal, j2k-metal-support +j2k-ml -> j2k, j2k-cuda, j2k-metal, j2k-metal-support j2k-transcode-metal -> j2k-codec-math, j2k-core, j2k-metal, j2k-metal-support, j2k-transcode j2k-transcode-cuda -> j2k-core, j2k-cuda-runtime, j2k-native, j2k-transcode j2k-cli -> j2k, j2k-jpeg, j2k-transcode diff --git a/docs/j2k-ml.md b/docs/j2k-ml.md index 67a028cb..96b69000 100644 --- a/docs/j2k-ml.md +++ b/docs/j2k-ml.md @@ -90,8 +90,9 @@ JP2 channel definitions are part of the grouping key. In particular, straight and premultiplied alpha are never combined into the same RGBA group, and the alpha interpretation remains available in `BatchGroupInfo`. -New batch sessions use strict decoding by default. Lenient decoding is -opt-in and reports warnings. An explicit CUDA or Metal route never substitutes +New batch sessions use strict decoding by default. Lenient decoding is opt-in, +and reports a warning only when one of its documented JP2/JPH metadata +recoveries was actually used. An explicit CUDA or Metal route never substitutes CPU codec decoding, but its completed decoded pixels are deliberately staged through host memory. Unsupported inputs and transfer failures are structured errors. diff --git a/docs/release.md b/docs/release.md index 9630c720..03f1cf59 100644 --- a/docs/release.md +++ b/docs/release.md @@ -1,6 +1,6 @@ # Release Policy -The `j2k` 0.7.3 public crate release is published and security-supported. +The `j2k` 0.7.5 public crate release is published and security-supported. Runtime backend selection defaults to `Auto`; CPU remains the portable baseline while supported device paths are selected only with validation and benchmark evidence. @@ -9,23 +9,51 @@ evidence. | Version | Distribution state | Security support | | --- | --- | --- | -| `0.7.6` | Planned `j2k-ml` adoption release with explicitly staged CUDA/Metal upload adapters; not staged or publishable until clean-consumer and hardware gates pass. | Not a release line. | -| `0.7.5` | Frozen source-incompatible release candidate; not published or tagged until the exact-SHA gates and annotated tag complete. | Not yet a published release line. | -| `0.7.3` | Latest publicly published crates and documentation. | Supported. | +| `0.8.0` | Staged maintenance candidate with strict-by-default decoding, explicit recovery settings, reviewed API tiers, a reusable CUDA lossless-encode façade, and the `j2k-ml` accelerator repair; not publishable until clean-consumer, exact-SHA, benchmark, and hardware gates pass. | Not a release line. | +| `0.7.5` | Published on crates.io from annotated tag `v0.7.5`. No GitHub Release object was created. The published `j2k-ml` CPU feature works; its CUDA and Metal features have the clean-consumer defect described below. | Supported, with the stated `j2k-ml` accelerator exception. | +| `0.7.3` | Previous published release line. | Supported. | | `0.7.2` | Previous published release line. | Supported. | | `0.7.1` | Previous published release line. | Supported. | | `0.7.0` | Previous published release line. | Supported. | | `0.6.x` | Previous published release line. | Supported for security fixes during the 0.7 transition. | | `<0.6` | Historical releases. | Unsupported. | -Version `0.7.3` is published from annotated tag `v0.7.3`, which peels to the -exact locally, hosted-CI, Metal, and CUDA verified release commit. GitHub Pages -is served directly from `main/docs`; this post-release state is documentation, -while the tag and crates.io records remain the publication evidence. +Version `0.7.5` is published from annotated tag `v0.7.5`, which peels to commit +`a89abb6e7eba469c44b3735740712c2a85be0499`. GitHub Pages is served directly +from `main/docs`; this post-release state is documentation, while the tag and +crates.io records remain the publication evidence. -Version `0.7.3` retains the API contract introduced by `0.7.1`, which -intentionally contracts parts of the published pre-1.0 `0.6.2` -API. It does not claim source compatibility with `0.6.x`. The +Version `0.7.5` is an explicitly reviewed source-compatibility exception to +the normal patch policy. Its wrapper-removal migrations are recorded under +the dated `0.7.5` heading in the [`CHANGELOG`](../CHANGELOG.md), and its reviewed +API evidence is compared directly with the published `v0.7.3` baseline. + +The published `j2k-ml 0.7.5` `cuda` and `metal` features do not compile for a +clean registry consumer because they reference CubeCL and wgpu interop methods +that are absent from the selected registry releases. The `cpu` feature is +unaffected. The staged `0.8.0` adapters instead perform accelerator codec +decode, explicit decoded-pixel readback, and ordinary Burn tensor upload using +released public APIs. Do not recommend the accelerator features or send the +Burn community notice until exact-version clean consumers and both hardware +gates pass for that release. + +The staged `0.8.0` candidate is intentionally source- and behavior-incompatible +with `0.7.5`: decode entry points become strict by default, explicit leniency +is limited to the documented JP2/JPH metadata recoveries, warnings report +actual recovery rather than lenient configuration, and +`J2kDecodeWarning::LenientDecodeMode` becomes +`J2kDecodeWarning::LenientMetadataRecovery`. The +[reviewed API report](../engineering/reviewed-public-api-diff-0.8.0.md) records +the generated signature diff, and the adjacent +[review configuration](../engineering/public-api-review-0.8.0.yml) contains the +exact source- and behavior-break ledger with migrations. The semver gate allows +only candidate `0.8.0` to compare against `v0.7.5`. If `0.8.0` is published, +the baseline version, tag, and peeled commit must rotate to that published +artifact before any later `0.8.x` candidate is prepared. + +Version `0.7.3` retained the API contract introduced by `0.7.1`, which +intentionally contracted parts of the published pre-1.0 `0.6.2` API. It does +not claim source compatibility with `0.6.x`. The [`CHANGELOG`](../CHANGELOG.md) provides migration notes, and the [reviewed API report](../engineering/reviewed-public-api-diff-0.7.3.md) records the additions, removals, and changed signatures. That report was @@ -33,13 +61,6 @@ regenerated, independently reviewed, and verified for the published tag. Any report prepared for a future release remains provisional until it is regenerated and verified after that release's final source freeze. -The `0.7.5` candidate is an explicit source-compatibility exception to -the normal patch policy. Its wrapper-removal migrations are recorded under -the dated `0.7.5` heading in the [`CHANGELOG`](../CHANGELOG.md), and its reviewed -API evidence is compared directly with the published `v0.7.3` baseline. This -candidate statement does not authorize publication or assert that the candidate -has passed exact-SHA release gates. - ## Candidate freeze and exact-SHA evidence Finish source, generated artifacts, documentation, changelog, and package @@ -101,8 +122,21 @@ prerequisite. [`release-crates.json`](../release-crates.json) is the ordered release manifest and source of truth for release-integrity, package construction, registry -recovery, and publication. Release scripts must use manifest versions and must -not publish from stale hard-coded crate/version pairs. +recovery, API tiers, documentation coverage, semver scope, and publication. +Schema 2 records only ordered crate names and one of `stable`, `experimental`, +`implementation`, or `binary` as each crate's `api_contract`. Release scripts +must not publish from stale hard-coded crate/version pairs or maintain a second +authored list of registry-independent crates. + +The manifest must contain every crates.io-eligible workspace member exactly +once; a member restricted to another registry is not crates.io eligible. +Library tiers require a library target, while the binary tier requires a binary +target and no library target. Every path dependency between release crates, +including dev-dependencies, must use the exact +`=` requirement and resolve to that workspace +crate—not to a same-named registry or Git dependency. Normal and build +dependencies, including optional and target-specific edges, determine publish +ordering. Dev-dependencies do not. Real publishes must run from tag `v`. All publishable crates must share that workspace version. If a crate version is @@ -152,9 +186,10 @@ cargo package --no-verify cargo publish --dry-run ``` -The gate lists all 19 package contents. It then constructs `.crate` archives -with `cargo package --no-verify` for the 15 staged packages whose workspace -dependencies are not yet available from crates.io. The four +The gate lists all 19 package contents. It derives dependency closure and +registry independence from locked Cargo metadata, then constructs `.crate` +archives with `cargo package --no-verify` for the 15 staged packages whose +workspace dependencies are not yet available from crates.io. The four derived registry-independent packages (`j2k-core`, `j2k-profile`, `j2k-types`, and `j2k-codec-math`) run `cargo publish --dry-run`, including Cargo's package verification build. Manual @@ -207,9 +242,10 @@ The codec-math codegen gate verifies generated Rust and Metal fragments against the Rust source of truth. The integrity gate parses lockfile-strict cargo metadata with `cargo metadata --locked --no-deps`, `release-crates.json`, manifests, `.github/workflows/publish.yml`, and this release document. It fails if a -publishable workspace crate is missing from the dependency-ordered manifest, docs.rs metadata, -semver/doc gates, or release docs, or if a workspace crate is neither -publishable nor explicitly `publish = false`. +crates.io-eligible workspace crate is missing from the dependency-ordered +manifest, docs.rs metadata, published-library semver/doc gates, or release +docs; if an API tier does not match its Cargo targets; if an internal +requirement is not exact; or if dependency order is invalid. The ordinary integrity mode is an offline pre-candidate check and accepts the structured `Unreleased` changelog state. `--publish` remains offline but diff --git a/docs/stable-api-1.0.implementation-public-api.txt b/docs/stable-api-1.0.implementation-public-api.txt index 7c36b76f..5dfcdd52 100644 --- a/docs/stable-api-1.0.implementation-public-api.txt +++ b/docs/stable-api-1.0.implementation-public-api.txt @@ -48,6 +48,7 @@ pub fn j2k::J2kScratchPool::bytes_allocated(&self) -> usize pub fn j2k::J2kScratchPool::reset(&mut self) pub fn j2k::PreparedClassicPlan::adapter_view(&self) -> &(dyn core::any::Any + core::marker::Send + core::marker::Sync) pub fn j2k::PreparedHtj2kPlan::adapter_view(&self) -> &(dyn core::any::Any + core::marker::Send + core::marker::Sync) +pub fn j2k::PreparedImage::used_lenient_metadata_recovery(&self) -> bool pub fn j2k::decode_tile_into_in_context(&[u8], &mut j2k::J2kContext, &mut j2k::J2kScratchPool, j2k::TileDecodeOutput<'_>) -> core::result::Result, j2k::J2kError> pub fn j2k::decode_tile_region_into_in_context(&[u8], &mut j2k::J2kContext, &mut j2k::J2kScratchPool, j2k::TileDecodeOutput<'_>, j2k_core::types::Rect) -> core::result::Result, j2k::J2kError> pub fn j2k::decode_tile_region_scaled_into_in_context(&[u8], &mut j2k::J2kContext, &mut j2k::J2kScratchPool, j2k::TileDecodeOutput<'_>, j2k_core::types::Rect, j2k_core::scale::Downscale) -> core::result::Result, j2k::J2kError> @@ -2203,6 +2204,7 @@ impl j2k_native::J2kRequiredBandRegion impl j2k_native::Reversible53CoefficientImage impl j2k_native::packet_math::HtSegmentLengthError impl<'a> j2k_native::J2kDirectDecodedPlane<'a> +pub const fn j2k_native::Image<'a>::used_lenient_metadata_recovery(&self) -> bool pub const fn j2k_native::J2kDirectCpuScratch::new() -> Self pub const fn j2k_native::J2kDirectDecodedComponents<'_>::component_count(&self) -> usize pub const fn j2k_native::J2kDirectDecodedComponents<'_>::dimensions(&self) -> (u32, u32) diff --git a/docs/stable-api-1.0.md b/docs/stable-api-1.0.md index 74d40b5a..abd894d4 100644 --- a/docs/stable-api-1.0.md +++ b/docs/stable-api-1.0.md @@ -25,7 +25,7 @@ This task must run on macOS with `cargo-public-api` `0.52.0` installed silently change with the runner host or floating nightly channel. The ordinary snapshot uses `RUSTDOCFLAGS=-D warnings` so its comparison with -the published 0.7.3 snapshot keeps the same scope. A second pass adds +the published 0.7.5 snapshot keeps the same scope. A second pass adds `--document-hidden-items` and records only the extra rustdoc-hidden items in the implementation snapshot. Rustdoc can rewrite equivalent re-export paths when hidden modules become visible, so the generator forms a conservative full @@ -34,22 +34,30 @@ sorted difference from the ordinary pass. This guarantees that the combined inventory remains a superset of the ordinary contract while retaining rewritten path variants for review rather than silently dropping reachable API. An empty full cargo-public-api pass fails the gate; an empty per-package hidden-only -difference is recorded truthfully. The 0.7.3 baseline comparison continues to +difference is recorded truthfully. The 0.7.5 baseline comparison continues to use only the ordinary snapshot. Those adapters are implementation-facing, but they are still reachable Rust API and therefore remain in the reviewed inventory. Do not use `#[doc(hidden)]` as a compatibility escape hatch. -The published 0.7.3 artifact recorded both ordinary and hidden-enabled passes -with the same generator, rustdoc, and target pins. The generated 0.7.5 semver -report compares the ordinary candidate inventory with 0.7.3 and also records +The published 0.7.5 artifact recorded both ordinary and hidden-enabled passes +with the same generator, rustdoc, and target pins. The generated 0.8.0 semver +report compares the ordinary candidate inventory with 0.7.5 and also records each package's complete hidden-inventory count and fingerprint. -Every semver invocation collects both live passes, compares both committed -companions, and requires exact ordinary added/removed fingerprints plus the -hidden count/fingerprint in `engineering/public-api-review-0.7.5.yml`. +Every semver invocation collects both live passes, compares both committed companions, and +requires exact ordinary added/removed fingerprints plus the hidden +count/fingerprint in `engineering/public-api-review-0.8.0.yml`. Nonempty hidden inventories also require a package-specific hidden rationale. -Consequently, additions and removals remain blocked until the snapshots, -report, and review evidence are updated together and their diffs are reviewed. + +The review file also contains the reviewed 0.7.5-to-0.8.0 break ledger. +Source-break entries must enumerate every exact removed API item, package, +summary, and migration. Validation requires that inventory to equal the +generated removed-item set: an omitted, duplicate, unknown, or stale item fails +the gate. Behavior-break entries carry the same summary and migration requirements +but contain no removed items, because their observable changes +cannot be inferred from a Rust signature diff. Consequently, additions, +removals, hidden reachability, and declared behavior breaks are reviewed +together rather than treating a pre-1.0 version bump as sufficient evidence. The two snapshot files are staged, synchronized, and committed as one rollback-capable transaction. API generation rejects ambient compiler, @@ -62,37 +70,70 @@ uses Rust `1.96` and does not accept the former `J2K_SEMVER_TOOLCHAIN` override. The snapshots record the published workspace's public items and the CLI exit-code contract expectations. Manual prose in this file must not duplicate that inventory. The staged comparison belongs in the generated -[`0.7.5` reviewed API report](../engineering/reviewed-public-api-diff-0.7.5.md). +[`0.8.0` reviewed API report](../engineering/reviewed-public-api-diff-0.8.0.md). It remains candidate evidence until source freeze and the exact-SHA local, hosted, Metal, and CUDA gates complete; it does not claim a release. -The published stable contract is the `0.7.x` line. Version `0.7.0` -intentionally contracts parts of the pre-1.0 `0.6.2` API and does not claim -source compatibility with `0.6.x`. +The currently published stable contract is the `0.7.x` line. The staged `0.8.0` +candidate intentionally changes the strict-decoding behavior and one warning +variant under Cargo's pre-1.0 compatibility rules. It does not claim source or +behavior compatibility with `0.7.x`; its exact breaks and migrations are in the +review file. Version `0.7.0` similarly contracted parts of the pre-1.0 `0.6.2` +API and did not claim source compatibility with `0.6.x`. ## Stability tiers -- Primary stable user-facing codec APIs: `j2k`, `j2k-core`, `j2k-jpeg`, - and `j2k-tilecodec`. -- Semver-gated published libraries: `j2k`, `j2k-core`, `j2k-jpeg`, - `j2k-tilecodec`, `j2k-jpeg-metal`, `j2k-metal`, - `j2k-jpeg-cuda`, `j2k-cuda`, `j2k-transcode`, `j2k-transcode-cuda`, - `j2k-metal-support`, `j2k-transcode-metal`, `j2k-native`, `j2k-types`, - `j2k-cuda-runtime`, `j2k-profile`, `j2k-codec-math`, and `j2k-ml`. -- Adapter and transcode crates are semver-gated published libraries, but their - supported runtime shapes remain limited by feature gates, hardware - availability, and `docs/public-support.md`. -- `j2k-codec-math` is included in the published `0.7.3` semver baseline. -- Unpublished tooling: test support, comparators, and xtask automation helpers. +`release-crates.json` assigns exactly one `api_contract` to every published +crate. The tier changes the support promise; it does not remove ordinary Rust +visibility or make a public item exempt from patch-release review. + +All 18 published libraries are built with missing-docs enforcement, recorded in +both the ordinary and rustdoc-hidden inventories, and checked for patch +compatibility. A public `#[doc(hidden)]` item remains callable Rust API and is +not private merely because rustdoc omits it. Use actual module or item privacy +for internals whenever possible. + +- `stable`: `j2k`, `j2k-core`, `j2k-jpeg`, and `j2k-tilecodec`. These are the + supported general-purpose consumer APIs. Their documented contracts are the + long-term compatibility promise and should grow conservatively. +- `experimental`: `j2k-native`, `j2k-cuda`, `j2k-metal`, `j2k-jpeg-cuda`, + `j2k-jpeg-metal`, `j2k-transcode`, `j2k-transcode-cuda`, + `j2k-transcode-metal`, and `j2k-ml`. Patch releases preserve their public + APIs. A pre-1.0 minor release may change them only with an explicit reviewed + break ledger and migration guidance. Their runtime support remains limited + by feature gates, hardware availability, and `docs/public-support.md`. +- `implementation`: `j2k-codec-math`, `j2k-profile`, `j2k-types`, + `j2k-cuda-runtime`, and `j2k-metal-support`. These are published sibling-crate + interfaces, not supported general-purpose extension APIs. They still receive + documentation, inventory, and patch-compatibility checks because downstream + Rust code can call them. A pre-1.0 minor release may revise them with the same + reviewed break evidence. +- `binary`: `j2k-cli`. Its command, output, and exit-status behavior is governed + by the CLI contract below rather than a Rust library inventory. + +`j2k-codec-math` and `j2k-ml` are included in the published `0.7.5` semver +baseline. Test support, comparators, and xtask automation helpers remain +unpublished and receive no public API compatibility promise. Patch releases normally preserve the active `0.x` public contract. Version `0.7.5` is an explicit, maintainer-approved source-compatibility exception for removing pass-through public wrappers. Its reviewed API diff must enumerate every contracted item and its changelog must provide migration guidance. This -exception applies only to `0.7.5`; it does not silently relax later patch -releases. Before `1.0`, a minor release may intentionally change the contract -under the same evidence requirements. Starting with `1.0`, stable crates follow -the normal compatibility guarantees for the declared major version. +exception applied only to `0.7.5`. + +The active transition lock is equally narrow: `0.8.0` is the only candidate permitted to compare against `v0.7.5` +as an intentional pre-1.0 break. It +rejects `0.8.1` or any later candidate while that older baseline remains +configured. After `0.8.0` is actually published, rotate the semver baseline to the published `v0.8.0` +tag, version, and peeled commit, then disable the +transition lock before preparing any `0.8.x` follow-up. That makes subsequent +patch-line checks compare with the real 0.8 contract instead of continuing to +receive permission for 0.7-to-0.8 breakage. + +Before `1.0`, a minor release may intentionally change the contract only under +the same generated evidence, explicit break-ledger, and migration requirements. +Starting with `1.0`, stable crates follow the normal compatibility guarantees +for the declared major version. ## CLI contract diff --git a/docs/stable-api-1.0.public-api.txt b/docs/stable-api-1.0.public-api.txt index b8f40af5..5e5c5bd6 100644 --- a/docs/stable-api-1.0.public-api.txt +++ b/docs/stable-api-1.0.public-api.txt @@ -252,6 +252,7 @@ pub fn j2k::J2kDecoder<'a>::info(&self) -> &j2k_core::types::Info pub fn j2k::J2kDecoder<'a>::inspect(&'a [u8]) -> core::result::Result pub fn j2k::J2kDecoder<'a>::inspect_support(&'a [u8]) -> core::result::Result pub fn j2k::J2kDecoder<'a>::new(&'a [u8]) -> core::result::Result +pub fn j2k::J2kDecoder<'a>::new_with_settings(&'a [u8], j2k::DecodeSettings) -> core::result::Result pub fn j2k::J2kDecoder<'a>::set_cpu_decode_parallelism(&mut self, j2k::CpuDecodeParallelism) pub fn j2k::J2kFileBoxMetadata<'a>::from_file_metadata(&'a j2k::J2kFileMetadata) -> Self pub fn j2k::J2kFileColorSpec<'a>::from_file_metadata(&'a j2k::J2kFileMetadata) -> core::option::Option @@ -296,6 +297,7 @@ pub fn j2k::J2kToHtj2kOptions::default() -> Self pub fn j2k::J2kView<'a>::bytes(&self) -> &'a [u8] pub fn j2k::J2kView<'a>::info(&self) -> &j2k_core::types::Info pub fn j2k::J2kView<'a>::parse(&'a [u8]) -> core::result::Result +pub fn j2k::J2kView<'a>::parse_with_settings(&'a [u8], j2k::DecodeSettings) -> core::result::Result pub fn j2k::J2kView<'a>::passthrough_candidate(&self) -> core::option::Option> pub fn j2k::J2kView<'a>::support_info(&self) -> core::option::Option<&j2k::J2kSupportInfo> pub fn j2k::NativeBackendError::fmt(&self, &mut core::fmt::Formatter<'_>) -> core::fmt::Result @@ -473,7 +475,7 @@ pub j2k::J2kComponentMappingType::Palette::column: u8 pub j2k::J2kComponentMappingType::Unknown pub j2k::J2kComponentMappingType::Unknown::column: u8 pub j2k::J2kComponentMappingType::Unknown::value: u8 -pub j2k::J2kDecodeWarning::LenientDecodeMode +pub j2k::J2kDecodeWarning::LenientMetadataRecovery pub j2k::J2kDecodedColorSpace::Cmyk pub j2k::J2kDecodedColorSpace::Gray pub j2k::J2kDecodedColorSpace::Icc @@ -2532,7 +2534,9 @@ pub use j2k_jpeg_cuda::ScratchPool ```text #[non_exhaustive] pub enum j2k_cuda::CudaBatchError +#[non_exhaustive] pub enum j2k_cuda::CudaEncodeFallbackReason #[non_exhaustive] pub enum j2k_cuda::Error +impl core::default::Default for j2k_cuda::CudaLosslessEncoder impl core::error::Error for j2k_cuda::NativeBackendError impl core::fmt::Debug for j2k_cuda::CudaSession impl core::fmt::Debug for j2k_cuda::SubmittedCudaExternalBatch @@ -2549,6 +2553,8 @@ impl j2k_cuda::CudaEncodeStageAccelerator impl j2k_cuda::CudaEncodeStageTimings impl j2k_cuda::CudaEncodedJ2k impl j2k_cuda::CudaExternalBatchGroup +impl j2k_cuda::CudaLosslessEncodeResult +impl j2k_cuda::CudaLosslessEncoder impl j2k_cuda::CudaResidentBatchBuffer impl j2k_cuda::CudaResidentCodestreamBuffer impl j2k_cuda::CudaSession @@ -2568,6 +2574,11 @@ pub const fn j2k_cuda::CudaEncodeStageAccelerator::collected_stage_timings(&self pub const fn j2k_cuda::CudaEncodeStageTimings::saturating_add(self, Self) -> Self pub const fn j2k_cuda::CudaEncodeStageTimings::total_us(self) -> u128 pub const fn j2k_cuda::CudaExternalBatchGroup::info(&self) -> &j2k::owned_batch::contracts::BatchGroupInfo +pub const fn j2k_cuda::CudaLosslessEncodeResult::actual_backend(&self) -> j2k_core::backend::BackendKind +pub const fn j2k_cuda::CudaLosslessEncodeResult::dispatch_report(&self) -> j2k_types::J2kEncodeDispatchReport +pub const fn j2k_cuda::CudaLosslessEncodeResult::encoded(&self) -> &j2k::encode::contracts::EncodedJ2k +pub const fn j2k_cuda::CudaLosslessEncodeResult::fallback_reason(&self) -> core::option::Option +pub const fn j2k_cuda::CudaLosslessEncodeResult::requested_backend(&self) -> j2k::encode::contracts::EncodeBackendPreference pub enum j2k_cuda::CudaExternalBatchTryFinish pub fn j2k_cuda::CudaBatchDecodeResult::errors(&self) -> &[j2k::owned_batch::errors::IndexedBatchError] pub fn j2k_cuda::CudaBatchDecodeResult::group_errors(&self) -> &[j2k_cuda::CudaBatchGroupError] @@ -2611,6 +2622,11 @@ pub fn j2k_cuda::CudaExternalBatchGroup::decoded_rects(&self) -> &[j2k_core::typ pub fn j2k_cuda::CudaExternalBatchGroup::ranges(&self) -> &[j2k_cuda_runtime::memory::CudaDeviceBufferRange] pub fn j2k_cuda::CudaExternalBatchGroup::source_indices(&self) -> &[usize] pub fn j2k_cuda::CudaExternalBatchGroup::warnings(&self) -> &[alloc::vec::Vec] +pub fn j2k_cuda::CudaLosslessEncodeResult::into_encoded(self) -> j2k::encode::contracts::EncodedJ2k +pub fn j2k_cuda::CudaLosslessEncoder::default() -> Self +pub fn j2k_cuda::CudaLosslessEncoder::encode(&mut self, j2k::encode::samples::J2kLosslessSamples<'_>, &j2k::encode::contracts::J2kLosslessEncodeOptions) -> core::result::Result +pub fn j2k_cuda::CudaLosslessEncoder::encode_strict_cuda(&mut self, j2k::encode::samples::J2kLosslessSamples<'_>, &j2k::encode::contracts::J2kLosslessEncodeOptions) -> core::result::Result +pub fn j2k_cuda::CudaLosslessEncoder::new() -> Self pub fn j2k_cuda::CudaResidentBatchBuffer::buffer(&self) -> &j2k_cuda_runtime::memory::CudaDeviceBuffer pub fn j2k_cuda::CudaResidentBatchBuffer::ranges(&self) -> &[j2k_cuda_runtime::memory::CudaDeviceBufferRange] pub fn j2k_cuda::CudaResidentCodestreamBuffer::buffer(&self) -> &j2k_cuda_runtime::memory::CudaDeviceBuffer @@ -2669,6 +2685,8 @@ pub j2k_cuda::CudaDecodePoolSnapshot::deferred_bytes: usize pub j2k_cuda::CudaDecodePoolSnapshot::peak_cached_bytes: usize pub j2k_cuda::CudaDecodePoolSnapshot::peak_deferred_bytes: usize pub j2k_cuda::CudaDecodePoolSnapshot::reuse_holds: usize +pub j2k_cuda::CudaEncodeFallbackReason::DeviceRouteIncomplete +pub j2k_cuda::CudaEncodeFallbackReason::DeviceUnavailable pub j2k_cuda::CudaEncodeStageTimings::deinterleave_us: u128 pub j2k_cuda::CudaEncodeStageTimings::dwt_us: u128 pub j2k_cuda::CudaEncodeStageTimings::ht_encode_us: u128 @@ -2740,7 +2758,9 @@ pub struct j2k_cuda::CudaEncodedJ2k pub struct j2k_cuda::CudaEncodedJ2kMetadata pub struct j2k_cuda::CudaExternalBatchGroup pub struct j2k_cuda::CudaLosslessEncodeResidency +pub struct j2k_cuda::CudaLosslessEncodeResult pub struct j2k_cuda::CudaLosslessEncodeTile<'a> +pub struct j2k_cuda::CudaLosslessEncoder pub struct j2k_cuda::CudaResidentBatchBuffer pub struct j2k_cuda::CudaResidentCodestreamBuffer pub struct j2k_cuda::CudaSession @@ -4400,24 +4420,24 @@ pub mod j2k_profile ## `j2k-ml` ```text -impl core::fmt::Debug for j2k_ml::MetalUploadBurnDecoder -impl core::fmt::Debug for j2k_ml::SubmittedCudaUploadBurnBatch -impl core::fmt::Debug for j2k_ml::SubmittedMetalUploadBurnBatch +impl core::fmt::Debug for j2k_ml::MetalBurnDecoder +impl core::fmt::Debug for j2k_ml::SubmittedCudaBurnBatch +impl core::fmt::Debug for j2k_ml::SubmittedMetalBurnBatch impl j2k_ml::BurnBatchGroupError -impl j2k_ml::CudaUploadBurnDecoder -impl j2k_ml::MetalUploadBurnDecoder -impl j2k_ml::SubmittedCudaUploadBurnBatch -impl j2k_ml::SubmittedMetalUploadBurnBatch +impl j2k_ml::CudaBurnDecoder +impl j2k_ml::MetalBurnDecoder +impl j2k_ml::SubmittedCudaBurnBatch +impl j2k_ml::SubmittedMetalBurnBatch impl j2k_ml::BurnBatchTensor impl j2k_ml::CpuBurnDecoder pub const fn j2k_ml::BurnBatchGroupError::source(&self) -> &j2k_ml::BurnDecodeError pub const fn j2k_ml::BurnBatchTensor::tensor(&self) -> &burn_tensor::tensor::api::base::Tensor pub const fn j2k_ml::CpuBurnDecoder::codec(&self) -> &j2k::owned_batch::cpu::CpuBatchDecoder pub const fn j2k_ml::CpuBurnDecoder::device(&self) -> &::Device -pub const fn j2k_ml::CudaUploadBurnDecoder::codec(&self) -> &j2k_cuda::batch::decoder::CudaBatchDecoder -pub const fn j2k_ml::CudaUploadBurnDecoder::device(&self) -> &cubecl_cuda::device::CudaDevice -pub const fn j2k_ml::MetalUploadBurnDecoder::codec(&self) -> &j2k_metal::batch_decoder::decoder::MetalBatchDecoder -pub const fn j2k_ml::MetalUploadBurnDecoder::device(&self) -> &cubecl_wgpu::device::WgpuDevice +pub const fn j2k_ml::CudaBurnDecoder::codec(&self) -> &j2k_cuda::batch::decoder::CudaBatchDecoder +pub const fn j2k_ml::CudaBurnDecoder::device(&self) -> &cubecl_cuda::device::CudaDevice +pub const fn j2k_ml::MetalBurnDecoder::codec(&self) -> &j2k_metal::batch_decoder::decoder::MetalBatchDecoder +pub const fn j2k_ml::MetalBurnDecoder::device(&self) -> &cubecl_wgpu::device::WgpuDevice pub enum j2k_ml::BurnBatchTensor pub enum j2k_ml::BurnDecodeError pub fn j2k_ml::BurnBatchGroupError::into_parts(self) -> (alloc::vec::Vec, j2k_ml::BurnDecodeError) @@ -4429,31 +4449,31 @@ pub fn j2k_ml::CpuBurnDecoder::decode_prepared_images(&mut self, alloc::vec:: pub fn j2k_ml::CpuBurnDecoder::new(::Device, j2k::owned_batch::contracts::BatchDecodeOptions) -> Self pub fn j2k_ml::CpuBurnDecoder::prepare(&self, alloc::vec::Vec) -> core::result::Result pub fn j2k_ml::CpuBurnDecoder::prepare_prepared_images(&self, alloc::vec::Vec) -> core::result::Result -pub fn j2k_ml::CudaUploadBurnDecoder::decode(&mut self, alloc::vec::Vec) -> core::result::Result, j2k_ml::BurnDecodeError> -pub fn j2k_ml::CudaUploadBurnDecoder::decode_prepared(&mut self, &j2k::owned_batch::prepared::PreparedBatch) -> core::result::Result, j2k_ml::BurnDecodeError> -pub fn j2k_ml::CudaUploadBurnDecoder::decode_prepared_images(&mut self, alloc::vec::Vec) -> core::result::Result, j2k_ml::BurnDecodeError> -pub fn j2k_ml::CudaUploadBurnDecoder::new(cubecl_cuda::device::CudaDevice, j2k::owned_batch::contracts::BatchDecodeOptions) -> Self -pub fn j2k_ml::CudaUploadBurnDecoder::prepare(&self, alloc::vec::Vec) -> core::result::Result -pub fn j2k_ml::CudaUploadBurnDecoder::prepare_prepared_images(&self, alloc::vec::Vec) -> core::result::Result -pub fn j2k_ml::CudaUploadBurnDecoder::submit(&mut self, alloc::vec::Vec) -> core::result::Result -pub fn j2k_ml::CudaUploadBurnDecoder::submit_prepared(&mut self, &j2k::owned_batch::prepared::PreparedBatch) -> core::result::Result -pub fn j2k_ml::MetalUploadBurnDecoder::decode(&mut self, alloc::vec::Vec) -> core::result::Result, j2k_ml::BurnDecodeError> -pub fn j2k_ml::MetalUploadBurnDecoder::decode_prepared(&mut self, &j2k::owned_batch::prepared::PreparedBatch) -> core::result::Result, j2k_ml::BurnDecodeError> -pub fn j2k_ml::MetalUploadBurnDecoder::decode_prepared_images(&mut self, alloc::vec::Vec) -> core::result::Result, j2k_ml::BurnDecodeError> -pub fn j2k_ml::MetalUploadBurnDecoder::fmt(&self, &mut core::fmt::Formatter<'_>) -> core::fmt::Result -pub fn j2k_ml::MetalUploadBurnDecoder::prepare(&self, alloc::vec::Vec) -> core::result::Result -pub fn j2k_ml::MetalUploadBurnDecoder::prepare_prepared_images(&self, alloc::vec::Vec) -> core::result::Result -pub fn j2k_ml::MetalUploadBurnDecoder::submit(&mut self, alloc::vec::Vec) -> core::result::Result -pub fn j2k_ml::MetalUploadBurnDecoder::submit_prepared(&mut self, &j2k::owned_batch::prepared::PreparedBatch) -> core::result::Result -pub fn j2k_ml::MetalUploadBurnDecoder::system_default(j2k::owned_batch::contracts::BatchDecodeOptions) -> core::result::Result -pub fn j2k_ml::SubmittedCudaUploadBurnBatch::fmt(&self, &mut core::fmt::Formatter<'_>) -> core::fmt::Result -pub fn j2k_ml::SubmittedCudaUploadBurnBatch::is_empty(&self) -> bool -pub fn j2k_ml::SubmittedCudaUploadBurnBatch::len(&self) -> usize -pub fn j2k_ml::SubmittedCudaUploadBurnBatch::wait(self) -> core::result::Result, j2k_ml::BurnDecodeError> -pub fn j2k_ml::SubmittedMetalUploadBurnBatch::fmt(&self, &mut core::fmt::Formatter<'_>) -> core::fmt::Result -pub fn j2k_ml::SubmittedMetalUploadBurnBatch::is_empty(&self) -> bool -pub fn j2k_ml::SubmittedMetalUploadBurnBatch::len(&self) -> usize -pub fn j2k_ml::SubmittedMetalUploadBurnBatch::wait(self) -> core::result::Result, j2k_ml::BurnDecodeError> +pub fn j2k_ml::CudaBurnDecoder::decode(&mut self, alloc::vec::Vec) -> core::result::Result, j2k_ml::BurnDecodeError> +pub fn j2k_ml::CudaBurnDecoder::decode_prepared(&mut self, &j2k::owned_batch::prepared::PreparedBatch) -> core::result::Result, j2k_ml::BurnDecodeError> +pub fn j2k_ml::CudaBurnDecoder::decode_prepared_images(&mut self, alloc::vec::Vec) -> core::result::Result, j2k_ml::BurnDecodeError> +pub fn j2k_ml::CudaBurnDecoder::new(cubecl_cuda::device::CudaDevice, j2k::owned_batch::contracts::BatchDecodeOptions) -> Self +pub fn j2k_ml::CudaBurnDecoder::prepare(&self, alloc::vec::Vec) -> core::result::Result +pub fn j2k_ml::CudaBurnDecoder::prepare_prepared_images(&self, alloc::vec::Vec) -> core::result::Result +pub fn j2k_ml::CudaBurnDecoder::submit(&mut self, alloc::vec::Vec) -> core::result::Result +pub fn j2k_ml::CudaBurnDecoder::submit_prepared(&mut self, &j2k::owned_batch::prepared::PreparedBatch) -> core::result::Result +pub fn j2k_ml::MetalBurnDecoder::decode(&mut self, alloc::vec::Vec) -> core::result::Result, j2k_ml::BurnDecodeError> +pub fn j2k_ml::MetalBurnDecoder::decode_prepared(&mut self, &j2k::owned_batch::prepared::PreparedBatch) -> core::result::Result, j2k_ml::BurnDecodeError> +pub fn j2k_ml::MetalBurnDecoder::decode_prepared_images(&mut self, alloc::vec::Vec) -> core::result::Result, j2k_ml::BurnDecodeError> +pub fn j2k_ml::MetalBurnDecoder::fmt(&self, &mut core::fmt::Formatter<'_>) -> core::fmt::Result +pub fn j2k_ml::MetalBurnDecoder::prepare(&self, alloc::vec::Vec) -> core::result::Result +pub fn j2k_ml::MetalBurnDecoder::prepare_prepared_images(&self, alloc::vec::Vec) -> core::result::Result +pub fn j2k_ml::MetalBurnDecoder::submit(&mut self, alloc::vec::Vec) -> core::result::Result +pub fn j2k_ml::MetalBurnDecoder::submit_prepared(&mut self, &j2k::owned_batch::prepared::PreparedBatch) -> core::result::Result +pub fn j2k_ml::MetalBurnDecoder::system_default(j2k::owned_batch::contracts::BatchDecodeOptions) -> core::result::Result +pub fn j2k_ml::SubmittedCudaBurnBatch::fmt(&self, &mut core::fmt::Formatter<'_>) -> core::fmt::Result +pub fn j2k_ml::SubmittedCudaBurnBatch::is_empty(&self) -> bool +pub fn j2k_ml::SubmittedCudaBurnBatch::len(&self) -> usize +pub fn j2k_ml::SubmittedCudaBurnBatch::wait(self) -> core::result::Result, j2k_ml::BurnDecodeError> +pub fn j2k_ml::SubmittedMetalBurnBatch::fmt(&self, &mut core::fmt::Formatter<'_>) -> core::fmt::Result +pub fn j2k_ml::SubmittedMetalBurnBatch::is_empty(&self) -> bool +pub fn j2k_ml::SubmittedMetalBurnBatch::len(&self) -> usize +pub fn j2k_ml::SubmittedMetalBurnBatch::wait(self) -> core::result::Result, j2k_ml::BurnDecodeError> pub j2k_ml::BurnBatchDecode::errors: alloc::vec::Vec pub j2k_ml::BurnBatchDecode::group_errors: alloc::vec::Vec pub j2k_ml::BurnBatchDecode::groups: alloc::vec::Vec> @@ -4469,15 +4489,10 @@ pub j2k_ml::BurnDecodeError::AcceleratorInterop pub j2k_ml::BurnDecodeError::AcceleratorInterop::backend: &'static str pub j2k_ml::BurnDecodeError::AcceleratorInterop::message: alloc::string::String pub j2k_ml::BurnDecodeError::Cuda(j2k_cuda::batch::types::CudaBatchError) -pub j2k_ml::BurnDecodeError::CudaCodec(j2k_cuda::error::Error) -pub j2k_ml::BurnDecodeError::CudaTransfer(j2k_cuda_runtime::error::CudaError) pub j2k_ml::BurnDecodeError::Infrastructure(j2k_core::batch::BatchInfrastructureError) pub j2k_ml::BurnDecodeError::Metal(j2k_metal::error::Error) pub j2k_ml::BurnDecodeError::SampleTypeMismatch pub j2k_ml::BurnDecodeError::SizeOverflow -pub j2k_ml::BurnDecodeError::StagingSizeMismatch -pub j2k_ml::BurnDecodeError::StagingSizeMismatch::actual: usize -pub j2k_ml::BurnDecodeError::StagingSizeMismatch::expected: usize pub j2k_ml::BurnDecodeError::UnsupportedCodecContract pub j2k_ml::BurnDecodeError::UnsupportedDType pub j2k_ml::BurnDecodeError::UnsupportedDType::dtype: burn_std::tensor::dtype::DType @@ -4489,23 +4504,23 @@ pub struct j2k_ml::BurnBatchDecode pub struct j2k_ml::BurnBatchGroup pub struct j2k_ml::BurnBatchGroupError pub struct j2k_ml::CpuBurnDecoder -pub struct j2k_ml::CudaUploadBurnDecoder -pub struct j2k_ml::MetalUploadBurnDecoder -pub struct j2k_ml::SubmittedCudaUploadBurnBatch -pub struct j2k_ml::SubmittedMetalUploadBurnBatch +pub struct j2k_ml::CudaBurnDecoder +pub struct j2k_ml::MetalBurnDecoder +pub struct j2k_ml::SubmittedCudaBurnBatch +pub struct j2k_ml::SubmittedMetalBurnBatch pub struct j2k_ml::cpu::CpuBurnDecoder -pub struct j2k_ml::cuda::CudaUploadBurnDecoder -pub struct j2k_ml::cuda::SubmittedCudaUploadBurnBatch -pub struct j2k_ml::metal::MetalUploadBurnDecoder -pub struct j2k_ml::metal::SubmittedMetalUploadBurnBatch -pub type j2k_ml::CudaBurnDecoder = j2k_ml::CudaUploadBurnDecoder -pub type j2k_ml::MetalBurnDecoder = j2k_ml::MetalUploadBurnDecoder -pub type j2k_ml::SubmittedCudaBurnBatch = j2k_ml::SubmittedCudaUploadBurnBatch -pub type j2k_ml::SubmittedMetalBurnBatch = j2k_ml::SubmittedMetalUploadBurnBatch -pub type j2k_ml::cuda::CudaBurnDecoder = j2k_ml::CudaUploadBurnDecoder -pub type j2k_ml::cuda::SubmittedCudaBurnBatch = j2k_ml::SubmittedCudaUploadBurnBatch -pub type j2k_ml::metal::MetalBurnDecoder = j2k_ml::MetalUploadBurnDecoder -pub type j2k_ml::metal::SubmittedMetalBurnBatch = j2k_ml::SubmittedMetalUploadBurnBatch +pub struct j2k_ml::cuda::CudaBurnDecoder +pub struct j2k_ml::cuda::SubmittedCudaBurnBatch +pub struct j2k_ml::metal::MetalBurnDecoder +pub struct j2k_ml::metal::SubmittedMetalBurnBatch +pub type j2k_ml::CudaUploadBurnDecoder = j2k_ml::CudaBurnDecoder +pub type j2k_ml::MetalUploadBurnDecoder = j2k_ml::MetalBurnDecoder +pub type j2k_ml::SubmittedCudaUploadBurnBatch = j2k_ml::SubmittedCudaBurnBatch +pub type j2k_ml::SubmittedMetalUploadBurnBatch = j2k_ml::SubmittedMetalBurnBatch +pub type j2k_ml::cuda::CudaUploadBurnDecoder = j2k_ml::CudaBurnDecoder +pub type j2k_ml::cuda::SubmittedCudaUploadBurnBatch = j2k_ml::SubmittedCudaBurnBatch +pub type j2k_ml::metal::MetalUploadBurnDecoder = j2k_ml::MetalBurnDecoder +pub type j2k_ml::metal::SubmittedMetalUploadBurnBatch = j2k_ml::SubmittedMetalBurnBatch ``` ## `j2k-cli` diff --git a/docs/unsafe-audit.md b/docs/unsafe-audit.md index 63613258..a6899db3 100644 --- a/docs/unsafe-audit.md +++ b/docs/unsafe-audit.md @@ -9,6 +9,7 @@ moved, or removed. | Path | Scope | Invariants | Regression guards | | --- | --- | --- | --- | +| `crates/j2k-alloc-probe/src/lib.rs` | Dev-only process-global allocation measurement across caller and joined Rayon worker threads. | Every allocator operation delegates unchanged to `System`; atomic bookkeeping never dereferences allocation pointers. Arming and epoch states prevent rejected or delayed operations from resetting or leaking across measurement windows, snapshots wait for admitted reporters, and deallocations never reduce the gross-byte budget. | Allocator self-tests cover allocation, reallocation, panic recovery, concurrent rejection, delayed cross-epoch observation, and a property-generated family of pre-existing-free traces; codec probes exercise warm decode reuse and bounded encode behavior. | | `crates/j2k-core/src/accelerator.rs` | Shared GPU ABI marker trait and byte-view helpers for host/device struct transfers. | Implementers are plain-data GPU ABI values with stable layout and valid byte views. | Core API tests plus backend layout/parity tests for GPU ABI structs. | | `crates/j2k-core/src/backend.rs` | CPU feature detection and backend probing. | CPU intrinsics are called only behind matching architecture/configuration checks. | Cross-architecture CI matrix and backend feature tests. | | `crates/j2k-cuda-runtime/src/bytes/abi.rs` | Unsafe `GpuAbi` marker implementations for CUDA host/device records. | Every marked type is `repr(C)`, accepts every bit pattern, and has a compile-time field-offset walk proving its complete representation contains no padding; explicit reserved-tail fields are initialized and mirrored in device records without changing ABI sizes or offsets. | Safe byte-view tests, ABI size/offset tests, CUDA ABI repository policy, strict cuda-oxide builds, and runtime parity tests. | diff --git a/engineering/ai-codebase-audit-remediation-plan.md b/engineering/ai-codebase-audit-remediation-plan.md index d5eb9aff..f1f53b96 100644 --- a/engineering/ai-codebase-audit-remediation-plan.md +++ b/engineering/ai-codebase-audit-remediation-plan.md @@ -87,8 +87,10 @@ one reopens review but does not predetermine a split. | `xtask/src/perf_guard.rs` | Benchmark snapshot/run/compare workflow | 813 before inline tests | One performance-evidence lifecycle | Snapshot schema v2, another runner, or 900 production lines | | `crates/j2k-test-support/src/jpeg_fixtures/builders.rs` | Explicit JPEG fixture construction | 3,478 test-support lines | Test data is intentionally explicit and excluded from production metrics | Repeated fixture bug fixes diverge or a reusable format owner emerges | -The focused CUDA registry, classic decode, direct-plan, and `j2k-ml` modules -remain covered by structural line and ownership ratchets. +At the historical baseline, focused CUDA registry, classic decode, +direct-plan, and `j2k-ml` modules were covered by structural line and ownership +ratchets. Current repository-lint scope is defined in +[`engineering/repo-lint-contract.md`](repo-lint-contract.md). ## Accepted clone register @@ -155,14 +157,14 @@ pass. | `cargo xtask clippy-strict` | pass | pending — final frozen-source run | | `cargo xtask test` including macOS Metal tests | pass | pending — final frozen-source run | | `cargo xtask doc` | covered by baseline test doctests; standalone pending | pending — final frozen-source run | -| `cargo xtask repo-lint --strict` | pass, 423 checks | pending — final frozen-source run | +| `cargo xtask repo-lint` | pass, 423 checks | pending — final frozen-source run | | `cargo xtask unsafe-audit` | pass | pending — final frozen-source run | | `cargo xtask clone-audit` | pass, 1.96% | pending — final frozen-source run | | test/support clone audit | not yet implemented | pending — final frozen-source run | | `cargo xtask panic-surface` | pass | pending — final frozen-source run | -| `cargo xtask typos` | pass | pending — final frozen-source run | +| `crate-ci/typos` workflow gate | pass | pending — final frozen-source run | | `cargo xtask machete` | pass | pending — final frozen-source run | -| `cargo xtask deny` | pass with reviewed warnings | pending — final frozen-source run | +| `EmbarkStudios/cargo-deny-action` workflow gate | pass with reviewed warnings | pending — final frozen-source run | | `cargo xtask codec-math-codegen` | pass | pending — final frozen-source run | | `cargo xtask stable-api` | pass | pending — generated inventory is stale until final API freeze | | `cargo xtask semver` | pass | pending — final API freeze review | diff --git a/engineering/j2k-ml-adoption-release.md b/engineering/j2k-ml-adoption-release.md index 7576c5c9..894ac87a 100644 --- a/engineering/j2k-ml-adoption-release.md +++ b/engineering/j2k-ml-adoption-release.md @@ -25,6 +25,30 @@ Burn, or any other external project on behalf of this work. Keep all further work inside `frames-sg/j2k` unless the user explicitly authorizes an exact external action. +## Current preparation evidence + +The following checks passed during local macOS preparation on 2026-07-24. +They prove the staged design is buildable and functional, but they are not +release evidence because they were not run from the final clean, versioned +candidate SHA: + +- `cargo xtask j2k-ml-package-smoke`, including clean packaged-source + consumers for `cpu`, `metal`, and `cpu,metal`, packaged examples, and docs; +- `cargo test -p j2k-ml --features cpu`; +- strict `j2k-ml` Clippy and docs for `cpu,metal`; +- `cargo xtask release-metal`; +- repository lint, formatting, unsafe-audit, and command-orchestration tests. + +The compatible lockfile refresh occurred after the package smoke and full +Metal gate. CPU tests, strict Clippy, repository lint, and formatting passed +again afterward. Rerun the package smoke and full Metal gate from the final +clean candidate SHA. + +The CUDA host remained on the published `v0.7.5` release commit during that +preparation. Run the full CUDA gate only after the corrected source is +committed and the same candidate SHA is available on that host. Do not treat a +different checkout or an uncommitted source copy as exact-SHA release evidence. + ## Release completion checklist 1. Run the package smoke with registry-only third-party dependencies: @@ -42,8 +66,8 @@ external action. content-distinct batches 1/8/32/64 for the staged accelerator adapters versus CPU-decode-and-upload. Record uncertainty, memory, and transfer counters without reusing old direct-route claims. -4. Stage the release version only after all clean-consumer and hardware gates - pass from a clean candidate SHA. +4. Keep the staged version under `Unreleased` until clean-consumer and hardware + gates pass; only then freeze a dated changelog and exact candidate SHA. 5. After publication, create fresh consumers pinned to the exact published version and repeat CPU, CUDA, and Metal checks before any community notice. diff --git a/engineering/public-api-review-0.8.0.yml b/engineering/public-api-review-0.8.0.yml new file mode 100644 index 00000000..ea097db3 --- /dev/null +++ b/engineering/public-api-review-0.8.0.yml @@ -0,0 +1,154 @@ +version: 3 +baseline_tag: v0.7.5 +baseline_version: 0.7.5 +candidate_version: "0.8.0" +break_ledger: + - id: strict-decode-default + kind: behavior + packages: + - j2k + - j2k-native + removed_items: [] + summary: "Default decode settings and the one-shot view path are strict, and explicit leniency is limited to the documented JP2/JPH metadata recoveries rather than relaxing raw codestream or entropy validation." + migration: "Use DecodeSettings::lenient() per job, including through J2kView::parse_with_settings or J2kDecoder::new_with_settings, only for the documented JP2/JPH metadata recoveries; malformed raw codestream or entropy data must now be rejected or repaired." + - id: recovery-warning-semantics + kind: behavior + packages: + - j2k + removed_items: [] + summary: "Lenient decoding now emits a warning only when an explicitly permitted metadata recovery was actually used." + migration: "Treat an empty warning list as no recovery having occurred; selecting DecodeSettings::lenient() by itself no longer produces a warning." + - id: rename-lenient-recovery-warning + kind: source + packages: + - j2k + removed_items: + - package: j2k + item: "pub j2k::J2kDecodeWarning::LenientDecodeMode" + summary: "The decode warning variant now describes actual lenient metadata recovery rather than configuration selection." + migration: "Replace matches on J2kDecodeWarning::LenientDecodeMode with J2kDecodeWarning::LenientMetadataRecovery and handle the warning only when recovery occurred." +reviews: + j2k: + removed_fingerprint: "fnv1a64:66f96e29b35dc974" + added_fingerprint: "fnv1a64:87ff4d4ed759e25b" + hidden_count: 104 + hidden_fingerprint: "fnv1a64:4c828a7f3b5307a4" + rationale: "Reviewed the warning-variant rename and the additive J2kView and J2kDecoder settings constructors against v0.7.5. The source break is confined to the exact ledger item, while the new constructors make explicit lenient recovery available without mutable shared context policy." + hidden_rationale: "Reviewed the complete hidden j2k inventory, including the new prepared-image recovery-use query, adapter-view borrows, retained classic and HT plans, and native pixel-format validation." + j2k-core: + removed_fingerprint: "none" + added_fingerprint: "none" + hidden_count: 189 + hidden_fingerprint: "fnv1a64:b87aa26da547e491" + rationale: "Reviewed the ordinary j2k-core inventory against v0.7.5; no public API item was added, removed, or changed." + hidden_rationale: "Reviewed the complete hidden j2k-core inventory, including bounded GPU job planning, pass buckets, source-preserving entries, and typed limit errors; it is unchanged from v0.7.5." + j2k-codec-math: + removed_fingerprint: "none" + added_fingerprint: "none" + hidden_count: 0 + hidden_fingerprint: "none" + rationale: "Reviewed the complete j2k-codec-math inventory against v0.7.5; no ordinary or hidden public API item changed." + j2k-jpeg: + removed_fingerprint: "none" + added_fingerprint: "none" + hidden_count: 550 + hidden_fingerprint: "fnv1a64:d2bd76732f32e066" + rationale: "Reviewed the ordinary j2k-jpeg inventory against v0.7.5; no public API item was added, removed, or changed." + hidden_rationale: "Reviewed the complete hidden j2k-jpeg inventory, including codec context, batch, decode, and typed error boundaries; it is unchanged from v0.7.5." + j2k-tilecodec: + removed_fingerprint: "none" + added_fingerprint: "none" + hidden_count: 32 + hidden_fingerprint: "fnv1a64:eb677842559d9b0e" + rationale: "Reviewed the ordinary j2k-tilecodec inventory against v0.7.5; no public API item was added, removed, or changed." + hidden_rationale: "Reviewed the complete hidden j2k-tilecodec inventory, including codec, scratch, and source-preserving error boundaries; it is unchanged from v0.7.5." + j2k-jpeg-metal: + removed_fingerprint: "none" + added_fingerprint: "none" + hidden_count: 83 + hidden_fingerprint: "fnv1a64:8c83730972447ab6" + rationale: "Reviewed the ordinary j2k-jpeg-metal inventory against v0.7.5; no public API item was added, removed, or changed." + hidden_rationale: "Reviewed the complete hidden JPEG Metal inventory, including resident-image ownership, session lifetime, and typed Metal failures; it is unchanged from v0.7.5." + j2k-metal: + removed_fingerprint: "none" + added_fingerprint: "none" + hidden_count: 299 + hidden_fingerprint: "fnv1a64:a496e2469837d3a8" + rationale: "Reviewed the ordinary j2k-metal inventory against v0.7.5; no public API item was added, removed, or changed." + hidden_rationale: "Reviewed the complete hidden J2K Metal inventory, including submission ownership, exact queue identity, event ordering, session usability, and typed failures; it is unchanged from v0.7.5." + j2k-jpeg-cuda: + removed_fingerprint: "none" + added_fingerprint: "none" + hidden_count: 124 + hidden_fingerprint: "fnv1a64:0b31ccb3e7974bc4" + rationale: "Reviewed the ordinary j2k-jpeg-cuda inventory against v0.7.5; no public API item was added, removed, or changed." + hidden_rationale: "Reviewed the complete hidden JPEG CUDA inventory, including session-bound execution, allocation accounting, and typed runtime failures; it is unchanged from v0.7.5." + j2k-cuda: + removed_fingerprint: "none" + added_fingerprint: "fnv1a64:a160bbc72d2879f1" + hidden_count: 181 + hidden_fingerprint: "fnv1a64:6bb25d5241c4e86a" + rationale: "Reviewed all eighteen additive j2k-cuda items. The concrete CudaLosslessEncoder accepts per-job backend preference, exposes a separate strict-CUDA method, returns an opaque result that distinguishes requested and actual execution plus fallback reason, resets cached accelerator state after failure, and requires exclusive mutable access while remaining movable between threads." + hidden_rationale: "Reviewed the complete hidden J2K CUDA inventory, including completion uncertainty, session usability, device identity, event ordering, pool diagnostics, and guarded interop; it is unchanged from v0.7.5." + j2k-transcode: + removed_fingerprint: "none" + added_fingerprint: "none" + hidden_count: 455 + hidden_fingerprint: "fnv1a64:fdd61f6fdda4a9ab" + rationale: "Reviewed the ordinary j2k-transcode inventory against v0.7.5; no public API item was added, removed, or changed." + hidden_rationale: "Reviewed the complete hidden j2k-transcode inventory, including transforms, packetization, resident handoff, accounting, and source-preserving errors; it is unchanged from v0.7.5." + j2k-transcode-cuda: + removed_fingerprint: "none" + added_fingerprint: "none" + hidden_count: 17 + hidden_fingerprint: "fnv1a64:3e3e69d35c4a62a5" + rationale: "Reviewed the ordinary j2k-transcode-cuda inventory against v0.7.5; no public API item was added, removed, or changed." + hidden_rationale: "Reviewed the complete hidden CUDA transcode inventory, including accelerator routing, resident handoff, allocation caps, and source-preserving runtime errors; it is unchanged from v0.7.5." + j2k-metal-support: + removed_fingerprint: "none" + added_fingerprint: "none" + hidden_count: 11 + hidden_fingerprint: "fnv1a64:671e50d4d121b419" + rationale: "Reviewed the ordinary j2k-metal-support inventory against v0.7.5; no public API item was added, removed, or changed." + hidden_rationale: "Reviewed the complete hidden Metal support inventory, including checked allocation, command, completion, buffer ownership, and audited unsafe boundaries; it is unchanged from v0.7.5." + j2k-transcode-metal: + removed_fingerprint: "none" + added_fingerprint: "none" + hidden_count: 49 + hidden_fingerprint: "fnv1a64:5bf8a22a33753a2d" + rationale: "Reviewed the ordinary j2k-transcode-metal inventory against v0.7.5; no public API item was added, removed, or changed." + hidden_rationale: "Reviewed the complete hidden Metal transcode inventory, including transforms, buffers, code blocks, resident handoff, fallback, and typed source chains; it is unchanged from v0.7.5." + j2k-native: + removed_fingerprint: "none" + added_fingerprint: "none" + hidden_count: 662 + hidden_fingerprint: "fnv1a64:c6883b9e9225ad87" + rationale: "Reviewed the ordinary j2k-native inventory against v0.7.5; no public API item was added, removed, or changed." + hidden_rationale: "Reviewed the complete hidden native inventory, including the new image recovery-use query, retained plans, payload descriptors, staged entropy and IDWT execution, decoded-plane borrows, and allocation metrics." + j2k-types: + removed_fingerprint: "none" + added_fingerprint: "none" + hidden_count: 54 + hidden_fingerprint: "fnv1a64:4b30413a7bfffeb5" + rationale: "Reviewed the ordinary j2k-types inventory against v0.7.5; no public API item was added, removed, or changed." + hidden_rationale: "Reviewed the complete hidden j2k-types inventory, including borrowed codestream payload descriptors and resident-job access; it is unchanged from v0.7.5." + j2k-cuda-runtime: + removed_fingerprint: "none" + added_fingerprint: "none" + hidden_count: 1089 + hidden_fingerprint: "fnv1a64:d33ffdcb13c73dd5" + rationale: "Reviewed the ordinary j2k-cuda-runtime inventory against v0.7.5; no public API item was added, removed, or changed." + hidden_rationale: "Reviewed the complete hidden CUDA runtime inventory, including queued owners, group status, native store ABIs, guarded writes, stream ordering, diagnostics, and drop-safe completion; it is unchanged from v0.7.5." + j2k-profile: + removed_fingerprint: "none" + added_fingerprint: "none" + hidden_count: 117 + hidden_fingerprint: "fnv1a64:a1546f56ae38f2e3" + rationale: "Reviewed the ordinary j2k-profile inventory against v0.7.5; its documented surface remains the crate module root only." + hidden_rationale: "Reviewed the complete hidden profiling inventory, including bounded capture, move-only ownership, fallible controls, and callable profiling surfaces; it is unchanged from v0.7.5." + j2k-ml: + removed_fingerprint: "none" + added_fingerprint: "fnv1a64:fa307c9021bb7e35" + hidden_count: 0 + hidden_fingerprint: "none" + rationale: "Reviewed all eight additive j2k-ml items. CudaBurnDecoder, MetalBurnDecoder, and their submitted-batch types remain the concrete structs with unchanged inherent method paths and unchanged exhaustive error contract. The new CudaUploadBurnDecoder, MetalUploadBurnDecoder, SubmittedCudaUploadBurnBatch, and SubmittedMetalUploadBurnBatch names are aliases at root and module scope. CUDA codec failures retain the published nested typed batch-error chain; staged transfer diagnostics use the published AcceleratorInterop variant. The staged adapters make no direct-destination or zero-copy claim." diff --git a/engineering/repo-lint-contract.md b/engineering/repo-lint-contract.md new file mode 100644 index 00000000..1c02a5da --- /dev/null +++ b/engineering/repo-lint-contract.md @@ -0,0 +1,70 @@ +# Repository lint contract + +`cargo xtask repo-lint` enforces repository properties that Rust compilation, +behavior tests, stable-API capture, or a standard external tool cannot express +directly. It is not a second compiler and does not prescribe where an internal +symbol or test must live. + +## Accepted policy checks + +Repository policy is appropriate for boundaries such as: + +- release-manifest completeness and dependency ordering; +- stable and implementation API inventory agreement; +- unsafe-code inventory and reviewed suppression scope; +- workflow permissions, immutable action references, secret isolation, and + required-gate wiring; +- dependency, corpus-license, generated-source, and public-document contracts; +- accelerator validation that must fail closed when required hardware is + absent. + +Checks should parse the relevant format when a parser exists. Source scanning +is reserved for narrow invariants that do not have a compiler or runtime +representation, and must fail closed on malformed input. + +## Rejected policy checks + +Do not add repository-lint assertions whose only purpose is to freeze: + +- physical file or module ownership; +- source line counts; +- exact test counts or test function names; +- private helper names or textual call shapes; +- formatting already owned by `rustfmt` or lints already owned by Clippy. + +Those checks make ordinary refactoring change two implementations without +proving behavior. File size remains a review signal, not a build failure. +Behavior belongs in the owning crate's unit, integration, property, +differential, or hardware tests. Public exposure belongs in the stable-API +inventory. Cheap type invariants should be compiler checked. + +## Allocation evidence + +Allocation ledgers and `try_reserve` behavior tests remain authoritative for +exact error categories, transactional failure, retained capacity, and +aggregate live-byte formulas. + +The unpublished `j2k-alloc-probe` crate supplements those tests at selected +real codec boundaries. It records successful allocation/reallocation calls +and gross requested bytes. Deallocations are diagnostic only and never credit +a budget. It intentionally does not claim retained bytes or peak live bytes. +Measurements are process-global, serial, and must join all worker work before +returning. + +Textual allocation policies may be retired only after their unique invariant +is covered by compiler enforcement, direct behavior/property tests, or a +measured boundary. Adding the probe does not by itself justify deleting the +remaining allocation policies. + +## Retirement rule + +Before removing a policy check: + +1. identify every property it claims to protect; +2. map each material property to direct replacement evidence; +3. run the owning crate's behavior suite before deletion; +4. run repository lint, formatting, and Clippy after deletion; +5. keep the old check when the replacement cannot run on the required target. + +Historical reports and reviewed API evidence remain provenance records even +when the command surface that produced them is simplified. diff --git a/engineering/reviewed-public-api-diff-0.8.0.md b/engineering/reviewed-public-api-diff-0.8.0.md new file mode 100644 index 00000000..35854dce --- /dev/null +++ b/engineering/reviewed-public-api-diff-0.8.0.md @@ -0,0 +1,285 @@ +# Reviewed public API diff for j2k 0.8.0 + +This report is generated by `cargo xtask semver --write-report`. Normal `cargo xtask semver` regenerates it in memory and fails if this committed file is stale. Every ordinary added/removed fingerprint and every full rustdoc-hidden candidate-inventory fingerprint requires an exact reviewed entry in `engineering/public-api-review-0.8.0.yml`; report regeneration never updates that review config. + +- Baseline registry version: `0.7.5` +- Baseline source snapshot: `v0.7.5` peeled to `a89abb6e7eba469c44b3735740712c2a85be0499` +- Candidate version: `0.8.0` +- Active intentional-break transition: only `0.8.0` may compare against `v0.7.5` +- Required next semver baseline: `v0.8.0` at version `0.8.0` before any later candidate +- Tool pins: Rust `1.96`, `cargo-semver-checks 0.48.0`, `cargo-public-api 0.52.0`, rustdoc `nightly-2026-06-28`, target `aarch64-apple-darwin` + +## Summary + +| Package | Baseline | Candidate | Computed release type | Added | Removed/changed | Removed fingerprint | Added fingerprint | Rustdoc-hidden items | Hidden inventory fingerprint | +| --- | --- | --- | --- | ---: | ---: | --- | --- | ---: | --- | +| `j2k-core` | `0.7.5` | `0.8.0` | `major` | 0 | 0 | `none` | `none` | 189 | `fnv1a64:b87aa26da547e491` | +| `j2k-profile` | `0.7.5` | `0.8.0` | `major` | 0 | 0 | `none` | `none` | 117 | `fnv1a64:a1546f56ae38f2e3` | +| `j2k-types` | `0.7.5` | `0.8.0` | `major` | 0 | 0 | `none` | `none` | 54 | `fnv1a64:4b30413a7bfffeb5` | +| `j2k-codec-math` | `0.7.5` | `0.8.0` | `major` | 0 | 0 | `none` | `none` | 0 | `none` | +| `j2k-cuda-runtime` | `0.7.5` | `0.8.0` | `major` | 0 | 0 | `none` | `none` | 1089 | `fnv1a64:d33ffdcb13c73dd5` | +| `j2k-metal-support` | `0.7.5` | `0.8.0` | `major` | 0 | 0 | `none` | `none` | 11 | `fnv1a64:671e50d4d121b419` | +| `j2k-native` | `0.7.5` | `0.8.0` | `major` | 0 | 0 | `none` | `none` | 662 | `fnv1a64:c6883b9e9225ad87` | +| `j2k-jpeg` | `0.7.5` | `0.8.0` | `major` | 0 | 0 | `none` | `none` | 550 | `fnv1a64:d2bd76732f32e066` | +| `j2k-tilecodec` | `0.7.5` | `0.8.0` | `major` | 0 | 0 | `none` | `none` | 32 | `fnv1a64:eb677842559d9b0e` | +| `j2k` | `0.7.5` | `0.8.0` | `major` | 3 | 1 | `fnv1a64:66f96e29b35dc974` | `fnv1a64:87ff4d4ed759e25b` | 104 | `fnv1a64:4c828a7f3b5307a4` | +| `j2k-transcode` | `0.7.5` | `0.8.0` | `major` | 0 | 0 | `none` | `none` | 455 | `fnv1a64:fdd61f6fdda4a9ab` | +| `j2k-transcode-cuda` | `0.7.5` | `0.8.0` | `major` | 0 | 0 | `none` | `none` | 17 | `fnv1a64:3e3e69d35c4a62a5` | +| `j2k-jpeg-metal` | `0.7.5` | `0.8.0` | `major` | 0 | 0 | `none` | `none` | 83 | `fnv1a64:8c83730972447ab6` | +| `j2k-metal` | `0.7.5` | `0.8.0` | `major` | 0 | 0 | `none` | `none` | 299 | `fnv1a64:a496e2469837d3a8` | +| `j2k-transcode-metal` | `0.7.5` | `0.8.0` | `major` | 0 | 0 | `none` | `none` | 49 | `fnv1a64:5bf8a22a33753a2d` | +| `j2k-jpeg-cuda` | `0.7.5` | `0.8.0` | `major` | 0 | 0 | `none` | `none` | 124 | `fnv1a64:0b31ccb3e7974bc4` | +| `j2k-cuda` | `0.7.5` | `0.8.0` | `major` | 18 | 0 | `none` | `fnv1a64:a160bbc72d2879f1` | 181 | `fnv1a64:6bb25d5241c4e86a` | +| `j2k-ml` | `0.7.5` | `0.8.0` | `major` | 8 | 0 | `none` | `fnv1a64:fa307c9021bb7e35` | 0 | `none` | + +## Published-package details + +### `j2k-core` + +Baseline items: 487. Candidate items: 487. Computed release type: `major`. Rustdoc-hidden candidate items: 189. Full hidden-inventory fingerprint: `fnv1a64:b87aa26da547e491`. + +#### Removed or changed baseline API items + +None. + +#### Added candidate API items + +None. + +### `j2k-profile` + +Baseline items: 1. Candidate items: 1. Computed release type: `major`. Rustdoc-hidden candidate items: 117. Full hidden-inventory fingerprint: `fnv1a64:a1546f56ae38f2e3`. + +#### Removed or changed baseline API items + +None. + +#### Added candidate API items + +None. + +### `j2k-types` + +Baseline items: 365. Candidate items: 365. Computed release type: `major`. Rustdoc-hidden candidate items: 54. Full hidden-inventory fingerprint: `fnv1a64:4b30413a7bfffeb5`. + +#### Removed or changed baseline API items + +None. + +#### Added candidate API items + +None. + +### `j2k-codec-math` + +Baseline items: 100. Candidate items: 100. Computed release type: `major`. Rustdoc-hidden candidate items: 0. Full hidden-inventory fingerprint: `none`. + +#### Removed or changed baseline API items + +None. + +#### Added candidate API items + +None. + +### `j2k-cuda-runtime` + +Baseline items: 100. Candidate items: 100. Computed release type: `major`. Rustdoc-hidden candidate items: 1089. Full hidden-inventory fingerprint: `fnv1a64:d33ffdcb13c73dd5`. + +#### Removed or changed baseline API items + +None. + +#### Added candidate API items + +None. + +### `j2k-metal-support` + +Baseline items: 178. Candidate items: 178. Computed release type: `major`. Rustdoc-hidden candidate items: 11. Full hidden-inventory fingerprint: `fnv1a64:671e50d4d121b419`. + +#### Removed or changed baseline API items + +None. + +#### Added candidate API items + +None. + +### `j2k-native` + +Baseline items: 377. Candidate items: 377. Computed release type: `major`. Rustdoc-hidden candidate items: 662. Full hidden-inventory fingerprint: `fnv1a64:c6883b9e9225ad87`. + +#### Removed or changed baseline API items + +None. + +#### Added candidate API items + +None. + +### `j2k-jpeg` + +Baseline items: 586. Candidate items: 586. Computed release type: `major`. Rustdoc-hidden candidate items: 550. Full hidden-inventory fingerprint: `fnv1a64:d2bd76732f32e066`. + +#### Removed or changed baseline API items + +None. + +#### Added candidate API items + +None. + +### `j2k-tilecodec` + +Baseline items: 31. Candidate items: 31. Computed release type: `major`. Rustdoc-hidden candidate items: 32. Full hidden-inventory fingerprint: `fnv1a64:eb677842559d9b0e`. + +#### Removed or changed baseline API items + +None. + +#### Added candidate API items + +None. + +### `j2k` + +Baseline items: 761. Candidate items: 763. Computed release type: `major`. Rustdoc-hidden candidate items: 104. Full hidden-inventory fingerprint: `fnv1a64:4c828a7f3b5307a4`. + +#### Removed or changed baseline API items + +```text +pub j2k::J2kDecodeWarning::LenientDecodeMode +``` + +#### Added candidate API items + +```text +pub fn j2k::J2kDecoder<'a>::new_with_settings(&'a [u8], j2k::DecodeSettings) -> core::result::Result +pub fn j2k::J2kView<'a>::parse_with_settings(&'a [u8], j2k::DecodeSettings) -> core::result::Result +pub j2k::J2kDecodeWarning::LenientMetadataRecovery +``` + +### `j2k-transcode` + +Baseline items: 426. Candidate items: 426. Computed release type: `major`. Rustdoc-hidden candidate items: 455. Full hidden-inventory fingerprint: `fnv1a64:fdd61f6fdda4a9ab`. + +#### Removed or changed baseline API items + +None. + +#### Added candidate API items + +None. + +### `j2k-transcode-cuda` + +Baseline items: 49. Candidate items: 49. Computed release type: `major`. Rustdoc-hidden candidate items: 17. Full hidden-inventory fingerprint: `fnv1a64:3e3e69d35c4a62a5`. + +#### Removed or changed baseline API items + +None. + +#### Added candidate API items + +None. + +### `j2k-jpeg-metal` + +Baseline items: 170. Candidate items: 170. Computed release type: `major`. Rustdoc-hidden candidate items: 83. Full hidden-inventory fingerprint: `fnv1a64:8c83730972447ab6`. + +#### Removed or changed baseline API items + +None. + +#### Added candidate API items + +None. + +### `j2k-metal` + +Baseline items: 272. Candidate items: 272. Computed release type: `major`. Rustdoc-hidden candidate items: 299. Full hidden-inventory fingerprint: `fnv1a64:a496e2469837d3a8`. + +#### Removed or changed baseline API items + +None. + +#### Added candidate API items + +None. + +### `j2k-transcode-metal` + +Baseline items: 99. Candidate items: 99. Computed release type: `major`. Rustdoc-hidden candidate items: 49. Full hidden-inventory fingerprint: `fnv1a64:5bf8a22a33753a2d`. + +#### Removed or changed baseline API items + +None. + +#### Added candidate API items + +None. + +### `j2k-jpeg-cuda` + +Baseline items: 71. Candidate items: 71. Computed release type: `major`. Rustdoc-hidden candidate items: 124. Full hidden-inventory fingerprint: `fnv1a64:0b31ccb3e7974bc4`. + +#### Removed or changed baseline API items + +None. + +#### Added candidate API items + +None. + +### `j2k-cuda` + +Baseline items: 229. Candidate items: 247. Computed release type: `major`. Rustdoc-hidden candidate items: 181. Full hidden-inventory fingerprint: `fnv1a64:6bb25d5241c4e86a`. + +#### Removed or changed baseline API items + +None. + +#### Added candidate API items + +```text +#[non_exhaustive] pub enum j2k_cuda::CudaEncodeFallbackReason +impl core::default::Default for j2k_cuda::CudaLosslessEncoder +impl j2k_cuda::CudaLosslessEncodeResult +impl j2k_cuda::CudaLosslessEncoder +pub const fn j2k_cuda::CudaLosslessEncodeResult::actual_backend(&self) -> j2k_core::backend::BackendKind +pub const fn j2k_cuda::CudaLosslessEncodeResult::dispatch_report(&self) -> j2k_types::J2kEncodeDispatchReport +pub const fn j2k_cuda::CudaLosslessEncodeResult::encoded(&self) -> &j2k::encode::contracts::EncodedJ2k +pub const fn j2k_cuda::CudaLosslessEncodeResult::fallback_reason(&self) -> core::option::Option +pub const fn j2k_cuda::CudaLosslessEncodeResult::requested_backend(&self) -> j2k::encode::contracts::EncodeBackendPreference +pub fn j2k_cuda::CudaLosslessEncodeResult::into_encoded(self) -> j2k::encode::contracts::EncodedJ2k +pub fn j2k_cuda::CudaLosslessEncoder::default() -> Self +pub fn j2k_cuda::CudaLosslessEncoder::encode(&mut self, j2k::encode::samples::J2kLosslessSamples<'_>, &j2k::encode::contracts::J2kLosslessEncodeOptions) -> core::result::Result +pub fn j2k_cuda::CudaLosslessEncoder::encode_strict_cuda(&mut self, j2k::encode::samples::J2kLosslessSamples<'_>, &j2k::encode::contracts::J2kLosslessEncodeOptions) -> core::result::Result +pub fn j2k_cuda::CudaLosslessEncoder::new() -> Self +pub j2k_cuda::CudaEncodeFallbackReason::DeviceRouteIncomplete +pub j2k_cuda::CudaEncodeFallbackReason::DeviceUnavailable +pub struct j2k_cuda::CudaLosslessEncodeResult +pub struct j2k_cuda::CudaLosslessEncoder +``` + +### `j2k-ml` + +Baseline items: 93. Candidate items: 101. Computed release type: `major`. Rustdoc-hidden candidate items: 0. Full hidden-inventory fingerprint: `none`. + +#### Removed or changed baseline API items + +None. + +#### Added candidate API items + +```text +pub type j2k_ml::CudaUploadBurnDecoder = j2k_ml::CudaBurnDecoder +pub type j2k_ml::MetalUploadBurnDecoder = j2k_ml::MetalBurnDecoder +pub type j2k_ml::SubmittedCudaUploadBurnBatch = j2k_ml::SubmittedCudaBurnBatch +pub type j2k_ml::SubmittedMetalUploadBurnBatch = j2k_ml::SubmittedMetalBurnBatch +pub type j2k_ml::cuda::CudaUploadBurnDecoder = j2k_ml::CudaBurnDecoder +pub type j2k_ml::cuda::SubmittedCudaUploadBurnBatch = j2k_ml::SubmittedCudaBurnBatch +pub type j2k_ml::metal::MetalUploadBurnDecoder = j2k_ml::MetalBurnDecoder +pub type j2k_ml::metal::SubmittedMetalUploadBurnBatch = j2k_ml::SubmittedMetalBurnBatch +``` diff --git a/release-crates.json b/release-crates.json index e2823989..8d705c6e 100644 --- a/release-crates.json +++ b/release-crates.json @@ -1,30 +1,81 @@ { - "schema": 1, - "ordered_crates": [ - "j2k-core", - "j2k-profile", - "j2k-types", - "j2k-codec-math", - "j2k-cuda-runtime", - "j2k-metal-support", - "j2k-native", - "j2k-jpeg", - "j2k-tilecodec", - "j2k", - "j2k-transcode", - "j2k-transcode-cuda", - "j2k-jpeg-metal", - "j2k-metal", - "j2k-transcode-metal", - "j2k-jpeg-cuda", - "j2k-cuda", - "j2k-ml", - "j2k-cli" - ], - "registry_independent": [ - "j2k-core", - "j2k-profile", - "j2k-types", - "j2k-codec-math" + "schema": 2, + "crates": [ + { + "name": "j2k-core", + "api_contract": "stable" + }, + { + "name": "j2k-profile", + "api_contract": "implementation" + }, + { + "name": "j2k-types", + "api_contract": "implementation" + }, + { + "name": "j2k-codec-math", + "api_contract": "implementation" + }, + { + "name": "j2k-cuda-runtime", + "api_contract": "implementation" + }, + { + "name": "j2k-metal-support", + "api_contract": "implementation" + }, + { + "name": "j2k-native", + "api_contract": "experimental" + }, + { + "name": "j2k-jpeg", + "api_contract": "stable" + }, + { + "name": "j2k-tilecodec", + "api_contract": "stable" + }, + { + "name": "j2k", + "api_contract": "stable" + }, + { + "name": "j2k-transcode", + "api_contract": "experimental" + }, + { + "name": "j2k-transcode-cuda", + "api_contract": "experimental" + }, + { + "name": "j2k-jpeg-metal", + "api_contract": "experimental" + }, + { + "name": "j2k-metal", + "api_contract": "experimental" + }, + { + "name": "j2k-transcode-metal", + "api_contract": "experimental" + }, + { + "name": "j2k-jpeg-cuda", + "api_contract": "experimental" + }, + { + "name": "j2k-cuda", + "api_contract": "experimental" + }, + { + "name": "j2k-ml", + "api_contract": "experimental" + }, + { + "name": "j2k-cli", + "api_contract": "binary" + } ] } diff --git a/scripts/publish-crate.sh b/scripts/publish-crate.sh index 43fc55b5..821ac01e 100755 --- a/scripts/publish-crate.sh +++ b/scripts/publish-crate.sh @@ -17,19 +17,11 @@ publishable_output="$( --manifest "$release_manifest" \ --field ordered-crates )" -registry_independent_output="$( - python3 "${script_dir}/publish_release.py" manifest \ - --manifest "$release_manifest" \ - --field registry-independent -)" publishable_crates=() while IFS= read -r crate; do [[ -n "$crate" ]] && publishable_crates+=("$crate") done <<< "$publishable_output" registry_independent_crates=() -while IFS= read -r crate; do - [[ -n "$crate" ]] && registry_independent_crates+=("$crate") -done <<< "$registry_independent_output" workspace_version() { awk ' @@ -378,6 +370,16 @@ crate="$requested" require_publishable_crate "$crate" if [[ "$dry_run" == "true" ]]; then + registry_independent_output="$( + python3 "${script_dir}/publish_release.py" manifest \ + --manifest "$release_manifest" \ + --field registry-independent + )" + while IFS= read -r independent_crate; do + [[ -n "$independent_crate" ]] \ + && registry_independent_crates+=("$independent_crate") + done <<< "$registry_independent_output" + if ! is_registry_independent_crate "$crate"; then echo "${crate}: constructing package without registry verification because its workspace dependencies are staged for publication" cargo package -p "$crate" --no-verify diff --git a/scripts/publish_release.py b/scripts/publish_release.py index bd15d939..7b84d87c 100644 --- a/scripts/publish_release.py +++ b/scripts/publish_release.py @@ -19,6 +19,29 @@ from dataclasses import dataclass from typing import Any, Callable, Mapping, Sequence +if __package__: + from .release_manifest import ( + CRATE_PATTERN, + ManifestError, + ReleaseManifest, + cargo_metadata as _cargo_metadata, + load_release_manifest as _load_release_manifest, + object_value as _object, + string_value as _string, + validate_release_graph as _validate_release_graph, + ) +else: + from release_manifest import ( + CRATE_PATTERN, + ManifestError, + ReleaseManifest, + cargo_metadata as _cargo_metadata, + load_release_manifest as _load_release_manifest, + object_value as _object, + string_value as _string, + validate_release_graph as _validate_release_graph, + ) + ROOT = pathlib.Path(__file__).resolve().parents[1] DEFAULT_MANIFEST = ROOT / "release-crates.json" @@ -26,145 +49,40 @@ MAX_RESPONSE_BYTES = 1_048_576 REQUEST_TIMEOUT_SECONDS = 15 RETRY_DELAYS_SECONDS = (5, 15, 30) -CRATE_PATTERN = re.compile(r"[A-Za-z0-9][A-Za-z0-9_-]*\Z") CHECKSUM_PATTERN = re.compile(r"[0-9a-f]{64}\Z") - -class PublishError(RuntimeError): - """Release state could not be validated or publication could not continue.""" +PublishError = ManifestError class TransientPublishError(PublishError): """A bounded retry may recover a registry or transport operation.""" -@dataclass(frozen=True) -class ReleaseManifest: - ordered_crates: tuple[str, ...] - registry_independent: frozenset[str] - - @dataclass(frozen=True) class RegistryRecord: published: bool checksum: str | None -def _object(value: Any, context: str) -> Mapping[str, Any]: - if not isinstance(value, dict): - raise PublishError(f"{context} must be an object") - return value - - -def _string(value: Any, context: str) -> str: - if not isinstance(value, str) or not value: - raise PublishError(f"{context} must be a non-empty string") - return value - - def load_release_manifest(path: pathlib.Path = DEFAULT_MANIFEST) -> ReleaseManifest: - try: - payload = json.loads(path.read_text(encoding="utf-8")) - except (OSError, UnicodeDecodeError, json.JSONDecodeError) as error: - raise PublishError(f"could not read release manifest {path}: {error}") from None - - root = _object(payload, "release manifest") - if root.get("schema") != 1: - raise PublishError("release manifest schema must be exactly 1") - raw_order = root.get("ordered_crates") - raw_independent = root.get("registry_independent") - if not isinstance(raw_order, list) or not raw_order: - raise PublishError("release manifest ordered_crates must be a non-empty array") - if not isinstance(raw_independent, list): - raise PublishError("release manifest registry_independent must be an array") - - ordered = tuple(_string(value, "release crate") for value in raw_order) - independent = frozenset( - _string(value, "registry-independent crate") for value in raw_independent - ) - if any(not CRATE_PATTERN.fullmatch(crate) for crate in ordered): - raise PublishError("release manifest contains a malformed crate name") - if len(set(ordered)) != len(ordered): - raise PublishError("release manifest contains duplicate crates") - if not independent.issubset(ordered): - raise PublishError("registry-independent crates must be in ordered_crates") - independent_prefix = frozenset(ordered[: len(independent)]) - if independent != independent_prefix: - raise PublishError("registry-independent crates must form the manifest prefix") - return ReleaseManifest(ordered, independent) + return _load_release_manifest(path) def cargo_metadata() -> Mapping[str, Any]: - result = subprocess.run( - ["cargo", "metadata", "--format-version", "1", "--no-deps"], - cwd=ROOT, - check=False, - capture_output=True, - text=True, - ) - if result.returncode != 0: - raise PublishError(f"cargo metadata failed:\n{result.stderr.strip()}") - try: - return _object(json.loads(result.stdout), "cargo metadata") - except (json.JSONDecodeError, UnicodeDecodeError) as error: - raise PublishError(f"cargo metadata returned invalid JSON: {error}") from None + return _cargo_metadata(ROOT) def validate_release_graph( manifest: ReleaseManifest, metadata: Mapping[str, Any] ) -> str: - raw_packages = metadata.get("packages") - if not isinstance(raw_packages, list): - raise PublishError("cargo metadata packages must be an array") + return _validate_release_graph(manifest, metadata).version - packages: dict[str, Mapping[str, Any]] = {} - for raw_package in raw_packages: - package = _object(raw_package, "cargo metadata package") - name = _string(package.get("name"), "cargo metadata package.name") - if name in packages: - raise PublishError(f"cargo metadata contains duplicate package {name}") - packages[name] = package - publishable = { - name for name, package in packages.items() if package.get("publish") != [] - } - ordered_set = set(manifest.ordered_crates) - if publishable != ordered_set: - missing = sorted(publishable - ordered_set) - extra = sorted(ordered_set - publishable) - raise PublishError( - "release manifest must contain all publishable workspace crates exactly once; " - f"missing={missing}, extra={extra}" - ) - - versions = { - _string(packages[name].get("version"), f"{name}.version") - for name in manifest.ordered_crates - } - if len(versions) != 1: - raise PublishError( - "publishable workspace crates must all have the same release version" - ) - - positions = {crate: index for index, crate in enumerate(manifest.ordered_crates)} - for crate in manifest.ordered_crates: - raw_dependencies = packages[crate].get("dependencies") - if not isinstance(raw_dependencies, list): - raise PublishError(f"cargo metadata dependencies for {crate} must be an array") - for raw_dependency in raw_dependencies: - dependency = _object(raw_dependency, f"{crate} dependency") - if dependency.get("kind") == "dev": - continue - dependency_name = _string( - dependency.get("name"), f"{crate} dependency.name" - ) - if dependency_name not in positions: - continue - if positions[dependency_name] >= positions[crate]: - raise PublishError( - f"release crate {crate} appears before dependency {dependency_name}" - ) - return versions.pop() +def registry_independent_crates( + manifest: ReleaseManifest, metadata: Mapping[str, Any] +) -> tuple[str, ...]: + independent = _validate_release_graph(manifest, metadata).registry_independent + return tuple(crate for crate in manifest.ordered_crates if crate in independent) class CratesIoApi: @@ -566,15 +484,10 @@ def main() -> int: if args.field is None: raise PublishError("manifest command requires --field") manifest = load_release_manifest(args.manifest.resolve()) - selected = ( - manifest.ordered_crates - if args.field == "ordered-crates" - else tuple( - crate - for crate in manifest.ordered_crates - if crate in manifest.registry_independent - ) - ) + if args.field == "ordered-crates": + selected = manifest.ordered_crates + else: + selected = registry_independent_crates(manifest, cargo_metadata()) print("\n".join(selected)) return 0 if args.field is not None: diff --git a/scripts/release_manifest.py b/scripts/release_manifest.py new file mode 100644 index 00000000..338a86ef --- /dev/null +++ b/scripts/release_manifest.py @@ -0,0 +1,371 @@ +# SPDX-License-Identifier: MIT OR Apache-2.0 +"""Strict release-manifest parsing and Cargo workspace graph validation.""" + +from __future__ import annotations + +import json +import pathlib +import re +import subprocess +from dataclasses import dataclass +from typing import Any, Mapping + + +CRATE_PATTERN = re.compile(r"[A-Za-z0-9][A-Za-z0-9_-]*\Z") +API_CONTRACTS = frozenset(("stable", "experimental", "implementation", "binary")) + + +class ManifestError(RuntimeError): + """The release manifest or workspace graph is invalid.""" + + +@dataclass(frozen=True) +class ReleaseManifest: + ordered_crates: tuple[str, ...] + api_contracts: Mapping[str, str] + + +@dataclass(frozen=True) +class ValidatedRelease: + version: str + registry_independent: frozenset[str] + + +def object_value(value: Any, context: str) -> Mapping[str, Any]: + if not isinstance(value, dict): + raise ManifestError(f"{context} must be an object") + return value + + +def string_value(value: Any, context: str) -> str: + if not isinstance(value, str) or not value: + raise ManifestError(f"{context} must be a non-empty string") + return value + + +def _exact_fields( + value: Mapping[str, Any], expected: frozenset[str], context: str +) -> None: + actual = frozenset(value) + if actual != expected: + raise ManifestError( + f"{context} has unexpected fields; " + f"expected={sorted(expected)}, actual={sorted(actual)}" + ) + + +def _reject_duplicate_fields(pairs: list[tuple[str, Any]]) -> dict[str, Any]: + value: dict[str, Any] = {} + for key, item in pairs: + if key in value: + raise ManifestError(f"release manifest contains duplicate field {key!r}") + value[key] = item + return value + + +def load_release_manifest(path: pathlib.Path) -> ReleaseManifest: + try: + payload = json.loads( + path.read_text(encoding="utf-8"), + object_pairs_hook=_reject_duplicate_fields, + ) + except (OSError, UnicodeDecodeError, json.JSONDecodeError) as error: + raise ManifestError(f"could not read release manifest {path}: {error}") from None + + root = object_value(payload, "release manifest") + _exact_fields(root, frozenset(("schema", "crates")), "release manifest") + schema = root.get("schema") + if type(schema) is not int or schema != 2: + raise ManifestError("release manifest schema must be exactly 2") + raw_crates = root.get("crates") + if not isinstance(raw_crates, list) or not raw_crates: + raise ManifestError("release manifest crates must be a non-empty array") + + ordered: list[str] = [] + api_contracts: dict[str, str] = {} + for index, raw_crate in enumerate(raw_crates): + crate = object_value(raw_crate, f"release manifest crates[{index}]") + _exact_fields( + crate, + frozenset(("name", "api_contract")), + f"release manifest crates[{index}]", + ) + name = string_value( + crate.get("name"), f"release manifest crates[{index}].name" + ) + contract = string_value( + crate.get("api_contract"), + f"release manifest crates[{index}].api_contract", + ) + if not CRATE_PATTERN.fullmatch(name): + raise ManifestError( + f"release manifest crates[{index}] has malformed crate name {name!r}" + ) + if name in api_contracts: + raise ManifestError(f"release manifest contains duplicate crate {name}") + if contract not in API_CONTRACTS: + raise ManifestError( + f"release manifest api_contract {contract!r} is not one of " + "stable, experimental, implementation, or binary" + ) + ordered.append(name) + api_contracts[name] = contract + + return ReleaseManifest(tuple(ordered), api_contracts) + + +def cargo_metadata(root: pathlib.Path) -> Mapping[str, Any]: + result = subprocess.run( + ["cargo", "metadata", "--locked", "--format-version", "1", "--no-deps"], + cwd=root, + check=False, + capture_output=True, + text=True, + ) + if result.returncode != 0: + raise ManifestError(f"cargo metadata failed:\n{result.stderr.strip()}") + try: + return object_value(json.loads(result.stdout), "cargo metadata") + except (json.JSONDecodeError, UnicodeDecodeError) as error: + raise ManifestError(f"cargo metadata returned invalid JSON: {error}") from None + + +def _workspace_packages(metadata: Mapping[str, Any]) -> dict[str, Mapping[str, Any]]: + raw_packages = metadata.get("packages") + if not isinstance(raw_packages, list): + raise ManifestError("cargo metadata packages must be an array") + + packages_by_id: dict[str, Mapping[str, Any]] = {} + for index, raw_package in enumerate(raw_packages): + package = object_value(raw_package, "cargo metadata package") + package_id = string_value( + package.get("id"), f"cargo metadata packages[{index}].id" + ) + if package_id in packages_by_id: + raise ManifestError( + f"cargo metadata contains duplicate package id {package_id}" + ) + packages_by_id[package_id] = package + + raw_members = metadata.get("workspace_members") + if not isinstance(raw_members, list): + raise ManifestError("cargo metadata workspace_members must be an array") + packages: dict[str, Mapping[str, Any]] = {} + seen_members: set[str] = set() + for index, raw_member in enumerate(raw_members): + member = string_value( + raw_member, f"cargo metadata workspace_members[{index}]" + ) + if member in seen_members: + raise ManifestError( + f"cargo metadata workspace_members contains duplicate id {member}" + ) + seen_members.add(member) + package = packages_by_id.get(member) + if package is None: + raise ManifestError( + f"cargo metadata workspace member {member} has no package record" + ) + name = string_value( + package.get("name"), f"cargo metadata package {member}.name" + ) + if name in packages: + raise ManifestError( + f"cargo metadata contains duplicate workspace package name {name}" + ) + packages[name] = package + return packages + + +def _crates_io_publishable(package: Mapping[str, Any], name: str) -> bool: + publish = package.get("publish") + if publish is None: + return True + if not isinstance(publish, list): + raise ManifestError( + f"cargo metadata package {name} has invalid publish eligibility" + ) + registries = tuple( + string_value(registry, f"cargo metadata package {name}.publish[{index}]") + for index, registry in enumerate(publish) + ) + return "crates-io" in registries + + +def _target_kinds(package: Mapping[str, Any], name: str) -> frozenset[str]: + raw_targets = package.get("targets") + if not isinstance(raw_targets, list): + raise ManifestError(f"cargo metadata targets for {name} must be an array") + kinds: set[str] = set() + for target_index, raw_target in enumerate(raw_targets): + target = object_value( + raw_target, f"cargo metadata {name} target[{target_index}]" + ) + raw_kinds = target.get("kind") + if not isinstance(raw_kinds, list): + raise ManifestError( + f"cargo metadata {name} target[{target_index}].kind must be an array" + ) + for kind_index, kind in enumerate(raw_kinds): + kinds.add( + string_value( + kind, + f"cargo metadata {name} " + f"target[{target_index}].kind[{kind_index}]", + ) + ) + return frozenset(kinds) + + +def _dependency_kind( + dependency: Mapping[str, Any], package: str, index: int +) -> str: + kind = dependency.get("kind") + if kind is None: + return "normal" + if not isinstance(kind, str): + raise ManifestError( + f"cargo metadata package {package} dependency[{index}] has invalid kind" + ) + return kind + + +def _workspace_release_dependency( + dependency: Mapping[str, Any], + package: str, + index: int, + release_roots: Mapping[str, pathlib.Path], +) -> str | None: + dependency_name = string_value( + dependency.get("name"), + f"cargo metadata package {package} dependency[{index}].name", + ) + expected_root = release_roots.get(dependency_name) + if expected_root is None: + return None + source = dependency.get("source") + if source is not None: + string_value( + source, + f"cargo metadata package {package} dependency[{index}].source", + ) + raise ManifestError( + f"release package {package} dependency {dependency_name} " + "must be workspace/path sourced" + ) + raw_path = dependency.get("path") + if raw_path is None: + raise ManifestError( + f"release package {package} dependency {dependency_name} " + "must be workspace/path sourced" + ) + dependency_root = pathlib.Path( + string_value( + raw_path, + f"cargo metadata package {package} dependency[{index}].path", + ) + ) + if dependency_root != expected_root: + raise ManifestError( + f"release package {package} dependency {dependency_name} path " + f"{dependency_root} does not match workspace package root {expected_root}" + ) + return dependency_name + + +def validate_release_graph( + manifest: ReleaseManifest, metadata: Mapping[str, Any] +) -> ValidatedRelease: + packages = _workspace_packages(metadata) + publishable = { + name + for name, package in packages.items() + if _crates_io_publishable(package, name) + } + ordered_set = set(manifest.ordered_crates) + if publishable != ordered_set: + missing = sorted(publishable - ordered_set) + extra = sorted(ordered_set - publishable) + raise ManifestError( + "release manifest must contain all publishable workspace crates exactly once; " + f"missing={missing}, extra={extra}" + ) + + versions_by_package = { + name: string_value(packages[name].get("version"), f"{name}.version") + for name in manifest.ordered_crates + } + versions = set(versions_by_package.values()) + if len(versions) != 1: + raise ManifestError( + "publishable workspace crates must all have the same release version" + ) + + release_roots: dict[str, pathlib.Path] = {} + for name in manifest.ordered_crates: + manifest_path = pathlib.Path( + string_value(packages[name].get("manifest_path"), f"{name}.manifest_path") + ) + release_roots[name] = manifest_path.parent + contract = manifest.api_contracts[name] + kinds = _target_kinds(packages[name], name) + has_library = "lib" in kinds + has_binary = "bin" in kinds + if contract == "binary": + if not has_binary or has_library: + raise ManifestError( + f"release package {name} has api_contract=binary but must have " + "a binary target and no library target" + ) + elif not has_library: + raise ManifestError( + f"release package {name} has api_contract={contract} " + "but has no library target" + ) + + positions = {crate: index for index, crate in enumerate(manifest.ordered_crates)} + dependencies_by_package: dict[str, set[str]] = {} + for crate in manifest.ordered_crates: + raw_dependencies = packages[crate].get("dependencies") + if not isinstance(raw_dependencies, list): + raise ManifestError( + f"cargo metadata dependencies for {crate} must be an array" + ) + release_dependencies: set[str] = set() + for dependency_index, raw_dependency in enumerate(raw_dependencies): + dependency = object_value(raw_dependency, f"{crate} dependency") + kind = _dependency_kind(dependency, crate, dependency_index) + dependency_name = _workspace_release_dependency( + dependency, + crate, + dependency_index, + release_roots, + ) + if dependency_name is None: + continue + requirement = string_value( + dependency.get("req"), + f"cargo metadata package {crate} " + f"dependency {dependency_name}.req", + ) + expected_requirement = f"={versions_by_package[dependency_name]}" + if requirement != expected_requirement: + raise ManifestError( + f"release package {crate} dependency {dependency_name} must use " + f"exact release requirement {expected_requirement}; " + f"found {requirement}" + ) + if kind == "dev": + continue + release_dependencies.add(dependency_name) + if positions[dependency_name] >= positions[crate]: + raise ManifestError( + f"release crate {crate} appears before dependency {dependency_name}" + ) + dependencies_by_package[crate] = release_dependencies + independent = frozenset( + crate + for crate in manifest.ordered_crates + if not dependencies_by_package[crate] + ) + return ValidatedRelease(versions.pop(), independent) diff --git a/scripts/tests/test_publish_release.py b/scripts/tests/test_publish_release.py index b7201f1f..5262f664 100644 --- a/scripts/tests/test_publish_release.py +++ b/scripts/tests/test_publish_release.py @@ -2,6 +2,7 @@ from __future__ import annotations +import json import subprocess import tempfile import unittest @@ -12,15 +13,39 @@ from scripts import publish_release -def package(name: str, dependencies: list[tuple[str, str | None]], *, publish: object = None) -> dict[str, object]: +def release_manifest(*crates: str) -> publish_release.ReleaseManifest: + return publish_release.ReleaseManifest( + ordered_crates=crates, + api_contracts={crate: "stable" for crate in crates}, + ) + + +def package( + name: str, + dependencies: list[tuple[str, str | None]], + *, + publish: object = None, + target_kinds: tuple[str, ...] = ("lib",), +) -> dict[str, object]: return { + "id": name, "name": name, "version": "0.7.3", "publish": publish, + "manifest_path": f"/workspace/{name}/Cargo.toml", "dependencies": [ - {"name": dependency, "kind": kind, "target": None} + { + "name": dependency, + "kind": kind, + "target": None, + "optional": False, + "path": f"/workspace/{dependency}", + "req": "=0.7.3", + "source": None, + } for dependency, kind in dependencies ], + "targets": [{"kind": list(target_kinds)}], } @@ -35,12 +60,113 @@ def version_record(self, crate: str, version: str) -> publish_release.RegistryRe class ManifestTests(unittest.TestCase): - def test_metadata_validation_requires_complete_dependency_ordered_publish_set(self) -> None: - manifest = publish_release.ReleaseManifest( - ordered_crates=("base", "consumer"), - registry_independent=frozenset({"base"}), + def test_schema_two_manifest_records_order_and_api_contracts(self) -> None: + payload = { + "schema": 2, + "crates": [ + {"name": "stable-api", "api_contract": "stable"}, + { + "name": "experimental-api", + "api_contract": "experimental", + }, + { + "name": "implementation-spi", + "api_contract": "implementation", + }, + {"name": "cli", "api_contract": "binary"}, + ], + } + with tempfile.TemporaryDirectory() as temporary_directory: + manifest_path = Path(temporary_directory) / "release-crates.json" + manifest_path.write_text(json.dumps(payload), encoding="utf-8") + manifest = publish_release.load_release_manifest(manifest_path) + + self.assertEqual( + manifest.ordered_crates, + ("stable-api", "experimental-api", "implementation-spi", "cli"), + ) + self.assertEqual( + manifest.api_contracts, + { + "stable-api": "stable", + "experimental-api": "experimental", + "implementation-spi": "implementation", + "cli": "binary", + }, + ) + + invalid_payloads = ( + ({"schema": 1, "crates": payload["crates"]}, "schema must be exactly 2"), + ({"schema": 2.0, "crates": payload["crates"]}, "schema must be exactly 2"), + ({"schema": True, "crates": payload["crates"]}, "schema must be exactly 2"), + ( + {"schema": 2, "crates": payload["crates"], "extra": True}, + "unexpected fields", + ), + ({"schema": 2, "crates": []}, "non-empty array"), + ( + { + "schema": 2, + "crates": [ + {"name": "duplicate", "api_contract": "stable"}, + {"name": "duplicate", "api_contract": "experimental"}, + ], + }, + "duplicate", + ), + ( + { + "schema": 2, + "crates": [ + {"name": "crate", "api_contract": "unsupported"}, + ], + }, + "api_contract", + ), + ( + { + "schema": 2, + "crates": [ + { + "name": "crate", + "api_contract": "stable", + "extra": True, + }, + ], + }, + "unexpected fields", + ), ) + for index, (invalid, expected) in enumerate(invalid_payloads): + with self.subTest(expected=expected): + with tempfile.TemporaryDirectory() as temporary_directory: + manifest_path = ( + Path(temporary_directory) / f"invalid-{index}.json" + ) + manifest_path.write_text( + json.dumps(invalid), encoding="utf-8" + ) + with self.assertRaisesRegex( + publish_release.PublishError, expected + ): + publish_release.load_release_manifest(manifest_path) + + with tempfile.TemporaryDirectory() as temporary_directory: + manifest_path = Path(temporary_directory) / "duplicate-field.json" + manifest_path.write_text( + '{"schema":2,"schema":2,"crates":' + '[{"name":"crate","api_contract":"stable"}]}', + encoding="utf-8", + ) + with self.assertRaisesRegex( + publish_release.PublishError, "duplicate field" + ): + publish_release.load_release_manifest(manifest_path) + + def test_metadata_validation_requires_complete_dependency_ordered_publish_set(self) -> None: + manifest = release_manifest("base", "consumer") metadata = { + "workspace_members": ["base", "consumer", "private-tests"], "packages": [ package("base", []), package("consumer", [("base", None), ("private-tests", "dev")]), @@ -52,26 +178,86 @@ def test_metadata_validation_requires_complete_dependency_ordered_publish_set(se publish_release.validate_release_graph(manifest, metadata), "0.7.3" ) - reversed_manifest = publish_release.ReleaseManifest( - ordered_crates=("consumer", "base"), - registry_independent=frozenset({"base"}), - ) + reversed_manifest = release_manifest("consumer", "base") with self.assertRaisesRegex(publish_release.PublishError, "before dependency"): publish_release.validate_release_graph(reversed_manifest, metadata) - incomplete = publish_release.ReleaseManifest( - ordered_crates=("base",), registry_independent=frozenset({"base"}) - ) + incomplete = release_manifest("base") with self.assertRaisesRegex(publish_release.PublishError, "publishable workspace crates"): publish_release.validate_release_graph(incomplete, metadata) + self.assertEqual( + publish_release.registry_independent_crates(manifest, metadata), + ("base",), + ) + + def test_metadata_validation_enforces_tiers_sources_and_exact_requirements( + self, + ) -> None: + manifest = release_manifest("base", "consumer") + metadata = { + "workspace_members": ["base", "consumer"], + "packages": [ + package("base", []), + package("consumer", [("base", "build")]), + ], + } + + with self.assertRaisesRegex(publish_release.PublishError, "before dependency"): + publish_release.validate_release_graph( + release_manifest("consumer", "base"), metadata + ) + + metadata["packages"][1]["dependencies"][0]["kind"] = "dev" + publish_release.validate_release_graph( + release_manifest("consumer", "base"), metadata + ) + + metadata["packages"][1]["dependencies"][0]["kind"] = None + metadata["packages"][1]["dependencies"][0]["req"] = "^0.7.3" + with self.assertRaisesRegex(publish_release.PublishError, "exact release"): + publish_release.validate_release_graph(manifest, metadata) + + metadata["packages"][1]["dependencies"][0]["req"] = "=0.7.3" + metadata["packages"][1]["dependencies"][0]["source"] = ( + "registry+https://github.com/rust-lang/crates.io-index" + ) + metadata["packages"][1]["dependencies"][0]["path"] = None + with self.assertRaisesRegex( + publish_release.PublishError, "workspace/path sourced" + ): + publish_release.validate_release_graph(manifest, metadata) + + metadata["packages"][1]["dependencies"][0]["source"] = None + metadata["packages"][1]["dependencies"][0]["path"] = "/workspace/base" + binary_manifest = publish_release.ReleaseManifest( + ordered_crates=manifest.ordered_crates, + api_contracts={"base": "stable", "consumer": "binary"}, + ) + with self.assertRaisesRegex(publish_release.PublishError, "binary target"): + publish_release.validate_release_graph(binary_manifest, metadata) + + def test_metadata_validation_requires_crates_io_publication_eligibility( + self, + ) -> None: + manifest = release_manifest("base") + metadata = { + "workspace_members": ["base"], + "packages": [package("base", [], publish=["private-registry"])], + } + + with self.assertRaisesRegex( + publish_release.PublishError, "publishable workspace crates" + ): + publish_release.validate_release_graph(manifest, metadata) + + metadata["packages"][0]["publish"] = ["crates-io"] + publish_release.validate_release_graph(manifest, metadata) + class PackagingTests(unittest.TestCase): def test_workspace_packages_resolve_unpublished_exact_dependencies_locally(self) -> None: - manifest = publish_release.ReleaseManifest( - ordered_crates=("base", "consumer"), - registry_independent=frozenset({"base"}), - ) + manifest = release_manifest("base", "consumer") with tempfile.TemporaryDirectory() as temporary_directory: root = Path(temporary_directory) metadata = { @@ -143,9 +329,7 @@ def opener(_request: object, *, timeout: int) -> object: ): api.version_record("a", "0.7.3") - manifest = publish_release.ReleaseManifest( - ordered_crates=("a",), registry_independent=frozenset() - ) + manifest = release_manifest("a") delays: list[int] = [] with self.assertRaises(publish_release.TransientPublishError): publish_release.validate_registry_state_with_retry( @@ -160,9 +344,7 @@ def opener(_request: object, *, timeout: int) -> object: self.assertEqual(attempts, 5) def test_registry_retry_requires_prefix_and_matching_local_checksums(self) -> None: - manifest = publish_release.ReleaseManifest( - ordered_crates=("a", "b", "c"), registry_independent=frozenset() - ) + manifest = release_manifest("a", "b", "c") api = FakeApi( { "a": publish_release.RegistryRecord(True, "aaa"), @@ -195,9 +377,7 @@ def test_registry_retry_requires_prefix_and_matching_local_checksums(self) -> No ) def test_transient_publish_retries_are_bounded_and_requery_registry(self) -> None: - manifest = publish_release.ReleaseManifest( - ordered_crates=("a",), registry_independent=frozenset() - ) + manifest = release_manifest("a") api = FakeApi({"a": publish_release.RegistryRecord(False, None)}) attempts = 0 delays: list[int] = [] diff --git a/xtask/Cargo.toml b/xtask/Cargo.toml index 5580a88d..3c368daa 100644 --- a/xtask/Cargo.toml +++ b/xtask/Cargo.toml @@ -20,6 +20,7 @@ adoption = [ flate2 = { workspace = true } j2k-codec-math = { path = "../crates/j2k-codec-math" } proc-macro2 = { workspace = true, features = ["span-locations"] } +serde = { workspace = true } serde_json = { workspace = true } serde_yaml_ng = { workspace = true } syn = { workspace = true, features = ["full", "visit"] } diff --git a/xtask/src/codegen_commands.rs b/xtask/src/codegen_commands.rs index 12705727..180b04f5 100644 --- a/xtask/src/codegen_commands.rs +++ b/xtask/src/codegen_commands.rs @@ -1,12 +1,14 @@ use std::fmt::Write as _; use std::fs; +use std::io::ErrorKind; use std::path::Path; +use std::{collections::BTreeMap, collections::BTreeSet}; mod transaction; use transaction::write_generated_pair_transactionally; -use crate::release_commands::STABLE_DOC_LIBRARY_PACKAGES; +use crate::release_commands::published_library_packages; use crate::stable_api::{ collect_package_apis, verify_cargo_public_api_version, CARGO_PUBLIC_API_VERSION, HIDDEN_API_SNAPSHOT, ORDINARY_RUSTDOCFLAGS, PUBLIC_API_SNAPSHOT, PUBLIC_API_TARGET, @@ -216,7 +218,19 @@ fn render_stable_api_snapshots() -> Result<(String, String), String> { ); } verify_cargo_public_api_version()?; - let inventories = collect_package_apis(STABLE_DOC_LIBRARY_PACKAGES)?; + let packages = published_library_packages()?; + let package_refs = packages.iter().map(String::as_str).collect::>(); + let inventories = collect_package_apis(&package_refs)?; + let committed_snapshot = match fs::read_to_string(PUBLIC_API_SNAPSHOT) { + Ok(source) => source, + Err(error) if error.kind() == ErrorKind::NotFound => String::new(), + Err(error) => { + return Err(format!( + "failed to read {PUBLIC_API_SNAPSHOT} for inventory presentation order: {error}" + )); + } + }; + let render_order = inventory_render_order(&packages, &committed_snapshot)?; let tool_version = format!("cargo-public-api {CARGO_PUBLIC_API_VERSION}"); let mut public_out = String::new(); @@ -268,8 +282,8 @@ fn render_stable_api_snapshots() -> Result<(String, String), String> { ) .unwrap(); - for package in STABLE_DOC_LIBRARY_PACKAGES { - let inventory = inventories.get(*package).ok_or_else(|| { + for package in render_order { + let inventory = inventories.get(package).ok_or_else(|| { format!("collected public API inventory is missing package `{package}`") })?; @@ -299,6 +313,41 @@ fn render_stable_api_snapshots() -> Result<(String, String), String> { )) } +fn inventory_render_order<'a>( + packages: &'a [String], + committed_snapshot: &str, +) -> Result, String> { + let by_name = packages + .iter() + .map(|package| (package.as_str(), package.as_str())) + .collect::>(); + let mut seen = BTreeSet::new(); + let mut ordered = Vec::with_capacity(packages.len()); + for line in committed_snapshot.lines() { + let Some(name) = line + .strip_prefix("## `") + .and_then(|line| line.strip_suffix('`')) + else { + continue; + }; + let Some(package) = by_name.get(name) else { + continue; + }; + if !seen.insert(name.to_string()) { + return Err(format!( + "{PUBLIC_API_SNAPSHOT} contains duplicate package heading `{name}`" + )); + } + ordered.push(*package); + } + for package in packages { + if seen.insert(package.clone()) { + ordered.push(package.as_str()); + } + } + Ok(ordered) +} + fn finalize_text_snapshot(snapshot: &str) -> String { let content = snapshot.trim_end_matches('\n'); if content.is_empty() { @@ -312,7 +361,7 @@ fn print_stable_api_help() { println!( "usage: cargo xtask stable-api [--write]\n\n\ Without --write, checks the ordinary and rustdoc-hidden API snapshots \ - against cargo-public-api output for all 1.0-stable library crates. \ + against cargo-public-api output for every published library crate. \ With --write, refreshes both snapshots. This task must run on macOS \ so target-gated Metal APIs are included." ); diff --git a/xtask/src/codegen_commands/tests/rendering.rs b/xtask/src/codegen_commands/tests/rendering.rs index 74d182af..2b5d176a 100644 --- a/xtask/src/codegen_commands/tests/rendering.rs +++ b/xtask/src/codegen_commands/tests/rendering.rs @@ -1,7 +1,7 @@ // SPDX-License-Identifier: MIT OR Apache-2.0 use super::super::{ - codec_math_codegen, compact_f32, finalize_text_snapshot, + codec_math_codegen, compact_f32, finalize_text_snapshot, inventory_render_order, render_codec_math_dwt97_metal_fragment, render_codec_math_dwt97_rust_fragment, stable_api, }; @@ -12,6 +12,22 @@ fn text_snapshots_end_with_exactly_one_newline() { assert_eq!(finalize_text_snapshot(""), ""); } +#[test] +fn inventory_rendering_preserves_committed_presentation_order_without_owning_scope() { + let packages = vec!["base".to_string(), "new".to_string(), "adapter".to_string()]; + let committed = "# Snapshot\n\n## `adapter`\n\n## `base`\n\n## `binary`\n"; + + assert_eq!( + inventory_render_order(&packages, committed).expect("render order"), + ["adapter", "base", "new"] + ); + + let duplicate = "## `base`\n## `base`\n"; + let error = inventory_render_order(&packages, duplicate) + .expect_err("duplicate committed package heading rejects"); + assert!(error.contains("duplicate")); +} + #[test] fn codec_math_fragments_render_from_one_ordered_constant_source() { let metal = render_codec_math_dwt97_metal_fragment(); diff --git a/xtask/src/coverage/model.rs b/xtask/src/coverage/model.rs index 060e18b4..cbbeb2db 100644 --- a/xtask/src/coverage/model.rs +++ b/xtask/src/coverage/model.rs @@ -48,6 +48,7 @@ const CUDA_ACCELERATOR_LANE: AcceleratorLaneSpec = AcceleratorLaneSpec { }; const ML_CUDA_MODULE_PREFIX: &str = "crates/j2k-ml/src/cuda/"; +const ML_METAL_MODULE_PREFIX: &str = "crates/j2k-ml/src/metal/"; const fn accelerator_package( name: &'static str, @@ -125,6 +126,7 @@ impl CoverageLane { Self::Host => !is_accelerator_path(path), Self::Metal => { METAL_ACCELERATOR_LANE.owns_path(path) + || path.starts_with(ML_METAL_MODULE_PREFIX) || is_shared_accelerator_path(path) || METAL_VENDOR_PATHS.contains(&path) } @@ -252,6 +254,7 @@ pub(super) fn parse_options(args: impl Iterator) -> Result bool { METAL_ACCELERATOR_LANE.owns_path(path) + || path.starts_with(ML_METAL_MODULE_PREFIX) || CUDA_ACCELERATOR_LANE.owns_path(path) || path.starts_with(ML_CUDA_MODULE_PREFIX) || is_shared_accelerator_path(path) diff --git a/xtask/src/coverage/tests.rs b/xtask/src/coverage/tests.rs index d6583c9f..9bbd9697 100644 --- a/xtask/src/coverage/tests.rs +++ b/xtask/src/coverage/tests.rs @@ -130,6 +130,8 @@ fn coverage_lanes_partition_host_and_accelerator_production_rust() { .includes_source("crates/j2k-metal/src/error.rs", SourceRole::Production)); assert!(!CoverageLane::Host .includes_source("crates/j2k-ml/src/cuda/batch.rs", SourceRole::Production)); + assert!(!CoverageLane::Host + .includes_source("crates/j2k-ml/src/metal/batch.rs", SourceRole::Production)); assert!(!CoverageLane::Host .includes_source("crates/j2k-core/src/accelerator.rs", SourceRole::Production)); assert!( @@ -137,6 +139,8 @@ fn coverage_lanes_partition_host_and_accelerator_production_rust() { ); assert!(CoverageLane::Cuda .includes_source("crates/j2k-ml/src/cuda/batch.rs", SourceRole::Production)); + assert!(CoverageLane::Metal + .includes_source("crates/j2k-ml/src/metal/batch.rs", SourceRole::Production)); assert!(CoverageLane::Metal .includes_source("crates/j2k-metal/src/error.rs", SourceRole::Production)); assert!(CoverageLane::Cuda diff --git a/xtask/src/main.rs b/xtask/src/main.rs index 2eb33b6c..a37dcf57 100644 --- a/xtask/src/main.rs +++ b/xtask/src/main.rs @@ -50,11 +50,11 @@ use codegen_commands::{codec_math_codegen, stable_api}; use command_support::passed_test_count; use panic_surface::panic_surface; use quality_commands::{ - ci, clippy, clippy_strict, deny, doc, downstream_smoke, fmt, fuzz_build, fuzz_run, machete, - miri, nextest, no_std, repo_lint, test, typos, verify_unsafe_audit, + ci, clippy, clippy_strict, doc, fmt, fuzz_build, fuzz_run, machete, miri, no_std, repo_lint, + test, verify_unsafe_audit, }; use release_commands::{ - j2k_ml_package_smoke, package, release_cpu, release_integrity, STABLE_SEMVER_PACKAGES, + j2k_ml_package_smoke, package, published_library_packages, release_cpu, release_integrity, }; use stable_api::CARGO_PUBLIC_API_VERSION; @@ -75,9 +75,7 @@ fn run() -> Result<(), String> { "clippy" => clippy(), "clippy-strict" => clippy_strict(), "test" => test(), - "nextest" => nextest(), "doc" | "docs" => doc(), - "typos" => typos(), "bench-build" => bench_build(env::args().skip(2)), "bench-report" => bench_report(env::args().skip(2)), "j2k-ml-bench-metal" => j2k_ml_batch_bench_metal(), @@ -109,19 +107,17 @@ fn run() -> Result<(), String> { "fuzz-build" => fuzz_build(), "fuzz-run" => fuzz_run(), "stable-api" => stable_api(env::args().skip(2)), - "semver" => semver::semver( - env::args().skip(2), - STABLE_SEMVER_PACKAGES, - CARGO_PUBLIC_API_VERSION, - ), - "deny" => deny(), + "semver" => { + let packages = published_library_packages()?; + let package_refs = packages.iter().map(String::as_str).collect::>(); + semver::semver(env::args().skip(2), &package_refs, CARGO_PUBLIC_API_VERSION) + } "miri" => miri(), "machete" => machete(), "clone-audit" => clone_audit(env::args().skip(2)), "panic-surface" => panic_surface(), "no-std" => no_std(), "unsafe-audit" => verify_unsafe_audit(), - "downstream-smoke" => downstream_smoke(), "repo-lint" => repo_lint(env::args().skip(2)), "release-integrity" => release_integrity(env::args().skip(2)), "release-status" => release_status::release_status(env::args().skip(2)), @@ -150,9 +146,7 @@ fn print_help() { clippy run clippy with warnings denied\n\ clippy-strict run stricter J2K clippy gates\n\ test run workspace tests\n\ - nextest run workspace tests with cargo-nextest\n\ doc build workspace docs with warnings denied\n\ - typos run typos\n\ bench-build compile benchmark targets [--lane host|cuda|metal|all]\n\ bench-report print or write a benchmark publication report\n\ j2k-ml-bench-metal benchmark Metal codec-resident and Burn-upload batch decode\n\ @@ -170,15 +164,13 @@ fn print_help() { fuzz-run run scheduled fuzz targets with J2K_FUZZ_RUNS\n\ stable-api check the generated 1.0 public API inventory snapshot\n\ semver verify computed release types and reviewed API diff [--write-report]\n\ - deny run cargo-deny\n\ miri run selected CPU/no_std crates under Miri\n\ machete run cargo-machete unused-dependency scan\n\ clone-audit stage source-aware production Rust and run pinned jscpd\n\ panic-surface run production-library unwrap/expect and explicit panic-macro ratchets\n\ no-std check no_std-compatible codec crates\n\ unsafe-audit verify docs/unsafe-audit.md lists unsafe Rust sources\n\ - downstream-smoke run facade and transcode examples used by integration docs\n\ - repo-lint run repository policy checks owned by xtask [--strict]\n\ + repo-lint run repository policy checks owned by xtask\n\ release-integrity validate offline release metadata; --publish requires final dated/signoff state\n\ release-status verify one frozen SHA's CI aggregate and both GPU jobs [--sha SHA] [--repository owner/name]\n\ release-cpu run release-mode CPU codec tests\n\ diff --git a/xtask/src/quality_commands.rs b/xtask/src/quality_commands.rs index e2029c50..b17976bc 100644 --- a/xtask/src/quality_commands.rs +++ b/xtask/src/quality_commands.rs @@ -13,7 +13,7 @@ use crate::command_support::{ run_nightly_cargo_in_dir_owned, run_program, rust_sources, }; use crate::panic_surface::panic_surface; -use crate::release_commands::STABLE_DOC_LIBRARY_PACKAGES; +use crate::release_commands::published_library_packages; const NO_STD_TARGET: &str = "aarch64-unknown-none"; const NO_STD_CORE_PORTABLE_TARGET: &str = "wasm32-unknown-unknown"; @@ -28,13 +28,9 @@ pub(super) fn ci() -> Result<(), String> { verify_unsafe_audit() } -pub(super) fn repo_lint(args: impl Iterator) -> Result<(), String> { - let mut strict = false; - for arg in args { - match arg.as_str() { - "--strict" => strict = true, - other => return Err(format!("unknown repo-lint argument `{other}`")), - } +pub(super) fn repo_lint(mut args: impl Iterator) -> Result<(), String> { + if let Some(argument) = args.next() { + return Err(format!("unknown repo-lint argument `{argument}`")); } run_cargo(&[ @@ -45,22 +41,7 @@ pub(super) fn repo_lint(args: impl Iterator) -> Result<(), String "repo_lint", "--", "--nocapture", - ])?; - - if strict { - run_cargo(&[ - "test", - "-p", - "xtask", - "--test", - "repo_lint", - "--", - "--nocapture", - "--ignored", - ])?; - } - - Ok(()) + ]) } pub(super) fn fmt() -> Result<(), String> { @@ -156,11 +137,13 @@ const STRICT_CLIPPY_BASELINE_ALLOWED_LINTS: &[&str] = &[ pub(super) fn test() -> Result<(), String> { if env::consts::OS != "macos" { - return test_workspace_without_benches(&[]); + test_workspace_without_benches(&[])?; + return test_downstream_examples(); } test_workspace_without_benches(&["--exclude", "j2k-metal"])?; - test_j2k_metal_without_benches() + test_j2k_metal_without_benches()?; + test_downstream_examples() } fn test_workspace_without_benches(extra_args: &[&str]) -> Result<(), String> { @@ -208,27 +191,15 @@ fn test_j2k_metal_without_benches() -> Result<(), String> { run_cargo(&["test", "-p", "j2k-metal", "--all-features", "--doc"]) } -pub(super) fn nextest() -> Result<(), String> { - run_cargo(&[ - "nextest", - "run", - "--workspace", - "--all-features", - "--lib", - "--bins", - "--tests", - ]) -} - pub(super) fn doc() -> Result<(), String> { run_cargo_with_env( &["doc", "--workspace", "--all-features", "--no-deps"], &[("RUSTDOCFLAGS", "-D warnings")], )?; - for package in STABLE_DOC_LIBRARY_PACKAGES { + for package in published_library_packages()? { run_cargo_with_env( - &["doc", "-p", package, "--lib", "--no-deps"], + &["doc", "-p", package.as_str(), "--lib", "--no-deps"], &[("RUSTDOCFLAGS", "-D warnings -D missing_docs")], )?; } @@ -239,10 +210,6 @@ pub(super) fn doc() -> Result<(), String> { ) } -pub(super) fn typos() -> Result<(), String> { - run_program(OsString::from("typos"), &[], &[]) -} - pub(super) fn fuzz_build() -> Result<(), String> { run_cargo(&["check", "--manifest-path", "crates/j2k/fuzz/Cargo.toml"])?; run_cargo(&[ @@ -318,10 +285,6 @@ fn fuzz_target_triple() -> Result { .ok_or_else(|| "failed to parse nightly host target from `rustc -vV`".to_string()) } -pub(super) fn deny() -> Result<(), String> { - run_cargo(&["deny", "check", "licenses", "advisories", "bans", "sources"]) -} - pub(super) fn miri() -> Result<(), String> { run_nightly_cargo(&["miri", "test", "-p", "j2k-core"])?; run_nightly_cargo(&["miri", "test", "-p", "j2k-tilecodec"])?; @@ -556,7 +519,7 @@ fn tokens_contain_unsafe(tokens: &TokenStream) -> bool { }) } -pub(super) fn downstream_smoke() -> Result<(), String> { +fn test_downstream_examples() -> Result<(), String> { run_cargo(&["test", "-p", "j2k", "--examples"])?; run_cargo(&["test", "-p", "j2k-transcode", "--examples"]) } diff --git a/xtask/src/quality_commands/command_tests.rs b/xtask/src/quality_commands/command_tests.rs index 5ef9bb25..d90cae3c 100644 --- a/xtask/src/quality_commands/command_tests.rs +++ b/xtask/src/quality_commands/command_tests.rs @@ -1,8 +1,7 @@ // SPDX-License-Identifier: MIT OR Apache-2.0 use super::{ - clippy, clippy_strict, deny, doc, downstream_smoke, fmt, fuzz_build, nextest, repo_lint, test, - test_workspace_without_benches, + clippy, clippy_strict, doc, fmt, fuzz_build, repo_lint, test, test_workspace_without_benches, }; use crate::command_support::use_test_cargo_program; use crate::test_command::RecordingProgram; @@ -17,13 +16,9 @@ fn quality_command_plans_are_complete_and_never_launch_real_tools() { clippy_strict().expect("strict Clippy plan"); test().expect("test plan"); test_workspace_without_benches(&["--exclude", "fixture-package"]).expect("custom test plan"); - nextest().expect("nextest plan"); doc().expect("documentation plan"); fuzz_build().expect("fuzz build plan"); - deny().expect("dependency policy plan"); - downstream_smoke().expect("downstream smoke plan"); repo_lint(std::iter::empty()).expect("repo-lint plan"); - repo_lint(["--strict".to_string()].into_iter()).expect("strict repo-lint plan"); let log = recording.log(); assert!(log.contains("fmt --all -- --check|")); @@ -32,23 +27,26 @@ fn quality_command_plans_are_complete_and_never_launch_real_tools() { assert!(log.contains("-W clippy::pedantic -W clippy::nursery -D warnings")); assert!(log.contains("test --workspace --all-features --lib --bins --tests")); assert!(log.contains("--exclude fixture-package")); - assert!(log.contains("nextest run --workspace --all-features --lib --bins --tests")); assert!(log.contains("doc --workspace --all-features --no-deps|RUSTDOCFLAGS=-D warnings")); assert!( log.contains("doc -p j2k-core --lib --no-deps|RUSTDOCFLAGS=-D warnings -D missing_docs") ); assert!(log.contains("check --manifest-path crates/j2k/fuzz/Cargo.toml")); assert!(log.contains("check --manifest-path crates/j2k-transcode/fuzz/Cargo.toml")); - assert!(log.contains("deny check licenses advisories bans sources")); assert!(log.contains("test -p j2k --examples")); assert!(log.contains("test -p j2k-transcode --examples")); assert!(log.contains("test -p xtask --test repo_lint -- --nocapture")); - assert!(log.contains("test -p xtask --test repo_lint -- --nocapture --ignored")); } #[test] -fn repo_lint_unknown_argument_fails_before_any_command() { - let error = - repo_lint(["--unknown".to_string()].into_iter()).expect_err("unknown repo-lint argument"); - assert!(error.contains("unknown repo-lint argument")); +fn repo_lint_rejects_retired_strict_and_unknown_arguments_before_any_command() { + let recording = RecordingProgram::new("repo-lint-argument-test", ""); + let _cargo = use_test_cargo_program(recording.program().as_os_str().to_owned()); + + for argument in ["--strict", "--unknown"] { + let error = repo_lint([argument.to_string()].into_iter()) + .expect_err("unsupported repo-lint argument"); + assert!(error.contains("unknown repo-lint argument")); + } + assert!(!recording.was_invoked()); } diff --git a/xtask/src/release_commands.rs b/xtask/src/release_commands.rs index ae4ff77c..ca47ba85 100644 --- a/xtask/src/release_commands.rs +++ b/xtask/src/release_commands.rs @@ -18,50 +18,10 @@ use release_integrity_policy::{ }; #[cfg(test)] use release_manifest::parse_release_manifest_source; -use release_manifest::{release_manifest_contract, validate_release_manifest_contract}; - -const PUBLISHABLE_PACKAGES: &[&str] = &[ - "j2k-core", - "j2k-profile", - "j2k-types", - "j2k-codec-math", - "j2k-cuda-runtime", - "j2k-metal-support", - "j2k-native", - "j2k-jpeg", - "j2k-tilecodec", - "j2k", - "j2k-transcode", - "j2k-transcode-cuda", - "j2k-jpeg-metal", - "j2k-metal", - "j2k-transcode-metal", - "j2k-jpeg-cuda", - "j2k-cuda", - "j2k-ml", - "j2k-cli", -]; - -const REGISTRY_INDEPENDENT_PACKAGES: &[&str] = - &["j2k-core", "j2k-profile", "j2k-types", "j2k-codec-math"]; - -const STAGED_DEPENDENCY_PACKAGES: &[&str] = &[ - "j2k-cuda-runtime", - "j2k-metal-support", - "j2k-native", - "j2k-jpeg", - "j2k-tilecodec", - "j2k", - "j2k-transcode", - "j2k-transcode-cuda", - "j2k-jpeg-metal", - "j2k-metal", - "j2k-transcode-metal", - "j2k-jpeg-cuda", - "j2k-cuda", - "j2k-ml", - "j2k-cli", -]; +use release_manifest::{ + crates_io_publishable, release_manifest_contract, validate_release_manifest_contract, + ReleaseManifestContract, +}; const CPU_RELEASE_PACKAGES: &[&str] = &[ "j2k-core", @@ -74,68 +34,22 @@ const CPU_RELEASE_PACKAGES: &[&str] = &[ "j2k-cli", ]; -pub(super) const STABLE_SEMVER_PACKAGES: &[&str] = &[ - "j2k", - "j2k-core", - "j2k-codec-math", - "j2k-jpeg", - "j2k-tilecodec", - "j2k-jpeg-metal", - "j2k-metal", - "j2k-jpeg-cuda", - "j2k-cuda", - "j2k-transcode", - "j2k-transcode-cuda", - "j2k-metal-support", - "j2k-transcode-metal", - "j2k-native", - "j2k-types", - "j2k-cuda-runtime", - "j2k-profile", - "j2k-ml", -]; - -pub(super) const STABLE_DOC_LIBRARY_PACKAGES: &[&str] = &[ - "j2k", - "j2k-core", - "j2k-codec-math", - "j2k-jpeg", - "j2k-tilecodec", - "j2k-jpeg-metal", - "j2k-metal", - "j2k-jpeg-cuda", - "j2k-cuda", - "j2k-transcode", - "j2k-transcode-cuda", - "j2k-metal-support", - "j2k-transcode-metal", - "j2k-native", - "j2k-types", - "j2k-cuda-runtime", - "j2k-profile", - "j2k-ml", -]; +pub(super) fn published_library_packages() -> Result, String> { + Ok(release_manifest_contract()? + .library_packages() + .map(ToString::to_string) + .collect()) +} -#[expect( - clippy::too_many_lines, - reason = "release integrity evaluates the complete package policy before reporting aggregated failures" -)] pub(super) fn release_integrity(args: impl Iterator) -> Result<(), String> { let mode = ReleaseIntegrityMode::parse(args)?; let metadata = cargo_metadata()?; let workspace_version = workspace_version()?; let release_manifest = release_manifest_contract()?; - let publishable_set = release_manifest - .ordered_crates - .iter() - .map(String::as_str) - .collect::>(); - let docs_set = str_set(STABLE_DOC_LIBRARY_PACKAGES); - let semver_set = str_set(STABLE_SEMVER_PACKAGES); + let publishable_set = release_manifest.ordered_crates().collect::>(); + let library_set = release_manifest.library_packages().collect::>(); let mut errors = Vec::new(); - validate_package_gate_partition(&mut errors); - let workspace_packages = workspace_package_records(&metadata)?; validate_release_manifest_contract(&release_manifest, &workspace_packages, &mut errors)?; let mut workspace_names = BTreeSet::new(); @@ -143,25 +57,32 @@ pub(super) fn release_integrity(args: impl Iterator) -> Result<() let unpublished_members = workspace_packages .iter() .copied() - .filter(|package| publish_false(package)) - .map(package_name) - .collect::, _>>()?; + .map(|package| { + let name = package_name(package)?; + Ok((name, crates_io_publishable(package, name)?)) + }) + .filter_map(|result| match result { + Ok((name, false)) => Some(Ok(name)), + Ok((_, true)) => None, + Err(error) => Some(Err(error)), + }) + .collect::, String>>()?; for package in workspace_packages { let name = package_name(package)?; workspace_names.insert(name.to_string()); let listed_publishable = publishable_set.contains(name); - let explicitly_unpublished = publish_false(package); + let crates_io_eligible = crates_io_publishable(package, name)?; - if listed_publishable && explicitly_unpublished { + if listed_publishable && !crates_io_eligible { errors.push(format!( - "`{name}` is listed as publishable but has `publish = false`" + "`{name}` is listed in release-crates.json but is not eligible for crates.io publication" )); continue; } - if !listed_publishable && !explicitly_unpublished { + if !listed_publishable && crates_io_eligible { errors.push(format!( - "`{name}` is neither in PUBLISHABLE_PACKAGES nor explicitly `publish = false`" + "`{name}` is eligible for crates.io publication but missing from release-crates.json" )); continue; } @@ -193,14 +114,9 @@ pub(super) fn release_integrity(args: impl Iterator) -> Result<() )); } if has_lib_target(package) { - if !docs_set.contains(name) { - errors.push(format!( - "`{name}` has a library target but is missing from STABLE_DOC_LIBRARY_PACKAGES" - )); - } - if !semver_set.contains(name) { + if !library_set.contains(name) { errors.push(format!( - "`{name}` has a library target but is missing from STABLE_SEMVER_PACKAGES" + "`{name}` has a library target but release-crates.json does not assign a library API contract" )); } } else if name != "j2k-cli" { @@ -210,31 +126,17 @@ pub(super) fn release_integrity(args: impl Iterator) -> Result<() } } - for package in &release_manifest.ordered_crates { + for package in release_manifest.ordered_crates() { if !workspace_names.contains(package) { errors.push(format!( - "`{package}` is listed in PUBLISHABLE_PACKAGES but is not a workspace member" - )); - } - } - for package in STABLE_DOC_LIBRARY_PACKAGES { - if !publishable_set.contains(package) { - errors.push(format!( - "`{package}` is in STABLE_DOC_LIBRARY_PACKAGES but is not publishable" - )); - } - } - for package in STABLE_SEMVER_PACKAGES { - if !publishable_set.contains(package) { - errors.push(format!( - "`{package}` is in STABLE_SEMVER_PACKAGES but is not publishable" + "`{package}` is listed in release-crates.json but is not a workspace member" )); } } validate_publish_workflow(&mut errors)?; validate_publish_script(&mut errors)?; - validate_release_docs(&mut errors)?; + validate_release_docs(&release_manifest, &mut errors)?; validate_release_metadata(&workspace_version, mode, &mut errors)?; if errors.is_empty() { @@ -387,13 +289,6 @@ fn package_name(package: &serde_json::Value) -> Result<&str, String> { .ok_or_else(|| "cargo metadata package missing name".to_string()) } -fn publish_false(package: &serde_json::Value) -> bool { - package - .get("publish") - .and_then(serde_json::Value::as_array) - .is_some_and(Vec::is_empty) -} - fn has_lib_target(package: &serde_json::Value) -> bool { package .get("targets") @@ -548,9 +443,40 @@ fn validate_publish_script(errors: &mut Vec) -> Result<(), String> { publisher_path.display() )); } + + let manifest_validator_path = Path::new("scripts/release_manifest.py"); + let manifest_validator = fs::read_to_string(manifest_validator_path).map_err(|err| { + format!( + "failed to read {}: {err}", + manifest_validator_path.display() + ) + })?; + validate_python_release_manifest_source(&manifest_validator, errors); Ok(()) } +fn validate_python_release_manifest_source(script: &str, errors: &mut Vec) { + let script_path = Path::new("scripts/release_manifest.py"); + for required in [ + "object_pairs_hook=_reject_duplicate_fields", + "if type(schema) is not int or schema != 2", + "\"workspace_members\"", + "return \"crates-io\" in registries", + "must be workspace/path sourced", + "expected_requirement = f\"={versions_by_package[dependency_name]}\"", + "if kind == \"dev\"", + "if positions[dependency_name] >= positions[crate]", + "registry_independent", + ] { + if !script.contains(required) { + errors.push(format!( + "{} does not enforce release-manifest check `{required}`", + script_path.display() + )); + } + } +} + fn validate_publish_script_source(script: &str, errors: &mut Vec) { let script_path = Path::new("scripts/publish-crate.sh"); for required in [ @@ -589,17 +515,25 @@ fn validate_publish_script_source(script: &str, errors: &mut Vec) { } } -fn validate_release_docs(errors: &mut Vec) -> Result<(), String> { +fn validate_release_docs( + manifest: &ReleaseManifestContract, + errors: &mut Vec, +) -> Result<(), String> { let release_doc_path = Path::new("docs/release.md"); let release_doc = fs::read_to_string(release_doc_path) .map_err(|err| format!("failed to read {}: {err}", release_doc_path.display()))?; - validate_release_docs_source(&release_doc, errors); + let packages = manifest.ordered_crates().collect::>(); + validate_release_docs_source(&release_doc, &packages, errors); Ok(()) } -fn validate_release_docs_source(release_doc: &str, errors: &mut Vec) { +fn validate_release_docs_source( + release_doc: &str, + publishable_packages: &[&str], + errors: &mut Vec, +) { let release_doc_path = Path::new("docs/release.md"); - for package in PUBLISHABLE_PACKAGES { + for package in publishable_packages { if !release_doc.contains(&format!("`{package}`")) { errors.push(format!( "{} does not document publishable crate `{package}`", @@ -625,41 +559,6 @@ fn validate_release_docs_source(release_doc: &str, errors: &mut Vec) { } } -fn str_set(values: &[&'static str]) -> BTreeSet<&'static str> { - values.iter().copied().collect() -} - -fn validate_package_gate_partition(errors: &mut Vec) { - let publishable = str_set(PUBLISHABLE_PACKAGES); - let independent = str_set(REGISTRY_INDEPENDENT_PACKAGES); - let staged = str_set(STAGED_DEPENDENCY_PACKAGES); - let partitioned = independent.union(&staged).copied().collect::>(); - - if independent.len() != REGISTRY_INDEPENDENT_PACKAGES.len() { - errors.push("REGISTRY_INDEPENDENT_PACKAGES contains duplicates".to_string()); - } - if staged.len() != STAGED_DEPENDENCY_PACKAGES.len() { - errors.push("STAGED_DEPENDENCY_PACKAGES contains duplicates".to_string()); - } - for package in independent.intersection(&staged) { - errors.push(format!( - "`{package}` appears in both package-gate partitions" - )); - } - for package in publishable.difference(&partitioned) { - errors.push(format!( - "publishable package `{package}` is missing from the package-gate partitions" - )); - } - for package in &partitioned { - if !publishable.contains(package) { - errors.push(format!( - "package-gate partition contains non-publishable package `{package}`" - )); - } - } -} - pub(super) fn release_cpu() -> Result<(), String> { let mut args = vec!["test", "--release"]; for package in CPU_RELEASE_PACKAGES { @@ -672,15 +571,17 @@ pub(super) fn release_cpu() -> Result<(), String> { pub(super) fn package() -> Result<(), String> { ensure_clean_worktree()?; let metadata = cargo_metadata()?; - for package in PUBLISHABLE_PACKAGES { + let release_manifest = release_manifest_contract()?; + for package in release_manifest.ordered_crates() { run_cargo(&["package", "-p", package, "--list"])?; } - package_gate::run(&metadata) + package_gate::run(&metadata, &release_manifest) } pub(super) fn j2k_ml_package_smoke() -> Result<(), String> { let metadata = cargo_metadata()?; - package_gate::run_j2k_ml_package_smoke(&metadata) + let release_manifest = release_manifest_contract()?; + package_gate::run_j2k_ml_package_smoke(&metadata, &release_manifest) } #[cfg(test)] diff --git a/xtask/src/release_commands/package_gate.rs b/xtask/src/release_commands/package_gate.rs index 809392a8..5e6b4826 100644 --- a/xtask/src/release_commands/package_gate.rs +++ b/xtask/src/release_commands/package_gate.rs @@ -10,59 +10,43 @@ use crate::process::{cargo, run_command_owned, CommandContext}; use consumer::{package_archive_path, run_j2k_ml_consumer_gate}; -use super::{ - package_name, str_set, workspace_package_records, PUBLISHABLE_PACKAGES, - REGISTRY_INDEPENDENT_PACKAGES, +use super::release_manifest::{ + registry_independent_packages, release_dependencies_by_package, + validate_release_manifest_contract, ReleaseManifestContract, }; +use super::{package_name, workspace_package_records}; #[derive(Debug)] struct PackageGateStep { - package: &'static str, + package: String, version: String, registry_independent: bool, patches: Vec<(String, String)>, } -fn package_gate_plan(metadata: &serde_json::Value) -> Result, String> { +fn package_gate_plan( + metadata: &serde_json::Value, + manifest: &ReleaseManifestContract, +) -> Result, String> { let packages = workspace_package_records(metadata)?; + let mut validation_errors = Vec::new(); + validate_release_manifest_contract(manifest, &packages, &mut validation_errors)?; + if !validation_errors.is_empty() { + return Err(format!( + "release manifest violations:\n- {}", + validation_errors.join("\n- ") + )); + } let package_by_name = packages - .into_iter() - .map(|package| Ok((package_name(package)?.to_string(), package))) - .collect::, String>>()?; - let publishable = str_set(PUBLISHABLE_PACKAGES); - let independent = str_set(REGISTRY_INDEPENDENT_PACKAGES); - let dependencies_by_package = PUBLISHABLE_PACKAGES .iter() - .map(|&package| { - let record = package_by_name.get(package).ok_or_else(|| { - format!("publishable package `{package}` is absent from cargo metadata") - })?; - let dependencies = record - .get("dependencies") - .and_then(serde_json::Value::as_array) - .ok_or_else(|| { - format!("cargo metadata package `{package}` has no dependency array") - })? - .iter() - .filter(|dependency| { - dependency - .get("kind") - .is_none_or(serde_json::Value::is_null) - && dependency - .get("source") - .is_none_or(serde_json::Value::is_null) - }) - .filter_map(|dependency| dependency.get("name")?.as_str()) - .filter(|dependency| publishable.contains(*dependency)) - .map(ToString::to_string) - .collect::>(); - Ok((package.to_string(), dependencies)) - }) + .map(|package| Ok((package_name(package)?.to_string(), package))) .collect::, String>>()?; + let dependencies_by_package = release_dependencies_by_package(manifest, &packages)?; + let independent = registry_independent_packages(manifest, &packages)?; let mut processed = BTreeSet::new(); - let mut plan = Vec::with_capacity(PUBLISHABLE_PACKAGES.len()); + let mut plan = Vec::with_capacity(manifest.ordered_crates().len()); - for &package in PUBLISHABLE_PACKAGES { + for package in manifest.ordered_crates() { let mut pending = dependencies_by_package .get(package) .cloned() @@ -103,8 +87,12 @@ fn package_gate_plan(metadata: &serde_json::Value) -> Result Result Result<(), String> { - let plan = package_gate_plan(metadata)?; +pub(super) fn run_j2k_ml_package_smoke( + metadata: &serde_json::Value, + manifest: &ReleaseManifestContract, +) -> Result<(), String> { + let plan = package_gate_plan(metadata, manifest)?; let step = plan .iter() .find(|step| step.package == "j2k-ml") @@ -131,7 +122,7 @@ fn run_staged_package(step: &PackageGateStep, allow_dirty: bool) -> Result<(), S let mut args = vec![ "package".to_string(), "-p".to_string(), - step.package.to_string(), + step.package.clone(), "--no-verify".to_string(), ]; if allow_dirty { @@ -154,10 +145,13 @@ fn append_patch_config_args(args: &mut Vec, step: &PackageGateStep) -> R Ok(()) } -pub(super) fn run(metadata: &serde_json::Value) -> Result<(), String> { - for step in package_gate_plan(metadata)? { +pub(super) fn run( + metadata: &serde_json::Value, + manifest: &ReleaseManifestContract, +) -> Result<(), String> { + for step in package_gate_plan(metadata, manifest)? { if step.registry_independent { - run_cargo(&["publish", "-p", step.package, "--dry-run"])?; + run_cargo(&["publish", "-p", step.package.as_str(), "--dry-run"])?; } else { run_staged_package(&step, false)?; } diff --git a/xtask/src/release_commands/package_gate/consumer.rs b/xtask/src/release_commands/package_gate/consumer.rs index 12064425..db3584c8 100644 --- a/xtask/src/release_commands/package_gate/consumer.rs +++ b/xtask/src/release_commands/package_gate/consumer.rs @@ -48,7 +48,7 @@ pub(super) fn j2k_ml_consumer_manifest( Ok(manifest) } -const CONSUMER_SOURCE: &str = r#"use j2k::BatchDecodeOptions; +pub(super) const CONSUMER_SOURCE: &str = r#"use j2k::BatchDecodeOptions; #[cfg(feature = "cpu")] fn check_cpu_api() { @@ -64,11 +64,17 @@ fn check_cuda_api() { Default::default(), BatchDecodeOptions::default(), ); + let _compat_decoder: j2k_ml::CudaBurnDecoder = j2k_ml::CudaBurnDecoder::new( + Default::default(), + BatchDecodeOptions::default(), + ); } #[cfg(feature = "metal")] fn check_metal_api() { let _decoder = j2k_ml::MetalUploadBurnDecoder::system_default(BatchDecodeOptions::default()); + let _compat_decoder: Result = + j2k_ml::MetalBurnDecoder::system_default(BatchDecodeOptions::default()); } fn main() { diff --git a/xtask/src/release_commands/package_gate/tests.rs b/xtask/src/release_commands/package_gate/tests.rs index 28148642..df88d163 100644 --- a/xtask/src/release_commands/package_gate/tests.rs +++ b/xtask/src/release_commands/package_gate/tests.rs @@ -6,8 +6,13 @@ use std::io::Cursor; use std::path::{Path, PathBuf}; use std::time::{SystemTime, UNIX_EPOCH}; -use super::consumer::{extract_packaged_crate, j2k_ml_consumer_checks, j2k_ml_consumer_manifest}; -use super::{package_gate_plan, PUBLISHABLE_PACKAGES, REGISTRY_INDEPENDENT_PACKAGES}; +use super::consumer::{ + extract_packaged_crate, j2k_ml_consumer_checks, j2k_ml_consumer_manifest, CONSUMER_SOURCE, +}; +use super::{package_gate_plan, PackageGateStep}; +use crate::release_commands::release_manifest::{ + parse_release_manifest_source, ReleaseManifestContract, +}; #[cfg(unix)] use super::run; @@ -15,37 +20,69 @@ use super::run; use crate::{command_support::use_test_cargo_program, test_command::RecordingProgram}; fn workspace_metadata(dependencies: &[(&str, &[&str])]) -> serde_json::Value { + let manifest = release_manifest(); + let publishable = manifest.ordered_crates().collect::>(); let dependencies = dependencies .iter() .map(|(package, dependencies)| (*package, *dependencies)) .collect::>(); - let packages = PUBLISHABLE_PACKAGES + let packages = publishable .iter() .map(|package| { + let default_dependencies = if matches!( + *package, + "j2k-core" | "j2k-profile" | "j2k-types" | "j2k-codec-math" + ) { + &[][..] + } else { + &["j2k-core"][..] + }; let package_dependencies = dependencies .get(package) - .into_iter() - .flat_map(|dependencies| dependencies.iter()) + .copied() + .unwrap_or(default_dependencies) + .iter() .map(|dependency| { - serde_json::json!({"name": dependency, "kind": null, "source": null}) + serde_json::json!({ + "name": dependency, + "kind": null, + "path": format!("/workspace/{dependency}"), + "req": "=0.7.5", + "source": null + }) }) .collect::>(); serde_json::json!({ "id": package, "name": package, "version": "0.7.5", + "publish": null, "manifest_path": format!("/workspace/{package}/Cargo.toml"), "dependencies": package_dependencies, + "targets": if *package == "j2k-cli" { + vec![serde_json::json!({"kind": ["bin"]})] + } else { + vec![serde_json::json!({"kind": ["lib"]})] + }, }) }) .collect::>(); serde_json::json!({ - "workspace_members": PUBLISHABLE_PACKAGES, + "workspace_members": publishable, "packages": packages, "target_directory": "/workspace/target", }) } +fn release_manifest() -> ReleaseManifestContract { + parse_release_manifest_source(include_str!("../../../../release-crates.json")) + .expect("checked-in release manifest") +} + +fn test_package_gate_plan(metadata: &serde_json::Value) -> Result, String> { + package_gate_plan(metadata, &release_manifest()) +} + fn write_packaged_fixture(path: &Path) { let file = fs::File::create(path).expect("create package fixture"); let encoder = flate2::write::GzEncoder::new(file, flate2::Compression::default()); @@ -81,16 +118,20 @@ fn package_gate_plan_is_ordered_and_includes_transitive_unpublished_patches() { ("j2k-cli", &["j2k"]), ]); - let plan = package_gate_plan(&metadata).expect("dependency-aware package plan"); + let plan = test_package_gate_plan(&metadata).expect("dependency-aware package plan"); + let manifest = release_manifest(); assert_eq!( - plan.iter().map(|step| step.package).collect::>(), - PUBLISHABLE_PACKAGES + plan.iter() + .map(|step| step.package.as_str()) + .collect::>(), + manifest.ordered_crates().collect::>() ); + let registry_independent = ["j2k-core", "j2k-profile", "j2k-types", "j2k-codec-math"]; for step in &plan { assert_eq!( step.registry_independent, - REGISTRY_INDEPENDENT_PACKAGES.contains(&step.package) + registry_independent.contains(&step.package.as_str()) ); } let cli = plan @@ -117,25 +158,69 @@ fn package_gate_plan_rejects_missing_or_malformed_publishable_records() { .as_array_mut() .expect("members") .retain(|member| member != "j2k-core"); - let error = package_gate_plan(&missing).expect_err("missing publishable package"); - assert!(error.contains("`j2k-core` is absent")); + let error = test_package_gate_plan(&missing).expect_err("missing publishable package"); + assert!( + error.contains("must contain every publishable workspace crate exactly once"), + "unexpected: {error}" + ); let mut malformed = workspace_metadata(&[]); malformed["packages"][0] .as_object_mut() .expect("package") .remove("dependencies"); - let error = package_gate_plan(&malformed).expect_err("missing dependency array"); - assert!(error.contains("has no dependency array")); + let error = test_package_gate_plan(&malformed).expect_err("missing dependency array"); + assert!( + error.contains("has no dependencies array"), + "unexpected: {error}" + ); } #[test] fn package_gate_plan_rejects_forward_dependency_before_any_packaging() { let metadata = workspace_metadata(&[("j2k-core", &["j2k"])]); - let error = package_gate_plan(&metadata).expect_err("forward dependency order"); + let error = test_package_gate_plan(&metadata).expect_err("forward dependency order"); - assert!(error.contains("processes `j2k-core` before unpublished workspace dependency `j2k`")); + assert!( + error.contains("places `j2k-core` before dependency `j2k`"), + "unexpected: {error}" + ); +} + +#[test] +fn package_gate_includes_build_optional_and_target_specific_release_edges() { + let mut metadata = workspace_metadata(&[]); + let runtime = metadata["packages"] + .as_array_mut() + .expect("packages") + .iter_mut() + .find(|package| package["name"] == "j2k-cuda-runtime") + .expect("CUDA runtime package"); + runtime["dependencies"] = serde_json::json!([{ + "name": "j2k-codec-math", + "kind": "build", + "optional": true, + "target": "cfg(target_os = \"linux\")", + "path": "/workspace/j2k-codec-math", + "req": "=0.7.5", + "source": null + }]); + + let plan = test_package_gate_plan(&metadata).expect("build dependency plan"); + let runtime = plan + .iter() + .find(|step| step.package == "j2k-cuda-runtime") + .expect("CUDA runtime plan"); + + assert!(!runtime.registry_independent); + assert_eq!( + runtime.patches, + [( + "j2k-codec-math".to_string(), + "/workspace/j2k-codec-math".to_string() + )] + ); } #[test] @@ -151,13 +236,16 @@ fn package_gate_plan_requires_manifest_paths_for_patch_dependencies() { .expect("core record") .remove("manifest_path"); - let error = package_gate_plan(&metadata).expect_err("missing dependency manifest path"); + let error = test_package_gate_plan(&metadata).expect_err("missing dependency manifest path"); - assert!(error.contains("`j2k-core` has no manifest path")); + assert!( + error.contains("`j2k-core` has no string manifest_path"), + "unexpected: {error}" + ); } #[test] -fn package_gate_ignores_registry_and_non_normal_dependencies() { +fn package_gate_ignores_dev_dependencies_and_rejects_registry_sibling_edges() { let mut metadata = workspace_metadata(&[]); let native = metadata["packages"] .as_array_mut() @@ -166,16 +254,68 @@ fn package_gate_ignores_registry_and_non_normal_dependencies() { .find(|package| package["name"] == "j2k-native") .expect("native package"); native["dependencies"] = serde_json::json!([ - {"name": "j2k-core", "kind": "dev", "source": null}, - {"name": "j2k-core", "kind": null, "source": "registry+https://example.invalid"} + { + "name": "j2k-core", + "kind": "dev", + "path": "/workspace/j2k-core", + "req": "=0.7.5", + "source": null + }, + { + "name": "external", + "kind": null, + "path": null, + "req": "1", + "source": "registry+https://example.invalid" + } ]); - let plan = package_gate_plan(&metadata).expect("ignored non-patch dependencies"); + let plan = test_package_gate_plan(&metadata).expect("ignored non-release dependencies"); let native = plan .iter() .find(|step| step.package == "j2k-native") .expect("native step"); assert!(native.patches.is_empty()); + + let native = metadata["packages"] + .as_array_mut() + .expect("packages") + .iter_mut() + .find(|package| package["name"] == "j2k-native") + .expect("native package"); + native["dependencies"][1]["name"] = serde_json::json!("j2k-core"); + let error = + test_package_gate_plan(&metadata).expect_err("registry sibling dependency must reject"); + assert!( + error.contains("workspace/path sourced"), + "unexpected: {error}" + ); +} + +#[test] +fn package_gate_validates_exact_dev_dependencies_before_planning() { + let mut metadata = workspace_metadata(&[]); + let native = metadata["packages"] + .as_array_mut() + .expect("packages") + .iter_mut() + .find(|package| package["name"] == "j2k-native") + .expect("native package"); + native["dependencies"] = serde_json::json!([{ + "name": "j2k-core", + "kind": "dev", + "path": "/workspace/j2k-core", + "req": "^0.7.5", + "source": null + }]); + + let error = test_package_gate_plan(&metadata) + .expect_err("package planning must enforce exact dev dependencies"); + + assert!( + error.contains("exact release requirement"), + "unexpected: {error}" + ); } #[test] @@ -188,6 +328,21 @@ fn j2k_ml_consumer_matrix_matches_the_host_accelerator() { assert_eq!(j2k_ml_consumer_checks("windows"), ["cpu"]); } +#[test] +fn packaged_consumer_compiles_new_and_0_7_compatibility_decoder_names() { + for decoder in [ + "CudaUploadBurnDecoder", + "MetalUploadBurnDecoder", + "CudaBurnDecoder", + "MetalBurnDecoder", + ] { + assert!( + CONSUMER_SOURCE.contains(decoder), + "packaged consumer must compile `{decoder}`" + ); + } +} + #[test] fn j2k_ml_consumer_manifest_patches_only_workspace_crates() { let metadata = workspace_metadata(&[ @@ -196,7 +351,7 @@ fn j2k_ml_consumer_manifest_patches_only_workspace_crates() { ("j2k-metal", &["j2k", "j2k-metal-support"]), ("j2k-ml", &["j2k", "j2k-cuda", "j2k-metal"]), ]); - let plan = package_gate_plan(&metadata).expect("package plan"); + let plan = test_package_gate_plan(&metadata).expect("package plan"); let ml = plan .iter() .find(|step| step.package == "j2k-ml") @@ -231,7 +386,7 @@ fn packaged_consumer_extracts_the_crate_archive_instead_of_using_workspace_sourc fs::create_dir_all(&root).expect("create package extraction test root"); let archive = root.join("j2k-ml-0.7.5.crate"); write_packaged_fixture(&archive); - let plan = package_gate_plan(&workspace_metadata(&[])).expect("package plan"); + let plan = test_package_gate_plan(&workspace_metadata(&[])).expect("package plan"); let step = plan .iter() .find(|step| step.package == "j2k-ml") @@ -264,7 +419,8 @@ fn package_gate_executes_registry_and_staged_steps_with_dependency_patches() { let recording = RecordingProgram::new("package-gate-command-test", ""); let _cargo = use_test_cargo_program(recording.program().as_os_str().to_owned()); - run(&metadata).expect("hermetic package gate"); + let manifest = release_manifest(); + run(&metadata, &manifest).expect("hermetic package gate"); fs::remove_dir_all(package_root).expect("remove package gate target"); let log = recording.log(); @@ -272,7 +428,7 @@ fn package_gate_executes_registry_and_staged_steps_with_dependency_patches() { let consumer_checks = j2k_ml_consumer_checks(std::env::consts::OS); assert_eq!( lines.len(), - PUBLISHABLE_PACKAGES.len() + consumer_checks.len() + 2 + manifest.ordered_crates().len() + consumer_checks.len() + 2 ); assert!(lines[0].starts_with("publish -p j2k-core --dry-run|")); assert!(lines[3].starts_with("publish -p j2k-codec-math --dry-run|")); diff --git a/xtask/src/release_commands/release_manifest.rs b/xtask/src/release_commands/release_manifest.rs index 5f4b7a70..8826b735 100644 --- a/xtask/src/release_commands/release_manifest.rs +++ b/xtask/src/release_commands/release_manifest.rs @@ -4,12 +4,89 @@ use std::collections::{BTreeMap, BTreeSet}; use std::fs; use std::path::Path; -use super::{package_name, publish_false, PUBLISHABLE_PACKAGES, REGISTRY_INDEPENDENT_PACKAGES}; +use serde::Deserialize; + +use super::{has_lib_target, package_name}; + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub(super) enum ApiContract { + Stable, + Experimental, + Implementation, + Binary, +} + +impl ApiContract { + fn parse(value: &str) -> Result { + match value { + "stable" => Ok(Self::Stable), + "experimental" => Ok(Self::Experimental), + "implementation" => Ok(Self::Implementation), + "binary" => Ok(Self::Binary), + _ => Err(format!( + "release manifest api_contract `{value}` is not one of stable, experimental, implementation, or binary" + )), + } + } + + pub(super) const fn as_str(self) -> &'static str { + match self { + Self::Stable => "stable", + Self::Experimental => "experimental", + Self::Implementation => "implementation", + Self::Binary => "binary", + } + } + + pub(super) const fn is_library(self) -> bool { + !matches!(self, Self::Binary) + } +} + +#[derive(Debug, Eq, PartialEq)] +struct ReleaseCrateContract { + name: String, + api_contract: ApiContract, +} #[derive(Debug, Eq, PartialEq)] pub(super) struct ReleaseManifestContract { - pub(super) ordered_crates: Vec, - pub(super) registry_independent: Vec, + crates: Vec, +} + +#[derive(Deserialize)] +#[serde(deny_unknown_fields)] +struct RawReleaseManifest { + schema: u64, + crates: Vec, +} + +#[derive(Deserialize)] +#[serde(deny_unknown_fields)] +struct RawReleaseCrate { + name: String, + api_contract: String, +} + +impl ReleaseManifestContract { + pub(super) fn ordered_crates(&self) -> impl ExactSizeIterator { + self.crates + .iter() + .map(|release_crate| release_crate.name.as_str()) + } + + pub(super) fn api_contracts(&self) -> impl Iterator { + self.crates + .iter() + .map(|release_crate| (release_crate.name.as_str(), release_crate.api_contract)) + } + + pub(super) fn library_packages(&self) -> impl Iterator { + self.crates + .iter() + .filter(|release_crate| release_crate.api_contract.is_library()) + .map(|release_crate| release_crate.name.as_str()) + } } pub(super) fn release_manifest_contract() -> Result { @@ -32,53 +109,104 @@ pub(super) fn release_manifest_contract() -> Result Result { - let value: serde_json::Value = serde_json::from_str(source) - .map_err(|error| format!("release manifest is not valid JSON: {error}"))?; - let object = value - .as_object() - .ok_or_else(|| "release manifest root must be an object".to_string())?; - if object.get("schema").and_then(serde_json::Value::as_u64) != Some(1) { - return Err("release manifest schema must be exactly 1".to_string()); - } - - let string_array = |field: &str| -> Result, String> { - object - .get(field) - .and_then(serde_json::Value::as_array) - .ok_or_else(|| format!("release manifest {field} must be an array"))? - .iter() - .enumerate() - .map(|(index, value)| { - value - .as_str() - .filter(|value| !value.is_empty()) - .map(ToString::to_string) - .ok_or_else(|| { - format!("release manifest {field}[{index}] must be a non-empty string") - }) - }) - .collect() - }; - let ordered_crates = string_array("ordered_crates")?; - let registry_independent = string_array("registry_independent")?; - if ordered_crates.is_empty() { - return Err("release manifest ordered_crates must not be empty".to_string()); + let raw: RawReleaseManifest = serde_json::from_str(source) + .map_err(|error| format!("release manifest is not valid schema 2 JSON: {error}"))?; + if raw.schema != 2 { + return Err("release manifest schema must be exactly 2".to_string()); } - if ordered_crates.iter().collect::>().len() != ordered_crates.len() { - return Err("release manifest ordered_crates contains duplicates".to_string()); + if raw.crates.is_empty() { + return Err("release manifest crates must not be empty".to_string()); } - if registry_independent.iter().collect::>().len() != registry_independent.len() { - return Err("release manifest registry_independent contains duplicates".to_string()); + + let mut crates = Vec::new(); + crates + .try_reserve_exact(raw.crates.len()) + .map_err(|error| format!("reserve release manifest crates: {error}"))?; + let mut names = BTreeSet::new(); + for (index, record) in raw.crates.into_iter().enumerate() { + if record.name.is_empty() { + return Err(format!( + "release manifest crates[{index}].name must be a non-empty string" + )); + } + if !valid_crate_name(&record.name) { + return Err(format!( + "release manifest crates[{index}] has malformed crate name `{}`", + record.name + )); + } + if !names.insert(record.name.clone()) { + return Err(format!( + "release manifest crates contains duplicate crate `{}`", + record.name + )); + } + let api_contract = ApiContract::parse(&record.api_contract)?; + crates.push(ReleaseCrateContract { + name: record.name, + api_contract, + }); } - if ordered_crates.get(..registry_independent.len()) != Some(®istry_independent) { - return Err( - "release manifest registry_independent must be an ordered manifest prefix".to_string(), - ); + Ok(ReleaseManifestContract { crates }) +} + +fn valid_crate_name(name: &str) -> bool { + let mut bytes = name.bytes(); + bytes + .next() + .is_some_and(|byte| byte.is_ascii_alphanumeric()) + && bytes.all(|byte| byte.is_ascii_alphanumeric() || matches!(byte, b'-' | b'_')) +} + +pub(super) fn registry_independent_packages( + manifest: &ReleaseManifestContract, + workspace_packages: &[&serde_json::Value], +) -> Result, String> { + let dependencies_by_package = release_dependencies_by_package(manifest, workspace_packages)?; + Ok(dependencies_by_package + .into_iter() + .filter_map(|(name, dependencies)| dependencies.is_empty().then_some(name)) + .collect()) +} + +pub(super) fn release_dependencies_by_package( + manifest: &ReleaseManifestContract, + workspace_packages: &[&serde_json::Value], +) -> Result>, String> { + let package_by_name = package_records_by_name(workspace_packages)?; + let release_roots = release_package_roots(manifest, &package_by_name)?; + let mut dependencies_by_package = BTreeMap::new(); + for package_name in manifest.ordered_crates() { + let Some(package) = package_by_name.get(package_name) else { + dependencies_by_package.insert(package_name.to_string(), BTreeSet::new()); + continue; + }; + let dependencies = package + .get("dependencies") + .and_then(serde_json::Value::as_array) + .ok_or_else(|| { + format!("cargo metadata package `{package_name}` has no dependencies array") + })?; + let release_dependencies = dependencies.iter().enumerate().try_fold( + BTreeSet::new(), + |mut found, (index, dependency)| { + let kind = dependency_kind(package_name, index, dependency)?; + if kind != "dev" { + if let Some(dependency_name) = workspace_release_dependency( + package_name, + index, + dependency, + &release_roots, + )? { + found.insert(dependency_name.to_string()); + } + } + Ok::<_, String>(found) + }, + )?; + dependencies_by_package.insert(package_name.to_string(), release_dependencies); } - Ok(ReleaseManifestContract { - ordered_crates, - registry_independent, - }) + Ok(dependencies_by_package) } pub(super) fn validate_release_manifest_contract( @@ -86,90 +214,217 @@ pub(super) fn validate_release_manifest_contract( workspace_packages: &[&serde_json::Value], errors: &mut Vec, ) -> Result<(), String> { - compare_declared_release_sets(manifest, errors); - let package_by_name = workspace_packages - .iter() - .map(|package| Ok((package_name(package)?.to_string(), *package))) - .collect::, String>>()?; + let package_by_name = package_records_by_name(workspace_packages)?; let actual_publishable = workspace_packages .iter() - .filter(|package| !publish_false(package)) - .map(|package| package_name(package).map(ToString::to_string)) - .collect::, _>>()?; + .map(|package| { + let name = package_name(package)?; + Ok((name, crates_io_publishable(package, name)?)) + }) + .filter_map(|result| match result { + Ok((name, true)) => Some(Ok(name.to_string())), + Ok((_, false)) => None, + Err(error) => Some(Err(error)), + }) + .collect::, String>>()?; let manifest_publishable = manifest - .ordered_crates - .iter() - .cloned() + .ordered_crates() + .map(ToString::to_string) .collect::>(); if actual_publishable != manifest_publishable { errors.push(format!( "release-crates.json must contain every publishable workspace crate exactly once; actual={actual_publishable:?}, manifest={manifest_publishable:?}" )); } - validate_dependency_order(manifest, &package_by_name, errors); + validate_common_release_version(manifest, &package_by_name, errors); + validate_contract_targets(manifest, &package_by_name, errors)?; + let dependencies_by_package = release_dependencies_by_package(manifest, workspace_packages)?; + validate_dependency_order(manifest, &dependencies_by_package, errors); + validate_exact_release_dependencies(manifest, &package_by_name, errors)?; Ok(()) } -fn compare_declared_release_sets(manifest: &ReleaseManifestContract, errors: &mut Vec) { - let expected = PUBLISHABLE_PACKAGES - .iter() - .map(ToString::to_string) - .collect::>(); - if manifest.ordered_crates != expected { - errors.push(format!( - "release-crates.json order is {:?}, expected {:?}", - manifest.ordered_crates, expected - )); +fn validate_common_release_version( + manifest: &ReleaseManifestContract, + package_by_name: &BTreeMap, + errors: &mut Vec, +) { + let mut versions = BTreeSet::new(); + for name in manifest.ordered_crates() { + let Some(package) = package_by_name.get(name) else { + continue; + }; + let Some(version) = package.get("version").and_then(serde_json::Value::as_str) else { + errors.push(format!( + "cargo metadata package `{name}` has no string version" + )); + continue; + }; + versions.insert(version); } - let expected_independent = REGISTRY_INDEPENDENT_PACKAGES - .iter() - .map(ToString::to_string) - .collect::>(); - if manifest.registry_independent != expected_independent { + if versions.len() > 1 { errors.push(format!( - "release-crates.json registry-independent prefix is {:?}, expected {:?}", - manifest.registry_independent, expected_independent + "release manifest packages must share one release version, found {versions:?}" )); } } -fn validate_dependency_order( +pub(super) fn crates_io_publishable( + package: &serde_json::Value, + name: &str, +) -> Result { + match package.get("publish") { + None | Some(serde_json::Value::Null) => Ok(true), + Some(serde_json::Value::Array(registries)) => { + let mut crates_io = false; + for (index, registry) in registries.iter().enumerate() { + let registry = registry.as_str().ok_or_else(|| { + format!("cargo metadata package `{name}` publish[{index}] is not a string") + })?; + crates_io |= registry == "crates-io"; + } + Ok(crates_io) + } + Some(_) => Err(format!( + "cargo metadata package `{name}` has invalid publish eligibility" + )), + } +} + +fn package_records_by_name<'a>( + workspace_packages: &[&'a serde_json::Value], +) -> Result, String> { + let mut package_by_name = BTreeMap::new(); + for package in workspace_packages { + let name = package_name(package)?.to_string(); + if package_by_name.insert(name.clone(), *package).is_some() { + return Err(format!( + "cargo metadata contains duplicate workspace package name `{name}`" + )); + } + } + Ok(package_by_name) +} + +fn validate_contract_targets( manifest: &ReleaseManifestContract, package_by_name: &BTreeMap, errors: &mut Vec, -) { - let positions = manifest - .ordered_crates - .iter() - .enumerate() - .map(|(index, crate_name)| (crate_name.as_str(), index)) - .collect::>(); - for (crate_index, crate_name) in manifest.ordered_crates.iter().enumerate() { - let Some(package) = package_by_name.get(crate_name) else { +) -> Result<(), String> { + for (name, contract) in manifest.api_contracts() { + let Some(package) = package_by_name.get(name) else { continue; }; - let Some(dependencies) = package - .get("dependencies") + let targets = package + .get("targets") .and_then(serde_json::Value::as_array) - else { - errors.push(format!( - "cargo metadata package `{crate_name}` has no dependencies array" - )); + .ok_or_else(|| format!("cargo metadata package `{name}` has no targets array"))?; + let mut kinds = BTreeSet::new(); + for (target_index, target) in targets.iter().enumerate() { + let target_kinds = target + .get("kind") + .and_then(serde_json::Value::as_array) + .ok_or_else(|| { + format!( + "cargo metadata package `{name}` target[{target_index}] has no kind array" + ) + })?; + for (kind_index, kind) in target_kinds.iter().enumerate() { + kinds.insert(kind.as_str().ok_or_else(|| { + format!( + "cargo metadata package `{name}` target[{target_index}].kind[{kind_index}] is not a string" + ) + })?); + } + } + let has_library = has_lib_target(package); + let has_binary = kinds.contains("bin"); + match contract { + ApiContract::Binary if !has_binary || has_library => errors.push(format!( + "`{name}` has api_contract=binary but must have a binary target and no library target" + )), + ApiContract::Stable | ApiContract::Experimental | ApiContract::Implementation + if !has_library => + { + errors.push(format!( + "`{name}` has api_contract={} but has no library target", + contract.as_str() + )); + } + _ => {} + } + } + Ok(()) +} + +fn validate_exact_release_dependencies( + manifest: &ReleaseManifestContract, + package_by_name: &BTreeMap, + errors: &mut Vec, +) -> Result<(), String> { + let release_roots = release_package_roots(manifest, package_by_name)?; + for package_name in manifest.ordered_crates() { + let Some(package) = package_by_name.get(package_name) else { continue; }; - for (dependency_index, dependency) in dependencies.iter().enumerate() { - if dependency.get("kind").and_then(serde_json::Value::as_str) == Some("dev") { + let dependencies = package + .get("dependencies") + .and_then(serde_json::Value::as_array) + .ok_or_else(|| { + format!("cargo metadata package `{package_name}` has no dependencies array") + })?; + for (index, dependency) in dependencies.iter().enumerate() { + let Some(dependency_name) = + workspace_release_dependency(package_name, index, dependency, &release_roots)? + else { continue; - } - let Some(dependency_name) = dependency.get("name").and_then(serde_json::Value::as_str) + }; + let Some(dependency_version) = package_by_name + .get(dependency_name) + .and_then(|dependency_package| dependency_package.get("version")) + .and_then(serde_json::Value::as_str) else { errors.push(format!( - "cargo metadata package `{crate_name}` dependency[{dependency_index}] has no string name" + "cargo metadata release dependency `{dependency_name}` has no string version" )); continue; }; + let requirement = dependency + .get("req") + .and_then(serde_json::Value::as_str) + .ok_or_else(|| { + format!( + "cargo metadata package `{package_name}` dependency `{dependency_name}` has no string requirement" + ) + })?; + let exact = format!("={dependency_version}"); + if requirement != exact { + errors.push(format!( + "`{package_name}` dependency `{dependency_name}` must use exact release requirement `{exact}`, found `{requirement}`" + )); + } + } + } + Ok(()) +} + +fn validate_dependency_order( + manifest: &ReleaseManifestContract, + dependencies_by_package: &BTreeMap>, + errors: &mut Vec, +) { + let positions = manifest + .ordered_crates() + .enumerate() + .map(|(index, crate_name)| (crate_name, index)) + .collect::>(); + for (crate_index, crate_name) in manifest.ordered_crates().enumerate() { + let Some(dependencies) = dependencies_by_package.get(crate_name) else { + continue; + }; + for dependency_name in dependencies { if positions - .get(dependency_name) + .get(dependency_name.as_str()) .is_some_and(|dependency_position| *dependency_position >= crate_index) { errors.push(format!( @@ -179,3 +434,75 @@ fn validate_dependency_order( } } } + +fn release_package_roots( + manifest: &ReleaseManifestContract, + package_by_name: &BTreeMap, +) -> Result, String> { + manifest + .ordered_crates() + .filter_map(|name| package_by_name.get(name).map(|package| (name, *package))) + .map(|(name, package)| { + let manifest_path = package + .get("manifest_path") + .and_then(serde_json::Value::as_str) + .ok_or_else(|| { + format!("cargo metadata package `{name}` has no string manifest_path") + })?; + let root = Path::new(manifest_path).parent().ok_or_else(|| { + format!("cargo metadata package `{name}` manifest_path has no parent") + })?; + Ok((name.to_string(), root.to_string_lossy().into_owned())) + }) + .collect() +} + +fn workspace_release_dependency<'a>( + package: &str, + index: usize, + dependency: &'a serde_json::Value, + release_roots: &BTreeMap, +) -> Result, String> { + let dependency_name = dependency + .get("name") + .and_then(serde_json::Value::as_str) + .ok_or_else(|| { + format!("cargo metadata package `{package}` dependency[{index}] has no string name") + })?; + let Some(expected_root) = release_roots.get(dependency_name) else { + return Ok(None); + }; + if dependency + .get("source") + .is_some_and(|source| !source.is_null()) + { + return Err(format!( + "release package `{package}` dependency `{dependency_name}` must be workspace/path sourced" + )); + } + let Some(path) = dependency.get("path").and_then(serde_json::Value::as_str) else { + return Err(format!( + "release package `{package}` dependency `{dependency_name}` must be workspace/path sourced" + )); + }; + if Path::new(path) != Path::new(expected_root) { + return Err(format!( + "release package `{package}` dependency `{dependency_name}` path `{path}` does not match workspace package root `{expected_root}`" + )); + } + Ok(Some(dependency_name)) +} + +fn dependency_kind<'a>( + package: &str, + index: usize, + dependency: &'a serde_json::Value, +) -> Result<&'a str, String> { + match dependency.get("kind") { + None | Some(serde_json::Value::Null) => Ok("normal"), + Some(serde_json::Value::String(kind)) => Ok(kind), + Some(_) => Err(format!( + "cargo metadata package `{package}` dependency[{index}] has invalid kind" + )), + } +} diff --git a/xtask/src/release_commands/tests.rs b/xtask/src/release_commands/tests.rs index c5df4339..8745ce4e 100644 --- a/xtask/src/release_commands/tests.rs +++ b/xtask/src/release_commands/tests.rs @@ -2,10 +2,10 @@ use std::collections::BTreeSet; +use super::release_manifest::{crates_io_publishable, release_manifest_contract}; use super::{ - has_docs_rs_metadata, has_lib_target, package_name, publish_false, release_cpu, - release_integrity, validate_package_gate_partition, validate_publish_script_source, - validate_publish_workflow_source, validate_release_docs_source, + has_docs_rs_metadata, has_lib_target, package_name, release_cpu, release_integrity, + validate_publish_script_source, validate_publish_workflow_source, validate_release_docs_source, validate_unpublished_dependencies, workspace_package_records, }; @@ -33,12 +33,12 @@ fn release_metadata_shape_helpers_fail_closed() { "metadata": {"docs": {"rs": {"all-features": true, "targets": []}}}, }); assert_eq!(package_name(&valid), Ok("j2k-core")); - assert!(publish_false(&valid)); + assert!(!crates_io_publishable(&valid, "j2k-core").expect("valid publish field")); assert!(has_lib_target(&valid)); assert!(has_docs_rs_metadata(&valid)); assert!(package_name(&serde_json::json!({})).is_err()); - assert!(!publish_false(&serde_json::json!({"publish": false}))); + assert!(crates_io_publishable(&serde_json::json!({"publish": false}), "malformed").is_err()); assert!(!has_lib_target( &serde_json::json!({"targets": [{"kind": "lib"}]}) )); @@ -146,9 +146,8 @@ fn unpublished_dependency_policy_distinguishes_publishable_dev_and_runtime_edges } #[test] -fn checked_in_publish_workflow_script_docs_and_partitions_agree() { +fn checked_in_publish_workflow_script_and_docs_agree_with_the_manifest() { let mut errors = Vec::new(); - validate_package_gate_partition(&mut errors); validate_publish_workflow_source( include_str!("../../../.github/workflows/publish.yml"), &mut errors, @@ -158,7 +157,13 @@ fn checked_in_publish_workflow_script_docs_and_partitions_agree() { include_str!("../../../scripts/publish-crate.sh"), &mut errors, ); - validate_release_docs_source(include_str!("../../../docs/release.md"), &mut errors); + let manifest = release_manifest_contract().expect("checked-in release manifest"); + let packages = manifest.ordered_crates().collect::>(); + validate_release_docs_source( + include_str!("../../../docs/release.md"), + &packages, + &mut errors, + ); assert!(errors.is_empty(), "release contract drift: {errors:#?}"); } diff --git a/xtask/src/release_commands/tests/file_boundaries.rs b/xtask/src/release_commands/tests/file_boundaries.rs index 8a6f4e21..75413d37 100644 --- a/xtask/src/release_commands/tests/file_boundaries.rs +++ b/xtask/src/release_commands/tests/file_boundaries.rs @@ -1,5 +1,6 @@ // SPDX-License-Identifier: MIT OR Apache-2.0 +use super::super::release_manifest::parse_release_manifest_source; use super::super::{ validate_publish_script, validate_publish_workflow, validate_release_docs, validate_release_metadata, ReleaseIntegrityMode, @@ -14,7 +15,12 @@ fn missing_release_contract_files_fail_with_path_context() { let error = match case.as_str() { "workflow" => validate_publish_workflow(&mut Vec::new()), "script" => validate_publish_script(&mut Vec::new()), - "docs" => validate_release_docs(&mut Vec::new()), + "docs" => { + let manifest = + parse_release_manifest_source(include_str!("../../../../release-crates.json")) + .expect("checked-in release manifest"); + validate_release_docs(&manifest, &mut Vec::new()) + } "changelog" => validate_release_metadata( "0.7.0", ReleaseIntegrityMode::PreCandidate, diff --git a/xtask/src/release_commands/tests/integrity.rs b/xtask/src/release_commands/tests/integrity.rs index f18cbf93..c15e1a0d 100644 --- a/xtask/src/release_commands/tests/integrity.rs +++ b/xtask/src/release_commands/tests/integrity.rs @@ -1,6 +1,7 @@ // SPDX-License-Identifier: MIT OR Apache-2.0 -use super::super::{release_integrity, PUBLISHABLE_PACKAGES}; +use super::super::release_integrity; +use super::super::release_manifest::parse_release_manifest_source; use crate::{command_support::use_test_cargo_program, test_command::RecordingProgram}; const WORKSPACE_CHILD_ENV: &str = "XTASK_TEST_RELEASE_INTEGRITY_WORKSPACE_CHILD"; @@ -31,6 +32,9 @@ pub(super) fn metadata_program(label: &str, metadata: &serde_json::Value) -> Rec } pub(super) fn complete_publishable_metadata() -> serde_json::Value { + let manifest = parse_release_manifest_source(include_str!("../../../../release-crates.json")) + .expect("checked-in release manifest"); + let publishable = manifest.ordered_crates().collect::>(); let workspace_version = include_str!("../../../../Cargo.toml") .lines() .find_map(|line| { @@ -39,21 +43,36 @@ pub(super) fn complete_publishable_metadata() -> serde_json::Value { .and_then(|rest| rest.split('"').nth(1)) }) .expect("workspace package version"); - let packages = PUBLISHABLE_PACKAGES + let packages = publishable .iter() .map(|name| { let targets = if *name == "j2k-cli" { - Vec::new() + vec![serde_json::json!({"kind": ["bin"]})] } else { vec![serde_json::json!({"kind": ["lib"]})] }; + let dependencies = if matches!( + *name, + "j2k-core" | "j2k-profile" | "j2k-types" | "j2k-codec-math" + ) { + Vec::new() + } else { + vec![serde_json::json!({ + "name": "j2k-core", + "kind": null, + "path": "/workspace/j2k-core", + "req": format!("={workspace_version}"), + "source": null, + })] + }; serde_json::json!({ "id": name, "name": name, "version": workspace_version, "publish": null, "readme": "README.md", - "dependencies": [], + "manifest_path": format!("/workspace/{name}/Cargo.toml"), + "dependencies": dependencies, "targets": targets, "metadata": { "docs": {"rs": {"all-features": true, "targets": []}} @@ -62,7 +81,7 @@ pub(super) fn complete_publishable_metadata() -> serde_json::Value { }) .collect::>(); serde_json::json!({ - "workspace_members": PUBLISHABLE_PACKAGES, + "workspace_members": publishable, "packages": packages, }) } @@ -97,6 +116,9 @@ fn release_integrity_aggregates_invalid_package_metadata_without_publishing() { "id": "core", "name": "j2k-core", "publish": [], + "manifest_path": "/workspace/j2k-core/Cargo.toml", + "dependencies": [], + "targets": [{"kind": ["lib"]}], }, { "id": "internal", @@ -114,6 +136,7 @@ fn release_integrity_aggregates_invalid_package_metadata_without_publishing() { "publish": null, "version": "9.9.9", "readme": null, + "manifest_path": "/workspace/j2k-native/Cargo.toml", "dependencies": [], "targets": [], "metadata": {}, @@ -133,13 +156,13 @@ fn release_integrity_aggregates_invalid_package_metadata_without_publishing() { .expect_err("invalid package metadata must fail release integrity"); for expected in [ - "`j2k-core` is listed as publishable but has `publish = false`", - "`outsider` is neither in PUBLISHABLE_PACKAGES nor explicitly `publish = false`", + "`j2k-core` is listed in release-crates.json but is not eligible for crates.io publication", + "`outsider` is eligible for crates.io publication but missing from release-crates.json", "`j2k-native` version 9.9.9 does not match workspace version", "`j2k-native` is publishable but has no package README", "`j2k-native` is publishable but missing [package.metadata.docs.rs]", "`j2k-native` is publishable but has no library target", - "`j2k-profile` is listed in PUBLISHABLE_PACKAGES but is not a workspace member", + "`j2k-profile` is listed in release-crates.json but is not a workspace member", ] { assert!(error.contains(expected), "missing `{expected}` in {error}"); } diff --git a/xtask/src/release_commands/tests/orchestration.rs b/xtask/src/release_commands/tests/orchestration.rs index 0718eefd..0d07511d 100644 --- a/xtask/src/release_commands/tests/orchestration.rs +++ b/xtask/src/release_commands/tests/orchestration.rs @@ -2,8 +2,9 @@ use std::os::unix::fs::symlink; +use super::super::release_manifest::parse_release_manifest_source; use super::{ - super::{package, release_integrity, PUBLISHABLE_PACKAGES}, + super::{package, release_integrity}, integrity::{complete_publishable_metadata, metadata_program}, package_fixture::packaged_metadata, }; @@ -72,6 +73,10 @@ fn release_integrity_publish_mode_accepts_hermetic_final_metadata() { "scripts/publish_release.py", include_str!("../../../../scripts/publish_release.py"), ), + ( + "scripts/release_manifest.py", + include_str!("../../../../scripts/release_manifest.py"), + ), ( "release-crates.json", include_str!("../../../../release-crates.json"), @@ -81,12 +86,12 @@ fn release_integrity_publish_mode_accepts_hermetic_final_metadata() { } std::fs::write( release_root.join("Cargo.toml"), - "[workspace.package]\nversion = \"0.7.5\"\n\n[patch.crates-io]\nblock = { path = \"third_party/block-0.1.6-patched\" }\n", + "[workspace.package]\nversion = \"0.8.0\"\n\n[patch.crates-io]\nblock = { path = \"third_party/block-0.1.6-patched\" }\n", ) .expect("write workspace manifest fixture"); std::fs::write( release_root.join("CHANGELOG.md"), - "# Changelog\n\n## [0.7.5] - 2026-07-22\n", + "# Changelog\n\n## [0.8.0] - 2026-07-27\n", ) .expect("write finalized changelog fixture"); std::fs::write( @@ -139,13 +144,18 @@ fn package_command_executes_list_and_dependency_aware_gates_hermetically() { assert!(git.log().starts_with("status --porcelain|")); let cargo_log = cargo.log(); let commands = cargo_log.lines().collect::>(); + let publishable_count = + parse_release_manifest_source(include_str!("../../../../release-crates.json")) + .expect("checked-in release manifest") + .ordered_crates() + .len(); let consumer_commands = match std::env::consts::OS { "linux" | "macos" => 5, _ => 3, }; assert_eq!( commands.len(), - 1 + 2 * PUBLISHABLE_PACKAGES.len() + consumer_commands + 1 + 2 * publishable_count + consumer_commands ); assert!(commands[0].starts_with("metadata --locked --no-deps --format-version 1|")); assert!(commands[1].starts_with("package -p j2k-core --list|")); @@ -154,7 +164,7 @@ fn package_command_executes_list_and_dependency_aware_gates_hermetically() { .any(|line| line.starts_with("publish -p j2k-core --dry-run|"))); assert!(commands .iter() - .any(|line| line.starts_with("package -p j2k-cli --no-verify|"))); + .any(|line| line.starts_with("package -p j2k-cli --no-verify"))); assert!(commands .iter() .any(|line| line.contains("check --examples --no-default-features --features"))); diff --git a/xtask/src/release_commands/tests/validation.rs b/xtask/src/release_commands/tests/validation.rs index 8a6a8518..30c11f97 100644 --- a/xtask/src/release_commands/tests/validation.rs +++ b/xtask/src/release_commands/tests/validation.rs @@ -2,36 +2,253 @@ use std::collections::BTreeSet; +use super::super::release_manifest::{ + registry_independent_packages, validate_release_manifest_contract, +}; use super::super::{ parse_release_manifest_source, validate_publish_script_source, - validate_publish_workflow_source, validate_release_docs_source, - validate_unpublished_dependencies, + validate_publish_workflow_source, validate_python_release_manifest_source, + validate_release_docs_source, validate_unpublished_dependencies, }; #[test] -fn release_manifest_parser_requires_unique_ordered_crates_and_prefix_partition() { +fn release_manifest_parser_requires_schema_two_tiered_crate_records() { let manifest = parse_release_manifest_source( - r#"{"schema":1,"ordered_crates":["base","consumer"],"registry_independent":["base"]}"#, + r#"{ + "schema": 2, + "crates": [ + {"name": "stable-api", "api_contract": "stable"}, + {"name": "experimental-api", "api_contract": "experimental"}, + {"name": "implementation-spi", "api_contract": "implementation"}, + {"name": "cli", "api_contract": "binary"} + ] + }"#, ) .expect("valid release manifest"); - assert_eq!(manifest.ordered_crates, ["base", "consumer"]); - assert_eq!(manifest.registry_independent, ["base"]); + assert_eq!( + manifest.ordered_crates().collect::>(), + [ + "stable-api", + "experimental-api", + "implementation-spi", + "cli" + ] + ); + assert_eq!( + manifest + .api_contracts() + .map(|(name, contract)| (name, contract.as_str())) + .collect::>(), + [ + ("stable-api", "stable"), + ("experimental-api", "experimental"), + ("implementation-spi", "implementation"), + ("cli", "binary"), + ] + ); + assert_eq!( + manifest.library_packages().collect::>(), + ["stable-api", "experimental-api", "implementation-spi"] + ); + assert_eq!( + manifest + .api_contracts() + .filter_map(|(name, contract)| (contract.as_str() == "stable").then_some(name)) + .collect::>(), + ["stable-api"] + ); for (source, expected) in [ ( - r#"{"schema":1,"ordered_crates":["base","base"],"registry_independent":["base"]}"#, + r#"{"schema":1,"crates":[{"name":"base","api_contract":"stable"}]}"#, + "schema must be exactly 2", + ), + ( + r#"{"schema":2.0,"crates":[{"name":"base","api_contract":"stable"}]}"#, + "invalid type", + ), + ( + r#"{"schema":true,"crates":[{"name":"base","api_contract":"stable"}]}"#, + "invalid type", + ), + ( + r#"{"schema":2,"schema":2,"crates":[{"name":"base","api_contract":"stable"}]}"#, + "duplicate field", + ), + ( + r#"{"schema":2,"crates":[{"name":"base","api_contract":"stable"}],"extra":true}"#, + "unknown field", + ), + ( + r#"{"schema":2,"crates":[{"name":"base","api_contract":"stable"},{"name":"base","api_contract":"experimental"}]}"#, "duplicate", ), ( - r#"{"schema":1,"ordered_crates":["base","consumer"],"registry_independent":["consumer"]}"#, - "prefix", + r#"{"schema":2,"crates":[{"name":"base","api_contract":"unsupported"}]}"#, + "api_contract", + ), + ( + r#"{"schema":2,"crates":[{"name":"bad name","api_contract":"stable"}]}"#, + "malformed crate name", + ), + ( + r#"{"schema":2,"crates":[{"name":"base","api_contract":"stable","extra":true}]}"#, + "unknown field", ), + (r#"{"schema":2,"crates":[]}"#, "must not be empty"), ] { let error = parse_release_manifest_source(source).expect_err("invalid manifest rejects"); assert!(error.contains(expected), "unexpected error: {error}"); } } +fn release_package( + name: &str, + target_kind: &str, + dependencies: &serde_json::Value, +) -> serde_json::Value { + serde_json::json!({ + "id": name, + "name": name, + "version": "0.7.6", + "publish": null, + "manifest_path": format!("/workspace/{name}/Cargo.toml"), + "targets": [{"kind": [target_kind]}], + "dependencies": dependencies, + }) +} + +#[test] +fn release_manifest_validation_derives_order_and_registry_independence() { + let manifest = parse_release_manifest_source( + r#"{ + "schema": 2, + "crates": [ + {"name": "base", "api_contract": "stable"}, + {"name": "consumer", "api_contract": "experimental"}, + {"name": "cli", "api_contract": "binary"} + ] + }"#, + ) + .expect("valid release manifest"); + let packages = [ + release_package("base", "lib", &serde_json::json!([])), + release_package( + "consumer", + "lib", + &serde_json::json!([{ + "name": "base", + "kind": "build", + "optional": true, + "target": "cfg(unix)", + "path": "/workspace/base", + "req": "=0.7.6", + "source": null + }]), + ), + release_package( + "cli", + "bin", + &serde_json::json!([{ + "name": "consumer", + "kind": null, + "path": "/workspace/consumer", + "req": "=0.7.6", + "source": null + }]), + ), + ]; + let records = packages.iter().collect::>(); + let mut errors = Vec::new(); + + validate_release_manifest_contract(&manifest, &records, &mut errors) + .expect("well-formed release metadata"); + + assert!(errors.is_empty(), "unexpected errors: {errors:?}"); + assert_eq!( + registry_independent_packages(&manifest, &records).expect("derived registry independence"), + BTreeSet::from(["base".to_string()]) + ); +} + +#[test] +fn release_manifest_validation_rejects_non_workspace_and_inexact_sibling_edges() { + let manifest = parse_release_manifest_source( + r#"{ + "schema": 2, + "crates": [ + {"name": "base", "api_contract": "stable"}, + {"name": "consumer", "api_contract": "experimental"} + ] + }"#, + ) + .expect("valid release manifest"); + let mut packages = [ + release_package("base", "lib", &serde_json::json!([])), + release_package( + "consumer", + "lib", + &serde_json::json!([{ + "name": "base", + "kind": null, + "path": "/workspace/base", + "req": "^0.7.6", + "source": null + }]), + ), + ]; + let records = packages.iter().collect::>(); + let mut errors = Vec::new(); + + validate_release_manifest_contract(&manifest, &records, &mut errors) + .expect("structurally valid metadata"); + + assert!(errors.iter().any(|error| error.contains("exact release"))); + + packages[1]["dependencies"][0]["req"] = serde_json::json!("=0.7.6"); + packages[1]["dependencies"][0]["path"] = serde_json::Value::Null; + packages[1]["dependencies"][0]["source"] = + serde_json::json!("registry+https://github.com/rust-lang/crates.io-index"); + let records = packages.iter().collect::>(); + let error = validate_release_manifest_contract(&manifest, &records, &mut Vec::new()) + .expect_err("same-name registry dependency must reject"); + assert!( + error.contains("workspace/path sourced"), + "unexpected: {error}" + ); +} + +#[test] +fn release_manifest_validation_requires_one_release_version() { + let manifest = parse_release_manifest_source( + r#"{ + "schema": 2, + "crates": [ + {"name": "base", "api_contract": "stable"}, + {"name": "consumer", "api_contract": "experimental"} + ] + }"#, + ) + .expect("valid release manifest"); + let packages = [release_package("base", "lib", &serde_json::json!([])), { + let mut package = release_package("consumer", "lib", &serde_json::json!([])); + package["version"] = serde_json::json!("0.8.0"); + package + }]; + let records = packages.iter().collect::>(); + let mut errors = Vec::new(); + + validate_release_manifest_contract(&manifest, &records, &mut errors) + .expect("structurally valid metadata"); + + assert!( + errors + .iter() + .any(|error| error.contains("must share one release version")), + "unexpected: {errors:?}" + ); +} + #[test] fn publish_workflow_validation_reports_parse_and_release_contract_failures() { let parse_error = validate_publish_workflow_source("jobs: [", &mut Vec::new()) @@ -65,11 +282,31 @@ fn publish_script_validation_fails_closed_for_missing_and_drifted_contracts() { .any(|error| error.contains("must not treat ambiguous cargo-info failures"))); } +#[test] +fn python_release_manifest_validation_fails_closed_for_missing_graph_contracts() { + let mut errors = Vec::new(); + + validate_python_release_manifest_source("", &mut errors); + + assert!(errors + .iter() + .any(|error| error.contains("object_pairs_hook=_reject_duplicate_fields"))); + assert!(errors + .iter() + .any(|error| error.contains("must be workspace/path sourced"))); + assert!(errors + .iter() + .any(|error| error.contains("expected_requirement = f\"="))); + assert!(errors + .iter() + .any(|error| error.contains("if kind == \"dev\""))); +} + #[test] fn release_docs_validation_reports_missing_packages_and_operational_guards() { let mut errors = Vec::new(); - validate_release_docs_source("", &mut errors); + validate_release_docs_source("", &["j2k-core"], &mut errors); assert!(errors .iter() diff --git a/xtask/src/semver.rs b/xtask/src/semver.rs index 928797ca..f6f553b4 100644 --- a/xtask/src/semver.rs +++ b/xtask/src/semver.rs @@ -25,12 +25,25 @@ use compatibility::{semver_check_args, semver_check_release_type}; const CARGO_SEMVER_CHECKS_VERSION: &str = "0.48.0"; const SEMVER_TOOLCHAIN: &str = "1.96"; -const SEMVER_BASELINE_VERSION: &str = "0.7.3"; -const SEMVER_BASELINE_TAG: &str = "v0.7.3"; -const SEMVER_BASELINE_COMMIT: &str = "494eebc3ef20895d331da86221b1d8c4bd4cabf8"; -const SOURCE_INCOMPATIBLE_PATCH_EXCEPTION_VERSION: &str = "0.7.5"; -const API_DIFF_REPORT: &str = "engineering/reviewed-public-api-diff-0.7.5.md"; -const API_REVIEW_CONFIG: &str = "engineering/public-api-review-0.7.5.yml"; +const SEMVER_BASELINE_VERSION: &str = "0.7.5"; +const SEMVER_BASELINE_TAG: &str = "v0.7.5"; +const SEMVER_BASELINE_COMMIT: &str = "a89abb6e7eba469c44b3735740712c2a85be0499"; +const API_DIFF_REPORT: &str = "engineering/reviewed-public-api-diff-0.8.0.md"; +const API_REVIEW_CONFIG: &str = "engineering/public-api-review-0.8.0.yml"; + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +struct BaselineTransition<'a> { + candidate_version: &'a str, + required_next_baseline_version: &'a str, + required_next_baseline_tag: &'a str, +} + +const INTENTIONAL_BREAK_TRANSITION: Option> = + Some(BaselineTransition { + candidate_version: "0.8.0", + required_next_baseline_version: "0.8.0", + required_next_baseline_tag: "v0.8.0", + }); const SEMVER_BASELINE_PACKAGES: &[&str] = &[ "j2k", @@ -50,9 +63,10 @@ const SEMVER_BASELINE_PACKAGES: &[&str] = &[ "j2k-types", "j2k-cuda-runtime", "j2k-profile", + "j2k-ml", ]; -const SEMVER_NEW_PACKAGES: &[&str] = &["j2k-ml"]; +const SEMVER_NEW_PACKAGES: &[&str] = &[]; #[derive(Clone, Copy, Debug, Eq, PartialEq, Ord, PartialOrd)] struct Version { @@ -172,22 +186,27 @@ struct Options { pub(crate) fn semver( args: impl Iterator, - stable_packages: &[&str], + published_library_packages: &[&str], cargo_public_api_version: &str, ) -> Result<(), String> { let options = parse_options(args)?; require_macos()?; verify_tool_versions(cargo_public_api_version)?; verify_baseline_tag()?; - validate_package_partition(stable_packages)?; + validate_package_partition(published_library_packages)?; let versions = workspace_package_versions()?; - let candidate_version = common_candidate_version(stable_packages, &versions)?; + let candidate_version = common_candidate_version(published_library_packages, &versions)?; + validate_baseline_transition( + SEMVER_BASELINE_VERSION, + &candidate_version, + INTENTIONAL_BREAK_TRANSITION, + )?; let baseline = Version::parse(SEMVER_BASELINE_VERSION)?; let baseline_snapshot = baseline_api_snapshot(cargo_public_api_version)?; let baseline_apis = parse_api_snapshot(&baseline_snapshot)?; validate_snapshot_scope( - "published 0.7.3 ordinary snapshot", + "published 0.7.5 ordinary snapshot", SEMVER_BASELINE_PACKAGES, &baseline_apis, )?; @@ -205,15 +224,15 @@ pub(crate) fn semver( let snapshot_hidden_apis = parse_api_snapshot(&hidden_snapshot)?; validate_snapshot_scope( "candidate ordinary snapshot", - stable_packages, + published_library_packages, &snapshot_apis, )?; validate_snapshot_scope( "candidate rustdoc-hidden snapshot", - stable_packages, + published_library_packages, &snapshot_hidden_apis, )?; - let live_inventories = collect_package_apis(stable_packages)?; + let live_inventories = collect_package_apis(published_library_packages)?; let current_apis = live_inventories .iter() .map(|(package, inventory)| (package.clone(), inventory.ordinary.clone())) @@ -222,12 +241,16 @@ pub(crate) fn semver( .iter() .map(|(package, inventory)| (package.clone(), inventory.hidden.clone())) .collect::>(); - let stale_ordinary_packages = snapshot_drift(stable_packages, &snapshot_apis, ¤t_apis); - let stale_hidden_packages = - snapshot_drift(stable_packages, &snapshot_hidden_apis, ¤t_hidden_apis); + let stale_ordinary_packages = + snapshot_drift(published_library_packages, &snapshot_apis, ¤t_apis); + let stale_hidden_packages = snapshot_drift( + published_library_packages, + &snapshot_hidden_apis, + ¤t_hidden_apis, + ); let diffs = build_package_diffs( - stable_packages, + published_library_packages, &versions, baseline, &baseline_apis, @@ -237,7 +260,7 @@ pub(crate) fn semver( if !stale_ordinary_packages.is_empty() || !stale_hidden_packages.is_empty() { return Err(format!( - "committed stable API snapshots are stale; ordinary packages: \ + "committed published-library API snapshots are stale; ordinary packages: \ {stale_ordinary_packages:?}; rustdoc-hidden packages: {stale_hidden_packages:?}; \ run `cargo xtask stable-api --write` and review both inventory diffs before \ regenerating the semver report" @@ -253,8 +276,47 @@ pub(crate) fn semver( Ok(()) } +fn validate_baseline_transition( + baseline_version: &str, + candidate_version: &str, + transition: Option>, +) -> Result<(), String> { + let Some(transition) = transition else { + return Ok(()); + }; + if transition.required_next_baseline_version != transition.candidate_version { + return Err(format!( + "intentional-break transition candidate {} must equal its required next baseline \ + version {}, so later 0.8.x checks cannot keep using the older baseline", + transition.candidate_version, transition.required_next_baseline_version + )); + } + let expected_tag = format!("v{}", transition.required_next_baseline_version); + if transition.required_next_baseline_tag != expected_tag { + return Err(format!( + "intentional-break transition requires next baseline tag `{expected_tag}`, found `{}`", + transition.required_next_baseline_tag + )); + } + let baseline = Version::parse(baseline_version)?; + let transition_candidate = Version::parse(transition.candidate_version)?; + computed_release_type(baseline, transition_candidate)?; + if candidate_version != transition.candidate_version { + return Err(format!( + "the active intentional-break transition permits only candidate {} against baseline \ + {baseline_version}, found {candidate_version}; after publishing {}, rotate the \ + semver baseline to {} and disable INTENTIONAL_BREAK_TRANSITION before preparing a \ + later release", + transition.candidate_version, + transition.candidate_version, + transition.required_next_baseline_tag + )); + } + Ok(()) +} + fn build_package_diffs( - stable_packages: &[&str], + published_library_packages: &[&str], versions: &BTreeMap, baseline: Version, baseline_apis: &BTreeMap>, @@ -265,16 +327,16 @@ fn build_package_diffs( .iter() .copied() .collect::>(); - let mut diffs = Vec::with_capacity(stable_packages.len()); - for package in stable_packages { + let mut diffs = Vec::with_capacity(published_library_packages.len()); + for package in published_library_packages { let current = current_apis.get(*package).cloned().ok_or_else(|| { - format!("current public API snapshot is missing stable package `{package}`") + format!("current public API snapshot is missing published library `{package}`") + })?; + let candidate = versions.get(*package).ok_or_else(|| { + format!("workspace metadata is missing published library `{package}`") })?; - let candidate = versions - .get(*package) - .ok_or_else(|| format!("workspace metadata is missing stable package `{package}`"))?; let hidden = current_hidden_apis.get(*package).cloned().ok_or_else(|| { - format!("current rustdoc-hidden API inventory is missing stable package `{package}`") + format!("current rustdoc-hidden API inventory is missing published library `{package}`") })?; if baseline_set.contains(package) { let baseline_api = baseline_apis.get(*package).ok_or_else(|| { @@ -405,10 +467,13 @@ fn require_version_token(output: &str, tool: &str, expected: &str) -> Result<(), } } -fn validate_package_partition(stable_packages: &[&str]) -> Result<(), String> { - let stable = stable_packages.iter().copied().collect::>(); - if stable.len() != stable_packages.len() { - return Err("STABLE_SEMVER_PACKAGES contains duplicates".to_string()); +fn validate_package_partition(published_library_packages: &[&str]) -> Result<(), String> { + let published = published_library_packages + .iter() + .copied() + .collect::>(); + if published.len() != published_library_packages.len() { + return Err("published library package set contains duplicates".to_string()); } let baseline = SEMVER_BASELINE_PACKAGES .iter() @@ -428,13 +493,19 @@ fn validate_package_partition(stable_packages: &[&str]) -> Result<(), String> { )); } let configured = baseline.union(&new).copied().collect::>(); - if configured == stable { + if configured == published { Ok(()) } else { - let missing = stable.difference(&configured).copied().collect::>(); - let unexpected = configured.difference(&stable).copied().collect::>(); + let missing = published + .difference(&configured) + .copied() + .collect::>(); + let unexpected = configured + .difference(&published) + .copied() + .collect::>(); Err(format!( - "semver baseline/new package partition drifted; missing: {missing:?}; unexpected: {unexpected:?}" + "published-library semver baseline/new partition drifted; missing: {missing:?}; unexpected: {unexpected:?}" )) } } @@ -464,26 +535,25 @@ fn workspace_package_versions() -> Result, String> { } fn common_candidate_version( - stable_packages: &[&str], + published_library_packages: &[&str], versions: &BTreeMap, ) -> Result { - let mut candidates = stable_packages + let mut candidates = published_library_packages .iter() .map(|package| { - versions - .get(*package) - .cloned() - .ok_or_else(|| format!("workspace metadata is missing stable package `{package}`")) + versions.get(*package).cloned().ok_or_else(|| { + format!("workspace metadata is missing published library `{package}`") + }) }) .collect::, _>>()?; if candidates.len() != 1 { return Err(format!( - "stable semver packages must share one candidate version, found {candidates:?}" + "published library packages must share one candidate version, found {candidates:?}" )); } candidates .pop_first() - .ok_or_else(|| "stable semver package list is empty".to_string()) + .ok_or_else(|| "published library package list is empty".to_string()) } fn baseline_api_snapshot(cargo_public_api_version: &str) -> Result { @@ -568,12 +638,15 @@ fn exact_metadata_line(lines: &[&str], prefix: &str, expected: &str) -> bool { } fn snapshot_drift( - stable_packages: &[&str], + published_library_packages: &[&str], snapshot_apis: &BTreeMap>, current_apis: &BTreeMap>, ) -> Vec { - let expected = stable_packages.iter().copied().collect::>(); - let mut drift = stable_packages + let expected = published_library_packages + .iter() + .copied() + .collect::>(); + let mut drift = published_library_packages .iter() .copied() .filter(|package| snapshot_apis.get(*package) != current_apis.get(*package)) @@ -713,10 +786,18 @@ fn render_report( ) .unwrap(); writeln!(&mut out, "- Candidate version: `{candidate_version}`").unwrap(); - if candidate_version == SOURCE_INCOMPATIBLE_PATCH_EXCEPTION_VERSION { + if let Some(transition) = INTENTIONAL_BREAK_TRANSITION { + writeln!( + &mut out, + "- Active intentional-break transition: only `{}` may compare against \ + `{SEMVER_BASELINE_TAG}`", + transition.candidate_version + ) + .unwrap(); writeln!( &mut out, - "- Compatibility exception: `{candidate_version}` is an explicitly reviewed source-incompatible patch candidate; compatibility checks use `major` while package versions remain `{candidate_version}`." + "- Required next semver baseline: `{}` at version `{}` before any later candidate", + transition.required_next_baseline_tag, transition.required_next_baseline_version ) .unwrap(); } diff --git a/xtask/src/semver/compatibility.rs b/xtask/src/semver/compatibility.rs index 214f7810..c4e82159 100644 --- a/xtask/src/semver/compatibility.rs +++ b/xtask/src/semver/compatibility.rs @@ -4,10 +4,7 @@ use std::ffi::OsString; use crate::process::{self, CommandContext}; -use super::{ - semver_cargo_args, PackageApiDiff, ReleaseType, SEMVER_BASELINE_VERSION, - SOURCE_INCOMPATIBLE_PATCH_EXCEPTION_VERSION, -}; +use super::{semver_cargo_args, PackageApiDiff, ReleaseType, SEMVER_BASELINE_VERSION}; pub(super) fn run_semver_checks(diffs: &[PackageApiDiff]) -> Result<(), String> { for diff in diffs.iter().filter(|diff| diff.release_type.is_some()) { @@ -35,9 +32,5 @@ pub(super) fn semver_check_args(diff: &PackageApiDiff) -> Vec { } pub(super) fn semver_check_release_type(diff: &PackageApiDiff) -> ReleaseType { - if diff.candidate_version == SOURCE_INCOMPATIBLE_PATCH_EXCEPTION_VERSION { - ReleaseType::Major - } else { - diff.release_type.expect("published diff release type") - } + diff.release_type.expect("published diff release type") } diff --git a/xtask/src/semver/review.rs b/xtask/src/semver/review.rs index ffc78a50..8e9c3062 100644 --- a/xtask/src/semver/review.rs +++ b/xtask/src/semver/review.rs @@ -12,6 +12,28 @@ use super::{ SEMVER_BASELINE_VERSION, }; +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub(super) enum BreakKind { + Source, + Behavior, +} + +#[derive(Debug)] +pub(super) struct RemovedApiItem { + pub(super) package: String, + pub(super) item: String, +} + +#[derive(Debug)] +pub(super) struct BreakLedgerEntry { + pub(super) id: String, + pub(super) kind: BreakKind, + pub(super) packages: BTreeSet, + pub(super) removed_items: Vec, + pub(super) summary: String, + pub(super) migration: String, +} + #[derive(Debug)] pub(super) struct ReviewEntry { pub(super) removed_fingerprint: String, @@ -25,6 +47,7 @@ pub(super) struct ReviewEntry { #[derive(Debug)] pub(super) struct ReviewConfig { pub(super) candidate_version: String, + pub(super) break_ledger: Vec, pub(super) reviews: BTreeMap, } @@ -47,16 +70,32 @@ pub(super) fn parse_review_config(value: &serde_yaml_ng::Value) -> Result Result Result { + reject_unknown_keys( + entry, + &[ + "id", + "kind", + "packages", + "removed_items", + "summary", + "migration", + ], + "API break-ledger entry", + )?; + let id = required_string(entry, "id")?.to_string(); + if !valid_break_id(&id) { + return Err(format!( + "API break-ledger id `{id}` must contain only lowercase ASCII letters, digits, and \ + interior hyphens" + )); + } + let kind = match required_string(entry, "kind")? { + "source" => BreakKind::Source, + "behavior" => BreakKind::Behavior, + other => { + return Err(format!( + "API break-ledger entry `{id}` has unknown kind `{other}`; expected `source` or \ + `behavior`" + )) + } + }; + let packages = parse_break_packages(entry, &id)?; + let removed_items = parse_removed_api_items(entry, &id, &packages)?; + validate_break_kind_shape(&id, kind, &packages, &removed_items)?; + let summary = reviewed_break_text(entry, "summary", &id)?; + let migration = reviewed_break_text(entry, "migration", &id)?; + Ok(BreakLedgerEntry { + id, + kind, + packages, + removed_items, + summary, + migration, + }) +} + +fn parse_break_packages( + entry: &serde_yaml_ng::Mapping, + id: &str, +) -> Result, String> { + let package_values = required_value(entry, "packages")? + .as_sequence() + .ok_or_else(|| format!("API break-ledger entry `{id}` packages must be a sequence"))?; + let mut packages = BTreeSet::new(); + for package in package_values { + let package = package.as_str().ok_or_else(|| { + format!("API break-ledger entry `{id}` package names must be strings") + })?; + if package.is_empty() || package.trim() != package { + return Err(format!( + "API break-ledger entry `{id}` contains an empty or whitespace-padded package name" + )); + } + if !packages.insert(package.to_string()) { + return Err(format!( + "API break-ledger entry `{id}` contains duplicate package `{package}`" + )); + } + } + if packages.is_empty() { + return Err(format!( + "API break-ledger entry `{id}` must name at least one affected package" + )); + } + Ok(packages) +} + +fn parse_removed_api_items( + entry: &serde_yaml_ng::Mapping, + id: &str, + packages: &BTreeSet, +) -> Result, String> { + let removed_values = required_value(entry, "removed_items")? + .as_sequence() + .ok_or_else(|| format!("API break-ledger entry `{id}` removed_items must be a sequence"))?; + let mut removed_items = Vec::with_capacity(removed_values.len()); + let mut unique_removed_items = BTreeSet::new(); + for value in removed_values { + let removed = value.as_mapping().ok_or_else(|| { + format!("API break-ledger entry `{id}` removed items must be mappings") + })?; + reject_unknown_keys( + removed, + &["package", "item"], + &format!("removed item in API break-ledger entry `{id}`"), + )?; + let package = required_string(removed, "package")?.to_string(); + let item = required_string(removed, "item")?.to_string(); + if !packages.contains(&package) { + return Err(format!( + "API break-ledger entry `{id}` records a removed item for unlisted package \ + `{package}`" + )); + } + if item.is_empty() || item.trim() != item { + return Err(format!( + "API break-ledger entry `{id}` contains an empty or whitespace-padded API item" + )); + } + if !unique_removed_items.insert((package.clone(), item.clone())) { + return Err(format!( + "API break-ledger entry `{id}` repeats removed item `{item}` for `{package}`" + )); + } + removed_items.push(RemovedApiItem { package, item }); + } + Ok(removed_items) +} + +fn validate_break_kind_shape( + id: &str, + kind: BreakKind, + packages: &BTreeSet, + removed_items: &[RemovedApiItem], +) -> Result<(), String> { + match kind { + BreakKind::Source if removed_items.is_empty() => { + return Err(format!( + "source break-ledger entry `{id}` must enumerate at least one removed API item" + )) + } + BreakKind::Source => { + let packages_with_items = removed_items + .iter() + .map(|removed| removed.package.as_str()) + .collect::>(); + let packages_without_items = packages + .iter() + .filter(|package| !packages_with_items.contains(package.as_str())) + .collect::>(); + if !packages_without_items.is_empty() { + return Err(format!( + "source break-ledger entry `{id}` names packages without removed items: \ + {packages_without_items:?}" + )); + } + } + BreakKind::Behavior if !removed_items.is_empty() => { + return Err(format!( + "behavior break-ledger entry `{id}` must not contain removed API items; record \ + source removals in a separate `source` entry" + )) + } + BreakKind::Behavior => {} + } + Ok(()) +} + +fn valid_break_id(id: &str) -> bool { + !id.is_empty() + && id + .bytes() + .all(|byte| byte.is_ascii_lowercase() || byte.is_ascii_digit() || byte == b'-') + && id.as_bytes().first().is_some_and(u8::is_ascii_alphanumeric) + && id.as_bytes().last().is_some_and(u8::is_ascii_alphanumeric) +} + +fn reviewed_break_text( + entry: &serde_yaml_ng::Mapping, + field: &str, + id: &str, +) -> Result { + let value = required_string(entry, field)?.trim().to_string(); + if value.len() < 20 { + return Err(format!( + "API break-ledger {field} for `{id}` must be at least 20 characters" + )); + } + if contains_pending_review_marker(&value) { + return Err(format!( + "API break-ledger {field} for `{id}` is still pending maintainer review" + )); + } + Ok(value) +} + fn parse_review_entry( package: &str, entry: &serde_yaml_ng::Mapping, @@ -164,64 +389,14 @@ pub(super) fn validate_reviews( config.candidate_version )); } - for diff in diffs { - let Some(review) = config.reviews.get(&diff.package) else { - errors.push(format!( - "`{}` has no review entry covering its ordinary diff and hidden inventory", - diff.package - )); - continue; - }; - let removed_fingerprint = diff.removed_fingerprint(); - if review.removed_fingerprint != removed_fingerprint { - errors.push(format!( - "`{}` removed/changed fingerprint is {removed_fingerprint}, but its review \ - records {}", - diff.package, review.removed_fingerprint - )); - } - let added_fingerprint = diff.added_fingerprint(); - if review.added_fingerprint != added_fingerprint { - errors.push(format!( - "`{}` added fingerprint is {added_fingerprint}, but its review records {}", - diff.package, review.added_fingerprint - )); - } - if review.hidden_count != diff.hidden.len() { - errors.push(format!( - "`{}` hidden inventory count is {}, but its review records {}", - diff.package, - diff.hidden.len(), - review.hidden_count - )); - } - let hidden_fingerprint = diff.hidden_fingerprint(); - if review.hidden_fingerprint != hidden_fingerprint { - errors.push(format!( - "`{}` hidden inventory fingerprint is {hidden_fingerprint}, but its review \ - records {}", - diff.package, review.hidden_fingerprint - )); - } - if review.rationale.trim().len() < 20 { - errors.push(format!("`{}` review rationale is too short", diff.package)); - } - if !diff.hidden.is_empty() - && review - .hidden_rationale - .as_deref() - .is_none_or(|rationale| rationale.trim().len() < 20) - { - errors.push(format!( - "`{}` nonempty hidden inventory lacks a package-specific rationale", - diff.package - )); - } - } let known_packages = diffs .iter() .map(|diff| diff.package.as_str()) .collect::>(); + let ledger_removed = validate_break_ledger(config, &known_packages, &mut errors); + for diff in diffs { + validate_package_review(config, diff, &ledger_removed, &mut errors); + } for package in config.reviews.keys() { if !known_packages.contains(package.as_str()) { errors.push(format!( @@ -233,12 +408,141 @@ pub(super) fn validate_reviews( Ok(()) } else { Err(format!( - "unreviewed or stale public API changes:\n- {}\nReview {API_DIFF_REPORT} and both API snapshot diffs, then update {API_REVIEW_CONFIG} with every exact generated fingerprint/count and the required package-specific rationales.", + "unreviewed or stale public API changes:\n- {}\nReview {API_DIFF_REPORT} and both API snapshot diffs, then update {API_REVIEW_CONFIG} with every exact generated fingerprint/count, every removed item in the source-break ledger, and the required package-specific rationales and migrations.", errors.join("\n- ") )) } } +fn validate_break_ledger( + config: &ReviewConfig, + known_packages: &BTreeSet<&str>, + errors: &mut Vec, +) -> BTreeMap> { + let mut ledger_removed = BTreeMap::>::new(); + let mut ledger_owners = BTreeMap::<(String, String), String>::new(); + for entry in &config.break_ledger { + if entry.summary.trim().len() < 20 || entry.migration.trim().len() < 20 { + errors.push(format!( + "break-ledger entry `{}` lacks a substantive summary or migration", + entry.id + )); + } + match entry.kind { + BreakKind::Source if entry.removed_items.is_empty() => errors.push(format!( + "source break-ledger entry `{}` has no removed API items", + entry.id + )), + BreakKind::Behavior if !entry.removed_items.is_empty() => errors.push(format!( + "behavior break-ledger entry `{}` contains removed API items", + entry.id + )), + BreakKind::Source | BreakKind::Behavior => {} + } + for package in &entry.packages { + if !known_packages.contains(package.as_str()) { + errors.push(format!( + "break-ledger entry `{}` contains unknown package `{package}`", + entry.id + )); + } + } + for removed in &entry.removed_items { + let key = (removed.package.clone(), removed.item.clone()); + if let Some(previous) = ledger_owners.insert(key, entry.id.clone()) { + errors.push(format!( + "removed API item `{}` for `{}` appears in both source-break entries \ + `{previous}` and `{}`", + removed.item, removed.package, entry.id + )); + } + ledger_removed + .entry(removed.package.clone()) + .or_default() + .insert(removed.item.clone()); + } + } + ledger_removed +} + +fn validate_package_review( + config: &ReviewConfig, + diff: &PackageApiDiff, + ledger_removed: &BTreeMap>, + errors: &mut Vec, +) { + let Some(review) = config.reviews.get(&diff.package) else { + errors.push(format!( + "`{}` has no review entry covering its ordinary diff and hidden inventory", + diff.package + )); + return; + }; + let removed_fingerprint = diff.removed_fingerprint(); + if review.removed_fingerprint != removed_fingerprint { + errors.push(format!( + "`{}` removed/changed fingerprint is {removed_fingerprint}, but its review records {}", + diff.package, review.removed_fingerprint + )); + } + let added_fingerprint = diff.added_fingerprint(); + if review.added_fingerprint != added_fingerprint { + errors.push(format!( + "`{}` added fingerprint is {added_fingerprint}, but its review records {}", + diff.package, review.added_fingerprint + )); + } + if review.hidden_count != diff.hidden.len() { + errors.push(format!( + "`{}` hidden inventory count is {}, but its review records {}", + diff.package, + diff.hidden.len(), + review.hidden_count + )); + } + let hidden_fingerprint = diff.hidden_fingerprint(); + if review.hidden_fingerprint != hidden_fingerprint { + errors.push(format!( + "`{}` hidden inventory fingerprint is {hidden_fingerprint}, but its review records {}", + diff.package, review.hidden_fingerprint + )); + } + if review.rationale.trim().len() < 20 { + errors.push(format!("`{}` review rationale is too short", diff.package)); + } + if !diff.hidden.is_empty() + && review + .hidden_rationale + .as_deref() + .is_none_or(|rationale| rationale.trim().len() < 20) + { + errors.push(format!( + "`{}` nonempty hidden inventory lacks a package-specific rationale", + diff.package + )); + } + let reviewed_removed = ledger_removed + .get(&diff.package) + .cloned() + .unwrap_or_default(); + let unlisted = diff + .removed + .difference(&reviewed_removed) + .cloned() + .collect::>(); + let stale = reviewed_removed + .difference(&diff.removed) + .cloned() + .collect::>(); + if !unlisted.is_empty() || !stale.is_empty() { + errors.push(format!( + "`{}` source-break ledger does not exactly cover its removed API items; unlisted: \ + {unlisted:?}; stale: {stale:?}", + diff.package + )); + } +} + fn required_value<'a>( mapping: &'a serde_yaml_ng::Mapping, key: &str, diff --git a/xtask/src/semver/tests.rs b/xtask/src/semver/tests.rs index fc88394b..5411ee00 100644 --- a/xtask/src/semver/tests.rs +++ b/xtask/src/semver/tests.rs @@ -2,12 +2,16 @@ use std::collections::{BTreeMap, BTreeSet}; -use super::review::{parse_review_config, validate_reviews, ReviewConfig, ReviewEntry}; +use super::review::{ + parse_review_config, validate_reviews, BreakKind, BreakLedgerEntry, RemovedApiItem, + ReviewConfig, ReviewEntry, +}; use super::{ computed_release_type, current_snapshot_uses_generation_contract, fingerprint, parse_api_snapshot, parse_options, render_report, semver_cargo_args, semver_check_release_type, - snapshot_uses_generator, validate_package_partition, PackageApiDiff, ReleaseType, SnapshotKind, - Version, SEMVER_BASELINE_PACKAGES, SEMVER_NEW_PACKAGES, + snapshot_uses_generator, validate_baseline_transition, validate_package_partition, + BaselineTransition, PackageApiDiff, ReleaseType, SnapshotKind, Version, + SEMVER_BASELINE_PACKAGES, SEMVER_NEW_PACKAGES, }; mod api_planning; @@ -65,12 +69,40 @@ fn release_type_uses_cargo_pre_one_compatibility_rules() { } #[test] -fn package_partition_lists_new_packages_explicitly() { - let mut stable = SEMVER_BASELINE_PACKAGES.to_vec(); - stable.extend_from_slice(SEMVER_NEW_PACKAGES); - assert!(validate_package_partition(&stable).is_ok()); - stable.pop(); - assert!(validate_package_partition(&stable).is_err()); +fn intentional_break_transition_is_exact_and_names_the_next_baseline() { + let transition = BaselineTransition { + candidate_version: "0.8.0", + required_next_baseline_version: "0.8.0", + required_next_baseline_tag: "v0.8.0", + }; + assert_eq!( + validate_baseline_transition("0.7.5", "0.8.0", Some(transition)), + Ok(()) + ); + + let error = validate_baseline_transition("0.7.5", "0.8.1", Some(transition)).unwrap_err(); + assert!(error.contains("only candidate 0.8.0")); + assert!(error.contains("rotate the semver baseline to v0.8.0")); + + let inconsistent = BaselineTransition { + required_next_baseline_version: "0.8.1", + ..transition + }; + let error = validate_baseline_transition("0.7.5", "0.8.0", Some(inconsistent)).unwrap_err(); + assert!(error.contains("required next baseline version")); + + assert_eq!(validate_baseline_transition("0.8.0", "0.8.1", None), Ok(())); +} + +#[test] +fn package_partition_treats_every_published_package_as_baseline() { + assert!(SEMVER_NEW_PACKAGES.is_empty()); + assert!(SEMVER_BASELINE_PACKAGES.contains(&"j2k-ml")); + assert!(validate_package_partition(SEMVER_BASELINE_PACKAGES).is_ok()); + + let mut incomplete = SEMVER_BASELINE_PACKAGES.to_vec(); + incomplete.pop(); + assert!(validate_package_partition(&incomplete).is_err()); } #[test] @@ -157,11 +189,11 @@ fn semver_checks_commands_use_the_pinned_rustup_toolchain() { } #[test] -fn source_incompatible_patch_exception_is_scoped_to_0_7_5() { - let mut diff = PackageApiDiff { +fn published_candidate_uses_its_computed_release_type() { + let diff = PackageApiDiff { package: "alpha".to_string(), - candidate_version: "0.7.5".to_string(), - release_type: Some(ReleaseType::Minor), + candidate_version: "0.8.0".to_string(), + release_type: Some(ReleaseType::Major), baseline_count: 1, candidate_count: 0, added: BTreeSet::new(), @@ -171,8 +203,6 @@ fn source_incompatible_patch_exception_is_scoped_to_0_7_5() { hidden: BTreeSet::new(), }; assert_eq!(semver_check_release_type(&diff), ReleaseType::Major); - diff.candidate_version = "0.7.6".to_string(); - assert_eq!(semver_check_release_type(&diff), ReleaseType::Minor); } #[test] @@ -191,15 +221,31 @@ fn report_has_one_published_details_section_and_hidden_evidence() { assert_eq!(report.matches("## Published-package details").count(), 1); assert!(report.contains("Rustdoc-hidden candidate items: 1")); assert!(report.contains("Full hidden-inventory fingerprint: `fnv1a64:")); + assert!(report.contains("Required next semver baseline: `v0.8.0`")); } #[test] fn parses_review_config_and_rejects_unknown_fields() { let source = "\ -version: 2 -baseline_tag: v0.7.3 -baseline_version: 0.7.3 -candidate_version: 0.7.4 +version: 3 +baseline_tag: v0.7.5 +baseline_version: 0.7.5 +candidate_version: 0.8.0 +break_ledger: + - id: strict-decode-default + kind: behavior + packages: [j2k, j2k-native] + removed_items: [] + summary: 'Decode entry points now reject malformed metadata by default.' + migration: 'Select lenient settings explicitly for documented metadata recovery.' + - id: renamed-warning + kind: source + packages: [j2k] + removed_items: + - package: j2k + item: 'pub variant j2k::DecodeWarning::LenientDecodeMode' + summary: 'The warning variant now identifies actual metadata recovery.' + migration: 'Match LenientMetadataRecovery instead of LenientDecodeMode.' reviews: alpha: removed_fingerprint: 'fnv1a64:1234' @@ -211,7 +257,10 @@ reviews: "; let value: serde_yaml_ng::Value = serde_yaml_ng::from_str(source).unwrap(); let parsed = parse_review_config(&value).unwrap(); - assert_eq!(parsed.candidate_version, "0.7.4"); + assert_eq!(parsed.candidate_version, "0.8.0"); + assert_eq!(parsed.break_ledger.len(), 2); + assert_eq!(parsed.break_ledger[0].kind, BreakKind::Behavior); + assert_eq!(parsed.break_ledger[1].kind, BreakKind::Source); assert_eq!(parsed.reviews.len(), 1); let invalid = source.replacen(" rationale:", " unknown: nope\n rationale:", 1); @@ -225,6 +274,19 @@ reviews: let value: serde_yaml_ng::Value = serde_yaml_ng::from_str(&pending).unwrap(); let error = parse_review_config(&value).unwrap_err(); assert!(error.contains("pending maintainer review")); + + let duplicate_id = source.replace(" - id: renamed-warning", " - id: strict-decode-default"); + let value: serde_yaml_ng::Value = serde_yaml_ng::from_str(&duplicate_id).unwrap(); + let error = parse_review_config(&value).unwrap_err(); + assert!(error.contains("duplicate break-ledger id")); + + let behavior_with_removed_item = source.replace( + " removed_items: []", + " removed_items:\n - package: j2k\n item: 'pub fn j2k::old()'", + ); + let value: serde_yaml_ng::Value = serde_yaml_ng::from_str(&behavior_with_removed_item).unwrap(); + let error = parse_review_config(&value).unwrap_err(); + assert!(error.contains("behavior break")); } #[test] @@ -247,6 +309,7 @@ fn unreviewed_and_stale_breaking_fingerprints_fail() { }; let empty = ReviewConfig { candidate_version: "0.7.0".to_string(), + break_ledger: Vec::new(), reviews: BTreeMap::new(), }; assert!(validate_reviews(&empty, "0.7.0", &[diff]).is_err()); @@ -283,7 +346,63 @@ fn unreviewed_and_stale_breaking_fingerprints_fail() { .collect(); let stale = ReviewConfig { candidate_version: "0.7.0".to_string(), + break_ledger: Vec::new(), reviews, }; assert!(validate_reviews(&stale, "0.7.0", &[diff]).is_err()); } + +#[test] +fn source_break_ledger_exactly_covers_removed_api_items() { + let removed_item = "pub fn alpha::old()".to_string(); + let removed = [removed_item.clone()].into_iter().collect::>(); + let diff = PackageApiDiff { + package: "alpha".to_string(), + candidate_version: "0.8.0".to_string(), + release_type: Some(ReleaseType::Major), + baseline_count: 1, + candidate_count: 0, + added: BTreeSet::new(), + removed: removed.clone(), + hidden: BTreeSet::new(), + }; + let reviews = [( + "alpha".to_string(), + ReviewEntry { + removed_fingerprint: fingerprint(&removed), + added_fingerprint: "none".to_string(), + hidden_count: 0, + hidden_fingerprint: "none".to_string(), + rationale: "Intentional reviewed compatibility change.".to_string(), + hidden_rationale: None, + }, + )] + .into_iter() + .collect(); + let source_break = BreakLedgerEntry { + id: "remove-old".to_string(), + kind: BreakKind::Source, + packages: ["alpha".to_string()].into_iter().collect(), + removed_items: vec![RemovedApiItem { + package: "alpha".to_string(), + item: removed_item, + }], + summary: "The obsolete alpha entry point is removed.".to_string(), + migration: "Call the replacement alpha entry point instead.".to_string(), + }; + let exact = ReviewConfig { + candidate_version: "0.8.0".to_string(), + break_ledger: vec![source_break], + reviews, + }; + assert_eq!( + validate_reviews(&exact, "0.8.0", std::slice::from_ref(&diff)), + Ok(()) + ); + + let mut missing = exact; + missing.break_ledger.clear(); + let error = validate_reviews(&missing, "0.8.0", std::slice::from_ref(&diff)).unwrap_err(); + assert!(error.contains("source-break ledger")); + assert!(error.contains("pub fn alpha::old()")); +} diff --git a/xtask/src/semver/tests/api_planning.rs b/xtask/src/semver/tests/api_planning.rs index 9aafc902..556a8096 100644 --- a/xtask/src/semver/tests/api_planning.rs +++ b/xtask/src/semver/tests/api_planning.rs @@ -26,7 +26,7 @@ fn common_candidate_version_rejects_empty_missing_and_mixed_sets() { .contains("share one candidate version")); assert!(common_candidate_version(&["missing"], &versions) .expect_err("missing version") - .contains("missing stable package")); + .contains("missing published library")); assert!(common_candidate_version(&["alpha", "beta"], &versions) .expect_err("mixed versions") .contains("share one candidate version")); @@ -89,7 +89,7 @@ fn package_diff_planning_distinguishes_published_and_new_packages() { assert!(diffs[1].removed.is_empty()); let report = render_report("0.7.4", &diffs, "0.52.0"); - assert!(report.contains("## New packages without a 0.7.3 registry baseline")); + assert!(report.contains("## New packages without a 0.7.5 registry baseline")); assert!(report.contains("- `j2k-future` `0.7.4`: 1 ordinary public API items")); assert!(report.contains("### `j2k`")); assert!(report.contains("```text\nremoved\n```")); diff --git a/xtask/src/semver/tests/command_boundaries.rs b/xtask/src/semver/tests/command_boundaries.rs index bdea959c..0b6abff5 100644 --- a/xtask/src/semver/tests/command_boundaries.rs +++ b/xtask/src/semver/tests/command_boundaries.rs @@ -74,7 +74,7 @@ fn committed_candidate_semver_inputs_match_the_pinned_workspace_contract() { assert!(hidden.starts_with("# J2K 1.0 Rustdoc-Hidden Public API Snapshot")); let versions = workspace_package_versions().expect("workspace package versions"); - assert_eq!(versions.get("j2k").map(String::as_str), Some("0.7.5")); + assert_eq!(versions.get("j2k").map(String::as_str), Some("0.8.0")); assert!(versions.keys().collect::>().len() > 10); } @@ -111,11 +111,11 @@ fn report_verification_is_workspace_anchored_and_empty_checks_are_a_noop() { } #[test] -fn semver_check_command_uses_the_reviewed_patch_exception_release_type() { +fn semver_check_command_uses_the_computed_candidate_release_type() { let diff = PackageApiDiff { package: "j2k-core".to_string(), - candidate_version: "0.7.5".to_string(), - release_type: Some(ReleaseType::Minor), + candidate_version: "0.8.0".to_string(), + release_type: Some(ReleaseType::Major), baseline_count: 1, candidate_count: 0, added: BTreeSet::new(), @@ -136,7 +136,7 @@ fn semver_check_command_uses_the_reviewed_patch_exception_release_type() { "--package", "j2k-core", "--baseline-version", - "0.7.3", + "0.7.5", "--release-type", "major", "--color", diff --git a/xtask/src/stable_api.rs b/xtask/src/stable_api.rs index d5d17c49..84d30a47 100644 --- a/xtask/src/stable_api.rs +++ b/xtask/src/stable_api.rs @@ -2,6 +2,8 @@ //! Deterministic collection of the ordinary and rustdoc-hidden public API. +mod native_boundary; + use std::{ collections::{BTreeMap, BTreeSet}, env, @@ -61,6 +63,7 @@ pub(super) fn collect_package_apis( for package in packages { eprintln!("collecting ordinary and rustdoc-hidden public API for `{package}`"); let inventory = collect_package_api(package)?; + native_boundary::validate(package, &inventory)?; if inventories .insert((*package).to_string(), inventory) .is_some() diff --git a/xtask/src/stable_api/native_boundary.rs b/xtask/src/stable_api/native_boundary.rs new file mode 100644 index 00000000..abd380d0 --- /dev/null +++ b/xtask/src/stable_api/native_boundary.rs @@ -0,0 +1,89 @@ +// SPDX-License-Identifier: MIT OR Apache-2.0 + +//! Native-backend isolation for collected public API inventories. + +use super::PackageApiInventory; + +pub(super) fn validate(package: &str, inventory: &PackageApiInventory) -> Result<(), String> { + if package == "j2k-native" { + return Ok(()); + } + + let ordinary = inventory + .ordinary + .iter() + .filter(|line| mentions_native_backend(line)) + .collect::>(); + let hidden = inventory + .hidden + .iter() + .filter(|line| mentions_native_backend(line)) + .collect::>(); + if ordinary.is_empty() && hidden.is_empty() { + return Ok(()); + } + + Err(format!( + "public API for package `{package}` exposes the private native backend; \ + ordinary items: {ordinary:#?}; rustdoc-hidden items: {hidden:#?}" + )) +} + +fn mentions_native_backend(line: &str) -> bool { + line.match_indices("j2k_native").any(|(start, name)| { + let before = line[..start].chars().next_back(); + let after = line[start + name.len()..].chars().next(); + before.is_none_or(|character| character != '_' && !character.is_alphanumeric()) + && after.is_none_or(|character| character != '_' && !character.is_alphanumeric()) + }) +} + +#[cfg(test)] +mod tests { + use std::collections::BTreeSet; + + use super::validate; + use crate::stable_api::{public_api_line_set, PackageApiInventory}; + + #[test] + fn non_native_packages_cannot_expose_native_backend_types() { + let clean = PackageApiInventory { + ordinary: public_api_line_set("pub fn j2k::decode(&[u8]) -> j2k::Image\n"), + hidden: BTreeSet::new(), + }; + assert!(validate("j2k", &clean).is_ok()); + assert!(validate("j2k-native", &clean).is_ok()); + + let ordinary_leak = PackageApiInventory { + ordinary: public_api_line_set("pub fn j2k::decode(&[u8]) -> j2k_native::RawBitmap\n"), + hidden: BTreeSet::new(), + }; + let error = validate("j2k", &ordinary_leak).expect_err("ordinary native type leak"); + assert!(error.contains("ordinary")); + assert!(error.contains("j2k_native::RawBitmap")); + validate("j2k-jpeg-cuda", &ordinary_leak) + .expect_err("new packages must not bypass the native boundary"); + + let hidden_leak = PackageApiInventory { + ordinary: public_api_line_set("pub struct j2k_metal::Decoder\n"), + hidden: public_api_line_set( + "pub fn j2k_metal::Decoder::native() -> j2k_native::Image\n", + ), + }; + let error = validate("j2k-metal", &hidden_leak).expect_err("hidden native type leak"); + assert!(error.contains("rustdoc-hidden")); + assert!(error.contains("j2k_native::Image")); + } + + #[test] + fn native_backend_detection_uses_identifier_boundaries() { + let similarly_named_items = PackageApiInventory { + ordinary: public_api_line_set( + "pub struct j2k::not_j2k_native_adapter\n\ + pub struct j2k_nativeish::Image\n", + ), + hidden: BTreeSet::new(), + }; + assert!(validate("j2k", &similarly_named_items).is_ok()); + } +} diff --git a/xtask/src/test_command.rs b/xtask/src/test_command.rs index ff4d9a1f..4ebca753 100644 --- a/xtask/src/test_command.rs +++ b/xtask/src/test_command.rs @@ -48,6 +48,10 @@ impl RecordingProgram { pub(crate) fn log(&self) -> String { fs::read_to_string(&self.log).expect("read recording-program log") } + + pub(crate) fn was_invoked(&self) -> bool { + self.log.exists() + } } impl Drop for RecordingProgram { diff --git a/xtask/tests/command_orchestration.rs b/xtask/tests/command_orchestration.rs index d48d2088..92cee8ab 100644 --- a/xtask/tests/command_orchestration.rs +++ b/xtask/tests/command_orchestration.rs @@ -157,7 +157,7 @@ fn assert_semver_fails_at_expected_boundary(output: &Output) { let stderr = String::from_utf8_lossy(&output.stderr); let expected_error = if cfg!(target_os = "macos") { - "committed stable API snapshots are stale" + "committed published-library API snapshots are stale" } else { "semver/API review must run on macOS so Metal public APIs are included" }; @@ -171,9 +171,17 @@ fn assert_semver_fails_at_expected_boundary(output: &Output) { fn external_quality_commands_preserve_their_complete_fake_tool_plans() { let harness = Harness::new(); - for task in ["typos", "miri", "machete", "no-std"] { + for task in ["miri", "machete", "no-std"] { assert_success(&harness.run(&[task]), task); } + for task in ["nextest", "typos", "deny", "downstream-smoke"] { + let output = harness.run(&[task]); + assert!(!output.status.success(), "{task} must remain retired"); + assert!( + String::from_utf8_lossy(&output.stderr).contains("unknown task"), + "{task} must fail at the dispatcher boundary" + ); + } assert_success( &harness.run_with_env( &["fuzz-run"], @@ -187,7 +195,6 @@ fn external_quality_commands_preserve_their_complete_fake_tool_plans() { ); let log = harness.log(); - assert!(log.contains("typos \n")); assert!(log.contains("cargo-machete --with-metadata\n")); assert!(log.contains("rustup run nightly cargo miri test -p j2k-core")); assert!(log.contains("rustup target add aarch64-unknown-none")); diff --git a/xtask/tests/command_orchestration/support.rs b/xtask/tests/command_orchestration/support.rs index 75c0f382..0add820f 100644 --- a/xtask/tests/command_orchestration/support.rs +++ b/xtask/tests/command_orchestration/support.rs @@ -2,13 +2,16 @@ use std::{ fmt::Write as _, - fs, + fs::{self, File}, + io::Cursor, os::unix::fs::PermissionsExt, path::{Path, PathBuf}, process::{Command, Output}, sync::atomic::{AtomicU64, Ordering}, }; +use flate2::{write::GzEncoder, Compression}; + static NEXT_TEMP_ID: AtomicU64 = AtomicU64::new(0); pub(crate) struct Harness { @@ -28,29 +31,27 @@ impl Harness { fs::create_dir_all(&root).expect("create orchestration test directory"); let cargo = root.join("cargo.sh"); let log = root.join("cargo.log"); - let real_cargo = std::env::var("CARGO").unwrap_or_else(|_| "cargo".to_string()); + let metadata = prepare_metadata_fixture(&root); fs::write( &cargo, format!( - "#!/bin/sh\nprintf '%s|RUSTDOCFLAGS=%s|RUST_TEST_THREADS=%s\\n' \"$*\" \"${{RUSTDOCFLAGS-unset}}\" \"${{RUST_TEST_THREADS-unset}}\" >> '{}'\nif [ \"$1\" = metadata ]; then exec \"{}\" \"$@\"; fi\nif [ \"$1\" = clippy ]; then printf '%s\\n' '{{\"reason\":\"build-finished\",\"success\":true}}'; exit 0; fi\nif [ \"$1\" = test ]; then printf 'test result: ok. 100 passed; 0 failed;\\n'; exit 0; fi\nif [ \"$1\" = -V ]; then printf 'cargo 1.96.0\\n'; fi\n", + "#!/bin/sh\nprintf '%s|RUSTDOCFLAGS=%s|RUST_TEST_THREADS=%s\\n' \"$*\" \"${{RUSTDOCFLAGS-unset}}\" \"${{RUST_TEST_THREADS-unset}}\" >> '{}'\nif [ \"$1\" = metadata ]; then exec cat '{}'; fi\nif [ \"$1\" = clippy ]; then printf '%s\\n' '{{\"reason\":\"build-finished\",\"success\":true}}'; exit 0; fi\nif [ \"$1\" = test ]; then printf 'test result: ok. 100 passed; 0 failed;\\n'; exit 0; fi\nif [ \"$1\" = -V ]; then printf 'cargo 1.96.0\\n'; fi\n", log.display(), - real_cargo + metadata.display() ), ) .expect("write fake Cargo"); make_executable(&cargo, "fake Cargo"); - for tool in ["typos", "cargo-machete"] { - let program = root.join(tool); - fs::write( - &program, - format!( - "#!/bin/sh\nprintf '%s %s\\n' \"$(basename \"$0\")\" \"$*\" >> '{}'\n", - log.display() - ), - ) - .expect("write fake external tool"); - make_executable(&program, "fake external tool"); - } + let cargo_machete = root.join("cargo-machete"); + fs::write( + &cargo_machete, + format!( + "#!/bin/sh\nprintf '%s %s\\n' \"$(basename \"$0\")\" \"$*\" >> '{}'\n", + log.display() + ), + ) + .expect("write fake cargo-machete"); + make_executable(&cargo_machete, "fake cargo-machete"); let git = root.join("git"); let baseline_snapshot = root.join("stable-api-0.7.3.public-api.txt"); fs::write(&baseline_snapshot, synthetic_baseline_snapshot()) @@ -187,6 +188,76 @@ fn make_executable(path: &Path, label: &str) { fs::set_permissions(path, permissions).expect(label); } +fn prepare_metadata_fixture(root: &Path) -> PathBuf { + let cargo = std::env::var("CARGO").unwrap_or_else(|_| "cargo".to_string()); + let output = Command::new(cargo) + .args(["metadata", "--format-version", "1", "--locked"]) + .current_dir(workspace_root()) + .output() + .expect("capture workspace metadata for orchestration fixture"); + assert!( + output.status.success(), + "workspace metadata fixture failed: {}", + String::from_utf8_lossy(&output.stderr) + ); + let mut metadata = + serde_json::from_slice::(&output.stdout).expect("parse cargo metadata"); + let version = metadata + .get("packages") + .and_then(serde_json::Value::as_array) + .and_then(|packages| { + packages.iter().find_map(|package| { + (package.get("name").and_then(serde_json::Value::as_str) == Some("j2k-ml")) + .then(|| package.get("version").and_then(serde_json::Value::as_str)) + .flatten() + }) + }) + .expect("metadata includes j2k-ml") + .to_string(); + let target = root.join("target"); + metadata["target_directory"] = serde_json::Value::String(target.to_string_lossy().into_owned()); + let metadata_path = root.join("metadata.json"); + fs::write( + &metadata_path, + serde_json::to_vec(&metadata).expect("serialize metadata fixture"), + ) + .expect("write metadata fixture"); + write_packaged_fixture( + &target + .join("package") + .join(format!("j2k-ml-{version}.crate")), + &version, + ); + metadata_path +} + +fn write_packaged_fixture(path: &Path, version: &str) { + fs::create_dir_all(path.parent().expect("package fixture has parent")) + .expect("create package fixture directory"); + let encoder = GzEncoder::new( + File::create(path).expect("create package fixture"), + Compression::default(), + ); + let mut archive = tar::Builder::new(encoder); + let manifest = format!("[package]\nname = \"j2k-ml\"\nversion = \"{version}\"\n"); + let mut header = tar::Header::new_gnu(); + header.set_mode(0o644); + header.set_size(u64::try_from(manifest.len()).expect("package manifest fixture fits in u64")); + header.set_cksum(); + archive + .append_data( + &mut header, + format!("j2k-ml-{version}/Cargo.toml"), + Cursor::new(manifest), + ) + .expect("append package manifest fixture"); + archive + .into_inner() + .expect("finish package archive") + .finish() + .expect("finish compressed package fixture"); +} + fn synthetic_baseline_snapshot() -> String { let mut snapshot = String::from("# J2K 1.0 Public API Snapshot\n\nGenerator: `cargo-public-api 0.52.0`.\n\n"); diff --git a/xtask/tests/repo_lint_support/architecture_policy.rs b/xtask/tests/repo_lint_support/architecture_policy.rs index 6fd4c626..3f8d2d83 100644 --- a/xtask/tests/repo_lint_support/architecture_policy.rs +++ b/xtask/tests/repo_lint_support/architecture_policy.rs @@ -1,12 +1,11 @@ // SPDX-License-Identifier: MIT OR Apache-2.0 -use std::fs; +use std::{collections::BTreeSet, fs}; use super::{ architecture_doc_dependency_edges, assert_file_pattern_checks, assert_pattern_checks, - cargo_metadata_workspace_edges, cargo_public_api_required, const_array_block, format_edge, - repo_root, rust_sources, stable_api_snapshot_sources, xtask_sources, FilePatternCheck, - PatternCheck, + cargo_metadata_workspace_edges, format_edge, release_manifest_entries, repo_root, rust_sources, + stable_api_snapshot_sources, FilePatternCheck, PatternCheck, }; #[test] @@ -94,8 +93,10 @@ fn architecture_docs_classify_workspace_and_in_repo_tool_crates() { #[test] fn tooling_and_validation_crates_stay_unpublished() { let root = repo_root(); - let xtask = xtask_sources(root); - let publishable = const_array_block(&xtask, "PUBLISHABLE_PACKAGES"); + let publishable = release_manifest_entries(root) + .into_iter() + .map(|entry| entry.name) + .collect::>(); assert_file_pattern_checks( root, @@ -122,52 +123,12 @@ fn tooling_and_validation_crates_stay_unpublished() { for package in ["j2k-test-support", "j2k-transcode-test-support", "xtask"] { assert!( - !publishable.contains(&format!("\"{package}\"")), - "xtask publishable package gate must not include {package}" + !publishable.contains(package), + "release manifest must not include {package}" ); } } -#[test] -fn public_crates_do_not_reexport_j2k_native() { - let root = repo_root(); - let mut offenders = Vec::new(); - - for crate_dir in [ - "crates/j2k/src", - "crates/j2k-jpeg/src", - "crates/j2k-transcode/src", - "crates/j2k-metal/src", - "crates/j2k-cuda/src", - "crates/j2k-transcode-metal/src", - "crates/j2k-transcode-cuda/src", - ] { - for path in rust_sources(&root.join(crate_dir)) { - let source = fs::read_to_string(&path) - .unwrap_or_else(|err| panic!("read {}: {err}", path.display())); - for (line_idx, line) in source.lines().enumerate() { - let trimmed = line.trim_start(); - if trimmed.starts_with("pub use j2k_native") - || trimmed.starts_with("pub type ") && trimmed.contains("j2k_native") - { - offenders.push(format!( - "{}:{}:{}", - path.strip_prefix(root).unwrap_or(&path).display(), - line_idx + 1, - line.trim() - )); - } - } - } - } - - assert!( - offenders.is_empty(), - "public crates must not re-export native J2K implementation types:\n{}", - offenders.join("\n") - ); -} - #[test] fn native_decode_error_boundaries_preserve_owned_sources_and_shared_classification() { let forbidden_native_matches = [ @@ -277,28 +238,6 @@ fn facade_backend_errors_require_explicit_classification() { ); } -#[test] -#[ignore = "strict repo-lint: shells out to cargo-public-api for all public API packages"] -fn rendered_public_api_does_not_expose_j2k_native() { - let root = repo_root(); - - for package in [ - "j2k", - "j2k-jpeg", - "j2k-transcode", - "j2k-metal", - "j2k-cuda", - "j2k-transcode-metal", - "j2k-transcode-cuda", - ] { - let api = cargo_public_api_required(root, package); - assert!( - !api.contains("j2k_native"), - "public API for package {package} exposes j2k_native:\n{api}" - ); - } -} - #[test] fn obsolete_adaptive_route_policy_model_cannot_return() { let root = repo_root(); diff --git a/xtask/tests/repo_lint_support/coverage_structure_policy.rs b/xtask/tests/repo_lint_support/coverage_structure_policy.rs deleted file mode 100644 index 0b7f40d4..00000000 --- a/xtask/tests/repo_lint_support/coverage_structure_policy.rs +++ /dev/null @@ -1,17 +0,0 @@ -// SPDX-License-Identifier: MIT OR Apache-2.0 - -//! Structural ownership and size ratchets for changed-line coverage tooling. - -use std::fs; - -use super::repo_root; - -mod coordination_parsing; -mod evaluation_exclusion; -mod line_ratchets; -mod source_analysis_tests; - -fn read(relative_path: &str) -> String { - fs::read_to_string(repo_root().join(relative_path)) - .unwrap_or_else(|error| panic!("read {relative_path}: {error}")) -} diff --git a/xtask/tests/repo_lint_support/coverage_structure_policy/coordination_parsing.rs b/xtask/tests/repo_lint_support/coverage_structure_policy/coordination_parsing.rs deleted file mode 100644 index 92721053..00000000 --- a/xtask/tests/repo_lint_support/coverage_structure_policy/coordination_parsing.rs +++ /dev/null @@ -1,183 +0,0 @@ -// SPDX-License-Identifier: MIT OR Apache-2.0 - -use super::super::{assert_pattern_checks, PatternCheck}; -use super::read; - -#[test] -fn coverage_coordination_model_and_build_outputs_stay_explicit() { - let coordinator = read("xtask/src/coverage.rs"); - let model = read("xtask/src/coverage/model.rs"); - let build_outputs = read("xtask/src/coverage/build_outputs.rs"); - let build_output_target = read("xtask/src/coverage/build_outputs/target.rs"); - let build_output_tests = read("xtask/src/coverage/build_outputs/tests.rs"); - - assert_pattern_checks(&[ - PatternCheck::new("coverage coordinator wiring", &coordinator) - .required(&[ - "mod accelerator_ownership;", - "mod build_outputs;", - "mod compiler_regions;", - "mod critical_path_policy;", - "mod evaluation;", - "mod exclusion_policy;", - "mod lane;", - "mod model;", - "mod parsing;", - "mod source_analysis;", - "mod summary;", - "pub(crate) fn coverage(", - "ensure_no_untracked_rust_sources()?;", - "validate_shared_accelerator_registry(&root)?;", - "parse_compiler_regions(&compiler_regions, &root)?", - ]) - .forbidden(&[ - "enum CoverageLane", - "fn run_lane(", - "fn parse_lcov(", - "fn evaluate_changed_coverage(", - "fn write_summary(", - "const COVERAGE_EXCLUSIONS", - ]), - PatternCheck::new("coverage model and option ownership", &model).required(&[ - "pub(super) const CHANGED_LINE_THRESHOLD_PERCENT: u64 = 80", - "struct AcceleratorLaneSpec", - "struct AcceleratorPackageSpec", - "const METAL_ACCELERATOR_LANE", - "const CUDA_ACCELERATOR_LANE", - "pub(super) enum CoverageLane", - "pub(super) fn coverage_packages(", - "pub(super) struct CoverageOptions", - "pub(super) struct ChangedCoverageResult", - "pub(super) critical: CoverageCounts", - "pub(super) fn parse_options(", - ]), - PatternCheck::new("coverage build-output cfg ownership", &build_outputs).required(&[ - "mod target;", - "pub(super) use target::CurrentBuildTarget;", - "pub(super) struct BuildOutputEvidence", - "pub(super) fn capture(", - "pub(super) fn current_cfg_flags(", - "fn scan_outputs(", - "fn reconcile_cfg_flags(", - "fn parse_build_cfg_output(", - ]), - PatternCheck::new("current coverage build target", &build_output_target).required(&[ - "pub(in crate::coverage) struct CurrentBuildTarget", - "pub(in crate::coverage) fn create(", - "CARGO_LLVM_COV_TARGET_DIR", - "CARGO_TARGET_DIR", - ".j2k-current-coverage-", - "fs::create_dir(", - "fs::remove_dir_all(", - ]), - PatternCheck::new("coverage build-output regressions", &build_output_tests).required(&[ - "fn identical_rerun_output_is_current_build_evidence()", - "fn stale_scope_output_is_outside_current_build_provenance()", - "fn missing_selected_package_build_output_fails_closed()", - "fn conflicting_current_scopes_fail_closed()", - "fn hyphenated_package_name_matches_its_full_build_scope()", - ]), - ]); -} - -#[test] -fn coverage_lane_and_diff_parsing_ownership_stays_explicit() { - let lane = read("xtask/src/coverage/lane.rs"); - let parsing = read("xtask/src/coverage/parsing.rs"); - - assert_pattern_checks(&[ - PatternCheck::new("coverage lane execution ownership", &lane) - .required(&[ - "const METAL_COVERAGE_ENV", - "const CUDA_COVERAGE_ENV", - "const REQUIRED_CARGO_LLVM_COV_VERSION: &str = \"0.8.7\"", - "pub(super) fn run_lane(", - "CurrentBuildTarget::create(root)", - "BuildOutputEvidence::capture(current_build_target)", - "CARGO_LLVM_COV_TARGET_DIR", - "CARGO_LLVM_COV_BUILD_DIR", - "fn run_host_coverage(", - "fn run_metal_coverage(", - "fn run_cuda_coverage(", - "fn report_compiler_regions(", - "fn report_compiler_regions_args(", - "fn coverage_tool_version(", - "fn parse_coverage_tool_version(", - "fn package_coverage_args(", - "fn accelerator_lane_package_args_include_every_shared_source_owner()", - "fn lane_spec_drives_package_args_and_source_ownership()", - "fn shared_accelerator_source_owners_drive_lane_package_selection()", - "fn coverage_tool_version_parser_requires_named_record()", - "fn llvm_cov_commands_share_unique_target_and_build_directory()", - "fn lane_orchestrators_execute_complete_hermetic_cargo_plans()", - "--include-build-script", - "fn run_llvm_cov(", - ]) - .forbidden(&["\"llvm-cov\", \"clean\""]), - PatternCheck::new("coverage diff and LCOV parser ownership", &parsing).required(&[ - "pub(super) fn ensure_no_untracked_rust_sources()", - "pub(super) fn validate_no_untracked_rust_sources(", - "pub(super) fn resolve_diff_base(", - "pub(super) fn git_output(", - "pub(super) fn parse_changed_lines(", - "pub(super) fn parse_lcov(", - "pub(super) fn normalize_coverage_path(", - ]), - ]); -} - -#[test] -fn coverage_compiler_region_ownership_stays_explicit() { - let compiler_regions = read("xtask/src/coverage/compiler_regions.rs"); - let compiler_region_parsing = read("xtask/src/coverage/compiler_regions/parsing.rs"); - let compiler_region_tests = read("xtask/src/coverage/compiler_regions/tests.rs"); - let compiler_region_line_tests = - read("xtask/src/coverage/compiler_regions/tests/line_evidence.rs"); - let compiler_line_evaluation_tests = - read("xtask/src/coverage/tests/evaluation/compiler_line_evidence.rs"); - - assert_pattern_checks(&[ - PatternCheck::new("compiler region evidence ownership", &compiler_regions).required(&[ - "mod parsing;", - "pub(super) use parsing::parse_compiler_regions;", - "pub(super) struct CompilerRegionReport", - "pub(super) fn evidence_for(", - "pub(super) fn evidence_for_line(", - "CompilerRegionEvidence::NonInstrumentable", - "mod tests;", - ]), - PatternCheck::new( - "compiler region JSON parsing ownership", - &compiler_region_parsing, - ) - .required(&[ - "llvm.coverage.json.export", - "pub(in crate::coverage) fn parse_compiler_regions(", - "must contain exactly 8 integer fields", - ]), - PatternCheck::new( - "compiler region evidence regressions", - &compiler_region_tests, - ) - .required(&[ - "fn parser_aggregates_code_regions_by_normalized_repository_path()", - "fn body_without_a_nested_code_region_is_compiler_noninstrumentable()", - "fn nested_zero_count_code_region_is_uncovered()", - "fn malformed_or_unrelated_reports_fail_closed()", - "fn dependency_macro_expansion_regions_are_ignored_without_hiding_repository_regions()", - ]), - PatternCheck::new( - "compiler line evidence regressions", - &compiler_region_line_tests, - ) - .required(&["fn line_evidence_uses_the_most_specific_intersecting_region()"]), - PatternCheck::new( - "compiler line evaluation regressions", - &compiler_line_evaluation_tests, - ) - .required(&[ - "fn covered_compiler_region_owns_multiline_expression_lines_without_da_records()", - "fn zero_compiler_region_keeps_an_executable_line_without_da_uncovered()", - ]), - ]); -} diff --git a/xtask/tests/repo_lint_support/coverage_structure_policy/evaluation_exclusion.rs b/xtask/tests/repo_lint_support/coverage_structure_policy/evaluation_exclusion.rs deleted file mode 100644 index 0ab4cfe3..00000000 --- a/xtask/tests/repo_lint_support/coverage_structure_policy/evaluation_exclusion.rs +++ /dev/null @@ -1,146 +0,0 @@ -// SPDX-License-Identifier: MIT OR Apache-2.0 - -use super::super::{assert_pattern_checks, PatternCheck}; -use super::read; - -#[test] -fn coverage_evaluation_and_reporting_ownership_stays_explicit() { - let evaluation = read("xtask/src/coverage/evaluation.rs"); - let summary = read("xtask/src/coverage/summary.rs"); - - assert_pattern_checks(&[ - PatternCheck::new("coverage changed-line evaluation ownership", &evaluation) - .required(&[ - "pub(super) fn evaluate_changed_coverage(", - "struct ChangedFileEvidence<'a>", - "fn evaluate_changed_lines(", - "evidence_for_line(self.path, line_number)", - "fn record_missing_body_evidence(", - "self.body_is_covered(function.body_start, function.body_end)", - "changed_functions_without_covered_body", - "changed_executable_bodies_without_covered_body", - "changed_deferred_bodies_without_covered_compiler_region", - "compiler_noninstrumentable_deferred_bodies", - "compiler_noninstrumentable_lines", - "mixed_test_production_lines", - "changed_opaque_macros", - "source_dispositions", - "pub(super) fn coverage_violations(", - "audited_zero_body_findings(lane, result)", - "lane.enforces_overall_changed_lines()", - "changed critical-path lines", - "critical executable bodies are absent", - "fn meets_threshold(", - ]) - .forbidden(&[ - "fn terminal_test_module_start(", - "fn source_has_instrumentable_function(", - ]), - PatternCheck::new("coverage summary ownership", &summary).required(&[ - "pub(super) fn write_summary(", - "pub(super) fn print_summary(", - "j2k-changed-line-coverage-v6", - "head_sha", - "lane_scope", - "cargo_llvm_cov_version", - "residual_unmeasured_lines", - "changed_functions_without_covered_body", - "changed_executable_bodies_without_covered_body", - "changed_deferred_bodies_without_covered_compiler_region", - "compiler_noninstrumentable_deferred_bodies", - "compiler_noninstrumentable_lines", - "compiler_regions_artifact", - "mixed_test_production_lines", - "changed_opaque_macros", - "critical_paths", - "zero_body_audit", - "accelerator_host_rust", - "narrow_exclusions", - ]), - ]); -} - -#[test] -fn coverage_critical_path_policy_ownership_stays_explicit() { - let policy = read("xtask/src/coverage/critical_path_policy.rs"); - let classification = read("xtask/src/coverage/critical_path_policy/classification.rs"); - let policy_modules = [policy.as_str(), classification.as_str()].join("\n"); - let tests = read("xtask/src/coverage/tests/critical_path_policy.rs"); - let release_gate_tests = read("xtask/src/coverage/tests/critical_path_policy/release_gates.rs"); - let test_modules = [tests.as_str(), release_gate_tests.as_str()].join("\n"); - - assert_pattern_checks(&[ - PatternCheck::new( - "coverage critical-path and residual ownership", - &policy_modules, - ) - .required(&[ - "pub(super) enum CriticalPathClass", - "pub(super) enum ResidualDisposition", - "pub(super) enum ZeroBodyAudit", - "pub(in crate::coverage) fn classify_path(", - "pub(super) fn audited_zero_body_findings(", - "Self::Unreachable => \"unreachable\"", - "Self::HardwareOnly => \"hardware-only\"", - "Self::Trivial => \"trivial\"", - "Self::LowRiskTooling => \"low-risk-tooling\"", - ]), - PatternCheck::new("coverage critical-path policy regressions", &test_modules).required(&[ - "fn critical_path_threshold_cannot_be_masked_by_low_risk_tooling_coverage()", - "fn accelerator_lane_uses_raw_implementation_coverage_as_audit_evidence()", - "fn host_lane_keeps_the_overall_changed_line_gate()", - "fn low_risk_tooling_absence_is_an_audited_residual_not_a_critical_failure()", - "fn critical_path_classification_covers_release_risk_boundaries()", - "fn accelerator_critical_paths_exclude_broad_compute_and_diagnostic_internals()", - "fn zero_body_audit_records_each_approved_residual_disposition()", - "fn critical_zero_body_findings_remain_in_the_audit_without_individual_failure()", - ]), - ]); -} - -#[test] -fn coverage_exclusion_policy_ownership_stays_explicit() { - let exclusions = read("xtask/src/coverage/exclusion_policy.rs"); - let evidence_modules = read("xtask/src/coverage/exclusion_policy/evidence_modules.rs"); - let tests = read("xtask/src/coverage/exclusion_policy/tests.rs"); - - assert_pattern_checks(&[ - PatternCheck::new("coverage exclusion policy ownership", &exclusions) - .required(&[ - "mod evidence_modules;", - "pub(super) const COVERAGE_EXCLUSIONS", - "enum EvidenceClass", - "fn require_primary_evidence(", - "fn enclosing_cfg_is_conditional(", - "cuda-simt-device-rust", - "metal-embedded-shader-body", - "generated-codec-math-fragment", - "vendored-block-ffi-binding", - "pub(super) fn matching_exclusion(", - "pub(super) fn validate_exclusion_policy(", - "fn collect_rust_files(", - ]) - .forbidden(&[ - "fn resolve_external_module_path(", - "fn explicit_module_path(", - ]), - PatternCheck::new( - "coverage exclusion external evidence traversal", - &evidence_modules, - ) - .required(&[ - "fn collect_evidence_symbols_from_file(", - "resolve_external_module(", - "existing_repository_source(", - ]) - .forbidden(&[ - "fn resolve_external_module_path(", - "fn explicit_module_path(", - ]), - PatternCheck::new("coverage exclusion evidence regressions", &tests).required(&[ - "fn direct_and_inherited_cfg_require_supplemental_classification()", - "fn exact_enclosing_cfg_test_is_harness_plumbing()", - "fn supplemental_only_exclusion_evidence_is_rejected()", - ]), - ]); -} diff --git a/xtask/tests/repo_lint_support/coverage_structure_policy/line_ratchets.rs b/xtask/tests/repo_lint_support/coverage_structure_policy/line_ratchets.rs deleted file mode 100644 index c45439e3..00000000 --- a/xtask/tests/repo_lint_support/coverage_structure_policy/line_ratchets.rs +++ /dev/null @@ -1,190 +0,0 @@ -// SPDX-License-Identifier: MIT OR Apache-2.0 - -use super::read; - -#[test] -fn coverage_structure_policy_has_focused_real_module_owners() { - let shell_relative = "xtask/tests/repo_lint_support/coverage_structure_policy.rs"; - let shell = read(shell_relative); - let include_macro = ["include", "!("].concat(); - - for (module, owned_symbols, max_lines) in [ - ( - "line_ratchets", - &[ - "fn coverage_structure_policy_has_focused_real_module_owners(", - "fn coverage_source_modules_stay_within_structural_ratchets(", - ][..], - 300usize, - ), - ( - "coordination_parsing", - &[ - "fn coverage_coordination_model_and_build_outputs_stay_explicit(", - "fn coverage_lane_and_diff_parsing_ownership_stays_explicit(", - "fn coverage_compiler_region_ownership_stays_explicit(", - ][..], - 300usize, - ), - ( - "evaluation_exclusion", - &[ - "fn coverage_evaluation_and_reporting_ownership_stays_explicit(", - "fn coverage_critical_path_policy_ownership_stays_explicit(", - "fn coverage_exclusion_policy_ownership_stays_explicit(", - ][..], - 250usize, - ), - ( - "source_analysis_tests", - &[ - "fn coverage_source_analysis_ast_ownership_stays_explicit(", - "fn coverage_source_analysis_boundaries_stay_explicit(", - "fn coverage_workspace_discovery_stays_explicit(", - "fn coverage_source_analysis_regression_ownership_stays_explicit(", - "fn coverage_source_roles_and_evaluation_regressions_stay_explicit(", - ][..], - 350usize, - ), - ] { - assert!( - shell.contains(&format!("mod {module};")), - "{shell_relative} must declare the {module} responsibility owner" - ); - let relative = - format!("xtask/tests/repo_lint_support/coverage_structure_policy/{module}.rs"); - let source = read(&relative); - for owned_symbol in owned_symbols { - let expected_count = usize::from(module == "line_ratchets") + 1; - assert_eq!( - source.matches(owned_symbol).count(), - expected_count, - "{relative} must own {owned_symbol} exactly once" - ); - } - assert!( - source.lines().count() < max_lines, - "{relative} exceeded its {max_lines}-line responsibility ratchet" - ); - assert!( - !source.lines().any(|line| line.trim() == "use super::*;"), - "{relative} must keep explicit imports" - ); - assert!( - !source.contains(&include_macro), - "{relative} must remain a real Rust module" - ); - } - - assert!( - shell.lines().count() < 20, - "{shell_relative} must remain a tiny shared-read module shell" - ); - assert!(!shell.contains("#[test]")); - assert!(!shell.contains("PatternCheck::new")); - assert!(!shell.contains(&include_macro)); - assert!(!shell.lines().any(|line| line.trim() == "use super::*;")); -} - -#[test] -fn coverage_source_modules_stay_within_structural_ratchets() { - let include_macro = ["include", "!("].concat(); - for (relative_path, max_lines) in [ - ("xtask/src/coverage.rs", 300), - ("xtask/src/coverage/accelerator_ownership.rs", 250), - ("xtask/src/coverage/build_outputs.rs", 350), - ("xtask/src/coverage/build_outputs/target.rs", 150), - ("xtask/src/coverage/build_outputs/tests.rs", 200), - ("xtask/src/coverage/compiler_regions.rs", 200), - ("xtask/src/coverage/compiler_regions/parsing.rs", 250), - ("xtask/src/coverage/compiler_regions/tests.rs", 180), - ( - "xtask/src/coverage/compiler_regions/tests/line_evidence.rs", - 100, - ), - ("xtask/src/coverage/critical_path_policy.rs", 350), - ( - "xtask/src/coverage/critical_path_policy/classification.rs", - 175, - ), - ("xtask/src/coverage/model.rs", 600), - ("xtask/src/coverage/lane.rs", 600), - ("xtask/src/coverage/parsing.rs", 600), - ("xtask/src/coverage/evaluation.rs", 600), - ("xtask/src/coverage/summary.rs", 600), - ("xtask/src/coverage/exclusion_policy.rs", 600), - ( - "xtask/src/coverage/exclusion_policy/evidence_modules.rs", - 150, - ), - ("xtask/src/coverage/exclusion_policy/tests.rs", 180), - ("xtask/src/coverage/source_analysis.rs", 300), - ("xtask/src/coverage/source_analysis/audit.rs", 100), - ("xtask/src/coverage/source_analysis/ast.rs", 300), - ("xtask/src/coverage/source_analysis/ast/executable.rs", 180), - ("xtask/src/coverage/source_analysis/ast/visitor.rs", 300), - ( - "xtask/src/coverage/source_analysis/ast/visitor/items.rs", - 300, - ), - ( - "xtask/src/coverage/source_analysis/ast/visitor/runtime.rs", - 200, - ), - ("xtask/src/coverage/source_analysis/cfg_eval.rs", 350), - ("xtask/src/coverage/source_analysis/graph.rs", 200), - ("xtask/src/coverage/source_analysis/module_resolver.rs", 180), - ("xtask/src/coverage/source_analysis/node_attrs.rs", 200), - ( - "xtask/src/coverage/source_analysis/test_constructors.rs", - 150, - ), - ("xtask/src/coverage/source_analysis/test_lines.rs", 150), - ("xtask/src/coverage/source_analysis/ast/test_spans.rs", 100), - ("xtask/src/coverage/source_analysis/workspace.rs", 500), - ( - "xtask/src/coverage/source_analysis/workspace/fuzz_manifests.rs", - 250, - ), - ("xtask/src/coverage/tests.rs", 250), - ("xtask/src/coverage/tests/attributes.rs", 200), - ("xtask/src/coverage/tests/cfg_provenance.rs", 100), - ("xtask/src/coverage/tests/critical_path_policy.rs", 125), - ( - "xtask/src/coverage/tests/critical_path_policy/release_gates.rs", - 150, - ), - ("xtask/src/coverage/tests/deferred_bodies.rs", 200), - ("xtask/src/coverage/tests/evaluation.rs", 250), - ("xtask/src/coverage/tests/exclusion_policy.rs", 150), - ("xtask/src/coverage/tests/evaluation/non_executable.rs", 100), - ( - "xtask/src/coverage/tests/evaluation/compiler_line_evidence.rs", - 100, - ), - ("xtask/src/coverage/tests/executable_evidence.rs", 180), - ("xtask/src/coverage/tests/presence.rs", 150), - ("xtask/src/coverage/tests/source_analysis.rs", 250), - ("xtask/src/coverage/tests/source_roles.rs", 250), - ("xtask/src/coverage/tests/support.rs", 100), - ] { - let source = read(relative_path); - let line_count = source.lines().count(); - assert!( - line_count < max_lines, - "{relative_path} has {line_count} lines; expected fewer than {max_lines}" - ); - assert!( - !source.contains("::*"), - "{relative_path} must keep explicit imports" - ); - assert!( - !source.contains(&include_macro), - "{relative_path} must remain a real Rust module" - ); - assert!( - !source.contains("#[allow("), - "{relative_path} must not add lint suppressions" - ); - } -} diff --git a/xtask/tests/repo_lint_support/coverage_structure_policy/source_analysis_tests.rs b/xtask/tests/repo_lint_support/coverage_structure_policy/source_analysis_tests.rs deleted file mode 100644 index 936e55ed..00000000 --- a/xtask/tests/repo_lint_support/coverage_structure_policy/source_analysis_tests.rs +++ /dev/null @@ -1,265 +0,0 @@ -// SPDX-License-Identifier: MIT OR Apache-2.0 - -use super::super::{assert_pattern_checks, PatternCheck}; -use super::read; - -#[test] -fn coverage_source_analysis_ast_ownership_stays_explicit() { - let facade = read("xtask/src/coverage/source_analysis.rs"); - let audit = read("xtask/src/coverage/source_analysis/audit.rs"); - let ast = read("xtask/src/coverage/source_analysis/ast.rs"); - let executable = read("xtask/src/coverage/source_analysis/ast/executable.rs"); - let visitor = read("xtask/src/coverage/source_analysis/ast/visitor.rs"); - let visitor_items = read("xtask/src/coverage/source_analysis/ast/visitor/items.rs"); - let visitor_runtime = read("xtask/src/coverage/source_analysis/ast/visitor/runtime.rs"); - let visitors = [ - visitor.as_str(), - visitor_items.as_str(), - visitor_runtime.as_str(), - ] - .join("\n"); - - assert_pattern_checks(&[ - PatternCheck::new("coverage source-analysis facade", &facade).required(&[ - "mod ast;", - "mod audit;", - "mod cfg_eval;", - "mod graph;", - "mod module_resolver;", - "mod node_attrs;", - "mod test_constructors;", - "mod workspace;", - "pub(super) struct FunctionSpan", - "pub(super) struct ExecutableBodySpan", - "pub(super) struct OpaqueMacroSpan", - "pub(super) executable_lines: BTreeSet", - "pub(super) body_start: usize", - "pub(super) struct SourceIndex", - ]), - PatternCheck::new("coverage production-audit facade", &audit).required(&[ - "pub(crate) struct SourceAuditTestSpan", - "pub(crate) struct SourceAuditSyntax", - "pub(crate) fn analyze_test_only_syntax(", - "CoverageCfgContext::for_current_target", - "ReachKind::Production", - ]), - PatternCheck::new("coverage syn AST analysis", &ast).required(&[ - "syn::parse_file(source)", - "mod executable;", - "mod visitor;", - "struct AstCollector", - "fn visit_attributed_node(", - "self.executable_lines.insert(body_start)", - ]), - PatternCheck::new("coverage executable-span collector", &executable).required(&[ - "fn record_executable_span(", - "fn record_closure(", - "fn record_opaque_macro(", - "fn visit_executable_node(", - ]), - PatternCheck::new("coverage syn AST visitor", &visitors).required(&[ - "impl<'ast> Visit<'ast> for AstCollector<'_>", - "mod items;", - "mod runtime;", - "function.block.span()", - "function.default", - "Expr::Closure(closure)", - "Expr::Macro(expression_macro)", - "Item::Verbatim(_)", - "unclassified cfg/test attribute", - "Attribute payloads are compile-time metadata", - "fn visit_fn_arg(", - "fn visit_pat(", - ]), - ]); -} - -#[test] -fn coverage_source_analysis_boundaries_stay_explicit() { - let cfg = read("xtask/src/coverage/source_analysis/cfg_eval.rs"); - let graph = read("xtask/src/coverage/source_analysis/graph.rs"); - let resolver = read("xtask/src/coverage/source_analysis/module_resolver.rs"); - let constructors = read("xtask/src/coverage/source_analysis/test_constructors.rs"); - let node_attrs = read("xtask/src/coverage/source_analysis/node_attrs.rs"); - - assert_pattern_checks(&[ - PatternCheck::new("coverage cfg evaluation", &cfg).required(&[ - "pub(super) struct CoverageCfgContext", - "enabled_features", - "custom_flags: Option>", - "SymbolicTruth::Unknown", - "conservatively_active", - "target_feature", - "structural cfg_attr", - ]), - PatternCheck::new("coverage module graph", &graph).required(&[ - "pub(super) enum ReachKind", - "pub(super) struct ReachState", - "pub(super) fn module_reachability(", - ]), - PatternCheck::new("coverage module path boundary", &resolver).required(&[ - "pub(in crate::coverage) fn resolve_external_module(", - "fs::canonicalize(root)", - "resolves outside repository root", - "more than one path attribute", - ]), - PatternCheck::new("coverage test constructors", &constructors).required(&[ - "impl SourceIndex", - "pub(in crate::coverage) fn single(", - "pub(in crate::coverage) fn repository_subset(", - "pub(in crate::coverage) fn repository_manifest_fuzz_subset(", - ]), - PatternCheck::new("coverage attribute-bearing node accessors", &node_attrs).required(&[ - "pub(super) fn expression(", - "Expr::Struct(value)", - "pub(super) fn foreign_item(", - "pub(super) fn function_argument(", - "pub(super) fn generic_parameter(", - "pub(super) fn pattern(", - "unclassified non-exhaustive expression variant", - ]), - ]); -} - -#[test] -fn coverage_workspace_discovery_stays_explicit() { - let workspace = read("xtask/src/coverage/source_analysis/workspace.rs"); - let fuzz_manifests = read("xtask/src/coverage/source_analysis/workspace/fuzz_manifests.rs"); - - assert_pattern_checks(&[ - PatternCheck::new("coverage workspace discovery", &workspace) - .required(&[ - "mod fuzz_manifests;", - "discover_source_roots(", - "cargo metadata", - "fuzz_manifests::discover(", - "capture_cfg_contexts(", - "has_build_script", - "current_cfg_flags(&selected_packages, &build_script_packages)", - "GENERATED_DWT_DISPOSITION", - "VENDORED_BLOCK_DISPOSITION", - ]) - .forbidden(&[ - "fresh_cfg_flags", - "unwrap_or_default()", - "fn is_test_target(", - "fn is_example_bench_or_fuzz(", - "path.starts_with(\"tests/\")", - "path.starts_with(\"examples/\")", - "path.starts_with(\"benches/\")", - "path.starts_with(\"fuzz/\")", - "path.contains(\"/tests/\")", - "path.contains(\"/examples/\")", - "path.contains(\"/benches/\")", - "path.contains(\"/fuzz/\")", - ]), - PatternCheck::new("manifest-backed cargo-fuzz roots", &fuzz_manifests).required(&[ - "pub(super) fn discover(", - "fn candidate_manifests(", - "fn parse_manifest(", - "metadata.get(\"cargo-fuzz\")", - "must explicitly declare [[bin]] targets", - "ReachKind::ExampleBenchFuzz", - ]), - ]); -} - -#[test] -fn coverage_source_analysis_regression_ownership_stays_explicit() { - let tests = read("xtask/src/coverage/tests.rs"); - let attributes = read("xtask/src/coverage/tests/attributes.rs"); - let cfg_provenance = read("xtask/src/coverage/tests/cfg_provenance.rs"); - let exclusion_policy = read("xtask/src/coverage/tests/exclusion_policy.rs"); - let source = read("xtask/src/coverage/tests/source_analysis.rs"); - - assert_pattern_checks(&[ - PatternCheck::new("coverage regression tests", &tests).required(&[ - "mod attributes;", - "mod cfg_provenance;", - "mod deferred_bodies;", - "mod exclusion_policy;", - "mod executable_evidence;", - "fn parses_added_diff_hunks_without_counting_deletions()", - "fn untracked_rust_sources_fail_the_local_coverage_preflight()", - "fn lcov_parser_merges_duplicate_line_records_by_max_count()", - "fn eighty_percent_changed_line_coverage_passes_exactly()", - "fn coverage_cli_defaults_to_host_and_accepts_explicit_lanes()", - ]), - PatternCheck::new("coverage attribute-disposition regressions", &attributes).required(&[ - "fn cfg_test_module_does_not_hide_later_production_items()", - "fn cfg_test_attributes_on_fields_locals_arms_and_expressions_are_test_only()", - "fn cfg_test_function_parameters_are_test_only_without_hiding_patterns()", - ]), - PatternCheck::new("coverage cfg provenance regressions", &cfg_provenance) - .required(&["fn cfg_active_changed_source_cannot_evade_coverage_gate()"]), - PatternCheck::new("coverage exclusion regressions", &exclusion_policy) - .required(&["fn exclusion_policy_maps_every_narrow_rule_to_existing_tests()"]), - PatternCheck::new("coverage source-analysis regressions", &source).required(&[ - "fn body_bearing_function_forms_have_item_and_body_spans()", - "fn nested_inline_module_uses_its_real_module_directory()", - "fn nonterminal_external_module_in_named_crate_root_uses_sibling_source()", - "fn module_path_cannot_escape_the_repository_root()", - "fn unknown_custom_cfg_is_conservatively_required()", - "fn unknown_cfg_in_either_polarity_is_conservatively_required()", - ]), - ]); -} - -#[test] -fn coverage_source_roles_and_evaluation_regressions_stay_explicit() { - let roles = read("xtask/src/coverage/tests/source_roles.rs"); - let evaluation = read("xtask/src/coverage/tests/evaluation.rs"); - let non_executable = read("xtask/src/coverage/tests/evaluation/non_executable.rs"); - let executable = read("xtask/src/coverage/tests/executable_evidence.rs"); - let deferred = read("xtask/src/coverage/tests/deferred_bodies.rs"); - let presence = read("xtask/src/coverage/tests/presence.rs"); - - assert_pattern_checks(&[ - PatternCheck::new("coverage source-role regressions", &roles).required(&[ - "fn nonterminal_external_test_modules_do_not_truncate_production_files()", - "crates/j2k-cuda-runtime/src/lib.rs", - "crates/j2k-jpeg/src/backend/mod.rs", - "crates/j2k-native/src/j2c/encode/single_tile.rs", - "fn cfg_test_helper_trees_are_not_production_source()", - "crates/j2k-cuda-runtime/src/context/test_kernels.rs", - "fn repository_owned_xtask_rust_fixtures_are_explicitly_test_only()", - "xtask/tests/fixtures/clone_audit/production_clone_b.rs", - "fn unreachable_role_named_directories_fail_closed()", - "crate/src/tests/orphan.rs", - "crate/src/examples/orphan.rs", - "crate/src/benches/orphan.rs", - "crate/src/fuzz/orphan.rs", - "xtask/tests/fixtures/other/orphan.rs", - "xtask/tests/fixtures/clone_audit/nested/orphan.rs", - "fn cargo_target_roots_retain_metadata_roles()", - "fn cargo_fuzz_manifest_only_grants_reachable_targets_the_fuzz_role()", - ]), - PatternCheck::new("coverage evaluation regressions", &evaluation).required(&[ - "mod non_executable;", - "fn changed_signature_requires_a_positive_da_record_in_the_function_body()", - "fn changed_function_without_covered_body_is_a_critical_audit_finding()", - "fn registered_shared_accelerator_sources_reach_both_gpu_denominators()", - "fn generated_and_vendored_sources_have_reviewed_dispositions()", - ]), - PatternCheck::new("coverage non-executable-line regressions", &non_executable).required(&[ - "fn residual_unmeasured_lines_remain_explicit()", - "fn compiler_mapped_documentation_is_not_an_executable_changed_line()", - ]), - PatternCheck::new("coverage executable-evidence regressions", &executable).required(&[ - "fn changed_uncalled_closure_requires_coverage_in_its_own_body()", - "fn changed_opaque_macro_definition_and_invocation_are_audited_without_blanket_failure()", - "fn cfg_test_macro_remains_test_only()", - ]), - PatternCheck::new("coverage deferred-body regressions", &deferred).required(&[ - "fn executed_one_line_closure_accepts_its_own_compiler_region()", - "fn unpolled_one_line_async_records_its_zero_count_compiler_region()", - "fn body_without_a_compiler_region_is_recorded_as_noninstrumentable()", - ]), - PatternCheck::new("coverage presence regressions", &presence).required(&[ - "fn partial_file_lcov_does_not_mask_second_changed_function_without_covered_body()", - "fn shared_accelerator_source_absent_from_metal_lcov_is_a_violation()", - "fn zero_count_body_record_does_not_prove_changed_signature_coverage()", - "fn changed_executable_body_line_without_da_is_uncovered()", - ]), - ]); -} diff --git a/xtask/tests/repo_lint_support/docs_and_workflows_policy/decoder_fixture_policy.rs b/xtask/tests/repo_lint_support/docs_and_workflows_policy/decoder_fixture_policy.rs index e5cd8f3f..7c1a0071 100644 --- a/xtask/tests/repo_lint_support/docs_and_workflows_policy/decoder_fixture_policy.rs +++ b/xtask/tests/repo_lint_support/docs_and_workflows_policy/decoder_fixture_policy.rs @@ -416,10 +416,12 @@ fn deinterleave_reference_has_only_checked_public_entrypoint() { } #[test] -fn decode_strictness_policy_is_explicit_and_warns_on_lenient_default() { +fn decode_strictness_policy_is_strict_by_default_and_warns_only_on_recovery() { let root = repo_root(); let native = fs::read_to_string(root.join("crates/j2k-native/src/image.rs")) .expect("read native image module"); + let facade_settings = fs::read_to_string(root.join("crates/j2k/src/decode/settings.rs")) + .expect("read facade decode settings"); let facade_decode = fs::read_to_string(root.join("crates/j2k/src/decode.rs")).expect("read facade decode"); let facade_view = read_source_files( @@ -428,7 +430,12 @@ fn decode_strictness_policy_is_explicit_and_warns_on_lenient_default() { ); let facade_batch = read_source_files( root, - &["crates/j2k/src/batch.rs", "crates/j2k/src/batch/direct.rs"], + &[ + "crates/j2k/src/batch.rs", + "crates/j2k/src/batch/direct.rs", + "crates/j2k/src/owned_batch/prepared.rs", + "crates/j2k/src/owned_batch/cpu_execute.rs", + ], ); let crate_readme = fs::read_to_string(root.join("crates/j2k/README.md")).expect("read j2k README"); @@ -440,29 +447,47 @@ fn decode_strictness_policy_is_explicit_and_warns_on_lenient_default() { "pub const fn lenient() -> Self", "pub const fn strict() -> Self", "pub const fn lenient_tolerance_enabled", - "Self::lenient()", + "Self::strict()", + "used_lenient_metadata_recovery", + "Raw codestream validation and entropy decoding remain strict", ]), - PatternCheck::new("j2k facade decode warnings", &facade_decode).required(&[ - "pub enum J2kDecodeWarning", - "LenientDecodeMode", - "decode_warnings_for_settings", - "DecodeOutcome", + PatternCheck::new("facade DecodeSettings strict default", &facade_settings).required(&[ + "pub const fn lenient() -> Self", + "pub const fn strict() -> Self", + "Self::strict()", + "resource-safety", ]), + PatternCheck::new("j2k facade decode warnings", &facade_decode) + .required(&[ + "pub enum J2kDecodeWarning", + "LenientMetadataRecovery", + "decode_warnings_for_recovery", + "used_lenient_metadata_recovery", + "DecodeOutcome", + ]) + .forbidden(&["LenientDecodeMode", "decode_warnings_for_settings"]), PatternCheck::new("j2k facade view warning propagation", &facade_view).required(&[ "type Warning = J2kDecodeWarning", - "decode_warnings_for_settings(DecodeSettings::default())", + "pub fn parse_with_settings(", + "pub fn new_with_settings(", + "decode_warnings_for_image(image)", ]), PatternCheck::new("j2k facade batch warning propagation", &facade_batch).required(&[ "DecodeOutcome", - "decode_warnings_for_settings(DecodeSettings::default())", + "used_lenient_metadata_recovery", + "decode_warnings_for_recovery(", ]), PatternCheck::new("j2k README decode strictness policy", &crate_readme).required(&[ - "DecodeSettings::strict()", - "J2kDecodeWarning::LenientDecodeMode", + "are strict", + "Selecting lenient mode is configuration, not a warning", + "J2kDecodeWarning::LenientMetadataRecovery", + "resource-safety checks remain strict", ]), PatternCheck::new("architecture decode strictness policy", &architecture).required(&[ - "DecodeSettings::strict()", - "J2kDecodeWarning::LenientDecodeMode", + "Decode settings are strict by default", + "never on shared `J2kContext`", + "J2kDecodeWarning::LenientMetadataRecovery", + "only when a recovery actually", ]), ]); } diff --git a/xtask/tests/repo_lint_support/docs_and_workflows_policy/stable_api_governance.rs b/xtask/tests/repo_lint_support/docs_and_workflows_policy/stable_api_governance.rs index 4081d7a6..213e1e82 100644 --- a/xtask/tests/repo_lint_support/docs_and_workflows_policy/stable_api_governance.rs +++ b/xtask/tests/repo_lint_support/docs_and_workflows_policy/stable_api_governance.rs @@ -3,7 +3,8 @@ use std::{collections::BTreeSet, fs}; use crate::repo_lint_support::{ - assert_pattern_checks, const_array_block, repo_root, workflow_job, xtask_sources, PatternCheck, + assert_pattern_checks, const_array_block, release_manifest_entries, repo_root, workflow_job, + PatternCheck, }; #[test] @@ -47,7 +48,6 @@ fn ci_stable_api_jobs_pin_inputs_and_never_write() { #[test] fn stable_api_and_semver_share_one_fail_closed_inventory_contract() { let root = repo_root(); - let xtask = xtask_sources(root); let semver = fs::read_to_string(root.join("xtask/src/semver.rs")).expect("read semver xtask"); let semver_review = fs::read_to_string(root.join("xtask/src/semver/review.rs")) .expect("read semver review policy"); @@ -72,7 +72,7 @@ fn stable_api_and_semver_share_one_fail_closed_inventory_contract() { &command_support, &policy, ); - assert_stable_package_partition(&xtask, &semver, &policy); + assert_published_library_partition(root, &semver, &policy); } fn assert_inventory_contracts( @@ -118,11 +118,14 @@ fn assert_inventory_contracts( PatternCheck::new("live semver inventory ratchet", semver) .required(&[ "SEMVER_TOOLCHAIN: &str = \"1.96\"", - "SEMVER_BASELINE_VERSION: &str = \"0.7.3\"", - "SEMVER_BASELINE_TAG: &str = \"v0.7.3\"", - "SOURCE_INCOMPATIBLE_PATCH_EXCEPTION_VERSION: &str = \"0.7.5\"", + "SEMVER_BASELINE_VERSION: &str = \"0.7.5\"", + "SEMVER_BASELINE_TAG: &str = \"v0.7.5\"", + "INTENTIONAL_BREAK_TRANSITION", + "candidate_version: \"0.8.0\"", + "required_next_baseline_tag: \"v0.8.0\"", + "validate_baseline_transition(", "SEMVER_BASELINE_TAG}:docs/stable-api-1.0.public-api.txt", - "collect_package_apis(stable_packages)?", + "collect_package_apis(published_library_packages)?", "SnapshotKind::Ordinary", "SnapshotKind::Hidden", "stale_ordinary_packages", @@ -132,9 +135,19 @@ fn assert_inventory_contracts( "[\"run\", SEMVER_TOOLCHAIN, \"cargo\"]", "validate_snapshot_scope(", ]) - .forbidden(&["unwrap_or_else(|_| \"1.96\".to_string())"]), + .forbidden(&[ + "unwrap_or_else(|_| \"1.96\".to_string())", + "SOURCE_INCOMPATIBLE_PATCH_EXCEPTION_VERSION", + ]), PatternCheck::new("semver review schema", semver_review).required(&[ - "API review config version must be 2", + "API review config version must be 3", + "break_ledger", + "BreakKind::Source", + "BreakKind::Behavior", + "removed_items", + "summary", + "migration", + "source-break ledger does not exactly cover", "hidden_count", "hidden_fingerprint", "hidden_rationale", @@ -148,9 +161,11 @@ fn assert_inventory_contracts( "String::from_utf8(output.stdout)", ]), PatternCheck::new("stable API policy", policy).required(&[ - "published 0.7.3 artifact recorded both ordinary and hidden-enabled passes", - "explicit, maintainer-approved source-compatibility exception", - "exception applies only to `0.7.5`", + "published 0.7.5 artifact recorded both ordinary and hidden-enabled passes", + "`0.8.0` is the only candidate permitted to compare against `v0.7.5`", + "rotate the semver baseline to the published `v0.8.0`", + "Source-break entries must enumerate every exact removed API item", + "Behavior-break entries carry the same summary and migration requirements", "complete hidden-inventory count and fingerprint", "Every semver invocation collects both live passes", "Nonempty hidden inventories also require a package-specific hidden rationale", @@ -162,21 +177,40 @@ fn assert_inventory_contracts( ]); } -fn assert_stable_package_partition(xtask: &str, semver: &str, policy: &str) { - let stable = const_string_array_values(xtask, "STABLE_SEMVER_PACKAGES"); +fn assert_published_library_partition(root: &std::path::Path, semver: &str, policy: &str) { + let entries = release_manifest_entries(root); + let published = entries + .iter() + .filter(|entry| entry.api_contract != "binary") + .map(|entry| entry.name.clone()) + .collect::>(); + let stable = entries + .iter() + .filter(|entry| entry.api_contract == "stable") + .map(|entry| entry.name.clone()) + .collect::>(); let baseline = const_string_array_values(semver, "SEMVER_BASELINE_PACKAGES"); let new = const_string_array_values(semver, "SEMVER_NEW_PACKAGES"); - assert!(!stable.is_empty()); + assert_eq!(published.len(), 18); + assert_eq!( + stable, + ["j2k", "j2k-core", "j2k-jpeg", "j2k-tilecodec"] + .into_iter() + .map(str::to_string) + .collect() + ); assert!(!baseline.is_empty()); + assert!(new.is_empty()); + assert!(baseline.contains("j2k-ml")); assert!(baseline.is_disjoint(&new)); assert_eq!( baseline.union(&new).cloned().collect::>(), - stable + published ); - for package in stable { + for package in published { assert!( policy.contains(&format!("`{package}`")), - "stable API policy must list `{package}`" + "published API policy must list `{package}`" ); } } diff --git a/xtask/tests/repo_lint_support/docs_and_workflows_policy/structural_ratchets.rs b/xtask/tests/repo_lint_support/docs_and_workflows_policy/structural_ratchets.rs index 167d2585..9bfda23a 100644 --- a/xtask/tests/repo_lint_support/docs_and_workflows_policy/structural_ratchets.rs +++ b/xtask/tests/repo_lint_support/docs_and_workflows_policy/structural_ratchets.rs @@ -1,175 +1,3 @@ // SPDX-License-Identifier: MIT OR Apache-2.0 -use std::fs; - -use crate::repo_lint_support::{ - assert_file_pattern_checks, assert_pattern_checks, read_source_files, repo_root, - FilePatternCheck, PatternCheck, -}; - mod lint_ratchets; -mod living_audit; -mod policy_line_limits; -mod policy_ownership; - -#[test] -fn large_test_files_stay_split_by_axis() { - let root = repo_root(); - for (relative, max_lines) in [ - ("crates/j2k-metal/src/encode/tests.rs", 150), - ("crates/j2k-metal/src/encode/tests/batch.rs", 450), - ("crates/j2k-metal/src/encode/tests/dwt_parity.rs", 250), - ("crates/j2k-metal/src/encode/tests/kernels.rs", 1_300), - ("crates/j2k-metal/src/encode/tests/layouts.rs", 250), - ("crates/j2k-metal/src/encode/tests/resident_batches.rs", 725), - ("crates/j2k-metal/src/encode/tests/resident_buffers.rs", 950), - ("crates/j2k-metal/src/encode/tests/routing.rs", 850), - ("crates/j2k-metal/src/encode/tests/stage_validation.rs", 650), - ("crates/j2k-metal/src/encode/tests/stats_inflight.rs", 950), - ("crates/j2k-jpeg-metal/src/tests.rs", 2_400), - ("crates/j2k-jpeg-metal/src/tests/reusable_output.rs", 250), - ("crates/j2k-jpeg-metal/src/tests/textures.rs", 2_300), - ("crates/j2k-jpeg-metal/src/tests/textures/residency.rs", 175), - ("crates/j2k-cuda-runtime/src/tests.rs", 2_300), - ("crates/j2k-cuda-runtime/src/tests/pipeline.rs", 2_400), - ("crates/j2k-jpeg-cuda/tests/host_surface.rs", 25), - ("crates/j2k-jpeg-cuda/tests/host_surface/batch.rs", 200), - ( - "crates/j2k-jpeg-cuda/tests/host_surface/caller_buffers.rs", - 225, - ), - ( - "crates/j2k-jpeg-cuda/tests/host_surface/diagnostics.rs", - 110, - ), - ( - "crates/j2k-jpeg-cuda/tests/host_surface/owned_decode.rs", - 100, - ), - ( - "crates/j2k-jpeg-cuda/tests/host_surface/output_initialization.rs", - 125, - ), - ("crates/j2k-jpeg-cuda/tests/host_surface/regions.rs", 180), - ("crates/j2k-jpeg-cuda/tests/host_surface/routing.rs", 200), - ( - "crates/j2k-jpeg-cuda/tests/host_surface/submissions.rs", - 125, - ), - ("crates/j2k-jpeg-cuda/tests/host_surface/support.rs", 125), - ("crates/j2k-jpeg/tests/decode_into.rs", 2_000), - ("crates/j2k-jpeg/tests/decode_into/lossless.rs", 1_600), - ("crates/j2k-jpeg/tests/decode_into/color.rs", 900), - ] { - let source = fs::read_to_string(root.join(relative)) - .unwrap_or_else(|error| panic!("read {relative}: {error}")); - assert!( - source.lines().count() < max_lines, - "{relative} must stay below the split test-file line-count ratchet" - ); - } -} - -#[test] -fn decode_capability_correctness_regressions_are_guarded() { - let root = repo_root(); - let native_codestream = read_source_files( - root, - &[ - "crates/j2k-native/src/j2c/codestream/header.rs", - "crates/j2k-native/src/j2c/codestream/tests.rs", - ], - ); - assert_pattern_checks(&[PatternCheck::new( - "target-resolution shrink-factor arithmetic", - &native_codestream, - ) - .required(&[ - ".checked_shl(u32::from(skipped_resolution_levels))", - ".checked_mul(resolution_shrink_factor)", - "size_data.checked_image_width()?;", - "size_data.checked_image_height()?;", - "checked_image_dimensions_reject_shrink_factor_overflow", - ])]); - assert_file_pattern_checks( - root, - &[FilePatternCheck::new("crates/j2k-jpeg/tests/inspect.rs") - .named("JPEG progressive inspect/decode agreement fixtures") - .required(&[ - "fn inspect_and_decoder_info_agree_for_progressive_fixtures()", - "progressive_8x8_jpeg()", - "progressive_12bit_grayscale_8x8_jpeg()", - "progressive_12bit_rgb_8x8_jpeg()", - "assert_eq!(decoder.info(), &inspected, \"{label}\");", - ])], - ); -} - -#[test] -fn docs_and_workflows_policy_children_stay_split_by_responsibility() { - let root = repo_root(); - let shell_relative = "xtask/tests/repo_lint_support/docs_and_workflows_policy.rs"; - let shell = fs::read_to_string(root.join(shell_relative)) - .unwrap_or_else(|error| panic!("read {shell_relative}: {error}")); - assert!( - shell.lines().count() < 25, - "{shell_relative} must remain a focused child-module shell" - ); - assert_pattern_checks(&[ - PatternCheck::new("docs/workflows policy module shell", &shell).required(&[ - "mod adoption_benchmark_policy;", - "mod decoder_fixture_policy;", - "mod documentation_api_evidence;", - "mod duplication_policy;", - "mod encoder_architecture_policy;", - "mod stable_api_evidence;", - "mod stable_api_governance;", - "mod structural_ratchets;", - "mod workflow_coverage_policy;", - ]), - ]); - - let policy_dir = "xtask/tests/repo_lint_support/docs_and_workflows_policy"; - for (leaf, max_lines) in [ - ("adoption_benchmark_policy.rs", 225), - ("documentation_api_evidence.rs", 100), - ("stable_api_evidence.rs", 175), - ("stable_api_governance.rs", 225), - ("workflow_coverage_policy.rs", 500), - ("structural_ratchets.rs", 225), - ("structural_ratchets/lint_ratchets.rs", 150), - ("structural_ratchets/policy_line_limits.rs", 275), - ("structural_ratchets/policy_ownership.rs", 175), - ("structural_ratchets/policy_ownership/gpu_adapter.rs", 100), - ("structural_ratchets/policy_ownership/j2k_ml.rs", 100), - ("structural_ratchets/policy_ownership/metal_compute.rs", 75), - ("structural_ratchets/policy_ownership/path_patches.rs", 75), - ("structural_ratchets/policy_ownership/public_docs.rs", 75), - ( - "structural_ratchets/policy_ownership/repository_domains.rs", - 175, - ), - ("duplication_policy.rs", 625), - ("duplication_policy/cache_identity.rs", 100), - ("duplication_policy/classic_mq.rs", 75), - ("duplication_policy/pass_through.rs", 175), - ("encoder_architecture_policy.rs", 700), - ("encoder_architecture_policy/native_contracts.rs", 150), - ("decoder_fixture_policy.rs", 475), - ] { - let relative = format!("{policy_dir}/{leaf}"); - let source = fs::read_to_string(root.join(&relative)) - .unwrap_or_else(|error| panic!("read {relative}: {error}")); - assert!( - source.lines().count() < max_lines, - "{relative} must stay below its focused policy line-count ratchet of {max_lines}" - ); - assert!( - !source.lines().any(|line| line.trim() == "use super::*;") - && !source - .lines() - .any(|line| line.trim_start().starts_with("include!(")), - "{relative} must use explicit real-Rust module boundaries" - ); - } -} diff --git a/xtask/tests/repo_lint_support/docs_and_workflows_policy/structural_ratchets/lint_ratchets.rs b/xtask/tests/repo_lint_support/docs_and_workflows_policy/structural_ratchets/lint_ratchets.rs index db7acfa5..5ba7fb32 100644 --- a/xtask/tests/repo_lint_support/docs_and_workflows_policy/structural_ratchets/lint_ratchets.rs +++ b/xtask/tests/repo_lint_support/docs_and_workflows_policy/structural_ratchets/lint_ratchets.rs @@ -2,13 +2,7 @@ use std::fs; -use crate::repo_lint_support::{repo_root, rust_sources}; -use syn::{ - parse::Parser, - punctuated::Punctuated, - visit::{self, Visit}, - Attribute, Meta, Token, -}; +use crate::repo_lint_support::repo_root; fn source_before_cfg_test_module<'a>(source: &'a str, relative: &str) -> &'a str { source.split_once("#[cfg(test)]\nmod tests").map_or_else( @@ -25,7 +19,6 @@ fn source_before_cfg_test_module<'a>(source: &'a str, relative: &str) -> &'a str #[test] fn panic_hotspot_production_paths_do_not_use_unwrap_or_expect() { - let root = repo_root(); for relative in [ "crates/j2k-cuda/src/encode.rs", "crates/j2k-jpeg/src/entropy/block.rs", @@ -38,8 +31,8 @@ fn panic_hotspot_production_paths_do_not_use_unwrap_or_expect() { "crates/j2k-jpeg/src/entropy/progressive/render.rs", "crates/j2k-jpeg/src/entropy/sequential.rs", ] { - let source = fs::read_to_string(root.join(relative)) - .unwrap_or_else(|err| panic!("read {relative}: {err}")); + let source = fs::read_to_string(repo_root().join(relative)) + .unwrap_or_else(|error| panic!("read {relative}: {error}")); let production = source_before_cfg_test_module(&source, relative); for forbidden in [".unwrap(", ".expect("] { assert!( @@ -49,82 +42,3 @@ fn panic_hotspot_production_paths_do_not_use_unwrap_or_expect() { } } } - -#[test] -fn too_many_arguments_suppressions_stay_below_current_ratchet() { - const MAX_SUPPRESSIONS: usize = 106; - let root = repo_root(); - let mut sources = rust_sources(&root.join("crates")); - sources.extend(rust_sources(&root.join("xtask"))); - assert!( - !sources.is_empty(), - "too_many_arguments ratchet must scan Rust sources" - ); - - let mut count = 0usize; - for path in sources { - let source = fs::read_to_string(&path).unwrap_or_else(|err| panic!("read {path:?}: {err}")); - count += count_too_many_arguments_suppressions(&source); - } - - assert!( - count <= MAX_SUPPRESSIONS, - "too_many_arguments suppression count must not exceed the current ratchet: found {count}, expected <= {MAX_SUPPRESSIONS}" - ); -} - -fn count_too_many_arguments_suppressions(source: &str) -> usize { - #[derive(Default)] - struct SuppressionCounter(usize); - - impl<'ast> Visit<'ast> for SuppressionCounter { - fn visit_attribute(&mut self, attribute: &'ast Attribute) { - if meta_suppresses_too_many_arguments(&attribute.meta) { - self.0 += 1; - } - visit::visit_attribute(self, attribute); - } - } - - fn meta_suppresses_too_many_arguments(meta: &Meta) -> bool { - let Meta::List(list) = meta else { - return false; - }; - let nested = Punctuated::::parse_terminated - .parse2(list.tokens.clone()) - .unwrap_or_default(); - if list.path.is_ident("allow") || list.path.is_ident("expect") { - return nested.iter().any(|meta| { - matches!(meta, Meta::Path(path) if path.segments.len() == 2 - && path.segments[0].ident == "clippy" - && path.segments[1].ident == "too_many_arguments") - }); - } - list.path.is_ident("cfg_attr") && nested.iter().any(meta_suppresses_too_many_arguments) - } - - let syntax = syn::parse_file(source).expect("parse Rust source for lint-suppression ratchet"); - let mut counter = SuppressionCounter::default(); - counter.visit_file(&syntax); - counter.0 -} - -#[test] -fn too_many_arguments_ratchet_counts_allow_and_expect_attributes() { - let lint = ["clippy::too_many_", "arguments"].concat(); - let source = [ - "#[", - "allow(", - lint.as_str(), - ", reason = \"device ABI\")]\nfn allowed() {}\n#[", - "expect(", - lint.as_str(), - ", reason = \"stable codec boundary\")]\nfn expected() {}\n#[cfg_attr(test, expect(", - lint.as_str(), - ", reason = \"test fixture\"))]\nfn test_expected() {}\n#[cfg_attr(feature = \"ffi\", allow(", - lint.as_str(), - ", reason = \"conditional ABI\"))]\nfn conditionally_allowed() {}\n", - ] - .concat(); - assert_eq!(count_too_many_arguments_suppressions(&source), 4); -} diff --git a/xtask/tests/repo_lint_support/docs_and_workflows_policy/structural_ratchets/living_audit.rs b/xtask/tests/repo_lint_support/docs_and_workflows_policy/structural_ratchets/living_audit.rs deleted file mode 100644 index d557bda3..00000000 --- a/xtask/tests/repo_lint_support/docs_and_workflows_policy/structural_ratchets/living_audit.rs +++ /dev/null @@ -1,68 +0,0 @@ -// SPDX-License-Identifier: MIT OR Apache-2.0 - -use std::fs; - -use crate::repo_lint_support::repo_root; - -const LIVING_AUDIT: &str = "engineering/ai-codebase-audit-remediation-plan.md"; -const LIVING_AUDIT_MAX_LINES: usize = 500; - -#[test] -fn living_audit_stays_current_and_bounded() { - let source = fs::read_to_string(repo_root().join(LIVING_AUDIT)) - .unwrap_or_else(|error| panic!("read {LIVING_AUDIT}: {error}")); - - assert!( - source.lines().count() < LIVING_AUDIT_MAX_LINES, - "{LIVING_AUDIT} is a current-state register, not a historical task diary; keep it below {LIVING_AUDIT_MAX_LINES} lines and use Git history for completed work" - ); - for stale_claim in [ - "workspace `0.7.0` remains staged", - "0.7.0 remains staged", - "blocked 0.7.0", - "Final candidate ratio: pending", - ] { - assert!( - !source.contains(stale_claim), - "{LIVING_AUDIT} contains stale present-tense release claim {stale_claim:?}" - ); - } - for required in [ - "v0.7.3 verdict", - "Active debt", - "Accepted large-file register", - "Accepted clone register", - "Verification matrix", - ] { - assert!( - source.contains(required), - "{LIVING_AUDIT} must retain the current-state section {required:?}" - ); - } -} - -#[test] -fn current_candidate_matrix_does_not_claim_invalidated_green_evidence() { - let source = fs::read_to_string(repo_root().join(LIVING_AUDIT)) - .unwrap_or_else(|error| panic!("read {LIVING_AUDIT}: {error}")); - let matrix = source - .split("## Verification matrix") - .nth(1) - .and_then(|tail| tail.split("## Living-document rule").next()) - .expect("verification matrix section"); - - for row in matrix.lines().filter(|line| line.starts_with("| `")) { - let columns = row.split('|').map(str::trim).collect::>(); - let candidate = columns - .get(3) - .unwrap_or_else(|| panic!("candidate evidence column in {row:?}")); - assert!( - !candidate.contains("pass"), - "current candidate evidence was invalidated by later source edits: {row}" - ); - assert!( - candidate.contains("pending") || candidate.contains("required"), - "current candidate row must state its pending or required final validation: {row}" - ); - } -} diff --git a/xtask/tests/repo_lint_support/docs_and_workflows_policy/structural_ratchets/policy_line_limits.rs b/xtask/tests/repo_lint_support/docs_and_workflows_policy/structural_ratchets/policy_line_limits.rs deleted file mode 100644 index 56b30654..00000000 --- a/xtask/tests/repo_lint_support/docs_and_workflows_policy/structural_ratchets/policy_line_limits.rs +++ /dev/null @@ -1,225 +0,0 @@ -// SPDX-License-Identifier: MIT OR Apache-2.0 - -use std::fs; - -use crate::repo_lint_support::repo_root; - -const REPO_LINT_POLICY_LINE_LIMITS: &[(&str, usize)] = &[ - ( - "xtask/tests/repo_lint_support/audit_integrity_policy.rs", - 200, - ), - ( - "xtask/tests/repo_lint_support/docs_and_workflows_policy.rs", - 2_750, - ), - ( - "xtask/tests/repo_lint_support/encode_compare_structure_policy.rs", - 250, - ), - ( - "xtask/tests/repo_lint_support/fixture_compare_structure_policy.rs", - 250, - ), - ("xtask/tests/repo_lint_support/gpu_adapter_policy.rs", 1_800), - ( - "xtask/tests/repo_lint_support/gpu_adapter_policy/cuda_decoder_policy.rs", - 250, - ), - ( - "xtask/tests/repo_lint_support/gpu_adapter_policy/cuda_decoder_policy/architecture.rs", - 175, - ), - ( - "xtask/tests/repo_lint_support/gpu_adapter_policy/cuda_decoder_policy/architecture/grayscale_batch.rs", - 100, - ), - ( - "xtask/tests/repo_lint_support/gpu_adapter_policy/cuda_decoder_policy/line_ratchets.rs", - 200, - ), - ( - "xtask/tests/repo_lint_support/gpu_adapter_policy/cuda_decoder_policy/color_runtime.rs", - 150, - ), - ("xtask/tests/repo_lint_support/gpu_adapter_policy/cuda_decoder_policy/resident_leaf_structure.rs", 75), - ("xtask/tests/repo_lint_support/gpu_adapter_policy/cuda_decoder_policy/resident_leaf_structure/classic.rs", 50), - ( - "xtask/tests/repo_lint_support/gpu_adapter_policy/cuda_runtime_safety_policy.rs", - 200, - ), - ( - "xtask/tests/repo_lint_support/gpu_adapter_policy/cuda_runtime_safety_policy/allocation.rs", - 225, - ), - ( - "xtask/tests/repo_lint_support/gpu_adapter_policy/cuda_runtime_safety_policy/lifecycle.rs", - 175, - ), - ( - "xtask/tests/repo_lint_support/gpu_adapter_policy/cuda_runtime_safety_policy/lifecycle/context.rs", - 75, - ), - ( - "xtask/tests/repo_lint_support/gpu_adapter_policy/cuda_runtime_safety_policy/lifecycle/context/transitions.rs", - 100, - ), - ( - "xtask/tests/repo_lint_support/gpu_adapter_policy/cuda_runtime_safety_policy/lifecycle/queued.rs", - 70, - ), - ( - "xtask/tests/repo_lint_support/gpu_adapter_policy/cuda_runtime_safety_policy/lifecycle/queued/status.rs", - 50, - ), - ( - "xtask/tests/repo_lint_support/gpu_adapter_policy/cuda_runtime_safety_policy/validation.rs", - 225, - ), - ( - "xtask/tests/repo_lint_support/gpu_adapter_policy/cuda_runtime_safety_policy/validation/htj2k_output.rs", - 175, - ), - ( - "xtask/tests/repo_lint_support/gpu_adapter_policy/cuda_runtime_safety_policy/validation/htj2k_output/planning.rs", - 75, - ), - ( - "xtask/tests/repo_lint_support/gpu_adapter_policy/jpeg_allocation_policy.rs", - 100, - ), - ( - "xtask/tests/repo_lint_support/gpu_adapter_policy/jpeg_allocation_policy/allocation_sources.rs", - 300, - ), - ( - "xtask/tests/repo_lint_support/gpu_adapter_policy/jpeg_allocation_policy/checks.rs", - 75, - ), - ( - "xtask/tests/repo_lint_support/gpu_adapter_policy/jpeg_allocation_policy/checks/adapter_checks.rs", - 75, - ), - ( - "xtask/tests/repo_lint_support/gpu_adapter_policy/jpeg_allocation_policy/checks/checkpoint_checks.rs", - 110, - ), - ( - "xtask/tests/repo_lint_support/gpu_adapter_policy/jpeg_allocation_policy/checks/packet_checks.rs", - 130, - ), - ( - "xtask/tests/repo_lint_support/gpu_adapter_policy/jpeg_allocation_policy/checks/structure_checks.rs", - 140, - ), - ( - "xtask/tests/repo_lint_support/gpu_adapter_policy/jpeg_allocation_policy/encoder_checks/contracts.rs", - 110, - ), - ( - "xtask/tests/repo_lint_support/gpu_adapter_policy/jpeg_allocation_policy/gpu_capacity.rs", - 100, - ), - ( - "xtask/tests/repo_lint_support/gpu_adapter_policy/jpeg_metal_compute_structure_policy.rs", - 300, - ), - ( - "xtask/tests/repo_lint_support/gpu_adapter_policy/jpeg_metal_viewport_structure_policy.rs", - 175, - ), - ( - "xtask/tests/repo_lint_support/gpu_adapter_policy/resident_encode_policy.rs", - 150, - ), - ( - "xtask/tests/repo_lint_support/gpu_device_structure_policy.rs", - 500, - ), - ( - "xtask/tests/repo_lint_support/jpeg_decoder_structure_policy.rs", - 400, - ), - ( - "xtask/tests/repo_lint_support/jpeg_decoder_structure_policy/adapter_tests.rs", - 260, - ), - ( - "xtask/tests/repo_lint_support/jpeg_decoder_structure_policy/owned_output.rs", - 75, - ), - ( - "xtask/tests/repo_lint_support/jpeg_decoder_structure_policy/support_contracts.rs", - 100, - ), - ( - "xtask/tests/repo_lint_support/jpeg_encoder_structure_policy.rs", - 225, - ), - ( - "xtask/tests/repo_lint_support/jpeg_restart_policy.rs", - 125, - ), - ( - "xtask/tests/repo_lint_support/jpeg_restart_policy/counts.rs", - 75, - ), - ( - "xtask/tests/repo_lint_support/jpeg_metal_resource_safety_policy.rs", - 350, - ), - ( - "xtask/tests/repo_lint_support/metal_compute_structure_policy.rs", - 550, - ), - ( - "xtask/tests/repo_lint_support/tilecodec_error_policy.rs", - 100, - ), - ("xtask/tests/repo_lint_support/transcode_api_policy.rs", 125), - ( - "xtask/tests/repo_lint_support/transcode_structure_policy.rs", - 375, - ), - ( - "xtask/tests/repo_lint_support/transcode_structure_policy/cpu.rs", - 250, - ), - ( - "xtask/tests/repo_lint_support/transcode_structure_policy/cpu/batch.rs", - 225, - ), - ( - "xtask/tests/repo_lint_support/transcode_structure_policy/metal.rs", - 150, - ), - ( - "xtask/tests/repo_lint_support/xtask_main_structure_policy.rs", - 300, - ), - ( - "xtask/tests/repo_lint_support/xtask_main_structure_policy/codegen.rs", - 100, - ), - ( - "xtask/tests/repo_lint_support/xtask_main_structure_policy/lint_policy.rs", - 50, - ), - ( - "xtask/tests/repo_lint_support/xtask_main_structure_policy/release_integrity.rs", - 100, - ), -]; - -#[test] -fn repo_lint_policy_support_files_stay_split_by_axis() { - let root = repo_root(); - for &(relative, max_lines) in REPO_LINT_POLICY_LINE_LIMITS { - let source = fs::read_to_string(root.join(relative)) - .unwrap_or_else(|error| panic!("read {relative}: {error}")); - assert!( - source.lines().count() < max_lines, - "{relative} must stay below the split repo-lint policy line-count ratchet" - ); - } -} diff --git a/xtask/tests/repo_lint_support/docs_and_workflows_policy/structural_ratchets/policy_ownership.rs b/xtask/tests/repo_lint_support/docs_and_workflows_policy/structural_ratchets/policy_ownership.rs deleted file mode 100644 index 9302625d..00000000 --- a/xtask/tests/repo_lint_support/docs_and_workflows_policy/structural_ratchets/policy_ownership.rs +++ /dev/null @@ -1,8 +0,0 @@ -// SPDX-License-Identifier: MIT OR Apache-2.0 - -mod gpu_adapter; -mod j2k_ml; -mod metal_compute; -mod path_patches; -mod public_docs; -mod repository_domains; diff --git a/xtask/tests/repo_lint_support/docs_and_workflows_policy/structural_ratchets/policy_ownership/gpu_adapter.rs b/xtask/tests/repo_lint_support/docs_and_workflows_policy/structural_ratchets/policy_ownership/gpu_adapter.rs deleted file mode 100644 index 3a2af6fa..00000000 --- a/xtask/tests/repo_lint_support/docs_and_workflows_policy/structural_ratchets/policy_ownership/gpu_adapter.rs +++ /dev/null @@ -1,71 +0,0 @@ -// SPDX-License-Identifier: MIT OR Apache-2.0 - -use std::fs; - -use crate::repo_lint_support::repo_root; - -#[test] -fn gpu_adapter_policy_is_a_shell_over_focused_contract_domains() { - let root = repo_root(); - let policy_root = "xtask/tests/repo_lint_support/gpu_adapter_policy.rs"; - let shell = fs::read_to_string(root.join(policy_root)) - .unwrap_or_else(|error| panic!("read {policy_root}: {error}")); - assert!(shell.lines().count() < 50); - assert!(!shell.contains("#[test]")); - assert!(!shell.lines().any(|line| line.trim() == "use super::*;")); - - for (module, symbol, max_lines) in [ - ( - "error_contracts", - "fn gpu_adapter_error_classification_uses_shared_core_impl(", - 200, - ), - ( - "jpeg_encode_orchestration", - "fn jpeg_gpu_encode_host_orchestration_uses_shared_adapter_helper(", - 200, - ), - ( - "metal_abi_contracts", - "fn jpeg_metal_huffman_derivation_uses_shared_entropy_canonical_tables(", - 200, - ), - ( - "jpeg_metal_fast_paths", - "fn fast444_region_scaled_batches_use_shared_region_scaled_metal_path(", - 250, - ), - ( - "cuda_adapter_contracts", - "fn cuda_htj2k_compact_jobs_use_shared_planner(", - 325, - ), - ( - "transcode_contracts", - "fn transcode_gpu_auto_threshold_policy_is_documented(", - 150, - ), - ( - "metal_public_api", - "fn metal_public_error_lives_in_focused_module(", - 325, - ), - ( - "decode_request_contracts", - "fn jpeg_metal_viewport_plane_rows_use_shared_target(", - 225, - ), - ] { - assert!(shell.contains(&format!("mod {module};"))); - assert!(!shell.contains(symbol)); - let relative = format!("xtask/tests/repo_lint_support/gpu_adapter_policy/{module}.rs"); - let source = fs::read_to_string(root.join(&relative)) - .unwrap_or_else(|error| panic!("read {relative}: {error}")); - assert!(source.contains(symbol), "{relative} must own {symbol}"); - assert!(source.lines().count() < max_lines); - assert!(!source.lines().any(|line| line.trim() == "use super::*;")); - assert!(!source - .lines() - .any(|line| line.trim_start().starts_with("include!("))); - } -} diff --git a/xtask/tests/repo_lint_support/docs_and_workflows_policy/structural_ratchets/policy_ownership/j2k_ml.rs b/xtask/tests/repo_lint_support/docs_and_workflows_policy/structural_ratchets/policy_ownership/j2k_ml.rs deleted file mode 100644 index 4d650f09..00000000 --- a/xtask/tests/repo_lint_support/docs_and_workflows_policy/structural_ratchets/policy_ownership/j2k_ml.rs +++ /dev/null @@ -1,84 +0,0 @@ -// SPDX-License-Identifier: MIT OR Apache-2.0 - -use std::fs; - -use crate::repo_lint_support::repo_root; - -#[test] -fn j2k_ml_policy_keeps_domain_specific_children() { - let root = repo_root().join("xtask/tests/repo_lint_support"); - let shell_path = root.join("j2k_ml_policy.rs"); - let shell = fs::read_to_string(&shell_path).expect("read j2k-ml policy root"); - - for module in [ - "mod adapter;", - "mod benchmark_evidence;", - "mod benchmark_prepare_policy;", - "mod benchmark_support_structure;", - "mod features;", - "mod interop;", - ] { - assert!( - shell.contains(module), - "j2k-ml policy root must own {module}" - ); - } - assert!(shell.contains("fn read(relative: &str) -> String")); - assert!(!shell.contains("#[test]")); - assert!(!shell.contains("RustEvidence")); - assert!(!shell.contains("include!")); - assert!( - shell.lines().count() < 30, - "j2k-ml policy root must remain a thin module/shared-read owner" - ); - - for (relative, symbols, max_lines) in [ - ( - "j2k_ml_policy/adapter.rs", - &[ - "fn j2k_ml_is_a_thin_persistent_batch_adapter(", - "fn metal_upload_burn_decoder_keeps_batch_options_in_the_codec_session_only(", - ][..], - 125usize, - ), - ( - "j2k_ml_policy/features.rs", - &[ - "fn j2k_ml_stays_independent_publishable_and_explicitly_feature_gated(", - "fn j2k_ml_uses_a_portable_arm_linux_test_backend(", - ][..], - 100usize, - ), - ( - "j2k_ml_policy/interop.rs", - &["fn j2k_ml_accelerator_upload_contracts_are_source_enforced("][..], - 100usize, - ), - ( - "j2k_ml_policy/benchmark_evidence.rs", - &[ - "struct RustEvidence", - "impl<'ast> Visit<'ast> for RustEvidence", - "fn rust_evidence(", - "fn j2k_ml_batch_benchmarks_cover_native_medical_outputs_and_all_requests(", - ][..], - 250usize, - ), - ] { - let source = fs::read_to_string(root.join(relative)) - .unwrap_or_else(|error| panic!("read {relative}: {error}")); - for symbol in symbols { - assert!(source.contains(symbol), "{relative} must own {symbol}"); - assert!( - !shell.contains(symbol), - "j2k-ml policy root must not own {symbol}" - ); - } - assert!( - source.lines().count() < max_lines, - "{relative} must stay below its domain-policy line-count ratchet" - ); - assert!(!source.lines().any(|line| line.trim() == "use super::*;")); - assert!(!source.contains("include!")); - } -} diff --git a/xtask/tests/repo_lint_support/docs_and_workflows_policy/structural_ratchets/policy_ownership/metal_compute.rs b/xtask/tests/repo_lint_support/docs_and_workflows_policy/structural_ratchets/policy_ownership/metal_compute.rs deleted file mode 100644 index 1bda8fa6..00000000 --- a/xtask/tests/repo_lint_support/docs_and_workflows_policy/structural_ratchets/policy_ownership/metal_compute.rs +++ /dev/null @@ -1,46 +0,0 @@ -// SPDX-License-Identifier: MIT OR Apache-2.0 - -use std::fs; - -use crate::repo_lint_support::repo_root; - -#[test] -fn metal_compute_policy_root_is_a_focused_module_shell() { - let root = repo_root(); - let shell_relative = "xtask/tests/repo_lint_support/metal_compute_structure_policy.rs"; - let shell = fs::read_to_string(root.join(shell_relative)) - .unwrap_or_else(|error| panic!("read {shell_relative}: {error}")); - assert!(shell.lines().count() < 25); - assert!(!shell - .lines() - .any(|line| line.trim_start().starts_with("use "))); - - for (module, symbol, max_lines) in [ - ( - "runtime_registry", - "fn metal_compute_runtime_registry_is_split_from_compute_god_file(", - 300, - ), - ( - "direct_plan_types", - "fn metal_direct_plan_types_live_in_focused_module(", - 300, - ), - ] { - assert!(shell.contains(&format!("mod {module};"))); - assert!(!shell.contains(symbol)); - let relative = - format!("xtask/tests/repo_lint_support/metal_compute_structure_policy/{module}.rs"); - let source = fs::read_to_string(root.join(&relative)) - .unwrap_or_else(|error| panic!("read {relative}: {error}")); - assert!(source.contains(symbol)); - assert!(source.lines().count() < max_lines); - assert!(!source.lines().any(|line| line.trim() == "use super::*;")); - } - - let status_relative = - "xtask/tests/repo_lint_support/metal_compute_structure_policy/status_contracts.rs"; - let status = fs::read_to_string(root.join(status_relative)) - .unwrap_or_else(|error| panic!("read {status_relative}: {error}")); - assert!(!status.lines().any(|line| line.trim() == "use super::*;")); -} diff --git a/xtask/tests/repo_lint_support/docs_and_workflows_policy/structural_ratchets/policy_ownership/path_patches.rs b/xtask/tests/repo_lint_support/docs_and_workflows_policy/structural_ratchets/policy_ownership/path_patches.rs deleted file mode 100644 index 32eac339..00000000 --- a/xtask/tests/repo_lint_support/docs_and_workflows_policy/structural_ratchets/policy_ownership/path_patches.rs +++ /dev/null @@ -1,46 +0,0 @@ -// SPDX-License-Identifier: MIT OR Apache-2.0 - -use std::fs; - -use crate::repo_lint_support::repo_root; - -#[test] -fn path_patch_governance_has_focused_owners() { - let root = repo_root(); - for (owner, child, module_decl, symbols, owner_limit, child_limit) in [ - ( - "xtask/tests/repo_lint_support/dependency_policy.rs", - "xtask/tests/repo_lint_support/dependency_policy/path_patches.rs", - "mod path_patches;", - &[ - "fn all_workspace_path_patches_have_pinned_provenance_and_local_digests(", - "fn patched_tree_sha256(", - ][..], - 250usize, - 325usize, - ), - ( - "xtask/src/release_commands.rs", - "xtask/src/release_commands/path_patches.rs", - "mod path_patches;", - &["fn workspace_path_patch_provenance_paths("][..], - 700usize, - 100usize, - ), - ] { - let owner_source = fs::read_to_string(root.join(owner)) - .unwrap_or_else(|error| panic!("read {owner}: {error}")); - let child_source = fs::read_to_string(root.join(child)) - .unwrap_or_else(|error| panic!("read {child}: {error}")); - assert!(owner_source.contains(module_decl)); - for symbol in symbols { - assert!(!owner_source.contains(symbol)); - assert!(child_source.contains(symbol)); - } - assert!(owner_source.lines().count() < owner_limit); - assert!(child_source.lines().count() < child_limit); - assert!(!child_source - .lines() - .any(|line| line.trim() == "use super::*;")); - } -} diff --git a/xtask/tests/repo_lint_support/docs_and_workflows_policy/structural_ratchets/policy_ownership/public_docs.rs b/xtask/tests/repo_lint_support/docs_and_workflows_policy/structural_ratchets/policy_ownership/public_docs.rs deleted file mode 100644 index e8c27ffd..00000000 --- a/xtask/tests/repo_lint_support/docs_and_workflows_policy/structural_ratchets/policy_ownership/public_docs.rs +++ /dev/null @@ -1,53 +0,0 @@ -// SPDX-License-Identifier: MIT OR Apache-2.0 - -use std::fs; - -use crate::repo_lint_support::repo_root; - -#[test] -fn public_docs_policy_is_a_shell_over_focused_contract_domains() { - let root = repo_root(); - let shell_relative = "xtask/tests/repo_lint_support/public_docs_policy.rs"; - let shell = fs::read_to_string(root.join(shell_relative)) - .unwrap_or_else(|error| panic!("read {shell_relative}: {error}")); - assert!(shell.lines().count() < 25); - - for (module, owned_symbol, max_lines) in [ - ("environment", "fn supported_j2k_env_vars_are_documented(", 125), - ( - "accelerator_evidence", - "fn accelerator_support_and_benchmark_evidence_have_single_document_owners(", - 250, - ), - ( - "navigation_packaging", - "fn published_crates_have_crates_io_landing_readmes(", - 400, - ), - ( - "benchmark_publication", - "fn benchmark_docs_define_publication_gate_for_openjpeg_and_grok(", - 250, - ), - ( - "metal_safety", - "fn metal_consistency_cleanup_keeps_names_status_buffers_and_marker_sizes_single_sourced(", - 300, - ), - ( - "repository_hygiene", - "fn public_text_does_not_embed_local_user_home_paths(", - 175, - ), - ] { - assert!(shell.contains(&format!("mod {module};"))); - assert!(!shell.contains(owned_symbol)); - let relative = format!("xtask/tests/repo_lint_support/public_docs_policy/{module}.rs"); - let source = fs::read_to_string(root.join(&relative)) - .unwrap_or_else(|error| panic!("read {relative}: {error}")); - assert!(source.contains(owned_symbol)); - assert!(source.lines().count() < max_lines); - assert!(!source.lines().any(|line| line.trim() == "use super::*;")); - assert!(!source.lines().any(|line| line.trim_start().starts_with("include!("))); - } -} diff --git a/xtask/tests/repo_lint_support/docs_and_workflows_policy/structural_ratchets/policy_ownership/repository_domains.rs b/xtask/tests/repo_lint_support/docs_and_workflows_policy/structural_ratchets/policy_ownership/repository_domains.rs deleted file mode 100644 index cea75f56..00000000 --- a/xtask/tests/repo_lint_support/docs_and_workflows_policy/structural_ratchets/policy_ownership/repository_domains.rs +++ /dev/null @@ -1,157 +0,0 @@ -// SPDX-License-Identifier: MIT OR Apache-2.0 - -use std::fs; - -use crate::repo_lint_support::repo_root; - -#[test] -fn repo_lint_checks_have_domain_specific_owners() { - let root = repo_root(); - let module_root = fs::read_to_string(root.join("xtask/tests/repo_lint_support/mod.rs")) - .expect("read repo-lint module root"); - assert!(!module_root.contains("mod source_policy;")); - assert!( - !root - .join("xtask/tests/repo_lint_support/source_policy.rs") - .exists(), - "repo-wide source assertions must live with their actual domains" - ); - let gpu_root = - fs::read_to_string(root.join("xtask/tests/repo_lint_support/gpu_adapter_policy.rs")) - .expect("read GPU adapter policy root"); - for module in [ - "mod accelerator_test_structure_policy;", - "mod adapter_boundary_policy;", - "mod cuda_product_source_policy;", - ] { - assert!(gpu_root.contains(module)); - } - assert!(!gpu_root.contains("mod accelerator_product_source_policy;")); - assert!(!root - .join( - "xtask/tests/repo_lint_support/gpu_adapter_policy/accelerator_product_source_policy.rs" - ) - .exists()); - - for (relative, symbols) in [ - ( - "xtask/tests/repo_lint_support/native_direct_plan_structure_policy.rs", - &["referenced_direct_plan_tests_keep_focused_owners"][..], - ), - ( - "xtask/tests/repo_lint_support/j2k_decode_structure_policy.rs", - &["owned_batch_tests_are_split_by_responsibility"][..], - ), - ( - "xtask/tests/repo_lint_support/metal_compute_structure_policy/compute_tests.rs", - &["referenced_plan_regression_has_focused_owner"][..], - ), - ( - "xtask/tests/repo_lint_support/gpu_adapter_policy/metal_batch_structure_policy.rs", - &["metal_multitile_tests_are_split_by_pixel_contract"][..], - ), - ( - "xtask/tests/repo_lint_support/gpu_adapter_policy/accelerator_test_structure_policy.rs", - &["accelerator_boundary_tests_keep_focused_owners"][..], - ), - ( - "xtask/tests/repo_lint_support/gpu_adapter_policy/adapter_boundary_policy.rs", - &[ - "adapter_crates_do_not_import_codec_private_modules", - "cuda_adapter_crates_keep_public_libs_as_module_shells", - ][..], - ), - ( - "xtask/tests/repo_lint_support/gpu_adapter_policy/cuda_product_source_policy.rs", - &[ - "production_j2k_cuda_code_does_not_reference_nvjpeg", - "cuda_runtime_rejects_product_cuda_c_and_checked_in_ptx", - "cuda_runtime_dispatch_does_not_read_deprecated_oxide_route_selectors", - ][..], - ), - ( - "xtask/tests/repo_lint_support/gpu_adapter_policy/cuda_profile_policy.rs", - &["cuda_trace_export_is_non_clobbering_and_documented"][..], - ), - ( - "xtask/tests/repo_lint_support/docs_and_workflows_policy/adoption_benchmark_policy.rs", - &["reusable_benchmark_generators_live_in_test_support"][..], - ), - ( - "xtask/tests/repo_lint_support/docs_and_workflows_policy/workflow_coverage_policy.rs", - &["gpu_runtime_tests_do_not_silently_return_on_missing_hardware_gates"][..], - ), - ] { - let source = fs::read_to_string(root.join(relative)) - .unwrap_or_else(|error| panic!("read {relative}: {error}")); - for symbol in symbols { - assert!(source.contains(symbol), "{relative} must own {symbol}"); - } - } -} - -#[test] -fn metal_policy_checks_live_in_focused_children() { - let root = repo_root(); - let gpu_root_relative = "xtask/tests/repo_lint_support/gpu_adapter_policy.rs"; - let gpu_root = fs::read_to_string(root.join(gpu_root_relative)) - .unwrap_or_else(|error| panic!("read {gpu_root_relative}: {error}")); - let metal_batch_relative = - "xtask/tests/repo_lint_support/gpu_adapter_policy/metal_batch_structure_policy.rs"; - let metal_batch = fs::read_to_string(root.join(metal_batch_relative)) - .unwrap_or_else(|error| panic!("read {metal_batch_relative}: {error}")); - assert!(gpu_root.contains("mod metal_batch_structure_policy;")); - for symbol in [ - "fn metal_batch_heuristics_live_in_focused_module(", - "fn metal_batch_routes_share_session_aware_implementations(", - ] { - assert!(!gpu_root.contains(symbol)); - assert!(metal_batch.contains(symbol)); - } - assert!(gpu_root.lines().count() < 50); - assert!(metal_batch.lines().count() < 300); - - let compute_root_relative = "xtask/tests/repo_lint_support/metal_compute_structure_policy.rs"; - let compute_root = fs::read_to_string(root.join(compute_root_relative)) - .unwrap_or_else(|error| panic!("read {compute_root_relative}: {error}")); - let batch_shell_relative = - "xtask/tests/repo_lint_support/metal_compute_structure_policy/batch_execution.rs"; - let batch_shell = fs::read_to_string(root.join(batch_shell_relative)) - .unwrap_or_else(|error| panic!("read {batch_shell_relative}: {error}")); - assert!(compute_root.contains("mod batch_execution;")); - assert!(compute_root.lines().count() < 25); - assert!(batch_shell.lines().count() < 25); - - for (module, symbol, max_lines) in [ - ( - "ht_chunks", - "fn metal_ht_chunk_tests_are_split_by_planning_cache_and_status_behavior(", - 250, - ), - ( - "stacked_execution", - "fn metal_stacked_execution_is_split_by_codec_stage(", - 75, - ), - ( - "direct_destination", - "fn metal_direct_destination_is_split_by_submission_and_group_encoding(", - 75, - ), - ( - "classic_batch", - "fn metal_distinct_classic_batch_execution_is_split_from_cleanup_dispatch(", - 75, - ), - ] { - assert!(batch_shell.contains(&format!("mod {module};"))); - let relative = format!( - "xtask/tests/repo_lint_support/metal_compute_structure_policy/batch_execution/{module}.rs" - ); - let source = fs::read_to_string(root.join(&relative)) - .unwrap_or_else(|error| panic!("read {relative}: {error}")); - assert!(source.contains(symbol), "{relative} must own {symbol}"); - assert!(source.lines().count() < max_lines); - assert!(!source.lines().any(|line| line.trim() == "use super::*;")); - } -} diff --git a/xtask/tests/repo_lint_support/docs_and_workflows_policy/workflow_coverage_policy.rs b/xtask/tests/repo_lint_support/docs_and_workflows_policy/workflow_coverage_policy.rs index 57ba70f2..8b8c9fb2 100644 --- a/xtask/tests/repo_lint_support/docs_and_workflows_policy/workflow_coverage_policy.rs +++ b/xtask/tests/repo_lint_support/docs_and_workflows_policy/workflow_coverage_policy.rs @@ -123,7 +123,7 @@ fn benchmark_targets_are_not_test_targets() { } #[test] -fn xtask_exposes_nextest_machete_and_strict_clippy_gates() { +fn xtask_exposes_machete_and_strict_clippy_gates_without_retired_aliases() { let xtask = xtask_sources(repo_root()); let workflow = fs::read_to_string(repo_root().join(".github/workflows/full-validation.yml")) .expect("read full validation workflow"); @@ -135,19 +135,18 @@ fn xtask_exposes_nextest_machete_and_strict_clippy_gates() { .expect("xtask help section"); assert_pattern_checks(&[ - PatternCheck::new("xtask nextest/machete/strict clippy dispatch", &xtask).required(&[ - "\"nextest\" =>", - "\"machete\" =>", - "\"clippy-strict\" =>", - ]), - PatternCheck::new("xtask nextest/machete/strict clippy help", help_section).required(&[ - "nextest", - "machete", - "clippy-strict", - ]), - PatternCheck::new("xtask nextest/machete/strict clippy gates", &xtask).required(&[ - "\"nextest\"", - "\"run\"", + PatternCheck::new("xtask machete/strict clippy dispatch", &xtask) + .required(&["\"machete\" =>", "\"clippy-strict\" =>"]) + .forbidden(&[ + "\"nextest\" =>", + "\"typos\" =>", + "\"deny\" =>", + "\"downstream-smoke\" =>", + ]), + PatternCheck::new("xtask machete/strict clippy help", help_section) + .required(&["machete", "clippy-strict"]) + .forbidden(&["nextest", "typos", "downstream-smoke"]), + PatternCheck::new("xtask machete/strict clippy gates", &xtask).required(&[ "\"cargo-machete\"", "\"--with-metadata\"", "\"--no-deps\"", diff --git a/xtask/tests/repo_lint_support/encode_compare_structure_policy.rs b/xtask/tests/repo_lint_support/encode_compare_structure_policy.rs deleted file mode 100644 index 4c6f5143..00000000 --- a/xtask/tests/repo_lint_support/encode_compare_structure_policy.rs +++ /dev/null @@ -1,120 +0,0 @@ -// SPDX-License-Identifier: MIT OR Apache-2.0 - -//! Structural ownership and size ratchets for the encode-comparison harness. - -use std::fs; - -use super::{assert_pattern_checks, repo_root, PatternCheck}; - -fn read(relative_path: &str) -> String { - fs::read_to_string(repo_root().join(relative_path)) - .unwrap_or_else(|error| panic!("read {relative_path}: {error}")) -} - -fn assert_line_budget(relative_path: &str, source: &str, max_lines: usize) { - let line_count = source.lines().count(); - assert!( - line_count < max_lines, - "{relative_path} has {line_count} lines; expected fewer than {max_lines}" - ); -} - -#[test] -fn encode_compare_stays_split_by_responsibility() { - let shell = read("crates/j2k-compare/src/encode_compare.rs"); - let types = read("crates/j2k-compare/src/encode_compare/types.rs"); - let cli = read("crates/j2k-compare/src/encode_compare/cli.rs"); - let images = read("crates/j2k-compare/src/encode_compare/images.rs"); - let tools = read("crates/j2k-compare/src/encode_compare/tools.rs"); - let validation = read("crates/j2k-compare/src/encode_compare/validation.rs"); - let measurement = read("crates/j2k-compare/src/encode_compare/measurement.rs"); - let render = read("crates/j2k-compare/src/encode_compare/render.rs"); - - for (path, source, max_lines) in [ - ("encode_compare.rs", shell.as_str(), 350), - ("encode_compare/types.rs", types.as_str(), 200), - ("encode_compare/cli.rs", cli.as_str(), 200), - ("encode_compare/images.rs", images.as_str(), 525), - ("encode_compare/tools.rs", tools.as_str(), 400), - ("encode_compare/validation.rs", validation.as_str(), 325), - ("encode_compare/measurement.rs", measurement.as_str(), 325), - ("encode_compare/render.rs", render.as_str(), 725), - ] { - assert_line_budget(path, source, max_lines); - assert!( - !source.contains("use super::*"), - "crates/j2k-compare/src/{path} must keep explicit module imports" - ); - assert!( - !source.contains("include!("), - "crates/j2k-compare/src/{path} must remain a real Rust module" - ); - } - - assert_pattern_checks(&[ - PatternCheck::new("encode_compare coordinator", &shell) - .required(&[ - "mod types;", - "mod cli;", - "mod images;", - "mod tools;", - "mod validation;", - "mod measurement;", - "mod render;", - "fn run() -> Result<(), String>", - ]) - .forbidden(&[ - "struct ImageCase", - "fn batch_size_config_from_env(", - "fn load_external_image_cases(", - "fn all_encoder_tools(", - "fn validate_encoded_profile(", - "fn measure_case_rows(", - "fn emit_metadata(", - ]), - PatternCheck::new("encode_compare type ownership", &types).required(&[ - "pub(super) struct ImageCase", - "pub(super) enum EncoderKind", - "pub(super) struct Measurement", - "pub(super) struct MetadataInput", - ]), - PatternCheck::new("encode_compare CLI ownership", &cli).required(&[ - "pub(super) fn encode_one", - "pub(super) fn validate_tool_gates", - "pub(super) fn batch_size_config_from_env", - "pub(super) fn encode_work_dir", - ]), - PatternCheck::new("encode_compare image ownership", &images).required(&[ - "pub(super) fn load_external_image_cases", - "pub(super) fn encode_manifest_from_env", - "pub(super) fn collect_source_image_paths", - "pub(super) fn read_raster_image", - "pub(super) fn read_pnm", - ]), - PatternCheck::new("encode_compare tool ownership", &tools).required(&[ - "pub(super) fn all_encoder_tools", - "pub(super) fn run_encoder_once", - "pub(super) fn command_version_label", - "pub(super) fn selected_encoders_label", - ]), - PatternCheck::new("encode_compare validation ownership", &validation).required(&[ - "pub(super) fn validate_encoded_profile", - "pub(super) struct CodProfile", - "pub(super) fn cod_profile", - "pub(super) fn decode_encoded_output", - ]), - PatternCheck::new("encode_compare measurement ownership", &measurement).required(&[ - "pub(super) fn measure_case_rows", - "pub(super) fn measure_mixed_rows", - "pub(super) fn measure_case_encoder_once", - "pub(super) fn measurement", - ]), - PatternCheck::new("encode_compare render ownership", &render).required(&[ - "pub(super) fn emit_metadata", - "pub(super) fn publication_blockers", - "pub(super) fn measurement_row", - "pub(super) fn mixed_measurement_row", - "pub(super) fn unique_image_count", - ]), - ]); -} diff --git a/xtask/tests/repo_lint_support/fixture_compare_structure_policy.rs b/xtask/tests/repo_lint_support/fixture_compare_structure_policy.rs deleted file mode 100644 index 9a09b833..00000000 --- a/xtask/tests/repo_lint_support/fixture_compare_structure_policy.rs +++ /dev/null @@ -1,142 +0,0 @@ -// SPDX-License-Identifier: MIT OR Apache-2.0 - -//! Structural ownership and size ratchets for the fixture-comparison harness. - -use std::fs; - -use super::{assert_pattern_checks, repo_root, PatternCheck}; - -mod decode; -mod fixtures; -mod manifest; - -fn read(relative_path: &str) -> String { - fs::read_to_string(repo_root().join(relative_path)) - .unwrap_or_else(|error| panic!("read {relative_path}: {error}")) -} - -fn assert_line_budget(relative_path: &str, source: &str, max_lines: usize) { - let line_count = source.lines().count(); - assert!( - line_count < max_lines, - "{relative_path} has {line_count} lines; expected fewer than {max_lines}" - ); -} - -fn assert_module_boundary(relative_path: &str, source: &str, max_lines: usize) { - assert_line_budget(relative_path, source, max_lines); - assert!( - !source.contains("use super::*"), - "crates/j2k-compare/src/{relative_path} must keep explicit module imports" - ); - assert!( - !source.contains("include!("), - "crates/j2k-compare/src/{relative_path} must remain a real Rust module" - ); -} - -fn assert_comparator_contracts(comparators: &str, comparator_tests: &str) { - assert_pattern_checks(&[ - PatternCheck::new("fixture comparator cleanup ownership", comparators).required(&[ - "fn cleanup_cli_staging(", - "let input_cleanup = cleanup_cli_temp(", - "let output_cleanup = cleanup_cli_temp(", - "staged input cleanup failed:", - "staged output cleanup failed:", - ]), - PatternCheck::new("fixture comparator command regressions", comparator_tests).required(&[ - "fn staging_cleanup_reports_both_failures_after_attempting_both_paths()", - "fn comparator_commands_decode_and_report_process_errors()", - "fn comparator_cleanup_attempts_output_after_input_failure()", - "output cleanup must still run after input cleanup fails", - "thread_local!", - ]), - ]); -} - -#[test] -fn fixture_compare_stays_split_by_responsibility() { - let shell = read("crates/j2k-compare/src/fixture_compare.rs"); - let cli = read("crates/j2k-compare/src/fixture_compare/cli.rs"); - let metadata = read("crates/j2k-compare/src/fixture_compare/metadata.rs"); - let validation = read("crates/j2k-compare/src/fixture_compare/validation.rs"); - let measurement = read("crates/j2k-compare/src/fixture_compare/measurement.rs"); - let comparators = read("crates/j2k-compare/src/fixture_compare/comparators.rs"); - let comparator_tests = read("crates/j2k-compare/src/fixture_compare/comparators/tests.rs"); - let gates = read("crates/j2k-compare/src/fixture_compare/gates.rs"); - let rows = read("crates/j2k-compare/src/fixture_compare/rows.rs"); - let types = read("crates/j2k-compare/src/fixture_compare/types.rs"); - - for (path, source, max_lines) in [ - ("fixture_compare.rs", shell.as_str(), 250), - ("fixture_compare/cli.rs", cli.as_str(), 200), - ("fixture_compare/metadata.rs", metadata.as_str(), 500), - ("fixture_compare/validation.rs", validation.as_str(), 250), - ("fixture_compare/measurement.rs", measurement.as_str(), 250), - ("fixture_compare/comparators.rs", comparators.as_str(), 350), - ( - "fixture_compare/comparators/tests.rs", - comparator_tests.as_str(), - 350, - ), - ("fixture_compare/gates.rs", gates.as_str(), 400), - ("fixture_compare/rows.rs", rows.as_str(), 350), - ("fixture_compare/types.rs", types.as_str(), 400), - ] { - assert_module_boundary(path, source, max_lines); - } - - assert_pattern_checks(&[ - PatternCheck::new("fixture_compare coordinator", &shell) - .required(&[ - "mod cli;", - "mod fixtures;", - "mod metadata;", - "mod validation;", - "mod measurement;", - "mod decode;", - "fn run() -> Result<(), String>", - ]) - .forbidden(&[ - "struct FixtureCase", - "fn batch_size_config_from_env(", - "fn all_fixture_cases(", - "fn emit_metadata(", - "fn validate_cases(", - "fn measure_case_batch_rows(", - "fn decode_j2k_batch(", - ]), - PatternCheck::new("fixture_compare CLI ownership", &cli).required(&[ - "pub(super) fn benchmark_mode_from_env", - "pub(super) fn batch_size_config_from_env", - "pub(super) fn validate_comparator_gates", - "pub(super) fn filter_cases_for_mode", - ]), - PatternCheck::new("fixture_compare metadata ownership", &metadata).required(&[ - "pub(super) fn emit_metadata", - "pub(super) fn external_unique_input_count", - "pub(super) fn skipped_comparators_label", - "pub(super) fn resolved_workers_label", - ]), - PatternCheck::new("fixture_compare validation ownership", &validation).required(&[ - "pub(super) fn validate_cases", - "pub(super) fn validate_mixed_batches", - "pub(super) fn skip_reason", - "pub(super) fn mixed_skip_reason", - ]), - PatternCheck::new("fixture_compare measurement ownership", &measurement).required(&[ - "pub(super) fn measure_case_batch_rows", - "pub(super) fn measure_mixed_batch_rows", - ]), - PatternCheck::new("fixture_compare type ownership", &types).required(&[ - "pub(super) struct FixtureCase", - "pub(super) struct Measurement", - "pub(super) struct MixedFixtureBatch", - "pub(super) struct MetadataContext", - ]), - ]); - assert_comparator_contracts(&comparators, &comparator_tests); - decode::assert_contract(); - fixtures::assert_contract(); - manifest::assert_contract(); -} diff --git a/xtask/tests/repo_lint_support/fixture_compare_structure_policy/decode.rs b/xtask/tests/repo_lint_support/fixture_compare_structure_policy/decode.rs deleted file mode 100644 index 8f9564e6..00000000 --- a/xtask/tests/repo_lint_support/fixture_compare_structure_policy/decode.rs +++ /dev/null @@ -1,35 +0,0 @@ -// SPDX-License-Identifier: MIT OR Apache-2.0 - -use super::{ - assert_line_budget, assert_module_boundary, assert_pattern_checks, read, PatternCheck, -}; - -pub(super) fn assert_contract() { - let production = read("crates/j2k-compare/src/fixture_compare/decode.rs"); - let tests = read("crates/j2k-compare/src/fixture_compare/decode/tests.rs"); - let policy = read("xtask/tests/repo_lint_support/fixture_compare_structure_policy/decode.rs"); - - assert_module_boundary("fixture_compare/decode.rs", &production, 650); - assert_module_boundary("fixture_compare/decode/tests.rs", &tests, 425); - assert_line_budget( - "xtask/tests/repo_lint_support/fixture_compare_structure_policy/decode.rs", - &policy, - 75, - ); - assert_pattern_checks(&[ - PatternCheck::new("fixture_compare decode ownership", &production).required(&[ - "pub(super) fn decode_j2k_batch", - "pub(super) fn decode_external_once", - "pub(super) fn decode_method_label", - "pub(super) fn crop_interleaved", - "#[cfg(test)]\nmod tests;", - ]), - PatternCheck::new("fixture_compare decode regressions", &tests).required(&[ - "j2k_decode_failures_preserve_operation_and_batch_context", - "openjpeg_router_matches_j2k_for_gray_and_rgb_operations", - "external_batch_dispatch_preserves_order_for_homogeneous_and_mixed_inputs", - "portable_emulation_crops_full_scaled_openjpeg_output", - "external_router_rejects_decoded_length_mismatches", - ]), - ]); -} diff --git a/xtask/tests/repo_lint_support/fixture_compare_structure_policy/fixtures.rs b/xtask/tests/repo_lint_support/fixture_compare_structure_policy/fixtures.rs deleted file mode 100644 index 3e05537c..00000000 --- a/xtask/tests/repo_lint_support/fixture_compare_structure_policy/fixtures.rs +++ /dev/null @@ -1,28 +0,0 @@ -// SPDX-License-Identifier: MIT OR Apache-2.0 - -use super::{assert_module_boundary, assert_pattern_checks, read, PatternCheck}; - -pub(super) fn assert_contract() { - let production = read("crates/j2k-compare/src/fixture_compare/fixtures.rs"); - let tests = read("crates/j2k-compare/src/fixture_compare/fixtures/tests.rs"); - - assert_module_boundary("fixture_compare/fixtures.rs", &production, 525); - assert_module_boundary("fixture_compare/fixtures/tests.rs", &tests, 350); - assert_pattern_checks(&[ - PatternCheck::new("fixture_compare fixture ownership", &production).required(&[ - "pub(super) fn all_fixture_cases", - "pub(super) fn fixture_cases", - "pub(super) fn load_external_fixture_cases", - "pub(super) fn encode_lossless", - "#[cfg(test)]\nmod tests;", - ]), - PatternCheck::new("fixture_compare fixture regressions", &tests).required(&[ - "fn fixture_classification_covers_supported_extensions_containers_and_codecs()", - "fn case_materialization_validates_codestream_shape_and_roi()", - "fn external_loader_reports_directory_and_fixture_failures()", - "fn external_loader_sorts_cases_and_applies_region_scaled_corpus_policy()", - "fn mixed_external_batches_group_compatible_distinct_inputs_only()", - "fn generated_encoder_rejects_invalid_samples_and_unknown_codec()", - ]), - ]); -} diff --git a/xtask/tests/repo_lint_support/fixture_compare_structure_policy/manifest.rs b/xtask/tests/repo_lint_support/fixture_compare_structure_policy/manifest.rs deleted file mode 100644 index ac37c3e6..00000000 --- a/xtask/tests/repo_lint_support/fixture_compare_structure_policy/manifest.rs +++ /dev/null @@ -1,32 +0,0 @@ -// SPDX-License-Identifier: MIT OR Apache-2.0 - -use super::{ - assert_line_budget, assert_module_boundary, assert_pattern_checks, read, PatternCheck, -}; - -pub(super) fn assert_contract() { - let production = read("crates/j2k-compare/src/fixture_compare/manifest.rs"); - let tests = read("crates/j2k-compare/src/fixture_compare/manifest/tests.rs"); - let policy = read("xtask/tests/repo_lint_support/fixture_compare_structure_policy/manifest.rs"); - - assert_module_boundary("fixture_compare/manifest.rs", &production, 325); - assert_module_boundary("fixture_compare/manifest/tests.rs", &tests, 425); - assert_line_budget( - "xtask/tests/repo_lint_support/fixture_compare_structure_policy/manifest.rs", - &policy, - 75, - ); - assert_pattern_checks(&[ - PatternCheck::new("fixture_compare manifest ownership", &production).required(&[ - "fn fixture_manifest_from_path(", - "pub(super) fn fixture_manifest_from_env", - "pub(super) fn external_fixture_metadata", - "#[cfg(test)]\nmod tests;", - ]), - PatternCheck::new("fixture_compare manifest regressions", &tests).required(&[ - "manifest_parser_reports_read_header_and_row_structure_errors", - "manifest_parser_rejects_duplicate_paths_and_invalid_type_pins", - "external_metadata_rejects_hash_codec_and_container_mismatches", - ]), - ]); -} diff --git a/xtask/tests/repo_lint_support/gpu_adapter_policy/cuda_encode_test_structure_policy.rs b/xtask/tests/repo_lint_support/gpu_adapter_policy/cuda_encode_test_structure_policy.rs index e9511af5..642364e8 100644 --- a/xtask/tests/repo_lint_support/gpu_adapter_policy/cuda_encode_test_structure_policy.rs +++ b/xtask/tests/repo_lint_support/gpu_adapter_policy/cuda_encode_test_structure_policy.rs @@ -5,6 +5,16 @@ use std::fs; use crate::repo_lint_support::repo_root; const TEST_MODULES: &[(&str, usize, &[&str])] = &[ + ( + "facade.rs", + 125, + &[ + "cuda_encoder_auto_reports_unavailable_fallback_and_reuses_after_success", + "cuda_encoder_cpu_only_reports_no_fallback", + "cuda_encoder_reuses_after_strict_route_failure", + "cuda_lossless_encoder_is_send", + ], + ), ( "htj2k.rs", 300, @@ -105,6 +115,7 @@ fn cuda_encode_tests_use_focused_real_modules_with_stable_inventory() { assert!(!encode.contains("#[cfg(test)]\nmod tests {") && !encode.contains("#[test]")); assert!(shell.lines().count() < 125, "CUDA encode test shell regrew"); for module in [ + "mod facade;", "mod htj2k;", "mod packetization;", "mod resident;", @@ -148,7 +159,7 @@ fn cuda_encode_tests_use_focused_real_modules_with_stable_inventory() { test_count += expected_tests.len(); } assert_eq!( - test_count, 43, + test_count, 47, "CUDA encode test inventory must remain exact" ); } diff --git a/xtask/tests/repo_lint_support/j2k_batch_allocation_policy.rs b/xtask/tests/repo_lint_support/j2k_batch_allocation_policy.rs index 2761676a..ff8fd534 100644 --- a/xtask/tests/repo_lint_support/j2k_batch_allocation_policy.rs +++ b/xtask/tests/repo_lint_support/j2k_batch_allocation_policy.rs @@ -66,10 +66,10 @@ fn j2k_batch_uses_one_authoritative_aggregate_plan() { PatternCheck::new("J2K allocation-free warning owner", &decode) .required(&[ "const _: [(); 0] = [(); core::mem::size_of::()];", - "warnings.push(J2kDecodeWarning::LenientDecodeMode)", + "warnings.push(J2kDecodeWarning::LenientMetadataRecovery)", "decode_warning_owner_is_statically_allocation_free", ]) - .forbidden(&["Vec::from([J2kDecodeWarning::LenientDecodeMode])"]), + .forbidden(&["Vec::from([J2kDecodeWarning::LenientMetadataRecovery])"]), ]); } diff --git a/xtask/tests/repo_lint_support/j2k_container_allocation_policy.rs b/xtask/tests/repo_lint_support/j2k_container_allocation_policy.rs index 3cd33fe6..3d610a91 100644 --- a/xtask/tests/repo_lint_support/j2k_container_allocation_policy.rs +++ b/xtask/tests/repo_lint_support/j2k_container_allocation_policy.rs @@ -452,6 +452,8 @@ fn native_jp2_responsibilities_stay_split_and_focused() { let root = repo_root(); let facade = read_source_files(root, &["crates/j2k-native/src/jp2/mod.rs"]); let container = read_source_files(root, &["crates/j2k-native/src/jp2/container.rs"]); + let container_header = + read_source_files(root, &["crates/j2k-native/src/jp2/container/header.rs"]); let metadata = read_source_files(root, &["crates/j2k-native/src/jp2/metadata.rs"]); let image_header = read_source_files(root, &["crates/j2k-native/src/jp2/image_header.rs"]); let validation = read_source_files(root, &["crates/j2k-native/src/jp2/validation.rs"]); @@ -478,12 +480,25 @@ fn native_jp2_responsibilities_stay_split_and_focused() { "pub fn extract_jp2_codestream_payload(", "pub(crate) fn parse_with_retained_baseline(", "fn parse_jp2_container_with_strict(", - "pub(super) fn parse_jp2_header_box(", - "fn count_color_specification_boxes(", + "mod header;", + "parse_jp2_header_box_tracked(", + "fn read_recoverable_metadata_box", ]) .forbidden(&[ "pub struct Jp2FileMetadata", "fn parse_component_descriptor(", + "fn count_color_specification_boxes(", + "fn validate_component_precision_metadata(", + ]), + PatternCheck::new("native JP2 header traversal ownership", &container_header) + .required(&[ + "pub(in crate::jp2) fn parse_jp2_header_box(", + "pub(super) fn parse_jp2_header_box_tracked(", + "fn count_color_specification_boxes(", + "read_recoverable_metadata_box(", + ]) + .forbidden(&[ + "pub fn inspect_jp2_container(", "fn validate_component_precision_metadata(", ]), PatternCheck::new("native JP2 metadata ownership", &metadata) @@ -511,6 +526,10 @@ fn native_jp2_responsibilities_stay_split_and_focused() { ]), ]); + assert_jp2_module_line_limits(root); +} + +fn assert_jp2_module_line_limits(root: &std::path::Path) { for (relative, max_lines) in [ ("crates/j2k-native/src/jp2/mod.rs", 60), ("crates/j2k-native/src/jp2/allocation.rs", 200), @@ -519,6 +538,7 @@ fn native_jp2_responsibilities_stay_split_and_focused() { ("crates/j2k-native/src/jp2/cmap.rs", 90), ("crates/j2k-native/src/jp2/colr.rs", 175), ("crates/j2k-native/src/jp2/container.rs", 425), + ("crates/j2k-native/src/jp2/container/header.rs", 175), ("crates/j2k-native/src/jp2/icc.rs", 100), ("crates/j2k-native/src/jp2/image_header.rs", 80), ("crates/j2k-native/src/jp2/metadata.rs", 500), diff --git a/xtask/tests/repo_lint_support/j2k_decode_structure_policy.rs b/xtask/tests/repo_lint_support/j2k_decode_structure_policy.rs deleted file mode 100644 index 93060a38..00000000 --- a/xtask/tests/repo_lint_support/j2k_decode_structure_policy.rs +++ /dev/null @@ -1,271 +0,0 @@ -// SPDX-License-Identifier: MIT OR Apache-2.0 - -//! Responsibility and regression ratchets for facade decode orchestration. - -use std::fs; - -use super::{assert_pattern_checks, read_source_files, repo_root, rust_sources, PatternCheck}; - -fn read(relative: &str) -> String { - fs::read_to_string(repo_root().join(relative)) - .unwrap_or_else(|error| panic!("read {relative}: {error}")) -} - -#[test] -fn facade_decode_keeps_pixel_layout_conversion_in_explicit_children() { - let root = read("crates/j2k/src/decode.rs"); - let output = read_source_files( - repo_root(), - &[ - "crates/j2k/src/decode/output.rs", - "crates/j2k/src/decode/output/u8.rs", - "crates/j2k/src/decode/output/u16.rs", - ], - ); - - assert_pattern_checks(&[ - PatternCheck::new("facade decode orchestrator", &root) - .required(&[ - "mod component_handoff;", - "mod output;", - "decode_image_into_with_native_context", - "decode_image_region_into_with_native_context", - "decode_warnings_for_settings", - ]) - .forbidden(&[ - "fn write_u8_output(", - "fn write_u16_output(", - "fn write_components_u8_output(", - "fn convert_or_copy_u16(", - "include!(", - ]), - PatternCheck::new("facade decode output modules", &output) - .required(&[ - "mod u16;", - "mod u8;", - "pub(in crate::decode) fn write_u8_output(", - "pub(in crate::decode) fn write_u16_output(", - "direct_u8_decode_accepts_exact_rgb_and_gray_layouts", - "eight_bit_samples_widen_across_the_complete_u16_domain", - "synthesized_alpha_matches_native_sample_storage", - ]) - .forbidden(&["use super::*", "include!("]), - ]); -} - -#[test] -fn facade_decode_responsibility_modules_stay_focused() { - for (relative, max_lines) in [ - ("crates/j2k/src/decode.rs", 220), - ("crates/j2k/src/decode/output.rs", 25), - ("crates/j2k/src/decode/output/u8.rs", 375), - ("crates/j2k/src/decode/output/u16.rs", 260), - ] { - let lines = read(relative).lines().count(); - assert!( - lines < max_lines, - "{relative} must stay below its {max_lines}-line responsibility ratchet" - ); - } -} - -#[test] -fn owned_batch_tests_are_split_by_responsibility() { - let root = repo_root(); - let test_root = root.join("crates/j2k/tests/owned_batch"); - let shell = fs::read_to_string(root.join("crates/j2k/tests/owned_batch.rs")) - .expect("read owned-batch integration-test shell"); - - for (module, owned_symbol, max_lines) in [ - ("fixtures", "fn htj2k_gray8_fixture(", 260usize), - ("oracles", "fn native_request_oracle(", 140usize), - ( - "payload_plan", - "fn assert_prepared_ht_payload_ranges_reconstruct_owned_bytes(", - 180usize, - ), - ( - "native_types_and_requests", - "fn prepared_htj2k_gray_and_rgb_support_native_types_and_requests_exactly(", - 150usize, - ), - ] { - assert!(shell.contains(&format!("mod {module};"))); - let path = test_root.join(format!("{module}.rs")); - let source = fs::read_to_string(&path) - .unwrap_or_else(|error| panic!("read {}: {error}", path.display())); - assert!(source.contains(owned_symbol)); - assert!(source.lines().count() < max_lines); - } - assert!(shell.lines().count() < 40); - for forbidden in [ - "fn htj2k_gray8_fixture(", - "fn native_request_oracle(", - "fn assert_prepared_ht_payload_ranges_reconstruct_owned_bytes(", - ] { - assert!(!shell.contains(forbidden)); - } - let rgba = fs::read_to_string(test_root.join("rgba.rs")).expect("read owned-batch RGBA tests"); - assert!(rgba.lines().count() < 400); - assert!( - !rgba.contains("fn prepared_htj2k_gray_and_rgb_support_native_types_and_requests_exactly(") - ); - for path in rust_sources(&test_root) { - let source = fs::read_to_string(&path) - .unwrap_or_else(|error| panic!("read {}: {error}", path.display())); - assert!(!source.lines().any(|line| line.trim() == "use super::*;")); - } -} - -#[test] -fn owned_batch_ht_matrix_tests_have_three_explicit_owners() { - let root = repo_root(); - let test_root = root.join("crates/j2k/tests/owned_batch_fixtures/ht_matrix"); - let shell_relative = "crates/j2k/tests/owned_batch_fixtures/ht_matrix.rs"; - let shell = fs::read_to_string(root.join(shell_relative)) - .unwrap_or_else(|error| panic!("read {shell_relative}: {error}")); - - for (module, owned_symbol, max_lines) in [ - ( - "reversible_native", - "fn reversible_batch_matrix_preserves_native_gray_and_rgb_samples(", - 100usize, - ), - ( - "raw_jph_request_geometry", - "fn independent_odd_openhtj2k_fixture_supports_roi_and_reduction(", - 175usize, - ), - ( - "pass_bucket_parity", - "fn external_cleanup_magref_and_generated_sigprop_jobs_decode_in_batches(", - 300usize, - ), - ] { - assert!( - shell.contains(&format!( - "#[path = \"ht_matrix/{module}.rs\"]\nmod {module};" - )), - "{shell_relative} must declare the {module} responsibility owner" - ); - let relative = format!("crates/j2k/tests/owned_batch_fixtures/ht_matrix/{module}.rs"); - let source = fs::read_to_string(test_root.join(format!("{module}.rs"))) - .unwrap_or_else(|error| panic!("read {relative}: {error}")); - assert!( - source.contains(owned_symbol), - "{relative} must own {owned_symbol}" - ); - assert!( - source.lines().count() < max_lines, - "{relative} exceeded its {max_lines}-line responsibility ratchet" - ); - assert!( - !source.lines().any(|line| line.trim() == "use super::*;"), - "{relative} must keep explicit imports" - ); - } - - assert!( - shell.lines().count() < 15, - "{shell_relative} must remain a focused module shell" - ); - for forbidden in [ - "fn reversible_batch_matrix_preserves_native_gray_and_rgb_samples(", - "fn independent_openhtj2k_raw_and_derived_jph_outputs_are_exact_and_indexed(", - "fn independent_odd_openhtj2k_fixture_supports_roi_and_reduction(", - "fn external_cleanup_magref_and_generated_sigprop_jobs_decode_in_batches(", - ] { - assert!( - !shell.contains(forbidden), - "{shell_relative} must not retain {forbidden}" - ); - } -} - -#[test] -fn cpu_fast_route_preparation_lives_in_focused_children() { - let shell = read("crates/j2k/src/owned_batch/cpu_fast.rs"); - let ht = read("crates/j2k/src/owned_batch/cpu_fast/ht.rs"); - let classic = read("crates/j2k/src/owned_batch/cpu_fast/classic.rs"); - - assert_pattern_checks(&[ - PatternCheck::new("CPU fast workspace lifecycle", &shell) - .required(&[ - "mod classic;", - "mod ht;", - "mod plan;", - "fn prepare_storage(", - "fn assign_image_spans(", - "fn finish_group(", - "fn record_job_buckets(", - "fn clear_active_plan(", - ]) - .forbidden(&["fn prepare_htj2k(", "fn prepare_classic("]), - PatternCheck::new("CPU fast HT preparation", &ht) - .required(&["impl CpuGroupFastWorkspace", "fn prepare_htj2k("]) - .forbidden(&["use super::*;", "fn prepare_classic("]), - PatternCheck::new("CPU fast classic preparation", &classic) - .required(&["impl CpuGroupFastWorkspace", "fn prepare_classic("]) - .forbidden(&["use super::*;", "fn prepare_htj2k("]), - ]); - - for (relative, source, max_lines) in [ - ("crates/j2k/src/owned_batch/cpu_fast.rs", &shell, 350usize), - ("crates/j2k/src/owned_batch/cpu_fast/ht.rs", &ht, 180usize), - ( - "crates/j2k/src/owned_batch/cpu_fast/classic.rs", - &classic, - 180usize, - ), - ] { - assert!( - source.lines().count() < max_lines, - "{relative} must stay below its {max_lines}-line responsibility ratchet" - ); - } -} - -#[test] -fn cpu_staged_group_orchestrator_delegates_explicit_phases() { - let source = read("crates/j2k/src/owned_batch/cpu_staged_execute.rs"); - let (_, function_tail) = source - .split_once("pub(super) fn run_staged_typed_group") - .expect("find staged group orchestrator"); - let (orchestrator, _) = function_tail - .split_once("\nfn prepare_staged_window<") - .expect("isolate staged group orchestrator"); - - for (phase, owner) in [ - ("prepare_staged_window(", "fn prepare_staged_window JpegView<'a>", - "parse_header(input)?", - ]), - PatternCheck::new("decoder/view.rs parsed-view API", &view).required(&[ - "pub struct JpegView", - "impl<'a> JpegView<'a>", - "pub fn parse(", - "pub fn parse_with_options(", - "pub fn passthrough_candidate(", - "pub fn restart_index(", - ]), - PatternCheck::new("decoder.rs output-format module shell", &decoder).required(&[ - "mod output_format;", - "output_format_from_parts", - "checked_output_geometry", - ]), - ]); - let output_format_patterns = [ - "pub(super) fn output_format_profile_name", - "pub(super) fn downscale_profile_name", - "pub(super) fn jpeg_downscale", - "pub(super) fn output_format_from_parts", - "pub(super) fn scaled_dimensions", - "pub(super) fn scaled_rect_covering", - "pub(super) fn checked_output_geometry", - ]; - assert_pattern_checks(&[ - PatternCheck::new("decoder.rs output-format helper exclusion", &decoder) - .forbidden(&output_format_patterns), - PatternCheck::new("decoder/output_format.rs helpers", &output_format) - .required(&output_format_patterns), - PatternCheck::new("decoder.rs focused helper module wiring", &decoder).required(&[ - "mod extended12;", - "mod lossless_helpers;", - "mod lossless_region;", - "mod color_convert;", - "mod warning_ownership;", - "mod core_traits;", - "mod scratch;", - "mod sink_writer;", - "mod plan;", - "mod routing;", - "mod rows;", - "mod sequential;", - "mod tile;", - "mod lossless_render;", - ]), - PatternCheck::new("decoder.rs focused ownership exclusions", &decoder).forbidden(&[ - "fn build_prepared_plan(", - "pub fn decode_into(", - "pub fn decode_rows<", - "pub fn decode_tile_into(", - "fn decode_with_writer<", - "fn decode_lossless_gray8_into(", - ]), - PatternCheck::new("decoder/plan.rs plan ownership", &plan).required(&[ - "pub(super) fn build_prepared_plan(", - "pub(super) fn build_lossless_plan(", - "pub(super) fn build_progressive_plan(", - "fn build_decode_plan(", - ]), - PatternCheck::new("decoder routing output ownership", &routing_sources).required(&[ - "mod dispatch;", - "mod profile;", - "pub fn decode_into(", - "pub fn decode_request(", - "pub(super) fn decode_region_into_output_format_with_scratch(", - "dispatch_full_output(", - "dispatch_region_output(", - ]), - PatternCheck::new("decoder/rows.rs row ownership", &rows).required(&[ - "pub fn decode_rows<", - "pub fn decode_component_rows_with_scratch<", - "pub fn decode_region_component_rows_with_scratch<", - ]), - PatternCheck::new("decoder/tile.rs tile facade ownership", &tile).required(&[ - "pub fn decode_tile_into(", - "pub fn decode_prepared_jpeg_tiles_rgb8(", - "pub fn decode_tiles_region_scaled_into(", - ]), - PatternCheck::new("decoder/sequential.rs writer ownership", &sequential).required(&[ - "pub(super) fn decode_scratch_bytes(", - "pub(super) fn decode_with_writer<", - "pub(super) fn decode_rgb_with_writer<", - ]), - PatternCheck::new( - "decoder/lossless_render.rs render ownership", - &lossless_render, - ) - .required(&[ - "pub(super) fn decode_lossless_gray8_into(", - "pub(super) fn decode_lossless_color_sampled_into<", - "pub(super) fn decode_lossless_gray16_into(", - ]), - ]); - let extended12_patterns = [ - "pub(super) enum Extended12Output", - "pub(super) struct Extended12WriteRegion", - "pub(super) fn decode_extended12_color_planes", - "pub(super) fn render_progressive12_color_planes", - "trait UpsampleSample", - ]; - assert_pattern_checks(&[ - PatternCheck::new("decoder/extended12.rs helpers", &extended12) - .required(&extended12_patterns), - PatternCheck::new("decoder.rs extended12 helper exclusion", &decoder) - .forbidden(&extended12_patterns), - ]); - let extended12_renderer_patterns = [ - "fn decode_12bit_rgba16_region_scaled_into", - "fn decode_progressive12_gray16_region_scaled_into", - "fn decode_progressive12_rgb16_region_scaled_into", - "fn decode_progressive12_region_into", - "fn decode_progressive12_color444_region_into", - "fn decode_progressive12_color_subsampled_region_into", - "fn decode_progressive12_four_component_region_into", - "fn decode_extended12_gray16_into", - "fn decode_extended12_rgb16_region_scaled_into", - "fn decode_extended12_region_into", - "fn decode_extended12_color444_region_into", - "fn decode_extended12_four_component_subsampled_region_into", - ]; - assert_pattern_checks(&[ - PatternCheck::new("decoder/extended12.rs 12-bit renderers", &extended12) - .required(&extended12_renderer_patterns), - PatternCheck::new("decoder.rs 12-bit renderer exclusion", &decoder) - .forbidden(&extended12_renderer_patterns), - ]); - let lossless_patterns = [ - "pub(super) fn restart_index_for_stream", - "pub(super) fn consume_lossless_restart", - "pub(super) struct LosslessRestartTracker", - "pub(super) struct Extended12RestartTracker", - "pub(super) fn validate_lossless_color_plan", - "pub(super) fn decode_lossless_plane_sample", - "pub(super) fn decode_lossless_color_sample", - "pub(super) struct LosslessColorIntoSample", - "pub(super) struct LosslessColorRowSample", - "pub(super) fn decode_lossless_sampled_color_mcu

", - "pub(super) struct LosslessSampledColorPlanesMut", - "pub(super) struct LosslessSampledMcu", - "pub(super) fn write_lossless_color16_sampled_output", - ]; - assert_pattern_checks(&[ - PatternCheck::new("decoder/lossless_helpers.rs helpers", &lossless) - .required(&lossless_patterns), - PatternCheck::new("decoder.rs lossless helper exclusion", &decoder) - .forbidden(&lossless_patterns), - ]); - assert!( - !decoder.contains("Extended12RestartTracker::new(") - && extended12 - .contains("Extended12RestartTracker::new(self.plan.restart_interval, total_mcus)") - && extended12 - .contains("Extended12RestartTracker::new(plan.restart_interval, total_mcus)") - && !decoder.contains("consume_extended12_restart(") - && !extended12.contains("consume_extended12_restart("), - "extended-12 restart cadence must be centralized through Extended12RestartTracker" - ); - assert!( - lossless_render - .matches("validate_lossless_color_plan::

") - .count() - == 3, - "lossless render paths must share validation through decoder/lossless_helpers.rs" - ); - let color_convert_patterns = [ - "pub(super) fn convert_ycbcr8_to_rgb8_in_place", - "pub(super) fn copy_rgb16_scaled_rect", - ]; - assert_pattern_checks(&[ - PatternCheck::new("decoder/color_convert.rs helpers", &color_convert) - .required(&color_convert_patterns), - PatternCheck::new("decoder.rs color-convert helper exclusion", &decoder) - .forbidden(&color_convert_patterns), - ]); - support_contracts::assert_decoder_support_contracts( - &decoder, - &warning_ownership, - &scratch, - &core_traits, - &sink_writer, - &bench_support, - ); -} diff --git a/xtask/tests/repo_lint_support/jpeg_decoder_structure_policy/adapter_tests.rs b/xtask/tests/repo_lint_support/jpeg_decoder_structure_policy/adapter_tests.rs deleted file mode 100644 index 85f6f649..00000000 --- a/xtask/tests/repo_lint_support/jpeg_decoder_structure_policy/adapter_tests.rs +++ /dev/null @@ -1,259 +0,0 @@ -// SPDX-License-Identifier: MIT OR Apache-2.0 - -//! Structural ownership ratchets for JPEG adapter production and integration tests. - -use std::{fs, path::Path}; - -use super::super::{ - assert_file_pattern_checks, assert_pattern_checks, repo_root, FilePatternCheck, PatternCheck, -}; - -const JPEG_DEVICE_PLAN_TEST_LEAVES: &[(&str, usize)] = &[ - ("crates/j2k-jpeg/tests/device_plan/support.rs", 175), - ( - "crates/j2k-jpeg/tests/device_plan/capability_routing.rs", - 200, - ), - ("crates/j2k-jpeg/tests/device_plan/color_components.rs", 225), - ( - "crates/j2k-jpeg/tests/device_plan/device_plan_basics.rs", - 185, - ), - ("crates/j2k-jpeg/tests/device_plan/malformed_inputs.rs", 240), - ("crates/j2k-jpeg/tests/device_plan/progressive.rs", 345), - ( - "crates/j2k-jpeg/tests/device_plan/extended/color_and_sampling.rs", - 565, - ), - ( - "crates/j2k-jpeg/tests/device_plan/extended/four_component.rs", - 350, - ), - ("crates/j2k-jpeg/tests/device_plan/lossless/app14.rs", 190), - ( - "crates/j2k-jpeg/tests/device_plan/lossless/grayscale.rs", - 140, - ), - ( - "crates/j2k-jpeg/tests/device_plan/lossless/sampling.rs", - 375, - ), - ("crates/j2k-jpeg/tests/device_plan/lossless/ycbcr.rs", 190), -]; - -fn read_repo_file(root: &Path, relative_path: &str) -> String { - fs::read_to_string(root.join(relative_path)) - .unwrap_or_else(|error| panic!("read {relative_path}: {error}")) -} - -#[test] -fn jpeg_device_plan_integration_tests_use_focused_real_modules() { - let root = repo_root(); - let target = read_repo_file(root, "crates/j2k-jpeg/tests/device_plan.rs"); - let suite = read_repo_file(root, "crates/j2k-jpeg/tests/device_plan/mod.rs"); - let extended = read_repo_file(root, "crates/j2k-jpeg/tests/device_plan/extended.rs"); - let lossless = read_repo_file(root, "crates/j2k-jpeg/tests/device_plan/lossless.rs"); - - assert!( - target.lines().count() <= 8 - && target.contains("#[path = \"device_plan/mod.rs\"]") - && target.contains("mod suite;") - && target.matches("mod ").count() == 1, - "device_plan.rs must remain a minimal integration-target shell" - ); - for declaration in [ - "mod capability_routing;", - "mod color_components;", - "mod device_plan_basics;", - "mod extended;", - "mod lossless;", - "mod malformed_inputs;", - "mod progressive;", - "mod support;", - ] { - assert!( - suite.contains(declaration), - "device_plan suite lost required boundary {declaration}" - ); - } - assert!(suite.lines().count() <= 15 && suite.matches("mod ").count() == 8); - for declaration in ["mod color_and_sampling;", "mod four_component;"] { - assert!(extended.contains(declaration)); - } - assert!(extended.lines().count() <= 8 && extended.matches("mod ").count() == 2); - for declaration in [ - "mod app14;", - "mod grayscale;", - "mod sampling;", - "mod ycbcr;", - ] { - assert!(lossless.contains(declaration)); - } - assert!(lossless.lines().count() <= 10 && lossless.matches("mod ").count() == 4); - - let mut test_count = 0usize; - for (relative_path, max_lines) in JPEG_DEVICE_PLAN_TEST_LEAVES { - let source = read_repo_file(root, relative_path); - let line_count = source.lines().count(); - assert!( - line_count <= *max_lines, - "{relative_path} grew to {line_count} lines; split it before exceeding {max_lines}" - ); - assert!( - !source.contains("include!(") && !source.contains("#[path") && !source.contains("::*"), - "{relative_path} must remain a real module with explicit imports" - ); - test_count += source.matches("#[test]").count(); - } - assert_eq!( - test_count, 70, - "the device-plan split must preserve its behavior-test inventory" - ); -} - -#[test] -fn jpeg_fast_packet_owner_uses_focused_real_modules() { - let root = repo_root(); - let owner = fs::read_to_string(root.join("crates/j2k-jpeg/src/adapter/fast_packet.rs")) - .expect("read JPEG fast-packet owner"); - let module_limits = [ - ("allocation.rs", 130usize), - ("build.rs", 250), - ("build/gray.rs", 100), - ("build/materialization.rs", 40), - ("cache.rs", 90), - ("checkpoints.rs", 175), - ("entropy.rs", 250), - ("error.rs", 125), - ("family.rs", 75), - ("header.rs", 230), - ("types.rs", 330), - ("tests.rs", 30), - ]; - - assert!( - owner.lines().count() <= 30 && owner.matches("mod ").count() == 10, - "JPEG fast-packet root must remain a minimal module owner" - ); - for declaration in [ - "mod allocation;", - "mod build;", - "mod cache;", - "mod checkpoints;", - "mod entropy;", - "mod error;", - "mod family;", - "mod header;", - "mod types;", - "mod tests;", - ] { - assert!( - owner.contains(declaration), - "JPEG fast-packet owner lost required boundary {declaration}" - ); - } - assert_pattern_checks(&[PatternCheck::new("JPEG fast-packet owner", &owner) - .required(&[ - "pub use build::{", - "build_fast420_packet", - "build_fast422_packet", - "build_fast444_packet", - "build_gray_packet", - "pub use cache::{", - "pub use error::{FastPacketError, TableKind};", - "pub use family::{classify_color_fast_packet_family, JpegFastPacketFamily};", - "pub use types::{", - "JpegCanonicalHuffmanTable", - "JpegEntropyCheckpointV1", - "JpegFast420PacketV1", - "JpegFast422PacketV1", - "JpegFast444PacketV1", - "JpegGrayPacketV1", - "JpegHuffmanTable", - ]) - .forbidden(&[ - "pub struct JpegFast420PacketV1", - "fn build_color_fast_packet", - "fn packet_checkpoints_from_device", - "fn extract_entropy_segments", - ])]); - - for (filename, max_lines) in module_limits { - let relative = format!("crates/j2k-jpeg/src/adapter/fast_packet/{filename}"); - let source = fs::read_to_string(root.join(&relative)) - .unwrap_or_else(|error| panic!("read {relative}: {error}")); - let line_count = source.lines().count(); - assert!( - line_count <= max_lines, - "{relative} grew to {line_count} lines; split it before exceeding {max_lines}" - ); - assert!( - !source.contains("include!(") && !source.contains("#[path") && !source.contains("::*"), - "{relative} must remain a real module with explicit imports" - ); - } - - let build = fs::read_to_string(root.join("crates/j2k-jpeg/src/adapter/fast_packet/build.rs")) - .expect("read JPEG fast-packet builder"); - let header = fs::read_to_string(root.join("crates/j2k-jpeg/src/adapter/fast_packet/header.rs")) - .expect("read JPEG fast-packet header adapter"); - assert_eq!( - header.matches(".get(usize::from(slot))").count(), - 2, - "quant and defensive Huffman selectors must remain bounds checked" - ); - assert!(!build.contains("tables[slot as usize]") && !header.contains("tables[slot as usize]")); -} - -#[test] -fn jpeg_fast_packet_accessors_stay_out_of_public_api() { - let root = repo_root(); - assert_file_pattern_checks( - root, - &[ - FilePatternCheck::new("crates/j2k-jpeg/src/adapter/fast_packet/types.rs") - .named("shared JPEG fast-packet ABI types") - .required(&[ - "pub struct JpegFast420PacketV1", - "pub struct JpegFast422PacketV1", - "pub struct JpegFast444PacketV1", - ]) - .forbidden(&[ - "pub trait JpegColorFastPacket", - "impl_color_fast_packet_access", - ]), - FilePatternCheck::new("crates/j2k-jpeg/src/adapter/mod.rs") - .named("j2k-jpeg adapter facade") - .forbidden(&["JpegColorFastPacket"]), - FilePatternCheck::new("crates/j2k-jpeg-cuda/src/owned_decode/plan.rs") - .named("CUDA owned-decode plan") - .required(&[ - "macro_rules! fast_rgb8_packet_parts", - "fn build_cuda_rgb8_plan_data", - ]) - .forbidden(&[ - "JpegColorFastPacket", - "trait JpegColorFastPacket", - "macro_rules! impl_color_fast_packet_access", - "macro_rules! cuda_decode_plan", - ]), - FilePatternCheck::new("crates/j2k-jpeg-metal/src/compute/fast_packets/descriptors.rs") - .named("Metal fast packets") - .required(&[ - "trait FastSubsampledPacket", - "macro_rules! impl_fast_subsampled_packet_accessors", - ]) - .forbidden(&[ - "JpegColorFastPacket", - "trait JpegColorFastPacket", - "macro_rules! impl_color_fast_packet_access", - ]), - FilePatternCheck::stable_api_snapshots() - .named("stable API snapshot union") - .forbidden(&[ - "pub trait j2k_jpeg::adapter::JpegColorFastPacket", - "j2k_jpeg::adapter::JpegColorFastPacket::", - ]), - ], - ); -} diff --git a/xtask/tests/repo_lint_support/jpeg_decoder_structure_policy/owned_output.rs b/xtask/tests/repo_lint_support/jpeg_decoder_structure_policy/owned_output.rs deleted file mode 100644 index e8ea7889..00000000 --- a/xtask/tests/repo_lint_support/jpeg_decoder_structure_policy/owned_output.rs +++ /dev/null @@ -1,47 +0,0 @@ -// SPDX-License-Identifier: MIT OR Apache-2.0 - -use std::fs; - -use super::super::{assert_pattern_checks, repo_root, PatternCheck}; - -#[test] -fn jpeg_decoder_owned_outputs_use_decode_request() { - let root = repo_root(); - let decoder = fs::read_to_string(root.join("crates/j2k-jpeg/src/decoder.rs")) - .expect("read j2k-jpeg decoder"); - let routing = fs::read_to_string(root.join("crates/j2k-jpeg/src/decoder/routing.rs")) - .expect("read j2k-jpeg decoder routing"); - let owned_output = - fs::read_to_string(root.join("crates/j2k-jpeg/src/decoder/routing/owned_output.rs")) - .expect("read j2k-jpeg owned-output routing"); - let decoder_api = format!("{decoder}\n{routing}\n{owned_output}"); - let lib = - fs::read_to_string(root.join("crates/j2k-jpeg/src/lib.rs")).expect("read j2k-jpeg lib"); - - assert_pattern_checks(&[ - PatternCheck::new("j2k-jpeg owned-output request API", &decoder_api).required(&[ - "pub struct DecodeRequest", - "pub const fn full(fmt: PixelFormat) -> Self", - "pub const fn scaled(fmt: PixelFormat, scale: Downscale) -> Self", - "pub const fn region(fmt: PixelFormat, region: Rect) -> Self", - "pub const fn region_scaled(fmt: PixelFormat, region: Rect, scale: Downscale) -> Self", - "pub fn decode_request(", - "pub fn decode_request_with_external_live(", - "pub fn decode_request_with_scratch_and_external_live(", - ]), - PatternCheck::new("j2k-jpeg owned-output wrapper removal", &decoder_api).forbidden(&[ - "pub fn decode(&self, fmt: PixelFormat)", - "pub fn decode_scaled(", - "pub fn decode_with_scratch(", - "pub fn decode_scaled_with_scratch(", - "pub fn decode_region(", - "pub fn decode_region_scaled(", - "pub fn decode_region_with_scratch(", - "pub fn decode_region_scaled_with_scratch(", - ]), - PatternCheck::new("j2k-jpeg DecodeRequest re-export", &lib).required(&[ - "DecodeOutcome, DecodeRequest", - "DecodedTile, Decoder, JpegView", - ]), - ]); -} diff --git a/xtask/tests/repo_lint_support/jpeg_decoder_structure_policy/profile_contract.rs b/xtask/tests/repo_lint_support/jpeg_decoder_structure_policy/profile_contract.rs deleted file mode 100644 index b4afb071..00000000 --- a/xtask/tests/repo_lint_support/jpeg_decoder_structure_policy/profile_contract.rs +++ /dev/null @@ -1,65 +0,0 @@ -// SPDX-License-Identifier: MIT OR Apache-2.0 - -use std::{fs, path::Path}; - -use super::{assert_pattern_checks, PatternCheck}; - -pub(super) fn assert_policy(root: &Path) { - let production = read(root, "crates/j2k-jpeg/src/decoder/routing/profile.rs"); - let support = read( - root, - "crates/j2k-jpeg/src/decoder/routing/profile/test_support.rs", - ); - let tests = read(root, "crates/j2k-jpeg/src/decoder/routing/profile/tests.rs"); - let policy = read( - root, - "xtask/tests/repo_lint_support/jpeg_decoder_structure_policy/profile_contract.rs", - ); - - for (label, source, max_lines) in [ - ("JPEG routing profile source", &production, 140), - ("JPEG routing profile test support", &support, 125), - ("JPEG routing profile tests", &tests, 300), - ("JPEG routing profile policy", &policy, 100), - ] { - assert!( - source.lines().count() < max_lines, - "{label} must stay below its focused {max_lines}-line ratchet" - ); - } - for (label, source) in [ - ("JPEG routing profile source", &production), - ("JPEG routing profile test support", &support), - ("JPEG routing profile tests", &tests), - ] { - assert!( - !source.contains("use super::*"), - "{label} must use explicit imports" - ); - } - - assert_pattern_checks(&[ - PatternCheck::new("JPEG routing profile ownership", &production).required(&[ - "struct DecodeProfileRecord", - "fn emit_decode_profile_fields(", - "emit_jpeg_profile_fields(operation, op, path, build)", - "#[cfg(test)]\nmod test_support;", - "#[cfg(test)]\nmod tests;", - ]), - PatternCheck::new("JPEG routing profile capture contract", &support).required(&[ - "thread_local!", - "PhantomData>", - "impl Drop for TestProfileSinkGuard", - ]), - PatternCheck::new("JPEG routing profile regressions", &tests).required(&[ - "emit_dispatches_full_profile_with_exact_bounded_fields", - "emit_dispatches_region_and_scaled_region_profiles_with_exact_fields", - "profile_capture_is_thread_local_nested_and_restored", - ]), - ]); -} - -fn read(root: &Path, relative: &str) -> String { - fs::read_to_string(root.join(relative)) - .unwrap_or_else(|error| panic!("read {relative}: {error}")) -} diff --git a/xtask/tests/repo_lint_support/jpeg_decoder_structure_policy/support_contracts.rs b/xtask/tests/repo_lint_support/jpeg_decoder_structure_policy/support_contracts.rs deleted file mode 100644 index 026eabd6..00000000 --- a/xtask/tests/repo_lint_support/jpeg_decoder_structure_policy/support_contracts.rs +++ /dev/null @@ -1,77 +0,0 @@ -// SPDX-License-Identifier: MIT OR Apache-2.0 - -use super::{assert_pattern_checks, PatternCheck}; - -pub(super) fn assert_decoder_support_contracts( - decoder: &str, - warning_ownership: &str, - scratch: &str, - core_traits: &str, - sink_writer: &str, - bench_support: &str, -) { - assert_pattern_checks(&[ - PatternCheck::new("decoder/warning_ownership.rs helpers", warning_ownership).required(&[ - "pub(super) fn merged_warnings", - "pub(super) fn try_clone_warnings", - ]), - PatternCheck::new("decoder.rs warning-owner wiring", decoder) - .required(&["use self::warning_ownership::{merged_warnings, try_clone_warnings};"]) - .forbidden(&[ - "pub(super) fn merged_warnings", - "pub(super) fn try_clone_warnings", - ]), - ]); - let scratch_patterns = [ - "pub(super) fn compute_decode_scratch_bytes", - "pub(super) fn compute_lossless_scratch_bytes", - "pub(super) fn compute_extended12_planes_scratch_bytes", - "pub(super) fn checked_scratch_len", - "pub(super) fn checked_usize_product", - ]; - assert_pattern_checks(&[ - PatternCheck::new("decoder/scratch.rs helpers", scratch).required(&scratch_patterns), - PatternCheck::new("decoder.rs scratch helper exclusion", decoder) - .forbidden(&scratch_patterns), - ]); - let core_trait_patterns = [ - "impl ImageCodec for Decoder<'_>", - "impl TileBatchDecode for JpegCodec", - "pub(super) struct CroppedWriter", - "impl OutputWriter for &mut W", - ]; - assert_pattern_checks(&[ - PatternCheck::new("decoder/core_traits.rs trait adapters", core_traits) - .required(&core_trait_patterns) - .forbidden(&["ComponentWriterAdapter"]), - PatternCheck::new("decoder.rs core trait adapter exclusion", decoder) - .forbidden(&core_trait_patterns), - PatternCheck::new("decoder.rs component writer adapter exclusion", decoder) - .forbidden(&["ComponentWriterAdapter"]), - PatternCheck::new("decoder.rs sink writer re-export", decoder) - .required(&["pub(crate) use self::sink_writer::SinkWriter;"]), - ]); - let sink_writer_patterns = [ - "pub(crate) struct SinkWriter", - "pub(crate) fn into_rows", - "impl InterleavedRgbWriter for SinkWriter<'_, S>", - "impl OutputWriter for SinkWriter<'_, S>", - ]; - assert_pattern_checks(&[ - PatternCheck::new("decoder/sink_writer.rs helpers", sink_writer) - .required(&sink_writer_patterns), - PatternCheck::new("decoder.rs sink writer helper exclusion", decoder) - .forbidden(&sink_writer_patterns), - PatternCheck::new("bench profile shared sink writer reuse", bench_support) - .required(&[ - "struct BlackBoxRowSink", - "impl RowSink for BlackBoxRowSink", - "SinkWriter::new(&mut sink, rows, dec.backend)", - ]) - .forbidden(&[ - "struct BenchProfileSinkWriter", - "impl InterleavedRgbWriter for BenchProfileSinkWriter", - "impl OutputWriter for BenchProfileSinkWriter", - ]), - ]); -} diff --git a/xtask/tests/repo_lint_support/jpeg_encoder_structure_policy.rs b/xtask/tests/repo_lint_support/jpeg_encoder_structure_policy.rs deleted file mode 100644 index b33e39d8..00000000 --- a/xtask/tests/repo_lint_support/jpeg_encoder_structure_policy.rs +++ /dev/null @@ -1,219 +0,0 @@ -// SPDX-License-Identifier: MIT OR Apache-2.0 - -//! Cohesive ownership ratchets for the baseline JPEG encoder. - -use std::fs; - -use super::{assert_contains_all, assert_not_contains_all, repo_root}; - -fn read(relative: &str) -> String { - fs::read_to_string(repo_root().join(relative)) - .unwrap_or_else(|error| panic!("read {relative}: {error}")) -} - -#[test] -fn jpeg_encoder_modules_keep_cohesive_ownership() { - let facade = read("crates/j2k-jpeg/src/encoder.rs"); - assert!( - facade.lines().count() <= 260, - "encoder.rs must remain a thin public-contract and orchestration facade" - ); - assert_contains_all( - "JPEG encoder facade", - &facade, - &[ - "mod api;", - "mod planning;", - "mod profile;", - "mod sample_planes;", - "mod transform;", - "pub use self::api::{", - "fn encode_jpeg_baseline_cpu(", - ], - ); - assert_not_contains_all( - "JPEG encoder facade", - &facade, - &[ - "pub enum JpegEncodeError", - "fn try_vec_with_live_budget", - "fn rgb_to_ycbcr", - "fn fdct_quantize", - "struct BitWriter", - "fn emit_cpu_encode_profile", - "mod tests {", - ], - ); - - assert_module_sizes(); - assert_module_owners(); - assert_shared_entropy_dependency_direction(); - assert_shared_allocation_ownership(); -} - -fn assert_module_sizes() { - for (module, ceiling) in [ - ("api.rs", 190usize), - ("planning.rs", 180), - ("profile.rs", 100), - ("sample_planes.rs", 220), - ("transform.rs", 125), - ("tests.rs", 260), - ] { - let source = read(&format!("crates/j2k-jpeg/src/encoder/{module}")); - assert!( - source.lines().count() <= ceiling, - "encoder/{module} has {} lines; cohesive ownership ceiling is {ceiling}", - source.lines().count() - ); - } -} - -fn assert_shared_entropy_dependency_direction() { - let api = read("crates/j2k-jpeg/src/encoder/api.rs"); - let facade = read("crates/j2k-jpeg/src/encoder.rs"); - let entropy = read("crates/j2k-jpeg/src/baseline_entropy.rs"); - let encode_contract = read("crates/j2k-jpeg/src/baseline_encode_contract.rs"); - let dct_contract = read("crates/j2k-jpeg/src/dct_contract.rs"); - let transcode = read("crates/j2k-jpeg/src/transcode.rs"); - let coefficients = read("crates/j2k-jpeg/src/transcode/validation/coefficients.rs"); - - assert_contains_all( - "shared baseline entropy owner", - &entropy, - &["struct BitWriter", "fn encode_block", "fn magnitude"], - ); - assert_contains_all( - "independent JPEG encode error owner", - &encode_contract, - &[ - "pub enum JpegBackend", - "pub enum JpegSubsampling", - "pub enum JpegEncodeError", - "crate::dct_contract::JpegDctImageError", - ], - ); - assert_contains_all( - "independent DCT contract owner", - &dct_contract, - &["pub enum JpegDctCodingMode", "pub enum JpegDctImageError"], - ); - assert_contains_all( - "transcode shared entropy imports", - &transcode, - &["crate::baseline_entropy::{encode_block, BitWriter}"], - ); - assert_contains_all( - "DCT validation shared entropy imports", - &coefficients, - &["crate::baseline_entropy::magnitude"], - ); - for (label, source) in [ - ("JPEG encoder API", api.as_str()), - ("JPEG encoder facade", facade.as_str()), - ("shared baseline entropy", entropy.as_str()), - ("JPEG encode contract", encode_contract.as_str()), - ("DCT contract", dct_contract.as_str()), - ] { - assert_not_contains_all( - label, - source, - &["crate::transcode", "super::bitstream", "mod bitstream;"], - ); - } -} - -fn assert_shared_allocation_ownership() { - let allocation = read("crates/j2k-jpeg/src/allocation.rs"); - let planning = read("crates/j2k-jpeg/src/encoder/planning.rs"); - let sample_planes = read("crates/j2k-jpeg/src/encoder/sample_planes.rs"); - - assert_contains_all( - "shared JPEG allocation owner", - &allocation, - &[ - "enum AllocationBudgetError", - "fn try_new_vec_with_live_budget", - ], - ); - assert_contains_all( - "JPEG encode allocation mapping boundary", - &planning, - &["AllocationBudgetError", "fn map_allocation_budget_error"], - ); - assert_contains_all( - "JPEG sample-plane shared allocation use", - &sample_planes, - &["try_new_vec_with_live_budget"], - ); - assert_not_contains_all( - "JPEG encoder planning", - &planning, - &["fn try_vec_with_live_budget", "try_reserve_exact"], - ); -} - -fn assert_module_owners() { - let api = read("crates/j2k-jpeg/src/encoder/api.rs"); - let contract = read("crates/j2k-jpeg/src/baseline_encode_contract.rs"); - assert_contains_all( - "JPEG encoder public contracts", - &contract, - &[ - "pub enum JpegBackend", - "pub enum JpegSubsampling", - "pub struct JpegEncodeOptions", - "pub enum JpegSamples<'a>", - "pub struct EncodedJpeg", - "pub enum JpegEncodeError", - ], - ); - assert_contains_all( - "JPEG encoder API re-export", - &api, - &["pub use crate::baseline_encode_contract::{"], - ); - - let planning = read("crates/j2k-jpeg/src/encoder/planning.rs"); - assert_contains_all( - "JPEG encoder capacity planning", - &planning, - &[ - "struct CpuEncodeCapacityPlan", - "checked_cpu_encode_capacity_plan", - "checked_sample_byte_len", - "component_plane_capacity_bytes", - ], - ); - - let sample_planes = read("crates/j2k-jpeg/src/encoder/sample_planes.rs"); - assert_contains_all( - "JPEG encoder sample planes", - &sample_planes, - &[ - "validate_sample_layout", - "component_planes", - "rgb_to_ycbcr", - "sample_block", - ], - ); - - let bitstream = read("crates/j2k-jpeg/src/baseline_entropy.rs"); - assert_contains_all( - "JPEG encoder entropy bitstream", - &bitstream, - &[ - "struct BitWriter", - "encode_block", - "write_huffman_symbol", - "magnitude", - ], - ); - - let transform = read("crates/j2k-jpeg/src/encoder/transform.rs"); - assert_contains_all( - "JPEG encoder transform", - &transform, - &["fdct_quantize", "cosine_table"], - ); -} diff --git a/xtask/tests/repo_lint_support/metal_compute_structure_policy.rs b/xtask/tests/repo_lint_support/metal_compute_structure_policy.rs index 683b88c2..aa7748f1 100644 --- a/xtask/tests/repo_lint_support/metal_compute_structure_policy.rs +++ b/xtask/tests/repo_lint_support/metal_compute_structure_policy.rs @@ -1,9 +1,3 @@ // SPDX-License-Identifier: MIT OR Apache-2.0 -mod batch_execution; -mod color_store; -mod compute_tests; -mod direct_plan_ratchets; -mod direct_plan_types; -mod runtime_registry; mod status_contracts; diff --git a/xtask/tests/repo_lint_support/metal_compute_structure_policy/batch_execution.rs b/xtask/tests/repo_lint_support/metal_compute_structure_policy/batch_execution.rs deleted file mode 100644 index 8817d402..00000000 --- a/xtask/tests/repo_lint_support/metal_compute_structure_policy/batch_execution.rs +++ /dev/null @@ -1,6 +0,0 @@ -// SPDX-License-Identifier: MIT OR Apache-2.0 - -mod classic_batch; -mod direct_destination; -mod ht_chunks; -mod stacked_execution; diff --git a/xtask/tests/repo_lint_support/metal_compute_structure_policy/batch_execution/classic_batch.rs b/xtask/tests/repo_lint_support/metal_compute_structure_policy/batch_execution/classic_batch.rs deleted file mode 100644 index 983d6edd..00000000 --- a/xtask/tests/repo_lint_support/metal_compute_structure_policy/batch_execution/classic_batch.rs +++ /dev/null @@ -1,50 +0,0 @@ -// SPDX-License-Identifier: MIT OR Apache-2.0 - -use std::fs; - -use crate::repo_lint_support::{assert_pattern_checks, repo_root, PatternCheck}; - -#[test] -fn metal_distinct_classic_batch_execution_is_split_from_cleanup_dispatch() { - let root = repo_root().join("crates/j2k-metal/src/compute/decode_dispatch"); - let cleanup = fs::read_to_string(root.join("classic_cleanup.rs")) - .expect("read Metal classic cleanup module"); - let distinct = fs::read_to_string(root.join("classic_cleanup/distinct_batch.rs")) - .expect("read Metal distinct classic batch module"); - - assert_pattern_checks(&[ - PatternCheck::new("Metal classic cleanup dispatch ownership", &cleanup) - .required(&["mod distinct_batch;"]) - .forbidden(&["fn append_distinct_classic_batch("]), - PatternCheck::new("Metal distinct classic batch ownership", &distinct).required(&[ - "fn append_distinct_classic_batch(", - "fn encode_distinct_classic_batches_to_buffer_in_encoder", - "dispatch_zero_u32_buffer_in_encoder", - ]), - ]); -} - -#[test] -fn repeated_classic_subband_variants_share_one_execution_lifecycle() { - let source = fs::read_to_string( - repo_root().join("crates/j2k-metal/src/compute/decode_dispatch/classic_subband.rs"), - ) - .expect("read Metal classic sub-band execution module"); - - assert_pattern_checks(&[PatternCheck::new( - "Metal classic sub-band shared execution ownership", - &source, - ) - .required(&[ - "struct ClassicBatchView", - "fn encode_repeated_classic_batch_to_buffer_in_command_buffer(", - ]) - .forbidden(&["clippy::too_many_lines"])]); - assert_eq!( - source - .matches("encode_repeated_classic_batch_to_buffer_in_command_buffer(") - .count(), - 3, - "the repeated execution owner must have one definition and two typed entry-point calls" - ); -} diff --git a/xtask/tests/repo_lint_support/metal_compute_structure_policy/batch_execution/direct_destination.rs b/xtask/tests/repo_lint_support/metal_compute_structure_policy/batch_execution/direct_destination.rs deleted file mode 100644 index db161995..00000000 --- a/xtask/tests/repo_lint_support/metal_compute_structure_policy/batch_execution/direct_destination.rs +++ /dev/null @@ -1,53 +0,0 @@ -// SPDX-License-Identifier: MIT OR Apache-2.0 - -use std::fs; - -use crate::repo_lint_support::{assert_pattern_checks, repo_root, PatternCheck}; - -#[test] -fn metal_direct_destination_is_split_by_submission_and_group_encoding() { - let root = repo_root().join("crates/j2k-metal/src/compute/direct_grayscale_execute"); - let facade = fs::read_to_string(root.join("destination.rs")) - .expect("read direct grayscale destination facade"); - let submission = fs::read_to_string(root.join("destination/submission.rs")) - .expect("read direct destination submission lifecycle module"); - let group_encode = fs::read_to_string(root.join("destination/group_encode.rs")) - .expect("read direct destination group encoding module"); - - assert_pattern_checks(&[ - PatternCheck::new("direct grayscale destination facade", &facade) - .required(&[ - "mod group_encode;", - "mod submission;", - "submit_prepared_direct_grayscale_plan_batch_into_group", - ]) - .forbidden(&[ - "struct SubmittedDirectDestination", - "struct GrayscaleGroupEncoder", - "fn commit_direct_destination(", - "fn encode_stacked_grayscale_destination(", - ]), - PatternCheck::new("direct destination submission lifecycle", &submission).required(&[ - "enum DirectDestinationConsumerOrdering", - "struct SubmittedDirectDestination", - "fn commit_direct_destination(", - "impl Drop for SubmittedDirectDestination", - ]), - PatternCheck::new("direct grayscale group encoding", &group_encode).required(&[ - "struct GrayscaleGroupEncoder", - "fn encode_stacked(", - "fn encode_individually(", - "fn encode_stacked_grayscale_destination(", - ]), - ]); - for (relative, source, max_lines) in [ - ("destination.rs", &facade, 150), - ("destination/submission.rs", &submission, 325), - ("destination/group_encode.rs", &group_encode, 275), - ] { - assert!( - source.lines().count() < max_lines, - "{relative} must stay below its focused-responsibility line-count ratchet" - ); - } -} diff --git a/xtask/tests/repo_lint_support/metal_compute_structure_policy/batch_execution/ht_chunks.rs b/xtask/tests/repo_lint_support/metal_compute_structure_policy/batch_execution/ht_chunks.rs deleted file mode 100644 index a934ea9e..00000000 --- a/xtask/tests/repo_lint_support/metal_compute_structure_policy/batch_execution/ht_chunks.rs +++ /dev/null @@ -1,221 +0,0 @@ -// SPDX-License-Identifier: MIT OR Apache-2.0 - -use std::fs; - -use crate::repo_lint_support::{assert_pattern_checks, repo_root, PatternCheck}; - -#[test] -#[expect( - clippy::too_many_lines, - reason = "one table-driven policy verifies the planning, cache, and status test ownership split" -)] -fn metal_ht_chunk_tests_are_split_by_planning_cache_and_status_behavior() { - let root = repo_root(); - let ht_chunks_root = root.join("crates/j2k-metal/src/compute/decode_dispatch/ht_chunks"); - let tests_root = root.join("crates/j2k-metal/src/compute/decode_dispatch/ht_chunks/tests"); - let ht_chunks_shell = - fs::read_to_string(root.join("crates/j2k-metal/src/compute/decode_dispatch/ht_chunks.rs")) - .expect("read Metal HT chunk module shell"); - let shell = fs::read_to_string( - root.join("crates/j2k-metal/src/compute/decode_dispatch/ht_chunks/tests.rs"), - ) - .expect("read Metal HT chunk test shell"); - - assert_pattern_checks(&[PatternCheck::new("Metal HT chunk test shell", &shell) - .required(&["mod cache;", "mod parity;", "mod planning;", "mod status;"]) - .forbidden(&[ - "fn tiny_caps_split_jobs_in_pass_order_and_keep_source_mapping(", - "fn second_prepared_ht_submission_reuses_immutable_gpu_arenas(", - "fn reused_ht_status_storage_is_overwritten_by_every_dispatched_job(", - ])]); - assert!( - shell.lines().count() < 25, - "HT chunk test shell must only own behavior-module wiring" - ); - - for (name, required, max_lines) in [ - ( - "planning.rs", - "tiny_caps_split_jobs_in_pass_order_and_keep_source_mapping", - 225, - ), - ( - "parity.rs", - "forced_multi_chunk_metal_output_matches_cpu_coefficients", - 150, - ), - ( - "cache.rs", - "second_prepared_ht_submission_reuses_immutable_gpu_arenas", - 150, - ), - ( - "status.rs", - "reused_ht_status_storage_is_overwritten_by_every_dispatched_job", - 425, - ), - ] { - let path = tests_root.join(name); - let source = fs::read_to_string(&path) - .unwrap_or_else(|error| panic!("read {}: {error}", path.display())); - assert!( - source.contains(required), - "{} must own {required}", - path.display() - ); - assert!( - source.lines().count() < max_lines, - "{} must stay below its focused-test line-count ratchet", - path.display() - ); - } - - let planning = fs::read_to_string(tests_root.join("planning.rs")) - .expect("read Metal HT chunk planning test owner"); - let cache = fs::read_to_string(tests_root.join("cache.rs")) - .expect("read Metal HT chunk cache test owner"); - assert_pattern_checks(&[ - PatternCheck::new("Metal HT chunk module shell", &ht_chunks_shell) - .forbidden(&["mod referenced_tests;"]), - PatternCheck::new("Metal HT chunk planning test owner", &planning) - .required(&["mod fixtures;", "mod referenced;"]), - PatternCheck::new("Metal HT chunk cache test owner", &cache).required(&["mod referenced;"]), - ]); - for (path, required, max_lines) in [ - ( - tests_root.join("planning/fixtures.rs"), - "prepared_fixtures_select_expected_dedicated_metal_ht_pipelines", - 150, - ), - ( - tests_root.join("planning/referenced.rs"), - "referenced_payload_pack_preserves_job_order_and_rebases_offsets", - 150, - ), - ( - tests_root.join("cache/referenced.rs"), - "second_referenced_prepared_submission_uploads_no_immutable_arenas", - 150, - ), - ] { - let source = fs::read_to_string(&path) - .unwrap_or_else(|error| panic!("read {}: {error}", path.display())); - assert!( - source.contains(required), - "{} must own {required}", - path.display() - ); - assert!( - source.lines().count() < max_lines, - "{} must stay below its focused-test line-count ratchet", - path.display() - ); - } - assert!( - !ht_chunks_root.join("referenced_tests.rs").exists(), - "referenced HT planning and cache tests must live with their behavior owners" - ); -} - -#[test] -fn metal_prepared_ht_execution_cache_is_split_from_immutable_execution() { - let root = repo_root().join("crates/j2k-metal/src/compute/decode_dispatch/ht_chunks"); - let prepared = - fs::read_to_string(root.join("prepared.rs")).expect("read prepared HT execution owner"); - let cache = fs::read_to_string(root.join("prepared/cache.rs")) - .expect("read prepared HT execution cache owner"); - let entry = fs::read_to_string(root.join("prepared/cache/entry.rs")) - .expect("read prepared HT execution cache-entry owner"); - - assert_pattern_checks(&[ - PatternCheck::new("prepared HT immutable execution ownership", &prepared) - .required(&[ - "pub(super) mod cache;", - "struct PreparedMetalHtExecution", - "struct PreparedMetalHtChunk", - "fn prepare(", - "let packed = plan.pack_chunk(chunk_index)?;", - ]) - .forbidden(&[ - "struct PreparedMetalHtInputKey", - "enum PreparedMetalHtPayloadKey", - "struct PreparedMetalHtExecutionCacheEntry", - "struct PreparedMetalHtExecutionCache", - "fn get_or_prepare(", - "fn find(", - "fn evict_until_fits(", - "fn retained_bytes(", - "fn host_bytes(", - "fn device_bytes(", - ]), - PatternCheck::new("prepared HT execution cache ownership", &cache) - .required(&[ - "mod entry;", - "struct PreparedMetalHtExecutionCache", - "fn get_or_prepare(", - "fn prepare_entry(", - "fn insert_entry(", - "fn find(", - "fn evict_until_fits(", - "fn prepared_metal_ht_execution(", - ]) - .forbidden(&[ - "struct PreparedMetalHtInputKey", - "enum PreparedMetalHtPayloadKey", - "fn retained_bytes(", - "fn host_bytes(", - "fn device_bytes(", - "let packed = plan.pack_chunk(chunk_index)?;", - "copied_slice_buffer(device, &packed.coded_data)", - "copied_slice_buffer(device, &packed.jobs)", - ]), - PatternCheck::new("prepared HT cache-entry ownership", &entry) - .required(&[ - "struct PreparedMetalHtInputKey", - "enum PreparedMetalHtPayloadKey", - "struct PreparedMetalHtExecutionCacheEntry", - "fn prepare(", - "fn matches(", - "fn retained_bytes(", - "fn host_bytes(", - "fn device_bytes(", - ]) - .forbidden(&[ - "fn get_or_prepare(", - "fn insert_entry(", - "fn evict_until_fits(", - ]), - ]); - assert!( - prepared.lines().count() < 175, - "prepared.rs must remain the focused immutable execution/chunk owner" - ); - assert!( - cache.lines().count() < 275, - "prepared/cache.rs must remain the focused cache-policy owner" - ); - assert!( - entry.lines().count() < 275, - "prepared/cache/entry.rs must remain the focused identity/accounting owner" - ); - - let (_, lookup_tail) = cache - .split_once("fn get_or_prepare(") - .expect("find prepared HT cache lookup orchestrator"); - let (lookup, _) = lookup_tail - .split_once("\n fn prepare_entry(") - .expect("isolate prepared HT cache lookup orchestrator"); - for phase in ["self.prepare_entry(", "self.insert_entry("] { - assert!(lookup.contains(phase), "cache lookup must delegate {phase}"); - } - for implementation_detail in [ - "PreparedMetalHtInputKey {", - "self.entries.push(", - "input_key_host_bytes(", - ] { - assert!( - !lookup.contains(implementation_detail), - "cache lookup must not own {implementation_detail}" - ); - } -} diff --git a/xtask/tests/repo_lint_support/metal_compute_structure_policy/batch_execution/stacked_execution.rs b/xtask/tests/repo_lint_support/metal_compute_structure_policy/batch_execution/stacked_execution.rs deleted file mode 100644 index a4a7d510..00000000 --- a/xtask/tests/repo_lint_support/metal_compute_structure_policy/batch_execution/stacked_execution.rs +++ /dev/null @@ -1,60 +0,0 @@ -// SPDX-License-Identifier: MIT OR Apache-2.0 - -use std::fs; - -use crate::repo_lint_support::{assert_pattern_checks, repo_root, PatternCheck}; - -#[test] -fn metal_stacked_execution_is_split_by_codec_stage() { - let root = repo_root().join("crates/j2k-metal/src/compute/direct_stacked_batch"); - let staged_modules = [ - ("command_submission.rs", 250), - ("command_submission/classic_tier1.rs", 400), - ("command_submission/ht_tier1.rs", 400), - ("command_submission/reconstruction.rs", 300), - ("command_submission/final_store.rs", 200), - ("repeated_grayscale/execution.rs", 200), - ("repeated_grayscale/execution/tier1.rs", 300), - ("repeated_grayscale/execution/reconstruction.rs", 250), - ("repeated_grayscale/execution/final_store.rs", 250), - ]; - - for (relative, max_lines) in staged_modules { - let path = root.join(relative); - let source = fs::read_to_string(&path) - .unwrap_or_else(|error| panic!("read {}: {error}", path.display())); - assert!( - source.lines().count() < max_lines, - "{} must stay below its codec-stage line-count ratchet", - path.display() - ); - } - - let submission = fs::read_to_string(root.join("command_submission.rs")) - .expect("read stacked command-submission shell"); - let repeated = fs::read_to_string(root.join("repeated_grayscale/execution.rs")) - .expect("read repeated grayscale execution shell"); - assert_pattern_checks(&[ - PatternCheck::new("stacked command-submission stage shell", &submission) - .required(&[ - "mod classic_tier1;", - "mod final_store;", - "mod ht_tier1;", - "mod reconstruction;", - ]) - .forbidden(&[ - "fn submit_classic_group(", - "fn submit_ht_group(", - "fn submit_idwt(", - "fn submit_store(", - ]), - PatternCheck::new("repeated grayscale execution stage shell", &repeated) - .required(&["mod final_store;", "mod reconstruction;", "mod tier1;"]) - .forbidden(&[ - "fn encode_classic_group(", - "fn encode_ht_group(", - "fn encode_idwt(", - "fn encode_store(", - ]), - ]); -} diff --git a/xtask/tests/repo_lint_support/metal_compute_structure_policy/color_store.rs b/xtask/tests/repo_lint_support/metal_compute_structure_policy/color_store.rs deleted file mode 100644 index 7232121c..00000000 --- a/xtask/tests/repo_lint_support/metal_compute_structure_policy/color_store.rs +++ /dev/null @@ -1,39 +0,0 @@ -// SPDX-License-Identifier: MIT OR Apache-2.0 - -use std::fs; - -use crate::repo_lint_support::{assert_pattern_checks, repo_root, PatternCheck}; - -#[test] -fn exact_native_color_store_separates_planning_from_encoder_dispatch() { - let root = - repo_root().join("crates/j2k-metal/src/compute/direct_grayscale_execute/color_destination"); - let dispatch = - fs::read_to_string(root.join("store.rs")).expect("read exact native color store dispatch"); - let plan = - fs::read_to_string(root.join("store/plan.rs")).expect("read exact native color store plan"); - - assert_pattern_checks(&[ - PatternCheck::new("exact native color store dispatch", &dispatch) - .required(&[ - "mod plan;", - "struct NativeColorStoreConfig", - "plan_exact_native_color_store(", - ]) - .forbidden(&[ - "validate_stacked_color_destination_indices(", - "J2kNativeColorBatchStoreParams {", - "fn native_color_store_pipeline(", - ]), - PatternCheck::new("exact native color store planning", &plan).required(&[ - "struct NativeColorStorePlan", - "fn plan_exact_native_color_store<'a>(", - "validate_stacked_color_destination_indices(", - "J2kNativeColorBatchStoreParams {", - "fn native_color_store_pipeline(", - ]), - ]); - assert!(dispatch.lines().count() < 100); - assert!(plan.lines().count() < 150); - assert!(!plan.lines().any(|line| line.trim() == "use super::*;")); -} diff --git a/xtask/tests/repo_lint_support/metal_compute_structure_policy/compute_tests.rs b/xtask/tests/repo_lint_support/metal_compute_structure_policy/compute_tests.rs deleted file mode 100644 index 80b36e40..00000000 --- a/xtask/tests/repo_lint_support/metal_compute_structure_policy/compute_tests.rs +++ /dev/null @@ -1,101 +0,0 @@ -// SPDX-License-Identifier: MIT OR Apache-2.0 - -use std::fs; - -use super::super::repo_root; - -#[test] -fn metal_compute_tests_are_split_by_behavior() { - let root = repo_root(); - let shell_path = root.join("crates/j2k-metal/src/compute/tests.rs"); - let test_root = root.join("crates/j2k-metal/src/compute/tests"); - let shell = fs::read_to_string(&shell_path).expect("read Metal compute test shell"); - - for (module, owned_symbol, max_lines) in [ - ( - "capacity", - "fn classic_encode_output_capacity_keeps_conservative_default(", - 300usize, - ), - ( - "classic", - "fn prepared_classic_sub_band_decodes_on_cpu_for_hybrid_upload(", - 400usize, - ), - ( - "grouping", - "fn direct_sub_band_grouping_groups_adjacent_ht_runs_without_runtime(", - 400usize, - ), - ( - "hybrid", - "fn hybrid_rgb8_batch_uses_stacked_component_graph(", - 450usize, - ), - ( - "hybrid_support", - "fn prepared_direct_color_tier1_input_count(", - 150usize, - ), - ( - "reuse", - "fn hybrid_rgb8_reused_plan_caches_cpu_tier1_inputs_across_calls(", - 350usize, - ), - ( - "roi", - "fn cropped_region_scaled_ht_direct_plan_prunes_codeblocks_outside_output_roi(", - 400usize, - ), - ( - "runtime", - "fn runtime_initialization_error_classifies_null_queue_as_unavailable(", - 300usize, - ), - ] { - assert!(shell.contains(&format!("mod {module};"))); - assert!( - !shell.contains(owned_symbol), - "Metal compute test shell must not retain {owned_symbol}" - ); - let module_path = test_root.join(format!("{module}.rs")); - let source = fs::read_to_string(&module_path) - .unwrap_or_else(|error| panic!("read {}: {error}", module_path.display())); - assert!( - source.contains(owned_symbol), - "{} must own {owned_symbol}", - module_path.display() - ); - assert!( - source.lines().count() < max_lines, - "{} exceeded its focused {max_lines}-line limit", - module_path.display() - ); - assert!( - !source.lines().any(|line| line.trim() == "use super::*;"), - "{} must import explicit test dependencies", - module_path.display() - ); - } - - assert!( - shell.lines().count() < 30, - "Metal compute test shell must remain a module inventory" - ); -} - -#[test] -fn referenced_plan_regression_has_focused_owner() { - let root = repo_root(); - let shell = fs::read_to_string(root.join("crates/j2k-metal/src/compute/tests.rs")) - .expect("read Metal compute test shell"); - let child = - fs::read_to_string(root.join("crates/j2k-metal/src/compute/tests/referenced_plan.rs")) - .expect("read Metal referenced-plan tests"); - let symbol = "fn referenced_htj2k_payload_ranges_reconstruct_owned_direct_plan_bytes"; - assert!(shell.contains("mod referenced_plan;")); - assert!(!shell.contains(symbol)); - assert!(child.contains(symbol)); - assert!(child.lines().count() < 100); - assert!(!child.lines().any(|line| line.trim() == "use super::*;")); -} diff --git a/xtask/tests/repo_lint_support/metal_compute_structure_policy/direct_plan_ratchets.rs b/xtask/tests/repo_lint_support/metal_compute_structure_policy/direct_plan_ratchets.rs deleted file mode 100644 index b3986838..00000000 --- a/xtask/tests/repo_lint_support/metal_compute_structure_policy/direct_plan_ratchets.rs +++ /dev/null @@ -1,231 +0,0 @@ -// SPDX-License-Identifier: MIT OR Apache-2.0 - -use std::{fs, path::Path}; - -use crate::repo_lint_support::repo_root; -use syn::spanned::Spanned; - -fn assert_top_level_function_below( - relative: &str, - source: &str, - function_name: &str, - max_lines: usize, -) { - let syntax = - syn::parse_file(source).unwrap_or_else(|error| panic!("parse {relative}: {error}")); - let function = syntax - .items - .iter() - .find_map(|item| match item { - syn::Item::Fn(function) if function.sig.ident == function_name => Some(function), - _ => None, - }) - .unwrap_or_else(|| panic!("find {function_name} in {relative}")); - let span = function.span(); - let lines = span.end().line.saturating_sub(span.start().line) + 1; - assert!( - lines < max_lines, - "{function_name} in {relative} must stay below {max_lines} lines, found {lines}" - ); -} - -pub(super) fn assert_direct_executor_line_ratchets(root: &Path) { - let source_root = root.join("crates/j2k-metal/src"); - for (relative, max_lines) in [ - ("compute/direct_commands.rs", 160), - ("compute/direct_grayscale_execute.rs", 500), - ("compute/direct_grayscale_execute/allocation.rs", 150), - ( - "compute/direct_grayscale_execute/color_batch_completion.rs", - 125, - ), - ("compute/direct_grayscale_execute/single.rs", 350), - ("compute/direct_grayscale_execute/component_plane.rs", 100), - ( - "compute/direct_grayscale_execute/component_plane/execution.rs", - 450, - ), - ( - "compute/direct_grayscale_execute/component_plane/execution/final_plane.rs", - 150, - ), - ("compute/direct_stacked_batch/repeated_grayscale.rs", 100), - ( - "compute/direct_stacked_batch/repeated_grayscale/execution.rs", - 600, - ), - ] { - let path = source_root.join(relative); - let source = fs::read_to_string(&path) - .unwrap_or_else(|error| panic!("read {}: {error}", path.display())); - let production = source - .split_once("#[cfg(test)]") - .map_or(source.as_str(), |(production, _)| production); - assert!( - production.lines().count() < max_lines, - "{relative} must stay below its direct-executor line-count ratchet" - ); - } -} - -#[test] -fn direct_color_batch_execution_has_explicit_phases() { - let source = fs::read_to_string( - repo_root().join("crates/j2k-metal/src/compute/direct_grayscale_execute.rs"), - ) - .expect("read Metal direct color batch execution"); - let completion = - fs::read_to_string(repo_root().join( - "crates/j2k-metal/src/compute/direct_grayscale_execute/color_batch_completion.rs", - )) - .expect("read Metal direct color batch completion"); - let (_, orchestrator) = source - .split_once("pub(super) fn execute_direct_color_plan_batch_with_tier1_options") - .expect("find direct color batch orchestrator"); - - for phase in [ - "fn allocate_direct_color_batch_execution(", - "fn encode_direct_color_batch_routes(", - ] { - assert!(source.contains(phase), "missing concrete phase {phase}"); - } - for phase in [ - "fn complete_direct_color_batch_command(", - "fn complete_split_direct_color_batch_command(", - "fn retire_direct_color_batch_resources(", - ] { - assert!(completion.contains(phase), "missing concrete phase {phase}"); - } - assert!(source.contains("mod color_batch_completion;")); - assert!(source.contains("use self::color_batch_completion::{")); - for delegated in [ - "allocate_direct_color_batch_execution(", - "encode_direct_color_batch_routes(", - ] { - assert!( - orchestrator.contains(delegated), - "orchestrator must delegate {delegated}" - ); - } - for implementation_detail in [ - "checked_count_sum(", - "direct_ht_job_count(", - "try_encode_stacked_mct_rgb8_direct_color_batch(", - "retire_direct_status_checks(", - "recycle_scratch_buffers(", - ] { - assert!( - !orchestrator.contains(implementation_detail), - "orchestrator must not inline {implementation_detail}" - ); - } - - let (_, allocation_tail) = source - .split_once("fn allocate_direct_color_batch_execution(") - .expect("find direct color allocation phase"); - let (allocation, _) = allocation_tail - .split_once("\nfn ") - .expect("isolate direct color allocation phase"); - for accounting in [ - "tier1_mode == DirectTier1Mode::Metal", - "direct_ht_job_count(", - ".flat_map(|plan| plan.component_plans.iter())", - "J2K Metal direct color batch HT jobs", - ] { - assert!( - allocation.contains(accounting), - "allocation phase must preserve {accounting}" - ); - } - assert!(orchestrator.lines().count() < 45); - assert!(!source.contains("clippy::too_many_lines")); - assert!(!completion.contains("clippy::too_many_lines")); - assert!(!completion.contains("use super::*;")); - assert!(source.lines().count() < 500); - assert!(completion.lines().count() < 125); -} - -#[test] -fn direct_prepare_hotspots_have_focused_owners_and_short_orchestrators() { - let source_root = repo_root().join("crates/j2k-metal/src/compute/direct_prepare"); - let sources = [ - ("ht.rs", 100), - ("ht/referenced.rs", 250), - ("ht/grouped.rs", 275), - ("classic.rs", 40), - ("classic/payload.rs", 200), - ("classic/sub_band.rs", 225), - ("classic/grouped.rs", 275), - ] - .into_iter() - .map(|(relative, max_lines)| { - let path = source_root.join(relative); - let source = fs::read_to_string(&path) - .unwrap_or_else(|error| panic!("read {}: {error}", path.display())); - assert!( - source.lines().count() < max_lines, - "{relative} must stay below its direct-prepare line-count ratchet" - ); - assert!( - !source.lines().any(|line| line.trim() == "use super::*;"), - "{relative} must use explicit imports" - ); - assert!( - !source.contains("clippy::too_many_lines"), - "{relative} must keep orchestration functions focused" - ); - (relative, source) - }) - .collect::>(); - - let ht = &sources["ht.rs"]; - for ownership in [ - "mod grouped;", - "mod referenced;", - "use self::grouped::prepare_ht_sub_band_groups;", - "use self::referenced::prepare_referenced_ht_sub_band;", - ] { - assert!(ht.contains(ownership), "ht.rs must declare {ownership}"); - } - assert!(!ht.contains("fn prepare_referenced_ht_sub_band(")); - assert!(!ht.contains("fn prepare_ht_sub_band_group(")); - - let classic = &sources["classic.rs"]; - for ownership in [ - "mod grouped;", - "mod payload;", - "mod sub_band;", - "use self::grouped::{", - "prepare_classic_sub_band_groups, prepare_sub_band_groups,", - "use self::payload::{", - "prepare_referenced_classic_sub_band, ReferencedClassicPayloadCursor,", - "use self::sub_band::prepare_classic_sub_band;", - ] { - assert!( - classic.contains(ownership), - "classic.rs must declare {ownership}" - ); - } - for misplaced in [ - "struct ReferencedClassicPayloadCursor", - "fn prepare_classic_sub_band_with_payloads(", - "fn prepare_classic_sub_band_group(", - ] { - assert!( - !classic.contains(misplaced), - "classic.rs must not own {misplaced}" - ); - } - - for (relative, function_name) in [ - ("ht/referenced.rs", "prepare_referenced_ht_sub_band"), - ("ht/grouped.rs", "prepare_ht_sub_band_group"), - ( - "classic/sub_band.rs", - "prepare_classic_sub_band_with_payloads", - ), - ("classic/grouped.rs", "prepare_classic_sub_band_group"), - ] { - assert_top_level_function_below(relative, &sources[relative], function_name, 80); - } -} diff --git a/xtask/tests/repo_lint_support/metal_compute_structure_policy/direct_plan_types.rs b/xtask/tests/repo_lint_support/metal_compute_structure_policy/direct_plan_types.rs deleted file mode 100644 index a63c72e4..00000000 --- a/xtask/tests/repo_lint_support/metal_compute_structure_policy/direct_plan_types.rs +++ /dev/null @@ -1,285 +0,0 @@ -// SPDX-License-Identifier: MIT OR Apache-2.0 - -use std::fs; - -use super::direct_plan_ratchets; -use crate::repo_lint_support::{assert_pattern_checks, repo_root, PatternCheck}; - -#[test] -#[expect( - clippy::too_many_lines, - reason = "direct-plan type ownership and source ratchets form one structural contract" -)] -fn metal_direct_plan_types_live_in_focused_module() { - let root = repo_root(); - let compute = fs::read_to_string(root.join("crates/j2k-metal/src/compute.rs")) - .expect("read Metal compute module"); - let direct_execute = - fs::read_to_string(root.join("crates/j2k-metal/src/compute/direct_execute.rs")) - .expect("read Metal direct execute module"); - let direct_commands = - fs::read_to_string(root.join("crates/j2k-metal/src/compute/direct_commands.rs")) - .expect("read Metal direct command lifecycle module"); - let plan_types = - fs::read_to_string(root.join("crates/j2k-metal/src/compute/direct_plan_types.rs")) - .expect("read Metal direct plan types module"); - let plane_pack = - fs::read_to_string(root.join("crates/j2k-metal/src/compute/direct_plane_pack.rs")) - .expect("read Metal direct plane-pack module"); - let prepare = fs::read_to_string(root.join("crates/j2k-metal/src/compute/direct_prepare.rs")) - .expect("read Metal direct prepare module"); - let prepare_grayscale = - fs::read_to_string(root.join("crates/j2k-metal/src/compute/direct_prepare/grayscale.rs")) - .expect("read Metal grayscale direct preparation module"); - let roi = fs::read_to_string(root.join("crates/j2k-metal/src/compute/direct_roi.rs")) - .expect("read Metal direct ROI module"); - let grayscale_execute = - fs::read_to_string(root.join("crates/j2k-metal/src/compute/direct_grayscale_execute.rs")) - .expect("read Metal direct grayscale executor module"); - let grayscale_allocation = fs::read_to_string( - root.join("crates/j2k-metal/src/compute/direct_grayscale_execute/allocation.rs"), - ) - .expect("read Metal direct grayscale allocation module"); - let grayscale_component = fs::read_to_string( - root.join("crates/j2k-metal/src/compute/direct_grayscale_execute/component_plane.rs"), - ) - .expect("read Metal direct grayscale component-plane facade"); - let grayscale_component_execution = fs::read_to_string(root.join( - "crates/j2k-metal/src/compute/direct_grayscale_execute/component_plane/execution.rs", - )) - .expect("read Metal direct grayscale component-plane execution stage"); - let grayscale_component_final_plane = fs::read_to_string(root.join( - "crates/j2k-metal/src/compute/direct_grayscale_execute/component_plane/execution/final_plane.rs", - )) - .expect("read Metal direct grayscale final component-plane owner"); - let stacked_batch = - fs::read_to_string(root.join("crates/j2k-metal/src/compute/direct_stacked_batch.rs")) - .expect("read Metal direct stacked batch module"); - let repeated_grayscale = fs::read_to_string( - root.join("crates/j2k-metal/src/compute/direct_stacked_batch/repeated_grayscale.rs"), - ) - .expect("read Metal repeated-grayscale shell"); - let repeated_grayscale_execution = - fs::read_to_string(root.join( - "crates/j2k-metal/src/compute/direct_stacked_batch/repeated_grayscale/execution.rs", - )) - .expect("read Metal repeated-grayscale execution module"); - let repeated_grayscale_reconstruction = fs::read_to_string(root.join( - "crates/j2k-metal/src/compute/direct_stacked_batch/repeated_grayscale/execution/reconstruction.rs", - )) - .expect("read Metal repeated-grayscale reconstruction stage"); - let surface_pack = - fs::read_to_string(root.join("crates/j2k-metal/src/compute/direct_surface_pack.rs")) - .expect("read Metal direct surface-pack module"); - - assert!( - compute.lines().count() < 450, - "compute.rs must stay below the honest post-include-removal line-count ratchet" - ); - assert!( - direct_execute.lines().count() < 150, - "direct_execute.rs must remain a focused planning and execution module" - ); - - assert_pattern_checks(&[ - PatternCheck::new("Metal compute direct module ownership", &compute) - .required(&[ - "mod direct_commands;", - "mod direct_plan_types;", - "pub(crate) use self::direct_plan_types", - "mod direct_execute;", - "pub(crate) use self::direct_execute", - ]) - .forbidden(&[ - "include!(\"compute/direct_execute_impl.rs\")", - "fn prepare_direct_color_plan_with_tier1_mode(", - "fn new_command_buffer(", - ]), - PatternCheck::new("Metal direct command lifecycle", &direct_commands).required(&[ - "fn new_command_buffer(", - "fn new_compute_command_encoder(", - "fn new_blit_command_encoder(", - ]), - PatternCheck::new("Metal direct execute module", &direct_execute) - .required(&[ - "direct_plan_types::{", - "direct_prepare::{", - "direct_roi::crop_prepared_direct_grayscale_plan_to_output_region", - "direct_tier1::DirectTier1Mode", - "pub(crate) fn prepare_direct_color_plan(", - "pub(crate) fn prepare_direct_color_plan_for_cpu_upload(", - "pub(crate) fn crop_prepared_direct_color_plan_to_output_region(", - ]) - .forbidden(&[ - "include!(", - "use super::*", - "mod direct_plan_types;", - "mod direct_prepare;", - "mod direct_roi;", - ]), - PatternCheck::new("Metal direct preparation facade", &prepare) - .required(&[ - "mod classic;", - "mod color;", - "mod grayscale;", - "mod ht;", - "mod referenced;", - ]) - .forbidden(&["fn prepare_direct_grayscale_plan("]), - ]); - assert_pattern_checks(&[ - PatternCheck::new("Metal direct grayscale execution shell", &grayscale_execute) - .required(&[ - "mod allocation;", - "mod component_plane;", - "mod single;", - "pub(in crate::compute) use self::component_plane::{", - "checked_coefficient_len", - "encode_prepared_direct_component_plane_in_command_buffer", - "upload_cpu_decoded_coefficients", - "DirectComponentPlaneRequest", - "allocate_direct_execution_metadata, direct_ht_job_count, DirectExecutionMetadata,", - "pub(in crate::compute) use self::single::encode_prepared_direct_grayscale_plan_in_command_buffer;", - ]) - .forbidden(&[ - "pub(in crate::compute) use self::component_plane::*;", - "pub(in crate::compute) use self::single::*;", - "fn allocate_direct_execution_metadata(", - "fn encode_prepared_direct_component_plane_in_command_buffer(", - ]), - PatternCheck::new("Metal component-plane facade", &grayscale_component) - .required(&[ - "mod execution;", - "pub(in crate::compute) use self::execution::encode_prepared_direct_component_plane_in_encoder;", - "pub(in crate::compute) struct DirectComponentPlaneRequest", - "fn encode_prepared_direct_component_plane_in_command_buffer(", - ]) - .forbidden(&[ - "struct ComponentPlaneExecution", - "fn encode_store(&mut self", - "use super::*", - ]), - PatternCheck::new( - "Metal component-plane execution stage", - &grayscale_component_execution, - ) - .required(&[ - "mod final_plane;", - "struct ComponentPlaneExecution", - "fn encode_store(&mut self", - "self.bands.clear();", - "fn encode_prepared_direct_component_plane_in_encoder(", - ]) - .forbidden(&["struct RetainedComponentPlane", "use super::*"]), - PatternCheck::new( - "Metal final component-plane lifecycle", - &grayscale_component_final_plane, - ) - .required(&[ - "struct FinalComponentPlane", - "struct RetainedComponentPlane", - "fn buffer_for_store(", - "fn validate_later_component_store(", - "later_component_store_requires_exact_final_plane_geometry", - ]) - .forbidden(&["use super::*"]), - PatternCheck::new("Metal direct execution allocation owner", &grayscale_allocation) - .required(&[ - "pub(super) struct DirectExecutionMetadata", - "pub(super) fn allocate_direct_execution_metadata(", - "direct_execution_resources_honor_exact_cap_and_one_byte_over", - ]), - PatternCheck::new( - "Metal repeated grayscale execution shell", - &repeated_grayscale, - ) - .required(&[ - "mod execution;", - "encode_repeated_direct_grayscale_plan_in_command_buffer", - ]) - .forbidden(&["struct RepeatedGrayscaleExecution"]), - PatternCheck::new( - "Metal repeated grayscale execution implementation", - &repeated_grayscale_execution, - ) - .required(&["struct RepeatedGrayscaleExecution", "fn encode_step(&mut self"]) - .forbidden(&[ - "fn encode_stacked_idwt(&mut self", - "fn encode_per_instance_idwt(&mut self", - ]), - PatternCheck::new( - "Metal repeated grayscale reconstruction stage", - &repeated_grayscale_reconstruction, - ) - .required(&[ - "fn encode_stacked_idwt(", - "fn encode_per_instance_idwt(", - ]), - ]); - direct_plan_ratchets::assert_direct_executor_line_ratchets(root); - for item in [ - "pub(crate) struct PreparedDirectGrayscalePlan", - "pub(crate) struct PreparedDirectColorPlan", - "pub(super) enum PreparedDirectGrayscaleStep", - "pub(super) struct PreparedDirectIdwt", - "pub(super) struct PreparedClassicSubBand", - "pub(super) struct PreparedClassicSubBandGroup", - "pub(super) struct PreparedHtSubBand", - "pub(super) struct PreparedHtSubBandGroup", - ] { - assert_pattern_checks(&[ - PatternCheck::new( - "direct_execute.rs direct plan type exclusion", - &direct_execute, - ) - .forbidden(&[item]), - PatternCheck::new( - "compute/direct_plan_types.rs direct plan type ownership", - &plan_types, - ) - .required(&[item]), - ]); - } - for required in [ - ( - "mod direct_plane_pack;", - &plane_pack, - "pub(super) struct PlaneStage", - ), - ( - "mod direct_prepare;", - &prepare_grayscale, - "pub(crate) fn prepare_direct_grayscale_plan", - ), - ( - "mod direct_roi;", - &roi, - "pub(crate) fn crop_prepared_direct_grayscale_plan_to_output_region", - ), - ( - "mod direct_grayscale_execute;", - &grayscale_component, - "pub(in crate::compute) fn encode_prepared_direct_component_plane_in_command_buffer", - ), - ( - "mod direct_stacked_batch;", - &stacked_batch, - "pub(super) fn encode_stacked_direct_component_plane_batch", - ), - ( - "mod direct_surface_pack;", - &surface_pack, - "pub(super) fn output_shape_for", - ), - ] { - let (module_wire, module_source, owned_item) = required; - assert_pattern_checks(&[ - PatternCheck::new("compute.rs direct child module wiring", &compute) - .required(&[module_wire]), - PatternCheck::new("direct split module owned item", module_source) - .required(&[owned_item]), - PatternCheck::new("direct_execute.rs owned item exclusion", &direct_execute) - .forbidden(&[owned_item]), - ]); - } -} diff --git a/xtask/tests/repo_lint_support/metal_compute_structure_policy/runtime_registry.rs b/xtask/tests/repo_lint_support/metal_compute_structure_policy/runtime_registry.rs deleted file mode 100644 index 350d52d7..00000000 --- a/xtask/tests/repo_lint_support/metal_compute_structure_policy/runtime_registry.rs +++ /dev/null @@ -1,258 +0,0 @@ -// SPDX-License-Identifier: MIT OR Apache-2.0 - -use std::fs; - -use crate::repo_lint_support::{assert_pattern_checks, repo_root, PatternCheck}; - -#[test] -#[expect( - clippy::too_many_lines, - reason = "the Metal compute registry split is enforced by one cohesive ownership matrix" -)] -fn metal_compute_runtime_registry_is_split_from_compute_god_file() { - let root = repo_root(); - let compute = fs::read_to_string(root.join("crates/j2k-metal/src/compute.rs")) - .expect("read Metal compute module"); - let runtime = fs::read_to_string(root.join("crates/j2k-metal/src/compute/runtime.rs")) - .expect("read Metal compute runtime module"); - let forward_transform = - fs::read_to_string(root.join("crates/j2k-metal/src/compute/forward_transform.rs")) - .expect("read Metal compute forward-transform module"); - let resident_tier1 = - fs::read_to_string(root.join("crates/j2k-metal/src/compute/resident_tier1.rs")) - .expect("read Metal compute resident tier1 module"); - let resident_tier1_types = - fs::read_to_string(root.join("crates/j2k-metal/src/compute/resident_tier1/types.rs")) - .expect("read Metal compute resident tier1 types module"); - let lossless_prepare = - fs::read_to_string(root.join("crates/j2k-metal/src/compute/lossless_prepare.rs")) - .expect("read Metal compute lossless prepare module"); - let lossless_prepare_single = - fs::read_to_string(root.join("crates/j2k-metal/src/compute/lossless_prepare/single.rs")) - .expect("read Metal compute single-item lossless prepare module"); - let decode_dispatch = - fs::read_to_string(root.join("crates/j2k-metal/src/compute/decode_dispatch.rs")) - .expect("read Metal compute decode dispatch module"); - let decode_dispatch_mct = - fs::read_to_string(root.join("crates/j2k-metal/src/compute/decode_dispatch/mct.rs")) - .expect("read Metal compute decode MCT module"); - let tier1_encode = - fs::read_to_string(root.join("crates/j2k-metal/src/compute/tier1_encode.rs")) - .expect("read Metal compute tier1 encode module"); - let resident_codestream = - fs::read_to_string(root.join("crates/j2k-metal/src/compute/resident_codestream.rs")) - .expect("read Metal compute resident codestream module"); - let resident_codestream_tier2 = fs::read_to_string( - root.join("crates/j2k-metal/src/compute/resident_codestream/tier2_packetization.rs"), - ) - .expect("read Metal compute resident codestream tier-2 packetization module"); - let resident_codestream_tier2_tests = fs::read_to_string( - root.join("crates/j2k-metal/src/compute/resident_codestream/tier2_packetization/tests.rs"), - ) - .expect("read Metal compute resident codestream tier-2 packetization tests"); - let resident_codestream_ht_cleanup = fs::read_to_string( - root.join("crates/j2k-metal/src/compute/resident_codestream/ht_cleanup.rs"), - ) - .expect("read Metal compute resident codestream HT cleanup module"); - let resident_codestream_classic_labels = fs::read_to_string( - root.join("crates/j2k-metal/src/compute/resident_codestream/classic_labels.rs"), - ) - .expect("read Metal compute resident codestream classic labels module"); - let decode_cleanup = - fs::read_to_string(root.join("crates/j2k-metal/src/compute/decode_cleanup.rs")) - .expect("read Metal compute decode cleanup module"); - assert!( - compute.lines().count() < 450, - "compute.rs must stay below the honest post-include-removal line-count ratchet" - ); - assert!( - resident_codestream.lines().count() < 150, - "resident_codestream.rs must remain a focused module shell" - ); - assert!( - resident_codestream_tier2.lines().count() < 500, - "tier2_packetization.rs must stay below its post-extraction line-count ratchet" - ); - assert!( - resident_codestream_tier2_tests.lines().count() < 200, - "tier2_packetization/tests.rs must stay below its focused-test line-count ratchet" - ); - assert!( - resident_codestream_tier2.contains("#[cfg(test)]\nmod tests;") - && !resident_codestream_tier2 - .contains("fn tier2_metadata_plan_honors_exact_aggregate_cap"), - "Tier-2 packetization regression tests must remain in their focused test module" - ); - assert!( - resident_codestream.contains("mod ht_cleanup;") - && resident_codestream_ht_cleanup - .contains("pub(in crate::compute) fn dispatch_ht_cleanup") - && !resident_codestream.contains("fn dispatch_ht_cleanup("), - "resident_codestream HT cleanup dispatch helpers must live in resident_codestream/ht_cleanup.rs" - ); - assert!( - resident_codestream.contains("mod classic_labels;") - && resident_codestream_classic_labels.contains("CLASSIC_TIER1_DENSITY_LABEL") - && resident_codestream_classic_labels.contains("next_enabled_classic_stage_label") - && !resident_codestream.contains("const CLASSIC_TIER1_DENSITY_LABEL") - && !resident_codestream.contains("fn next_enabled_classic_stage_label("), - "resident_codestream classic profiling labels must live in resident_codestream/classic_labels.rs" - ); - - assert_pattern_checks(&[ - PatternCheck::new("Metal compute runtime module shell", &compute) - .required(&[ - "mod runtime;", - "pub(crate) use self::runtime", - "MetalRuntime", - "runtime_initialization_error", - ]) - .forbidden(&[ - "pub(crate) struct MetalRuntime", - "MetalPipelineLoader::new(device", - ]), - PatternCheck::new("Metal compute runtime implementation", &runtime).required(&[ - "pub(crate) struct MetalRuntime", - "MetalPipelineLoader::new(device", - "DEFAULT_METAL_SESSION", - "METAL_RUNTIME_OVERRIDE", - "with_runtime_for_session", - ]), - PatternCheck::new("Metal resident codestream module wiring", &compute) - .required(&["mod resident_codestream;"]) - .forbidden(&["pub(crate) fn encode_tier2_packetization"]), - PatternCheck::new( - "Metal resident codestream tier-2 module shell", - &resident_codestream, - ) - .required(&[ - "mod tier2_packetization;", - "pub(crate) use self::tier2_packetization::encode_tier2_packetization;", - ]) - .forbidden(&[ - "pub(crate) use self::tier2_packetization::*;", - "pub(crate) fn encode_tier2_packetization", - ]), - PatternCheck::new( - "Metal resident codestream tier-2 packetization implementation", - &resident_codestream_tier2, - ) - .required(&["pub(crate) fn encode_tier2_packetization"]), - PatternCheck::new("Metal resident Tier-1 module shell", &resident_tier1) - .required(&[ - "mod counter_validation;", - "mod profile_dispatch;", - "mod readback;", - "mod result_harvest;", - "mod types;", - ]) - .forbidden(&["pub(crate) struct J2kLosslessDeviceCodeBlock"]), - PatternCheck::new("Metal lossless prepare module shell", &lossless_prepare) - .required(&[ - "mod batch;", - "mod batch_item;", - "mod commands;", - "mod forward_encode;", - "mod single;", - "mod sizes;", - ]) - .forbidden(&["pub(crate) fn prepare_lossless_device_code_blocks("]), - PatternCheck::new("Metal decode dispatch module shell", &decode_dispatch) - .required(&[ - "mod classic_cleanup;", - "mod classic_subband;", - "mod ht_distinct;", - "mod ht_subband;", - "mod idwt;", - "mod mct;", - "mod store;", - ]) - .forbidden(&["pub(crate) fn decode_inverse_mct"]), - ]); - for (relative, max_lines) in [ - ("compute/resident_tier1.rs", 100), - ("compute/resident_tier1/types.rs", 450), - ("compute/resident_tier1/readback.rs", 250), - ("compute/resident_tier1/result_harvest.rs", 500), - ("compute/resident_tier1/profile_dispatch/analysis.rs", 500), - ("compute/resident_tier1/profile_dispatch/tokens.rs", 700), - ("compute/resident_tier1/counter_validation/record.rs", 500), - ("compute/resident_tier1/counter_validation/validate.rs", 500), - ("compute/lossless_prepare.rs", 100), - ("compute/lossless_prepare/batch.rs", 150), - ("compute/lossless_prepare/batch_item.rs", 400), - ("compute/lossless_prepare/commands.rs", 700), - ("compute/lossless_prepare/forward_encode.rs", 300), - ("compute/lossless_prepare/single.rs", 200), - ("compute/decode_dispatch.rs", 100), - ("compute/decode_dispatch/classic_cleanup.rs", 525), - ( - "compute/decode_dispatch/classic_cleanup/status_sources.rs", - 100, - ), - ( - "compute/decode_dispatch/classic_cleanup/distinct_allocation.rs", - 100, - ), - ( - "compute/decode_dispatch/classic_cleanup/distinct_batch.rs", - 350, - ), - ("compute/decode_dispatch/classic_subband.rs", 450), - ("compute/decode_dispatch/ht_distinct.rs", 250), - ("compute/decode_dispatch/ht_subband.rs", 300), - ("compute/decode_dispatch/idwt.rs", 550), - ("compute/decode_dispatch/idwt/irreversible.rs", 325), - ("compute/decode_dispatch/mct.rs", 250), - ("compute/decode_dispatch/store.rs", 400), - ] { - let path = root.join("crates/j2k-metal/src").join(relative); - let source = fs::read_to_string(&path) - .unwrap_or_else(|error| panic!("read {}: {error}", path.display())); - assert!( - source.lines().count() < max_lines, - "{} must stay below its focused-module line-count ratchet", - path.display() - ); - } - for (module_wire, module_source, owned_item) in [ - ( - "mod forward_transform;", - &forward_transform, - "pub(crate) fn encode_forward_dwt53", - ), - ( - "mod resident_tier1;", - &resident_tier1_types, - "pub(crate) struct J2kLosslessDeviceCodeBlock", - ), - ( - "mod lossless_prepare;", - &lossless_prepare_single, - "pub(crate) fn prepare_lossless_device_code_blocks", - ), - ( - "mod decode_dispatch;", - &decode_dispatch_mct, - "pub(crate) fn decode_inverse_mct", - ), - ( - "mod tier1_encode;", - &tier1_encode, - "pub(crate) fn encode_classic_tier1_code_blocks", - ), - ( - "mod decode_cleanup;", - &decode_cleanup, - "pub(crate) fn decode_classic_cleanup_code_block", - ), - ] { - assert_pattern_checks(&[ - PatternCheck::new("Metal compute module wiring", &compute).required(&[module_wire]), - PatternCheck::new("split Metal compute module owned item", module_source) - .required(&[owned_item]), - PatternCheck::new("Metal compute module shell owned-item exclusion", &compute) - .forbidden(&[owned_item]), - ]); - } -} diff --git a/xtask/tests/repo_lint_support/metal_direct_plan_structure_policy.rs b/xtask/tests/repo_lint_support/metal_direct_plan_structure_policy.rs deleted file mode 100644 index 54ccd295..00000000 --- a/xtask/tests/repo_lint_support/metal_direct_plan_structure_policy.rs +++ /dev/null @@ -1,40 +0,0 @@ -// SPDX-License-Identifier: MIT OR Apache-2.0 - -use std::fs; - -use super::repo_root; - -#[test] -fn metal_direct_plan_validation_is_split_by_runtime_and_shape_ownership() { - let root = repo_root(); - let compute = fs::read_to_string(root.join("crates/j2k-metal/src/compute.rs")) - .expect("read Metal compute module"); - let validation = - fs::read_to_string(root.join("crates/j2k-metal/src/compute/direct_plan_validation.rs")) - .expect("read direct-plan validation façade"); - let runtime = fs::read_to_string( - root.join("crates/j2k-metal/src/compute/direct_plan_validation/runtime.rs"), - ) - .expect("read direct-plan runtime validation"); - let shape = fs::read_to_string( - root.join("crates/j2k-metal/src/compute/direct_plan_validation/shape.rs"), - ) - .expect("read direct-plan shape validation"); - let compute_test_support = - fs::read_to_string(root.join("crates/j2k-metal/src/compute/tests/hybrid_support.rs")) - .expect("read Metal compute hybrid test support"); - - assert!(compute.contains("mod direct_plan_validation;")); - assert!(!compute.contains("mod direct_plan_support;")); - assert!(validation.lines().count() < 40); - assert!(validation.contains("mod runtime;")); - assert!(validation.contains("mod shape;")); - assert!(runtime.contains("fn prepared_direct_color_plan_supports_runtime")); - assert!(runtime.contains("fn classic_prepared_job_supports_runtime")); - assert!(shape.contains("fn classic_group_shapes_match")); - assert!(shape.contains("fn store_shapes_match")); - assert!(compute_test_support.contains("fn prepared_direct_color_tier1_input_count")); - assert!(!root - .join("crates/j2k-metal/src/compute/direct_plan_support.rs") - .exists()); -} diff --git a/xtask/tests/repo_lint_support/mod.rs b/xtask/tests/repo_lint_support/mod.rs index 4c340b0f..d50702c8 100644 --- a/xtask/tests/repo_lint_support/mod.rs +++ b/xtask/tests/repo_lint_support/mod.rs @@ -12,18 +12,14 @@ use std::{ pub(crate) mod architecture_policy; pub(crate) mod audit_integrity_policy; pub(crate) mod corpus_policy; -pub(crate) mod coverage_structure_policy; pub(crate) mod dependency_policy; pub(crate) mod docs_and_workflows_policy; -pub(crate) mod encode_compare_structure_policy; pub(crate) mod encode_stage_error_policy; -pub(crate) mod fixture_compare_structure_policy; pub(crate) mod gpu_adapter_policy; pub(crate) mod gpu_device_structure_policy; pub(crate) mod j2k_batch_allocation_policy; pub(crate) mod j2k_component_handoff_policy; pub(crate) mod j2k_container_allocation_policy; -pub(crate) mod j2k_decode_structure_policy; pub(crate) mod j2k_encode_validation_policy; pub(crate) mod j2k_error_source_policy; pub(crate) mod j2k_ml_policy; @@ -31,8 +27,6 @@ pub(crate) mod j2k_scratch_allocation_policy; pub(crate) mod jpeg_batch_allocation_policy; pub(crate) mod jpeg_dct_reemit_policy; pub(crate) mod jpeg_decode_allocation_policy; -pub(crate) mod jpeg_decoder_structure_policy; -pub(crate) mod jpeg_encoder_structure_policy; pub(crate) mod jpeg_header_allocation_policy; pub(crate) mod jpeg_metal_resource_safety_policy; pub(crate) mod jpeg_prepared_table_policy; @@ -43,13 +37,12 @@ pub(crate) mod jpeg_transcode_allocation_policy; pub(crate) mod metal_buffer_pool_policy; pub(crate) mod metal_compute_structure_policy; pub(crate) mod metal_compute_symbol_policy; -pub(crate) mod metal_direct_plan_structure_policy; pub(crate) mod metal_resource_construction_policy; pub(crate) mod native_decode_allocation_policy; pub(crate) mod native_decode_context_reuse_policy; -pub(crate) mod native_direct_plan_structure_policy; pub(crate) mod native_encode_allocation_policy; pub(crate) mod native_tile_metadata_policy; +pub(crate) mod phase_order_policy; pub(crate) mod profile_allocation_policy; pub(crate) mod public_docs_policy; pub(crate) mod public_typed_helper_error_policy; @@ -60,9 +53,8 @@ pub(crate) mod suppression_policy; pub(crate) mod tilecodec_error_policy; pub(crate) mod transcode_api_policy; pub(crate) mod transcode_cuda_policy; -pub(crate) mod transcode_structure_policy; pub(crate) mod workflow_policy; -pub(crate) mod xtask_main_structure_policy; +pub(crate) mod workflow_structure_policy; pub(crate) fn repo_root() -> &'static Path { Path::new(env!("CARGO_MANIFEST_DIR")) @@ -211,8 +203,6 @@ pub(crate) struct FilePatternCheck<'a> { normalized_forbidden: &'a [&'a str], } -const STABLE_API_SNAPSHOT_UNION: &str = ""; - pub(crate) struct PatternCheck<'a> { source_name: &'a str, source: &'a str, @@ -301,10 +291,6 @@ impl<'a> FilePatternCheck<'a> { } } - pub(crate) fn stable_api_snapshots() -> FilePatternCheck<'static> { - FilePatternCheck::new(STABLE_API_SNAPSHOT_UNION) - } - pub(crate) fn named(mut self, source_name: &'a str) -> Self { self.source_name = Some(source_name); self @@ -429,12 +415,8 @@ pub(crate) fn assert_file_pattern_checks(root: &Path, checks: &[FilePatternCheck "{source_name} file-pattern check must define at least one pattern" ); - let source = if check.relative_path == STABLE_API_SNAPSHOT_UNION { - stable_api_snapshot_sources(root) - } else { - fs::read_to_string(root.join(check.relative_path)) - .unwrap_or_else(|err| panic!("read {}: {err}", check.relative_path)) - }; + let source = fs::read_to_string(root.join(check.relative_path)) + .unwrap_or_else(|err| panic!("read {}: {err}", check.relative_path)); assert_pattern_sets( source_name, &source, @@ -497,41 +479,57 @@ pub(crate) fn workflow_job<'a>(workflow: &'a str, job_name: &str) -> &'a str { } pub(crate) fn publishable_crate_dirs(root: &Path) -> Vec { - let xtask = xtask_sources(root); - const_array_values(&xtask, "PUBLISHABLE_PACKAGES") + release_manifest_entries(root) .into_iter() - .map(|package| { - let dir = root.join("crates").join(&package); + .map(|entry| { + let dir = root.join("crates").join(&entry.name); assert!( dir.exists(), - "PUBLISHABLE_PACKAGES entry `{package}` must resolve to an existing crate dir" + "release manifest entry `{}` must resolve to an existing crate dir", + entry.name ); dir }) .collect() } -pub(crate) fn const_array_values(source: &str, name: &str) -> Vec { - let values = const_array_block(source, name) - .lines() - .filter_map(|line| { - let value = line.trim().trim_matches([',', '"']); - if value.is_empty() - || value.starts_with("const ") - || value.starts_with(']') - || value.starts_with('&') - { - None - } else { - Some(value.to_string()) - } +#[derive(Debug, Eq, PartialEq)] +pub(crate) struct ReleaseManifestEntry { + pub(crate) name: String, + pub(crate) api_contract: String, +} + +pub(crate) fn release_manifest_entries(root: &Path) -> Vec { + let source = + fs::read_to_string(root.join("release-crates.json")).expect("read release manifest"); + let manifest: serde_json::Value = + serde_json::from_str(&source).expect("parse release manifest"); + assert_eq!(manifest["schema"], 2, "release manifest schema"); + let crates = manifest["crates"] + .as_array() + .expect("release manifest crates array"); + assert!(!crates.is_empty(), "release manifest must not be empty"); + crates + .iter() + .map(|entry| { + let name = entry["name"] + .as_str() + .expect("release manifest crate name") + .to_string(); + let api_contract = entry["api_contract"] + .as_str() + .expect("release manifest api_contract") + .to_string(); + assert!( + matches!( + api_contract.as_str(), + "stable" | "experimental" | "implementation" | "binary" + ), + "unsupported API contract for {name}: {api_contract}" + ); + ReleaseManifestEntry { name, api_contract } }) - .collect::>(); - assert!( - !values.is_empty(), - "const array {name} must contain at least one parsed value" - ); - values + .collect() } pub(crate) fn const_array_block<'a>(source: &'a str, name: &str) -> &'a str { @@ -545,8 +543,6 @@ pub(crate) fn const_array_block<'a>(source: &'a str, name: &str) -> &'a str { &rest[..end] } -const CARGO_PUBLIC_API_VERSION: &str = "0.52.0"; - static CARGO_METADATA_WORKSPACE_EDGES: OnceLock> = OnceLock::new(); pub(crate) fn cargo_metadata_workspace_edges(root: &Path) -> BTreeSet<(String, String)> { @@ -649,52 +645,6 @@ pub(crate) fn format_edge(edge: &(String, String)) -> String { format!("{} -> {}", edge.0, edge.1) } -pub(crate) fn cargo_public_api_required(root: &Path, package: &str) -> String { - let version = Command::new("cargo") - .args(["+nightly-2026-06-28", "public-api", "--version"]) - .current_dir(root) - .output() - .unwrap_or_else(|err| { - panic!( - "run cargo public-api --version: {err}; install cargo-public-api {CARGO_PUBLIC_API_VERSION}" - ) - }); - let stdout = String::from_utf8_lossy(&version.stdout); - let stderr = String::from_utf8_lossy(&version.stderr); - let version_text = stdout.trim(); - assert!( - version.status.success(), - "cargo public-api --version failed\nstdout:\n{stdout}\nstderr:\n{stderr}" - ); - assert!( - version_text == format!("cargo-public-api {CARGO_PUBLIC_API_VERSION}"), - "cargo-public-api version must be {CARGO_PUBLIC_API_VERSION}; found `{version_text}`" - ); - - let output = Command::new("cargo") - .args([ - "+nightly-2026-06-28", - "public-api", - "-p", - package, - "--all-features", - "--target", - "aarch64-apple-darwin", - ]) - .env("RUSTDOCFLAGS", "-D warnings --document-hidden-items") - .env("RUSTFLAGS", "") - .current_dir(root) - .output() - .unwrap_or_else(|err| panic!("run cargo public-api for {package}: {err}")); - let stdout = String::from_utf8_lossy(&output.stdout); - let stderr = String::from_utf8_lossy(&output.stderr); - assert!( - output.status.success(), - "cargo public-api failed for package {package}\nstdout:\n{stdout}\nstderr:\n{stderr}" - ); - format!("{stdout}{stderr}") -} - pub(crate) fn rust_sources(dir: &Path) -> Vec { let mut out = Vec::new(); collect_rust_sources(dir, &mut out); diff --git a/xtask/tests/repo_lint_support/native_decode_allocation_policy/move_only.rs b/xtask/tests/repo_lint_support/native_decode_allocation_policy/move_only.rs index e5e3c3cb..ed8d82c3 100644 --- a/xtask/tests/repo_lint_support/native_decode_allocation_policy/move_only.rs +++ b/xtask/tests/repo_lint_support/native_decode_allocation_policy/move_only.rs @@ -1,61 +1,10 @@ // SPDX-License-Identifier: MIT OR Apache-2.0 -//! Move-only native decode owners and allocation-free tile-part cursor ratchets. +//! Allocation-free tile-part cursor ratchets. use super::read; use crate::repo_lint_support::{assert_pattern_checks, PatternCheck}; -fn assert_does_not_derive_clone(source: &str, declaration: &str) { - let declaration_offset = source - .find(declaration) - .unwrap_or_else(|| panic!("missing owner declaration `{declaration}`")); - let prefix = &source[..declaration_offset]; - let derive_offset = prefix - .rfind("#[derive(") - .unwrap_or_else(|| panic!("missing derive for owner `{declaration}`")); - let derive = prefix[derive_offset..].lines().next().expect("derive line"); - assert!( - !derive.contains("Clone"), - "{declaration} must remain move-only; found `{derive}`" - ); -} - -#[test] -fn retained_native_decode_owner_graphs_remain_move_only() { - let roi = read("crates/j2k-native/src/j2c/roi.rs"); - assert_does_not_derive_clone(&roi, "pub(crate) struct RoiPlan"); - - let math = read("crates/j2k-native/src/math.rs"); - assert_does_not_derive_clone(&math, "pub(crate) struct SimdBuffer"); - - let component = read("crates/j2k-native/src/j2c/mod.rs"); - assert_does_not_derive_clone(&component, "pub(crate) struct ComponentData"); - - let tile = read("crates/j2k-native/src/j2c/tile.rs"); - for declaration in [ - "pub(crate) struct Tile<'a>", - "pub(crate) struct MergedTilePart<'a>", - "pub(crate) struct SeparatedTilePart<'a>", - "pub(crate) enum TilePart<'a>", - "struct PacketLengthMetadata", - ] { - assert_does_not_derive_clone(&tile, declaration); - } - - let model = read("crates/j2k-native/src/j2c/codestream/model.rs"); - for declaration in [ - "pub(crate) struct PpmMarkerData<'a>", - "pub(crate) struct PacketLengthMarker", - "pub(crate) struct ComponentInfo", - "pub(crate) struct QuantizationInfo", - "pub(crate) struct CodingStyleDefault", - "pub(crate) struct CodingStyleComponent", - "pub(crate) struct CodingStyleParameters", - ] { - assert_does_not_derive_clone(&model, declaration); - } -} - #[test] fn tile_part_decode_uses_a_borrowing_nonallocating_cursor() { let cursor = read("crates/j2k-native/src/j2c/tile/cursor.rs"); diff --git a/xtask/tests/repo_lint_support/native_direct_plan_structure_policy.rs b/xtask/tests/repo_lint_support/native_direct_plan_structure_policy.rs deleted file mode 100644 index 3204ad97..00000000 --- a/xtask/tests/repo_lint_support/native_direct_plan_structure_policy.rs +++ /dev/null @@ -1,99 +0,0 @@ -// SPDX-License-Identifier: MIT OR Apache-2.0 - -//! Responsibility ratchets for native direct-plan construction. - -use std::fs; - -use super::{assert_pattern_checks, repo_root, PatternCheck}; - -#[test] -fn native_direct_plan_storage_is_split_by_component_and_sub_band_ownership() { - let root = repo_root(); - let component = - fs::read_to_string(root.join("crates/j2k-native/src/j2c/decode/direct_plan/storage.rs")) - .expect("read native direct-plan component storage"); - let sub_band = fs::read_to_string( - root.join("crates/j2k-native/src/j2c/decode/direct_plan/storage/sub_band.rs"), - ) - .expect("read native direct-plan sub-band storage"); - - assert_pattern_checks(&[ - PatternCheck::new("native direct-plan component assembly", &component) - .required(&[ - "mod sub_band;", - "fn build_component_plan_from_storage(", - "fn append_idwt_steps(", - "fn direct_store_geometry(", - "J2kDirectStoreStep", - ]) - .forbidden(&[ - "fn build_grayscale_sub_band_step(", - "HtOwnedCodeBlockBatchJob", - "J2kOwnedCodeBlockBatchJob", - "fn encoded_input_range(", - "fn strip_grayscale_payload_owners(", - "fn strip_classic_payload_owners(", - "clippy::too_many_lines", - ]), - PatternCheck::new( - "native direct-plan sub-band payload construction", - &sub_band, - ) - .required(&[ - "fn build_grayscale_sub_band_step(", - "fn build_ht_sub_band_step(", - "fn build_classic_sub_band_step(", - "HtOwnedCodeBlockBatchJob", - "J2kOwnedCodeBlockBatchJob", - "fn encoded_input_range(", - "fn strip_grayscale_payload_owners(", - "fn strip_classic_payload_owners(", - ]) - .forbidden(&[ - "fn build_component_plan_from_storage(", - "fn append_idwt_steps(", - "fn direct_store_geometry(", - "J2kDirectIdwtStep", - "J2kDirectStoreStep", - "clippy::too_many_lines", - ]), - ]); - - assert!( - component.lines().count() < 350, - "native direct-plan component assembly must stay below its focused 350-line ratchet" - ); - assert!( - sub_band.lines().count() < 425, - "native direct-plan sub-band payload construction must stay below its focused 425-line ratchet" - ); -} - -#[test] -fn referenced_direct_plan_tests_keep_focused_owners() { - let root = repo_root(); - let direct_plan = fs::read_to_string(root.join("crates/j2k-native/src/direct_plan.rs")) - .expect("read j2k-native direct plan"); - let referenced = - fs::read_to_string(root.join("crates/j2k-native/src/direct_plan/referenced_ht.rs")) - .expect("read referenced HT direct plan"); - assert!(direct_plan.contains("mod referenced_ht;")); - assert!(direct_plan.contains("pub use referenced_ht::")); - assert!(!direct_plan.contains("pub enum J2kReferencedHtj2kPlan")); - assert!(referenced.contains("pub enum J2kReferencedHtj2kPlan")); - assert!(!referenced - .lines() - .any(|line| line.trim() == "use super::*;")); - assert!(direct_plan.lines().count() < 250); - - let tests = fs::read_to_string(root.join("crates/j2k-native/src/tests.rs")) - .expect("read native tests shell"); - let workspace = fs::read_to_string(root.join("crates/j2k-native/src/tests/workspace_reuse.rs")) - .expect("read native workspace-reuse tests"); - let symbol = "fn decoder_workspace_reuses_component_owners_across_distinct_input_lifetimes"; - assert!(tests.contains("mod workspace_reuse;")); - assert!(!tests.contains(symbol)); - assert!(workspace.contains(symbol)); - assert!(workspace.lines().count() < 300); - assert!(!workspace.lines().any(|line| line.trim() == "use super::*;")); -} diff --git a/xtask/tests/repo_lint_support/native_encode_allocation_policy.rs b/xtask/tests/repo_lint_support/native_encode_allocation_policy.rs index 78970b88..2e7acb03 100644 --- a/xtask/tests/repo_lint_support/native_encode_allocation_policy.rs +++ b/xtask/tests/repo_lint_support/native_encode_allocation_policy.rs @@ -24,86 +24,6 @@ fn production_source(relative: &str) -> String { .to_owned() } -fn assert_does_not_derive_clone(source: &str, declaration: &str) { - let declaration_offset = source - .find(declaration) - .unwrap_or_else(|| panic!("missing public owner declaration `{declaration}`")); - let prefix = &source[..declaration_offset]; - let derive_offset = prefix - .rfind("#[derive(") - .unwrap_or_else(|| panic!("missing derive for public owner `{declaration}`")); - let derive = prefix[derive_offset..].lines().next().expect("derive line"); - assert!( - !derive.contains("Clone"), - "{declaration} must remain move-only; found `{derive}`" - ); -} - -#[test] -fn large_encode_owner_graphs_remain_move_only() { - let types = read("crates/j2k-types/src/lib.rs"); - for declaration in [ - "pub struct EncodedJ2kCodeBlock", - "pub struct EncodedHtJ2kCodeBlock", - "pub struct J2kPacketizationSubband", - "pub struct J2kPacketizationResolution", - "pub struct J2kForwardDwt53Output", - "pub struct J2kForwardDwt53Level", - "pub struct J2kForwardDwt97Output", - "pub struct J2kForwardDwt97Level", - "pub struct PrecomputedHtj2k53Component", - "pub struct PrecomputedHtj2k53Image", - "pub struct PrecomputedHtj2k97Component", - "pub struct PrecomputedHtj2k97Image", - "pub struct PrequantizedHtj2k97Image", - "pub struct PrequantizedHtj2k97Component", - "pub struct PrequantizedHtj2k97Resolution", - "pub struct PrequantizedHtj2k97Subband", - "pub struct PrequantizedHtj2k97CodeBlock", - "pub struct PreencodedHtj2k97Image", - "pub struct PreencodedHtj2k97Component", - "pub struct PreencodedHtj2k97Resolution", - "pub struct PreencodedHtj2k97Subband", - "pub struct PreencodedHtj2k97CodeBlock", - "pub struct PreencodedHtj2k97CompactImage", - "pub struct PreencodedHtj2k97CompactComponent", - "pub struct PreencodedHtj2k97CompactResolution", - "pub struct PreencodedHtj2k97CompactSubband", - "pub struct PreencodedHtj2k97CompactCodeBlock", - ] { - assert_does_not_derive_clone(&types, declaration); - } - - let transcode_contracts = read("crates/j2k-transcode/src/accelerator_contracts.rs"); - for declaration in [ - "pub struct PreencodedHtj2k97CompactBatch", - "pub struct PreencodedHtj2k97CompactBatchGroups", - ] { - assert_does_not_derive_clone(&transcode_contracts, declaration); - } - - let transcode = read("crates/j2k-transcode/src/lib.rs"); - for declaration in [ - "pub struct Dwt53TwoDimensional", - "pub struct Dwt97TwoDimensional", - "pub struct ReversibleDwt53FirstLevel", - ] { - assert_does_not_derive_clone(&transcode, declaration); - } - - let transcode_outputs = read("crates/j2k-transcode/src/jpeg_to_htj2k/output.rs"); - assert_does_not_derive_clone(&transcode_outputs, "pub struct EncodedTranscode"); - - let transcode_test_support = read("crates/j2k-transcode-test-support/src/dct53_multilevel.rs"); - assert_does_not_derive_clone(&transcode_test_support, "pub struct Dwt53MultiLevel"); - - let native_params = read("crates/j2k-native/src/j2c/codestream_write.rs"); - assert_does_not_derive_clone(&native_params, "pub(crate) struct EncodeParams"); - - let native_roi = read("crates/j2k-native/src/j2c/encode/roi_plan.rs"); - assert_does_not_derive_clone(&native_roi, "pub(super) struct ComponentRoiEncodePlan"); -} - #[test] fn native_encode_errors_preserve_resource_and_stage_categories() { let error = read("crates/j2k-native/src/error.rs"); diff --git a/xtask/tests/repo_lint_support/phase_order_policy.rs b/xtask/tests/repo_lint_support/phase_order_policy.rs new file mode 100644 index 00000000..9936c5f7 --- /dev/null +++ b/xtask/tests/repo_lint_support/phase_order_policy.rs @@ -0,0 +1,142 @@ +// SPDX-License-Identifier: MIT OR Apache-2.0 + +//! Safety- and budget-critical phase ordering checked against the Rust AST. + +use std::fs; + +use super::{repo_root, rust_function_policy::FunctionCalls}; + +fn read(relative: &str) -> String { + fs::read_to_string(repo_root().join(relative)) + .unwrap_or_else(|error| panic!("read {relative}: {error}")) +} + +#[test] +fn native_decode_preflights_and_propagates_before_roi_or_allocation() { + let decode = read("crates/j2k-native/src/j2c/decode.rs"); + let decode_tile = FunctionCalls::parse("native tile decode", &decode, "decode_tile"); + decode_tile.assert_ordered( + "native full-tile preflight before ROI planning", + &[ + "build::build", + "RoiPlan::build", + "segment::parse", + "decode_component_tile_bit_planes_budgeted", + ], + ); + decode_tile.assert_propagated( + "native tile workspace failures", + &[ + "build::build", + "segment::parse", + "decode_component_tile_bit_planes_budgeted", + ], + ); + + let allocation = read("crates/j2k-native/src/j2c/build/allocation.rs"); + FunctionCalls::parse( + "native decomposition allocation", + &allocation, + "prepare_decomposition_storage", + ) + .assert_ordered( + "plan, stale-capacity normalization, and actual-capacity validation", + &[ + "plan::build_allocation_plan", + "reuse::discard_stale_capacity", + "account_live_workspace", + "reuse::reserve_decomposition_storage", + "account_live_workspace", + ], + ); +} + +#[test] +fn native_context_and_tile_owner_handoffs_remain_transactional() { + let decode = read("crates/j2k-native/src/j2c/decode.rs"); + FunctionCalls::parse("native decode", &decode, "decode").assert_ordered( + "retained component accounting before parse and reset", + &["prepare_reused_decode_baseline", "tile::parse", "reset"], + ); + + let tile = read("crates/j2k-native/src/j2c/tile.rs"); + FunctionCalls::parse("native tile parser", &tile, "parse").assert_ordered( + "native tile construction and owner handoff", + &[ + "TileMetadataBudget::for_image", + "try_reserve_retained", + "inherit_tile_metadata", + "parse_tile_part", + "validate_owner_graph", + "ParsedTiles::new", + ], + ); +} + +#[test] +fn cuda_status_and_packet_owners_exist_before_launch_or_completion() { + let decode = read("crates/j2k-cuda-runtime/src/htj2k_decode/completion.rs"); + FunctionCalls::parse( + "CUDA cleanup completion", + &decode, + "run_htj2k_cleanup_multi_kernel", + ) + .assert_ordered( + "CUDA cleanup status allocation before launch", + &["try_vec_filled", "launch_htj2k_decode_codeblocks_multi"], + ); + + let encode = read("crates/j2k-cuda-runtime/src/htj2k_encode/completion.rs"); + for function in [ + "encode_htj2k_kernel_jobs_device_with_resources_and_pool", + "encode_htj2k_multi_input_kernel_jobs_device_compact_with_resources_and_pool", + ] { + FunctionCalls::parse("CUDA HTJ2K encode completion", &encode, function).assert_ordered( + "CUDA encode status allocation before launch", + &[ + "HostPhaseBudget::with_live_bytes", + "try_vec_filled", + "time_default_stream_named_us", + ], + ); + } + + let packetize = read("crates/j2k-cuda-runtime/src/htj2k_packetize.rs"); + FunctionCalls::parse( + "CUDA packetization completion", + &packetize, + "packetize_htj2k_cleanup_packets_with_tag_state_and_live_host_bytes", + ) + .assert_ordered( + "CUDA packetization launch and output ownership", + &[ + "HostPhaseBudget::with_live_bytes", + "htj2k_packetization_kernel_packets", + "try_vec_filled", + "drop", + "time_default_stream_named_us", + "complete_htj2k_packetization", + ], + ); +} + +#[test] +fn cuda_grayscale_batch_preserves_prepare_execute_complete_order() { + let execution = read("crates/j2k-cuda/src/decoder/grayscale_batch/execution.rs"); + FunctionCalls::parse( + "CUDA grayscale batch orchestration", + &execution, + "decode_grayscale_cuda_batch_with_profile", + ) + .assert_ordered( + "CUDA grayscale batch phases", + &[ + "prepare_grayscale_batch", + "upload_grayscale_decode_resources", + "build_grayscale_component_work", + "enqueue_grayscale_entropy", + "enqueue_grayscale_idwt", + "finish_grayscale_components_and_store", + ], + ); +} diff --git a/xtask/tests/repo_lint_support/release_policy.rs b/xtask/tests/repo_lint_support/release_policy.rs index 9ef0f47f..ed242f4a 100644 --- a/xtask/tests/repo_lint_support/release_policy.rs +++ b/xtask/tests/repo_lint_support/release_policy.rs @@ -1,13 +1,11 @@ // SPDX-License-Identifier: MIT OR Apache-2.0 -use std::fs; - -use serde_json::Value; +use std::{collections::BTreeSet, fs}; use super::{ assert_contains_all, assert_file_pattern_checks, assert_pattern_checks, - cargo_metadata_workspace_edges, const_array_block, repo_root, xtask_sources, FilePatternCheck, - PatternCheck, + cargo_metadata_workspace_edges, release_manifest_entries, repo_root, xtask_sources, + FilePatternCheck, PatternCheck, }; #[path = "../../src/release_commands/release_integrity_policy/markdown.rs"] @@ -20,19 +18,13 @@ fn j2k_ml_is_a_publishable_release_crate() { let root = repo_root(); let manifest = fs::read_to_string(root.join("crates/j2k-ml/Cargo.toml")).expect("read j2k-ml manifest"); - let release_manifest = fs::read_to_string(root.join("release-crates.json")) - .expect("read ordered release manifest"); - let release_manifest: Value = - serde_json::from_str(&release_manifest).expect("parse ordered release manifest"); - let ordered_crates = release_manifest["ordered_crates"] - .as_array() - .expect("ordered_crates array") - .iter() - .map(|value| value.as_str().expect("crate name")) + let ordered_crates = release_manifest_entries(root) + .into_iter() + .map(|entry| entry.name) .collect::>(); let j2k_ml_index = ordered_crates .iter() - .position(|name| *name == "j2k-ml") + .position(|name| name == "j2k-ml") .expect("j2k-ml must be in the release manifest"); assert!( @@ -54,7 +46,7 @@ fn j2k_ml_is_a_publishable_release_crate() { for dependency in ["j2k", "j2k-cuda-runtime", "j2k-cuda", "j2k-metal"] { let dependency_index = ordered_crates .iter() - .position(|name| *name == dependency) + .position(|name| name == dependency) .unwrap_or_else(|| panic!("{dependency} must be in the release manifest")); assert!( dependency_index < j2k_ml_index, @@ -68,8 +60,6 @@ fn crates_io_publish_policy_is_explicit() { let root = repo_root(); let workspace = fs::read_to_string(root.join("Cargo.toml")).expect("read workspace manifest"); let changelog = fs::read_to_string(root.join("CHANGELOG.md")).expect("read changelog"); - let xtask = xtask_sources(root); - let publishable = const_array_block(&xtask, "PUBLISHABLE_PACKAGES"); let publish_workflow = fs::read_to_string(root.join(".github/workflows/publish.yml")) .expect("read publish workflow"); let release_manifest = fs::read_to_string(root.join("release-crates.json")) @@ -95,23 +85,29 @@ fn crates_io_publish_policy_is_explicit() { ); assert_pattern_checks(&[ - PatternCheck::new("xtask publishable package gate", publishable) - .required(&[ - "\"j2k-core\"", - "\"j2k-cuda-runtime\"", - "\"j2k-profile\"", - "\"j2k-native\"", - "\"j2k-tilecodec\"", - "\"j2k-jpeg\"", - "\"j2k\"", - "\"j2k-jpeg-metal\"", - "\"j2k-jpeg-cuda\"", - "\"j2k-metal\"", - "\"j2k-cuda\"", - "\"j2k-ml\"", - "\"j2k-cli\"", - ]) - .forbidden(&["\"j2k-compare\""]), + PatternCheck::new( + "release manifest package and API contracts", + &release_manifest, + ) + .required(&[ + "\"schema\": 2", + "\"crates\"", + "\"api_contract\"", + "\"j2k-core\"", + "\"j2k-cuda-runtime\"", + "\"j2k-profile\"", + "\"j2k-native\"", + "\"j2k-tilecodec\"", + "\"j2k-jpeg\"", + "\"j2k\"", + "\"j2k-jpeg-metal\"", + "\"j2k-jpeg-cuda\"", + "\"j2k-metal\"", + "\"j2k-cuda\"", + "\"j2k-ml\"", + "\"j2k-cli\"", + ]) + .forbidden(&["\"j2k-compare\""]), PatternCheck::new("single-runner publish workflow", &publish_workflow) .required(&[ "preflight:", @@ -123,7 +119,7 @@ fn crates_io_publish_policy_is_explicit() { ]) .forbidden(&["publish-j2k-core:", "cargo publish --workspace", "sleep "]), PatternCheck::new("ordered release manifest", &release_manifest) - .required(&["\"ordered_crates\"", "\"j2k-core\"", "\"j2k-cli\""]) + .required(&["\"crates\"", "\"j2k-core\"", "\"j2k-cli\""]) .forbidden(&["j2k-compare"]), PatternCheck::new("checksum-aware release publisher", &publisher).required(&[ "hashlib.sha256", @@ -136,7 +132,7 @@ fn crates_io_publish_policy_is_explicit() { } #[test] -fn release_docs_use_manifest_versions_for_publish_order() { +fn release_docs_use_schema_two_manifest_and_workspace_versions() { let xtask = xtask_sources(repo_root()); assert_file_pattern_checks( @@ -144,7 +140,8 @@ fn release_docs_use_manifest_versions_for_publish_order() { &[FilePatternCheck::new("docs/release.md") .named("release docs") .required(&[ - "manifest versions", + "Schema 2 records only ordered crate names", + "workspace.package.version", "cargo metadata --locked --no-deps", "block v0.1.6", "metal v0.33.0", @@ -342,7 +339,6 @@ fn j2k_compare_stays_unpublished_and_out_of_j2k_package_deps() { #[test] fn package_preflight_is_staged_dependency_aware() { let root = repo_root(); - let xtask = xtask_sources(root); assert_package_gate_file_contracts(root); let package_gate = fs::read_to_string(root.join("xtask/src/release_commands/package_gate.rs")) .expect("read package-gate module"); @@ -350,30 +346,40 @@ fn package_preflight_is_staged_dependency_aware() { package_gate.lines().count() <= 175, "package-gate module must stay within its focused 175-line ownership ceiling" ); - let publishable_packages = - const_array_entries(const_array_block(&xtask, "PUBLISHABLE_PACKAGES")); - let strict_packages = const_array_block(&xtask, "REGISTRY_INDEPENDENT_PACKAGES"); - let staged_packages = const_array_block(&xtask, "STAGED_DEPENDENCY_PACKAGES"); + let publishable_packages = release_manifest_entries(root) + .into_iter() + .map(|entry| entry.name) + .collect::>(); + let workspace_edges = cargo_metadata_workspace_edges(root); + let staged_packages = workspace_edges + .iter() + .map(|(package, _)| package.clone()) + .collect::>(); + let strict_packages = publishable_packages + .iter() + .filter(|package| !staged_packages.contains(*package)) + .cloned() + .collect::>(); for package in &publishable_packages { - let strict = strict_packages.contains(&format!("\"{package}\"")); - let staged = staged_packages.contains(&format!("\"{package}\"")); + let strict = strict_packages.contains(package); + let staged = staged_packages.contains(package); assert_ne!( strict, staged, "publishable package `{package}` must appear in exactly one package-gate partition" ); } - assert_package_gate_dependency_order(root, &publishable_packages, strict_packages); + assert_package_gate_dependency_order(&publishable_packages, &strict_packages, &workspace_edges); assert!( - staged_packages.contains("\"j2k-cuda-runtime\""), + staged_packages.contains("j2k-cuda-runtime"), "j2k-cuda-runtime depends on staged j2k-core and must not run strict package verification before publication" ); let codec_math = publishable_packages .iter() - .position(|package| *package == "j2k-codec-math") + .position(|package| package == "j2k-codec-math") .expect("j2k-codec-math publish position"); let cuda_runtime = publishable_packages .iter() - .position(|package| *package == "j2k-cuda-runtime") + .position(|package| package == "j2k-cuda-runtime") .expect("j2k-cuda-runtime publish position"); assert!( codec_math < cuda_runtime, @@ -388,17 +394,18 @@ fn assert_package_gate_file_contracts(root: &std::path::Path) { FilePatternCheck::new("xtask/src/release_commands.rs") .named("xtask package preflight") .required(&[ - "PUBLISHABLE_PACKAGES", - "REGISTRY_INDEPENDENT_PACKAGES", - "STAGED_DEPENDENCY_PACKAGES", "mod package_gate;", - "package_gate::run(&metadata)", + "release_manifest_contract()?", + "package_gate::run(&metadata, &release_manifest)", "\"--list\"", ]), FilePatternCheck::new("xtask/src/release_commands/package_gate.rs") .named("dependency-aware package construction") .required(&[ - "package_gate_plan(metadata)?", + "package_gate_plan(metadata, manifest)?", + "validate_release_manifest_contract(manifest, &packages, &mut validation_errors)?", + "release_dependencies_by_package(manifest, &packages)?", + "registry_independent_packages(manifest, &packages)?", "dependency_closure", "processed.contains(dependency_name.as_str())", "--config", @@ -446,6 +453,19 @@ fn assert_package_gate_file_contracts(root: &std::path::Path) { "cargo\", \"publish\", \"--locked", ]) .forbidden(&["cargo publish --workspace"]), + FilePatternCheck::new("scripts/release_manifest.py") + .named("strict release manifest and graph validator") + .required(&[ + "object_pairs_hook=_reject_duplicate_fields", + "if type(schema) is not int or schema != 2", + "\"workspace_members\"", + "return \"crates-io\" in registries", + "must be workspace/path sourced", + "expected_requirement = f\"={versions_by_package[dependency_name]}\"", + "if kind == \"dev\"", + "if positions[dependency_name] >= positions[crate]", + "registry_independent", + ]), FilePatternCheck::new("scripts/crates_io_version.py") .named("fail-closed crates.io version helper") .required(&[ @@ -470,14 +490,13 @@ fn assert_package_gate_file_contracts(root: &std::path::Path) { } fn assert_package_gate_dependency_order( - root: &std::path::Path, - publishable_packages: &[&str], - strict_packages: &str, + publishable_packages: &[String], + strict_packages: &BTreeSet, + workspace_edges: &BTreeSet<(String, String)>, ) { - let workspace_edges = cargo_metadata_workspace_edges(root); - for (package, dependency) in &workspace_edges { + for (package, dependency) in workspace_edges { assert!( - !strict_packages.contains(&format!("\"{package}\"")), + !strict_packages.contains(package), "strict package preflight must not include `{package}` while it depends on staged workspace crate `{dependency}`" ); let package_position = publishable_packages @@ -501,44 +520,46 @@ fn assert_package_gate_dependency_order( ); } -fn const_array_entries(block: &str) -> Vec<&str> { - block - .lines() - .map(|line| line.trim().trim_matches([',', '"'])) - .filter(|entry| { - !entry.is_empty() - && !entry.starts_with("const ") - && !entry.starts_with(']') - && !entry.starts_with('&') - }) - .collect() -} - #[test] -fn release_manifest_covers_all_publishable_crates() { +fn release_manifest_is_schema_two_and_drives_the_legacy_preflight() { let root = repo_root(); - let xtask = xtask_sources(root); let release_manifest = fs::read_to_string(root.join("release-crates.json")).expect("read release manifest"); let publish_script = fs::read_to_string(root.join("scripts/publish-crate.sh")).expect("read publish script"); - let publishable = const_array_block(&xtask, "PUBLISHABLE_PACKAGES"); - let publishable_packages: Vec<&str> = publishable - .lines() - .map(|line| line.trim().trim_matches([',', '"'])) - .filter(|package| { - !package.is_empty() - && !package.starts_with("const ") - && !package.starts_with(']') - && !package.starts_with('&') - }) - .collect(); + let entries = release_manifest_entries(root); + let publishable_packages = entries + .iter() + .map(|entry| entry.name.as_str()) + .collect::>(); + assert_eq!(entries.len(), 19); + assert_eq!( + entries + .iter() + .map(|entry| entry.name.as_str()) + .collect::>() + .len(), + entries.len(), + "release manifest crate names must be unique" + ); assert_contains_all( "release manifest publishable package coverage", &release_manifest, &publishable_packages, ); + assert_contains_all( + "release manifest schema and tiers", + &release_manifest, + &[ + "\"schema\": 2", + "\"crates\"", + "\"api_contract\": \"stable\"", + "\"api_contract\": \"experimental\"", + "\"api_contract\": \"implementation\"", + "\"api_contract\": \"binary\"", + ], + ); assert_contains_all( "legacy preflight consumes release manifest", &publish_script, diff --git a/xtask/tests/repo_lint_support/suppression_policy/manifest.rs b/xtask/tests/repo_lint_support/suppression_policy/manifest.rs index 6ac820e3..f7c38e53 100644 --- a/xtask/tests/repo_lint_support/suppression_policy/manifest.rs +++ b/xtask/tests/repo_lint_support/suppression_policy/manifest.rs @@ -8,6 +8,7 @@ use crate::repo_lint_support::{repo_root, repo_text_files}; // Safety suppressions are an exact inventory. Removing one from a manifest must // remove it here too, so stale policy entries cannot silently re-authorize it. const REVIEWED_SAFETY_ALLOWS: &[(&str, &str, &str)] = &[ + ("crates/j2k-alloc-probe/Cargo.toml", "rust", "unsafe_code"), ("crates/j2k-core/Cargo.toml", "rust", "unsafe_code"), ("crates/j2k-cuda-runtime/Cargo.toml", "rust", "unsafe_code"), ("crates/j2k-cuda/Cargo.toml", "rust", "unsafe_code"), diff --git a/xtask/tests/repo_lint_support/transcode_structure_policy.rs b/xtask/tests/repo_lint_support/transcode_structure_policy.rs deleted file mode 100644 index 7a137ff2..00000000 --- a/xtask/tests/repo_lint_support/transcode_structure_policy.rs +++ /dev/null @@ -1,270 +0,0 @@ -// SPDX-License-Identifier: MIT OR Apache-2.0 - -use std::fs; - -use super::{assert_pattern_checks, repo_root, PatternCheck}; - -fn read(relative_path: &str) -> String { - fs::read_to_string(repo_root().join(relative_path)) - .unwrap_or_else(|error| panic!("read {relative_path}: {error}")) -} - -fn assert_line_budget(relative_path: &str, source: &str, max_lines: usize) { - let line_count = source.lines().count(); - assert!( - line_count < max_lines, - "{relative_path} has {line_count} lines; expected fewer than {max_lines}" - ); -} - -mod cpu; -mod metal; - -#[test] -#[expect( - clippy::too_many_lines, - reason = "CUDA and Metal residency-stage ownership is one cross-backend structural policy" -)] -fn cuda_and_metal_transcode_backends_stay_split_by_residency_stage() { - let cuda = read("crates/j2k-transcode-cuda/src/cuda.rs"); - let cuda_transform = read("crates/j2k-transcode-cuda/src/cuda/transform.rs"); - let cuda_transform_components = - read("crates/j2k-transcode-cuda/src/cuda/transform/components.rs"); - let cuda_transform_staging = read("crates/j2k-transcode-cuda/src/cuda/transform/staging.rs"); - let cuda_resident_dispatch = read("crates/j2k-transcode-cuda/src/cuda/resident_dispatch.rs"); - let cuda_resident_dispatch_grouped = - read("crates/j2k-transcode-cuda/src/cuda/resident_dispatch/grouped.rs"); - let cuda_resident_encode = read("crates/j2k-transcode-cuda/src/cuda/resident_encode.rs"); - let cuda_resident_encode_orchestration = - read("crates/j2k-transcode-cuda/src/cuda/resident_encode/orchestration.rs"); - let cuda_resident_encode_output = - read("crates/j2k-transcode-cuda/src/cuda/resident_encode/output.rs"); - let cuda_resident_encode_planning = - read("crates/j2k-transcode-cuda/src/cuda/resident_encode/planning.rs"); - let metal = read("crates/j2k-transcode-metal/src/metal.rs"); - let metal_runtime = read("crates/j2k-transcode-metal/src/metal/runtime.rs"); - let metal_reversible = read("crates/j2k-transcode-metal/src/metal/reversible.rs"); - let metal_irreversible = read("crates/j2k-transcode-metal/src/metal/irreversible.rs"); - let metal_projection = read("crates/j2k-transcode-metal/src/metal/projection.rs"); - let metal_resident = read("crates/j2k-transcode-metal/src/metal/resident.rs"); - let metal_geometry = read("crates/j2k-transcode-metal/src/metal/geometry.rs"); - let metal_buffers = read("crates/j2k-transcode-metal/src/metal/buffers.rs"); - - for (path, source, max_lines) in [ - ("j2k-transcode-cuda/src/cuda.rs", cuda.as_str(), 250), - ( - "j2k-transcode-cuda/src/cuda/transform.rs", - cuda_transform.as_str(), - 625, - ), - ( - "j2k-transcode-cuda/src/cuda/transform/components.rs", - cuda_transform_components.as_str(), - 425, - ), - ( - "j2k-transcode-cuda/src/cuda/transform/staging.rs", - cuda_transform_staging.as_str(), - 425, - ), - ( - "j2k-transcode-cuda/src/cuda/resident_dispatch.rs", - cuda_resident_dispatch.as_str(), - 425, - ), - ( - "j2k-transcode-cuda/src/cuda/resident_dispatch/grouped.rs", - cuda_resident_dispatch_grouped.as_str(), - 425, - ), - ( - "j2k-transcode-cuda/src/cuda/resident_encode.rs", - cuda_resident_encode.as_str(), - 125, - ), - ( - "j2k-transcode-cuda/src/cuda/resident_encode/orchestration.rs", - cuda_resident_encode_orchestration.as_str(), - 425, - ), - ( - "j2k-transcode-cuda/src/cuda/resident_encode/output.rs", - cuda_resident_encode_output.as_str(), - 425, - ), - ( - "j2k-transcode-cuda/src/cuda/resident_encode/planning.rs", - cuda_resident_encode_planning.as_str(), - 425, - ), - ("j2k-transcode-metal/src/metal.rs", metal.as_str(), 125), - ( - "j2k-transcode-metal/src/metal/runtime.rs", - metal_runtime.as_str(), - 275, - ), - ( - "j2k-transcode-metal/src/metal/reversible.rs", - metal_reversible.as_str(), - 300, - ), - ( - "j2k-transcode-metal/src/metal/irreversible.rs", - metal_irreversible.as_str(), - 700, - ), - ( - "j2k-transcode-metal/src/metal/projection.rs", - metal_projection.as_str(), - 300, - ), - ( - "j2k-transcode-metal/src/metal/resident.rs", - metal_resident.as_str(), - 825, - ), - ( - "j2k-transcode-metal/src/metal/geometry.rs", - metal_geometry.as_str(), - 425, - ), - ( - "j2k-transcode-metal/src/metal/buffers.rs", - metal_buffers.as_str(), - 225, - ), - ] { - assert_line_budget(path, source, max_lines); - } - - assert_pattern_checks(&[ - PatternCheck::new("CUDA transcode facade", &cuda) - .required(&[ - "mod transform;", - "mod resident_dispatch;", - "mod resident_encode;", - ]) - .forbidden(&["fn run_dwt97(", "fn encode_resident_subbands("]), - PatternCheck::new("CUDA transform dispatch ownership", &cuda_transform).required(&[ - "mod components;", - "mod staging;", - "fn dispatch_reversible_dwt53_batch(", - "fn dispatch_dwt97_batch(", - "fn dispatch_htj2k97_preencoded_batch(", - ]), - PatternCheck::new( - "CUDA transform component ownership", - &cuda_transform_components, - ) - .required(&[ - "fn preflight_component_allocation_budget(", - "fn codeblock_bands_to_components(", - "fn component_from_subbands(", - "fn account_codeblock_bands(", - ]), - PatternCheck::new("CUDA transform staging ownership", &cuda_transform_staging).required(&[ - "fn validate_staging_and_readback_workspace(", - "fn preflight_dwt97_conversion_budget(", - "fn flatten_f64_blocks_to_f32(", - ]), - PatternCheck::new("CUDA resident dispatch ownership", &cuda_resident_dispatch).required(&[ - "mod grouped;", - "fn dispatch_htj2k97_preencoded_i16_batch_with_sink", - "fn device_bands_to_preencoded_components", - ]), - PatternCheck::new( - "CUDA grouped resident dispatch ownership", - &cuda_resident_dispatch_grouped, - ) - .required(&[ - "fn live_staging_budget<", - "fn dispatch_with_sink<", - "fn dispatch_htj2k97_compact_preencoded_i16_batch_groups", - ]), - PatternCheck::new("CUDA resident encode facade", &cuda_resident_encode) - .required(&["mod orchestration;", "mod output;", "mod planning;"]) - .forbidden(&[ - "fn encode_resident_subbands(", - "fn split_resident_subband_blocks(", - ]), - PatternCheck::new( - "CUDA resident encode orchestration ownership", - &cuda_resident_encode_orchestration, - ) - .required(&[ - "fn encode_resident_subbands(", - "fn encode_resident_compact_subbands(", - "fn device_band_groups_to_preencoded_components", - ]), - PatternCheck::new( - "CUDA resident output ownership", - &cuda_resident_encode_output, - ) - .required(&[ - "fn assemble_compact_preencoded_components", - "fn split_resident_subband_blocks(", - "fn split_resident_compact_subband_blocks(", - ]), - PatternCheck::new( - "CUDA resident planning ownership", - &cuda_resident_encode_planning, - ) - .required(&[ - "type ResidentMetadataBudget = HostPhaseBudget", - "fn reserve_component_assembly_budget", - "fn resident_group_targets", - "fn resident_subband_encode_plan", - ]), - PatternCheck::new("Metal transcode facade", &metal) - .required(&[ - "mod runtime;", - "mod reversible;", - "mod irreversible;", - "mod projection;", - "mod resident;", - "mod codeblock_output;", - "mod geometry;", - "mod buffers;", - ]) - .forbidden(&[ - "fn shader_source(", - "fn dispatch_dct_grid_to_reversible_dwt53(", - "fn dispatch_dct_grid_to_dwt97(", - ]), - PatternCheck::new("Metal runtime ownership", &metal_runtime).required(&[ - "fn shader_source(", - "struct MetalRuntime", - "pub struct MetalTranscodeSession", - ]), - PatternCheck::new("Metal reversible ownership", &metal_reversible).required(&[ - "fn dispatch_dct_grid_to_reversible_dwt53(", - "fn dispatch_dct_grid_to_reversible_dwt53_batch(", - "batch::dispatch_with_runtime(", - ]), - PatternCheck::new("Metal irreversible ownership", &metal_irreversible).required(&[ - "fn dispatch_dct_grid_to_dwt97(", - "fn encode_dwt97_quantize_codeblocks(", - "fn dispatch_dct_grid_to_dwt97_batch_staged_with_runtime(", - ]), - PatternCheck::new("Metal projection ownership", &metal_projection).required(&[ - "fn dispatch_projection_threads(", - "struct ProjectionBatchJob", - "fn dispatch_projected_bands_batch_with_runtime(", - ]), - PatternCheck::new("Metal resident handoff ownership", &metal_resident).required(&[ - "fn validate_resident_dct_handoffs(", - "fn validate_resident_dwt_handoffs(", - "fn projection_batch_output_buffers(", - ]), - PatternCheck::new("Metal geometry ownership", &metal_geometry).required(&[ - "struct BandGeometry", - "fn validate_reversible_batch_geometry(", - "fn metal_sparse_rows(", - ]), - PatternCheck::new("Metal buffer ownership", &metal_buffers).required(&[ - "fn dwt97_batch_blocks_buffer(", - "fn read_f32_buffer(", - "fn idct8_basis_table(", - ]), - ]); -} diff --git a/xtask/tests/repo_lint_support/transcode_structure_policy/cpu.rs b/xtask/tests/repo_lint_support/transcode_structure_policy/cpu.rs deleted file mode 100644 index bc1dcec4..00000000 --- a/xtask/tests/repo_lint_support/transcode_structure_policy/cpu.rs +++ /dev/null @@ -1,156 +0,0 @@ -// SPDX-License-Identifier: MIT OR Apache-2.0 - -use super::super::{assert_pattern_checks, PatternCheck}; -use super::{assert_line_budget, read}; - -mod batch; - -#[test] -#[expect( - clippy::too_many_lines, - reason = "core and batch transcode ownership is enforced as one cohesive stage matrix" -)] -fn jpeg_to_htj2k_core_and_batch_stay_split_by_stage() { - let core = read("crates/j2k-transcode/src/jpeg_to_htj2k.rs"); - let validation = read("crates/j2k-transcode/src/jpeg_to_htj2k/validation.rs"); - let scratch = read("crates/j2k-transcode/src/jpeg_to_htj2k/scratch.rs"); - let output = read("crates/j2k-transcode/src/jpeg_to_htj2k/output.rs"); - let component_plan = read("crates/j2k-transcode/src/jpeg_to_htj2k/component_plan.rs"); - let component_groups = read("crates/j2k-transcode/src/jpeg_to_htj2k/component_groups.rs"); - let float_reference = read("crates/j2k-transcode/src/jpeg_to_htj2k/float_reference.rs"); - let float_output = read("crates/j2k-transcode/src/jpeg_to_htj2k/float_output.rs"); - let integer_reference = read("crates/j2k-transcode/src/jpeg_to_htj2k/integer_reference.rs"); - let integer_storage = read("crates/j2k-transcode/src/jpeg_to_htj2k/integer_storage.rs"); - let single = read("crates/j2k-transcode/src/jpeg_to_htj2k/single.rs"); - let single_tile_encode = read("crates/j2k-transcode/src/jpeg_to_htj2k/single_tile_encode.rs"); - for (path, source, max_lines) in [ - ("jpeg_to_htj2k.rs", core.as_str(), 400), - ("jpeg_to_htj2k/validation.rs", validation.as_str(), 175), - ("jpeg_to_htj2k/scratch.rs", scratch.as_str(), 75), - ("jpeg_to_htj2k/output.rs", output.as_str(), 75), - ( - "jpeg_to_htj2k/component_plan.rs", - component_plan.as_str(), - 500, - ), - ( - "jpeg_to_htj2k/component_groups.rs", - component_groups.as_str(), - 75, - ), - ( - "jpeg_to_htj2k/float_reference.rs", - float_reference.as_str(), - 450, - ), - ("jpeg_to_htj2k/float_output.rs", float_output.as_str(), 250), - ( - "jpeg_to_htj2k/integer_reference.rs", - integer_reference.as_str(), - 475, - ), - ( - "jpeg_to_htj2k/integer_storage.rs", - integer_storage.as_str(), - 100, - ), - ("jpeg_to_htj2k/single.rs", single.as_str(), 325), - ( - "jpeg_to_htj2k/single_tile_encode.rs", - single_tile_encode.as_str(), - 100, - ), - ] { - assert_line_budget(path, source, max_lines); - } - - assert_pattern_checks(&[ - PatternCheck::new("JPEG-to-HTJ2K facade", &core) - .required(&[ - "mod validation;", - "mod component_plan;", - "mod component_groups;", - "mod float_reference;", - "mod float_output;", - "mod integer_reference;", - "mod integer_storage;", - "mod single;", - "mod single_tile_encode;", - ]) - .forbidden(&[ - "fn transcode_component_batch(", - "fn same_geometry_component_groups(", - "fn integer_direct_wavelet_from_component(", - "fn float_direct_wavelet_from_component(", - ]), - PatternCheck::new("JPEG-to-HTJ2K validation ownership", &validation).required(&[ - "fn validate_transcode_options(", - "fn validate_component_block_grid(", - "fn decomposition_levels_for_components(", - ]), - PatternCheck::new( - "JPEG-to-HTJ2K component planning ownership", - &component_plan, - ) - .required(&[ - "fn transcode_component_batch(", - "struct ComponentTranscodePlan", - "fn component_to_precomputed_htj2k(", - ]) - .forbidden(&["fn same_geometry_component_groups("]), - PatternCheck::new( - "JPEG-to-HTJ2K component grouping ownership", - &component_groups, - ) - .required(&[ - "fn same_geometry_component_groups(", - "fn same_component_geometry(", - "try_vec_filled(", - "try_vec_with_capacity(", - "try_vec_reserve_len(", - ]), - PatternCheck::new("JPEG-to-HTJ2K float reference ownership", &float_reference).required(&[ - "struct ComponentWavelet97", - "fn float_direct_97_wavelet_from_component(", - "fn float97_reference_coefficients(", - ]), - PatternCheck::new("JPEG-to-HTJ2K float output ownership", &float_output).required(&[ - "fn j2k_dwt_from_wavelet(", - "fn rounded_wavelet97_i32(", - "fn wavelet_coefficient_count(", - ]), - PatternCheck::new( - "JPEG-to-HTJ2K integer reference ownership", - &integer_reference, - ) - .required(&[ - "struct IntegerWavelet", - "fn integer_direct_wavelet_from_component(", - "fn integer_reference_coefficients(", - ]), - PatternCheck::new("JPEG-to-HTJ2K integer storage ownership", &integer_storage).required(&[ - "fn idct_component_samples_i32(", - "fn checked_product(", - "fn validate_band_len(", - ]), - PatternCheck::new("JPEG-to-HTJ2K single orchestration ownership", &single) - .required(&[ - "fn prepare_single_transcode(", - "fn jpeg_to_htj2k_with_scratch<", - "fn finish_single_transcode(", - "struct PreparedSingleTranscode", - "struct CompletedSingleTranscode", - ]) - .forbidden(&["include!(", "use super::*;"]), - PatternCheck::new( - "JPEG-to-HTJ2K single-tile encode ownership", - &single_tile_encode, - ) - .required(&[ - "fn encode_component_batch", - "record_encode_dispatch_delta(", - "timings.htj2k_encode_us = encode_us;", - ]), - ]); - batch::assert_batch_structure(); -} diff --git a/xtask/tests/repo_lint_support/transcode_structure_policy/cpu/batch.rs b/xtask/tests/repo_lint_support/transcode_structure_policy/cpu/batch.rs deleted file mode 100644 index a32ba099..00000000 --- a/xtask/tests/repo_lint_support/transcode_structure_policy/cpu/batch.rs +++ /dev/null @@ -1,205 +0,0 @@ -// SPDX-License-Identifier: MIT OR Apache-2.0 - -use super::super::{assert_line_budget, read}; -use super::{assert_pattern_checks, PatternCheck}; - -struct BatchSources { - batch: String, - result_slots: String, - result_slot_tests: String, - actual_live: String, - individual: String, - prepare: String, - group_budget: String, - transform: String, - accelerated_storage: String, - storage: String, - encode: String, - float97_input: String, - precomputed: String, - encode_live: String, -} - -impl BatchSources { - fn read() -> Self { - Self { - batch: read("crates/j2k-transcode/src/jpeg_to_htj2k/batch.rs"), - result_slots: read("crates/j2k-transcode/src/jpeg_to_htj2k/batch/result_slots.rs"), - result_slot_tests: read( - "crates/j2k-transcode/src/jpeg_to_htj2k/batch/result_slots/tests.rs", - ), - actual_live: read("crates/j2k-transcode/src/jpeg_to_htj2k/batch/actual_live.rs"), - individual: read("crates/j2k-transcode/src/jpeg_to_htj2k/batch/individual.rs"), - prepare: read("crates/j2k-transcode/src/jpeg_to_htj2k/batch/prepare.rs"), - group_budget: read("crates/j2k-transcode/src/jpeg_to_htj2k/batch/group_budget.rs"), - transform: read("crates/j2k-transcode/src/jpeg_to_htj2k/batch/transform.rs"), - accelerated_storage: read( - "crates/j2k-transcode/src/jpeg_to_htj2k/batch/accelerated_storage.rs", - ), - storage: read("crates/j2k-transcode/src/jpeg_to_htj2k/batch/storage.rs"), - encode: read("crates/j2k-transcode/src/jpeg_to_htj2k/batch/encode.rs"), - float97_input: read( - "crates/j2k-transcode/src/jpeg_to_htj2k/batch/encode/float97_input.rs", - ), - precomputed: read("crates/j2k-transcode/src/jpeg_to_htj2k/batch/encode/precomputed.rs"), - encode_live: read("crates/j2k-transcode/src/jpeg_to_htj2k/batch/encode/live.rs"), - } - } - - fn assert_line_budgets(&self) { - for (path, source, max_lines) in [ - ("jpeg_to_htj2k/batch.rs", self.batch.as_str(), 350), - ( - "jpeg_to_htj2k/batch/result_slots.rs", - self.result_slots.as_str(), - 125, - ), - ( - "jpeg_to_htj2k/batch/result_slots/tests.rs", - self.result_slot_tests.as_str(), - 100, - ), - ( - "jpeg_to_htj2k/batch/actual_live.rs", - self.actual_live.as_str(), - 100, - ), - ( - "jpeg_to_htj2k/batch/individual.rs", - self.individual.as_str(), - 125, - ), - ("jpeg_to_htj2k/batch/prepare.rs", self.prepare.as_str(), 325), - ( - "jpeg_to_htj2k/batch/group_budget.rs", - self.group_budget.as_str(), - 75, - ), - ( - "jpeg_to_htj2k/batch/transform.rs", - self.transform.as_str(), - 425, - ), - ( - "jpeg_to_htj2k/batch/accelerated_storage.rs", - self.accelerated_storage.as_str(), - 475, - ), - ("jpeg_to_htj2k/batch/storage.rs", self.storage.as_str(), 475), - ("jpeg_to_htj2k/batch/encode.rs", self.encode.as_str(), 600), - ( - "jpeg_to_htj2k/batch/encode/float97_input.rs", - self.float97_input.as_str(), - 150, - ), - ( - "jpeg_to_htj2k/batch/encode/precomputed.rs", - self.precomputed.as_str(), - 350, - ), - ( - "jpeg_to_htj2k/batch/encode/live.rs", - self.encode_live.as_str(), - 250, - ), - ] { - assert_line_budget(path, source, max_lines); - } - } - - fn assert_contracts(&self) { - assert_pattern_checks(&[ - PatternCheck::new("JPEG-to-HTJ2K batch facade", &self.batch) - .required(&[ - "mod prepare;", - "mod group_budget;", - "mod result_slots;", - "mod transform;", - "mod accelerated_storage;", - "mod storage;", - "mod encode;", - ]) - .forbidden(&[ - "struct IntegerBatchTile", - "fn transform_float97_batch_tiles(", - "fn store_integer_batch_wavelet(", - "fn encode_float97_batch_tile(", - "tile_results[", - ]), - PatternCheck::new("batch result-slot invariants", &self.result_slots).required(&[ - "struct BatchResultSlots", - "fn insert(", - "mod tests;", - "fn into_results(", - "JpegToHtj2kError::InternalInvariant", - ]), - PatternCheck::new("batch result-slot regressions", &self.result_slot_tests).required( - &[ - "fn missing_worker_result_is_an_internal_invariant(", - "fn duplicate_worker_result_is_an_internal_invariant(", - "fn out_of_range_worker_result_is_an_internal_invariant(", - "fn complete_worker_results_preserve_input_order(", - ], - ), - PatternCheck::new("batch 9/7 encode-input ownership", &self.float97_input) - .required(&[ - "enum Float97BatchEncodingInput", - "fn select_float97_batch_encoding(", - "PreencodedHtj2k97CompactImage", - "PrequantizedHtj2k97Image", - ]) - .forbidden(&["include!(", "use super::*;"]), - PatternCheck::new("JPEG-to-HTJ2K batch preparation ownership", &self.prepare).required( - &[ - "struct IntegerBatchTile", - "fn prepare_float97_batch_tile(", - "fn batch_component_groups(", - ], - ), - PatternCheck::new( - "JPEG-to-HTJ2K batch group budget ownership", - &self.group_budget, - ) - .required(&[ - "fn batch_component_count(", - "fn validate_group_workspace(", - "fn next_group_len(", - ]), - PatternCheck::new("JPEG-to-HTJ2K batch transform ownership", &self.transform).required( - &[ - "fn transform_integer_batch_tiles", - "fn float97_wavelets_for_batch_group", - "fn record_cpu_fallback", - ], - ), - PatternCheck::new("accelerated batch storage", &self.accelerated_storage).required(&[ - "fn store_compact_preencoded_component(", - "fn try_store_grouped_i16_preencoded_float97_batches", - "fn try_store_prequantized_float97_batch_group", - ]), - PatternCheck::new("wavelet batch storage", &self.storage).required(&[ - "fn store_integer_batch_wavelet(", - "fn store_float97_batch_wavelet(", - ]), - PatternCheck::new("JPEG-to-HTJ2K batch encode ownership", &self.encode).required(&[ - "fn record_encode_dispatch_delta(", - "mod precomputed;", - "fn encode_float97_batch_tile", - ]), - PatternCheck::new( - "JPEG-to-HTJ2K precomputed batch encode ownership", - &self.precomputed, - ) - .required(&[ - "fn encode_float97_precomputed_tiles_batch", - "fn can_encode_float97_precomputed_tiles_batch", - ]), - ]); - } -} - -pub(super) fn assert_batch_structure() { - let sources = BatchSources::read(); - sources.assert_line_budgets(); - sources.assert_contracts(); -} diff --git a/xtask/tests/repo_lint_support/transcode_structure_policy/metal.rs b/xtask/tests/repo_lint_support/transcode_structure_policy/metal.rs deleted file mode 100644 index 4a87e256..00000000 --- a/xtask/tests/repo_lint_support/transcode_structure_policy/metal.rs +++ /dev/null @@ -1,123 +0,0 @@ -// SPDX-License-Identifier: MIT OR Apache-2.0 - -use super::super::{assert_pattern_checks, PatternCheck}; -use super::{assert_line_budget, read}; - -#[test] -fn metal_transcode_facades_and_extracted_children_stay_focused() { - let focused_modules = [ - ("lib.rs", 150), - ("route.rs", 275), - ("accelerator.rs", 350), - ("accelerator/dispatch.rs", 250), - ("metal/reversible/batch.rs", 275), - ("metal/irreversible/staging.rs", 150), - ("metal/geometry/allocation.rs", 125), - ("metal/buffers/storage.rs", 100), - ("metal/buffers/basis.rs", 50), - ("metal/codeblock_output.rs", 375), - ("weights.rs", 50), - ("weights/budget.rs", 425), - ("weights/dense.rs", 425), - ("weights/error.rs", 425), - ("weights/shared.rs", 425), - ("weights/sparse.rs", 425), - ("weights/symbolic.rs", 425), - ("weights/tests.rs", 425), - ("weights/transform.rs", 425), - ]; - for (relative, max_lines) in focused_modules { - let path = format!("crates/j2k-transcode-metal/src/{relative}"); - let source = read(&path); - assert_line_budget(&path, &source, max_lines); - } - - let library = read("crates/j2k-transcode-metal/src/lib.rs"); - let accelerator = read("crates/j2k-transcode-metal/src/accelerator.rs"); - let reversible = read("crates/j2k-transcode-metal/src/metal/reversible.rs"); - let irreversible = read("crates/j2k-transcode-metal/src/metal/irreversible.rs"); - let geometry = read("crates/j2k-transcode-metal/src/metal/geometry.rs"); - let buffers = read("crates/j2k-transcode-metal/src/metal/buffers.rs"); - let weights = read("crates/j2k-transcode-metal/src/weights.rs"); - assert_pattern_checks(&[ - PatternCheck::new("Metal transcode crate facade", &library) - .required(&["mod accelerator;", "mod error;", "mod route;"]) - .forbidden(&["impl DctToWaveletStageAccelerator for"]), - PatternCheck::new("Metal accelerator facade", &accelerator) - .required(&["mod dispatch;"]) - .forbidden(&["impl DctToWaveletStageAccelerator for"]), - PatternCheck::new("Metal reversible facade", &reversible).required(&["mod batch;"]), - PatternCheck::new("Metal irreversible facade", &irreversible).required(&["mod staging;"]), - PatternCheck::new("Metal geometry facade", &geometry).required(&["mod allocation;"]), - PatternCheck::new("Metal buffer facade", &buffers) - .required(&["mod basis;", "mod storage;"]), - PatternCheck::new("Metal weight facade", &weights) - .required(&[ - "mod budget;", - "mod dense;", - "mod error;", - "mod shared;", - "mod sparse;", - "mod symbolic;", - "mod transform;", - ]) - .forbidden(&[ - "fn linearized_53_from_sample_slice(", - "fn sparse_rows_from_dense(", - ]), - ]); -} - -#[test] -fn metal_transcode_extracted_children_own_their_focused_stages() { - let route = read("crates/j2k-transcode-metal/src/route.rs"); - let dispatch = read("crates/j2k-transcode-metal/src/accelerator/dispatch.rs"); - let reversible_batch = read("crates/j2k-transcode-metal/src/metal/reversible/batch.rs"); - let irreversible_staging = read("crates/j2k-transcode-metal/src/metal/irreversible/staging.rs"); - let geometry_allocation = read("crates/j2k-transcode-metal/src/metal/geometry/allocation.rs"); - let buffer_storage = read("crates/j2k-transcode-metal/src/metal/buffers/storage.rs"); - let buffer_basis = read("crates/j2k-transcode-metal/src/metal/buffers/basis.rs"); - let codeblock_output = read("crates/j2k-transcode-metal/src/metal/codeblock_output.rs"); - assert_pattern_checks(&[ - PatternCheck::new("Metal route ownership", &route).required(&[ - "pub fn jpeg_to_htj2k_with_metal_route(", - "pub fn jpeg_to_htj2k_batch_with_metal_route(", - "fn fallback_reason(", - ]), - PatternCheck::new("Metal accelerator dispatch ownership", &dispatch).required(&[ - "impl DctToWaveletStageAccelerator for MetalDctToWaveletStageAccelerator", - "fn dct_grid_to_dwt53(", - "fn dct_grid_to_htj2k97_codeblock_batch(", - ]), - PatternCheck::new("Metal reversible batch ownership", &reversible_batch).required(&[ - "fn dispatch_with_runtime(", - "fn reversible_batch_shapes(", - "fn read_reversible_batch_outputs(", - ]), - PatternCheck::new( - "Metal irreversible staging ownership", - &irreversible_staging, - ) - .required(&[ - "fn dwt97_staged_batch_shape(", - "fn dwt97_codeblock_batch_shape(", - "fn dwt97_staged_row_buffers(", - ]), - PatternCheck::new("Metal geometry allocation ownership", &geometry_allocation).required(&[ - "fn validate_float_projection_allocations(", - "fn validate_codeblock_projection_allocations(", - ]), - PatternCheck::new("Metal buffer storage ownership", &buffer_storage).required(&[ - "fn buffer_with_slice<", - "fn dwt97_blocks_buffer(", - "fn output_i32_buffer(", - ]), - PatternCheck::new("Metal basis ownership", &buffer_basis) - .required(&["fn idct8_basis_table(", "fn idct8_basis("]), - PatternCheck::new("Metal code-block readback ownership", &codeblock_output).required(&[ - "fn validate_codeblock_output_host_workspace(", - "fn read_prequantized_97_codeblock_outputs(", - "fn try_vec_from_array<", - ]), - ]); -} diff --git a/xtask/tests/repo_lint_support/workflow_policy.rs b/xtask/tests/repo_lint_support/workflow_policy.rs index 9a6e2087..e2993e33 100644 --- a/xtask/tests/repo_lint_support/workflow_policy.rs +++ b/xtask/tests/repo_lint_support/workflow_policy.rs @@ -344,15 +344,14 @@ fn release_candidate_and_publish_evidence_are_fail_closed() { "cargo-machete@0.9.2", "cargo xtask machete", ]), - PatternCheck::new("CI normal and strict repository policy gate", repo_lint).required(&[ - "runs-on: macos-latest", - "toolchain: nightly-2026-06-28", - "targets: aarch64-apple-darwin", - "cargo-public-api@0.52.0", - "cargo install cargo-public-api --version 0.52.0 --locked", - "Run normal and strict repository policy", - "cargo xtask repo-lint --strict", - ]), + PatternCheck::new("CI repository policy gate", repo_lint) + .required(&["runs-on: ubuntu-latest", "cargo xtask repo-lint"]) + .forbidden(&[ + "toolchain: nightly-2026-06-28", + "targets: aarch64-apple-darwin", + "cargo-public-api", + "repo-lint --strict", + ]), PatternCheck::new("xtask release-status dispatch", &xtask).required(&[ "mod release_status;", "\"release-status\" => release_status::release_status(env::args().skip(2))", diff --git a/xtask/tests/repo_lint_support/workflow_structure_policy.rs b/xtask/tests/repo_lint_support/workflow_structure_policy.rs new file mode 100644 index 00000000..11c2405f --- /dev/null +++ b/xtask/tests/repo_lint_support/workflow_structure_policy.rs @@ -0,0 +1,298 @@ +// SPDX-License-Identifier: MIT OR Apache-2.0 + +//! Structural GitHub Actions policy with workflow YAML as the source of truth. + +mod github_expression; +mod yaml; + +use std::collections::BTreeSet; +use std::path::{Component, PathBuf}; + +use serde_yaml_ng::Value; + +use self::github_expression::{untrusted_event_run_tokens, value_contains_secret_reference}; +use self::yaml::{ + display_key, is_numeric_zero, jobs, mapping_get, string_set, value_mapping, visit_mappings, + workflow, workflows, +}; +use super::repo_root; + +#[test] +fn every_action_reference_is_pinned_to_a_forty_hex_sha() { + let mut violations = Vec::new(); + for workflow in workflows() { + visit_mappings(&workflow.document, &mut |mapping| { + let Some(reference) = mapping_get(mapping, "uses").and_then(Value::as_str) else { + return; + }; + if reference.starts_with("./") || reference.starts_with("docker://") { + return; + } + let pinned = reference.rsplit_once('@').is_some_and(|(_, revision)| { + revision.len() == 40 && revision.bytes().all(|byte| byte.is_ascii_hexdigit()) + }); + if !pinned { + violations.push(format!("{}: {reference}", workflow.file_name)); + } + }); + } + assert!( + violations.is_empty(), + "external actions must use a full forty-hex commit SHA:\n{}", + violations.join("\n") + ); +} + +#[test] +fn every_workflow_declares_default_permissions_including_contents_read() { + let mut violations = Vec::new(); + for workflow in workflows() { + let root = value_mapping(&workflow.document, &workflow.file_name); + let contents = mapping_get(root, "permissions") + .and_then(Value::as_mapping) + .and_then(|permissions| mapping_get(permissions, "contents")) + .and_then(Value::as_str); + if contents != Some("read") { + violations.push(workflow.file_name.clone()); + } + } + assert!( + violations.is_empty(), + "workflows missing top-level contents: read permission: {violations:?}" + ); +} + +#[test] +fn no_job_requests_a_write_permission_scope() { + let mut violations = Vec::new(); + for workflow in workflows() { + visit_mappings(&workflow.document, &mut |mapping| { + let Some(permissions) = mapping_get(mapping, "permissions") else { + return; + }; + if permissions.as_str() == Some("write-all") + || permissions.as_mapping().is_some_and(|scopes| { + scopes.values().any(|value| value.as_str() == Some("write")) + }) + { + violations.push(workflow.file_name.clone()); + } + }); + } + violations.sort(); + violations.dedup(); + assert!( + violations.is_empty(), + "workflow permission scopes must remain read-only: {violations:?}" + ); +} + +#[test] +fn every_downloaded_release_asset_is_checksum_verified() { + let mut violations = Vec::new(); + for workflow in workflows() { + visit_mappings(&workflow.document, &mut |mapping| { + let Some(script) = mapping_get(mapping, "run").and_then(Value::as_str) else { + return; + }; + if script.contains("/releases/download/") && !script.contains("sha256sum -c") { + violations.push(workflow.file_name.clone()); + } + }); + } + assert!( + violations.is_empty(), + "downloaded release assets need same-step SHA-256 verification: {violations:?}" + ); +} + +#[test] +fn the_secret_scan_redacts_findings_and_scans_full_history() { + let secret_scan = workflow("secret-scan.yml"); + let mut full_checkout = false; + let mut fail_closed_scan = false; + visit_mappings(&secret_scan.document, &mut |mapping| { + if mapping_get(mapping, "uses") + .and_then(Value::as_str) + .is_some_and(|reference| reference.starts_with("actions/checkout@")) + { + full_checkout = mapping_get(mapping, "with") + .and_then(Value::as_mapping) + .and_then(|with| mapping_get(with, "fetch-depth")) + .is_some_and(is_numeric_zero); + } + if let Some(script) = mapping_get(mapping, "run").and_then(Value::as_str) { + if script.contains("gitleaks detect") { + fail_closed_scan = script.contains("--source .") && script.contains("--redact"); + } + } + }); + assert!( + full_checkout, + "secret scan checkout must fetch full history" + ); + assert!( + fail_closed_scan, + "secret scan must redact findings and scan the repository" + ); +} + +#[test] +fn manual_publish_dispatch_is_dry_run_only() { + let publish = workflow("publish.yml"); + let root = value_mapping(&publish.document, &publish.file_name); + let triggers = mapping_get(root, "on") + .and_then(Value::as_mapping) + .expect("publish workflow trigger mapping"); + assert!( + mapping_get(triggers, "workflow_dispatch").is_some(), + "publish workflow must keep its manual dry-run entry point" + ); + let dry_run = mapping_get(root, "env") + .and_then(Value::as_mapping) + .and_then(|environment| mapping_get(environment, "DRY_RUN_ONLY")) + .and_then(Value::as_str) + .expect("publish DRY_RUN_ONLY expression"); + assert!( + dry_run.contains("github.event_name == 'workflow_dispatch'"), + "manual publish dispatch must set DRY_RUN_ONLY" + ); + + let publish_job = jobs(root, &publish.file_name) + .get(Value::String("publish".to_owned())) + .and_then(Value::as_mapping) + .expect("publish job"); + let condition = mapping_get(publish_job, "if") + .and_then(Value::as_str) + .expect("publish job event guard"); + assert!( + condition.contains("github.event_name == 'push'"), + "the credentialed publish job must not run for workflow_dispatch" + ); +} + +#[test] +fn every_job_that_reads_a_secret_declares_a_protected_environment() { + let mut violations = Vec::new(); + for workflow in workflows() { + let root = value_mapping(&workflow.document, &workflow.file_name); + for (job_name, job) in jobs(root, &workflow.file_name) { + if value_contains_secret_reference(job) + && job + .as_mapping() + .and_then(|mapping| mapping_get(mapping, "environment")) + .is_none_or(Value::is_null) + { + violations.push(format!("{}:{}", workflow.file_name, display_key(job_name))); + } + } + } + assert!( + violations.is_empty(), + "jobs reading secrets need a protected environment:\n{}", + violations.join("\n") + ); +} + +#[test] +fn no_workflow_uses_pull_request_target_or_interpolates_untrusted_event_fields() { + let mut violations = Vec::new(); + for workflow in workflows() { + let root = value_mapping(&workflow.document, &workflow.file_name); + let triggers = mapping_get(root, "on") + .and_then(Value::as_mapping) + .unwrap_or_else(|| panic!("{} trigger mapping", workflow.file_name)); + if mapping_get(triggers, "pull_request_target").is_some() { + violations.push(format!("{}: pull_request_target", workflow.file_name)); + } + visit_mappings(&workflow.document, &mut |mapping| { + let Some(script) = mapping_get(mapping, "run").and_then(Value::as_str) else { + return; + }; + for token in untrusted_event_run_tokens(script) { + violations.push(format!("{}: {token}", workflow.file_name)); + } + }); + } + assert!( + violations.is_empty(), + "untrusted event text must not reach shell bodies:\n{}", + violations.join("\n") + ); +} + +#[test] +fn every_required_pr_gate_is_listed_in_the_aggregate_job_needs() { + let ci = workflow("ci.yml"); + let root = value_mapping(&ci.document, &ci.file_name); + let jobs = jobs(root, &ci.file_name); + let aggregate_name = Value::String("pr-required-checks".to_owned()); + let aggregate = jobs + .get(&aggregate_name) + .and_then(Value::as_mapping) + .expect("PR aggregate job"); + + let mut expected = jobs + .keys() + .filter_map(Value::as_str) + .map(str::to_owned) + .collect::>(); + expected.remove("pr-required-checks"); + let actual = string_set( + mapping_get(aggregate, "needs").expect("PR aggregate needs"), + "PR aggregate needs", + ); + assert_eq!( + actual, expected, + "PR aggregate needs must cover every sibling gate" + ); +} + +#[test] +fn local_reusable_workflow_references_resolve_to_files_in_this_repo() { + let mut violations = Vec::new(); + for workflow in workflows() { + visit_mappings(&workflow.document, &mut |mapping| { + let Some(reference) = mapping_get(mapping, "uses").and_then(Value::as_str) else { + return; + }; + if !reference.starts_with("./.github/workflows/") { + return; + } + let relative = reference.trim_start_matches("./"); + let path = PathBuf::from(relative); + if path + .components() + .any(|component| matches!(component, Component::ParentDir)) + { + violations.push(format!("{}: path escape {reference}", workflow.file_name)); + return; + } + let target = repo_root().join(&path); + if !target.is_file() { + violations.push(format!("{}: missing {reference}", workflow.file_name)); + return; + } + let Some(target_name) = target.file_name().and_then(|name| name.to_str()) else { + violations.push(format!("{}: non-UTF-8 {reference}", workflow.file_name)); + return; + }; + let target_workflow = yaml::workflow(target_name); + let target_root = value_mapping(&target_workflow.document, &target_workflow.file_name); + let callable = mapping_get(target_root, "on") + .and_then(Value::as_mapping) + .is_some_and(|triggers| mapping_get(triggers, "workflow_call").is_some()); + if !callable { + violations.push(format!( + "{}: target is not reusable {reference}", + workflow.file_name + )); + } + }); + } + assert!( + violations.is_empty(), + "local reusable workflow references must resolve in-repo:\n{}", + violations.join("\n") + ); +} diff --git a/xtask/tests/repo_lint_support/workflow_structure_policy/github_expression.rs b/xtask/tests/repo_lint_support/workflow_structure_policy/github_expression.rs new file mode 100644 index 00000000..a2c7d716 --- /dev/null +++ b/xtask/tests/repo_lint_support/workflow_structure_policy/github_expression.rs @@ -0,0 +1,142 @@ +// SPDX-License-Identifier: MIT OR Apache-2.0 + +use std::collections::BTreeSet; + +use serde_yaml_ng::Value; + +pub(super) fn value_contains_secret_reference(value: &Value) -> bool { + match value { + Value::String(text) => github_expression_tokens(text) + .iter() + .any(|token| token == "secrets" || token.starts_with("secrets.")), + Value::Sequence(values) => values.iter().any(value_contains_secret_reference), + Value::Mapping(mapping) => mapping.iter().any(|(key, value)| { + value_contains_secret_reference(key) || value_contains_secret_reference(value) + }), + _ => false, + } +} + +pub(super) fn untrusted_event_run_tokens(script: &str) -> BTreeSet { + github_expression_tokens(script) + .into_iter() + .filter(|token| is_untrusted_event_run_token(token)) + .collect() +} + +fn is_untrusted_event_run_token(token: &str) -> bool { + token == "github.head_ref" + || (token.starts_with("github.event.") + && [".title", ".body", ".head.ref"] + .iter() + .any(|suffix| token.ends_with(suffix))) +} + +fn github_expression_tokens(text: &str) -> BTreeSet { + let compact = text + .chars() + .filter(|character| !character.is_ascii_whitespace()) + .collect::() + .to_ascii_lowercase(); + let canonical = compact + .replace("['", ".") + .replace("']", "") + .replace("[\"", ".") + .replace("\"]", ""); + + let mut tokens = BTreeSet::new(); + let mut remaining = canonical.as_str(); + while let Some(start) = remaining.find("${{") { + let expression = &remaining[start + 3..]; + let (expression, next) = expression.find("}}").map_or((expression, None), |end| { + (&expression[..end], Some(&expression[end + 2..])) + }); + tokens.extend( + expression + .split(|character: char| { + !(character.is_ascii_alphanumeric() || character == '_' || character == '.') + }) + .filter(|token| !token.is_empty()) + .map(str::to_owned), + ); + let Some(next) = next else { + break; + }; + remaining = next; + } + tokens +} + +#[test] +fn untrusted_event_run_tokens_cover_event_families_without_rejecting_commit_shas() { + for token in [ + "github.event.pull_request.title", + "github.event.issue.body", + "github.event.discussion.title", + "github.event.pull_request.head.ref", + "github.head_ref", + ] { + assert!(is_untrusted_event_run_token(token), "{token}"); + } + for token in [ + "github.event.pull_request.head.sha", + "github.event.pull_request.base.sha", + "github.event.before", + "github.event_name", + ] { + assert!(!is_untrusted_event_run_token(token), "{token}"); + } +} + +#[test] +fn untrusted_event_run_tokens_normalize_bracket_and_dotted_property_access() { + for script in [ + "echo '${{ github.event.issue.title }}'", + "echo '${{ github ['event'] ['discussion'] ['body'] }}'", + "echo '${{ github[\"event\"].pull_request['head'][\"ref\"] }}'", + "echo '${{ github ['head_ref'] }}'", + ] { + assert!( + !untrusted_event_run_tokens(script).is_empty(), + "expected untrusted reference in {script}" + ); + } + + for script in [ + "echo '${{ github.event.pull_request.head.sha }}'", + "echo '${{ github ['event'] ['pull_request'] ['base'] ['sha'] }}'", + "echo '${{ github.event.before }}'", + ] { + assert!( + untrusted_event_run_tokens(script).is_empty(), + "unexpected untrusted reference in {script}" + ); + } +} + +#[test] +fn secret_reference_detection_normalizes_bracket_and_dotted_property_access() { + for reference in [ + "${{ secrets.CARGO_TOKEN }}", + "${{ secrets ['CARGO_TOKEN'] }}", + "${{ toJson(secrets) }}", + ] { + let value = Value::Sequence(vec![Value::String(reference.to_owned())]); + assert!( + value_contains_secret_reference(&value), + "expected secret reference in {reference}" + ); + } + + for reference in [ + "${{ env.CARGO_TOKEN }}", + "${{ vars.CARGO_TOKEN }}", + "documentation mentioning secrets.CARGO_TOKEN outside an expression", + ] { + let value = Value::String(reference.to_owned()); + assert!( + !value_contains_secret_reference(&value), + "unexpected secret reference in {reference}" + ); + } +} diff --git a/xtask/tests/repo_lint_support/workflow_structure_policy/yaml.rs b/xtask/tests/repo_lint_support/workflow_structure_policy/yaml.rs new file mode 100644 index 00000000..e4fe7131 --- /dev/null +++ b/xtask/tests/repo_lint_support/workflow_structure_policy/yaml.rs @@ -0,0 +1,126 @@ +// SPDX-License-Identifier: MIT OR Apache-2.0 + +use std::collections::BTreeSet; +use std::fs; +use std::sync::OnceLock; + +use serde_yaml_ng::{Mapping, Value}; + +use super::super::repo_root; + +#[derive(Debug)] +pub(super) struct Workflow { + pub(super) file_name: String, + pub(super) document: Value, +} + +pub(super) fn workflows() -> &'static [Workflow] { + static WORKFLOWS: OnceLock> = OnceLock::new(); + WORKFLOWS.get_or_init(|| { + let directory = repo_root().join(".github/workflows"); + let mut paths = fs::read_dir(&directory) + .expect("read workflow directory") + .map(|entry| entry.expect("read workflow entry").path()) + .filter(|path| { + matches!( + path.extension().and_then(|extension| extension.to_str()), + Some("yml" | "yaml") + ) + }) + .collect::>(); + paths.sort(); + assert!(!paths.is_empty(), "no GitHub Actions workflows found"); + + paths + .into_iter() + .map(|path| { + let source = fs::read_to_string(&path) + .unwrap_or_else(|error| panic!("read {}: {error}", path.display())); + let document = serde_yaml_ng::from_str(&source) + .unwrap_or_else(|error| panic!("parse {}: {error}", path.display())); + let file_name = path + .file_name() + .and_then(|name| name.to_str()) + .unwrap_or_else(|| { + panic!("workflow path has no UTF-8 name: {}", path.display()) + }) + .to_owned(); + Workflow { + file_name, + document, + } + }) + .collect() + }) +} + +pub(super) fn workflow(file_name: &str) -> &'static Workflow { + workflows() + .iter() + .find(|workflow| workflow.file_name == file_name) + .unwrap_or_else(|| panic!("missing workflow {file_name}")) +} + +pub(super) fn value_mapping<'a>(value: &'a Value, label: &str) -> &'a Mapping { + value + .as_mapping() + .unwrap_or_else(|| panic!("{label} must contain a YAML mapping")) +} + +pub(super) fn jobs<'a>(root: &'a Mapping, label: &str) -> &'a Mapping { + mapping_get(root, "jobs") + .and_then(Value::as_mapping) + .unwrap_or_else(|| panic!("{label} must contain a jobs mapping")) +} + +pub(super) fn mapping_get<'a>(mapping: &'a Mapping, key: &str) -> Option<&'a Value> { + mapping.get(Value::String(key.to_owned())).or_else(|| { + (key == "on") + .then(|| mapping.get(Value::Bool(true))) + .flatten() + }) +} + +pub(super) fn visit_mappings(value: &Value, visitor: &mut impl FnMut(&Mapping)) { + match value { + Value::Mapping(mapping) => { + visitor(mapping); + for (key, value) in mapping { + visit_mappings(key, visitor); + visit_mappings(value, visitor); + } + } + Value::Sequence(values) => { + for value in values { + visit_mappings(value, visitor); + } + } + _ => {} + } +} + +pub(super) fn string_set(value: &Value, label: &str) -> BTreeSet { + match value { + Value::String(value) => [value.clone()].into_iter().collect(), + Value::Sequence(values) => values + .iter() + .map(|value| { + value + .as_str() + .unwrap_or_else(|| panic!("{label} entries must be strings")) + .to_owned() + }) + .collect(), + _ => panic!("{label} must be a string or sequence"), + } +} + +pub(super) fn is_numeric_zero(value: &Value) -> bool { + value.as_u64() == Some(0) || value.as_i64() == Some(0) || value.as_str() == Some("0") +} + +pub(super) fn display_key(value: &Value) -> String { + value + .as_str() + .map_or_else(|| format!("{value:?}"), str::to_owned) +} diff --git a/xtask/tests/repo_lint_support/xtask_main_structure_policy.rs b/xtask/tests/repo_lint_support/xtask_main_structure_policy.rs deleted file mode 100644 index f3a83001..00000000 --- a/xtask/tests/repo_lint_support/xtask_main_structure_policy.rs +++ /dev/null @@ -1,284 +0,0 @@ -// SPDX-License-Identifier: MIT OR Apache-2.0 - -//! Structural ownership and size ratchets for the xtask command dispatcher. - -use std::fs; - -use super::{assert_pattern_checks, repo_root, PatternCheck}; - -mod codegen; -mod lint_policy; -mod release_commands; -mod release_integrity; -mod release_status; - -fn read(relative_path: &str) -> String { - fs::read_to_string(repo_root().join(relative_path)) - .unwrap_or_else(|error| panic!("read {relative_path}: {error}")) -} - -struct XtaskSources { - main: String, - benchmark: String, - support: String, - panic_surface: String, - quality: String, - release: String, - semver: String, - semver_review: String, - semver_tests: String, - stable_api: String, - release_integrity_policy: String, - release_integrity_markdown: String, - release_integrity_html: String, - release_integrity_policy_tests: String, - release_integrity_changelog_tests: String, - release_integrity_metadata_tests: String, - release_integrity_provenance_tests: String, - release_integrity_provenance_boundary_tests: String, -} - -impl XtaskSources { - fn read() -> Self { - Self { - main: read("xtask/src/main.rs"), - benchmark: read("xtask/src/benchmark_commands.rs"), - support: read("xtask/src/command_support.rs"), - panic_surface: read("xtask/src/panic_surface.rs"), - quality: read("xtask/src/quality_commands.rs"), - release: read("xtask/src/release_commands.rs"), - semver: read("xtask/src/semver.rs"), - semver_review: read("xtask/src/semver/review.rs"), - semver_tests: read("xtask/src/semver/tests.rs"), - stable_api: read("xtask/src/stable_api.rs"), - release_integrity_policy: read( - "xtask/src/release_commands/release_integrity_policy.rs", - ), - release_integrity_markdown: read( - "xtask/src/release_commands/release_integrity_policy/markdown.rs", - ), - release_integrity_html: read( - "xtask/src/release_commands/release_integrity_policy/markdown/html.rs", - ), - release_integrity_policy_tests: read( - "xtask/src/release_commands/release_integrity_policy/tests.rs", - ), - release_integrity_changelog_tests: read( - "xtask/src/release_commands/release_integrity_policy/tests/changelog.rs", - ), - release_integrity_metadata_tests: read( - "xtask/src/release_commands/release_integrity_policy/tests/metadata.rs", - ), - release_integrity_provenance_tests: read( - "xtask/src/release_commands/release_integrity_policy/tests/provenance.rs", - ), - release_integrity_provenance_boundary_tests: read( - "xtask/src/release_commands/release_integrity_policy/tests/provenance_boundaries.rs", - ), - } - } -} - -fn assert_modules_stay_focused(sources: &XtaskSources) { - for (relative_path, source, max_lines) in [ - ("xtask/src/main.rs", sources.main.as_str(), 700), - ( - "xtask/src/benchmark_commands.rs", - sources.benchmark.as_str(), - 800, - ), - ( - "xtask/src/command_support.rs", - sources.support.as_str(), - 300, - ), - ( - "xtask/src/panic_surface.rs", - sources.panic_surface.as_str(), - 650, - ), - ( - "xtask/src/quality_commands.rs", - sources.quality.as_str(), - 800, - ), - ( - "xtask/src/release_commands.rs", - sources.release.as_str(), - 800, - ), - ("xtask/src/semver.rs", sources.semver.as_str(), 900), - ( - "xtask/src/semver/review.rs", - sources.semver_review.as_str(), - 350, - ), - ( - "xtask/src/semver/tests.rs", - sources.semver_tests.as_str(), - 300, - ), - ("xtask/src/stable_api.rs", sources.stable_api.as_str(), 350), - ( - "xtask/src/release_commands/release_integrity_policy.rs", - sources.release_integrity_policy.as_str(), - 300, - ), - ( - "xtask/src/release_commands/release_integrity_policy/markdown.rs", - sources.release_integrity_markdown.as_str(), - 100, - ), - ( - "xtask/src/release_commands/release_integrity_policy/markdown/html.rs", - sources.release_integrity_html.as_str(), - 100, - ), - ( - "xtask/src/release_commands/release_integrity_policy/tests.rs", - sources.release_integrity_policy_tests.as_str(), - 50, - ), - ( - "xtask/src/release_commands/release_integrity_policy/tests/changelog.rs", - sources.release_integrity_changelog_tests.as_str(), - 175, - ), - ( - "xtask/src/release_commands/release_integrity_policy/tests/metadata.rs", - sources.release_integrity_metadata_tests.as_str(), - 100, - ), - ( - "xtask/src/release_commands/release_integrity_policy/tests/provenance.rs", - sources.release_integrity_provenance_tests.as_str(), - 75, - ), - ( - "xtask/src/release_commands/release_integrity_policy/tests/provenance_boundaries.rs", - sources.release_integrity_provenance_boundary_tests.as_str(), - 75, - ), - ] { - assert_module_stays_focused(relative_path, source, max_lines); - } -} - -fn assert_module_stays_focused(relative_path: &str, source: &str, max_lines: usize) { - let line_count = source.lines().count(); - assert!( - line_count < max_lines, - "{relative_path} has {line_count} lines; expected fewer than {max_lines}" - ); - assert!( - !source.contains("::*"), - "{relative_path} must keep explicit imports" - ); - assert!( - !source.contains("include!("), - "{relative_path} must remain a real Rust module" - ); - assert!( - !source.contains("#[allow("), - "{relative_path} must not add lint suppressions" - ); -} - -fn assert_dispatcher_and_command_ownership(sources: &XtaskSources) { - assert_pattern_checks(&[ - PatternCheck::new("xtask root dispatcher", &sources.main) - .required(&[ - "mod benchmark_commands;", - "mod clone_audit;", - "mod codegen_commands;", - "mod command_support;", - "mod panic_surface;", - "mod quality_commands;", - "mod release_commands;", - "mod stable_api;", - "mod source_audit;", - "\"clone-audit\" => clone_audit(env::args().skip(2))", - "\"release-integrity\" => release_integrity(env::args().skip(2))", - "fn run() -> Result<(), String>", - "fn print_help()", - ]) - .forbidden(&[ - "fn bench_build()", - "fn codec_math_codegen(", - "fn ensure_clean_worktree()", - "fn panic_surface()", - "fn release_integrity(", - ]), - PatternCheck::new("xtask benchmark command ownership", &sources.benchmark).required(&[ - "pub(super) fn bench_build(args:", - "pub(super) fn j2k_bench_signoff()", - "pub(super) fn bench_report(", - "fn render_benchmark_report(", - ]), - PatternCheck::new("xtask stable API collection ownership", &sources.stable_api).required( - &[ - "pub(super) fn collect_package_apis(", - "fn collect_package_api(", - "fn package_public_api(", - "fn validate_public_api_environment_keys", - ], - ), - PatternCheck::new("xtask semver module boundaries", &sources.semver).required(&[ - "mod review;", - "mod tests;", - "review::load_review_config()?", - "review::validate_reviews(", - ]), - PatternCheck::new("xtask semver review ownership", &sources.semver_review).required(&[ - "pub(super) fn parse_review_config(", - "pub(super) fn validate_reviews(", - "hidden_fingerprint", - "hidden_rationale", - ]), - PatternCheck::new("xtask process and path helper ownership", &sources.support).required(&[ - "pub(super) fn ensure_clean_worktree()", - "pub(super) fn run_cargo(", - "pub(super) fn run_cargo_test_with_pass_floor(", - "pub(super) fn command_output_os(", - "pub(super) fn command_output_os_detailed_with_env(", - "pub(super) fn rust_sources(", - ]), - PatternCheck::new( - "xtask panic-surface command ownership", - &sources.panic_surface, - ) - .required(&[ - "mod source_inventory;", - "pub(super) fn panic_surface()", - "enforce_panic_macro_inventory", - "fn parse_panic_surface_selection(", - "fn parse_panic_surface_output(", - ]), - PatternCheck::new("xtask quality command ownership", &sources.quality).required(&[ - "pub(super) fn ci()", - "pub(super) fn clippy_strict()", - "pub(super) fn fuzz_build()", - "pub(super) fn verify_unsafe_audit()", - ]), - PatternCheck::new("xtask release command ownership", &sources.release).required(&[ - "mod release_integrity_policy;", - "const PUBLISHABLE_PACKAGES:", - "pub(super) const STABLE_SEMVER_PACKAGES:", - "pub(super) fn release_integrity(", - "pub(super) fn release_cpu()", - "pub(super) fn package()", - ]), - ]); -} - -#[test] -fn xtask_dispatcher_stays_split_by_command_family() { - let sources = XtaskSources::read(); - assert_modules_stay_focused(&sources); - assert_dispatcher_and_command_ownership(&sources); - codegen::assert_ownership_and_focus(); - release_commands::assert_regressions_stay_focused(); - release_integrity::assert_ownership(&sources); - release_integrity::assert_regressions(&sources); - release_status::assert_ownership_and_focus(); -} diff --git a/xtask/tests/repo_lint_support/xtask_main_structure_policy/codegen.rs b/xtask/tests/repo_lint_support/xtask_main_structure_policy/codegen.rs deleted file mode 100644 index 4ed92038..00000000 --- a/xtask/tests/repo_lint_support/xtask_main_structure_policy/codegen.rs +++ /dev/null @@ -1,35 +0,0 @@ -// SPDX-License-Identifier: MIT OR Apache-2.0 - -use super::{assert_module_stays_focused, read}; -use crate::repo_lint_support::{assert_pattern_checks, PatternCheck}; - -pub(super) fn assert_ownership_and_focus() { - let commands = read("xtask/src/codegen_commands.rs"); - let transaction = read("xtask/src/codegen_commands/transaction.rs"); - - assert_module_stays_focused("xtask/src/codegen_commands.rs", &commands, 400); - assert_module_stays_focused( - "xtask/src/codegen_commands/transaction.rs", - &transaction, - 350, - ); - assert_pattern_checks(&[ - PatternCheck::new("xtask codegen command ownership", &commands).required(&[ - "mod transaction;", - "pub(super) fn stable_api(", - "pub(super) fn codec_math_codegen(", - "fn render_codec_math_dwt97_metal_fragment()", - "fn render_stable_api_snapshots()", - "write_generated_pair_transactionally(&snapshots)", - "write_generated_pair_transactionally(&fragments)", - ]), - PatternCheck::new("xtask generated-pair transaction ownership", &transaction).required(&[ - "pub(super) fn write_generated_pair_transactionally(", - "fn stage_generated_entries(", - "fn stage_generated_file(", - "fn restore_originals(", - "pub(super) fn rollback_generated_pair_install(", - "fn sync_generated_directories(", - ]), - ]); -} diff --git a/xtask/tests/repo_lint_support/xtask_main_structure_policy/lint_policy.rs b/xtask/tests/repo_lint_support/xtask_main_structure_policy/lint_policy.rs deleted file mode 100644 index 4ff2d844..00000000 --- a/xtask/tests/repo_lint_support/xtask_main_structure_policy/lint_policy.rs +++ /dev/null @@ -1,27 +0,0 @@ -// SPDX-License-Identifier: MIT OR Apache-2.0 - -use std::fs; - -use super::super::{assert_pattern_checks, repo_root, rust_sources, PatternCheck}; -use super::read; - -#[test] -fn xtask_manifest_keeps_pedantic_clippy_enabled() { - let manifest = read("xtask/Cargo.toml"); - assert_pattern_checks(&[PatternCheck::new("xtask manifest", &manifest) - .normalized_required(&["pedantic = { level = \"warn\", priority = -1 }"]) - .normalized_forbidden(&["pedantic = \"allow\"", "pedantic = { level = \"allow\""])]); - for relative_dir in ["xtask/src", "xtask/tests"] { - for path in rust_sources(&repo_root().join(relative_dir)) { - let source = fs::read_to_string(&path) - .unwrap_or_else(|error| panic!("read {}: {error}", path.display())); - assert!( - source - .lines() - .all(|line| !line.trim_start().starts_with(concat!("#!", "[allow"))), - "{} must not use crate- or module-wide lint allows", - path.strip_prefix(repo_root()).unwrap_or(&path).display() - ); - } - } -} diff --git a/xtask/tests/repo_lint_support/xtask_main_structure_policy/release_commands.rs b/xtask/tests/repo_lint_support/xtask_main_structure_policy/release_commands.rs deleted file mode 100644 index 8777c28e..00000000 --- a/xtask/tests/repo_lint_support/xtask_main_structure_policy/release_commands.rs +++ /dev/null @@ -1,94 +0,0 @@ -// SPDX-License-Identifier: MIT OR Apache-2.0 - -use std::fs; - -use super::super::{assert_pattern_checks, repo_root, PatternCheck}; - -pub(super) fn assert_regressions_stay_focused() { - let root = repo_root(); - let tests = fs::read_to_string(root.join("xtask/src/release_commands/tests.rs")) - .expect("read release command tests"); - let file_boundaries = - fs::read_to_string(root.join("xtask/src/release_commands/tests/file_boundaries.rs")) - .expect("read release file-boundary tests"); - let integrity = fs::read_to_string(root.join("xtask/src/release_commands/tests/integrity.rs")) - .expect("read release integrity command tests"); - let orchestration = - fs::read_to_string(root.join("xtask/src/release_commands/tests/orchestration.rs")) - .expect("read release orchestration command tests"); - let path_patches = - fs::read_to_string(root.join("xtask/src/release_commands/tests/path_patches.rs")) - .expect("read release path-patch command tests"); - let validation = - fs::read_to_string(root.join("xtask/src/release_commands/tests/validation.rs")) - .expect("read release validation command tests"); - - for (relative, source, max_lines) in [ - ("xtask/src/release_commands/tests.rs", &tests, 250), - ( - "xtask/src/release_commands/tests/file_boundaries.rs", - &file_boundaries, - 125, - ), - ( - "xtask/src/release_commands/tests/integrity.rs", - &integrity, - 200, - ), - ( - "xtask/src/release_commands/tests/orchestration.rs", - &orchestration, - 175, - ), - ( - "xtask/src/release_commands/tests/path_patches.rs", - &path_patches, - 125, - ), - ( - "xtask/src/release_commands/tests/validation.rs", - &validation, - 180, - ), - ] { - assert!( - source.lines().count() < max_lines, - "{relative} must stay below its focused {max_lines}-line ownership ratchet" - ); - } - - assert_pattern_checks(&[ - PatternCheck::new("release command test modules", &tests) - .required(&[ - "mod file_boundaries;", - "mod integrity;", - "mod orchestration;", - "mod path_patches;", - "mod validation;", - ]), - PatternCheck::new("release file-boundary regressions", &file_boundaries) - .required(&["missing_release_contract_files_fail_with_path_context"]), - PatternCheck::new("release integrity command regressions", &integrity).required(&[ - "fn release_integrity_accepts_complete_hermetic_metadata_in_pre_candidate_mode()", - "fn release_integrity_aggregates_invalid_package_metadata_without_publishing()", - "fn release_integrity_rejects_non_json_cargo_metadata()", - "fn release_integrity_rejects_publishable_packages_without_versions()", - "run_test_from_workspace(", - ]), - PatternCheck::new("release command orchestration regressions", &orchestration) - .required(&[ - "release_integrity_publish_mode_accepts_hermetic_final_metadata", - "package_command_executes_list_and_dependency_aware_gates_hermetically", - ]), - PatternCheck::new("release path-patch regressions", &path_patches).required(&[ - "path_patch_provenance_discovery_rejects_repository_escape", - "release_integrity_publish_mode_requires_approval_for_every_workspace_path_patch", - ]), - PatternCheck::new("release validation command regressions", &validation).required(&[ - "fn publish_workflow_validation_reports_parse_and_release_contract_failures()", - "fn publish_script_validation_fails_closed_for_missing_and_drifted_contracts()", - "fn release_docs_validation_reports_missing_packages_and_operational_guards()", - "fn unpublished_dependency_validation_skips_external_edges_and_accepts_path_only_dev_edges()", - ]), - ]); -} diff --git a/xtask/tests/repo_lint_support/xtask_main_structure_policy/release_integrity.rs b/xtask/tests/repo_lint_support/xtask_main_structure_policy/release_integrity.rs deleted file mode 100644 index eb50e440..00000000 --- a/xtask/tests/repo_lint_support/xtask_main_structure_policy/release_integrity.rs +++ /dev/null @@ -1,74 +0,0 @@ -// SPDX-License-Identifier: MIT OR Apache-2.0 - -use super::{assert_pattern_checks, PatternCheck, XtaskSources}; - -pub(super) fn assert_ownership(sources: &XtaskSources) { - assert_pattern_checks(&[ - PatternCheck::new( - "release integrity pure policy ownership", - &sources.release_integrity_policy, - ) - .required(&[ - "mod markdown;", - "enum ReleaseIntegrityMode", - "fn validate_changelog_state(", - "fn validate_patch_provenance(", - "fn is_calendar_date(", - "mod tests;", - ]), - PatternCheck::new( - "release integrity Markdown evidence ownership", - &sources.release_integrity_markdown, - ) - .required(&[ - "mod html;", - "fn content_lines(", - "fn non_indented_line(", - "fn opening_fence(", - "fn closes_fence(", - ]), - PatternCheck::new( - "release integrity HTML evidence ownership", - &sources.release_integrity_html, - ) - .required(&[ - "enum HtmlBlock", - "fn opening(", - "fn closes_on(", - "fn starts_open_tag(", - "fn starts_block_tag(", - ]), - ]); -} - -pub(super) fn assert_regressions(sources: &XtaskSources) { - let regressions = [ - sources.release_integrity_changelog_tests.as_str(), - sources.release_integrity_metadata_tests.as_str(), - sources.release_integrity_provenance_tests.as_str(), - sources.release_integrity_provenance_boundary_tests.as_str(), - ] - .join("\n"); - assert_pattern_checks(&[PatternCheck::new( - "release integrity behavior regressions", - ®ressions, - ) - .required(&[ - "pre_candidate_changelog_accepts_exact_unreleased_state", - "publish_changelog_accepts_one_calendar_valid_dated_heading", - "publish_changelog_rejects_provisional_or_duplicate_state", - "fenced_only", - "html_comment_only", - "real_with_fenced_example", - "real_after_div", - "patch_provenance_accepts_structured_reviewer_and_calendar_date", - "patch_provenance_rejects_missing_duplicate_or_placeholder_fields", - "fenced_approval", - "indented_fields", - "html_comment_approval", - "raw_html_approval", - "div_html_approval", - "patch_provenance_requires_one_bounded_release_approval_section", - "patch_provenance_rejects_ambiguous_reviewer_placeholders", - ])]); -} diff --git a/xtask/tests/repo_lint_support/xtask_main_structure_policy/release_status.rs b/xtask/tests/repo_lint_support/xtask_main_structure_policy/release_status.rs deleted file mode 100644 index c6b18e5d..00000000 --- a/xtask/tests/repo_lint_support/xtask_main_structure_policy/release_status.rs +++ /dev/null @@ -1,38 +0,0 @@ -// SPDX-License-Identifier: MIT OR Apache-2.0 - -use super::{assert_module_stays_focused, assert_pattern_checks, read, PatternCheck}; - -pub(super) fn assert_ownership_and_focus() { - let policy = - read("xtask/tests/repo_lint_support/xtask_main_structure_policy/release_status.rs"); - let production = read("xtask/src/release_status.rs"); - let tests = read("xtask/src/release_status/tests.rs"); - let boundary_tests = read("xtask/src/release_status/tests/boundary_errors.rs"); - - assert_module_stays_focused( - "xtask/tests/repo_lint_support/xtask_main_structure_policy/release_status.rs", - &policy, - 75, - ); - assert_module_stays_focused("xtask/src/release_status.rs", &production, 350); - assert_module_stays_focused("xtask/src/release_status/tests.rs", &tests, 225); - assert_module_stays_focused( - "xtask/src/release_status/tests/boundary_errors.rs", - &boundary_tests, - 175, - ); - assert_pattern_checks(&[ - PatternCheck::new("release-status test module ownership", &production) - .required(&["#[cfg(test)]", "mod tests;"]), - PatternCheck::new("release-status command regressions", &tests).required(&[ - "mod boundary_errors;", - "options_reject_missing_values_duplicates_help_and_unknown_arguments", - "repository_components_reject_reserved_and_git_suffixed_names", - "release_status_derives_remote_and_executes_exact_verifier_contract", - ]), - PatternCheck::new("release-status boundary regressions", &boundary_tests).required(&[ - "repository_environment_empty_and_present_paths_execute_exact_contracts", - "repository_discovery_rejects_process_and_unicode_failures", - ]), - ]); -}