Skip to content

More extensive tests #874

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Draft
wants to merge 1 commit into
base: master
Choose a base branch
from
Draft

More extensive tests #874

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion libm-test/benches/random.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -57,7 +57,7 @@ where
let name = Op::NAME;

let ctx = CheckCtx::new(Op::IDENTIFIER, CheckBasis::Musl, GeneratorKind::Random);
let benchvec: Vec<_> = random::get_test_cases::<Op::RustArgs>(&ctx)
let benchvec: Vec<_> = random::get_test_cases::<Op>(ctx.clone())
.0
.take(BENCH_ITER_ITEMS)
.collect();
Expand Down
4 changes: 2 additions & 2 deletions libm-test/examples/plot_domains.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -61,15 +61,15 @@ where
&ctx,
"logspace",
config,
spaced::get_test_cases::<Op>(&ctx).0,
spaced::get_test_cases::<Op>(ctx.clone()).0,
);
ctx.gen_kind = GeneratorKind::EdgeCases;
plot_one_generator(
out_dir,
&ctx,
"edge_cases",
config,
edge_cases::get_test_cases::<Op>(&ctx).0,
edge_cases::get_test_cases::<Op>(ctx.clone()).0,
);
}

Expand Down
42 changes: 26 additions & 16 deletions libm-test/src/run_cfg.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -24,9 +24,9 @@ static EXTENSIVE_ITER_OVERRIDE: LazyLock<Option<u64>> = LazyLock::new(|| {
/// amount of time.
///
/// Contains the itentifier+generator combo to match on, plus the factor to reduce by.
const EXTEMELY_SLOW_TESTS: &[(Identifier, GeneratorKind, u64)] = &[
(Identifier::Fmodf128, GeneratorKind::QuickSpaced, 50),
(Identifier::Fmodf128, GeneratorKind::Extensive, 50),
const EXTEMELY_SLOW_TESTS: &[(Identifier, GeneratorKind, bool, u64)] = &[
(Identifier::Fmodf128, GeneratorKind::QuickSpaced, false, 50),
(Identifier::Fmodf128, GeneratorKind::Extensive, true, 50),
];

/// Maximum number of iterations to run for a single routine.
Expand Down Expand Up @@ -54,6 +54,7 @@ pub struct CheckCtx {
/// Source of truth for tests.
pub basis: CheckBasis,
pub gen_kind: GeneratorKind,
pub extensive: bool,
/// If specified, this value will override the value returned by [`iteration_count`].
pub override_iterations: Option<u64>,
}
Expand All @@ -69,12 +70,19 @@ impl CheckCtx {
base_name_str: fn_ident.base_name().as_str(),
basis,
gen_kind,
extensive: false,
override_iterations: None,
};
ret.ulp = crate::default_ulp(&ret);
ret
}

/// Configure that this is an extensive test.
pub fn extensive(mut self, extensive: bool) -> Self {
self.extensive = extensive;
self
}

/// The number of input arguments for this function.
pub fn input_count(&self) -> usize {
self.fn_ident.math_op().rust_sig.args.len()
Expand All @@ -101,8 +109,7 @@ pub enum CheckBasis {
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
pub enum GeneratorKind {
EdgeCases,
Extensive,
QuickSpaced,
Spaced,
Random,
List,
}
Expand Down Expand Up @@ -216,17 +223,17 @@ pub fn iteration_count(ctx: &CheckCtx, argnum: usize) -> u64 {
let random_iter_count = domain_iter_count / 100;

let mut total_iterations = match ctx.gen_kind {
GeneratorKind::QuickSpaced => domain_iter_count,
GeneratorKind::Spaced if ctx.extensive => extensive_max_iterations(),
GeneratorKind::Spaced => domain_iter_count,
GeneratorKind::Random => random_iter_count,
GeneratorKind::Extensive => extensive_max_iterations(),
GeneratorKind::EdgeCases | GeneratorKind::List => {
unimplemented!("shoudn't need `iteration_count` for {:?}", ctx.gen_kind)
}
};

// Larger float types get more iterations.
if t_env.large_float_ty && ctx.gen_kind != GeneratorKind::Extensive {
if ctx.gen_kind == GeneratorKind::Extensive {
if t_env.large_float_ty {
if ctx.extensive {
// Extensive already has a pretty high test count.
total_iterations *= 2;
} else {
Expand All @@ -244,12 +251,15 @@ pub fn iteration_count(ctx: &CheckCtx, argnum: usize) -> u64 {
}

// Some tests are significantly slower than others and need to be further reduced.
if let Some((_id, _gen, scale)) = EXTEMELY_SLOW_TESTS
.iter()
.find(|(id, generator, _scale)| *id == ctx.fn_ident && *generator == ctx.gen_kind)
if let Some((_id, _generator, _extensive, scale)) =
EXTEMELY_SLOW_TESTS
.iter()
.find(|(id, generator, extensive, _scale)| {
*id == ctx.fn_ident && *generator == ctx.gen_kind && *extensive == ctx.extensive
})
{
// However, do not override if the extensive iteration count has been manually set.
if !(ctx.gen_kind == GeneratorKind::Extensive && EXTENSIVE_ITER_OVERRIDE.is_some()) {
if !(ctx.extensive && EXTENSIVE_ITER_OVERRIDE.is_some()) {
total_iterations /= scale;
}
}
Expand Down Expand Up @@ -279,7 +289,7 @@ pub fn iteration_count(ctx: &CheckCtx, argnum: usize) -> u64 {
let total = ntests.pow(t_env.input_count.try_into().unwrap());

let seed_msg = match ctx.gen_kind {
GeneratorKind::QuickSpaced | GeneratorKind::Extensive => String::new(),
GeneratorKind::Spaced => String::new(),
GeneratorKind::Random => {
format!(
" using `{SEED_ENV}={}`",
Expand Down Expand Up @@ -327,8 +337,8 @@ pub fn int_range(ctx: &CheckCtx, argnum: usize) -> RangeInclusive<i32> {
let extensive_range = (-0xfff)..=0xfffff;

match ctx.gen_kind {
GeneratorKind::Extensive => extensive_range,
GeneratorKind::QuickSpaced | GeneratorKind::Random => non_extensive_range,
_ if ctx.extensive => extensive_range,
GeneratorKind::Spaced | GeneratorKind::Random => non_extensive_range,
GeneratorKind::EdgeCases => extensive_range,
GeneratorKind::List => unimplemented!("shoudn't need range for {:?}", ctx.gen_kind),
}
Expand Down
10 changes: 5 additions & 5 deletions libm-test/tests/compare_built_musl.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -48,7 +48,7 @@ macro_rules! musl_tests {
fn [< musl_random_ $fn_name >]() {
type Op = libm_test::op::$fn_name::Routine;
let ctx = CheckCtx::new(Op::IDENTIFIER, BASIS, GeneratorKind::Random);
let cases = random::get_test_cases::<<Op as MathOp>::RustArgs>(&ctx).0;
let cases = random::get_test_cases::<Op>(ctx.clone()).0;
musl_runner::<Op>(&ctx, cases, musl_math_sys::$fn_name);
}

Expand All @@ -57,16 +57,16 @@ macro_rules! musl_tests {
fn [< musl_edge_case_ $fn_name >]() {
type Op = libm_test::op::$fn_name::Routine;
let ctx = CheckCtx::new(Op::IDENTIFIER, BASIS, GeneratorKind::EdgeCases);
let cases = edge_cases::get_test_cases::<Op>(&ctx).0;
let cases = edge_cases::get_test_cases::<Op>(ctx.clone()).0;
musl_runner::<Op>(&ctx, cases, musl_math_sys::$fn_name);
}

#[test]
$(#[$attr])*
fn [< musl_quickspace_ $fn_name >]() {
fn [< musl_logspace_ $fn_name >]() {
type Op = libm_test::op::$fn_name::Routine;
let ctx = CheckCtx::new(Op::IDENTIFIER, BASIS, GeneratorKind::QuickSpaced);
let cases = spaced::get_test_cases::<Op>(&ctx).0;
let ctx = CheckCtx::new(Op::IDENTIFIER, BASIS, GeneratorKind::Spaced);
let cases = spaced::get_test_cases::<Op>(ctx.clone()).0;
musl_runner::<Op>(&ctx, cases, musl_math_sys::$fn_name);
}
}
Expand Down
10 changes: 5 additions & 5 deletions libm-test/tests/multiprecision.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -38,7 +38,7 @@ macro_rules! mp_tests {
fn [< mp_random_ $fn_name >]() {
type Op = libm_test::op::$fn_name::Routine;
let ctx = CheckCtx::new(Op::IDENTIFIER, BASIS, GeneratorKind::Random);
let cases = random::get_test_cases::<<Op as MathOp>::RustArgs>(&ctx).0;
let cases = random::get_test_cases::<Op>(ctx.clone()).0;
mp_runner::<Op>(&ctx, cases);
}

Expand All @@ -47,16 +47,16 @@ macro_rules! mp_tests {
fn [< mp_edge_case_ $fn_name >]() {
type Op = libm_test::op::$fn_name::Routine;
let ctx = CheckCtx::new(Op::IDENTIFIER, BASIS, GeneratorKind::EdgeCases);
let cases = edge_cases::get_test_cases::<Op>(&ctx).0;
let cases = edge_cases::get_test_cases::<Op>(ctx.clone()).0;
mp_runner::<Op>(&ctx, cases);
}

#[test]
$(#[$attr])*
fn [< mp_quickspace_ $fn_name >]() {
fn [< mp_logspace_ $fn_name >]() {
type Op = libm_test::op::$fn_name::Routine;
let ctx = CheckCtx::new(Op::IDENTIFIER, BASIS, GeneratorKind::QuickSpaced);
let cases = spaced::get_test_cases::<Op>(&ctx).0;
let ctx = CheckCtx::new(Op::IDENTIFIER, BASIS, GeneratorKind::Spaced);
let cases = spaced::get_test_cases::<Op>(ctx.clone()).0;
mp_runner::<Op>(&ctx, cases);
}
}
Expand Down
58 changes: 47 additions & 11 deletions libm-test/tests/z_extensive/run.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,9 @@ use std::sync::atomic::{AtomicU64, Ordering};
use std::time::Duration;

use indicatif::{ProgressBar, ProgressStyle};
use libm_test::generate::spaced;
use libm_test::generate::edge_cases::EdgeCaseInput;
use libm_test::generate::random::RandomInput;
use libm_test::generate::{edge_cases, random, spaced};
use libm_test::mpfloat::MpOp;
use libm_test::{
CheckBasis, CheckCtx, CheckOutput, GeneratorKind, MathOp, TestResult, TupleCall,
Expand All @@ -17,7 +19,6 @@ use rayon::prelude::*;
use spaced::SpacedInput;

const BASIS: CheckBasis = CheckBasis::Mpfr;
const GEN_KIND: GeneratorKind = GeneratorKind::Extensive;

/// Run the extensive test suite.
pub fn run() {
Expand Down Expand Up @@ -48,7 +49,7 @@ macro_rules! mp_extensive_tests {
extra: [$push_to:ident],
) => {
$(#[$attr])*
register_single_test::<libm_test::op::$fn_name::Routine>(&mut $push_to);
register_single_op::<libm_test::op::$fn_name::Routine>(&mut $push_to);
};
}

Expand All @@ -71,21 +72,51 @@ fn register_all_tests() -> Vec<Trial> {
}

/// Add a single test to the list.
fn register_single_test<Op>(all: &mut Vec<Trial>)
fn register_single_op<Op>(all: &mut Vec<Trial>)
where
Op: MathOp + MpOp + 'static,
Op::RustArgs: SpacedInput<Op> + RandomInput + EdgeCaseInput<Op> + Send,
{
// All generators need to take `ctx` by value to avoid
// https://github.com/rust-lang/rust/issues/42940

let ctx = CheckCtx::new(Op::IDENTIFIER, BASIS, GeneratorKind::EdgeCases).extensive(true);
let (cases, total) = edge_cases::get_test_cases::<Op>(ctx.clone());
register_single_test::<Op>(all, ctx, cases, total);

let ctx = CheckCtx::new(Op::IDENTIFIER, BASIS, GeneratorKind::Spaced).extensive(true);
let (cases, total) = spaced::get_test_cases::<Op>(ctx.clone());
register_single_test::<Op>(all, ctx, cases, total);

let ctx = CheckCtx::new(Op::IDENTIFIER, BASIS, GeneratorKind::Random).extensive(true);
let (cases, total) = random::get_test_cases::<Op>(ctx.clone());
register_single_test::<Op>(all, ctx, cases, total);
}

fn register_single_test<Op>(
all: &mut Vec<Trial>,
ctx: CheckCtx,
cases: impl Iterator<Item = Op::RustArgs> + Send + 'static,
total: u64,
) where
Op: MathOp + MpOp,
Op::RustArgs: SpacedInput<Op> + Send,
Op::RustArgs: SpacedInput<Op> + RandomInput + EdgeCaseInput<Op> + Send,
{
let test_name = format!("mp_extensive_{}", Op::NAME);
let ctx = CheckCtx::new(Op::IDENTIFIER, BASIS, GEN_KIND);
let x = match ctx.gen_kind {
GeneratorKind::EdgeCases => "edge_cases",
GeneratorKind::Spaced => "logspace",
GeneratorKind::Random => "random",
};

let test_name = format!("mp_extensive_{x}_{}", Op::NAME);
let skip = skip_extensive_test(&ctx);

let runner = move || {
if !cfg!(optimizations_enabled) {
panic!("extensive tests should be run with --release");
}

let res = run_single_test::<Op>(&ctx);
let res = run_single_test::<Op>(&ctx, cases, total);
let e = match res {
Ok(()) => return Ok(()),
Err(e) => e,
Expand All @@ -103,7 +134,11 @@ where
}

/// Test runner for a signle routine.
fn run_single_test<Op>(ctx: &CheckCtx) -> TestResult
fn run_single_test<Op>(
ctx: &CheckCtx,
mut cases: impl Iterator<Item = Op::RustArgs> + Send,
total: u64,
) -> TestResult
where
Op: MathOp + MpOp,
Op::RustArgs: SpacedInput<Op> + Send,
Expand All @@ -113,7 +148,8 @@ where
eprintln!();

let completed = AtomicU64::new(0);
let (ref mut cases, total) = spaced::get_test_cases::<Op>(ctx);
// let ref mut cases = cases;
// let (ref mut cases, total) = spaced::get_test_cases::<Op>(ctx);
let pb = Progress::new(Op::NAME, total);

let test_single_chunk = |mp_vals: &mut Op::MpTy, input_vec: Vec<Op::RustArgs>| -> TestResult {
Expand All @@ -136,7 +172,7 @@ where
let chunk_size = 50_000;
let chunks = std::iter::from_fn(move || {
let mut v = Vec::with_capacity(chunk_size);
v.extend(cases.take(chunk_size));
v.extend((&mut cases).take(chunk_size));
(!v.is_empty()).then_some(v)
});

Expand Down
Loading