Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 4 additions & 0 deletions CHANGELOG.md
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,10 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.

## [Unreleased]

### Fixed

- Make the non-default `html_reports` feature control HTML report generation again.

## [0.8.2](https://github.com/criterion-rs/criterion.rs/compare/criterion-v0.8.1...criterion-v0.8.2) - 2026-02-04

### Fixed
Expand Down
6 changes: 3 additions & 3 deletions Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -36,15 +36,15 @@ exclude = ["book/*"]

[dependencies]
anes = "0.1.4"
criterion-plot = { path = "plot", version = "0.8.2" }
criterion-plot = { path = "plot", version = "0.8.2", optional = true }
itertools = "0.13"
page_size = "0.6"
serde = { version = "1.0.100", features = ["derive"] }
serde_json = "1.0.100"
ciborium = "0.2.0"
clap = { version = "4.5", default-features = false, features = ["std", "help"] }
walkdir = "2.3"
tinytemplate = "1.1"
tinytemplate = { version = "1.1", optional = true }
cast = "0.3"
num-traits = { version = "0.2", default-features = false, features = ["std"] }
oorandom = "11.1"
Expand Down Expand Up @@ -106,7 +106,7 @@ async_smol = ["dep:smol", "async"]
async_tokio = ["dep:tokio", "async"]

# Make Criterion.rs generate its own plots (as opposed to using cargo-criterion)
html_reports = []
html_reports = ["dep:criterion-plot", "dep:tinytemplate"]
Comment thread
WaterWhisperer marked this conversation as resolved.

# Make Criterion.rs usable outside of cargo-criterion.
cargo_bench_support = []
Expand Down
6 changes: 4 additions & 2 deletions book/src/user_guide/html_report.md
Original file line number Diff line number Diff line change
@@ -1,10 +1,12 @@
# HTML Report

Starting with Criterion.rs 0.4.0 HTML reports must be explicitly enabled via the `html_reports` [feature](https://doc.rust-lang.org/cargo/reference/features.html#dependency-features):
HTML reports must be explicitly enabled via the `html_reports` [feature](https://doc.rust-lang.org/cargo/reference/features.html#dependency-features):

```toml
[dev-dependencies]
criterion = { version = "0.5", features = ["html_reports"] }
criterion = { version = "0.8", features = ["html_reports"] }
```

Criterion.rs can generate an HTML report displaying the results of the benchmark under
`target/criterion/reports/index.html`. By default, the plots are generated using
[gnuplot](http://www.gnuplot.info/) if it is available, or the
Expand Down
7 changes: 3 additions & 4 deletions book/src/user_guide/plots_and_graphs.md
Original file line number Diff line number Diff line change
Expand Up @@ -6,12 +6,11 @@ understanding of the behavior of the benchmark. These charts will be generated w
it is not available. The examples below were generated using the gnuplot backend, but the plotters
ones are similar.

Note that in older versions of criterion.rs html reports were enabled by default. Recent versions
have introduced a cargo feature for plot and html generation. In order to activate the html report
generation make sure that your `Cargo.toml` activates the feature:
HTML reports and plot generation are controlled by the non-default `html_reports` feature. Enable it
in your `Cargo.toml` to generate HTML reports:

```toml
criterion = { version = "0.5", features = ["html_reports"] }
criterion = { version = "0.8", features = ["html_reports"] }
```

## File Structure
Expand Down
62 changes: 53 additions & 9 deletions src/lib.rs
Original file line number Diff line number Diff line change
Expand Up @@ -48,10 +48,12 @@ mod error;
mod estimate;
mod format;
mod fs;
#[cfg(feature = "html_reports")]
mod html;
mod kde;
mod macros;
pub mod measurement;
#[cfg(feature = "html_reports")]
mod plot;
pub mod profiler;
mod report;
Expand All @@ -70,19 +72,20 @@ use std::{
time::Duration,
};

use {
criterion_plot::{Version, VersionError},
std::sync::OnceLock,
};
#[cfg(feature = "html_reports")]
use criterion_plot::{Version, VersionError};
use std::sync::OnceLock;

#[cfg(feature = "plotters")]
#[cfg(feature = "html_reports")]
use crate::html::Html;
#[cfg(all(feature = "html_reports", feature = "plotters"))]
use crate::plot::PlottersBackend;
#[cfg(feature = "html_reports")]
use crate::plot::{Gnuplot, Plotter};
use crate::{
benchmark::BenchmarkConfig,
connection::{Connection, OutgoingMessage},
html::Html,
measurement::{Measurement, WallTime},
plot::{Gnuplot, Plotter},
profiler::{ExternalProfiler, Profiler},
report::{BencherReport, CliReport, CliVerbosity, Report, ReportContext, Reports},
};
Expand All @@ -94,12 +97,14 @@ pub use crate::{
benchmark_group::{BenchmarkGroup, BenchmarkId},
};

#[cfg(feature = "html_reports")]
fn gnuplot_version() -> &'static Result<Version, VersionError> {
static GNUPLOT_VERSION: OnceLock<Result<Version, VersionError>> = OnceLock::new();

GNUPLOT_VERSION.get_or_init(criterion_plot::version)
}

#[cfg(feature = "html_reports")]
fn default_plotting_backend() -> &'static PlottingBackend {
static DEFAULT_PLOTTING_BACKEND: OnceLock<PlottingBackend> = OnceLock::new();

Expand Down Expand Up @@ -278,6 +283,7 @@ pub enum PlottingBackend {
/// Null plotting backend which outputs nothing,
None,
}
#[cfg(feature = "html_reports")]
impl PlottingBackend {
fn create_plotter(&self) -> Option<Box<dyn Plotter>> {
match self {
Expand Down Expand Up @@ -405,6 +411,7 @@ impl Default for Criterion {
cli: CliReport::new(false, false, CliVerbosity::Normal),
bencher_enabled: false,
bencher: BencherReport,
#[cfg(feature = "html_reports")]
html: default_plotting_backend().create_plotter().map(Html::new),
csv_enabled: cfg!(feature = "csv_output"),
};
Expand Down Expand Up @@ -442,7 +449,10 @@ impl Default for Criterion {
criterion.report.cli_enabled = false;
criterion.report.bencher_enabled = false;
criterion.report.csv_enabled = false;
criterion.report.html = None;
#[cfg(feature = "html_reports")]
{
criterion.report.html = None;
}
}
criterion
}
Expand Down Expand Up @@ -487,6 +497,7 @@ impl<M: Measurement> Criterion<M> {
/// Panics if `backend` is [`PlottingBackend::Gnuplot`] and `gnuplot` is not available.
///
/// [plotting backend]: PlottingBackend
#[cfg(feature = "html_reports")]
pub fn plotting_backend(mut self, backend: PlottingBackend) -> Criterion<M> {
if let PlottingBackend::Gnuplot = backend {
assert!(
Expand All @@ -501,6 +512,16 @@ impl<M: Measurement> Criterion<M> {
self
}

#[must_use]
/// Set the plotting backend.
///
/// This has no effect when Criterion.rs is built without the `html_reports` feature.
#[cfg(not(feature = "html_reports"))]
pub fn plotting_backend(self, backend: PlottingBackend) -> Criterion<M> {
let _ = backend;
self
}

#[must_use]
/// Changes the default size of the sample for benchmarks run with this runner.
///
Expand Down Expand Up @@ -641,6 +662,7 @@ impl<M: Measurement> Criterion<M> {

#[must_use]
/// Enables plotting
#[cfg(feature = "html_reports")]
pub fn with_plots(mut self) -> Criterion<M> {
// If running under cargo-criterion then don't re-enable the reports; let it do the reporting.
if self.connection.is_none() && self.report.html.is_none() {
Expand All @@ -654,13 +676,32 @@ impl<M: Measurement> Criterion<M> {
self
}

#[must_use]
/// Enables plotting.
///
/// This has no effect when Criterion.rs is built without the `html_reports` feature.
#[cfg(not(feature = "html_reports"))]
pub fn with_plots(self) -> Criterion<M> {
self
}

#[must_use]
/// Disables plotting
#[cfg(feature = "html_reports")]
pub fn without_plots(mut self) -> Criterion<M> {
self.report.html = None;
self
}

#[must_use]
/// Disables plotting.
///
/// This has no effect when Criterion.rs is built without the `html_reports` feature.
#[cfg(not(feature = "html_reports"))]
pub fn without_plots(self) -> Criterion<M> {
self
}

#[must_use]
/// Names an explicit baseline and enables overwriting the previous results.
pub fn save_baseline(mut self, baseline: String) -> Criterion<M> {
Expand Down Expand Up @@ -1057,7 +1098,10 @@ https://criterion-rs.github.io/book/faq.html
self.report.cli_enabled = false;
self.report.bencher_enabled = false;
self.report.csv_enabled = false;
self.report.html = None;
#[cfg(feature = "html_reports")]
{
self.report.html = None;
}
} else {
match matches.get_one("output-format").map(String::as_str) {
Some("bencher") => {
Expand Down
6 changes: 5 additions & 1 deletion src/report.rs
Original file line number Diff line number Diff line change
@@ -1,8 +1,10 @@
#[cfg(feature = "csv_output")]
use crate::csv_report::FileCsvReport;
#[cfg(feature = "html_reports")]
use crate::html::Html;
use crate::stats::bivariate::regression::Slope;
use crate::stats::bivariate::Data;
use crate::stats::univariate::outliers::tukey::LabeledSample;
use crate::{html::Html, stats::bivariate::Data};

use crate::estimate::{ChangeDistributions, ChangeEstimates, Distributions, Estimate, Estimates};
use crate::format;
Expand Down Expand Up @@ -319,6 +321,7 @@ pub(crate) struct Reports {
pub(crate) bencher_enabled: bool,
pub(crate) bencher: BencherReport,
pub(crate) csv_enabled: bool,
#[cfg(feature = "html_reports")]
pub(crate) html: Option<Html>,
}
macro_rules! reports_impl {
Expand All @@ -334,6 +337,7 @@ macro_rules! reports_impl {
if self.csv_enabled {
FileCsvReport.$name($($argn),*);
}
#[cfg(feature = "html_reports")]
if let Some(reporter) = &self.html {
reporter.$name($($argn),*);
}
Expand Down
12 changes: 12 additions & 0 deletions tests/criterion_tests.rs
Original file line number Diff line number Diff line change
Expand Up @@ -137,6 +137,18 @@ fn test_without_plots() {
}
}

#[cfg(all(feature = "plotters", not(feature = "html_reports")))]
Comment thread
WaterWhisperer marked this conversation as resolved.
#[test]
fn test_plotters_without_html_reports_does_not_generate_report() {
let dir = temp_dir();
short_benchmark(&dir).bench_function("test_html_reports_disabled", |b| b.iter(|| 10));

let benchmark_dir = dir.path().join("test_html_reports_disabled");
verify_stats(&benchmark_dir, "new");
verify_stats(&benchmark_dir, "base");
verify_not_exists(&benchmark_dir, "report");
}

#[test]
fn test_save_baseline() {
let dir = temp_dir();
Expand Down