diff --git a/HISTORY.md b/HISTORY.md index 9b42b274d..814d83c73 100644 --- a/HISTORY.md +++ b/HISTORY.md @@ -6,6 +6,7 @@ #### API +- Add `aggregate_cdf` to visualize CDF/CCDF of multiple archives ({pr}`722`) - Support fields as a single str for return_type="dict"|"tuple"|"pandas" in archive.data() ({pr}`721`) - Add `archive_ecdf` to visualize ECDF and ECCDF of archives ({pr}`719`) diff --git a/ribs/visualize/__init__.py b/ribs/visualize/__init__.py index 3103ed083..3a9f62325 100644 --- a/ribs/visualize/__init__.py +++ b/ribs/visualize/__init__.py @@ -14,6 +14,7 @@ .. autosummary:: :toctree: + aggregate_cdf archive_ecdf archive_histogram cvt_archive_3d_plot @@ -26,6 +27,7 @@ qdax_repertoire_heatmap """ +from ribs.visualize._aggregate_cdf import aggregate_cdf from ribs.visualize._archive_ecdf import archive_ecdf from ribs.visualize._archive_histogram import archive_histogram from ribs.visualize._cvt_archive_3d_plot import cvt_archive_3d_plot @@ -42,6 +44,7 @@ ) __all__ = [ + "aggregate_cdf", "archive_ecdf", "archive_histogram", "cvt_archive_3d_plot", diff --git a/ribs/visualize/_aggregate_cdf.py b/ribs/visualize/_aggregate_cdf.py new file mode 100644 index 000000000..3f08e5296 --- /dev/null +++ b/ribs/visualize/_aggregate_cdf.py @@ -0,0 +1,239 @@ +"""Provides aggregate_cdf.""" + +from __future__ import annotations + +from collections.abc import Collection, Sequence +from typing import Literal + +import matplotlib.pyplot as plt +import numpy as np +import scipy.stats +from matplotlib.axes import Axes +from matplotlib.patches import StepPatch +from pandas import DataFrame + +from ribs.archives import ArchiveBase, ArchiveDataFrame +from ribs.visualize._utils import compute_vmin_vmax, validate_df + + +def aggregate_cdf( + archives: Collection[ArchiveBase], + ax: Axes | None = None, + dfs: Collection[DataFrame] | Collection[ArchiveDataFrame] | None = None, + cumulative: bool | Literal[-1] = True, + bins: int | Sequence[float] | str | None = 100, + vmin: float | None = None, + vmax: float | None = None, + estimator: Literal["mean", "median"] = "mean", + errorbar: None | Literal["se", "sd", "iqr"] = "sd", + show_edges: bool = True, +) -> tuple[StepPatch, StepPatch]: + """Plots a CDF/CCDF aggregated over multiple archives. + + Generally, the `CDF (cumulative distribution function) + `_ represents the + number of observations that fall below each output value. In the case of archives, a + CDF counts the number of elites that perform worse than or equal to each objective + value. Conversely, a CCDF (*complementary* cumulative distribution function) counts + the number of elites that perform better than or equal to each objective value. + + This function approximates a CDF/CCDF over multiple archives by first computing a + histogram over the objective values in each archive. Then, it performs a cumulative + sum on the histograms to obtain a CDF/CCDF. Finally, it aggregates the CDFs/CCDFs by + aggregating the number of values in each histogram's bin. For example, if Archive + 0's CDF has 10 values in the bin [0, 0.1), and Archive 1's CDF has 20 values in the + bin [0, 0.1), and we aggregate values using the "mean" estimator, then the final + histogram shows 15 values in the bin [0, 0.1). + + This function also supports plotting histograms by setting the `cumulative` + parameter to False. + + .. info:: + + The idea of using a CDF/CCDF to evaluate QD algorithms was introduced and + formalized in `Vassiliades + 2018 `_. + + Examples: + .. plot:: + :context: close-figs + + import numpy as np + import matplotlib.pyplot as plt + from ribs.archives import GridArchive + from ribs.visualize import aggregate_cdf + + # Populate 5 archives with slightly offset versions of the negative sphere + # function. + archives = [] + for i in range(5): + archive = GridArchive( + solution_dim=2, ranges=[(-1, 1), (-1, 1)], dims=[100, 100] + ) + xxs, yys = np.meshgrid(np.linspace(-1, 1, 100), np.linspace(-1, 1, 100)) + xxs, yys = xxs.ravel(), yys.ravel() + coords = np.stack((xxs, yys), axis=1) + archive.add( + solution=coords, + objective=-(xxs**2 + yys**2) + 0.2 * i, # Negative sphere, with offset. + measures=coords, + ) + archives.append(archive) + + plt.figure(figsize=(8, 6)) + line, _ = aggregate_cdf(archives, cumulative=True) + line.set_label("CDF using Mean and Std") + line, _ = aggregate_cdf(archives, cumulative=True, estimator="median", errorbar="iqr") + line.set_label("CDF using Median and IQR") + plt.title("CDF") + plt.xlabel("Objective Value") + plt.ylabel("Num. Elites") + plt.legend() + + plt.figure(figsize=(8, 6)) + aggregate_cdf(archives, cumulative=-1, estimator="median", errorbar="iqr", vmin=-3, vmax=3) + plt.title("CCDF with Median and IQR, and Using Custom Bounds (vmin/vmax)") + plt.xlabel("Objective Value") + plt.ylabel("Num. Elites") + + plt.figure(figsize=(8, 6)) + aggregate_cdf(archives, cumulative=False) + plt.title("Histogram") + plt.xlabel("Objective Value") + plt.ylabel("Num. Elites") + + plt.figure(figsize=(8, 6)) + aggregate_cdf(archives[:1], cumulative=False) + plt.title("Histogram of Just One Archive") + plt.xlabel("Objective Value") + plt.ylabel("Num. Elites") + + Args: + archives: Archives to aggregate for the CDF/CCDF. + ax: Axes on which to plot the CDF/CCDF. If ``None``, the current axis will be + used. + dfs: If provided, we will plot data from this sequence of dataframes instead of + the data currently in the archives. This data can be obtained by, for + instance, calling :meth:`ribs.archives.ArchiveBase.data` with + ``return_type="pandas"`` and modifying the resulting + :class:`~ribs.archives.ArchiveDataFrame`. Note that, at a minimum, each + dataframe must contain a column for "objective". The number of dataframes + must be the same as the number of archives. + cumulative: Pass True to plot a CDF, -1 to plot a CCDF, and False to plot a + histogram. + bins: Bins for the CDF/CCDF. The default of 100 indicates that there will be 100 + equally-sized bins. + vmin: Minimum objective value to use in the plot. If ``None``, the minimum + objective value across all the archives is used. + vmax: Maximum objective value to use in the plot. If ``None``, the maximum + objective value across all the archives is used. + estimator: Method for aggregating the CDF/CCDF or histogram across the multiple + archives. For example, if "mean" is passed, we count the number of entries + in each histogram bin for each archive, and the final plot shows the mean + number of entries in each bin. + errorbar: Method for computing the errorbar for the CDF/CCDF or histogram. For + example, if "sd" is passed, we display an errorbar showing the standard + deviation of the number of entries in each histogram bin. Options are "sd" + (standard deviation), "se" (standard error of the mean), "iqr" + (interquartile range, i.e., the interval from the 25th to 75th percentile), + and None (no error bar). + show_edges: Whether to show the left and right edges of the CDF/CCDF or + histogram (these show up as vertical lines). + + Returns: + Tuple of two Matplotlib patches. The first patch is for the line, while the + second is for the errorbar. + + Raises: + AttributeError: The data() method is not implemented on one of the archives. + ValueError: Number of dfs passed in is not the same as the number of archives. + """ + if dfs is None: + objectives = [] + for archive in archives: + try: + objectives.append(archive.data("objective")) + except NotImplementedError as e: + raise AttributeError( + "To use aggregate_cdf, each archive must have the data() method." + ) from e + else: + if len(dfs) != len(archives): + raise ValueError( + "If passed in, the number of dfs must equal the number of archives." + ) + objectives = [] + for df in dfs: + df = validate_df(df) + objectives.append(np.asarray(df["objective"])) + + vmin, vmax = compute_vmin_vmax(vmin, vmax, np.concatenate(objectives)) + + # Initialize axis. + ax = plt.gca() if ax is None else ax + + # Compute histogram for each archive. + histograms = [] + for objs in objectives: + hist, bin_edges = np.histogram(objs, bins, range=(vmin, vmax)) + histograms.append(hist) + histograms = np.stack(histograms, axis=0) + + # Apply the cumulative parameter if needed. + if cumulative > 0: + # CDF. + histograms = np.cumsum(histograms, axis=1) + elif cumulative < 0: + # CCDF -- We want to do a cumsum from right to left, so we flip the histograms, + # compute the cumsum from left to right, and then flip back. + histograms = np.flip(np.cumsum(np.flip(histograms, axis=1), axis=1), axis=1) + else: # cumulative is False (i.e., 0). + # Leave the histograms as is. + pass + + # Aggregate values with `estimator`. + if estimator == "mean": + agg_hist = np.mean(histograms, axis=0) + elif estimator == "median": + agg_hist = np.median(histograms, axis=0) + else: + raise ValueError(f"Unknown estimator {estimator}") + + # Compute errors/spread with `errorbar`. + if errorbar is None: + err_low = histograms + err_high = histograms + elif errorbar == "sd": + std_hist = histograms.std(axis=0) + err_low = agg_hist - std_hist + err_high = agg_hist + std_hist + elif errorbar == "se": + sem_hist = scipy.stats.sem(histograms, axis=0) + err_low = agg_hist - sem_hist + err_high = agg_hist + sem_hist + elif errorbar == "iqr": + err_low, err_high = np.percentile(histograms, (25, 75), axis=0) + else: + raise ValueError(f"Unknown errorbar {errorbar}") + + # Plot the line. + line_patch = ax.stairs( + values=agg_hist, + edges=bin_edges, + baseline=0 if show_edges else None, + ) + + # Plot errorbar with same color as the line, but transparent. + if errorbar is not None: + errorbar_patch = ax.stairs( + values=err_high, + edges=bin_edges, + baseline=err_low, + fill=True, + alpha=0.2, + color=line_patch.get_edgecolor(), + ) + else: + errorbar_patch = None + + return line_patch, errorbar_patch diff --git a/ribs/visualize/_archive_histogram.py b/ribs/visualize/_archive_histogram.py index bf47da798..c430221eb 100644 --- a/ribs/visualize/_archive_histogram.py +++ b/ribs/visualize/_archive_histogram.py @@ -35,6 +35,11 @@ def archive_histogram( from the archive and then applies a number of (opinionated) customizations. As such, many of this function's arguments are shared with ``hist``. + .. note:: + This function is intended to plot a single archive, similar to heatmap + functions. To aggregate multiple archives into a CDF/CCDF or histogram, see + :func:`~ribs.visualize.aggregate_cdf`. + Examples: Basic Histogram of a 2D GridArchive diff --git a/tests/visualize/aggregate_cdf_test.py b/tests/visualize/aggregate_cdf_test.py new file mode 100644 index 000000000..f04a5018b --- /dev/null +++ b/tests/visualize/aggregate_cdf_test.py @@ -0,0 +1,338 @@ +"""Tests for aggregate_cdf. + +See README.md for instructions on writing tests. +""" + +import matplotlib.pyplot as plt +import numpy as np +import pytest +from matplotlib.testing.decorators import image_comparison + +from ribs.archives import DensityArchive, GridArchive +from ribs.visualize import aggregate_cdf + +# pylint: disable=redefined-outer-name + + +# +# Fixtures +# + + +@pytest.fixture(scope="module") +def three_archives(): + """Three archives intended to have three bins of objective values.""" + rng = np.random.default_rng(42) + + archives = [] + for obj_count in [ + # Number of items in [0, 1), [1, 2), and [2, 3). + # + # Keep in mind that when plotting a CDF, the CDF does a cumulative sum over the + # number of items in the bins. Hence, there should be a high std in the count in + # [0, 1), then [1, 2) will have lower std (sums are 55, 60, 65), and finally, + # [2, 3) will have zero std (all sums are 100 at that point). + [20, 35, 45], + [30, 30, 40], + [40, 25, 35], + ]: + # Populate the archive with the negative sphere function. + archive = GridArchive(solution_dim=2, dims=[100], ranges=[(0, 1)]) + + objectives = np.concatenate( + ( + 0.0 + rng.uniform(0, 1, size=obj_count[0]), + 1.0 + rng.uniform(0, 1, size=obj_count[1]), + 2.0 + rng.uniform(0, 1, size=obj_count[2]), + ) + ) + + archive.add( + solution=np.zeros((100, 2)), + objective=objectives, + measures=np.arange(0, 1, 0.01)[:, None], + ) + archives.append(archive) + + return archives + + +@pytest.fixture(scope="module") +def three_archives_skewed(): + """Same as above, but the distribution for each range is skewed.""" + rng = np.random.default_rng(42) + + archives = [] + for obj_count in [ + # Number of items in [0, 1), [1, 2), and [2, 3). + [30, 20, 50], + [30, 20, 50], + [50, 35, 15], + # Skewed -- e.g., [30, 30, 50] for [0, 1) has mean of 36.67 but median of 30. + ]: + # Populate the archive with the negative sphere function. + archive = GridArchive(solution_dim=2, dims=[100], ranges=[(0, 1)]) + + objectives = np.concatenate( + ( + 0.0 + rng.uniform(0, 1, size=obj_count[0]), + 1.0 + rng.uniform(0, 1, size=obj_count[1]), + 2.0 + rng.uniform(0, 1, size=obj_count[2]), + ) + ) + + archive.add( + solution=np.zeros((100, 2)), + objective=objectives, + measures=np.arange(0, 1, 0.01)[:, None], + ) + archives.append(archive) + + return archives + + +# +# Tests +# + + +def test_no_data_method_available(): + with pytest.raises( + AttributeError, + match=r"To use aggregate_cdf, each archive must have the data\(\) method\.", + ): + aggregate_cdf([DensityArchive(measure_dim=2) for _ in range(3)]) + + +@image_comparison( + baseline_images=["basic_cdf"], + remove_text=False, + extensions=["png"], + style="mpl20", +) +def test_basic_cdf(three_archives): + """Basic CDF. + + The line should be the mean, which is 30, 60, 100. + + The errorbar should be the std, which is std(20, 30, 40) ~= 8.16, std(55, 60, 65) ~= + 4.08, std(100, 100, 100) = 0. + """ + plt.figure(figsize=(8, 6)) + aggregate_cdf(three_archives, bins=3, cumulative=True) + + +@image_comparison( + baseline_images=["basic_cdf"], + remove_text=False, + extensions=["png"], + style="mpl20", +) +def test_basic_cdf_with_dfs(three_archives): + plt.figure(figsize=(8, 6)) + aggregate_cdf( + three_archives, + dfs=[ + archive.data("objective", return_type="pandas") + for archive in three_archives + ], + bins=3, + cumulative=True, + ) + + +@image_comparison( + baseline_images=["cdf_with_labels"], + remove_text=False, + extensions=["png"], + style="mpl20", +) +def test_cdf_with_labels(three_archives): + """Take advantage of the patches returned from the function.""" + plt.figure(figsize=(8, 6)) + line, errorbar = aggregate_cdf( + three_archives, + bins=3, + cumulative=True, + ) + line.set_label("Mean") + errorbar.set_label("Error Bar") + plt.legend() + + +def test_wrong_num_dfs(three_archives): + plt.figure(figsize=(8, 6)) + with pytest.raises( + ValueError, + match=r"If passed in, the number of dfs must equal the number of archives\.", + ): + aggregate_cdf( + three_archives, + # Only provide dfs for two archives. + dfs=[ + archive.data("objective", return_type="pandas") + for archive in three_archives[:2] + ], + ) + + +@image_comparison( + baseline_images=["basic_ccdf"], + remove_text=False, + extensions=["png"], + style="mpl20", +) +def test_basic_ccdf(three_archives): + """Basic CCDF. + + The line should be the mean, which is 100, 70, 40. + + The errorbar should be the std, which is std(100, 100, 100) = 0, std(80, 70, 60) ~= + 8.16, std(45, 40, 35) ~= 4.08. + """ + plt.figure(figsize=(8, 6)) + aggregate_cdf(three_archives, bins=3, cumulative=-1) + + +@image_comparison( + baseline_images=["basic_histogram"], + remove_text=False, + extensions=["png"], + style="mpl20", +) +def test_basic_histogram(three_archives): + """Basic histogram. + + The line should be the mean, which is 30, 30, 40. + + The errorbar should be the std, which is std(20, 30, 40) = 8.16, std(35, 30, 25) ~= + 4.08, std(45, 40, 35) ~= 4.08. + """ + plt.figure(figsize=(8, 6)) + aggregate_cdf(three_archives, bins=3, cumulative=False) + + +@image_comparison( + baseline_images=["vmin_vmax"], + remove_text=False, + extensions=["png"], + style="mpl20", +) +def test_vmin_vmax(three_archives): + """CDF with vmin and vmax. + + On average, the total number of items in the range [1, 2) for the archives is 30, so + the bin on the right should have 30 items. + """ + plt.figure(figsize=(8, 6)) + aggregate_cdf(three_archives, bins=2, vmin=1.0, vmax=2.0) + + +@image_comparison( + baseline_images=["errorbar_se"], + remove_text=False, + extensions=["png"], + style="mpl20", +) +def test_errorbar_se(three_archives): + """errorbar should be a bit smaller than in test_basic_cdf.""" + plt.figure(figsize=(8, 6)) + aggregate_cdf(three_archives, bins=3, cumulative=True, errorbar="se") + + +@image_comparison( + baseline_images=["errorbar_none"], + remove_text=False, + extensions=["png"], + style="mpl20", +) +def test_errorbar_none(three_archives): + """errorbar should be a lot smaller than in test_basic_cdf.""" + plt.figure(figsize=(8, 6)) + aggregate_cdf(three_archives, bins=3, cumulative=True, errorbar=None) + + +@image_comparison( + baseline_images=["median_with_iqr_cdf"], + remove_text=False, + extensions=["png"], + style="mpl20", +) +def test_median_with_iqr_cdf(three_archives_skewed): + plt.figure(figsize=(8, 6)) + aggregate_cdf( + three_archives_skewed, + bins=3, + estimator="median", + errorbar="iqr", + cumulative=True, + ) + + +@image_comparison( + baseline_images=["median_with_iqr_hist"], + remove_text=False, + extensions=["png"], + style="mpl20", +) +def test_median_with_iqr_hist(three_archives_skewed): + plt.figure(figsize=(8, 6)) + aggregate_cdf( + three_archives_skewed, + bins=3, + estimator="median", + errorbar="iqr", + cumulative=False, + ) + + +@image_comparison( + baseline_images=["no_edges"], + remove_text=False, + extensions=["png"], + style="mpl20", +) +def test_no_edges(three_archives): + plt.figure(figsize=(8, 6)) + aggregate_cdf(three_archives, bins=3, cumulative=True, show_edges=False) + + +@image_comparison( + baseline_images=[ + "full_scale_single_hist", + "full_scale_hist", + "full_scale_cdf_mean", + "full_scale_cdf_median", + ], + remove_text=False, + extensions=["png"], + style="mpl20", +) +def test_full_scale(): + """Larger-scale test.""" + archives = [] + for i in range(5): + archive = GridArchive( + solution_dim=2, ranges=[(-1, 1), (-1, 1)], dims=[100, 100] + ) + xxs, yys = np.meshgrid(np.linspace(-1, 1, 100), np.linspace(-1, 1, 100)) + xxs, yys = xxs.ravel(), yys.ravel() + coords = np.stack((xxs, yys), axis=1) + archive.add( + solution=coords, + objective=-(xxs**2 + yys**2) + i, # Negative sphere, offset by i. + measures=coords, + ) + archives.append(archive) + + plt.figure(figsize=(8, 6)) + aggregate_cdf(archives[:1], cumulative=False) + + plt.figure(figsize=(8, 6)) + aggregate_cdf(archives, cumulative=False) + + plt.figure(figsize=(8, 6)) + aggregate_cdf(archives, cumulative=True) + + plt.figure(figsize=(8, 6)) + aggregate_cdf(archives, cumulative=True, estimator="median", errorbar="iqr") diff --git a/tests/visualize/baseline_images/aggregate_cdf_test/basic_ccdf.png b/tests/visualize/baseline_images/aggregate_cdf_test/basic_ccdf.png new file mode 100644 index 000000000..948054d9a Binary files /dev/null and b/tests/visualize/baseline_images/aggregate_cdf_test/basic_ccdf.png differ diff --git a/tests/visualize/baseline_images/aggregate_cdf_test/basic_cdf.png b/tests/visualize/baseline_images/aggregate_cdf_test/basic_cdf.png new file mode 100644 index 000000000..ebe2c7e00 Binary files /dev/null and b/tests/visualize/baseline_images/aggregate_cdf_test/basic_cdf.png differ diff --git a/tests/visualize/baseline_images/aggregate_cdf_test/basic_histogram.png b/tests/visualize/baseline_images/aggregate_cdf_test/basic_histogram.png new file mode 100644 index 000000000..f3e64fca9 Binary files /dev/null and b/tests/visualize/baseline_images/aggregate_cdf_test/basic_histogram.png differ diff --git a/tests/visualize/baseline_images/aggregate_cdf_test/cdf_with_labels.png b/tests/visualize/baseline_images/aggregate_cdf_test/cdf_with_labels.png new file mode 100644 index 000000000..c55c44164 Binary files /dev/null and b/tests/visualize/baseline_images/aggregate_cdf_test/cdf_with_labels.png differ diff --git a/tests/visualize/baseline_images/aggregate_cdf_test/errorbar_none.png b/tests/visualize/baseline_images/aggregate_cdf_test/errorbar_none.png new file mode 100644 index 000000000..372782613 Binary files /dev/null and b/tests/visualize/baseline_images/aggregate_cdf_test/errorbar_none.png differ diff --git a/tests/visualize/baseline_images/aggregate_cdf_test/errorbar_se.png b/tests/visualize/baseline_images/aggregate_cdf_test/errorbar_se.png new file mode 100644 index 000000000..4c051a8f0 Binary files /dev/null and b/tests/visualize/baseline_images/aggregate_cdf_test/errorbar_se.png differ diff --git a/tests/visualize/baseline_images/aggregate_cdf_test/full_scale_cdf_mean.png b/tests/visualize/baseline_images/aggregate_cdf_test/full_scale_cdf_mean.png new file mode 100644 index 000000000..d7a36578d Binary files /dev/null and b/tests/visualize/baseline_images/aggregate_cdf_test/full_scale_cdf_mean.png differ diff --git a/tests/visualize/baseline_images/aggregate_cdf_test/full_scale_cdf_median.png b/tests/visualize/baseline_images/aggregate_cdf_test/full_scale_cdf_median.png new file mode 100644 index 000000000..6aa58c8f4 Binary files /dev/null and b/tests/visualize/baseline_images/aggregate_cdf_test/full_scale_cdf_median.png differ diff --git a/tests/visualize/baseline_images/aggregate_cdf_test/full_scale_hist.png b/tests/visualize/baseline_images/aggregate_cdf_test/full_scale_hist.png new file mode 100644 index 000000000..0b2380c47 Binary files /dev/null and b/tests/visualize/baseline_images/aggregate_cdf_test/full_scale_hist.png differ diff --git a/tests/visualize/baseline_images/aggregate_cdf_test/full_scale_single_hist.png b/tests/visualize/baseline_images/aggregate_cdf_test/full_scale_single_hist.png new file mode 100644 index 000000000..50d1011c8 Binary files /dev/null and b/tests/visualize/baseline_images/aggregate_cdf_test/full_scale_single_hist.png differ diff --git a/tests/visualize/baseline_images/aggregate_cdf_test/median_with_iqr_cdf.png b/tests/visualize/baseline_images/aggregate_cdf_test/median_with_iqr_cdf.png new file mode 100644 index 000000000..01f907502 Binary files /dev/null and b/tests/visualize/baseline_images/aggregate_cdf_test/median_with_iqr_cdf.png differ diff --git a/tests/visualize/baseline_images/aggregate_cdf_test/median_with_iqr_hist.png b/tests/visualize/baseline_images/aggregate_cdf_test/median_with_iqr_hist.png new file mode 100644 index 000000000..2f78aa55a Binary files /dev/null and b/tests/visualize/baseline_images/aggregate_cdf_test/median_with_iqr_hist.png differ diff --git a/tests/visualize/baseline_images/aggregate_cdf_test/no_edges.png b/tests/visualize/baseline_images/aggregate_cdf_test/no_edges.png new file mode 100644 index 000000000..56d229824 Binary files /dev/null and b/tests/visualize/baseline_images/aggregate_cdf_test/no_edges.png differ diff --git a/tests/visualize/baseline_images/aggregate_cdf_test/vmin_vmax.png b/tests/visualize/baseline_images/aggregate_cdf_test/vmin_vmax.png new file mode 100644 index 000000000..c5fff52cf Binary files /dev/null and b/tests/visualize/baseline_images/aggregate_cdf_test/vmin_vmax.png differ