diff --git a/Project.toml b/Project.toml index 9cde33f9..d7d29431 100644 --- a/Project.toml +++ b/Project.toml @@ -6,6 +6,7 @@ version = "0.5.5" BenchmarkProfiles = "ecbce9bc-3e5e-569d-9e29-55181f61f8d0" BenchmarkTools = "6e4b80f9-dd63-53aa-95a3-0cdb28fa8baf" ColorSchemes = "35d6a980-a343-548e-a6ea-1d62b119f2f4" +CSV = "336ed68f-0bac-5ca0-87d4-7b16caf5d00b" DataFrames = "a93c6f00-e57d-5684-b7b6-d8193f3e46c0" GitHub = "bc5e4493-9b4d-5f90-b8aa-2b2bcaad7a26" JLD2 = "033835bb-8acc-5ee8-8aae-3f567f8a3819" @@ -29,6 +30,7 @@ UnicodePlots = "b8865327-cd53-5732-bb35-84acbb429228" BenchmarkProfiles = "0.4.2" BenchmarkTools = "^0.4.2, 0.5, 0.6, 0.7, 1" ColorSchemes = "^3.9" +CSV = "0.10" DataFrames = "^0.21, 1" GitHub = "^5.0.2" JLD2 = "0.1.12, 0.2, 0.3, 0.4" diff --git a/src/profiles.jl b/src/profiles.jl index dd9d2c80..5c4cc0f5 100644 --- a/src/profiles.jl +++ b/src/profiles.jl @@ -1,7 +1,7 @@ import BenchmarkProfiles: performance_profile -using BenchmarkProfiles, Plots +using BenchmarkProfiles, Plots, CSV -export performance_profile, profile_solvers +export performance_profile, profile_solvers, export_profile_solvers_data """ performance_profile(stats, cost, args...; b = PlotsBackend(), kwargs...) @@ -137,3 +137,129 @@ function profile_solvers( end p end + +""" + get_profile_solvers_data(stats, costs; kwargs...) + +Exports performance profiles plot data comparing `solvers` based on the data in `stats` in a .csv file. + +Inputs: +- `stats::Dict{Symbol,DataFrame}`: a dictionary of `DataFrame`s containing the + benchmark results per solver (e.g., produced by `bmark_results_to_dataframes()`) +- `costs::Vector{Function}`: a vector of functions specifying the measures to use in the profiles + +Keyword arguments are passed to `BenchmarkProfiles.performance_profile_data()`. + +Output: +x_mat, y_mat: vectors which elements are matrices containing the x and y coordinate of the plots. Each matrix correspond to a cost, matrices columns correspond to solvers. +Matrices are padded with NaN if necessary (happens if plots do not have the same number of points). +""" +function get_profile_solvers_data( + stats::Dict{Symbol, DataFrame}, + costs::Vector{<:Function}; + kwargs... + ) + + solvers = collect(keys(stats)) + dfs = (stats[solver] for solver in solvers) + Ps = [hcat([Float64.(cost(df)) for df in dfs]...) for cost in costs] + + nprobs = size(stats[first(solvers)], 1) + nsolvers = length(solvers) + ncosts = length(costs) + npairs = div(nsolvers * (nsolvers - 1), 2) + x_data, y_data = performance_profile_data(Ps[1]; kwargs...) + nmaxrow = maximum(length.(x_data)) + for i in eachindex(x_data) + append!(x_data[i],[NaN for i=1:nprobs-length(x_data[i])]) + append!(y_data[i],[NaN for i=1:nprobs-length(y_data[i])]) + end + x_mat = [hcat(x_data...)] + y_mat = [hcat(y_data...)] + for k in 2:ncosts + x_data, y_data = performance_profile_data(Ps[k];kwargs...) + nmaxrow = max(nmaxrow,maximum(length.(x_data))) + for i in eachindex(x_data) + append!(x_data[i],[NaN for i=1:nprobs-length(x_data[i])]) + append!(y_data[i],[NaN for i=1:nprobs-length(y_data[i])]) + end + push!(x_mat, hcat(x_data...)) + push!(y_mat, hcat(y_data...)) + end + return [m[1:nmaxrow,:] for m in x_mat], [m[1:nmaxrow,:] for m in y_mat] +end + +""" + export_profile_solvers_data(stats, costs, costnames, filename; one_file=true, two_by_two=false, kwargs...) + +Exports performance profiles plot data comparing `solvers` based on the data in `stats` in a .csv file. +Data are padded with NaN to ensure .csv consistency. + +Inputs: +- `stats::Dict{Symbol,DataFrame}`: a dictionary of `DataFrame`s containing the + benchmark results per solver (e.g., produced by `bmark_results_to_dataframes()`) +- `costs::Vector{Function}`: a vector of functions specifying the measures to use in the profiles +- `costnames::Vector{String}`: names to be used as titles of the profiles. +- `filename::String`: path to the export file without the .csv extention. + +Keyword arguments: +- `one_file::Bool`: export one file per cost if false, otherwise profiles for all costs are exported in a single file +- `header::Vector{Vector{String}}`: Contains .csv file(s) column names for each files. Example for two costs exported in two files and two solvers "alpha" and "beta": `[ ["alpha_x","alpha_y","beta_x","beta_y"] for _=1:2]`. Note that `header` value does not change columns order in .csv exported files (see Output). + +Additional `kwargs` are passed to `BenchmarkProfiles.performance_profile_data()`. + +Output: +File(s) containing profile data in .csv format. +* If one_file=true, returns one file containing the data for all solvers and cost. + Columns are cost1_solver1_x, cost1_solver1_y, cost1_solver2_x, ... cost2_solver1_x, cost2_solver1_y, ... +* If one_file=false, returns as many files as the number of cost. + The names of the files contain the name of the cost, and the columns are + solver1_x, solver1_y, solver2_x, ... +""" +function export_profile_solvers_data( + stats::Dict{Symbol, DataFrame}, + costs::Vector{<:Function}, + costnames::S, + filename::String; + header = [], + one_file=true, + kwargs... + ) where {S <: Vector{String}} + solvers = collect(keys(stats)) + nprobs = size(stats[first(solvers)], 1) + nsolvers = length(solvers) + solver_names = String.(keys(stats)) + csv_header = Vector{String}[] + + x_mat, y_mat = get_profile_solvers_data(stats,costs;kwargs...) + if one_file + if isempty(header) + csv_header = vcat([vcat([[cname*"_"*sname*"_x",cname*"_"*sname*"_y"] for sname in solver_names]...) for cname in costnames]...) + else + csv_header = vcat(header...) + end + x_mat = hcat(x_mat...) + y_mat = hcat(y_mat...) + ncol = size(x_mat)[2] + nrow = size(x_mat)[1] + data = Matrix{Float64}(undef,nrow,ncol*2) + for i =0:ncol-1 + data[:,2*i+1] .= x_mat[:,i+1] + data[:,2*i+2] .= y_mat[:,i+1] + end + CSV.write(filename*".csv",Tables.table(data),header=csv_header) + else + csv_header = vcat([[sname*"_x",sname*"_y"] for sname in solver_names]...) + data = Matrix{Float64}(undef,nprobs,nsolvers*2) + for k in eachindex(costs) + if !isempty(header) + csv_header = header[k] + end + for i =0:nsolvers-1 + data[:,2*i+1] .= x_mat[k][:,i+1] + data[:,2*i+2] .= y_mat[k][:,i+1] + end + CSV.write(filename*"_$(costnames[k]).csv",Tables.table(data),header=csv_header) + end + end +end diff --git a/test/profiles.jl b/test/profiles.jl index 33088a11..52c1bbb6 100644 --- a/test/profiles.jl +++ b/test/profiles.jl @@ -18,6 +18,27 @@ function test_profiles() b = SolverBenchmark.BenchmarkProfiles.PGFPlotsXBackend(), ) end + + @info "Exporting perfomance profiles" + filename = "profiles" + export_profile_solvers_data(stats,[df -> df.t, df -> df.iter],["Time", "Iterations"],"profiles") + @test isfile(filename * ".csv") + rm(filename * ".csv") + export_profile_solvers_data(stats,[df -> df.t, df -> df.iter],["Time", "Iterations"],"profiles";header=[["x" for _ in 1:6] for _ in 1:2]) + @test isfile(filename * ".csv") + rm(filename * ".csv") + + export_profile_solvers_data(stats,[df -> df.t, df -> df.iter],["Time", "Iterations"],"profiles";one_file=false) + @test isfile(filename * "_Time.csv") + @test isfile(filename * "_Iterations.csv") + rm(filename * "_Time.csv") + rm(filename * "_Iterations.csv") + export_profile_solvers_data(stats,[df -> df.t, df -> df.iter],["Time", "Iterations"],"profiles";one_file=false,header=[["x" for _ in 1:6] for _ in 1:2]) + @test isfile(filename * "_Time.csv") + @test isfile(filename * "_Iterations.csv") + rm(filename * "_Time.csv") + rm(filename * "_Iterations.csv") + nothing end diff --git a/test/runtests.jl b/test/runtests.jl index 5238ed76..5518d3a1 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -13,6 +13,6 @@ using SolverBenchmark include("data.jl") include("tables.jl") -include("profiles.jl") include("pkgbmark.jl") include("test_bmark.jl") +include("profiles.jl")