export profile data

Project.toml

@@ -6,6 +6,7 @@ version = "0.5.5"
 BenchmarkProfiles = "ecbce9bc-3e5e-569d-9e29-55181f61f8d0"
 BenchmarkTools = "6e4b80f9-dd63-53aa-95a3-0cdb28fa8baf"
 ColorSchemes = "35d6a980-a343-548e-a6ea-1d62b119f2f4"
+CSV = "336ed68f-0bac-5ca0-87d4-7b16caf5d00b"
 DataFrames = "a93c6f00-e57d-5684-b7b6-d8193f3e46c0"
 GitHub = "bc5e4493-9b4d-5f90-b8aa-2b2bcaad7a26"
 JLD2 = "033835bb-8acc-5ee8-8aae-3f567f8a3819"
@@ -29,6 +30,7 @@ UnicodePlots = "b8865327-cd53-5732-bb35-84acbb429228"
 BenchmarkProfiles = "0.4.2"
 BenchmarkTools = "^0.4.2, 0.5, 0.6, 0.7, 1"
 ColorSchemes = "^3.9"
+CSV = "0.10"
 DataFrames = "^0.21, 1"
 GitHub = "^5.0.2"
 JLD2 = "0.1.12, 0.2, 0.3, 0.4"

src/profiles.jl

@@ -1,7 +1,7 @@
 import BenchmarkProfiles: performance_profile
-using BenchmarkProfiles, Plots
+using BenchmarkProfiles, Plots, CSV
 
-export performance_profile, profile_solvers
+export performance_profile, profile_solvers, export_profile_solvers_data
 
 """
     performance_profile(stats, cost, args...; b = PlotsBackend(), kwargs...)
@@ -137,3 +137,129 @@ function profile_solvers(
   end
   p
 end
+
+"""
+    get_profile_solvers_data(stats, costs; kwargs...)
+
+Exports performance profiles plot data comparing `solvers` based on the data in `stats` in a .csv file.
+
+Inputs:
+- `stats::Dict{Symbol,DataFrame}`: a dictionary of `DataFrame`s containing the
+    benchmark results per solver (e.g., produced by `bmark_results_to_dataframes()`)
+- `costs::Vector{Function}`: a vector of functions specifying the measures to use in the profiles
+
+Keyword arguments are passed to `BenchmarkProfiles.performance_profile_data()`.
+
+Output:
+x_mat, y_mat: vectors which elements are matrices containing the x and y coordinate of the plots. Each matrix correspond to a cost, matrices columns correspond to solvers.
+Matrices are padded with NaN if necessary (happens if plots do not have the same number of points).
+"""
+function get_profile_solvers_data(
+  stats::Dict{Symbol, DataFrame},
+  costs::Vector{<:Function};
+  kwargs...
+  )
+
+  solvers = collect(keys(stats))
+  dfs = (stats[solver] for solver in solvers)
+  Ps = [hcat([Float64.(cost(df)) for df in dfs]...) for cost in costs]
+
+  nprobs = size(stats[first(solvers)], 1)
+  nsolvers = length(solvers)
+  ncosts = length(costs)
+  npairs = div(nsolvers * (nsolvers - 1), 2)
+  x_data, y_data = performance_profile_data(Ps[1]; kwargs...)
+  nmaxrow = maximum(length.(x_data))
+  for i in eachindex(x_data)
+    append!(x_data[i],[NaN for i=1:nprobs-length(x_data[i])])
+    append!(y_data[i],[NaN for i=1:nprobs-length(y_data[i])])
+  end
+  x_mat = [hcat(x_data...)]
+  y_mat = [hcat(y_data...)]
+  for k in 2:ncosts
+    x_data, y_data = performance_profile_data(Ps[k];kwargs...)
+    nmaxrow = max(nmaxrow,maximum(length.(x_data)))
+    for i in eachindex(x_data)
+      append!(x_data[i],[NaN for i=1:nprobs-length(x_data[i])])
+      append!(y_data[i],[NaN for i=1:nprobs-length(y_data[i])])
+    end
+    push!(x_mat, hcat(x_data...))
+    push!(y_mat, hcat(y_data...))
+  end
+  return [m[1:nmaxrow,:] for m in x_mat], [m[1:nmaxrow,:] for m in y_mat]
+end
+
+"""
+    export_profile_solvers_data(stats, costs, costnames, filename; one_file=true, two_by_two=false, kwargs...)
+
+Exports performance profiles plot data comparing `solvers` based on the data in `stats` in a .csv file.
+Data are padded with NaN to ensure .csv consistency.
+
+Inputs:
+- `stats::Dict{Symbol,DataFrame}`: a dictionary of `DataFrame`s containing the
+    benchmark results per solver (e.g., produced by `bmark_results_to_dataframes()`)
+- `costs::Vector{Function}`: a vector of functions specifying the measures to use in the profiles
+- `costnames::Vector{String}`: names to be used as titles of the profiles.
+- `filename::String`: path to the export file without the .csv extention.
+
+Keyword arguments:
+- `one_file::Bool`: export one file per cost if false, otherwise profiles for all costs are exported in a single file
+- `header::Vector{Vector{String}}`: Contains .csv file(s) column names for each files. Example for two costs exported in two files and two solvers "alpha" and "beta": `[ ["alpha_x","alpha_y","beta_x","beta_y"] for _=1:2]`. Note that `header` value does not change columns order in .csv exported files (see Output). 
+
+Additional `kwargs` are passed to `BenchmarkProfiles.performance_profile_data()`.
+
+Output:
+File(s) containing profile data in .csv format.
+* If one_file=true, returns one file containing the data for all solvers and cost. 
+  Columns are cost1_solver1_x, cost1_solver1_y, cost1_solver2_x, ... cost2_solver1_x, cost2_solver1_y, ...
+* If one_file=false, returns as many files as the number of cost. 
+  The names of the files contain the name of the cost, and the columns are
+  solver1_x, solver1_y, solver2_x, ...
+"""
+function export_profile_solvers_data(
+  stats::Dict{Symbol, DataFrame},
+  costs::Vector{<:Function},
+  costnames::S,
+  filename::String;
+  header = [],
+  one_file=true,
+  kwargs...
+  ) where {S <: Vector{String}}
+  solvers = collect(keys(stats))
+  nprobs = size(stats[first(solvers)], 1)
+  nsolvers = length(solvers)
+  solver_names = String.(keys(stats))
+  csv_header = Vector{String}[]
+
+  x_mat, y_mat = get_profile_solvers_data(stats,costs;kwargs...)
+  if one_file
+    if isempty(header) 
+      csv_header = vcat([vcat([[cname*"_"*sname*"_x",cname*"_"*sname*"_y"] for sname in solver_names]...) for cname in costnames]...)
+    else 
+      csv_header = vcat(header...)
+    end
+    x_mat = hcat(x_mat...)
+    y_mat = hcat(y_mat...)
+    ncol = size(x_mat)[2]
+    nrow = size(x_mat)[1]
+    data = Matrix{Float64}(undef,nrow,ncol*2)
+    for i =0:ncol-1
+      data[:,2*i+1] .= x_mat[:,i+1]
+      data[:,2*i+2] .= y_mat[:,i+1]
+    end
+    CSV.write(filename*".csv",Tables.table(data),header=csv_header)
+  else
+    csv_header = vcat([[sname*"_x",sname*"_y"] for sname in solver_names]...)
+    data = Matrix{Float64}(undef,nprobs,nsolvers*2)
+    for k in eachindex(costs)
+      if !isempty(header)
+        csv_header = header[k]
+      end
+      for i =0:nsolvers-1
+        data[:,2*i+1] .= x_mat[k][:,i+1]
+        data[:,2*i+2] .= y_mat[k][:,i+1]
+      end
+      CSV.write(filename*"_$(costnames[k]).csv",Tables.table(data),header=csv_header)
+    end 
+  end
+end

test/profiles.jl

@@ -18,6 +18,27 @@ function test_profiles()
       b = SolverBenchmark.BenchmarkProfiles.PGFPlotsXBackend(),
     )
   end
+
+  @info "Exporting perfomance profiles"
+  filename = "profiles"
+  export_profile_solvers_data(stats,[df -> df.t, df -> df.iter],["Time", "Iterations"],"profiles")
+  @test isfile(filename * ".csv")
+  rm(filename * ".csv")
+  export_profile_solvers_data(stats,[df -> df.t, df -> df.iter],["Time", "Iterations"],"profiles";header=[["x" for _ in 1:6] for _ in 1:2])
+  @test isfile(filename * ".csv")
+  rm(filename * ".csv")
+
+  export_profile_solvers_data(stats,[df -> df.t, df -> df.iter],["Time", "Iterations"],"profiles";one_file=false)
+  @test isfile(filename * "_Time.csv")
+  @test isfile(filename * "_Iterations.csv")
+  rm(filename * "_Time.csv")
+  rm(filename * "_Iterations.csv")
+  export_profile_solvers_data(stats,[df -> df.t, df -> df.iter],["Time", "Iterations"],"profiles";one_file=false,header=[["x" for _ in 1:6] for _ in 1:2])
+  @test isfile(filename * "_Time.csv")
+  @test isfile(filename * "_Iterations.csv")
+  rm(filename * "_Time.csv")
+  rm(filename * "_Iterations.csv")
+
   nothing
 end
 

test/runtests.jl

@@ -13,6 +13,6 @@ using SolverBenchmark
 
 include("data.jl")
 include("tables.jl")
-include("profiles.jl")
 include("pkgbmark.jl")
 include("test_bmark.jl")
+include("profiles.jl")