diff --git a/.github/dependabot.yml b/.github/dependabot.yml index 5c03104..4117611 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -1,7 +1,19 @@ -# https://docs.github.com/github/administering-a-repository/configuration-options-for-dependency-updates +# To get started with Dependabot version updates, you'll need to specify which +# package ecosystems to update and where the package manifests are located. +# Please see the documentation for all configuration options: +# https://docs.github.com/code-security/dependabot/dependabot-version-updates/configuration-options-for-the-dependabot.yml-file + version: 2 updates: - package-ecosystem: "github-actions" + # Workflow files stored in the default location of `.github/workflows` + # You don't need to specify `/.github/workflows` for `directory`. You can use `directory: "/"`. directory: "/" schedule: interval: "weekly" + + - package-ecosystem: "julia" + directory: "/" + schedule: + interval: "weekly" + diff --git a/.github/workflows/CI.yml b/.github/workflows/CI.yml index ecb629f..d5848a8 100644 --- a/.github/workflows/CI.yml +++ b/.github/workflows/CI.yml @@ -1,30 +1,57 @@ name: CI on: + pull_request: + paths-ignore: + - 'docs/**' + - '*.md' push: branches: - main tags: ['*'] - pull_request: concurrency: + # Skip intermediate builds: always. + # Cancel intermediate builds: only if it is a pull request build. group: ${{ github.workflow }}-${{ github.ref }} cancel-in-progress: ${{ startsWith(github.ref, 'refs/pull/') }} jobs: - precompile: - name: Julia ${{ matrix.version }} - runs-on: ubuntu-latest + test: + name: Julia ${{ matrix.version }} - ${{ matrix.os }} - ${{ matrix.arch }} - ${{ github.event_name }} + runs-on: ${{ matrix.os }} strategy: fail-fast: false matrix: version: + - '1.12' - '1.10' - '1.11' - - '1.12' + os: + - ubuntu-latest + arch: + - x64 + env: + PYTHON: "" steps: - uses: actions/checkout@v6 - uses: julia-actions/setup-julia@v3 with: version: ${{ matrix.version }} + arch: ${{ matrix.arch }} - uses: julia-actions/cache@v3 + # - uses: actions/cache@v1 + # env: + # cache-name: cache-artifacts + # with: + # path: ~/.julia/artifacts + # key: ${{ runner.os }}-test-${{ env.cache-name }}-${{ hashFiles('**/Project.toml') }} + # restore-keys: | + # ${{ runner.os }}-test-${{ env.cache-name }}- + # ${{ runner.os }}-test- + # ${{ runner.os }}- - uses: julia-actions/julia-buildpkg@v1 - - name: Load package - run: julia --project=. -e 'using HarmoniqsBenchmarks' + - uses: julia-actions/julia-runtest@v1 + - uses: julia-actions/julia-processcoverage@v1 + - uses: codecov/codecov-action@v6 + with: + files: lcov.info + token: ${{ secrets.CODECOV_TOKEN }} + fail_ci_if_error: false diff --git a/.github/workflows/Formatter.yml b/.github/workflows/Formatter.yml new file mode 100644 index 0000000..5dd9c73 --- /dev/null +++ b/.github/workflows/Formatter.yml @@ -0,0 +1,41 @@ +name: Formatter +on: + pull_request: + push: + branches: + - main + workflow_dispatch: +jobs: + julia-format: + name: Formatter + runs-on: ubuntu-latest + steps: + - uses: julia-actions/setup-julia@latest + - uses: actions/checkout@v6 + with: + token: ${{ secrets.GITHUB_TOKEN }} + - name: Install JuliaFormatter and format + run: | + julia --color=yes -e 'using Pkg; Pkg.add(PackageSpec(name="JuliaFormatter"))' + julia --color=yes -e 'using JuliaFormatter; format(".", verbose=true)' + - name: Commit formatted code + if: github.event_name == 'workflow_dispatch' + run: | + git config user.name "github-actions[bot]" + git config user.email "github-actions[bot]@users.noreply.github.com" + git diff --quiet && exit 0 + git add -A + git commit -m "chore: autoformat" + git push + - name: Format check + if: github.event_name != 'workflow_dispatch' + run: | + julia --color=yes -e ' + out = Cmd(`git diff --name-only`) |> read |> String + if out == "" + exit(0) + else + @error "Some files have not been formatted !!!" + write(stdout, out) + exit(1) + end' diff --git a/.github/workflows/TagBot.yml b/.github/workflows/TagBot.yml index 50fa1ce..5efc8cb 100644 --- a/.github/workflows/TagBot.yml +++ b/.github/workflows/TagBot.yml @@ -6,7 +6,7 @@ on: workflow_dispatch: inputs: lookback: - default: "3" + default: 3 permissions: actions: read checks: read diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml new file mode 100644 index 0000000..c48b33b --- /dev/null +++ b/.github/workflows/nightly.yml @@ -0,0 +1,35 @@ +name: Nightly +on: + schedule: + - cron: '00 00 * * *' + workflow_dispatch: +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: ${{ startsWith(github.ref, 'refs/pull/') }} +jobs: + test: + name: Julia nightly - ${{ matrix.os }} - ${{ matrix.arch }} - ${{ github.event_name }} + runs-on: ${{ matrix.os }} + timeout-minutes: 60 + permissions: + actions: write + contents: read + strategy: + fail-fast: false + matrix: + version: + - 'pre' + os: + - ubuntu-latest + arch: + - x64 + steps: + - uses: actions/checkout@v6 + - uses: julia-actions/setup-julia@v3 + with: + version: ${{ matrix.version }} + arch: ${{ matrix.arch }} + - uses: julia-actions/cache@v3 + - name: Check installation + uses: julia-actions/julia-buildpkg@v1 + diff --git a/examples/convergence_template.jl b/examples/convergence_template.jl index 859853c..8ef0d1d 100644 --- a/examples/convergence_template.jl +++ b/examples/convergence_template.jl @@ -18,8 +18,10 @@ using DirectTrajOpt # ---------------------------------------------------------------------------- # # Replace this with your real problem builder. function build_my_problem() - error("Replace `build_my_problem()` with the builder for the problem you ", - "want to benchmark (e.g. a Piccolo `UnitarySmoothPulseProblem`).") + error( + "Replace `build_my_problem()` with the builder for the problem you ", + "want to benchmark (e.g. a Piccolo `UnitarySmoothPulseProblem`).", + ) end post_solve_fidelity(prob) = error("Return `1 - F(prob)` after the solve.") @@ -34,15 +36,17 @@ function bench_ipopt() state, cb = ipopt_capture() - result = benchmark_solve!(prob, opts; + result = benchmark_solve!( + prob, + opts; benchmark_name = "x_gate_ipopt", - runner = "self_hosted", - callback = cb, - convergence = InfidelityConvergence( - target_infidelity = 1e-4, - final_infidelity = post_solve_fidelity(prob), + runner = "self_hosted", + callback = cb, + convergence = InfidelityConvergence( + target_infidelity = 1e-4, + final_infidelity = post_solve_fidelity(prob), primal_infeasibility = ipopt_primal_infeasibility(state), - feas_tol = 1e-6, + feas_tol = 1e-6, ), ) @@ -63,15 +67,15 @@ function bench_external() # opts = MySolver.Options(...) result = benchmark_solve!( - package = "MyConsumerRepo", - solver = "MySolver", + package = "MyConsumerRepo", + solver = "MySolver", benchmark_name = "x_gate_mysolver", - N = prob.trajectory.N, - state_dim = prob.trajectory.dims[:x], - control_dim = prob.trajectory.dims[:u], - n_constraints = 0, # fill in from your problem - n_variables = 0, - runner = "self_hosted", + N = prob.trajectory.N, + state_dim = prob.trajectory.dims[:x], + control_dim = prob.trajectory.dims[:u], + n_constraints = 0, # fill in from your problem + n_variables = 0, + runner = "self_hosted", ) do # Replace with: MySolver.solve!(prob, opts) — must return whatever the # solver returns; the post_solve closure (below) reads it. diff --git a/src/extractors.jl b/src/extractors.jl index 349be19..521f931 100644 --- a/src/extractors.jl +++ b/src/extractors.jl @@ -31,11 +31,11 @@ to compose with other callbacks via use [`ipopt_capture`](@ref) instead. """ function ipopt_capture_callback(state::IpoptCapture) - return function(_optimizer, optimizer_state; kwargs...) + return function (_optimizer, optimizer_state; kwargs...) state.iter_count = Int(optimizer_state.iter_count) - state.inf_pr = Float64(optimizer_state.inf_pr) - state.inf_du = Float64(optimizer_state.inf_du) - state.objective = Float64(optimizer_state.obj_value) + state.inf_pr = Float64(optimizer_state.inf_pr) + state.inf_du = Float64(optimizer_state.inf_du) + state.objective = Float64(optimizer_state.obj_value) return true end end diff --git a/src/harness.jl b/src/harness.jl index ddd0271..4719b4a 100644 --- a/src/harness.jl +++ b/src/harness.jl @@ -20,7 +20,8 @@ function build_evaluator( prob::DirectTrajOpt.Problems.DirectTrajOptProblem; eval_hessian::Bool = true, ) - evaluator = DirectTrajOpt.Solvers.Evaluator(prob; eval_hessian = eval_hessian, verbose = false) + evaluator = + DirectTrajOpt.Solvers.Evaluator(prob; eval_hessian = eval_hessian, verbose = false) traj = prob.trajectory Z_vec = vcat(collect(traj.datavec), collect(traj.global_data)) return (evaluator, Z_vec) @@ -37,10 +38,10 @@ function evaluator_dims(evaluator::DirectTrajOpt.Solvers.Evaluator) traj = evaluator.trajectory n_variables = traj.dim * traj.N + traj.global_dim return ( - n_constraints = evaluator.n_constraints, - n_variables = n_variables, + n_constraints = evaluator.n_constraints, + n_variables = n_variables, n_jacobian_entries = length(evaluator.jacobian_structure), - n_hessian_entries = length(evaluator.hessian_structure), + n_hessian_entries = length(evaluator.hessian_structure), ) end @@ -66,11 +67,11 @@ Control dim is the sum of all non-timestep control component dimensions. function problem_dims(prob::DirectTrajOptProblem) traj = prob.trajectory return ( - N = traj.N, - state_dim = _infer_state_dim(prob), - control_dim = _infer_control_dim(prob), - n_constraints = _count_constraints(prob), - n_variables = traj.dim * traj.N + traj.global_dim, + N = traj.N, + state_dim = _infer_state_dim(prob), + control_dim = _infer_control_dim(prob), + n_constraints = _count_constraints(prob), + n_variables = traj.dim * traj.N + traj.global_dim, ) end @@ -97,9 +98,9 @@ function evaluate_post_solve(prob::DirectTrajOptProblem) solver_status = constraint_violation < 1e-4 ? :Optimal : :Suboptimal return ( - objective_value = objective_value, + objective_value = objective_value, constraint_violation = constraint_violation, - solver_status = solver_status, + solver_status = solver_status, ) end @@ -157,8 +158,8 @@ end function _count_constraints(prob::DirectTrajOptProblem) dynamics_dim = sum(integrator.dim for integrator in prob.integrators; init = 0) n_nonlinear = sum( - c.dim for c in prob.constraints - if c isa DirectTrajOpt.Constraints.AbstractNonlinearConstraint; + c.dim for c in prob.constraints if + c isa DirectTrajOpt.Constraints.AbstractNonlinearConstraint; init = 0, ) return dynamics_dim + n_nonlinear @@ -207,13 +208,13 @@ function _capture_solve_metrics(solve_fn::Function) # Clean up garbage so peak/heap deltas reflect this solve's work, not # leftover state from earlier code in the same process. GC.gc(true) - gc_before = Base.gc_num() - rss_before = Sys.maxrss() - live_before = Base.gc_live_bytes() + gc_before = Base.gc_num() + rss_before = Sys.maxrss() + live_before = Base.gc_live_bytes() timed = @timed solve_fn() - rss_after = Sys.maxrss() + rss_after = Sys.maxrss() gc_after_solve = Base.gc_num() # Full GC to measure heap retained by the solve (persistent state / leaks). @@ -229,22 +230,24 @@ function _capture_solve_metrics(solve_fn::Function) oom_margin_bytes = Int(Sys.total_memory()) - Int(rss_after) return ( - result = timed.value, - wall_time_s = timed.time, + result = timed.value, + wall_time_s = timed.time, total_allocations_bytes = Int(timed.bytes), - total_allocs_count = Int(timed.gcstats.malloc + timed.gcstats.poolalloc + timed.gcstats.bigalloc), - gc_time_ns = Int(round(timed.gctime * 1e9)), - gc_count = Int(gc_diff.pause), - gc_full_count = Int(gc_diff.full_sweep), + total_allocs_count = Int( + timed.gcstats.malloc + timed.gcstats.poolalloc + timed.gcstats.bigalloc, + ), + gc_time_ns = Int(round(timed.gctime * 1e9)), + gc_count = Int(gc_diff.pause), + gc_full_count = Int(gc_diff.full_sweep), # Sys.maxrss() is monotonic over the process lifetime, so this delta # is a LOWER BOUND on the peak added by this solve. First solve in a # process gives the true peak; later solves give 0 unless they exceed # prior peaks. See schema.jl docstring. - peak_rss_delta_bytes = Int(max(rss_after - rss_before, 0)), + peak_rss_delta_bytes = Int(max(rss_after - rss_before, 0)), # Retained Julia heap after a full post-solve GC. Signed — negative # means the solve freed more of the pre-existing heap than it kept. - live_heap_delta_bytes = Int(live_after - live_before), - oom_margin_bytes = oom_margin_bytes, + live_heap_delta_bytes = Int(live_after - live_before), + oom_margin_bytes = oom_margin_bytes, ) end @@ -288,35 +291,35 @@ function benchmark_solve!( post = evaluate_post_solve(prob) return BenchmarkResult( - package = "DirectTrajOpt", - package_version = _get_package_version("DirectTrajOpt"), - commit = _get_git_commit(), - benchmark_name = benchmark_name, - N = dims.N, - state_dim = dims.state_dim, - control_dim = dims.control_dim, - n_constraints = dims.n_constraints, - n_variables = dims.n_variables, - wall_time_s = metrics.wall_time_s, - iterations = -1, # solve! returns nothing; sentinel - objective_value = post.objective_value, - constraint_violation = post.constraint_violation, - solver_status = post.solver_status, - solver = _solver_name(options), + package = "DirectTrajOpt", + package_version = _get_package_version("DirectTrajOpt"), + commit = _get_git_commit(), + benchmark_name = benchmark_name, + N = dims.N, + state_dim = dims.state_dim, + control_dim = dims.control_dim, + n_constraints = dims.n_constraints, + n_variables = dims.n_variables, + wall_time_s = metrics.wall_time_s, + iterations = -1, # solve! returns nothing; sentinel + objective_value = post.objective_value, + constraint_violation = post.constraint_violation, + solver_status = post.solver_status, + solver = _solver_name(options), total_allocations_bytes = metrics.total_allocations_bytes, - total_allocs_count = metrics.total_allocs_count, - gc_time_ns = metrics.gc_time_ns, - gc_count = metrics.gc_count, - gc_full_count = metrics.gc_full_count, - peak_rss_delta_bytes = metrics.peak_rss_delta_bytes, - live_heap_delta_bytes = metrics.live_heap_delta_bytes, - oom_margin_bytes = metrics.oom_margin_bytes, - solver_options = opts_snapshot, - convergence = convergence, - julia_version = string(VERSION), - timestamp = Dates.now(), - runner = runner, - n_threads = Threads.nthreads(), + total_allocs_count = metrics.total_allocs_count, + gc_time_ns = metrics.gc_time_ns, + gc_count = metrics.gc_count, + gc_full_count = metrics.gc_full_count, + peak_rss_delta_bytes = metrics.peak_rss_delta_bytes, + live_heap_delta_bytes = metrics.live_heap_delta_bytes, + oom_margin_bytes = metrics.oom_margin_bytes, + solver_options = opts_snapshot, + convergence = convergence, + julia_version = string(VERSION), + timestamp = Dates.now(), + runner = runner, + n_threads = Threads.nthreads(), ) end @@ -385,43 +388,44 @@ function benchmark_solve!( if post_solve !== nothing override = post_solve(metrics.result) if override !== nothing - iterations = get(override, :iterations, iterations) - objective_value = get(override, :objective_value, objective_value) - constraint_violation = get(override, :constraint_violation, constraint_violation) - solver_status = get(override, :solver_status, solver_status) + iterations = get(override, :iterations, iterations) + objective_value = get(override, :objective_value, objective_value) + constraint_violation = + get(override, :constraint_violation, constraint_violation) + solver_status = get(override, :solver_status, solver_status) end end return BenchmarkResult( - package = package, - package_version = package_version, - commit = commit, - benchmark_name = benchmark_name, - N = N, - state_dim = state_dim, - control_dim = control_dim, - n_constraints = n_constraints, - n_variables = n_variables, - wall_time_s = metrics.wall_time_s, - iterations = iterations, - objective_value = objective_value, - constraint_violation = constraint_violation, - solver_status = solver_status, - solver = solver, + package = package, + package_version = package_version, + commit = commit, + benchmark_name = benchmark_name, + N = N, + state_dim = state_dim, + control_dim = control_dim, + n_constraints = n_constraints, + n_variables = n_variables, + wall_time_s = metrics.wall_time_s, + iterations = iterations, + objective_value = objective_value, + constraint_violation = constraint_violation, + solver_status = solver_status, + solver = solver, total_allocations_bytes = metrics.total_allocations_bytes, - total_allocs_count = metrics.total_allocs_count, - gc_time_ns = metrics.gc_time_ns, - gc_count = metrics.gc_count, - gc_full_count = metrics.gc_full_count, - peak_rss_delta_bytes = metrics.peak_rss_delta_bytes, - live_heap_delta_bytes = metrics.live_heap_delta_bytes, - oom_margin_bytes = metrics.oom_margin_bytes, - solver_options = solver_options, - convergence = convergence, - julia_version = string(VERSION), - timestamp = Dates.now(), - runner = runner, - n_threads = Threads.nthreads(), + total_allocs_count = metrics.total_allocs_count, + gc_time_ns = metrics.gc_time_ns, + gc_count = metrics.gc_count, + gc_full_count = metrics.gc_full_count, + peak_rss_delta_bytes = metrics.peak_rss_delta_bytes, + live_heap_delta_bytes = metrics.live_heap_delta_bytes, + oom_margin_bytes = metrics.oom_margin_bytes, + solver_options = solver_options, + convergence = convergence, + julia_version = string(VERSION), + timestamp = Dates.now(), + runner = runner, + n_threads = Threads.nthreads(), ) end @@ -436,10 +440,10 @@ Convert a `BenchmarkTools.Trial` into an `EvalBenchmark`. """ function trial_to_eval_benchmark(trial::BenchmarkTools.Trial)::EvalBenchmark return EvalBenchmark( - times_ns = Float64.(trial.times), - gctimes_ns = Float64.(trial.gctimes), + times_ns = Float64.(trial.times), + gctimes_ns = Float64.(trial.gctimes), memory_bytes = Int(trial.memory), - allocs = Int(trial.allocs), + allocs = Int(trial.allocs), ) end @@ -478,7 +482,7 @@ function per_op_benchmark( function _set_params!(b) b.params.seconds = seconds - b.params.evals = evals + b.params.evals = evals b.params.samples = samples return b end @@ -491,7 +495,8 @@ function per_op_benchmark( results[:eval_objective] = trial_to_eval_benchmark(BenchmarkTools.run(bench_obj)) # Objective gradient - bench_grad = BenchmarkTools.@benchmarkable MOI.eval_objective_gradient($evaluator, $g, $Z_vec) + bench_grad = + BenchmarkTools.@benchmarkable MOI.eval_objective_gradient($evaluator, $g, $Z_vec) _set_params!(bench_grad) results[:eval_objective_gradient] = trial_to_eval_benchmark(BenchmarkTools.run(bench_grad)) @@ -504,7 +509,11 @@ function per_op_benchmark( # Constraint Jacobian n_jac = length(evaluator.jacobian_structure) jac_buf = zeros(n_jac) - bench_jac = BenchmarkTools.@benchmarkable MOI.eval_constraint_jacobian($evaluator, $jac_buf, $Z_vec) + bench_jac = BenchmarkTools.@benchmarkable MOI.eval_constraint_jacobian( + $evaluator, + $jac_buf, + $Z_vec, + ) _set_params!(bench_jac) results[:eval_constraint_jacobian] = trial_to_eval_benchmark(BenchmarkTools.run(bench_jac)) @@ -516,7 +525,11 @@ function per_op_benchmark( σ = 1.0 μ = zeros(n_con) bench_hess = BenchmarkTools.@benchmarkable MOI.eval_hessian_lagrangian( - $evaluator, $hess_buf, $Z_vec, $σ, $μ + $evaluator, + $hess_buf, + $Z_vec, + $σ, + $μ, ) _set_params!(bench_hess) results[:eval_hessian_lagrangian] = diff --git a/src/report.jl b/src/report.jl index e28f48c..39429cf 100644 --- a/src/report.jl +++ b/src/report.jl @@ -47,14 +47,17 @@ function compare_convergence(results::Vector{BenchmarkResult})::Vector{Convergen for r in results crit = r.convergence crit === nothing && continue - push!(rows, ConvergenceRow( - r.benchmark_name, - r.solver, - converged(crit), - r.iterations, - r.wall_time_s, - crit, - )) + push!( + rows, + ConvergenceRow( + r.benchmark_name, + r.solver, + converged(crit), + r.iterations, + r.wall_time_s, + crit, + ), + ) end return rows end @@ -84,7 +87,7 @@ Vector of `ComparisonRow` structs, one per matched benchmark. function compare_results( baseline::Vector{BenchmarkResult}, current::Vector{BenchmarkResult}; - regression_threshold=10.0, + regression_threshold = 10.0, )::Vector{ComparisonRow} rows = ComparisonRow[] @@ -128,8 +131,9 @@ function compare_results( end # Flag regression if either metric increased beyond threshold - has_regression = (wall_time_pct_change > regression_threshold) || - (alloc_bytes_pct_change > regression_threshold) + has_regression = + (wall_time_pct_change > regression_threshold) || + (alloc_bytes_pct_change > regression_threshold) row = ComparisonRow( curr.benchmark_name, diff --git a/src/schema.jl b/src/schema.jl index f153937..63ab024 100644 --- a/src/schema.jl +++ b/src/schema.jl @@ -34,7 +34,10 @@ function InfidelityConvergence(; feas_tol::Float64, ) return InfidelityConvergence( - target_infidelity, final_infidelity, primal_infeasibility, feas_tol, + target_infidelity, + final_infidelity, + primal_infeasibility, + feas_tol, ) end @@ -59,7 +62,10 @@ function ObjectiveConvergence(; feas_tol::Float64, ) return ObjectiveConvergence( - target_objective, final_objective, primal_infeasibility, feas_tol, + target_objective, + final_objective, + primal_infeasibility, + feas_tol, ) end @@ -71,12 +77,10 @@ Did this benchmark meet its success bar AND satisfy the dynamics tolerance? function converged end converged(c::InfidelityConvergence) = - c.final_infidelity ≤ c.target_infidelity && - c.primal_infeasibility ≤ c.feas_tol + c.final_infidelity ≤ c.target_infidelity && c.primal_infeasibility ≤ c.feas_tol converged(c::ObjectiveConvergence) = - c.final_objective ≤ c.target_objective && - c.primal_infeasibility ≤ c.feas_tol + c.final_objective ≤ c.target_objective && c.primal_infeasibility ≤ c.feas_tol """ EvalBenchmark @@ -104,7 +108,15 @@ function EvalBenchmark(; median_ns = median(times_ns) min_ns = minimum(times_ns) mean_ns = mean(times_ns) - return EvalBenchmark(times_ns, gctimes_ns, memory_bytes, allocs, median_ns, min_ns, mean_ns) + return EvalBenchmark( + times_ns, + gctimes_ns, + memory_bytes, + allocs, + median_ns, + min_ns, + mean_ns, + ) end """ diff --git a/src/storage.jl b/src/storage.jl index 5fef464..a4b31ca 100644 --- a/src/storage.jl +++ b/src/storage.jl @@ -6,7 +6,11 @@ using JLD2 Save a vector of `BenchmarkResult` to `dir/name_commit.jld2`. Returns the path of the saved file. """ -function save_results(dir::AbstractString, name::AbstractString, results::Vector{BenchmarkResult})::String +function save_results( + dir::AbstractString, + name::AbstractString, + results::Vector{BenchmarkResult}, +)::String mkpath(dir) # Use commit from first result (all should share the same commit in a benchmark run) commit = isempty(results) ? "unknown" : results[1].commit @@ -33,7 +37,11 @@ end Save a `MicroBenchmarkResult` to `dir/name_commit.jld2`. Returns the path of the saved file. """ -function save_micro_results(dir::AbstractString, name::AbstractString, result::MicroBenchmarkResult)::String +function save_micro_results( + dir::AbstractString, + name::AbstractString, + result::MicroBenchmarkResult, +)::String mkpath(dir) filename = "$(name)_$(result.commit).jld2" path = joinpath(dir, filename) @@ -59,7 +67,11 @@ Save an `AllocProfileResult` to `dir/name_commit_allocs.jld2` (a file distinct from `save_results`/`save_micro_results` so allocation artifacts do not bloat the main benchmark JLD2). Returns the path of the saved file. """ -function save_alloc_profile(dir::AbstractString, name::AbstractString, profile::AllocProfileResult)::String +function save_alloc_profile( + dir::AbstractString, + name::AbstractString, + profile::AllocProfileResult, +)::String mkpath(dir) filename = "$(name)_$(profile.commit)_allocs.jld2" path = joinpath(dir, filename) diff --git a/test/runtests.jl b/test/runtests.jl index 5931ff1..84f8c05 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -108,10 +108,7 @@ using LinearAlgebra N = 50, state_dim = 4, control_dim = 2, - eval_benchmarks = Dict{Symbol,EvalBenchmark}( - :jacobian => eb1, - :hessian => eb2, - ), + eval_benchmarks = Dict{Symbol,EvalBenchmark}(:jacobian => eb1, :hessian => eb2), julia_version = "1.12.0", timestamp = DateTime(2026, 4, 14), runner = "local", @@ -186,18 +183,26 @@ using LinearAlgebra # ------------------------------------------------------------------ # # Helper: build the bilinear test problem used across harness tests - function _make_bilinear_prob(; N=10, Δt=0.1, u_bound=0.1, ω=0.1) + function _make_bilinear_prob(; N = 10, Δt = 0.1, u_bound = 0.1, ω = 0.1) Gx = sparse(Float64[0 0 0 1; 0 0 1 0; 0 -1 0 0; -1 0 0 0]) Gy = sparse(Float64[0 -1 0 0; 1 0 0 0; 0 0 0 -1; 0 0 1 0]) Gz = sparse(Float64[0 0 1 0; 0 0 0 -1; -1 0 0 0; 0 1 0 0]) G(u) = ω * Gz + u[1] * Gx + u[2] * Gy traj = NamedTrajectory( - (x=2rand(4,N).-1, u=u_bound*(2rand(2,N).-1), du=randn(2,N), ddu=randn(2,N), Δt=fill(Δt,N)); - controls=(:ddu,:Δt), timestep=:Δt, - bounds=(u=u_bound, Δt=(0.01,0.5)), - initial=(x=[1.0,0.0,0.0,0.0], u=zeros(2)), final=(u=zeros(2),), - goal=(x=[0.0,1.0,0.0,0.0],), + ( + x = 2rand(4, N) .- 1, + u = u_bound*(2rand(2, N) .- 1), + du = randn(2, N), + ddu = randn(2, N), + Δt = fill(Δt, N), + ); + controls = (:ddu, :Δt), + timestep = :Δt, + bounds = (u = u_bound, Δt = (0.01, 0.5)), + initial = (x = [1.0, 0.0, 0.0, 0.0], u = zeros(2)), + final = (u = zeros(2),), + goal = (x = [0.0, 1.0, 0.0, 0.0],), ) integrators = [ BilinearIntegrator(G, :x, :u, traj), @@ -224,7 +229,7 @@ using LinearAlgebra @test isfinite(obj) # eval_hessian=false variant also works - (ev2, Z_vec2) = build_evaluator(prob; eval_hessian=false) + (ev2, Z_vec2) = build_evaluator(prob; eval_hessian = false) @test length(Z_vec2) == expected_len @test !ev2.eval_hessian end @@ -256,13 +261,14 @@ using LinearAlgebra @test dims.state_dim == 4 # :x component has dim 4 @test dims.control_dim == 2 # :ddu (2); Δt is timestep, excluded @test dims.n_constraints > 0 - @test dims.n_variables == prob.trajectory.dim * prob.trajectory.N + prob.trajectory.global_dim + @test dims.n_variables == + prob.trajectory.dim * prob.trajectory.N + prob.trajectory.global_dim end @testset "evaluate_post_solve" begin prob = _make_bilinear_prob() # Solve first so trajectory has meaningful values - solve!(prob; options=IpoptOptions(max_iter=5, print_level=0), verbose=false) + solve!(prob; options = IpoptOptions(max_iter = 5, print_level = 0), verbose = false) post = evaluate_post_solve(prob) @test isfinite(post.objective_value) @@ -272,7 +278,7 @@ using LinearAlgebra end @testset "snapshot_options" begin - opts = IpoptOptions(max_iter=42, tol=1e-6, print_level=0) + opts = IpoptOptions(max_iter = 42, tol = 1e-6, print_level = 0) snap = snapshot_options(opts) @test snap isa Dict{Symbol,Any} @@ -285,9 +291,10 @@ using LinearAlgebra @testset "benchmark_solve!" begin prob = _make_bilinear_prob() - opts = IpoptOptions(max_iter=5, print_level=0) + opts = IpoptOptions(max_iter = 5, print_level = 0) - result = benchmark_solve!(prob, opts; benchmark_name="test_bilinear", runner="test") + result = + benchmark_solve!(prob, opts; benchmark_name = "test_bilinear", runner = "test") # Identity fields @test result.package == "DirectTrajOpt" @@ -330,26 +337,26 @@ using LinearAlgebra @testset "benchmark_solve! do-block (generic)" begin # Run a fake "solve" that just allocates and sleeps result = benchmark_solve!( - package = "TestPkg", + package = "TestPkg", package_version = "0.0.0", - commit = "abc1234", - solver = "FakeSolver", - benchmark_name = "fake_solve", - N = 42, - state_dim = 7, - control_dim = 3, - n_constraints = 100, - n_variables = 200, - iterations = 5, + commit = "abc1234", + solver = "FakeSolver", + benchmark_name = "fake_solve", + N = 42, + state_dim = 7, + control_dim = 3, + n_constraints = 100, + n_variables = 200, + iterations = 5, objective_value = 0.001, constraint_violation = 1e-8, - solver_status = :Optimal, - solver_options = Dict{Symbol,Any}(:max_iter => 5, :fidelity => 0.999), - runner = "test", + solver_status = :Optimal, + solver_options = Dict{Symbol,Any}(:max_iter => 5, :fidelity => 0.999), + runner = "test", ) do # Simulate a solve that allocates v = zeros(10_000) - for _ in 1:100 + for _ = 1:100 v .+= rand(10_000) end return nothing @@ -393,22 +400,22 @@ using LinearAlgebra solver_state = Ref(0) result = benchmark_solve!( - package = "TestPkg", - solver = "PostSolveTest", + package = "TestPkg", + solver = "PostSolveTest", benchmark_name = "post_solve_test", - N = 10, - state_dim = 2, - control_dim = 1, - n_constraints = 20, - n_variables = 30, + N = 10, + state_dim = 2, + control_dim = 1, + n_constraints = 20, + n_variables = 30, solver_options = Dict{Symbol,Any}(:max_iter => 42), - commit = "abc1234", - post_solve = function(_) + commit = "abc1234", + post_solve = function (_) # post_solve runs AFTER the closure; access state written there return ( - iterations = solver_state[], + iterations = solver_state[], objective_value = 1e-6, - solver_status = :Optimal, + solver_status = :Optimal, ) end, ) do @@ -447,13 +454,10 @@ using LinearAlgebra @testset "per_op_benchmark" begin prob = _make_bilinear_prob() - (evaluator, Z_vec) = build_evaluator(prob; eval_hessian=true) + (evaluator, Z_vec) = build_evaluator(prob; eval_hessian = true) # Tiny budgets to keep CI fast - results = per_op_benchmark( - evaluator, Z_vec; - seconds=0.05, evals=1, samples=5, - ) + results = per_op_benchmark(evaluator, Z_vec; seconds = 0.05, evals = 1, samples = 5) expected_keys = [ :eval_objective, @@ -472,11 +476,8 @@ using LinearAlgebra end # Without Hessian: key should be absent - (ev_nohess, Z2) = build_evaluator(prob; eval_hessian=false) - results2 = per_op_benchmark( - ev_nohess, Z2; - seconds=0.05, evals=1, samples=5, - ) + (ev_nohess, Z2) = build_evaluator(prob; eval_hessian = false) + results2 = per_op_benchmark(ev_nohess, Z2; seconds = 0.05, evals = 1, samples = 5) @test !haskey(results2, :eval_hessian_lagrangian) @test haskey(results2, :eval_objective) end @@ -831,15 +832,17 @@ using LinearAlgebra feas_tol = 1e-6, ) - br_quantum = BenchmarkResult(; kw..., convergence = crit) - br_dto = BenchmarkResult(; kw..., - convergence = ObjectiveConvergence( - target_objective = 1.0, - final_objective = 0.5, - primal_infeasibility = 1e-8, - feas_tol = 1e-6, - )) - br_timing = BenchmarkResult(; kw...) # convergence === nothing + br_quantum = BenchmarkResult(; kw..., convergence = crit) + br_dto = BenchmarkResult(; + kw..., + convergence = ObjectiveConvergence( + target_objective = 1.0, + final_objective = 0.5, + primal_infeasibility = 1e-8, + feas_tol = 1e-6, + ), + ) + br_timing = BenchmarkResult(; kw...) # convergence === nothing dir = mktempdir() path = save_results(dir, "convergence_mix", [br_quantum, br_dto, br_timing]) @@ -882,17 +885,17 @@ using LinearAlgebra state = IpoptCapture() cb = ipopt_capture_callback(state) fake_state = ( - alg_mod = Int32(0), - iter_count = Int32(7), - obj_value = 3.14, - inf_pr = 1.5e-5, - inf_du = 2.0e-4, - mu = 0.1, - d_norm = 0.5, - regularization_size = 0.0, - alpha_du = 1.0, - alpha_pr = 1.0, - ls_trials = Int32(2), + alg_mod = Int32(0), + iter_count = Int32(7), + obj_value = 3.14, + inf_pr = 1.5e-5, + inf_du = 2.0e-4, + mu = 0.1, + d_norm = 0.5, + regularization_size = 0.0, + alpha_du = 1.0, + alpha_pr = 1.0, + ls_trials = Int32(2), ) @test cb(nothing, fake_state) === true @@ -934,40 +937,50 @@ using LinearAlgebra n_threads = 1, ) - br_quantum_pass = BenchmarkResult(; kw_common..., + br_quantum_pass = BenchmarkResult(; + kw_common..., benchmark_name = "x_gate", solver = "Ipopt", iterations = 15, wall_time_s = 1.2, convergence = InfidelityConvergence( - target_infidelity = 1e-4, final_infidelity = 5e-5, - primal_infeasibility = 1e-7, feas_tol = 1e-6, + target_infidelity = 1e-4, + final_infidelity = 5e-5, + primal_infeasibility = 1e-7, + feas_tol = 1e-6, ), ) - br_quantum_fail_altissimo = BenchmarkResult(; kw_common..., + br_quantum_fail_altissimo = BenchmarkResult(; + kw_common..., benchmark_name = "x_gate", solver = "Altissimo", iterations = 100, wall_time_s = 2.5, convergence = InfidelityConvergence( - target_infidelity = 1e-4, final_infidelity = 5e-3, - primal_infeasibility = 1e-7, feas_tol = 1e-6, + target_infidelity = 1e-4, + final_infidelity = 5e-3, + primal_infeasibility = 1e-7, + feas_tol = 1e-6, ), ) - br_dto_fail = BenchmarkResult(; kw_common..., + br_dto_fail = BenchmarkResult(; + kw_common..., benchmark_name = "cartpole", solver = "Ipopt", iterations = 200, wall_time_s = 10.0, convergence = ObjectiveConvergence( - target_objective = 0.1, final_objective = 0.5, - primal_infeasibility = 1e-7, feas_tol = 1e-6, + target_objective = 0.1, + final_objective = 0.5, + primal_infeasibility = 1e-7, + feas_tol = 1e-6, ), ) - br_timing_only = BenchmarkResult(; kw_common..., + br_timing_only = BenchmarkResult(; + kw_common..., benchmark_name = "ignore_me", solver = "Ipopt", iterations = 1, @@ -1010,13 +1023,17 @@ using LinearAlgebra @testset "benchmark_solve! threads convergence kwarg" begin crit = InfidelityConvergence( - target_infidelity = 1e-4, final_infidelity = 5e-5, - primal_infeasibility = 1e-7, feas_tol = 1e-6, + target_infidelity = 1e-4, + final_infidelity = 5e-5, + primal_infeasibility = 1e-7, + feas_tol = 1e-6, ) # DTO-typed method prob = _make_bilinear_prob() - result_dto = benchmark_solve!(prob, IpoptOptions(max_iter=5, print_level=0); + result_dto = benchmark_solve!( + prob, + IpoptOptions(max_iter = 5, print_level = 0); benchmark_name = "bilinear_conv", runner = "test", convergence = crit, @@ -1025,7 +1042,9 @@ using LinearAlgebra @test result_dto.convergence isa InfidelityConvergence # Omitting it still works → defaults to nothing. - result_dto_default = benchmark_solve!(prob, IpoptOptions(max_iter=5, print_level=0); + result_dto_default = benchmark_solve!( + prob, + IpoptOptions(max_iter = 5, print_level = 0); benchmark_name = "bilinear_no_conv", runner = "test", ) @@ -1033,23 +1052,29 @@ using LinearAlgebra # do-block form result_doblock = benchmark_solve!( - package = "TestPkg", - solver = "FakeSolver", + package = "TestPkg", + solver = "FakeSolver", benchmark_name = "fake_conv", - N = 10, state_dim = 4, control_dim = 2, - n_constraints = 80, n_variables = 100, - convergence = crit, + N = 10, + state_dim = 4, + control_dim = 2, + n_constraints = 80, + n_variables = 100, + convergence = crit, ) do nothing end @test result_doblock.convergence === crit result_doblock_default = benchmark_solve!( - package = "TestPkg", - solver = "FakeSolver", + package = "TestPkg", + solver = "FakeSolver", benchmark_name = "fake_no_conv", - N = 10, state_dim = 4, control_dim = 2, - n_constraints = 80, n_variables = 100, + N = 10, + state_dim = 4, + control_dim = 2, + n_constraints = 80, + n_variables = 100, ) do nothing end