diff --git a/.github/workflows/benchmark-convergence.yml b/.github/workflows/benchmark-convergence.yml new file mode 100644 index 0000000..c2fa2fe --- /dev/null +++ b/.github/workflows/benchmark-convergence.yml @@ -0,0 +1,80 @@ +name: Convergence Benchmarks +on: + push: + tags: ['v*'] + pull_request: + paths: + - 'src/**' + - 'benchmark/convergence/**' + - '.github/workflows/benchmark-convergence.yml' + workflow_dispatch: + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: ${{ startsWith(github.ref, 'refs/pull/') }} + +jobs: + convergence: + name: Convergence suite (Ipopt + MadNLP) + runs-on: ubuntu-latest + timeout-minutes: 30 + permissions: + actions: write + # contents: write so github-action-benchmark can auto-push the time-series + # to gh-pages (gated to main below); pull-requests: write for its + # regression alert comments. + contents: write + pull-requests: write + steps: + - uses: actions/checkout@v6 + + - uses: julia-actions/setup-julia@v2 + with: + version: '1.12' + arch: x64 + + - uses: julia-actions/cache@v2 + + - name: Instantiate convergence environment + run: julia --project=benchmark/convergence -e 'using Pkg; Pkg.instantiate(); Pkg.precompile()' + + - name: Run convergence benchmarks + env: + BENCHMARK_RUNNER: github-actions + run: | + julia --project=benchmark/convergence -t auto -e ' + using TestItemRunner + TestItemRunner.run_tests("benchmark/convergence/") + ' + + - name: Generate convergence report + if: always() + run: julia --project=benchmark/convergence benchmark/convergence/report.jl + + - name: Upload convergence artifacts + if: always() + uses: actions/upload-artifact@v4 + with: + name: convergence-${{ github.event.pull_request.number || github.ref_name }}-${{ github.sha }} + path: benchmark/convergence/results/ + retention-days: 90 + + # Publish to /bench-convergence on gh-pages (separate dashboard from the + # timing suite's /bench). Per-commit time series for wall/alloc/iters and + # achieved infidelity, with 120% regression alerts. save-data-file / + # auto-push gated to main so PR runs render a comparison only. + - name: Publish convergence benchmark to gh-pages + if: success() && hashFiles('benchmark/convergence/results/bench.json') != '' + uses: benchmark-action/github-action-benchmark@v1 + with: + name: DirectTrajOpt.jl convergence + tool: customSmallerIsBetter + output-file-path: benchmark/convergence/results/bench.json + github-token: ${{ secrets.GITHUB_TOKEN }} + gh-pages-branch: gh-pages + benchmark-data-dir-path: bench-convergence + alert-threshold: '120%' + comment-on-alert: true + fail-on-alert: false + save-data-file: ${{ github.ref == 'refs/heads/main' }} + auto-push: ${{ github.ref == 'refs/heads/main' }} diff --git a/.github/workflows/benchmark.yml b/.github/workflows/benchmark.yml index 67628ea..fd3aba7 100644 --- a/.github/workflows/benchmark.yml +++ b/.github/workflows/benchmark.yml @@ -38,16 +38,21 @@ jobs: - name: Instantiate benchmark environment run: julia --project=benchmark -e 'using Pkg; Pkg.instantiate()' - - name: Run benchmarks (excluding alloc profile) + - name: Run timing benchmarks only env: BENCHMARK_RUNNER: github-actions run: | julia --project=benchmark -t auto -e ' using TestItemRunner - # The alloc-profile testitem has its own workflow - # (.github/workflows/alloc-profile.yml) — its Profile.Allocs run is - # far slower and would blow this suite''s 60-min budget. - TestItemRunner.run_tests("benchmark/"; filter = ti -> !occursin("alloc_profile", ti.filename)) + # Run only the timing testitems (benchmark/benchmarks.jl). The + # alloc-profile (benchmark/alloc_profile.jl) and convergence + # (benchmark/convergence/) suites each have their own workflow + + # project env (alloc-profile.yml, benchmark-convergence.yml); running + # them here would use the wrong env and/or blow this suite''s 60-min + # budget. + TestItemRunner.run_tests("benchmark/"; + filter = ti -> !occursin("alloc_profile", ti.filename) && + !occursin("/convergence/", ti.filename)) ' - name: Generate benchmark report diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml index 8250dc2..93c1bdb 100644 --- a/.github/workflows/docs.yml +++ b/.github/workflows/docs.yml @@ -18,6 +18,11 @@ jobs: docs: name: Documentation runs-on: ubuntu-latest + # Bound the build+deploy. makedocs is ~5 min; the rest is the deploydocs + # gh-pages push. Without this, a transient deploy/git-push stall runs to the + # 6h default and blocks the Documentation concurrency group (observed + # 2026-06-08: a deploy hung 90+ min, queueing the next main build behind it). + timeout-minutes: 30 permissions: contents: write statuses: write diff --git a/benchmark/convergence/.gitignore b/benchmark/convergence/.gitignore new file mode 100644 index 0000000..ca28c11 --- /dev/null +++ b/benchmark/convergence/.gitignore @@ -0,0 +1,2 @@ +results/ +Manifest.toml diff --git a/benchmark/convergence/Project.toml b/benchmark/convergence/Project.toml new file mode 100644 index 0000000..908b429 --- /dev/null +++ b/benchmark/convergence/Project.toml @@ -0,0 +1,24 @@ +[deps] +Dates = "ade2ca70-3891-5945-98fb-dc099432e06a" +DirectTrajOpt = "c823fa1f-8872-4af5-b810-2b9b72bbbf56" +ExponentialAction = "e24c0720-ea99-47e8-929e-571b494574d3" +HarmoniqsBenchmarks = "f45d0b76-2d23-4568-9599-481e0da131db" +Ipopt = "b6b21f68-93f8-5de0-b562-5493be1d77c9" +JLD2 = "033835bb-8acc-5ee8-8aae-3f567f8a3819" +LinearAlgebra = "37e2e46d-f89d-539d-b4ee-838fcccc9c8e" +MadNLP = "2621e9c9-9eb4-46b1-8089-e8c72242dfb6" +NamedTrajectories = "538bc3a1-5ab9-4fc3-b776-35ca1e893e08" +Printf = "de0858da-6303-5e67-8744-51eddeeeb8d7" +Random = "9a3f8284-a2c9-5f02-9a11-845980a1fd5c" +SparseArrays = "2f01184e-e22b-5df5-ae63-d93ebab69eaf" +Statistics = "10745b16-79ce-11e8-11f9-7d13ad32a3b2" +TestItemRunner = "f8b46487-2199-4994-9208-9a1283c18c0a" +TestItems = "1c621080-faea-4a02-84b6-bbd5e436b8fe" + +[sources] +DirectTrajOpt = {path = "../.."} +# HBJ v0.2.0 not yet registered in General; pin to a specific commit so +# convergence results are reproducible across CI re-runs. Bump this SHA +# (and the local Manifest) when HBJ ships a new feature we want to use. +# Drop in favor of `[compat]` once HBJ registers in General. +HarmoniqsBenchmarks = {url = "https://github.com/harmoniqs/HarmoniqsBenchmarks.jl", rev = "c38418cb7f932f2ff9a9c6c6eacf9a11ff1018c1"} diff --git a/benchmark/convergence/README.md b/benchmark/convergence/README.md new file mode 100644 index 0000000..6f3b36f --- /dev/null +++ b/benchmark/convergence/README.md @@ -0,0 +1,38 @@ +# DirectTrajOpt — Convergence Benchmarks + +Convergence-quality benchmarks built on HarmoniqsBenchmarks.jl v0.2.0's new +convergence API (`InfidelityConvergence`, `ipopt_capture`, +`compare_convergence`). Each `@testitem` runs a small X-gate state-transfer +on Ipopt or MadNLP and asserts the result meets a problem-specific success +bar before saving a JLD2 artifact under `benchmark/convergence/results/`. + +**Scope — this is a regression/sanity baseline, not a solver-difficulty +benchmark.** The 1-qubit X gate is deliberately easy: both Ipopt and MadNLP +drive it to ~machine precision (infidelity ~1e-11 / ~1e-14). Its job is to (a) +catch a regression that stops a solver from converging on a known-good problem, +and (b) track each solver's wall-time / allocations / iterations-to-converge +over commits via the dashboard. Harder, solver-discriminating problems +(multi-qubit, cavity, free-phase) live in the platform demos and the +Piccolissimo Altissimo suite — not here. + +## Running locally + +```bash +# from DirectTrajOpt.jl root +julia --project=benchmark/convergence -e 'using Pkg; Pkg.instantiate()' +julia --project=benchmark/convergence -e ' + using TestItemRunner + @run_package_tests filter = ti -> occursin("convergence", ti.name) +' +``` + +## What's covered + +- **X gate convergence: Ipopt** — uses `ipopt_capture()` to grab final + `iter_count` + `inf_pr`, builds an `InfidelityConvergence`, passes it + through `benchmark_solve!` to populate `BenchmarkResult.convergence`. +- **X gate convergence: MadNLP** — same problem, MadNLP solver. No capture + hook yet, so `primal_infeasibility` is taken from the post-solve + evaluator's `constraint_violation`. + +Atoms / spin-qubit / bosonic demo problems land in separate follow-up PRs. diff --git a/benchmark/convergence/convergence.jl b/benchmark/convergence/convergence.jl new file mode 100644 index 0000000..2652e74 --- /dev/null +++ b/benchmark/convergence/convergence.jl @@ -0,0 +1,118 @@ +using TestItems + +@testitem "X gate convergence: Ipopt" begin + using HarmoniqsBenchmarks + using DirectTrajOpt + using NamedTrajectories + using SparseArrays, ExponentialAction, Random, Dates, Printf, LinearAlgebra + + include(joinpath(@__DIR__, "problem_utils.jl")) + + runner = get(ENV, "BENCHMARK_RUNNER", "local") + + prob = _make_xgate_prob(; N = 51, seed = 42) + + # Wire Ipopt callback that captures final iter_count + inf_pr. + state, cb = ipopt_capture() + ipopt_opts = IpoptOptions(max_iter = 500, print_level = 0) + + # benchmark_solve! forwards extra kwargs to DirectTrajOpt.solve!, so we + # can inject the capture callback through the same call. + result = benchmark_solve!( + prob, + ipopt_opts; + benchmark_name = "xgate_convergence_ipopt_N51", + runner = runner, + callback = cb, + ) + + final_inf = _xgate_infidelity(prob) + primal_inf = ipopt_primal_infeasibility(state) + iters = ipopt_iterations(state) + + crit = InfidelityConvergence( + target_infidelity = 1e-3, + final_infidelity = final_inf, + primal_infeasibility = primal_inf, + feas_tol = 1e-6, + ) + + result_with_conv = _build_convergence_result(result, crit; iterations = iters) + + @printf( + "\n=== X gate convergence (Ipopt) ===\n iters=%d final_inf=%.3e inf_pr=%.3e wall=%.3fs converged=%s\n", + iters, + final_inf, + primal_inf, + result_with_conv.wall_time_s, + converged(crit), + ) + + @test converged(result_with_conv.convergence) == true + + results_dir = joinpath(@__DIR__, "results") + saved = save_results(results_dir, "xgate_convergence_ipopt_N51", [result_with_conv]) + println(" Saved $(saved)") + + # Exercise the reporting path. + rows = compare_convergence([result_with_conv]) + @test length(rows) == 1 + @test rows[1].converged == true +end + + +@testitem "X gate convergence: MadNLP" begin + using HarmoniqsBenchmarks + using DirectTrajOpt + using NamedTrajectories + using SparseArrays, ExponentialAction, Random, Dates, Printf, LinearAlgebra + import MadNLP + + include(joinpath(@__DIR__, "problem_utils.jl")) + + runner = get(ENV, "BENCHMARK_RUNNER", "local") + + prob = _make_xgate_prob(; N = 51, seed = 42) + + madnlp_opts = MadNLPOptions(max_iter = 500, print_level = 6) + + # MadNLP doesn't have an ipopt_capture analogue yet — use the post-solve + # constraint_violation that benchmark_solve! already extracted as the + # primal-infeasibility proxy. + result = benchmark_solve!( + prob, + madnlp_opts; + benchmark_name = "xgate_convergence_madnlp_N51", + runner = runner, + ) + + final_inf = _xgate_infidelity(prob) + primal_inf = result.constraint_violation + + crit = InfidelityConvergence( + target_infidelity = 1e-3, + final_infidelity = final_inf, + primal_infeasibility = primal_inf, + feas_tol = 1e-6, + ) + + result_with_conv = _build_convergence_result(result, crit) + + @printf( + "\n=== X gate convergence (MadNLP) ===\n final_inf=%.3e cviol=%.3e wall=%.3fs converged=%s\n", + final_inf, + primal_inf, + result_with_conv.wall_time_s, + converged(crit), + ) + + @test converged(result_with_conv.convergence) == true + + results_dir = joinpath(@__DIR__, "results") + saved = save_results(results_dir, "xgate_convergence_madnlp_N51", [result_with_conv]) + println(" Saved $(saved)") + + rows = compare_convergence([result_with_conv]) + @test length(rows) == 1 + @test rows[1].converged == true +end diff --git a/benchmark/convergence/problem_utils.jl b/benchmark/convergence/problem_utils.jl new file mode 100644 index 0000000..7dab748 --- /dev/null +++ b/benchmark/convergence/problem_utils.jl @@ -0,0 +1,111 @@ +# Shared helpers for the convergence test items. +# Included by each @testitem via +# include(joinpath(@__DIR__, "problem_utils.jl")) + +""" + _make_xgate_prob(; N=51, seed=42) + +X-gate-style bilinear state-transfer problem: 4D real Bloch-like rep with +`x_init = [1,0,0,0]`, `x_goal = [0,1,0,0]`. Mirrors the shape of +`get_seeded_prob` in `test/solver_test_utils.jl` — terminal cost pulls `x` +toward the goal, and bounds/regularizers are sized so both Ipopt and MadNLP +actually drive infidelity below the convergence target. +""" +function _make_xgate_prob(; N::Int = 51, seed::Int = 42) + Random.seed!(seed) + Δt = 0.1 + u_bound = 1.0 + ω = 0.1 + Gx = sparse(Float64[0 0 0 1; 0 0 1 0; 0 -1 0 0; -1 0 0 0]) + Gy = sparse(Float64[0 -1 0 0; 1 0 0 0; 0 0 0 -1; 0 0 1 0]) + Gz = sparse(Float64[0 0 1 0; 0 0 0 -1; -1 0 0 0; 0 1 0 0]) + G(u) = ω * Gz + u[1] * Gx + u[2] * Gy + + x_init = [1.0, 0.0, 0.0, 0.0] + x_goal = [0.0, 1.0, 0.0, 0.0] + + traj = NamedTrajectory( + ( + x = 2rand(4, N) .- 1, + u = u_bound * (2rand(2, N) .- 1), + du = randn(2, N), + ddu = randn(2, N), + Δt = fill(Δt, N), + ); + controls = (:ddu, :Δt), + timestep = :Δt, + bounds = (u = (-u_bound, u_bound), Δt = (Δt, Δt)), + initial = (x = x_init, u = zeros(2)), + final = (u = zeros(2),), + goal = (x = x_goal,), + ) + integrators = [ + BilinearIntegrator(G, :x, :u, traj), + DerivativeIntegrator(:u, :du, traj), + DerivativeIntegrator(:du, :ddu, traj), + ] + J = TerminalObjective(x -> 1e3 * sum(abs2, x - x_goal), :x, traj) + J += QuadraticRegularizer(:u, traj, 1e-2) + J += QuadraticRegularizer(:du, traj, 1e-2) + return DirectTrajOptProblem(traj, J, integrators) +end + +""" + _xgate_infidelity(prob) -> Float64 + +Infidelity = 1 - , clamped to [0, 1]. Cheap because both +vectors are unit-norm in this representation. +""" +_xgate_infidelity(prob) = clamp( + 1.0 - LinearAlgebra.dot(prob.trajectory.x[:, end], [0.0, 1.0, 0.0, 0.0]), + 0.0, + 1.0, +) + +""" + _build_convergence_result(result::BenchmarkResult, crit::ConvergenceCriterion; + iterations::Union{Nothing,Int}=nothing) + +Return a copy of `result` with `crit` attached as its `convergence` field +(optionally overriding `iterations`). `BenchmarkResult` is immutable, so we +rebuild it positionally; pulling this out of the testitems keeps them +readable. +""" +function _build_convergence_result( + result::HarmoniqsBenchmarks.BenchmarkResult, + crit::HarmoniqsBenchmarks.ConvergenceCriterion; + iterations::Union{Nothing,Int} = nothing, +) + iters = iterations === nothing ? result.iterations : iterations + return HarmoniqsBenchmarks.BenchmarkResult( + package = result.package, + package_version = result.package_version, + commit = result.commit, + benchmark_name = result.benchmark_name, + N = result.N, + state_dim = result.state_dim, + control_dim = result.control_dim, + n_constraints = result.n_constraints, + n_variables = result.n_variables, + wall_time_s = result.wall_time_s, + iterations = iters, + objective_value = result.objective_value, + constraint_violation = result.constraint_violation, + solver_status = result.solver_status, + solver = result.solver, + total_allocations_bytes = result.total_allocations_bytes, + total_allocs_count = result.total_allocs_count, + gc_time_ns = result.gc_time_ns, + gc_count = result.gc_count, + gc_full_count = result.gc_full_count, + peak_rss_delta_bytes = result.peak_rss_delta_bytes, + live_heap_delta_bytes = result.live_heap_delta_bytes, + oom_margin_bytes = result.oom_margin_bytes, + solver_options = result.solver_options, + convergence = crit, + julia_version = result.julia_version, + timestamp = result.timestamp, + runner = result.runner, + n_threads = result.n_threads, + ) +end diff --git a/benchmark/convergence/report.jl b/benchmark/convergence/report.jl new file mode 100644 index 0000000..55293e9 --- /dev/null +++ b/benchmark/convergence/report.jl @@ -0,0 +1,13 @@ +# Post-process the convergence suite's saved artifacts into display surfaces. +# +# julia --project=benchmark/convergence benchmark/convergence/report.jl +# +# Reuses the shared BenchmarkReporting module (benchmark/BenchmarkUtils.jl): +# writes benchmark/convergence/results/bench.json (github-action-benchmark +# schema, incl. per-result [iters]/[infidelity] series) and appends a markdown +# table to $GITHUB_STEP_SUMMARY when run in CI. +using HarmoniqsBenchmarks +include(joinpath(@__DIR__, "..", "BenchmarkUtils.jl")) +using .BenchmarkReporting + +BenchmarkReporting.write_report(joinpath(@__DIR__, "results"))