benchmarks: convergence suite v1 (Ipopt + MadNLP on X gate)

.github/workflows/benchmark-convergence.yml

@@ -0,0 +1,80 @@
+name: Convergence Benchmarks
+on:
+  push:
+    tags: ['v*']
+  pull_request:
+    paths:
+      - 'src/**'
+      - 'benchmark/convergence/**'
+      - '.github/workflows/benchmark-convergence.yml'
+  workflow_dispatch:
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: ${{ startsWith(github.ref, 'refs/pull/') }}
+
+jobs:
+  convergence:
+    name: Convergence suite (Ipopt + MadNLP)
+    runs-on: ubuntu-latest
+    timeout-minutes: 30
+    permissions:
+      actions: write
+      # contents: write so github-action-benchmark can auto-push the time-series
+      # to gh-pages (gated to main below); pull-requests: write for its
+      # regression alert comments.
+      contents: write
+      pull-requests: write
+    steps:
+      - uses: actions/checkout@v6
+
+      - uses: julia-actions/setup-julia@v2
+        with:
+          version: '1.12'
+          arch: x64
+
+      - uses: julia-actions/cache@v2
+
+      - name: Instantiate convergence environment
+        run: julia --project=benchmark/convergence -e 'using Pkg; Pkg.instantiate(); Pkg.precompile()'
+
+      - name: Run convergence benchmarks
+        env:
+          BENCHMARK_RUNNER: github-actions
+        run: |
+          julia --project=benchmark/convergence -t auto -e '
+            using TestItemRunner
+            TestItemRunner.run_tests("benchmark/convergence/")
+          '
+
+      - name: Generate convergence report
+        if: always()
+        run: julia --project=benchmark/convergence benchmark/convergence/report.jl
+
+      - name: Upload convergence artifacts
+        if: always()
+        uses: actions/upload-artifact@v4
+        with:
+          name: convergence-${{ github.event.pull_request.number || github.ref_name }}-${{ github.sha }}
+          path: benchmark/convergence/results/
+          retention-days: 90
+
+      # Publish to /bench-convergence on gh-pages (separate dashboard from the
+      # timing suite's /bench). Per-commit time series for wall/alloc/iters and
+      # achieved infidelity, with 120% regression alerts. save-data-file /
+      # auto-push gated to main so PR runs render a comparison only.
+      - name: Publish convergence benchmark to gh-pages
+        if: success() && hashFiles('benchmark/convergence/results/bench.json') != ''
+        uses: benchmark-action/github-action-benchmark@v1
+        with:
+          name: DirectTrajOpt.jl convergence
+          tool: customSmallerIsBetter
+          output-file-path: benchmark/convergence/results/bench.json
+          github-token: ${{ secrets.GITHUB_TOKEN }}
+          gh-pages-branch: gh-pages
+          benchmark-data-dir-path: bench-convergence
+          alert-threshold: '120%'
+          comment-on-alert: true
+          fail-on-alert: false
+          save-data-file: ${{ github.ref == 'refs/heads/main' }}
+          auto-push: ${{ github.ref == 'refs/heads/main' }}

.github/workflows/benchmark.yml

@@ -38,16 +38,21 @@ jobs:
       - name: Instantiate benchmark environment
         run: julia --project=benchmark -e 'using Pkg; Pkg.instantiate()'
 
-      - name: Run benchmarks (excluding alloc profile)
+      - name: Run timing benchmarks only
         env:
           BENCHMARK_RUNNER: github-actions
         run: |
           julia --project=benchmark -t auto -e '
             using TestItemRunner
-            # The alloc-profile testitem has its own workflow
-            # (.github/workflows/alloc-profile.yml) — its Profile.Allocs run is
-            # far slower and would blow this suite''s 60-min budget.
-            TestItemRunner.run_tests("benchmark/"; filter = ti -> !occursin("alloc_profile", ti.filename))
+            # Run only the timing testitems (benchmark/benchmarks.jl). The
+            # alloc-profile (benchmark/alloc_profile.jl) and convergence
+            # (benchmark/convergence/) suites each have their own workflow +
+            # project env (alloc-profile.yml, benchmark-convergence.yml); running
+            # them here would use the wrong env and/or blow this suite''s 60-min
+            # budget.
+            TestItemRunner.run_tests("benchmark/";
+                filter = ti -> !occursin("alloc_profile", ti.filename) &&
+                               !occursin("/convergence/", ti.filename))
           '
 
       - name: Generate benchmark report

.github/workflows/docs.yml

@@ -18,6 +18,11 @@ jobs:
   docs:
     name: Documentation
     runs-on: ubuntu-latest
+    # Bound the build+deploy. makedocs is ~5 min; the rest is the deploydocs
+    # gh-pages push. Without this, a transient deploy/git-push stall runs to the
+    # 6h default and blocks the Documentation concurrency group (observed
+    # 2026-06-08: a deploy hung 90+ min, queueing the next main build behind it).
+    timeout-minutes: 30
     permissions:
       contents: write
       statuses: write

benchmark/convergence/.gitignore

@@ -0,0 +1,2 @@
+results/
+Manifest.toml

benchmark/convergence/Project.toml

@@ -0,0 +1,24 @@
+[deps]
+Dates = "ade2ca70-3891-5945-98fb-dc099432e06a"
+DirectTrajOpt = "c823fa1f-8872-4af5-b810-2b9b72bbbf56"
+ExponentialAction = "e24c0720-ea99-47e8-929e-571b494574d3"
+HarmoniqsBenchmarks = "f45d0b76-2d23-4568-9599-481e0da131db"
+Ipopt = "b6b21f68-93f8-5de0-b562-5493be1d77c9"
+JLD2 = "033835bb-8acc-5ee8-8aae-3f567f8a3819"
+LinearAlgebra = "37e2e46d-f89d-539d-b4ee-838fcccc9c8e"
+MadNLP = "2621e9c9-9eb4-46b1-8089-e8c72242dfb6"
+NamedTrajectories = "538bc3a1-5ab9-4fc3-b776-35ca1e893e08"
+Printf = "de0858da-6303-5e67-8744-51eddeeeb8d7"
+Random = "9a3f8284-a2c9-5f02-9a11-845980a1fd5c"
+SparseArrays = "2f01184e-e22b-5df5-ae63-d93ebab69eaf"
+Statistics = "10745b16-79ce-11e8-11f9-7d13ad32a3b2"
+TestItemRunner = "f8b46487-2199-4994-9208-9a1283c18c0a"
+TestItems = "1c621080-faea-4a02-84b6-bbd5e436b8fe"
+
+[sources]
+DirectTrajOpt = {path = "../.."}
+# HBJ v0.2.0 not yet registered in General; pin to a specific commit so
+# convergence results are reproducible across CI re-runs. Bump this SHA
+# (and the local Manifest) when HBJ ships a new feature we want to use.
+# Drop in favor of `[compat]` once HBJ registers in General.
+HarmoniqsBenchmarks = {url = "https://github.com/harmoniqs/HarmoniqsBenchmarks.jl", rev = "c38418cb7f932f2ff9a9c6c6eacf9a11ff1018c1"}

benchmark/convergence/README.md

@@ -0,0 +1,38 @@
+# DirectTrajOpt — Convergence Benchmarks
+
+Convergence-quality benchmarks built on HarmoniqsBenchmarks.jl v0.2.0's new
+convergence API (`InfidelityConvergence`, `ipopt_capture`,
+`compare_convergence`). Each `@testitem` runs a small X-gate state-transfer
+on Ipopt or MadNLP and asserts the result meets a problem-specific success
+bar before saving a JLD2 artifact under `benchmark/convergence/results/`.
+
+**Scope — this is a regression/sanity baseline, not a solver-difficulty
+benchmark.** The 1-qubit X gate is deliberately easy: both Ipopt and MadNLP
+drive it to ~machine precision (infidelity ~1e-11 / ~1e-14). Its job is to (a)
+catch a regression that stops a solver from converging on a known-good problem,
+and (b) track each solver's wall-time / allocations / iterations-to-converge
+over commits via the dashboard. Harder, solver-discriminating problems
+(multi-qubit, cavity, free-phase) live in the platform demos and the
+Piccolissimo Altissimo suite — not here.
+
+## Running locally
+
+```bash
+# from DirectTrajOpt.jl root
+julia --project=benchmark/convergence -e 'using Pkg; Pkg.instantiate()'
+julia --project=benchmark/convergence -e '
+    using TestItemRunner
+    @run_package_tests filter = ti -> occursin("convergence", ti.name)
+'
+```
+
+## What's covered
+
+- **X gate convergence: Ipopt** — uses `ipopt_capture()` to grab final
+  `iter_count` + `inf_pr`, builds an `InfidelityConvergence`, passes it
+  through `benchmark_solve!` to populate `BenchmarkResult.convergence`.
+- **X gate convergence: MadNLP** — same problem, MadNLP solver. No capture
+  hook yet, so `primal_infeasibility` is taken from the post-solve
+  evaluator's `constraint_violation`.
+
+Atoms / spin-qubit / bosonic demo problems land in separate follow-up PRs.

benchmark/convergence/convergence.jl

@@ -0,0 +1,118 @@
+using TestItems
+
+@testitem "X gate convergence: Ipopt" begin
+    using HarmoniqsBenchmarks
+    using DirectTrajOpt
+    using NamedTrajectories
+    using SparseArrays, ExponentialAction, Random, Dates, Printf, LinearAlgebra
+
+    include(joinpath(@__DIR__, "problem_utils.jl"))
+
+    runner = get(ENV, "BENCHMARK_RUNNER", "local")
+
+    prob = _make_xgate_prob(; N = 51, seed = 42)
+
+    # Wire Ipopt callback that captures final iter_count + inf_pr.
+    state, cb = ipopt_capture()
+    ipopt_opts = IpoptOptions(max_iter = 500, print_level = 0)
+
+    # benchmark_solve! forwards extra kwargs to DirectTrajOpt.solve!, so we
+    # can inject the capture callback through the same call.
+    result = benchmark_solve!(
+        prob,
+        ipopt_opts;
+        benchmark_name = "xgate_convergence_ipopt_N51",
+        runner = runner,
+        callback = cb,
+    )
+
+    final_inf = _xgate_infidelity(prob)
+    primal_inf = ipopt_primal_infeasibility(state)
+    iters = ipopt_iterations(state)
+
+    crit = InfidelityConvergence(
+        target_infidelity = 1e-3,
+        final_infidelity = final_inf,
+        primal_infeasibility = primal_inf,
+        feas_tol = 1e-6,
+    )
+
+    result_with_conv = _build_convergence_result(result, crit; iterations = iters)
+
+    @printf(
+        "\n=== X gate convergence (Ipopt) ===\n  iters=%d  final_inf=%.3e  inf_pr=%.3e  wall=%.3fs  converged=%s\n",
+        iters,
+        final_inf,
+        primal_inf,
+        result_with_conv.wall_time_s,
+        converged(crit),
+    )
+
+    @test converged(result_with_conv.convergence) == true
+
+    results_dir = joinpath(@__DIR__, "results")
+    saved = save_results(results_dir, "xgate_convergence_ipopt_N51", [result_with_conv])
+    println("  Saved $(saved)")
+
+    # Exercise the reporting path.
+    rows = compare_convergence([result_with_conv])
+    @test length(rows) == 1
+    @test rows[1].converged == true
+end
+
+
+@testitem "X gate convergence: MadNLP" begin
+    using HarmoniqsBenchmarks
+    using DirectTrajOpt
+    using NamedTrajectories
+    using SparseArrays, ExponentialAction, Random, Dates, Printf, LinearAlgebra
+    import MadNLP
+
+    include(joinpath(@__DIR__, "problem_utils.jl"))
+
+    runner = get(ENV, "BENCHMARK_RUNNER", "local")
+
+    prob = _make_xgate_prob(; N = 51, seed = 42)
+
+    madnlp_opts = MadNLPOptions(max_iter = 500, print_level = 6)
+
+    # MadNLP doesn't have an ipopt_capture analogue yet — use the post-solve
+    # constraint_violation that benchmark_solve! already extracted as the
+    # primal-infeasibility proxy.
+    result = benchmark_solve!(
+        prob,
+        madnlp_opts;
+        benchmark_name = "xgate_convergence_madnlp_N51",
+        runner = runner,
+    )
+
+    final_inf = _xgate_infidelity(prob)
+    primal_inf = result.constraint_violation
+
+    crit = InfidelityConvergence(
+        target_infidelity = 1e-3,
+        final_infidelity = final_inf,
+        primal_infeasibility = primal_inf,
+        feas_tol = 1e-6,
+    )
+
+    result_with_conv = _build_convergence_result(result, crit)
+
+    @printf(
+        "\n=== X gate convergence (MadNLP) ===\n  final_inf=%.3e  cviol=%.3e  wall=%.3fs  converged=%s\n",
+        final_inf,
+        primal_inf,
+        result_with_conv.wall_time_s,
+        converged(crit),
+    )
+
+    @test converged(result_with_conv.convergence) == true
+
+    results_dir = joinpath(@__DIR__, "results")
+    saved = save_results(results_dir, "xgate_convergence_madnlp_N51", [result_with_conv])
+    println("  Saved $(saved)")
+
+    rows = compare_convergence([result_with_conv])
+    @test length(rows) == 1
+    @test rows[1].converged == true
+end