Surf dems (#293)

* Wire Selene HUGR programs through the QIS path

* Make surface native DEM generation ancilla-budget aware

* Normalize Rust doc test README casing

* Improve Selene parity checks and surface DEM tooling

* Align traced-QIS DEM sampling with gate-level reference

* Profile and optimize PECOS sim hot paths

* Optimize PECOS sim control-path overhead

* Speed up byte message builder fast paths

* Remove PECOS profiling API cleanup

* Handle implicit static QIS qubit handles

* Restore surface metadata descriptors and DEM behavior

* Restore DEM bindings and docs compatibility

* Tighten Python test setup and DEM parity

* lint

* Address code review: dedupe, doc fixes, BTreeMap, alias cleanup, loud Selene

* Lint

* Fix Windows path separator mismatch in selene plugin workspace test

* Restore three ignored doctests to compiling examples

* Drop dangling selene_runtime re-export from pecos.engines

* Bump pytest from 8.3.3 to 9.0.3

* Bump hypothesis to 6.152.1 for pytest 9 compatibility

* Bump pytest-cov to 7.1.0 alongside pytest 9

* Align sub-package pyproject pins with workspace (maturin, pytest)

* Separate dev/test/examples dependency groups properly

* Document optional dependency groups in DEVELOPMENT.md

* Pin all workspace test deps to exact versions for reproducibility

* Rename tests/pecos/slr/ast to ast_tests to avoid shadowing stdlib ast

* Add matplotlib to test group for surface-patch render tests

* Refresh Cargo.lock and uv.lock with upstream patch-level bumps

* Apply second-round review fixes: notebook alias, doc comments

* Mirror lookup-only memo comment on search_decomposition cache

* Replace remaining HashMap/HashSet with BTreeMap/BTreeSet in dem_builder::types

* Add DEM build/render benchmarks for decomposition hot paths

* Swap BTreeMap to Vec<Vec<_>> for dense candidates_by_detector hot path

* Lift SingletonDecompositionIndex out of per-call hot path

* Apply Vec<Vec<_>> pattern to GraphlikeDecompositionIndex for consistency

* Rename Error* types to Fault* in dem_builder (ErrorMechanism, ErrorContribution, ErrorSourceType, DecomposedError)

* Add profiling binary for DEM decomposition hot path

* Fix clippy format_push_string in DEM benchmark and profiler

* Address dependabot alerts: rand in isolated lockfiles, drop orphan cuda uv.lock

---------

Co-authored-by: ciaranra <ciaranra@gmail.com>

Author: qciaran

.github/workflows/python-test.yml

@@ -218,5 +218,17 @@ jobs:
     - name: Run Python tests
       run: just pytest
 
+    - name: Run slow Python tests
+      # Only run on the primary Linux/newest-Python cell to keep the full
+      # matrix fast. Dropping this entirely would let the `slow` lane
+      # silently atrophy, which is exactly what the `slow` marker is
+      # supposed to prevent.
+      #
+      # The `3.14` literal below must track the newest entry of the
+      # `python-version` matrix at the top of this file (line ~34). Bump
+      # both together.
+      if: runner.os == 'Linux' && matrix.python-version == '3.14'
+      run: just pytest-slow
+
     - name: Run linting
       run: just lint check

Cargo.lock

@@ -4,7 +4,9 @@ members = [
   "python/pecos-rslib",
   "python/pecos-rslib-cuda",
   "python/pecos-rslib-llvm",
-  "python/selene-plugins/pecos-selene-*",
+  "python/selene-plugins/pecos-selene-clifford-rz",
+  "python/selene-plugins/pecos-selene-stabilizer",
+  "python/selene-plugins/pecos-selene-statevec",
   "julia/pecos-julia-ffi",
   "go/pecos-go-ffi",
   "crates/pecos*",

Cargo.toml

@@ -156,7 +156,7 @@ pytest *args:
     else
         uv run pytest python/pecos-rslib/tests -m "not performance"
         uv run --group numpy-compat pytest python/pecos-rslib/tests -m "numpy and not performance"
-        uv run pytest python/quantum-pecos/tests -m "not optional_dependency"
+        uv run pytest python/quantum-pecos/tests -m "not optional_dependency and not slow"
         uv run pytest python/selene-plugins
     fi
 
@@ -481,6 +481,11 @@ pytest-dep:
     uv run pytest python/pecos-rslib/tests -m "optional_dependency"
     uv run pytest python/quantum-pecos/tests -m "optional_dependency"
 
+# Run the slower integration lane (excluded from the default fast lane)
+[group('test')]
+pytest-slow:
+    uv run pytest python/quantum-pecos/tests -m "slow and not optional_dependency"
+
 
 
 
@@ -497,20 +502,28 @@ setup-quiet:
 sync-deps:
     #!/usr/bin/env bash
     set -euo pipefail
-    # Quick check: if quantum-pecos is importable, deps are likely fine
-    if uv run --frozen python -c "import pecos" 2>/dev/null; then
+    # Quick check: ensure the packages used by the default dev/test lane are importable.
+    # This catches newly added workspace members that an older .venv may be missing.
+    if uv run --frozen python -c "import importlib.util, sys; required = ('pecos', 'pecos_rslib', 'pecos_selene_clifford_rz', 'pecos_selene_stabilizer', 'pecos_selene_statevec'); missing = [name for name in required if importlib.util.find_spec(name) is None]; sys.exit(1 if missing else 0)" 2>/dev/null; then
         exit 0
     fi
-    echo "Python deps not installed, running uv sync..."
+    echo "Python deps incomplete, running uv sync..."
     uv sync --project . --all-packages
 
 [private]
 build-selene:
     #!/usr/bin/env bash
     set -euo pipefail
+    PLUGIN_DIRS=()
+    for DIR in python/selene-plugins/pecos-selene-*/; do
+        [ -d "$DIR" ] || continue
+        [ -f "$DIR/Cargo.toml" ] || continue
+        [ -f "$DIR/pyproject.toml" ] || continue
+        PLUGIN_DIRS+=("$DIR")
+    done
     # Check if any selene source changed since last install
     NEEDS_BUILD=false
-    for DIR in python/selene-plugins/pecos-selene-*/; do
+    for DIR in "${PLUGIN_DIRS[@]}"; do
         PKG=$(basename "$DIR")
         DEST="$DIR/python/${PKG//-/_}/_dist/lib/"
         SO=$(find "$DEST" -name "*.so" 2>/dev/null | head -1 || true)
@@ -531,7 +544,7 @@ build-selene:
     fi
     echo "Building Selene plugins..."
     CARGO_ARGS=""
-    for DIR in python/selene-plugins/pecos-selene-*/; do
+    for DIR in "${PLUGIN_DIRS[@]}"; do
         CARGO_ARGS="$CARGO_ARGS -p $(basename "$DIR")"
     done
     if [ -n "$CARGO_ARGS" ]; then

Justfile

@@ -19,6 +19,7 @@ mod modules {
     pub mod allocation_overhead;
     pub mod clifford_rz;
     pub mod cpu_stabilizer_comparison;
+    pub mod dem_builder;
     pub mod dem_sampler;
     pub mod dod_statevec;
     pub mod quizx_eval;
@@ -51,10 +52,10 @@ use modules::gpu_influence_sampler;
 #[cfg(feature = "cppsparsestab")]
 use modules::sparse_stab_vs_cpp;
 use modules::{
-    allocation_overhead, clifford_rz, cpu_stabilizer_comparison, dem_sampler, dod_statevec,
-    measurement_sampling, native_statevec_comparison, noise_models, pecos_neo_comparison,
-    quizx_eval, rng, set_ops, sparse_stab_w_vs_y, sparse_state_vec, stabilizer_sims,
-    state_vec_sims, surface_code, trig,
+    allocation_overhead, clifford_rz, cpu_stabilizer_comparison, dem_builder, dem_sampler,
+    dod_statevec, measurement_sampling, native_statevec_comparison, noise_models,
+    pecos_neo_comparison, quizx_eval, rng, set_ops, sparse_stab_w_vs_y, sparse_state_vec,
+    stabilizer_sims, state_vec_sims, surface_code, trig,
 };
 
 fn all_benchmarks(c: &mut Criterion) {
@@ -64,6 +65,7 @@ fn all_benchmarks(c: &mut Criterion) {
     quizx_eval::benchmarks(c);
     #[cfg(feature = "cuquantum")]
     cuquantum::benchmarks(c);
+    dem_builder::benchmarks(c);
     dem_sampler::benchmarks(c);
     dod_statevec::benchmarks(c);
     #[cfg(feature = "gpu-sims")]