From f829681b7a6372b3c8f9c6206b03234ab5888b6b Mon Sep 17 00:00:00 2001 From: frstrtr Date: Mon, 13 Jul 2026 08:48:42 +0000 Subject: [PATCH 1/6] ci: auto-heal corrupt boost conan cache on self-hosted runners Add a conan-cache-guard composite action that runs "conan cache check-integrity boost/*" before each conan install and purges the boost package on any manifest mismatch, so the install rebuilds it clean. Fixes the recurring intermittent boost reds (missing asio/log headers, libboost_process/coroutine "expected but not built") caused by a partially-failed conan build leaving an incomplete boost package in the shared ~/.conan2 that persists and poisons later jobs. Wired before every conan install across build.yml, coin-matrix.yml, release.yml, codeql-analysis.yml (presence/build-mode conditions mirrored). --- .github/actions/conan-cache-guard/action.yml | 20 ++++++++++++++++++++ .github/workflows/build.yml | 12 ++++++++++++ .github/workflows/codeql-analysis.yml | 3 +++ .github/workflows/coin-matrix.yml | 3 +++ .github/workflows/release.yml | 6 ++++++ 5 files changed, 44 insertions(+) create mode 100644 .github/actions/conan-cache-guard/action.yml diff --git a/.github/actions/conan-cache-guard/action.yml b/.github/actions/conan-cache-guard/action.yml new file mode 100644 index 000000000..9eb5e90fe --- /dev/null +++ b/.github/actions/conan-cache-guard/action.yml @@ -0,0 +1,20 @@ +name: "Conan cache integrity guard" +description: > + Auto-heal a corrupt/incomplete cached boost package on self-hosted runners. + The self-hosted runners reuse ~/.conan2 across jobs; a partially-failed conan + build can leave an incomplete boost package (missing headers/libs) that + persists and reddens later jobs intermittently. This verifies the cached + boost package against its manifest and purges it on any mismatch so the + following `conan install` rebuilds it clean. No-op on a healthy/empty cache. +runs: + using: composite + steps: + - name: Verify boost conan package integrity (auto-purge if corrupt) + shell: bash + run: | + if conan cache check-integrity "boost/*"; then + echo "conan-cache-guard: boost cache integrity OK" + else + echo "::warning title=conan-cache-guard::boost package integrity check FAILED — purging so conan install rebuilds it clean" + conan remove "boost/*" -c || true + fi diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index eda47e2d1..78552ba30 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -84,6 +84,8 @@ jobs: if: runner.environment != 'github-hosted' run: rm -rf build_ci + - name: Conan cache guard (auto-heal corrupt boost) + uses: ./.github/actions/conan-cache-guard - name: Conan install run: conan install . --build=missing --output-folder=build_ci --settings=build_type=Release @@ -216,6 +218,8 @@ jobs: if: runner.environment != 'github-hosted' run: rm -rf build_asan + - name: Conan cache guard (auto-heal corrupt boost) + uses: ./.github/actions/conan-cache-guard - name: Conan install run: conan install . --build=missing --output-folder=build_asan --settings=build_type=Release @@ -339,6 +343,8 @@ jobs: if: runner.environment != 'github-hosted' run: rm -rf build_bch + - name: Conan cache guard (auto-heal corrupt boost) + uses: ./.github/actions/conan-cache-guard - name: Conan install run: conan install . --build=missing --output-folder=build_bch --settings=build_type=Release @@ -428,6 +434,8 @@ jobs: if: runner.environment != 'github-hosted' run: rm -rf build_bch_asan + - name: Conan cache guard (auto-heal corrupt boost) + uses: ./.github/actions/conan-cache-guard - name: Conan install run: conan install . --build=missing --output-folder=build_bch_asan --settings=build_type=Release @@ -802,6 +810,8 @@ jobs: path: ~/.conan2 key: conan2-windows-msvc194-${{ hashFiles('conanfile.txt') }} + - name: Conan cache guard (auto-heal corrupt boost) + uses: ./.github/actions/conan-cache-guard - name: Conan install run: conan install . --build=missing --output-folder=build_ci @@ -903,6 +913,8 @@ jobs: if: runner.environment != 'github-hosted' run: rm -rf build_dgb_auxdoge + - name: Conan cache guard (auto-heal corrupt boost) + uses: ./.github/actions/conan-cache-guard - name: Conan install run: conan install . --build=missing --output-folder=build_dgb_auxdoge --settings=build_type=Release diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml index 246ab01a7..5b79c6b63 100644 --- a/.github/workflows/codeql-analysis.yml +++ b/.github/workflows/codeql-analysis.yml @@ -90,6 +90,9 @@ jobs: name: Clean stale build dir (self-hosted workspace is reused) run: rm -rf build_codeql + - if: matrix.build-mode == 'manual' + name: Conan cache guard (auto-heal corrupt boost) + uses: ./.github/actions/conan-cache-guard - if: matrix.build-mode == 'manual' name: Install Conan dependencies run: | diff --git a/.github/workflows/coin-matrix.yml b/.github/workflows/coin-matrix.yml index 5e39b4220..cb2a66def 100644 --- a/.github/workflows/coin-matrix.yml +++ b/.github/workflows/coin-matrix.yml @@ -98,6 +98,9 @@ jobs: run: | find ~/.conan2/p -type f \( -name 'conan_package.tgz' -o -name 'conan_sources.tgz' \) -delete 2>/dev/null || true + - name: Conan cache guard (auto-heal corrupt boost) + if: steps.presence.outputs.exists == '1' + uses: ./.github/actions/conan-cache-guard - name: Conan install if: steps.presence.outputs.exists == '1' run: | diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index ad5fab59e..01ed9cee0 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -130,6 +130,9 @@ jobs: conan2-ubuntu24-gcc13-${{ matrix.coin }}- conan2-ubuntu24-gcc13- + - name: Conan cache guard (auto-heal corrupt boost) + if: steps.presence.outputs.exists == '1' + uses: ./.github/actions/conan-cache-guard - name: Conan install if: steps.presence.outputs.exists == '1' run: | @@ -476,6 +479,9 @@ jobs: path: ~/.conan2 key: conan2-windows-msvc194-${{ hashFiles('conanfile.txt') }} + - name: Conan cache guard (auto-heal corrupt boost) + if: steps.presence.outputs.exists == '1' + uses: ./.github/actions/conan-cache-guard - name: Conan install if: steps.presence.outputs.exists == '1' run: conan install . --build=missing --output-folder=build_ci From 46ea150dc81b6cb0ec27354b21bcd329f452b299 Mon Sep 17 00:00:00 2001 From: frstrtr Date: Mon, 13 Jul 2026 09:15:09 +0000 Subject: [PATCH 2/6] ci: make conan-cache-guard OS-aware so it runs on VM217 (no bash on service PATH) Windows self-hosted (VM217) has no bash on the service PATH, so the single shell: bash guard step errored with 'bash: command not found'. Split the composite action by runner.os: bash on POSIX, pwsh on Windows with the equivalent check-integrity/remove and exit 0 to preserve the never-fails invariant. Guard now runs on every lane including win-217. --- .github/actions/conan-cache-guard/action.yml | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-) diff --git a/.github/actions/conan-cache-guard/action.yml b/.github/actions/conan-cache-guard/action.yml index 9eb5e90fe..0a3705596 100644 --- a/.github/actions/conan-cache-guard/action.yml +++ b/.github/actions/conan-cache-guard/action.yml @@ -9,7 +9,11 @@ description: > runs: using: composite steps: - - name: Verify boost conan package integrity (auto-purge if corrupt) + # VM217 (Windows self-hosted) has no bash on the service PATH — same reason + # build.yml Windows conan steps run under pwsh. Split by runner.os so the + # guard runs on every lane incl. win-217. + - name: Verify boost conan package integrity (auto-purge if corrupt) [posix] + if: runner.os != 'Windows' shell: bash run: | if conan cache check-integrity "boost/*"; then @@ -18,3 +22,15 @@ runs: echo "::warning title=conan-cache-guard::boost package integrity check FAILED — purging so conan install rebuilds it clean" conan remove "boost/*" -c || true fi + - name: Verify boost conan package integrity (auto-purge if corrupt) [windows] + if: runner.os == 'Windows' + shell: pwsh + run: | + conan cache check-integrity "boost/*" + if ($LASTEXITCODE -ne 0) { + Write-Output "::warning title=conan-cache-guard::boost package integrity check FAILED — purging so conan install rebuilds it clean" + conan remove "boost/*" -c + } else { + Write-Output "conan-cache-guard: boost cache integrity OK" + } + exit 0 From b5a0b1760400a9cf6de2fdb556fd168c5bdad77d Mon Sep 17 00:00:00 2001 From: frstrtr Date: Mon, 13 Jul 2026 13:05:49 +0000 Subject: [PATCH 3/6] ci: conan-cache-guard Windows step uses shell: powershell (VM217 has WinPS 5.1, no pwsh Core) --- .github/actions/conan-cache-guard/action.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/actions/conan-cache-guard/action.yml b/.github/actions/conan-cache-guard/action.yml index 0a3705596..e0bbbb7be 100644 --- a/.github/actions/conan-cache-guard/action.yml +++ b/.github/actions/conan-cache-guard/action.yml @@ -24,7 +24,7 @@ runs: fi - name: Verify boost conan package integrity (auto-purge if corrupt) [windows] if: runner.os == 'Windows' - shell: pwsh + shell: powershell run: | conan cache check-integrity "boost/*" if ($LASTEXITCODE -ne 0) { From 4f12d72213ec64b8786e90005f4231da372481c8 Mon Sep 17 00:00:00 2001 From: frstrtr Date: Mon, 13 Jul 2026 14:03:46 +0000 Subject: [PATCH 4/6] ci: conan-cache-guard Windows step uses shell: cmd (VM217 execution-policy blocks unsigned powershell bodies) cmd has no execution-policy gate, mirroring the windows-2022 fork-fallback. if errorlevel 1 purges the corrupt boost package; exit /b 0 keeps the step non-failing on a healthy or empty cache. --- .github/actions/conan-cache-guard/action.yml | 22 +++++++++++--------- 1 file changed, 12 insertions(+), 10 deletions(-) diff --git a/.github/actions/conan-cache-guard/action.yml b/.github/actions/conan-cache-guard/action.yml index e0bbbb7be..f2fdbafef 100644 --- a/.github/actions/conan-cache-guard/action.yml +++ b/.github/actions/conan-cache-guard/action.yml @@ -9,9 +9,11 @@ description: > runs: using: composite steps: - # VM217 (Windows self-hosted) has no bash on the service PATH — same reason - # build.yml Windows conan steps run under pwsh. Split by runner.os so the - # guard runs on every lane incl. win-217. + # VM217 (Windows self-hosted) has no bash on the service PATH. Split by + # runner.os so the guard runs on every lane incl. win-217. The Windows leg + # uses shell: cmd (not powershell) because VM217 execution-policy blocks + # unsigned script bodies; cmd has no execution-policy gate, matching what + # the windows-2022 fork-fallback already does. - name: Verify boost conan package integrity (auto-purge if corrupt) [posix] if: runner.os != 'Windows' shell: bash @@ -24,13 +26,13 @@ runs: fi - name: Verify boost conan package integrity (auto-purge if corrupt) [windows] if: runner.os == 'Windows' - shell: powershell + shell: cmd run: | conan cache check-integrity "boost/*" - if ($LASTEXITCODE -ne 0) { - Write-Output "::warning title=conan-cache-guard::boost package integrity check FAILED — purging so conan install rebuilds it clean" + if errorlevel 1 ( + echo ::warning title=conan-cache-guard::boost package integrity check FAILED -- purging so conan install rebuilds it clean conan remove "boost/*" -c - } else { - Write-Output "conan-cache-guard: boost cache integrity OK" - } - exit 0 + ) else ( + echo conan-cache-guard: boost cache integrity OK + ) + exit /b 0 From a422f55e6396886c2664448ffe7e8ddc8d1cab39 Mon Sep 17 00:00:00 2001 From: frstrtr Date: Mon, 13 Jul 2026 21:40:40 +0000 Subject: [PATCH 5/6] ci: conan-cache-guard purges wrong-cppstd (gnu17) boost, not just corrupt The per-job profile pin (sed compiler.cppstd=20) sets what conan install REQUESTS but not what it SELECTS: an intact gnu17-built boost in the reused self-hosted cache passes check-integrity, and Conan compatibility fallback re-selects it instead of rebuilding a cppstd=20 boost -> ABI mismatch reddens later jobs (boost.asio thread_pool_/partial_redirect_error, cobalt assertion). Guard now purges boost on any cached compiler.cppstd != 20, making the cppstd=20 pin durable at the cache layer. Mirrored on the Windows cmd leg. --- .github/actions/conan-cache-guard/action.yml | 51 ++++++++++++++------ 1 file changed, 36 insertions(+), 15 deletions(-) diff --git a/.github/actions/conan-cache-guard/action.yml b/.github/actions/conan-cache-guard/action.yml index f2fdbafef..3a1123e5d 100644 --- a/.github/actions/conan-cache-guard/action.yml +++ b/.github/actions/conan-cache-guard/action.yml @@ -1,11 +1,17 @@ name: "Conan cache integrity guard" description: > - Auto-heal a corrupt/incomplete cached boost package on self-hosted runners. - The self-hosted runners reuse ~/.conan2 across jobs; a partially-failed conan - build can leave an incomplete boost package (missing headers/libs) that - persists and reddens later jobs intermittently. This verifies the cached - boost package against its manifest and purges it on any mismatch so the - following `conan install` rebuilds it clean. No-op on a healthy/empty cache. + Auto-heal a corrupt/incomplete OR wrong-config cached boost package on + self-hosted runners. The self-hosted runners reuse ~/.conan2 across jobs. + A partially-failed conan build can leave an incomplete boost package + (missing headers/libs), and an out-of-band/manual `conan install` can leave + an intact-but-wrong-std boost (compiler.cppstd=gnu17) that PASSES an integrity + check yet is ABI-incompatible with the cppstd=20 c2pool TUs. Conan's + compatibility fallback then re-selects that gnu17 boost instead of rebuilding, + reddening later jobs intermittently (the per-job profile cppstd=20 pin cannot + prevent this; it only sets what is *requested*, not what is *selected*). This + purges boost on integrity mismatch OR on any cached cppstd != 20 so the + following `conan install` rebuilds it clean. No-op on a healthy cppstd=20 or + empty cache. runs: using: composite steps: @@ -14,25 +20,40 @@ runs: # uses shell: cmd (not powershell) because VM217 execution-policy blocks # unsigned script bodies; cmd has no execution-policy gate, matching what # the windows-2022 fork-fallback already does. - - name: Verify boost conan package integrity (auto-purge if corrupt) [posix] + - name: Verify boost integrity + cppstd (auto-purge if corrupt or wrong-std) [posix] if: runner.os != 'Windows' shell: bash run: | - if conan cache check-integrity "boost/*"; then - echo "conan-cache-guard: boost cache integrity OK" - else - echo "::warning title=conan-cache-guard::boost package integrity check FAILED — purging so conan install rebuilds it clean" + purge="" + if ! conan cache check-integrity "boost/*"; then + purge="integrity check FAILED" + elif conan list "boost/*:*" -f json 2>/dev/null \ + | grep -oE '"compiler.cppstd": "[^"]*"' \ + | grep -qv '"compiler.cppstd": "20"'; then + purge="a cached binary was built with compiler.cppstd != 20 (wrong-std; Conan compatibility fallback would re-select it)" + fi + if [ -n "$purge" ]; then + echo "::warning title=conan-cache-guard::boost $purge -- purging so conan install rebuilds it clean" conan remove "boost/*" -c || true + else + echo "conan-cache-guard: boost cache OK (integrity + cppstd=20)" fi - - name: Verify boost conan package integrity (auto-purge if corrupt) [windows] + - name: Verify boost integrity + cppstd (auto-purge if corrupt or wrong-std) [windows] if: runner.os == 'Windows' shell: cmd run: | + set purge= conan cache check-integrity "boost/*" - if errorlevel 1 ( - echo ::warning title=conan-cache-guard::boost package integrity check FAILED -- purging so conan install rebuilds it clean + if errorlevel 1 set purge=integrity check FAILED + if not defined purge ( + conan list "boost/*:*" -f json > "%TEMP%\cg_boost.json" 2>nul + findstr /C:"compiler.cppstd" "%TEMP%\cg_boost.json" | findstr /V /C:": \"20\"" >nul + if not errorlevel 1 set purge=a cached binary was built with compiler.cppstd ^!= 20 + ) + if defined purge ( + echo ::warning title=conan-cache-guard::boost %purge% -- purging so conan install rebuilds it clean conan remove "boost/*" -c ) else ( - echo conan-cache-guard: boost cache integrity OK + echo conan-cache-guard: boost cache OK ^(integrity + cppstd=20^) ) exit /b 0 From a905da5f59e0f3359e885e60a3b2734c3b5db91a Mon Sep 17 00:00:00 2001 From: frstrtr Date: Mon, 13 Jul 2026 22:13:07 +0000 Subject: [PATCH 6/6] ci: flock-serialize conan-cache-guard boost purge+rebuild on shared cache The cppstd!=20 conditional purge is correct, but on a cold VM905 cache all 8 Linux runners share one ~/.conan2 and purge+rebuild boost concurrently, racing (Reference already exists / boost headers deleted mid-build). Wrap the whole check+purge+rebuild in an flock so exactly one job rebuilds a clean cppstd=20 boost from source and the rest wait then cache-hit. Windows (single VM217 runner) has no herd, leg unchanged. --- .github/actions/conan-cache-guard/action.yml | 52 +++++++++++++------- 1 file changed, 34 insertions(+), 18 deletions(-) diff --git a/.github/actions/conan-cache-guard/action.yml b/.github/actions/conan-cache-guard/action.yml index 3a1123e5d..4486fe5f0 100644 --- a/.github/actions/conan-cache-guard/action.yml +++ b/.github/actions/conan-cache-guard/action.yml @@ -1,29 +1,33 @@ name: "Conan cache integrity guard" description: > Auto-heal a corrupt/incomplete OR wrong-config cached boost package on - self-hosted runners. The self-hosted runners reuse ~/.conan2 across jobs. - A partially-failed conan build can leave an incomplete boost package - (missing headers/libs), and an out-of-band/manual `conan install` can leave - an intact-but-wrong-std boost (compiler.cppstd=gnu17) that PASSES an integrity - check yet is ABI-incompatible with the cppstd=20 c2pool TUs. Conan's - compatibility fallback then re-selects that gnu17 boost instead of rebuilding, - reddening later jobs intermittently (the per-job profile cppstd=20 pin cannot - prevent this; it only sets what is *requested*, not what is *selected*). This - purges boost on integrity mismatch OR on any cached cppstd != 20 so the - following `conan install` rebuilds it clean. No-op on a healthy cppstd=20 or - empty cache. + self-hosted runners. The self-hosted Linux runners (VM905) reuse ONE + ~/.conan2 across up to 8 concurrent jobs. A partially-failed conan build + can leave an incomplete boost package (missing headers/libs), and an + out-of-band/manual `conan install` can leave an intact-but-wrong-std boost + (compiler.cppstd=gnu17) that PASSES an integrity check yet is ABI-incompatible + with the cppstd=20 c2pool TUs. Conan compatibility fallback then re-selects + that gnu17 boost instead of rebuilding, reddening later jobs intermittently + (the per-job profile cppstd=20 pin only sets what is *requested*, not what is + *selected*). This purges boost on integrity mismatch OR on any cached + cppstd != 20 and rebuilds a clean cppstd=20 boost. On the shared Linux cache + the whole check+purge+rebuild is serialized under an flock so exactly ONE + cold-cache job rebuilds boost while the rest wait and then cache-hit -- without + the lock, every concurrent job purges+rebuilds into the same cache at once and + races ("Reference ... already exists" / boost headers deleted mid-build). + No-op (fast, unlocked-fast) on a healthy cppstd=20 cache. runs: using: composite steps: - # VM217 (Windows self-hosted) has no bash on the service PATH. Split by - # runner.os so the guard runs on every lane incl. win-217. The Windows leg - # uses shell: cmd (not powershell) because VM217 execution-policy blocks - # unsigned script bodies; cmd has no execution-policy gate, matching what - # the windows-2022 fork-fallback already does. - - name: Verify boost integrity + cppstd (auto-purge if corrupt or wrong-std) [posix] + - name: Verify boost integrity + cppstd, rebuild-once under lock (shared cache) [posix] if: runner.os != 'Windows' shell: bash run: | + # Serialize check+purge+rebuild across all jobs sharing ~/.conan2 on + # this self-hosted host (mirrors the apt-get flock guard, PR #670). + lock="${HOME}/.conan2/.c2pool-boost-guard.lock" + exec 9>"$lock" 2>/dev/null || exec 9>/tmp/.c2pool-boost-guard.lock + flock 9 purge="" if ! conan cache check-integrity "boost/*"; then purge="integrity check FAILED" @@ -33,15 +37,27 @@ runs: purge="a cached binary was built with compiler.cppstd != 20 (wrong-std; Conan compatibility fallback would re-select it)" fi if [ -n "$purge" ]; then - echo "::warning title=conan-cache-guard::boost $purge -- purging so conan install rebuilds it clean" + echo "::warning title=conan-cache-guard::boost $purge -- purging + rebuilding cppstd=20 boost from source under lock" conan remove "boost/*" -c || true + # Rebuild boost NOW, while holding the lock, using the job's already + # cppstd=20-pinned default profile, so every subsequent (unlocked) + # `conan install` is a clean cache hit and no job ever installs against + # a boost another job is deleting. Log is intentionally not suppressed + # so the from-source rebuild is visible as evidence. + conan install . --build="boost/*" --settings=build_type=Release \ + --output-folder="${RUNNER_TEMP:-/tmp}/conan-boost-warm" else echo "conan-cache-guard: boost cache OK (integrity + cppstd=20)" fi + flock -u 9 - name: Verify boost integrity + cppstd (auto-purge if corrupt or wrong-std) [windows] if: runner.os == 'Windows' shell: cmd run: | + rem VM217 is a single Windows runner -- no concurrent siblings share its + rem cache, so no thundering-herd race; a purge here is healed by this + rem job's own following conan install. cmd (not powershell) because + rem VM217 execution-policy blocks unsigned script bodies. set purge= conan cache check-integrity "boost/*" if errorlevel 1 set purge=integrity check FAILED