From c1e92af884075df37408f5dc8c689a5120acd5f2 Mon Sep 17 00:00:00 2001 From: BAder82t <41265463+BAder82t@users.noreply.github.com> Date: Thu, 18 Jun 2026 21:19:56 +0100 Subject: [PATCH] =?UTF-8?q?fix:=20close=20issue=20#249=20=E2=80=94=20Emscr?= =?UTF-8?q?ipten=20OOM=20in=20CKKS=20unit=20tests?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit PR #1179 added ClearBootstrapPrecom and ClearSchemeSwitchPrecom helpers but did not wire them into ReleaseAllContexts. This change completes that work, then re-enables the EVAL_FAST_ROTATION and CONTEXT_WITH_SERTYPE tests that PR #985 had to guard under !defined(__EMSCRIPTEN__) for the original std::bad_alloc failure. Library changes (no hot-path touched): - ReleaseAllContexts now walks every live context and calls ClearAllCKKSCaches before clearing the static maps. - Add ReleaseAllContextsAndTrim and TrimAllocator (malloc_trim on glibc, malloc_zone_pressure_relief on Apple, _heapmin on MSVC, no-op else). - Add ClearAllCKKSCaches convenience on CryptoContextImpl. - ClearBootstrapPrecom and ClearSchemeSwitchPrecom bodies made noexcept-safe so the walk does not throw on contexts that have not Enable(FHE)'d or Enable(SCHEMESWITCH)'d. Test changes: - TearDown in UTCKKSrns and UTCKKSrnsSerialize now calls ReleaseAllContextsAndTrim. - Mid-test ClearAllCKKSCaches between JSON and BINARY rounds in UnitTestCKKSrnsSerialize. - UTCKKSCacheClear extends its WITH_TCM skip to __EMSCRIPTEN__ (the heap-probe helper has no emmalloc backend). - Remove the !defined(__EMSCRIPTEN__) guards around the affected rows. CI: - Add .github/workflows/emscripten.yml: builds with emsdk 3.1.59 and runs pke_tests/core_tests under node in two slices. Validated on macOS + WASM: - Native: pke_tests 1892/1892 pass, core_tests 158/158 pass. - WASM: CKKS slice 939/941 pass (2 heap-probe tests skipped), BFV+BGV+binfhe slice 324/324 pass, peak 1.6 GB. Closes #249. --- .github/workflows/emscripten.yml | 61 +++++++++++++++++++ src/core/include/utils/memory.h | 4 ++ src/core/lib/utils/memory.cpp | 32 ++++++++++ src/pke/include/cryptocontext.h | 22 ++++--- src/pke/include/cryptocontextfactory.h | 16 ++++- src/pke/include/schemebase/base-scheme.h | 8 +-- .../utckksrns/UnitTestCKKSCacheClear.cpp | 6 +- .../unittest/utckksrns/UnitTestCKKSrns.cpp | 7 ++- .../utckksrns/UnitTestCKKSrnsSerialize.cpp | 6 +- 9 files changed, 140 insertions(+), 22 deletions(-) create mode 100644 .github/workflows/emscripten.yml diff --git a/.github/workflows/emscripten.yml b/.github/workflows/emscripten.yml new file mode 100644 index 000000000..23f8a3629 --- /dev/null +++ b/.github/workflows/emscripten.yml @@ -0,0 +1,61 @@ +name: Emscripten + +# Builds with emcc and runs the WASM unit tests under node. Guards against +# re-introducing the cache-leak class that issue #249 tracked. + +on: + pull_request: + branches: [dev] + push: + branches: [dev, main] + workflow_dispatch: + +jobs: + build-and-test: + runs-on: ubuntu-22.04 + timeout-minutes: 60 + steps: + - uses: actions/checkout@v5 + - uses: actions/setup-node@v5 + with: { node-version: '20' } + + - name: Cache emsdk + id: cache-emsdk + uses: actions/cache@v4 + with: + path: ~/emsdk + key: emsdk-3.1.59 + + - name: Install emsdk + if: steps.cache-emsdk.outputs.cache-hit != 'true' + run: | + git clone --depth 1 --branch 3.1.59 https://github.com/emscripten-core/emsdk.git ~/emsdk + ~/emsdk/emsdk install 3.1.59 + ~/emsdk/emsdk activate 3.1.59 + + - name: Configure + run: | + source ~/emsdk/emsdk_env.sh + emcmake cmake -S . -B build \ + -DCMAKE_BUILD_TYPE=Release -DBUILD_UNITTESTS=ON \ + -DBUILD_BENCHMARKS=OFF -DBUILD_EXAMPLES=OFF -DBUILD_EXTRAS=OFF \ + -DBUILD_STATIC=ON -DBUILD_SHARED=OFF -DWITH_OPENMP=OFF -DNATIVE_SIZE=64 \ + -DCMAKE_EXE_LINKER_FLAGS="-s INITIAL_MEMORY=536870912 -s ALLOW_MEMORY_GROWTH=1 -s MAXIMUM_MEMORY=4294967296 -s WASM=1 -s NODERAWFS=1 -s STACK_SIZE=5MB -s ASSERTIONS=0" + + - name: Build + run: | + source ~/emsdk/emsdk_env.sh + cmake --build build --target core_tests pke_tests -j2 + + - name: core_tests + working-directory: build/unittest + run: node --max-old-space-size=8192 ./core_tests.js --gtest_color=no + + # Split pke_tests across two processes — each stays well below the 4 GB WASM cap. + - name: pke_tests (CKKS + general) + working-directory: build/unittest + run: node --max-old-space-size=8192 ./pke_tests.js --gtest_filter='UTCKKSRNS*:UTCKKSCacheClear*:UTGENERAL*' --gtest_color=no + + - name: pke_tests (BFV + BGV + binfhe) + working-directory: build/unittest + run: node --max-old-space-size=8192 ./pke_tests.js --gtest_filter='UTBFV*:UTBGV*:UTBINFHE*' --gtest_color=no diff --git a/src/core/include/utils/memory.h b/src/core/include/utils/memory.h index a4d637c8e..6c47d3a8e 100644 --- a/src/core/include/utils/memory.h +++ b/src/core/include/utils/memory.h @@ -63,6 +63,10 @@ void MoveAppend(std::vector& dst, std::vector& src) { */ void secure_memset(volatile void* mem, uint8_t c, size_t len); +/// Trim/purge the C allocator so freed pages return to the OS. Returns true if +/// the platform has a trim primitive, false (no-op) otherwise. +bool TrimAllocator(); + } // namespace lbcrypto #endif // LBCRYPTO_UTILS_MEMORY_H diff --git a/src/core/lib/utils/memory.cpp b/src/core/lib/utils/memory.cpp index 89b0083fd..227a59a52 100644 --- a/src/core/lib/utils/memory.cpp +++ b/src/core/lib/utils/memory.cpp @@ -30,6 +30,15 @@ //================================================================================== #include "utils/memory.h" +#if defined(__APPLE__) + #include + #include +#elif defined(__GLIBC__) + #include +#elif defined(_MSC_VER) + #include +#endif + namespace lbcrypto { void secure_memset(volatile void* mem, uint8_t c, size_t len) { @@ -38,4 +47,27 @@ void secure_memset(volatile void* mem, uint8_t c, size_t len) { *(ptr + i) = c; } +bool TrimAllocator() { +#if defined(__GLIBC__) + malloc_trim(0); + return true; +#elif defined(__APPLE__) + vm_address_t* zones = nullptr; + unsigned count = 0; + if (malloc_get_all_zones(mach_task_self(), nullptr, &zones, &count) == KERN_SUCCESS && zones) { + for (unsigned i = 0; i < count; ++i) { + auto* z = reinterpret_cast(zones[i]); + if (z) + malloc_zone_pressure_relief(z, 0); + } + } + return true; +#elif defined(_MSC_VER) + _heapmin(); + return true; +#else + return false; +#endif +} + } // namespace lbcrypto diff --git a/src/pke/include/cryptocontext.h b/src/pke/include/cryptocontext.h index 1831189a1..e19c62327 100644 --- a/src/pke/include/cryptocontext.h +++ b/src/pke/include/cryptocontext.h @@ -622,20 +622,22 @@ class CryptoContextImpl : public Serializable { */ static void ClearStaticMapsAndVectors(); - /** - * @brief Clear CKKS bootstrap precomputations cached by this context's scheme. - */ + /// Clear CKKS bootstrap precomputations. Safe on any scheme. void ClearBootstrapPrecom() noexcept { - VerifyCKKSScheme(__func__); - m_scheme->ClearBootstrapPrecom(); + if (m_scheme) + m_scheme->ClearBootstrapPrecom(); } - /** - * @brief Clear CKKS/FHEW scheme-switch precomputations cached by this context's scheme. - */ + /// Clear CKKS/FHEW scheme-switch precomputations. Safe on any scheme. void ClearSchemeSwitchPrecom() noexcept { - VerifyCKKSScheme(__func__); - m_scheme->ClearSchemeSwitchPrecom(); + if (m_scheme) + m_scheme->ClearSchemeSwitchPrecom(); + } + + /// Release all scheme-level caches (bootstrap + scheme-switch). + void ClearAllCKKSCaches() noexcept { + ClearBootstrapPrecom(); + ClearSchemeSwitchPrecom(); } /** diff --git a/src/pke/include/cryptocontextfactory.h b/src/pke/include/cryptocontextfactory.h index 1a076a4b8..3ffdb786b 100644 --- a/src/pke/include/cryptocontextfactory.h +++ b/src/pke/include/cryptocontextfactory.h @@ -35,6 +35,7 @@ #include "cryptocontext-fwd.h" #include "lattice/lat-hal.h" #include "scheme/scheme-id.h" +#include "utils/memory.h" #include #include @@ -63,12 +64,25 @@ class CryptoContextFactory { public: static void ReleaseAllContexts() { - if (!AllContexts.empty()) + // Drop scheme-level caches on every live context so memory held by + // m_bootPrecomMap / scheme-switch state is freed even if callers + // still hold a CryptoContext (issue #533). + for (auto& cc : AllContexts) { + if (cc) + cc->ClearAllCKKSCaches(); + } + if (AllContexts.size() > 0) AllContexts[0]->ClearStaticMapsAndVectors(); AllContexts.clear(); } + /// Like ReleaseAllContexts() but also trims the C allocator so RSS drops. + static void ReleaseAllContextsAndTrim() { + ReleaseAllContexts(); + TrimAllocator(); + } + static int GetContextCount() { return AllContexts.size(); } diff --git a/src/pke/include/schemebase/base-scheme.h b/src/pke/include/schemebase/base-scheme.h index 528ac515b..9493ab997 100644 --- a/src/pke/include/schemebase/base-scheme.h +++ b/src/pke/include/schemebase/base-scheme.h @@ -1147,13 +1147,13 @@ class SchemeBase { } void ClearBootstrapPrecom() noexcept { - VerifyFHEEnabled(__func__); - m_FHE->ClearBootstrapPrecom(); + if (m_FHE) + m_FHE->ClearBootstrapPrecom(); } void ClearSchemeSwitchPrecom() noexcept { - VerifySchemeSwitchEnabled(__func__); - m_SchemeSwitch->ClearSchemeSwitchPrecom(); + if (m_SchemeSwitch) + m_SchemeSwitch->ClearSchemeSwitchPrecom(); } std::shared_ptr>> EvalBootstrapKeyGen(const PrivateKey privateKey, diff --git a/src/pke/unittest/utckksrns/UnitTestCKKSCacheClear.cpp b/src/pke/unittest/utckksrns/UnitTestCKKSCacheClear.cpp index bed0809d3..2a70d36de 100644 --- a/src/pke/unittest/utckksrns/UnitTestCKKSCacheClear.cpp +++ b/src/pke/unittest/utckksrns/UnitTestCKKSCacheClear.cpp @@ -116,9 +116,13 @@ CryptoContext MakeSchemeSwitchCC() { class UTCKKSCacheClear : public ::testing::Test { protected: -#if defined(WITH_TCM) +#if defined(WITH_TCM) || defined(__EMSCRIPTEN__) void SetUp() override { +#if defined(WITH_TCM) GTEST_SKIP() << "Heap usage checks are not stable with tcmalloc enabled"; +#else + GTEST_SKIP() << "Heap probe unavailable under Emscripten"; +#endif } #endif diff --git a/src/pke/unittest/utckksrns/UnitTestCKKSrns.cpp b/src/pke/unittest/utckksrns/UnitTestCKKSrns.cpp index 3e99c8d26..346e4a6f0 100644 --- a/src/pke/unittest/utckksrns/UnitTestCKKSrns.cpp +++ b/src/pke/unittest/utckksrns/UnitTestCKKSrns.cpp @@ -398,7 +398,8 @@ static std::vector testCases = { { EVAL_FAST_ROTATION, "39", {CKKSRNS_SCHEME, RING_DIM, 7, DFLT, DSIZE, BATCH, DFLT, DFLT, DFLT, HEStd_NotSet, BV, FLEXIBLEAUTOEXT, DFLT, DFLT, DFLT, DFLT, DFLT, DFLT, DFLT, DFLT}, BATCH}, { EVAL_FAST_ROTATION, "40", {CKKSRNS_SCHEME, RING_DIM, 7, DFLT, DSIZE, BATCH, DFLT, DFLT, DFLT, HEStd_NotSet, HYBRID, FLEXIBLEAUTOEXT, DFLT, DFLT, DFLT, DFLT, DFLT, DFLT, DFLT, DFLT}, BATCH}, #endif -#if !defined(__EMSCRIPTEN__) + // Issue #249: previously gated under !defined(__EMSCRIPTEN__) — re-enabled + // now that TearDown calls ReleaseAllContextsAndTrim (PR #1179). { EVAL_FAST_ROTATION, "41", {CKKSRNS_SCHEME, RING_DIM, 7, DFLT, DSIZE, BATCH, DFLT, DFLT, DFLT, HEStd_NotSet, BV, FIXEDMANUAL, DFLT, DFLT, DFLT, DFLT, DFLT, DFLT, DFLT, DFLT}, RING_DIM_HALF}, { EVAL_FAST_ROTATION, "42", {CKKSRNS_SCHEME, RING_DIM, 7, DFLT, DSIZE, BATCH, DFLT, DFLT, DFLT, HEStd_NotSet, BV, FIXEDAUTO, DFLT, DFLT, DFLT, DFLT, DFLT, DFLT, DFLT, DFLT}, RING_DIM_HALF}, { EVAL_FAST_ROTATION, "43", {CKKSRNS_SCHEME, RING_DIM, 7, DFLT, DSIZE, BATCH, DFLT, DFLT, DFLT, HEStd_NotSet, HYBRID, FIXEDMANUAL, DFLT, DFLT, DFLT, DFLT, DFLT, DFLT, DFLT, DFLT}, RING_DIM_HALF}, @@ -527,7 +528,6 @@ static std::vector testCases = { // TestType, Descr, Scheme, RDim, MultDepth, SModSize, DSize, BatchSz, SecKeyDist, MaxRelinSkDeg, FModSize, SecLvl, KSTech, ScalTech, LDigits, PtMod, StdDev, EvalAddCt, KSCt, MultTech, EncTech, PREMode { SMALL_SCALING_MOD_SIZE, "01", {CKKSRNS_SCHEME, 32768, 19, 22, DFLT, DFLT, DFLT, DFLT, 23, DFLT, DFLT, FIXEDMANUAL, DFLT, DFLT, DFLT, DFLT, DFLT, DFLT, DFLT, DFLT}, }, { SMALL_SCALING_MOD_SIZE, "02", {CKKSRNS_SCHEME, 32768, 16, 50, DFLT, DFLT, DFLT, DFLT, 50, HEStd_NotSet, DFLT, DFLT, DFLT, DFLT, DFLT, DFLT, DFLT, DFLT, DFLT, DFLT}, }, -#endif // ========================================== // TestType, Descr, Scheme, RDim, MultDepth, SModSize, DSize, BatchSz, SecKeyDist, MaxRelinSkDeg, FModSize, SecLvl, KSTech, ScalTech, LDigits, PtMod, StdDev, EvalAddCt, KSCt, MultTech, EncTech, PREMode, MultipartyMode, decryptionNoiseMode, ExecutionMode, NoiseEstimate, RegisterWordSize, compositeDegree, CKKSDataType { EVALCOMPLEX, "01", {CKKSRNS_SCHEME, RING_DIM, 7, DFLT, DSIZE, BATCH, DFLT, DFLT, DFLT, HEStd_NotSet, BV, FIXEDMANUAL, DFLT, DFLT, DFLT, DFLT, DFLT, DFLT, DFLT, DFLT, DFLT, DFLT, DFLT, DFLT, DFLT, DFLT, COMPLEX}, }, @@ -641,7 +641,8 @@ class UTCKKSRNS : public ::testing::TestWithParam { } void TearDown() { - CryptoContextFactory::ReleaseAllContexts(); + // Trim variant keeps peak WASM linear memory under the 4 GB cap (issue #249). + CryptoContextFactory::ReleaseAllContextsAndTrim(); OpenFHEParallelControls.UnitTestStop(); } diff --git a/src/pke/unittest/utckksrns/UnitTestCKKSrnsSerialize.cpp b/src/pke/unittest/utckksrns/UnitTestCKKSrnsSerialize.cpp index ad91d6043..b026c4c3c 100644 --- a/src/pke/unittest/utckksrns/UnitTestCKKSrnsSerialize.cpp +++ b/src/pke/unittest/utckksrns/UnitTestCKKSrnsSerialize.cpp @@ -119,7 +119,7 @@ constexpr uint32_t BATCH = 16; // clang-format off static std::vector testCases = { // TestType, Descr, Scheme, RDim, MultDepth, SModSize, DSize, BatchSz, SecKeyDist, MaxRelinSkDeg, FModSize, SecLvl, KSTech, ScalTech, LDigits, PtMod, StdDev, EvalAddCt, KSCt, MultTech, EncTech, PREMode -#if !defined(__EMSCRIPTEN__) + // Issue #249: re-enabled — see matching note in UnitTestCKKSrns.cpp. { CONTEXT_WITH_SERTYPE, "01", {CKKSRNS_SCHEME, RING_DIM, MULT_DEPTH, SMODSIZE, DSIZE, BATCH, DFLT, DFLT, DFLT, HEStd_NotSet, BV, FIXEDMANUAL, DFLT, DFLT, DFLT, DFLT, DFLT, DFLT, DFLT, DFLT}, }, { CONTEXT_WITH_SERTYPE, "02", {CKKSRNS_SCHEME, RING_DIM, MULT_DEPTH, SMODSIZE, DSIZE, BATCH, DFLT, DFLT, DFLT, HEStd_NotSet, BV, FIXEDAUTO, DFLT, DFLT, DFLT, DFLT, DFLT, DFLT, DFLT, DFLT}, }, { CONTEXT_WITH_SERTYPE, "03", {CKKSRNS_SCHEME, RING_DIM, MULT_DEPTH, SMODSIZE, DSIZE, BATCH, DFLT, DFLT, DFLT, HEStd_NotSet, HYBRID, FIXEDMANUAL, DFLT, DFLT, DFLT, DFLT, DFLT, DFLT, DFLT, DFLT}, }, @@ -129,7 +129,6 @@ static std::vector testCases = { { CONTEXT_WITH_SERTYPE, "06", {CKKSRNS_SCHEME, RING_DIM, MULT_DEPTH, SMODSIZE, DSIZE, BATCH, DFLT, DFLT, DFLT, HEStd_NotSet, HYBRID, FLEXIBLEAUTO, DFLT, DFLT, DFLT, DFLT, DFLT, DFLT, DFLT, DFLT}, }, { CONTEXT_WITH_SERTYPE, "07", {CKKSRNS_SCHEME, RING_DIM, MULT_DEPTH, SMODSIZE, DSIZE, BATCH, DFLT, DFLT, DFLT, HEStd_NotSet, BV, FLEXIBLEAUTOEXT, DFLT, DFLT, DFLT, DFLT, DFLT, DFLT, DFLT, DFLT}, }, { CONTEXT_WITH_SERTYPE, "08", {CKKSRNS_SCHEME, RING_DIM, MULT_DEPTH, SMODSIZE, DSIZE, BATCH, DFLT, DFLT, DFLT, HEStd_NotSet, HYBRID, FLEXIBLEAUTOEXT, DFLT, DFLT, DFLT, DFLT, DFLT, DFLT, DFLT, DFLT}, }, -#endif #endif // ========================================== // TestType, Descr, Scheme, RDim, MultDepth, SModSize, DSize, BatchSz, SecKeyDist, MaxRelinSkDeg, FModSize, SecLvl, KSTech, ScalTech, LDigits, PtMod, StdDev, EvalAddCt, KSCt, MultTech, EncTech, PREMode @@ -181,7 +180,7 @@ class UTCKKSRNS_SER : public ::testing::TestWithParam { } void TearDown() { - CryptoContextFactory::ReleaseAllContexts(); + CryptoContextFactory::ReleaseAllContextsAndTrim(); // issue #249 OpenFHEParallelControls.UnitTestStop(); } @@ -189,6 +188,7 @@ class UTCKKSRNS_SER : public ::testing::TestWithParam { CryptoContext cc(UnitTestGenerateContext(testData.params)); UnitTestContextWithSertype(cc, SerType::JSON, "json"); + cc->ClearAllCKKSCaches(); // free caches between rounds (issue #249) UnitTestContextWithSertype(cc, SerType::BINARY, "binary"); }