Move configuring V8 tests into CMake (#803)

I kept forgetting how to replicate CI so make it a flag to CMake which
slims down CI configuration and moves more into something that can be
more easily replicated locally

Author: alexcrichton

.github/workflows/main.yml

@@ -90,15 +90,13 @@ jobs:
             os: ubuntu-24.04
             clang_version: 20
             test: true
-            test_with_v8: true
-            args: -DTARGET_TRIPLE=wasm32-wasip1
+            args: -DTARGET_TRIPLE=wasm32-wasip1 -DTEST_WITH_V8=ON
 
           - name: Test wasm32-wasip1-threads in V8
             os: ubuntu-24.04
             clang_version: 20
             test: true
-            test_with_v8: true
-            args: -DTARGET_TRIPLE=wasm32-wasip1-threads
+            args: -DTARGET_TRIPLE=wasm32-wasip1-threads -DTEST_WITH_V8=ON
 
           - name: Test wasm32-wasi-simd
             os: ubuntu-24.04
@@ -177,14 +175,6 @@ jobs:
         clang_version: ${{ matrix.clang_version }}
         llvm_asset_suffix: ${{ matrix.llvm_asset_suffix }}
 
-    - name: Install V8 dependencies
-      if: matrix.test_with_v8
-      run: |
-        npm -C test/scripts/browser-test install
-        npx -C test/scripts/browser-test playwright install chromium-headless-shell
-        echo ENGINE="$PWD/test/scripts/browser-test/harness.mjs" >> $GITHUB_ENV
-        echo CTEST_ARGS="--label-exclude v8fail" >> $GITHUB_ENV
-
     - name: Configure libc
       run: |
         cmake -G Ninja -S . -B build \
@@ -194,14 +184,14 @@ jobs:
 
     - name: Configure testing
       if: matrix.test
-      run: cmake -S . -B build -DENABLE_WERROR=ON -DBUILD_TESTS=ON -DENGINE=${{ env.ENGINE }}
+      run: cmake -S . -B build -DENABLE_WERROR=ON -DBUILD_TESTS=ON
 
     - name: Build
       run: ninja -C build -v
 
     - name: Test
       if: matrix.test
-      run: ctest --test-dir build/test --output-on-failure -j4 $CTEST_ARGS
+      run: ctest --test-dir build/test --output-on-failure -j4
 
     - uses: actions/upload-artifact@v6
       if: matrix.upload

test/CMakeLists.txt

@@ -20,10 +20,38 @@ ExternalProject_Get_Property(libc-test SOURCE_DIR)
 set(LIBC_TEST "${SOURCE_DIR}")
 message(STATUS "libc-test source directory: ${LIBC_TEST}")
 
-# ========= Download wasmtime as a test runner ==================
+# ========= Download a test runner =============================
 include(ba-download)
 
-if(NOT ENGINE OR ENGINE STREQUAL "")
+option(TEST_WITH_V8 "Run tests in headless Chromium (V8) via Playwright instead of Wasmtime" OFF)
+
+if(TEST_WITH_V8)
+  find_program(NPM_EXECUTABLE npm REQUIRED)
+  find_program(NPX_EXECUTABLE npx REQUIRED)
+
+  set(browser_test_src "${CMAKE_CURRENT_SOURCE_DIR}/scripts/browser-test")
+  set(browser_test_dir "${CMAKE_CURRENT_BINARY_DIR}/browser-test")
+  set(browser_test_stamp "${browser_test_dir}/deps.stamp")
+
+  set(browser_test_inputs
+    "${browser_test_src}/package.json"
+    "${browser_test_src}/package-lock.json"
+  )
+
+  add_custom_command(
+    OUTPUT ${browser_test_stamp}
+    COMMAND ${NPM_EXECUTABLE} -C ${browser_test_src} install
+    COMMAND ${NPX_EXECUTABLE} -C ${browser_test_src} playwright install chromium-headless-shell
+    COMMAND ${CMAKE_COMMAND} -E touch ${browser_test_stamp}
+    DEPENDS ${browser_test_inputs}
+  )
+
+  set(ENGINE "${browser_test_src}/harness.mjs")
+  add_custom_target(engine ALL DEPENDS ${browser_test_stamp})
+  message(STATUS "Test engine: V8 (headless browser) via ${ENGINE}")
+
+elseif(NOT ENGINE OR ENGINE STREQUAL "")
+  # ====== Download Wasmtime as a test runner ==================
   set(wasmtime_version dev)
   if (TARGET_TRIPLE MATCHES "-threads")
     set(wasmtime_version "v44.0.0")
@@ -41,6 +69,13 @@ else()
   add_custom_target(engine)
 endif()
 
+# Helper to completely disable tests which currently fail in v8.
+function(expect_fail_under_v8 test_name)
+  if(TEST_WITH_V8)
+    set_tests_properties(${test_name} PROPERTIES WILL_FAIL ON)
+  endif()
+endfunction()
+
 # All tests are compiled against the wasi-libc sysroot
 add_compile_options(--sysroot ${SYSROOT})
 add_link_options(--sysroot ${SYSROOT})
@@ -339,7 +374,7 @@ add_wasilibc_test(fwscanf.c FS)
 # main thread when doing file I/O. This is used to exercise this test's
 # behavior where file streams use fallback code for nonblocking reads/writes.
 add_wasilibc_test(fs-nonblocking.c FS ENGINE_ARG -Wtimeout=10s)
-set_tests_properties(fs-nonblocking.wasm PROPERTIES LABELS v8fail)
+expect_fail_under_v8(fs-nonblocking.wasm)
 add_wasilibc_test(getentropy.c)
 add_wasilibc_test(hello.c PASS_REGULAR_EXPRESSION "Hello, World!")
 add_wasilibc_test(ioctl.c FS)
@@ -390,7 +425,7 @@ if (CMAKE_C_COMPILER_VERSION VERSION_GREATER 20.0)
   # doesn't work well with tags exported from shared libraries. Probably related
   # to llvm-project/llvm#188367 and/or llvm/llvm-project#188120
   add_wasilibc_test(setjmp.c SETJMP NOSHARED)
-  set_tests_properties(setjmp.wasm PROPERTIES LABELS v8fail)
+  expect_fail_under_v8(setjmp.wasm)
 endif()
 
 # This test works for single-threaded targets too
@@ -504,19 +539,18 @@ if (NOT (WASI STREQUAL "p1"))
     sockets-client-handle-hangups.wasm hello.wasm)
 endif()
 
-# Flag some tests as failing in V8
-set_tests_properties(hello.wasm PROPERTIES LABELS v8fail)
-set_tests_properties(clock_nanosleep.wasm PROPERTIES LABELS v8fail)
+# Disable some tests that can't run in V8
+expect_fail_under_v8(hello.wasm)
 # Skip test that uses environment variables
-set_tests_properties(external_env.wasm PROPERTIES LABELS v8fail)
+expect_fail_under_v8(external_env.wasm)
 # Skip test that uses command-line arguments
-set_tests_properties(argv_two_args.wasm PROPERTIES LABELS v8fail)
+expect_fail_under_v8(argv_two_args.wasm)
 if (TARGET_TRIPLE MATCHES "-threads")
   # atomic.wait32 can't be executed on the main thread
-  set_tests_properties(libc_test_functional_pthread_mutex.wasm PROPERTIES LABELS v8fail)
-  set_tests_properties(libc_test_functional_pthread_tsd.wasm PROPERTIES LABELS v8fail)
+  expect_fail_under_v8(libc_test_functional_pthread_mutex.wasm)
+  expect_fail_under_v8(libc_test_functional_pthread_tsd.wasm)
   # "poll_oneoff" can't be implemented in the browser
-  set_tests_properties(libc_test_functional_pthread_cond.wasm PROPERTIES LABELS v8fail)
+  expect_fail_under_v8(libc_test_functional_pthread_cond.wasm)
 endif()
 
 # If enabled add a copy of Python which is built against `wasi-libc` and run