Merge branch 'main' into wasm_split_fuzz_fix

Author: aheejin

.flake8

@@ -38,7 +38,6 @@ jobs:
         sudo chmod +x llvm.sh
         sudo ./llvm.sh ${LLVM_VERSION}
         sudo apt-get install clang-format clang-format-${LLVM_VERSION} clang-tidy-${LLVM_VERSION}
-    - run: flake8
     - run: ruff check
     - run: ./scripts/clang-format-diff.sh
     - name: clang-tidy
@@ -219,19 +218,30 @@ jobs:
     - name: test
       run: python check.py --binaryen-bin=out/bin
 
-  # Build with gcc 6.3 and run tests on Alpine Linux (inside chroot).
+  # Run tests on Alpine Linux, which we use to make our release builds.
   # Note: Alpine uses musl libc.
-  # Keep in sync with build_release.yml
+  # Keep in sync with create_release.yml. The only difference is that here we
+  # do not have the "archive" and "upload tarball" jobs.
   build-alpine:
     name: alpine
-    runs-on: ubuntu-24.04-arm
+    runs-on: ${{ matrix.os }}
+    strategy:
+      matrix:
+        os: [ubuntu-latest, ubuntu-24.04-arm]
     steps:
+    - uses: actions/setup-python@v5
+      with:
+        python-version: '3.x'
     - uses: actions/checkout@v4
       with:
         submodules: true
+
     - name: start docker
       run: |
-        docker run -w /src -dit --platform=linux/arm64 --name alpine -v $PWD:/src node:lts-alpine
+        if [[ "${{ matrix.docker_platform }}" == "ubuntu-24.04-arm" ]]; then
+          platform="--platform=linux/arm64"
+        fi
+        docker run -w /src -dit $platform --name alpine -v $PWD:/src node:lts-alpine
         echo 'docker exec alpine "$@";' > ./alpine.sh
         chmod +x ./alpine.sh
 
@@ -248,8 +258,11 @@ jobs:
       run: ./alpine.sh pip3 install --break-system-packages -r requirements-dev.txt
 
     - name: cmake
+      # Build with an 8MB stack size, as otherwise Alpine/musl's default stack
+      # size for pthreads is tiny,
+      # https://github.com/WebAssembly/binaryen/issues/8594
       run: |
-        ./alpine.sh cmake . -G Ninja -DCMAKE_INSTALL_PREFIX=out/install -DCMAKE_CXX_FLAGS="-static" -DCMAKE_C_FLAGS="-static" -DCMAKE_BUILD_TYPE=Release -DBUILD_STATIC_LIB=ON -DBUILD_MIMALLOC=ON -DCMAKE_INSTALL_PREFIX=install
+        ./alpine.sh cmake . -G Ninja -DCMAKE_CXX_FLAGS="-static" -DCMAKE_C_FLAGS="-static" -DCMAKE_EXE_LINKER_FLAGS="-Wl,-z,stack-size=8388608" -DCMAKE_BUILD_TYPE=Release -DBUILD_STATIC_LIB=ON -DBUILD_MIMALLOC=ON -DCMAKE_INSTALL_PREFIX=install
 
     - name: build
       run: |
@@ -346,23 +359,20 @@ jobs:
     - name: install ninja
       run: sudo apt-get install ninja-build
     - name: emsdk install
-      run: |
-        mkdir $HOME/emsdk
-        git clone --depth 1 https://github.com/emscripten-core/emsdk.git $HOME/emsdk
-        $HOME/emsdk/emsdk update-tags
-        $HOME/emsdk/emsdk install tot
-        $HOME/emsdk/emsdk activate tot
+      uses: emscripten-core/setup-emsdk@v15
+      with:
+        version: tot
     - name: override emscripten repository
       if: ${{ env.EMSCRIPTEN_REPO != '' }}
       run: |
-        $HOME/emsdk/emsdk install emscripten-main-64bit \
+        $EMSDK/emsdk install emscripten-main-64bit \
           --override-repository emscripten-main-64bit@$EMSCRIPTEN_REPO
-        $HOME/emsdk/emsdk activate emscripten-main-64bit
+        $EMSDK/emsdk activate emscripten-main-64bit
     - name: update path
-      run:  echo "PATH=$PATH:$HOME/emsdk" >> $GITHUB_ENV
+      run:  echo "PATH=$PATH:$EMSDK" >> $GITHUB_ENV
     - name: emcc-tests
       run: |
-        source $HOME/emsdk/emsdk_env.sh
+        source $EMSDK/emsdk_env.sh
         ./scripts/emcc-tests.sh
 
   # Windows + gcc needs work before the tests will run, so just test the compile
@@ -383,40 +393,6 @@ jobs:
     - name: build
       run: cmake --build out
 
-  # Duplicates build-asan.  Please keep in sync
-  build-cxx20:
-    name: c++20
-    # Make sure we can still build on older Ubuntu
-    runs-on: ubuntu-22.04
-    env:
-      CC: "gcc"
-      CXX: "g++"
-    steps:
-    - uses: actions/setup-python@v5
-      with:
-        python-version: '3.x'
-    - uses: actions/checkout@v4
-      with:
-        submodules: true
-    - name: install ninja
-      run: sudo apt-get install ninja-build
-    - name: install v8
-      run: |
-        npm install jsvu -g
-        jsvu --os=default --engines=v8
-    - name: install Python dev dependencies
-      run: pip3 install -r requirements-dev.txt
-    - name: cmake
-      run: |
-        mkdir -p out
-        cmake -S . -B out -G Ninja -DCMAKE_INSTALL_PREFIX=out/install -DCMAKE_C_COMPILER="$CC" -DCMAKE_CXX_COMPILER="$CXX" -DCMAKE_BUILD_TYPE=Release -DCMAKE_CXX_STANDARD=20
-    - name: build
-      run: cmake --build out
-    - name: test
-      run: |
-        python check.py --binaryen-bin=out/bin lit
-        python check.py --binaryen-bin=out/bin gtest
-
   # Ensures we can build in no-asserts mode (just building).
   build-noasserts:
     name: noasserts

.github/workflows/ci.yml

@@ -108,6 +108,7 @@ jobs:
           ${{ steps.archive-arm64.outputs.SHASUM }}
 
   # Build with gcc 6.3 and run tests on Alpine Linux (inside chroot).
+  # Keep in sync with ci.yml.
   # Note: Alpine uses musl libc.
   build-alpine:
     name: alpine
@@ -145,8 +146,11 @@ jobs:
       run: ./alpine.sh pip3 install --break-system-packages -r requirements-dev.txt
 
     - name: cmake
+      # Build with an 8MB stack size, as otherwise Alpine/musl's default stack
+      # size for pthreads is tiny,
+      # https://github.com/WebAssembly/binaryen/issues/8594
       run: |
-        ./alpine.sh cmake . -G Ninja -DCMAKE_CXX_FLAGS="-static" -DCMAKE_C_FLAGS="-static" -DCMAKE_BUILD_TYPE=Release -DBUILD_STATIC_LIB=ON -DBUILD_MIMALLOC=ON -DCMAKE_INSTALL_PREFIX=install
+        ./alpine.sh cmake . -G Ninja -DCMAKE_CXX_FLAGS="-static" -DCMAKE_C_FLAGS="-static" -DCMAKE_EXE_LINKER_FLAGS="-Wl,-z,stack-size=8388608" -DCMAKE_BUILD_TYPE=Release -DBUILD_STATIC_LIB=ON -DBUILD_MIMALLOC=ON -DCMAKE_INSTALL_PREFIX=install
 
     - name: build
       run: |

.github/workflows/create_release.yml

@@ -61,3 +61,5 @@ CMakeUserPresets.json
 
 # files related to clangd cache
 .cache/*
+
+.venv/

.gitignore

@@ -1,16 +1,25 @@
 target-version = "py310"
+preview = true
 
 exclude = [
   'third_party',
   'test/lit/lit.cfg.py',
   'test/spec/testsuite',
 ]
 
+[lint.pylint]
+max-locals = 30
+max-positional-args = 7
+
+[lint.per-file-ignores]
+"test/**.py" = ["PLR6301", "PLW1514", "PLR0914"]
+
 [lint]
 select = [
   "ARG",
   "ASYNC",
   "B",
+  "D",
   "C4",
   "C90",
   "COM",
@@ -27,17 +36,29 @@ select = [
 
 ignore = [
   "C901", # https://docs.astral.sh/ruff/rules/complex-structure/
+  "D100", # https://docs.astral.sh/ruff/rules/undocumented-public-module/
+  "D101", # https://docs.astral.sh/ruff/rules/undocumented-public-class/
+  "D102", # https://docs.astral.sh/ruff/rules/undocumented-public-method/
+  "D103", # https://docs.astral.sh/ruff/rules/undocumented-public-function/
+  "D104", # https://docs.astral.sh/ruff/rules/undocumented-public-package/
+  "D105", # https://docs.astral.sh/ruff/rules/undocumented-magic-method/
+  "D203", # https://docs.astral.sh/ruff/rules/incorrect-blank-line-before-class/
+  "D213", # https://docs.astral.sh/ruff/rules/multi-line-summary-second-line/
+  "D107", # https://docs.astral.sh/ruff/rules/undocumented-public-init/
   "B006", # https://docs.astral.sh/ruff/rules/mutable-argument-default/
   "B011", # https://docs.astral.sh/ruff/rules/assert-false/
   "B023", # https://docs.astral.sh/ruff/rules/function-uses-loop-variable/
   "E501", # https://docs.astral.sh/ruff/rules/line-too-long/
   "E741", # https://docs.astral.sh/ruff/rules/ambiguous-variable-name/
   "PERF401", # https://docs.astral.sh/ruff/rules/manual-list-comprehension/
+  "PLR0904", # https://docs.astral.sh/ruff/rules/too-many-public-methods/
   "PLR0912", # https://docs.astral.sh/ruff/rules/too-many-branches/
   "PLR0913", # https://docs.astral.sh/ruff/rules/too-many-arguments/
   "PLR0915", # https://docs.astral.sh/ruff/rules/too-many-statements/
+  "PLR1702", # https://docs.astral.sh/ruff/rules/too-many-nested-blocks/
   "PLR2004", # https://docs.astral.sh/ruff/rules/magic-value-comparison/
   "PLW0603", # https://docs.astral.sh/ruff/rules/global-statement/
   "PLW1510", # https://docs.astral.sh/ruff/rules/subprocess-run-without-check/
   "PLW2901", # https://docs.astral.sh/ruff/rules/redefined-loop-name/
+  "PLW1514", # https://docs.astral.sh/ruff/rules/unspecified-encoding/
 ]

.ruff.toml

@@ -15,11 +15,34 @@ full changeset diff at the end of each section.
 Current Trunk
 -------------
 
-- Add support for non-nullable table types and initialization expressions for
+ - Rename `MemorySegment` functions to `DataSegment` in the c and js apis
+   - Rename `BinaryenGetNumMemorySegments` to `BinaryenGetNumDataSegments` in c api.
+   - Rename `BinaryenGetMemorySegmentByteOffset` to `BinaryenGetDataSegmentByteOffset` in c api.
+   - Rename `BinaryenGetMemorySegmentByteLength` to `BinaryenGetDataSegmentByteLength` in c api.
+   - Rename `BinaryenGetMemorySegmentPassive` to `BinaryenGetDataSegmentPassive` in c api.
+   - Rename `BinaryenCopyMemorySegmentData` to `BinaryenCopyDataSegmentData` in c api.
+   - Rename `module.getNumMemorySegments` to `module.getNumDataSegments` in js api.
+   - Rename `module.getMemorySegmentInfo` to `module.getDataSegmentInfo` in js api.
+
+v129
+----
+
+ - Add a `BinaryenDataSegmentRef` type to the C API. (#8286)
+   - Add `BinaryenGetDataSegment` and `BinaryenGetDataSegmentByIndex` to the C API, which allow looking up a data segment by name or index.
+   - Add `BinaryenDataSegmentGetName` to the C API, which allows looking up a data segment's name.
+   - Convert `BinaryenGetMemorySegmentByteOffset`, `BinaryenGetMemorySegmentByteLength`, `BinaryenGetMemorySegmentPassive` and `BinaryenCopyMemorySegmentData` to take a `BinaryenDataSegmentRef` instead of a name.
+   - Add `module.getDataSegment`, `module.getDataSegmentByIndex` to the JS API, which allows looking up a data segment by name or index.
+   - Convert `module.getMemorySegmentInfo` to take a data segment reference instead of a name, and return the name as part of the info.
+ - Add support for non-nullable table types and initialization expressions for
    tables. This comes with a breaking change to C API: `BinaryenAddTable` takes
    an additional `BinaryenExpressionRef` parameter to provide an initialization
    expression. This may be set to NULL for tables without an initializer. In JS
    this parameter is optional and so is not breaking. (#8405)
+ - [multibyte] Add multibyte array store and load instructions. (#8059, #8504)
+ - MinifyImportsAndExports now has a new output format using JSON. This was
+   changed while fixing bugs with colliding module names (to avoid two breaking
+   changes to the output). (#8550)
+ - Update from C++17 to C++20 (#8218)
 
 v128
 ----

CHANGELOG.md

@@ -5,13 +5,13 @@ cmake_minimum_required(VERSION 3.16.3)
 # Needed for C++17 (std::path)
 set(CMAKE_OSX_DEPLOYMENT_TARGET "10.15" CACHE STRING "Minimum OS X deployment version")
 
-project(binaryen LANGUAGES C CXX VERSION 128)
+project(binaryen LANGUAGES C CXX VERSION 129)
 include(GNUInstallDirs)
 
 # The C++ standard whose features are required to build Binaryen.
 # Keep in sync with scripts/test/shared.py cxx_standard
 # The if condition allows embedding in a project with a higher default C++ standard set
-set(REQUIRED_CXX_STANDARD 17)
+set(REQUIRED_CXX_STANDARD 20)
 if(NOT CMAKE_CXX_STANDARD)
   set(CMAKE_CXX_STANDARD ${REQUIRED_CXX_STANDARD})
 elseif(CMAKE_CXX_STANDARD LESS ${REQUIRED_CXX_STANDARD})
@@ -550,8 +550,8 @@ if(EMSCRIPTEN)
   if(JS_OF_OCAML)
     # js_of_ocaml needs a specified variable with special comment to provide the library to consumer
     target_link_libraries(binaryen_js PRIVATE "--extern-pre-js=${CMAKE_CURRENT_SOURCE_DIR}/src/js/binaryen.jsoo-extern-pre.js")
-    # Currently, js_of_ocaml can only process ES5 code
-    target_link_libraries(binaryen_js PRIVATE optimized "--closure-args=\"--language_out=ECMASCRIPT5\"")
+    # js_of_ocaml does not support top level await
+    target_link_libraries(binaryen_js PRIVATE "-sWASM_ASYNC_COMPILATION=0")
   else()
     target_link_libraries(binaryen_js PRIVATE "-sEXPORT_ES6")
   endif()

CMakeLists.txt

@@ -412,7 +412,9 @@ After that you can build with CMake:
 cmake . && make
 ```
 
-A C++17 compiler is required. On macOS, you need to install `cmake`, for example, via `brew install cmake`. Note that you can also use `ninja` as your generator: `cmake -G Ninja . && ninja`.
+A C++20 compiler is required. On macOS, you need to install `cmake`, for
+example, via `brew install cmake`. Note that you can also use `ninja` as your
+generator: `cmake -G Ninja . && ninja`.
 
 To avoid the gtest dependency, you can pass `-DBUILD_TESTS=OFF` to cmake.
 

README.md

@@ -25,7 +25,7 @@
 from multiprocessing.pool import ThreadPool
 from pathlib import Path
 
-from scripts.test import binaryenjs, lld, shared, support, wasm2js, wasm_opt
+from scripts.test import binaryenjs, finalize, shared, support, wasm2js, wasm_opt
 
 assert sys.version_info >= (3, 10), 'requires Python 3.10'
 
@@ -265,7 +265,9 @@ def run_one_spec_test(wast: Path, stdout=None):
 
 
 def run_spec_test_with_wrapped_stdout(wast: Path):
-    """Return (bool, str) where the first element is whether the test was
+    """Run a single spec test while capturing stdout.
+
+    Return (bool, str) where the first element is whether the test was
     successful and the second is the combined stdout and stderr of the test.
     """
     out = io.StringIO()
@@ -436,7 +438,7 @@ def wrapper(*args, **kwargs):
     'wasm-metadce': run_wasm_metadce_tests,
     'wasm-reduce': run_wasm_reduce_tests,
     'spec': run_spec_tests,
-    'lld': lld.test_wasm_emscripten_finalize,
+    'finalize': finalize.test_wasm_emscripten_finalize,
     'wasm2js': wasm2js.test_wasm2js,
     'validator': run_validator_tests,
     'example': run_example_tests,

check.py

@@ -1,9 +1,8 @@
 # These requirements are only needed for developers who want to run the test
-# suite or flake8, not for end users.
+# suite, or CI checks, not for end users.
 
 # Install with `pip3 install -r requirements-dev.txt`
 
-flake8==7.3.0
 ruff==0.14.1
 filecheck==0.0.22
 lit==0.11.0.post1