Update playground-how-it-works for on-demand WASM module loading #595
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Docs | |
| on: | |
| push: | |
| branches: | |
| - main | |
| pull_request: | |
| branches: | |
| - main | |
| schedule: | |
| - cron: "0 2 * * *" | |
| jobs: | |
| # The WASM modules compile in parallel: coreutils has its own job (it needs | |
| # l10n + feat_wasm parsing), while grep/find/diffutils share the matrix job | |
| # below. Each leg uploads its `.wasm` plus small fragments (command names and | |
| # a version block) that the `build` job stitches together into the | |
| # playground's commands.js / version.js. Fragment files are numbered so the | |
| # merge order is deterministic (coreutils base list first, then the rest). | |
| wasm-coreutils: | |
| name: Build uutils (coreutils) WASM | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Checkout Coreutils Repository | |
| uses: actions/checkout@v7 | |
| with: | |
| repository: uutils/coreutils | |
| path: './coreutils' | |
| fetch-depth: 0 | |
| - name: Checkout Coreutils L10n Repository | |
| uses: actions/checkout@v7 | |
| with: | |
| repository: uutils/coreutils-l10n | |
| path: './coreutils-l10n' | |
| fetch-depth: 0 | |
| - name: Install `rust` toolchain | |
| uses: dtolnay/rust-toolchain@stable | |
| with: | |
| targets: wasm32-wasip1 | |
| - name: Cache cargo build | |
| uses: Swatinem/rust-cache@v2 | |
| with: | |
| workspaces: coreutils | |
| - name: Copy l10n locales into coreutils | |
| run: | | |
| # Only copy locales into utilities that already exist in coreutils | |
| # Exclude en-US.ftl to preserve the coreutils English originals | |
| for util_dir in coreutils-l10n/src/uu/*/; do | |
| util=$(basename "$util_dir") | |
| if [ -d "coreutils/src/uu/$util/locales" ]; then | |
| for ftl in "$util_dir"/locales/*.ftl; do | |
| [ -f "$ftl" ] || continue | |
| [ "$(basename "$ftl")" = "en-US.ftl" ] && continue | |
| cp "$ftl" "coreutils/src/uu/$util/locales/" | |
| done | |
| fi | |
| done | |
| # Copy uucore l10n files, excluding en-US.ftl | |
| if [ -d "coreutils-l10n/src/uucore/locales" ]; then | |
| for ftl in coreutils-l10n/src/uucore/locales/*.ftl; do | |
| [ -f "$ftl" ] || continue | |
| [ "$(basename "$ftl")" = "en-US.ftl" ] && continue | |
| cp "$ftl" "coreutils/src/uucore/locales/" | |
| done | |
| fi | |
| - name: Build uutils WASM binary | |
| run: | | |
| cd coreutils | |
| # Build the multicall binary for WASI target | |
| # Use --no-default-features to avoid platform-specific dependencies | |
| cargo build --release --target wasm32-wasip1 -p coreutils --no-default-features --features feat_wasm | |
| test -f target/wasm32-wasip1/release/coreutils.wasm | |
| mkdir -p ../wasm-out | |
| cp target/wasm32-wasip1/release/coreutils.wasm ../wasm-out/uutils.wasm | |
| # Optimize WASM size if wasm-opt is available | |
| if command -v wasm-opt &> /dev/null; then | |
| wasm-opt -Oz ../wasm-out/uutils.wasm -o ../wasm-out/uutils.wasm | |
| fi | |
| echo "WASM binary size: $(du -h ../wasm-out/uutils.wasm | cut -f1)" | |
| # Generate the list of available locales from .ftl files | |
| locales=$(find src/uu/*/locales src/uucore/locales -name '*.ftl' 2>/dev/null \ | |
| | sed 's|.*/||; s|\.ftl$||' | sort -u | paste -sd, -) | |
| echo "const WASM_LOCALES = [$(echo "$locales" | sed 's/[^,]*/\"&\"/g')];" \ | |
| > ../wasm-out/locales.js | |
| echo "Available locales: $locales" | |
| # Generate the list of available commands from the feat_wasm feature | |
| # in Cargo.toml so the playground stays in sync with the WASM build. | |
| # HIDDEN_CMDS: utilities that ship in the binary but we don't want to | |
| # advertise in the playground (e.g. `yes` produces infinite output | |
| # that just spams the terminal). | |
| HIDDEN_CMDS='^(false|true|yes)$' | |
| commands=$(sed -n '/^feat_wasm = \[/,/^\]/p' Cargo.toml \ | |
| | grep -oE '"[a-zA-Z0-9_]+"' | tr -d '"' \ | |
| | sort -u | grep -vE "$HIDDEN_CMDS" | paste -sd, -) | |
| echo "$commands" | sed 's/[^,]*/"&"/g' > ../wasm-out/commands-00-coreutils.txt | |
| echo "Available commands: $commands" | |
| # Record the coreutils commit used to build the WASM binary | |
| commit_hash=$(git rev-parse HEAD) | |
| commit_short=$(git rev-parse --short HEAD) | |
| commit_date=$(git show -s --format=%cI HEAD) | |
| { | |
| echo "const UUTILS_WASM_VERSION = {" | |
| echo " commit: \"${commit_hash}\"," | |
| echo " short: \"${commit_short}\"," | |
| echo " date: \"${commit_date}\"" | |
| echo "};" | |
| } > ../wasm-out/version-00-coreutils.js | |
| echo "uutils WASM build: ${commit_short} (${commit_date})" | |
| - name: Upload uutils WASM artifact | |
| uses: actions/upload-artifact@v7 | |
| with: | |
| name: wasm-coreutils | |
| path: wasm-out | |
| # grep, findutils, diffutils and sed ship standalone binaries built the same | |
| # way, so they share one matrix job. Each entry lists its `bins` (findutils | |
| # provides three: find, locate, updatedb); every bin emits its own .wasm. | |
| # grep/findutils pull in the Oniguruma C library (onig_sys) and need a WASI | |
| # sysroot to compile its bundled C sources; diffutils and sed are pure Rust and | |
| # set needs_wasi: false. The `order` field keeps the merged command/version | |
| # lists in a stable sequence in `build`. | |
| wasm-standalone: | |
| name: Build ${{ matrix.name }} WASM | |
| runs-on: ubuntu-latest | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| include: | |
| - name: grep | |
| repo: uutils/grep | |
| bins: grep | |
| needs_wasi: true | |
| commands: '"grep"' | |
| version_const: UUTILS_GREP_VERSION | |
| order: 10 | |
| # findutils ships several standalone binaries; build find, locate and | |
| # updatedb so the playground can offer them. (xargs is omitted: it must | |
| # spawn child processes, which the browser WASI sandbox can't do.) | |
| - name: find | |
| repo: uutils/findutils | |
| bins: find locate updatedb | |
| needs_wasi: true | |
| commands: '"find","locate","updatedb"' | |
| version_const: UUTILS_FINDUTILS_VERSION | |
| order: 20 | |
| - name: diffutils | |
| repo: uutils/diffutils | |
| bins: diffutils | |
| needs_wasi: false | |
| commands: '"diff","cmp"' | |
| version_const: UUTILS_DIFFUTILS_VERSION | |
| order: 30 | |
| - name: sed | |
| repo: uutils/sed | |
| bins: sed | |
| needs_wasi: false | |
| commands: '"sed"' | |
| version_const: UUTILS_SED_VERSION | |
| order: 40 | |
| steps: | |
| - name: Checkout ${{ matrix.name }} Repository | |
| uses: actions/checkout@v7 | |
| with: | |
| repository: ${{ matrix.repo }} | |
| path: ${{ matrix.name }} | |
| fetch-depth: 0 | |
| - name: Install `rust` toolchain | |
| uses: dtolnay/rust-toolchain@stable | |
| with: | |
| targets: wasm32-wasip1 | |
| - name: Cache cargo build | |
| uses: Swatinem/rust-cache@v2 | |
| with: | |
| workspaces: ${{ matrix.name }} | |
| - name: Build ${{ matrix.name }} WASM binary | |
| run: | | |
| if [ "${{ matrix.needs_wasi }}" = "true" ]; then | |
| # Build a WASI sysroot so the onig_sys C sources can be compiled. | |
| WASI_SDK_VERSION=25 | |
| WASI_SDK_DIR="wasi-sdk-${WASI_SDK_VERSION}.0-x86_64-linux" | |
| curl -sL "https://github.com/WebAssembly/wasi-sdk/releases/download/wasi-sdk-${WASI_SDK_VERSION}/${WASI_SDK_DIR}.tar.gz" | tar xz | |
| export WASI_SDK_PATH="$PWD/${WASI_SDK_DIR}" | |
| export CC_wasm32_wasip1="$WASI_SDK_PATH/bin/clang" | |
| export CFLAGS_wasm32_wasip1="--sysroot=$WASI_SDK_PATH/share/wasi-sysroot" | |
| fi | |
| cd ${{ matrix.name }} | |
| mkdir -p ../wasm-out | |
| # A module may ship several standalone binaries (findutils → find, | |
| # locate, updatedb); build each and emit its own .wasm file. | |
| for bin in ${{ matrix.bins }}; do | |
| wasm="${bin}.wasm" | |
| cargo build --release --target wasm32-wasip1 --bin "${bin}" | |
| test -f "target/wasm32-wasip1/release/${wasm}" | |
| cp "target/wasm32-wasip1/release/${wasm}" "../wasm-out/${wasm}" | |
| # Optimize WASM size if wasm-opt is available | |
| if command -v wasm-opt &> /dev/null; then | |
| wasm-opt -Oz "../wasm-out/${wasm}" -o "../wasm-out/${wasm}" | |
| fi | |
| echo "${bin} WASM binary size: $(du -h ../wasm-out/${wasm} | cut -f1)" | |
| done | |
| # Advertise this module's commands in the playground's command list. | |
| echo '${{ matrix.commands }}' > ../wasm-out/commands-${{ matrix.order }}-${{ matrix.name }}.txt | |
| # Record the commit used to build this WASM module. | |
| h=$(git rev-parse HEAD) | |
| s=$(git rev-parse --short HEAD) | |
| d=$(git show -s --format=%cI HEAD) | |
| { | |
| echo "const ${{ matrix.version_const }} = {" | |
| echo " commit: \"${h}\"," | |
| echo " short: \"${s}\"," | |
| echo " date: \"${d}\"" | |
| echo "};" | |
| } > ../wasm-out/version-${{ matrix.order }}-${{ matrix.name }}.js | |
| echo "${{ matrix.name }} WASM build: ${s} (${d})" | |
| - name: Upload ${{ matrix.name }} WASM artifact | |
| uses: actions/upload-artifact@v7 | |
| with: | |
| name: wasm-${{ matrix.name }} | |
| path: wasm-out | |
| lint-js: | |
| name: Lint playground JavaScript | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Checkout uutils.github.io Repository | |
| uses: actions/checkout@v7 | |
| - name: Set up Node.js | |
| uses: actions/setup-node@v6 | |
| with: | |
| node-version: 24 | |
| - name: Install dependencies | |
| run: npm install | |
| - name: Run ESLint | |
| run: npm run lint | |
| build: | |
| name: Build website artifacts | |
| runs-on: ubuntu-latest | |
| needs: [wasm-coreutils, wasm-standalone] | |
| steps: | |
| - name: Checkout uutils.github.io Repository | |
| uses: actions/checkout@v7 | |
| with: | |
| repository: uutils/uutils.github.io | |
| path: './uutils.github.io' | |
| fetch-depth: 0 | |
| - name: Checkout Coreutils Repository | |
| uses: actions/checkout@v7 | |
| with: | |
| repository: uutils/coreutils | |
| path: './coreutils' | |
| fetch-depth: 0 | |
| - name: Checkout Coreutils L10n Repository | |
| uses: actions/checkout@v7 | |
| with: | |
| repository: uutils/coreutils-l10n | |
| path: './coreutils-l10n' | |
| fetch-depth: 0 | |
| - name: Checkout Findutils Repository | |
| uses: actions/checkout@v7 | |
| with: | |
| repository: uutils/findutils | |
| path: './findutils' | |
| fetch-depth: 0 | |
| - name: Install `rust` toolchain | |
| uses: dtolnay/rust-toolchain@stable | |
| - name: Install system deps | |
| run: | | |
| sudo apt update | |
| sudo apt install libacl1-dev libselinux1-dev libsystemd-dev | |
| pip install babel | |
| - name: Install necessary tools (mdbook and mdbook-toc) | |
| uses: taiki-e/install-action@v2 | |
| with: | |
| tool: mdbook@0.5.0,mdbook-toc@0.15.3 | |
| - name: Copy l10n locales into coreutils | |
| run: | | |
| # Only copy locales into utilities that already exist in coreutils | |
| # Exclude en-US.ftl to preserve the coreutils English originals | |
| for util_dir in coreutils-l10n/src/uu/*/; do | |
| util=$(basename "$util_dir") | |
| if [ -d "coreutils/src/uu/$util/locales" ]; then | |
| for ftl in "$util_dir"/locales/*.ftl; do | |
| [ -f "$ftl" ] || continue | |
| [ "$(basename "$ftl")" = "en-US.ftl" ] && continue | |
| cp "$ftl" "coreutils/src/uu/$util/locales/" | |
| done | |
| fi | |
| done | |
| # Copy uucore l10n files, excluding en-US.ftl | |
| if [ -d "coreutils-l10n/src/uucore/locales" ]; then | |
| for ftl in coreutils-l10n/src/uucore/locales/*.ftl; do | |
| [ -f "$ftl" ] || continue | |
| [ "$(basename "$ftl")" = "en-US.ftl" ] && continue | |
| cp "$ftl" "coreutils/src/uucore/locales/" | |
| done | |
| fi | |
| - name: Patch mdbook theme with language selector | |
| run: | | |
| uutils.github.io/scripts/patch-mdbook-theme.sh coreutils/docs coreutils-l10n | |
| - name: Build Coreutils Docs | |
| run: | | |
| cd coreutils | |
| # Download and repack tldr zip: uudoc expects pages/common/ prefix | |
| curl -sfL https://github.com/tldr-pages/tldr/releases/download/v2.3/tldr-pages.zip -o /tmp/tldr-raw.zip || true | |
| if [ -f /tmp/tldr-raw.zip ]; then | |
| mkdir -p /tmp/tldr-repack/pages | |
| cd /tmp/tldr-repack && unzip -o /tmp/tldr-raw.zip -d pages/ > /dev/null | |
| zip -r "$OLDPWD/docs/tldr.zip" pages/ > /dev/null | |
| cd "$OLDPWD" | |
| fi | |
| cargo run --bin uudoc --all-features | |
| cd docs | |
| # Remove deprecated 'multilingual' field unsupported by newer mdbook | |
| sed -i '/^multilingual/d' book.toml | |
| # Strip the legacy FA4 `fa` class from uudoc-generated platform icons | |
| # so mdbook 0.5's Font Awesome parser picks up the `fa-brands` family | |
| # instead of defaulting to `regular` and warning about linux/windows/apple. | |
| find src/utils -name '*.md' -exec sed -i 's|class="fa fa-brands |class="fa-brands |g' {} + | |
| mdbook build | |
| - name: Build Coreutils Docs (translations) | |
| run: | | |
| uutils.github.io/scripts/build-docs-l10n.sh coreutils | |
| - name: Build Findutils Docs | |
| run: | | |
| cd findutils/docs | |
| # Remove deprecated 'multilingual' field unsupported by newer mdbook | |
| sed -i '/^multilingual/d' book.toml | |
| mdbook build | |
| - name: Download WASM artifacts | |
| uses: actions/download-artifact@v8 | |
| with: | |
| pattern: wasm-* | |
| path: wasm-artifacts | |
| merge-multiple: true | |
| - name: Assemble WASM playground assets | |
| run: | | |
| # The four wasm-* artifacts each carry their .wasm plus numbered | |
| # command/version fragments. Stitch them into the single commands.js / | |
| # version.js the playground loads. SITE_VERSION is generated here since | |
| # it describes the uutils.github.io repo, not any of the tool repos. | |
| mkdir -p uutils.github.io/static/wasm | |
| cp wasm-artifacts/*.wasm uutils.github.io/static/wasm/ | |
| cp wasm-artifacts/locales.js uutils.github.io/static/wasm/ | |
| # Merge the command-name fragments in numeric (filename) order. | |
| commands=$(cat $(ls wasm-artifacts/commands-*.txt | sort) | paste -sd, -) | |
| echo "const WASM_COMMANDS = [${commands}];" \ | |
| > uutils.github.io/static/wasm/commands.js | |
| echo "Playground commands: ${commands}" | |
| # Record the uutils.github.io commit the site was built from. | |
| site_hash=$(git -C uutils.github.io rev-parse HEAD) | |
| site_short=$(git -C uutils.github.io rev-parse --short HEAD) | |
| site_date=$(git -C uutils.github.io show -s --format=%cI HEAD) | |
| { | |
| cat $(ls wasm-artifacts/version-*.js | sort) | |
| echo "const SITE_VERSION = {" | |
| echo " commit: \"${site_hash}\"," | |
| echo " short: \"${site_short}\"," | |
| echo " date: \"${site_date}\"" | |
| echo "};" | |
| } > uutils.github.io/static/wasm/version.js | |
| echo "site build: ${site_short} (${site_date})" | |
| - name: Run Zola | |
| uses: shalzz/zola-deploy-action@v0.22.1 | |
| env: | |
| BUILD_DIR: uutils.github.io | |
| BUILD_ONLY: true | |
| - name: Collect results into `public` folder | |
| run: | | |
| cp -r uutils.github.io/public public | |
| cp -r coreutils/docs/book public/coreutils/docs | |
| cp -r findutils/docs/book public/findutils/docs | |
| # Copy translated docs to /coreutils/docs-{lang}/ | |
| for lang_dir in coreutils/docs/book-*/; do | |
| [ -d "$lang_dir" ] || continue | |
| lang=$(basename "$lang_dir" | sed 's/^book-//') | |
| cp -r "$lang_dir" "public/coreutils/docs-${lang}" | |
| done | |
| - name: Run playground JS tests | |
| run: | | |
| npm install puppeteer@24 | |
| node uutils.github.io/scripts/run-tests.js --dir public --port 8080 | |
| - name: Remove test files from deploy output | |
| run: rm -f public/js/wasm-terminal.test.html | |
| - name: Upload artifact for checking the output | |
| uses: actions/upload-artifact@v7 | |
| with: | |
| path: ./public | |
| - name: Upload artifact for pages | |
| uses: actions/upload-pages-artifact@v5 | |
| with: | |
| path: ./public | |
| # Deployment job | |
| deploy: | |
| permissions: | |
| contents: read | |
| pages: write | |
| id-token: write | |
| environment: | |
| name: github-pages | |
| url: ${{ steps.deployment.outputs.page_url }} | |
| runs-on: ubuntu-latest | |
| needs: build | |
| if: (github.event_name == 'push' && github.ref == 'refs/heads/main') || github.event_name == 'schedule' | |
| steps: | |
| - name: Deploy to GitHub Pages | |
| id: deployment | |
| uses: actions/deploy-pages@v5 |