website.yml

name: Docs

on:
  push:
    branches:
    - main
  pull_request:
    branches:
    - main
  schedule:
  - cron: "0 2 * * *"

jobs:
  # The WASM modules compile in parallel: coreutils has its own job (it needs
  # l10n + feat_wasm parsing), while grep/find/diffutils share the matrix job
  # below. Each leg uploads its `.wasm` plus small fragments (command names and
  # a version block) that the `build` job stitches together into the
  # playground's commands.js / version.js. Fragment files are numbered so the
  # merge order is deterministic (coreutils base list first, then the rest).
  wasm-coreutils:
    name: Build uutils (coreutils) WASM
    runs-on: ubuntu-latest
    steps:
    - name: Checkout Coreutils Repository
      uses: actions/checkout@v7
      with:
        repository: uutils/coreutils
        path: './coreutils'
        fetch-depth: 0

    - name: Checkout Coreutils L10n Repository
      uses: actions/checkout@v7
      with:
        repository: uutils/coreutils-l10n
        path: './coreutils-l10n'
        fetch-depth: 0

    - name: Install `rust` toolchain
      uses: dtolnay/rust-toolchain@stable
      with:
        targets: wasm32-wasip1

    - name: Cache cargo build
      uses: Swatinem/rust-cache@v2
      with:
        workspaces: coreutils

    - name: Copy l10n locales into coreutils
      run: |
        # Only copy locales into utilities that already exist in coreutils
        # Exclude en-US.ftl to preserve the coreutils English originals
        for util_dir in coreutils-l10n/src/uu/*/; do
          util=$(basename "$util_dir")
          if [ -d "coreutils/src/uu/$util/locales" ]; then
            for ftl in "$util_dir"/locales/*.ftl; do
              [ -f "$ftl" ] || continue
              [ "$(basename "$ftl")" = "en-US.ftl" ] && continue
              cp "$ftl" "coreutils/src/uu/$util/locales/"
            done
          fi
        done
        # Copy uucore l10n files, excluding en-US.ftl
        if [ -d "coreutils-l10n/src/uucore/locales" ]; then
          for ftl in coreutils-l10n/src/uucore/locales/*.ftl; do
            [ -f "$ftl" ] || continue
            [ "$(basename "$ftl")" = "en-US.ftl" ] && continue
            cp "$ftl" "coreutils/src/uucore/locales/"
          done
        fi

    - name: Build uutils WASM binary
      run: |
        cd coreutils
        # Build the multicall binary for WASI target
        # Use --no-default-features to avoid platform-specific dependencies
        cargo build --release --target wasm32-wasip1 -p coreutils --no-default-features --features feat_wasm
        test -f target/wasm32-wasip1/release/coreutils.wasm
        mkdir -p ../wasm-out
        cp target/wasm32-wasip1/release/coreutils.wasm ../wasm-out/uutils.wasm
        # Optimize WASM size if wasm-opt is available
        if command -v wasm-opt &> /dev/null; then
          wasm-opt -Oz ../wasm-out/uutils.wasm -o ../wasm-out/uutils.wasm
        fi
        echo "WASM binary size: $(du -h ../wasm-out/uutils.wasm | cut -f1)"
        # Generate the list of available locales from .ftl files
        locales=$(find src/uu/*/locales src/uucore/locales -name '*.ftl' 2>/dev/null \
          | sed 's|.*/||; s|\.ftl$||' | sort -u | paste -sd, -)
        echo "const WASM_LOCALES = [$(echo "$locales" | sed 's/[^,]*/\"&\"/g')];" \
          > ../wasm-out/locales.js
        echo "Available locales: $locales"
        # Generate the list of available commands from the feat_wasm feature
        # in Cargo.toml so the playground stays in sync with the WASM build.
        # HIDDEN_CMDS: utilities that ship in the binary but we don't want to
        # advertise in the playground (e.g. `yes` produces infinite output
        # that just spams the terminal).
        HIDDEN_CMDS='^(false|true|yes)$'
        commands=$(sed -n '/^feat_wasm = \[/,/^\]/p' Cargo.toml \
          | grep -oE '"[a-zA-Z0-9_]+"' | tr -d '"' \
          | sort -u | grep -vE "$HIDDEN_CMDS" | paste -sd, -)
        echo "$commands" | sed 's/[^,]*/"&"/g' > ../wasm-out/commands-00-coreutils.txt
        echo "Available commands: $commands"
        # Record the coreutils commit used to build the WASM binary
        commit_hash=$(git rev-parse HEAD)
        commit_short=$(git rev-parse --short HEAD)
        commit_date=$(git show -s --format=%cI HEAD)
        {
          echo "const UUTILS_WASM_VERSION = {"
          echo "  commit: \"${commit_hash}\","
          echo "  short: \"${commit_short}\","
          echo "  date: \"${commit_date}\""
          echo "};"
        } > ../wasm-out/version-00-coreutils.js
        echo "uutils WASM build: ${commit_short} (${commit_date})"

    - name: Upload uutils WASM artifact
      uses: actions/upload-artifact@v7
      with:
        name: wasm-coreutils
        path: wasm-out

  # grep, findutils, diffutils and sed ship standalone binaries built the same
  # way, so they share one matrix job. Each entry lists its `bins` (findutils
  # provides three: find, locate, updatedb); every bin emits its own .wasm.
  # grep/findutils pull in the Oniguruma C library (onig_sys) and need a WASI
  # sysroot to compile its bundled C sources; diffutils and sed are pure Rust and
  # set needs_wasi: false. The `order` field keeps the merged command/version
  # lists in a stable sequence in `build`.
  wasm-standalone:
    name: Build ${{ matrix.name }} WASM
    runs-on: ubuntu-latest
    strategy:
      fail-fast: false
      matrix:
        include:
        - name: grep
          repo: uutils/grep
          bins: grep
          needs_wasi: true
          commands: '"grep"'
          version_const: UUTILS_GREP_VERSION
          order: 10
        # findutils ships several standalone binaries; build find, locate and
        # updatedb so the playground can offer them. (xargs is omitted: it must
        # spawn child processes, which the browser WASI sandbox can't do.)
        - name: find
          repo: uutils/findutils
          bins: find locate updatedb
          needs_wasi: true
          commands: '"find","locate","updatedb"'
          version_const: UUTILS_FINDUTILS_VERSION
          order: 20
        - name: diffutils
          repo: uutils/diffutils
          bins: diffutils
          needs_wasi: false
          commands: '"diff","cmp"'
          version_const: UUTILS_DIFFUTILS_VERSION
          order: 30
        - name: sed
          repo: uutils/sed
          bins: sed
          needs_wasi: false
          commands: '"sed"'
          version_const: UUTILS_SED_VERSION
          order: 40
    steps:
    - name: Checkout ${{ matrix.name }} Repository
      uses: actions/checkout@v7
      with:
        repository: ${{ matrix.repo }}
        path: ${{ matrix.name }}
        fetch-depth: 0

    - name: Install `rust` toolchain
      uses: dtolnay/rust-toolchain@stable
      with:
        targets: wasm32-wasip1

    - name: Cache cargo build
      uses: Swatinem/rust-cache@v2
      with:
        workspaces: ${{ matrix.name }}

    - name: Build ${{ matrix.name }} WASM binary
      run: |
        if [ "${{ matrix.needs_wasi }}" = "true" ]; then
          # Build a WASI sysroot so the onig_sys C sources can be compiled.
          WASI_SDK_VERSION=25
          WASI_SDK_DIR="wasi-sdk-${WASI_SDK_VERSION}.0-x86_64-linux"
          curl -sL "https://github.com/WebAssembly/wasi-sdk/releases/download/wasi-sdk-${WASI_SDK_VERSION}/${WASI_SDK_DIR}.tar.gz" | tar xz
          export WASI_SDK_PATH="$PWD/${WASI_SDK_DIR}"
          export CC_wasm32_wasip1="$WASI_SDK_PATH/bin/clang"
          export CFLAGS_wasm32_wasip1="--sysroot=$WASI_SDK_PATH/share/wasi-sysroot"
        fi
        cd ${{ matrix.name }}
        mkdir -p ../wasm-out
        # A module may ship several standalone binaries (findutils → find,
        # locate, updatedb); build each and emit its own .wasm file.
        for bin in ${{ matrix.bins }}; do
          wasm="${bin}.wasm"
          cargo build --release --target wasm32-wasip1 --bin "${bin}"
          test -f "target/wasm32-wasip1/release/${wasm}"
          cp "target/wasm32-wasip1/release/${wasm}" "../wasm-out/${wasm}"
          # Optimize WASM size if wasm-opt is available
          if command -v wasm-opt &> /dev/null; then
            wasm-opt -Oz "../wasm-out/${wasm}" -o "../wasm-out/${wasm}"
          fi
          echo "${bin} WASM binary size: $(du -h ../wasm-out/${wasm} | cut -f1)"
        done
        # Advertise this module's commands in the playground's command list.
        echo '${{ matrix.commands }}' > ../wasm-out/commands-${{ matrix.order }}-${{ matrix.name }}.txt
        # Record the commit used to build this WASM module.
        h=$(git rev-parse HEAD)
        s=$(git rev-parse --short HEAD)
        d=$(git show -s --format=%cI HEAD)
        {
          echo "const ${{ matrix.version_const }} = {"
          echo "  commit: \"${h}\","
          echo "  short: \"${s}\","
          echo "  date: \"${d}\""
          echo "};"
        } > ../wasm-out/version-${{ matrix.order }}-${{ matrix.name }}.js
        echo "${{ matrix.name }} WASM build: ${s} (${d})"

    - name: Upload ${{ matrix.name }} WASM artifact
      uses: actions/upload-artifact@v7
      with:
        name: wasm-${{ matrix.name }}
        path: wasm-out

  lint-js:
    name: Lint playground JavaScript
    runs-on: ubuntu-latest
    steps:
    - name: Checkout uutils.github.io Repository
      uses: actions/checkout@v7

    - name: Set up Node.js
      uses: actions/setup-node@v6
      with:
        node-version: 24

    - name: Install dependencies
      run: npm install

    - name: Run ESLint
      run: npm run lint

  build:
    name: Build website artifacts
    runs-on: ubuntu-latest
    needs: [wasm-coreutils, wasm-standalone]
    steps:
    - name: Checkout uutils.github.io Repository
      uses: actions/checkout@v7
      with:
        repository: uutils/uutils.github.io
        path: './uutils.github.io'
        fetch-depth: 0

    - name: Checkout Coreutils Repository
      uses: actions/checkout@v7
      with:
        repository: uutils/coreutils
        path: './coreutils'
        fetch-depth: 0

    - name: Checkout Coreutils L10n Repository
      uses: actions/checkout@v7
      with:
        repository: uutils/coreutils-l10n
        path: './coreutils-l10n'
        fetch-depth: 0

    - name: Checkout Findutils Repository
      uses: actions/checkout@v7
      with:
        repository: uutils/findutils
        path: './findutils'
        fetch-depth: 0

    - name: Install `rust` toolchain
      uses: dtolnay/rust-toolchain@stable

    - name: Install system deps
      run: |
        sudo apt update
        sudo apt install libacl1-dev libselinux1-dev libsystemd-dev
        pip install babel

    - name: Install necessary tools (mdbook and mdbook-toc)
      uses: taiki-e/install-action@v2
      with:
        tool: mdbook@0.5.0,mdbook-toc@0.15.3

    - name: Copy l10n locales into coreutils
      run: |
        # Only copy locales into utilities that already exist in coreutils
        # Exclude en-US.ftl to preserve the coreutils English originals
        for util_dir in coreutils-l10n/src/uu/*/; do
          util=$(basename "$util_dir")
          if [ -d "coreutils/src/uu/$util/locales" ]; then
            for ftl in "$util_dir"/locales/*.ftl; do
              [ -f "$ftl" ] || continue
              [ "$(basename "$ftl")" = "en-US.ftl" ] && continue
              cp "$ftl" "coreutils/src/uu/$util/locales/"
            done
          fi
        done
        # Copy uucore l10n files, excluding en-US.ftl
        if [ -d "coreutils-l10n/src/uucore/locales" ]; then
          for ftl in coreutils-l10n/src/uucore/locales/*.ftl; do
            [ -f "$ftl" ] || continue
            [ "$(basename "$ftl")" = "en-US.ftl" ] && continue
            cp "$ftl" "coreutils/src/uucore/locales/"
          done
        fi

    - name: Patch mdbook theme with language selector
      run: |
        uutils.github.io/scripts/patch-mdbook-theme.sh coreutils/docs coreutils-l10n

    - name: Build Coreutils Docs
      run: |
        cd coreutils
        # Download and repack tldr zip: uudoc expects pages/common/ prefix
        curl -sfL https://github.com/tldr-pages/tldr/releases/download/v2.3/tldr-pages.zip -o /tmp/tldr-raw.zip || true
        if [ -f /tmp/tldr-raw.zip ]; then
          mkdir -p /tmp/tldr-repack/pages
          cd /tmp/tldr-repack && unzip -o /tmp/tldr-raw.zip -d pages/ > /dev/null
          zip -r "$OLDPWD/docs/tldr.zip" pages/ > /dev/null
          cd "$OLDPWD"
        fi
        cargo run --bin uudoc --all-features
        cd docs
        # Remove deprecated 'multilingual' field unsupported by newer mdbook
        sed -i '/^multilingual/d' book.toml
        # Strip the legacy FA4 `fa` class from uudoc-generated platform icons
        # so mdbook 0.5's Font Awesome parser picks up the `fa-brands` family
        # instead of defaulting to `regular` and warning about linux/windows/apple.
        find src/utils -name '*.md' -exec sed -i 's|class="fa fa-brands |class="fa-brands |g' {} +
        mdbook build

    - name: Build Coreutils Docs (translations)
      run: |
        uutils.github.io/scripts/build-docs-l10n.sh coreutils

    - name: Build Findutils Docs
      run: |
        cd findutils/docs
        # Remove deprecated 'multilingual' field unsupported by newer mdbook
        sed -i '/^multilingual/d' book.toml
        mdbook build

    - name: Download WASM artifacts
      uses: actions/download-artifact@v8
      with:
        pattern: wasm-*
        path: wasm-artifacts
        merge-multiple: true

    - name: Assemble WASM playground assets
      run: |
        # The four wasm-* artifacts each carry their .wasm plus numbered
        # command/version fragments. Stitch them into the single commands.js /
        # version.js the playground loads. SITE_VERSION is generated here since
        # it describes the uutils.github.io repo, not any of the tool repos.
        mkdir -p uutils.github.io/static/wasm
        cp wasm-artifacts/*.wasm uutils.github.io/static/wasm/
        cp wasm-artifacts/locales.js uutils.github.io/static/wasm/

        # Merge the command-name fragments in numeric (filename) order.
        commands=$(cat $(ls wasm-artifacts/commands-*.txt | sort) | paste -sd, -)
        echo "const WASM_COMMANDS = [${commands}];" \
          > uutils.github.io/static/wasm/commands.js
        echo "Playground commands: ${commands}"

        # Record the uutils.github.io commit the site was built from.
        site_hash=$(git -C uutils.github.io rev-parse HEAD)
        site_short=$(git -C uutils.github.io rev-parse --short HEAD)
        site_date=$(git -C uutils.github.io show -s --format=%cI HEAD)
        {
          cat $(ls wasm-artifacts/version-*.js | sort)
          echo "const SITE_VERSION = {"
          echo "  commit: \"${site_hash}\","
          echo "  short: \"${site_short}\","
          echo "  date: \"${site_date}\""
          echo "};"
        } > uutils.github.io/static/wasm/version.js
        echo "site build: ${site_short} (${site_date})"

    - name: Run Zola
      uses: shalzz/zola-deploy-action@v0.22.1
      env:
        BUILD_DIR: uutils.github.io
        BUILD_ONLY: true

    - name: Collect results into `public` folder
      run: |
        cp -r uutils.github.io/public public
        cp -r coreutils/docs/book public/coreutils/docs
        cp -r findutils/docs/book public/findutils/docs
        # Copy translated docs to /coreutils/docs-{lang}/
        for lang_dir in coreutils/docs/book-*/; do
          [ -d "$lang_dir" ] || continue
          lang=$(basename "$lang_dir" | sed 's/^book-//')
          cp -r "$lang_dir" "public/coreutils/docs-${lang}"
        done

    - name: Run playground JS tests
      run: |
        npm install puppeteer@24
        node uutils.github.io/scripts/run-tests.js --dir public --port 8080

    - name: Remove test files from deploy output
      run: rm -f public/js/wasm-terminal.test.html

    - name: Upload artifact for checking the output
      uses: actions/upload-artifact@v7
      with:
        path: ./public

    - name: Upload artifact for pages
      uses: actions/upload-pages-artifact@v5
      with:
        path: ./public

  # Deployment job
  deploy:
    permissions:
      contents: read
      pages: write
      id-token: write
    environment:
      name: github-pages
      url: ${{ steps.deployment.outputs.page_url }}
    runs-on: ubuntu-latest
    needs: build
    if: (github.event_name == 'push' && github.ref == 'refs/heads/main') || github.event_name == 'schedule'
    steps:
      - name: Deploy to GitHub Pages
        id: deployment
        uses: actions/deploy-pages@v5