Revert "build: compile plugins in parallel with a central dist merge"

This reverts commit 82ffa41. Running up to 4 Nuitka processes in parallel did
not speed up the Windows build: Nuitka already parallelises its C compilation
across all CPU cores per plugin, so concurrent builds only oversubscribe the
4-vCPU runner (measured ~41+ min vs ~63 min sequential, no meaningful gain).
The sequential script is simpler and just as fast. A persisted Nuitka cache is
the more promising lever and will be evaluated separately.

Author: markuslf

build/compile-multiple.sh

@@ -1,23 +1,19 @@
 #!/usr/bin/env bash
-# 2026053001
+# 2025021702
 
 # This script can run in a container (absolute paths) or in a Windows-VM.
 
 set -e -x
 
 if uname -a | grep -q "_NT"; then
     # We are on Windows.
-    IS_WINDOWS='yes'
     REPO_DIR="$LFMP_DIR_REPOS"
-    COMPILE_DIR="$LFMP_DIR_COMPILED"
 else
     # We are in a container.
-    IS_WINDOWS='no'
     REPO_DIR="/repos"
-    COMPILE_DIR="/compiled"
 fi
 
-if [ "$IS_WINDOWS" = 'no' ]; then
+if ! uname -a | grep -q "_NT"; then
     # We are in a container.
     source /opt/venv/bin/activate
     PY_TAG="py$(python3 -c 'import sys; print(f"{sys.version_info.major}{sys.version_info.minor}")')"
@@ -30,23 +26,10 @@ if [ "$IS_WINDOWS" = 'no' ]; then
 fi
 python3 --version
 
-SCRIPT_DIR="$(dirname "$0")"
-COMPILE_ONE="$SCRIPT_DIR/compile-one.sh"
-
-# Number of plugins to compile concurrently. Defaults to 4 (the GitHub Windows
-# runner has 4 vCPUs). Override with LFMP_COMPILE_JOBS.
-JOBS="${LFMP_COMPILE_JOBS:-4}"
-
-# Capture the optional user-provided plugin list once. The per-group logic
-# below mirrors the original behaviour: when a list is given it applies to
-# check-plugins only; event-plugins and notification-plugins are skipped.
-USER_PLUGIN_LIST="$LFMP_COMPILE_PLUGINS"
-
-# Build a flat list of "<group> <plugin>" tasks to compile.
-TASKS=()
+# Loop through each plugin in the list.
 for PLUGINS in event-plugins notification-plugins check-plugins ; do
-    if [[ -n "$USER_PLUGIN_LIST" && "$PLUGINS" != "check-plugins" ]]; then
-        # if a check-plugin list is given, skip event-plugins and notification-plugins to save time
+    if [[ ! -z "$LFMP_COMPILE_PLUGINS" && "$PLUGINS" != "check-plugins" ]]; then
+        # if check-plugin list is given, skip event-plugins and notification-plugins to save time
         echo "✅ Skipping $REPO_DIR/monitoring-plugins/$PLUGINS..."
         continue
     fi
@@ -56,68 +39,30 @@ for PLUGINS in event-plugins notification-plugins check-plugins ; do
     fi
     echo "✅ Processing $PLUGINS..."
 
-    if [[ -z "$USER_PLUGIN_LIST" ]]; then
+    # If $LFMP_COMPILE_PLUGINS is empty, find all plugin directories under
+    # $REPO_DIR/monitoring-plugins/$PLUGINS/ and create a space-separated list.
+    if [[ -z "$LFMP_COMPILE_PLUGINS" ]]; then
         echo "✅ No plugin list provided. Discovering all plugins..."
-        # List the directories immediately under $PLUGINS/ by basename.
-        # avoid using `find` as this is sometimes problematic in Github runners
-        GROUP_PLUGINS=$(
+        # Find directories immediately under $PLUGINS/, extract their basenames, and join them with commas.
+        LFMP_COMPILE_PLUGINS=$(
             for d in "$REPO_DIR/monitoring-plugins/$PLUGINS"/*; do
                 [ -d "$d" ] && basename "$d"
             done | sort
-        )
-        echo "✅ Found '$GROUP_PLUGINS'"
-    else
-        GROUP_PLUGINS="$USER_PLUGIN_LIST"
+        )  # avoid using `find` as this is sometimes problematic in Github runners
+        echo "✅ Found '$LFMP_COMPILE_PLUGINS'"
     fi
 
-    for PLUGIN in $GROUP_PLUGINS; do
+    for PLUGIN in $LFMP_COMPILE_PLUGINS; do
         if [ "$PLUGIN" == "example" ]; then
             continue
         fi
-        if [[ ! -d "$REPO_DIR/monitoring-plugins/$PLUGINS/$PLUGIN" ]]; then
+        if [[ -d "$REPO_DIR/monitoring-plugins/$PLUGINS/$PLUGIN" ]]; then
+            bash $(dirname "$0")/compile-one.sh $PLUGINS $PLUGIN
+        else
             echo "✅ Directory $REPO_DIR/$PLUGINS/$PLUGIN does not exist. Ignoring..."
-            continue
-        fi
-        # On Windows only plugins carrying a `.windows` marker get compiled
-        # (compile-one.sh enforces this too). Filter them out here so the
-        # parallel pool below is not filled with immediate no-op invocations.
-        if [[ "$IS_WINDOWS" = 'yes' && ! -f "$REPO_DIR/monitoring-plugins/$PLUGINS/$PLUGIN/.windows" ]]; then
-            echo "✅ Ignoring '$PLUGIN' on Windows (no .windows marker)"
-            continue
-        fi
-        TASKS+=("$PLUGINS $PLUGIN")
-    done
-done
-
-# Compile. The first plugin runs alone to warm up Nuitka's download/cache dir
-# (depends.exe, ccache, ...), so the parallel batch that follows does not race
-# on first-time downloads. The rest run up to $JOBS at a time. xargs returns
-# non-zero if any invocation fails, so `set -e` aborts the whole build.
-if (( ${#TASKS[@]} > 0 )); then
-    echo "✅ Compiling ${#TASKS[@]} plugin(s), up to $JOBS in parallel..."
-    read -r WARM_GROUP WARM_PLUGIN <<< "${TASKS[0]}"
-    bash "$COMPILE_ONE" "$WARM_GROUP" "$WARM_PLUGIN"
-    if (( ${#TASKS[@]} > 1 )); then
-        printf '%s\n' "${TASKS[@]:1}" | xargs -P "$JOBS" -L 1 bash "$COMPILE_ONE"
-    fi
-fi
-
-# Merge each plugin's standalone `*.dist` directory into the shared, flattened
-# group directory to save disk space (important on Github runners). Done here,
-# serially, after all parallel compiles finished, so the shared runtime files
-# the plugins all carry (python3xx.dll etc.) are never written concurrently.
-# The special `/.` suffix copies only the *contents* of each dist directory.
-for PLUGINS in event-plugins notification-plugins check-plugins ; do
-    GROUP_DIR="$COMPILE_DIR/$PLUGINS"
-    [ -d "$GROUP_DIR" ] || continue
-    for DIST in "$GROUP_DIR"/*.dist ; do
-        [ -d "$DIST" ] || continue
-        if [ -n "$(ls --almost-all "$DIST")" ]; then
-            echo "✅ cp --archive $DIST/. $GROUP_DIR/"
-            \cp --archive "$DIST"/. "$GROUP_DIR"/
         fi
-        rm -rf "$DIST"
     done
+    LFMP_COMPILE_PLUGINS=""
 done
 
 # On RHEL? Then also compile the Linuxfabrik Type Enforcement Policy

build/compile-one.sh

@@ -1,5 +1,5 @@
 #!/usr/bin/env bash
-# 2026053001
+# 2025021602
 
 # This script can run in a container (absolute paths) or in a Windows-VM.
 
@@ -43,8 +43,12 @@ if [[ -e "$COMPILE_DIR/$PLUGINS/$PLUGIN.dist/$PLUGIN.bin" ]]; then
     mv "$COMPILE_DIR/$PLUGINS/$PLUGIN.dist/$PLUGIN.bin" "$COMPILE_DIR/$PLUGINS/$PLUGIN.dist/$PLUGIN"
 fi
 
-# The compiled plugin is left in its own `$PLUGIN.dist` directory on purpose.
-# Flattening every plugin's dist into the shared `$COMPILE_DIR/$PLUGINS/`
-# directory is done centrally by compile-multiple.sh *after* all plugins are
-# built, so that several compile-one.sh runs can execute in parallel without
-# racing on the shared runtime files (python3xx.dll etc.) they all carry.
+# move files to a merged flattened directory to save disk space if directory exists and is not empty.
+if [[ -d "$COMPILE_DIR/$PLUGINS/$PLUGIN.dist" && -n "$(ls --almost-all $COMPILE_DIR/$PLUGINS/$PLUGIN.dist)" ]]; then
+    # Note that we use the special /.' suffix to copy **only the contents** of the *.dist directory (preserving attributes).
+    # Unfortunately, mv does not have a built‐in equivalent to "move the contents of a directory" using the/.' trick.
+    # Therefore we copy and remove later to save disk space, important in github runners.
+    echo "✅ cp --archive $COMPILE_DIR/$PLUGINS/$PLUGIN.dist/. $COMPILE_DIR/$PLUGINS/"
+    \cp --archive --verbose $COMPILE_DIR/$PLUGINS/$PLUGIN.dist/. $COMPILE_DIR/$PLUGINS/
+    rm -rf $COMPILE_DIR/$PLUGINS/$PLUGIN.dist
+fi