Bugfixes and improvements (#43)

* also run snapshot test (and correctly treat the "snapshot" case, in which images and generated
files are named a bit differently (without a version))
* minor fixes
* refactor the tests

Author: jandelgado

.github/workflows/test.yml

@@ -18,7 +18,7 @@ jobs:
     name: test nix builds
     strategy:
       matrix:
-        example: ["x86_64", "rpi4", "glinet-gl-ar750", "glinet-gl-mt300n-v2", "nexx-wt3020", "rpi2", "wrt1043nd"]
+        example: ["x86_64", "x86_64-snapshot", "rpi4", "glinet-gl-ar750", "glinet-gl-mt300n-v2", "nexx-wt3020", "rpi2", "wrt1043nd"]
     steps:
       - name: Checkout
         uses: actions/checkout@v4
@@ -36,7 +36,7 @@ jobs:
     name: test container builds
     strategy:
       matrix:
-        example: ["x86_64", "rpi4", "glinet-gl-ar750", "glinet-gl-mt300n-v2", "nexx-wt3020", "rpi2", "wrt1043nd"]
+        example: ["x86_64", "x86_64-snapshot", "rpi4", "glinet-gl-ar750", "glinet-gl-mt300n-v2", "nexx-wt3020", "rpi2", "wrt1043nd"]
         option: ["--docker --sudo", "--podman", "--podman --sudo"]
     steps:
       - name: Checkout

.test/run_all.sh

@@ -1,26 +1,45 @@
 #!/bin/bash
 # run all tests for a given configuration. tests are run after images are
-# built. we do some simple assertions on the generated artifiacts to e.g. make
+# built. we do some simple assertions on the generated artifacts to e.g. make
 # sure the configured packages are included etc.
 set -eou pipefail
 
 SCRIPT_DIR=$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )
+
+if (( $# != 2 )); then
+    echo "Usage: $(basename "$0") <configuration> <output-directory>"
+    echo "Example: $(basename "$0") example-x86_64.conf output"
+    exit 1
+fi
+
 CONF=$1
 OUT=$2
-RC=0
+PASSED=0
+FAILED=0
+
+if ! source "$CONF" 2>/dev/null; then
+    echo "ERROR: cannot load configuration: $CONF"
+    exit 1
+fi
+
+echo "======================================================================"
+echo " TEST RESULTS  $CONF in $OUT"
+echo "======================================================================"
 
-echo "----------------------------------------------------------------------"
-echo "run tests for $CONF"
-echo "----------------------------------------------------------------------"
 for t in "$SCRIPT_DIR"/test_*.sh; do
-	echo "******************** RUNNING CASE TEST $t ************************"
-    if "$t" "$CONF" "$OUT"; then
-        echo ">>> TEST OK"
+    name=$(basename "$t" .sh | sed 's/^test_//')
+    if output=$("$t" "$CONF" "$OUT" 2>&1); then
+        echo "  [ PASS ] $name"
+        PASSED=$(( PASSED + 1 ))
     else
-        echo ">>> TEST FAILED"
-        RC=1
+        echo "  [ FAIL ] $name"
+        echo "$output" | sed 's/^/             /'
+        FAILED=$(( FAILED + 1 ))
     fi
-	echo "******************************************************************"
 done
 
-exit $RC
+total=$(( PASSED + FAILED ))
+echo "----------------------------------------------------------------------"
+echo " $total tests: $PASSED passed, $FAILED failed"
+echo "======================================================================"
+[[ $FAILED -eq 0 ]] || exit 1

.test/test-common

@@ -0,0 +1,22 @@
+CONF="$1"
+source "$1"
+DIR="$2"
+FAIL=0
+
+if [[ "$LEDE_RELEASE" == "snapshots" ]]; then
+    PREFIX="openwrt-$LEDE_TARGET-$LEDE_SUBTARGET-$LEDE_PROFILE"
+else
+    PREFIX="openwrt-$LEDE_RELEASE-$LEDE_TARGET-$LEDE_SUBTARGET-$LEDE_PROFILE"
+fi
+
+fail() {
+    echo "FAIL: $*"
+    FAIL=1
+}
+
+dump_context() {
+    echo "conf:     $CONF"
+    echo "output:   $DIR"
+    echo "prefix:   $PREFIX"
+    echo "packages: $LEDE_PACKAGES"
+}

.test/test_images_are_generated.sh

@@ -1,20 +1,13 @@
 #!/bin/bash
-source $1
-DIR=$2  # e.g. output/
-FAIL=0
+source "$(dirname "${BASH_SOURCE[0]}")/test-common" "$@"
 
-fail() {
-    echo "  ERROR: $*"
-    FAIL=1
-}
+FILES=$(find "$DIR" -type f \( -name "$PREFIX-*.img.gz" -o -name "$PREFIX-*.bin" \) | wc -l)
 
-PREFIX="openwrt-$LEDE_RELEASE-$LEDE_TARGET-$LEDE_SUBTARGET-$LEDE_PROFILE"
-
-echo "Test if images are generated for $PREFIX ..."
-
-FILES=$(find "$DIR"  -type f -name "$PREFIX-*.img.gz" \
-        -o -name "$PREFIX-*.bin" | wc -l)
-
-[ "$FILES" -eq "0" ] && fail "no images were built for $PREFIX"
+if (( FILES == 0 )); then
+    fail "no images found for prefix '$PREFIX'"
+    dump_context
+    echo "files in $DIR:"
+    ls "$DIR" 2>/dev/null || echo "(directory empty or missing)"
+fi
 
 exit $FAIL

.test/test_manifest_contains_packages.sh

@@ -1,33 +1,23 @@
 #!/bin/bash
+source "$(dirname "${BASH_SOURCE[0]}")/test-common" "$@"
 
-source $1
-DIR=$2  # e.g. output/
-FAIL=0
-
-fail() {
-    echo "  ERROR: $*"
-    FAIL=1
-}
-
-PREFIX="openwrt-$LEDE_RELEASE-$LEDE_TARGET-$LEDE_SUBTARGET-$LEDE_PROFILE"
 MANIFEST="$DIR/$PREFIX.manifest"
 
-echo "Test all configured packages are in manifest file $MANIFEST..."
 if [ ! -f "$MANIFEST" ]; then
-    fail "$MANIFEST does not exist"
+    fail "manifest not found: $MANIFEST"
+    dump_context
     exit 1
 fi
 
 for p in $LEDE_PACKAGES; do
     [[ $p == -* ]] && continue
-	grep -q -E "^$p " "$MANIFEST" || fail "package not in manifest: $p"
+    grep -q -E "^$p " "$MANIFEST" || fail "package not in manifest: $p"
 done
 
-if [ $FAIL -ne 0 ]; then
-	echo "======== TEST FAILED ======="
-	echo "LEDE_PACKAGES: $LEDE_PACKAGES"
-	echo "MANIFEST: $MANIFEST"
-	cat "$MANIFEST"
+if (( FAIL )); then
+    dump_context
+    echo "manifest:"
+    cat "$MANIFEST"
 fi
 
 exit $FAIL

.test/test_manifest_does_not_contain_excluded_packages.sh

@@ -1,28 +1,22 @@
 #!/bin/bash
+source "$(dirname "${BASH_SOURCE[0]}")/test-common" "$@"
 
-source $1
-DIR=$2  # e.g. output/
-FAIL=0
-
-fail() {
-    echo "  ERROR: $*"
-    FAIL=1
-}
-
-PREFIX="openwrt-$LEDE_RELEASE-$LEDE_TARGET-$LEDE_SUBTARGET-$LEDE_PROFILE"
 MANIFEST="$DIR/$PREFIX.manifest"
 
-echo "Test all excluded packages are not in manifest file $MANIFEST..."
 if [ ! -f "$MANIFEST" ]; then
-    fail "$MANIFEST does not exist"
+    fail "manifest not found: $MANIFEST"
+    dump_context
     exit 1
 fi
 
-for p in $LEDE_PACKAGES; do 
-    # consider only excluded packages, starting with "-", e.g. "-ppp"
-    [[ ! $p == -* ]] && continue
+for p in $LEDE_PACKAGES; do
+    [[ $p != -* ]] && continue
     p=${p:1}
-    grep -q -E "^$p " "$MANIFEST" && fail "package not expected to be in manifest: $p"
+    grep -q -E "^$p " "$MANIFEST" && fail "excluded package found in manifest: $p"
 done
 
+if (( FAIL )); then
+    dump_context
+fi
+
 exit $FAIL

CHANGELOG.md

@@ -3,6 +3,7 @@
 ## v3.9 [2026-05-30]
 
 - Upgrade to OpenWrt 25.12.4
+- Bugfixes and test refactoring
 
 ## v3.8 [2025-02-09]
 

builder.sh

@@ -63,7 +63,6 @@ Example:
   # use nix to build the OpenWrt image, no need to build a container first
   $PROG build example-x86_64.conf --nix
 EOT
-    exit 0
 }
 
 # return given file path as absolute path. path to file must exist.
@@ -83,7 +82,7 @@ function build_docker_image  {
 }
 
 function run_cmd_in_nix_shell {
-    nix-shell --pure --run "bash -c '$*'"
+    nix-shell --pure --run "$*"
 }
 
 function run_cmd_in_container {
@@ -103,9 +102,9 @@ function run_cmd_in_container {
         -v "$(abspath "$ROOTFS_OVERLAY")":/lede/rootfs-overlay \
         -v "$(abspath "$OUTPUT_DIR")":/lede/output \
         "${repositories_volume[@]}" \
-        ${DOCKER_OPTS[@]} \
+        "${DOCKER_OPTS[@]}" \
         ${_DOCKER_OPTS:-} \
-        --rm "$(image_tag)" bash -c "$*"
+        "$(image_tag)" bash -c "$*"
 }
 
 function run_cmd {
@@ -162,11 +161,16 @@ download_builder() {
         return
     fi
 
-    mkdir -p "$dir"
-    echo curl "download $url -> $dir" 
-    curl --progress-bar "$url" -o "$dir/tmpbuilder" \
-         && tar xfa "$dir/tmpbuilder" --strip-components=1 -C "$dir" \
-         && rm "$dir/tmpbuilder"
+    local tmpdir="${dir}.tmp"
+    rm -rf "$tmpdir"
+    mkdir -p "$tmpdir"
+    trap 'rm -rf "$tmpdir"' ERR INT TERM
+    echo "downloading $url -> $dir"
+    curl --progress-bar "$url" -o "$tmpdir/tmpbuilder" \
+         && tar xfa "$tmpdir/tmpbuilder" --strip-components=1 -C "$tmpdir" \
+         && rm "$tmpdir/tmpbuilder" \
+         && mv "$tmpdir" "$dir"
+    trap - ERR INT TERM
 }
 
 function print_config {
@@ -195,7 +199,7 @@ EOT
 # tests if given version is at least provided reference version.
 # version_ge_than("23.05.1", "24") -> false
 # version_ge_than("24.0.10", "24") -> true
-function version_ge_than { [ "${1%%.*}" -ge "$2" ]; }
+function version_ge_than { local major="${1%%.*}"; [[ "$major" =~ ^[0-9]+$ ]] && [ "$major" -ge "$2" ]; }
 
 # return default LEDE_BUILDER_URL if not overriden in configuration file
 function builder_url {
@@ -300,7 +304,7 @@ function dispatch_command {
              run_cmd "$cmd"
              ;;
          build-docker-image)
-             [ "$RUNTIME" == "nix" ] && warn "refusing to build docker image when using --nix" && exit
+             [ "$RUNTIME" == "nix" ] && warn "refusing to build docker image when using --nix" && exit 1
              print_config
              build_docker_image "$(builder_url)" "$(image_tag)"
              ;;
@@ -313,15 +317,13 @@ function dispatch_command {
              ;;
          *)
             usage "$0"
-            # shellcheck disable=2317
             exit 0
             ;;
     esac
 }
 
 if [ $# -lt 2 ]; then
     usage "$0"
-    # shellcheck disable=2317
     exit 1
 fi
 

example-x86_64-snapshot.conf

@@ -10,7 +10,7 @@ LEDE_SUBTARGET=64
 LEDE_PACKAGES="base-files busybox dropbear mtd uci netifd \
 fstools uclient-fetch logd kmod-usb-hid dnsmasq luci-nginx "
 
-LEDE_BUILDER_URL="https://downloads.openwrt.org/$LEDE_RELEASE/targets/$LEDE_TARGET/$LEDE_SUBTARGET/openwrt-imagebuilder-$LEDE_TARGET-$LEDE_SUBTARGET.Linux-x86_64.tar.xz"
+LEDE_BUILDER_URL="https://downloads.openwrt.org/$LEDE_RELEASE/targets/$LEDE_TARGET/$LEDE_SUBTARGET/openwrt-imagebuilder-$LEDE_TARGET-$LEDE_SUBTARGET.Linux-x86_64.tar.zst"
 
 # optionally override OUTPUT_DIR and ROOTFS_OVERLAY directory location here