Complete scoped regression smoke checks

Author: ranimandepudi

.github/actions/run-missing-package-smoke/action.yml

@@ -182,6 +182,144 @@ runs:
           test6_duration=0
         }
 
+        pypi_latest_version() {
+          local package="$1"
+          python3 - "$package" <<'PY'
+        import json
+        import sys
+        import urllib.request
+
+        package = sys.argv[1]
+        with urllib.request.urlopen(f"https://pypi.org/pypi/{package}/json", timeout=30) as response:
+            data = json.load(response)
+        print(data["info"]["version"])
+        PY
+        }
+
+        run_fastai_regression() {
+          local current="${1:-unknown}"
+          local start latest work_next
+          start=$(date +%s)
+          regression_current_version="$current"
+          latest="$(pypi_latest_version fastai 2>/tmp/fastai-pypi-latest-error.log || true)"
+          regression_latest_version="${latest:-unknown}"
+          if [ -z "$latest" ]; then
+            regression_decision="next_lookup_failed"
+            regression_next_installed_version="not_installed"
+            regression_result="Next FastAI version lookup failed on Arm64"
+            regression_details="Baseline FastAI ${current} passed Tests 1-5, but PyPI latest-version lookup failed, so Test 6 could not honestly select a next candidate."
+            set_test 6 failed "$start"
+            return
+          fi
+          if [ "$latest" = "$current" ]; then
+            regression_decision="current_is_latest_stable"
+            regression_next_installed_version="not_applicable"
+            regression_result="Current FastAI version is already latest stable"
+            regression_details="Baseline FastAI ${current} passed Tests 1-5, and PyPI reports the same version as latest stable. There is no newer stable FastAI candidate for Test 6."
+            set_test 6 skipped "$start"
+            return
+          fi
+          if ! apt_bootstrap podman; then
+            regression_decision="next_install_failed"
+            regression_next_installed_version="not_installed"
+            regression_result="Podman CLI dependency failed to install on Arm64"
+            regression_details="Baseline Podman Compose ${current} passed Tests 1-5, but Test 6 could not install the Podman CLI dependency required to validate PyPI latest ${latest} on the Arm64 runner."
+            set_test 6 failed "$start"
+            return
+          fi
+          work_next=$(mktemp -d)
+          if python3 -m venv "$work_next/venv" && \
+             "$work_next/venv/bin/python" -m pip install --upgrade pip >/dev/null && \
+             "$work_next/venv/bin/python" -m pip install "fastai==${latest}" ipython >/tmp/fastai-next-install.log 2>&1 && \
+             "$work_next/venv/bin/python" - <<'PY'
+        import fastai
+        import fastai.torch_core
+        import platform
+        import torch
+        from fastai.torch_core import tensor
+
+        assert platform.machine() in ("aarch64", "arm64")
+        assert fastai.__version__
+        value = tensor([1, 2, 3])
+        assert torch.equal(value + 1, torch.tensor([2, 3, 4]))
+        PY
+          then
+            regression_decision="next_install_validated"
+            regression_next_installed_version="$latest"
+            regression_result="Next FastAI version installed and passed bounded Arm64 tensor smoke"
+            regression_details="Baseline FastAI ${current} passed Tests 1-5, and PyPI latest ${latest} installed on the Arm64 runner, imported FastAI/Torch, verified the Arm64 Python runtime, and ran the same bounded tensor smoke."
+            set_test 6 passed "$start"
+          else
+            cat /tmp/fastai-next-install.log 2>/dev/null || true
+            regression_decision="next_install_failed"
+            regression_next_installed_version="not_installed"
+            regression_result="Next FastAI version failed bounded Arm64 validation"
+            regression_details="Baseline FastAI ${current} passed Tests 1-5, but PyPI latest ${latest} did not complete the same bounded install/import/tensor smoke on the Arm64 runner."
+            set_test 6 failed "$start"
+          fi
+        }
+
+        run_podman_compose_regression() {
+          local current="${1:-unknown}"
+          local start latest work_next
+          start=$(date +%s)
+          regression_current_version="$current"
+          latest="$(pypi_latest_version podman-compose 2>/tmp/podman-compose-pypi-latest-error.log || true)"
+          regression_latest_version="${latest:-unknown}"
+          if [ -z "$latest" ]; then
+            regression_decision="next_lookup_failed"
+            regression_next_installed_version="not_installed"
+            regression_result="Next Podman Compose version lookup failed on Arm64"
+            regression_details="Baseline Podman Compose ${current} passed Tests 1-5, but PyPI latest-version lookup failed, so Test 6 could not honestly select a next candidate."
+            set_test 6 failed "$start"
+            return
+          fi
+          if [ "$latest" = "$current" ]; then
+            regression_decision="current_is_latest_stable"
+            regression_next_installed_version="not_applicable"
+            regression_result="Current Podman Compose version is already latest stable"
+            regression_details="Baseline Podman Compose ${current} passed Tests 1-5, and PyPI reports the same version as latest stable. There is no newer stable Podman Compose candidate for Test 6."
+            set_test 6 skipped "$start"
+            return
+          fi
+          work_next=$(mktemp -d)
+          if python3 -m venv "$work_next/venv" && \
+             "$work_next/venv/bin/python" -m pip install --upgrade pip >/dev/null && \
+             "$work_next/venv/bin/python" -m pip install "podman-compose==${latest}" >/tmp/podman-compose-next-install.log 2>&1; then
+            cat > "$work_next/compose.yml" <<'EOF'
+        services:
+          smoke:
+            image: quay.io/podman/hello
+            command: ["echo", "arm64"]
+        EOF
+            if "$work_next/venv/bin/podman-compose" --version | grep -q "$latest" && \
+               "$work_next/venv/bin/podman-compose" --help | grep -qiE 'usage|podman-compose' && \
+               "$work_next/venv/bin/python" -m pip check && \
+               [ "$(uname -m)" = "aarch64" ] && \
+               "$work_next/venv/bin/podman-compose" --dry-run -f "$work_next/compose.yml" config >/tmp/podman-compose-next-config.log 2>&1 && \
+               grep -qi 'smoke' /tmp/podman-compose-next-config.log; then
+              regression_decision="next_install_validated"
+              regression_next_installed_version="$latest"
+              regression_result="Next Podman Compose version installed and passed bounded Arm64 dry-run validation"
+              regression_details="Baseline Podman Compose ${current} passed Tests 1-5, and PyPI latest ${latest} installed on the Arm64 runner with the Podman CLI dependency, reported the expected version/help output, passed dependency checks, and rendered the same dry-run Compose config."
+              set_test 6 passed "$start"
+            else
+              regression_decision="next_install_failed"
+              regression_next_installed_version="not_installed"
+              regression_result="Next Podman Compose version failed bounded Arm64 validation"
+              regression_details="Baseline Podman Compose ${current} passed Tests 1-5, but PyPI latest ${latest} did not complete the same version/help/dependency/dry-run config validation on the Arm64 runner."
+              set_test 6 failed "$start"
+            fi
+          else
+            cat /tmp/podman-compose-next-install.log 2>/dev/null || true
+            regression_decision="next_install_failed"
+            regression_next_installed_version="not_installed"
+            regression_result="Next Podman Compose version failed to install on Arm64"
+            regression_details="Baseline Podman Compose ${current} passed Tests 1-5, but PyPI latest ${latest} did not install on the Arm64 runner."
+            set_test 6 failed "$start"
+          fi
+        }
+
         write_outputs() {
           {
             echo "version=${version}"
@@ -398,7 +536,7 @@ runs:
         assert torch.equal(value + 1, torch.tensor([2, 3, 4]))
         PY
             fi
-            finish_package_manager_test6 "$version"
+            run_fastai_regression "$version"
             ;;
 
           podman-compose)
@@ -428,7 +566,7 @@ runs:
         EOF
               start=$(date +%s); "$work/venv/bin/podman-compose" --dry-run -f "$work/compose.yml" config >/tmp/podman-compose-config.log 2>&1 && grep -qi 'smoke' /tmp/podman-compose-config.log && set_test 5 passed "$start" || set_test 5 failed "$start"
             fi
-            finish_package_manager_test6 "$version"
+            run_podman_compose_regression "$version"
             ;;
 
           luigi-source)

.github/workflows/test-amazon-vpc-cni-k8s.yml

@@ -600,7 +600,7 @@ jobs:
           lane_kind: ${{ env.LANE_KIND }}
           limited_cpu_probe: |
             cd next-src
-            go list ./... >/tmp/amazon-vpc-cni-next-go-packages.txt
+            GOFLAGS=-mod=mod go list ./... >/tmp/amazon-vpc-cni-next-go-packages.txt
             BUILT=0
             while IFS= read -r main_file; do
               target="$(dirname "$main_file")"
@@ -618,7 +618,7 @@ jobs:
             assert any(marker in text for marker in ("DaemonSet", "amazon-k8s-cni", "aws-node", "CNI")), "missing CNI/Kubernetes markers"
             PY
           limited_cpu_description: "Test 6 reran the scoped Amazon VPC CNI Arm preflight against the next stable source tag: Go package discovery, linux/arm64 command builds, AArch64 binary checks, and Kubernetes/CNI manifest validation. Full pod networking still requires Kubernetes plus AWS VPC/ENI infrastructure."
-          defer_on_limited_cpu_probe_failure: "true"
+          defer_on_limited_cpu_probe_failure: "false"
       - name: Calculate test summary
         id: summary
         if: always()

.github/workflows/test-ampere-optimized-llama.yml

@@ -371,20 +371,28 @@ jobs:
           lane_kind: ${{ env.LANE_KIND }}
           limited_cpu_probe: |
             sudo apt-get update -qq
-            sudo apt-get install -y --no-install-recommends build-essential cmake >/dev/null
-            cd next-src
-            SOURCE_DIR="$(dirname "$(find . -maxdepth 4 -name CMakeLists.txt -print -quit)")"
-            [ -n "$SOURCE_DIR" ] && [ "$SOURCE_DIR" != "." ] || SOURCE_DIR="."
-            test -f "$SOURCE_DIR/CMakeLists.txt"
-            cmake -S "$SOURCE_DIR" -B build -DCMAKE_BUILD_TYPE=Release -DLLAMA_NATIVE=OFF -DLLAMA_BUILD_TESTS=OFF
-            cmake --build build --target llama-cli -j2 || cmake --build build --target main -j2 || cmake --build build -j2
-            BIN=$(find build -type f \( -name llama-cli -o -name main -o -name llama-simple \) -perm -111 | head -n 1)
+            sudo apt-get install -y --no-install-recommends curl file jq tar >/dev/null
+            WORK_DIR="$(mktemp -d)"
+            RELEASE_JSON="$WORK_DIR/release.json"
+            curl -fsSL "https://api.github.com/repos/${GITHUB_REPO}/releases/tags/${CANDIDATE_TAG}" -o "$RELEASE_JSON"
+            ASSET_URL="$(jq -r '.assets[] | select(.name | test("^llama_aio_.*\\.tar\\.gz$"; "i")) | .browser_download_url' "$RELEASE_JSON" | head -n 1)"
+            [ -n "$ASSET_URL" ] && [ "$ASSET_URL" != "null" ]
+            curl -fsSL "$ASSET_URL" -o "$WORK_DIR/llama-aio.tar.gz"
+            tar -xzf "$WORK_DIR/llama-aio.tar.gz" -C "$WORK_DIR"
+            BIN="$(find "$WORK_DIR" -type f \( -name llama-cli -o -name main \) -perm -111 | head -n 1)"
             [ -n "$BIN" ]
             file "$BIN" | grep -Eiq 'aarch64|arm64'
+            export LD_LIBRARY_PATH="$(dirname "$BIN"):${LD_LIBRARY_PATH:-}"
             "$BIN" --help >/tmp/llama-next-help.txt 2>&1 || "$BIN" -h >/tmp/llama-next-help.txt 2>&1
             grep -Eiq 'usage|llama|model|prompt' /tmp/llama-next-help.txt
-          limited_cpu_description: "Test 6 attempts the same scoped Ampere llama.cpp Arm CLI build against the next stable source candidate when that candidate contains buildable llama.cpp CMake sources. No GGUF model inference is claimed without weights."
-          defer_on_limited_cpu_probe_failure: "true"
+            SERVER_BIN="$(find "$WORK_DIR" -type f -name llama-server -perm -111 | head -n 1 || true)"
+            if [ -n "$SERVER_BIN" ]; then
+              file "$SERVER_BIN" | grep -Eiq 'aarch64|arm64'
+              "$SERVER_BIN" --help >/tmp/llama-server-next-help.txt 2>&1 || "$SERVER_BIN" -h >/tmp/llama-server-next-help.txt 2>&1
+              grep -Eiq 'usage|server|model|host|port' /tmp/llama-server-next-help.txt
+            fi
+          limited_cpu_description: "Test 6 reran a scoped Ampere llama.cpp Arm64 release-artifact preflight against the next stable release: downloaded the official AIO tarball, verified the included llama binaries are AArch64, and ran CLI/server help on the Arm64 runner. No source build, GGUF model inference, model quality, or performance claim is made."
+          defer_on_limited_cpu_probe_failure: "false"
       - name: Calculate test summary
         id: summary
         if: always()

.github/workflows/test-fastai.yml

@@ -53,7 +53,7 @@ jobs:
       regression_current_version: ${{ steps.smoke.outputs.regression_current_version || steps.smoke.outputs.version || 'unknown' }}
       regression_latest_version: ${{ steps.smoke.outputs.regression_latest_version || 'unknown' }}
       regression_next_installed_version: ${{ steps.smoke.outputs.regression_next_installed_version || 'not_installed' }}
-      regression_policy: package_manager_installed
+      regression_policy: applicable
       run_id: ${{ github.run_id }}
       run_attempt: ${{ github.run_attempt }}
       job_name: ${{ github.job }}

.github/workflows/test-flyte.yml

@@ -334,7 +334,20 @@ jobs:
             . /tmp/flytekit-smoke/bin/activate
             python -m pip install --upgrade pip
             python -m pip install flytekit
-            python -c "from flytekit import task, workflow; exec('@task\\ndef add_one(value: int) -> int:\\n    return value + 1\\n\\n@workflow\\ndef wf(value: int) -> int:\\n    return add_one(value=value)\\n'); assert wf(value=41) == 42"
+            {
+              printf '%s\n' 'from flytekit import task, workflow'
+              printf '%s\n' ''
+              printf '%s\n' '@task'
+              printf '%s\n' 'def add_one(value: int) -> int:'
+              printf '%s\n' '    return value + 1'
+              printf '%s\n' ''
+              printf '%s\n' '@workflow'
+              printf '%s\n' 'def wf(value: int) -> int:'
+              printf '%s\n' '    return add_one(value=value)'
+              printf '%s\n' ''
+              printf '%s\n' 'assert wf(value=41) == 42'
+            } >/tmp/flytekit-smoke.py
+            python /tmp/flytekit-smoke.py
             echo "status=passed" >> "$GITHUB_OUTPUT"
             echo "note=Compiled Flyte Go packages for linux/arm64, rendered or validated Flyte Kubernetes manifests, and executed a tiny local flytekit workflow. Kubernetes control-plane deployment remains out of scope." >> "$GITHUB_OUTPUT"
             END_TIME=$(date +%s)
@@ -434,10 +447,12 @@ jobs:
                 tar -xzf /tmp/helm.tgz -C /tmp
                 export PATH="/tmp/linux-arm64:$PATH"
               fi
-              helm dependency build "$(dirname "$CHART")" >/tmp/flyte-next-helm-deps.txt 2>&1 || true
-              if ! helm template flyte-next-smoke "$(dirname "$CHART")" >/tmp/flyte-next-rendered.yaml 2>/tmp/flyte-next-helm-render-error.txt; then
-                find "$(dirname "$CHART")" -type f \( -name '*.yaml' -o -name '*.yml' -o -name '*.tpl' \) -print0 | xargs -0r cat > /tmp/flyte-next-rendered.yaml
-              fi
+              helm repo add flyte-docker-registry https://twuni.github.io/docker-registry.helm >/tmp/flyte-next-helm-repo-docker-registry.txt 2>&1 || true
+              helm repo add flyte-knative-serving https://deeploy-knative-serving-charts.storage.googleapis.com/ >/tmp/flyte-next-helm-repo-knative-serving.txt 2>&1 || true
+              helm repo add flyte-rustfs https://rustfs.github.io/helm >/tmp/flyte-next-helm-repo-rustfs.txt 2>&1 || true
+              helm repo update >/tmp/flyte-next-helm-repo-update.txt 2>&1
+              helm dependency build "$(dirname "$CHART")" >/tmp/flyte-next-helm-deps.txt 2>&1
+              helm template flyte-next-smoke "$(dirname "$CHART")" >/tmp/flyte-next-rendered.yaml
               grep -Eq 'kind:[[:space:]]*(Deployment|Service|ConfigMap|CustomResourceDefinition|ServiceAccount)' /tmp/flyte-next-rendered.yaml
             else
               find . -type f \( -name '*.yaml' -o -name '*.yml' \) -print0 | xargs -0 grep -Eiq 'flyte|flyteadmin|flytepropeller|CustomResourceDefinition'
@@ -446,9 +461,22 @@ jobs:
             . /tmp/flytekit-next-smoke/bin/activate
             python -m pip install --upgrade pip
             python -m pip install flytekit
-            python -c "from flytekit import task, workflow; exec('@task\ndef add_one(value: int) -> int:\n    return value + 1\n\n@workflow\ndef wf(value: int) -> int:\n    return add_one(value=value)\n'); assert wf(value=41) == 42"
-          limited_cpu_description: "Test 6 reran the scoped Flyte Arm preflight against the next stable source tag: Go package discovery, linux/arm64 compile smoke, and chart/manifest validation. A live Flyte control plane and Kubernetes execution backend remain out of scope."
-          defer_on_limited_cpu_probe_failure: "true"
+            {
+              printf '%s\n' 'from flytekit import task, workflow'
+              printf '%s\n' ''
+              printf '%s\n' '@task'
+              printf '%s\n' 'def add_one(value: int) -> int:'
+              printf '%s\n' '    return value + 1'
+              printf '%s\n' ''
+              printf '%s\n' '@workflow'
+              printf '%s\n' 'def wf(value: int) -> int:'
+              printf '%s\n' '    return add_one(value=value)'
+              printf '%s\n' ''
+              printf '%s\n' 'assert wf(value=41) == 42'
+            } >/tmp/flytekit-next-smoke.py
+            python /tmp/flytekit-next-smoke.py
+          limited_cpu_description: "Test 6 reran the scoped Flyte Arm preflight against the next stable source tag: Go package discovery, linux/arm64 compile smoke, chart/manifest validation, and a tiny local Flytekit workflow. A live Flyte control plane and Kubernetes execution backend remain out of scope."
+          defer_on_limited_cpu_probe_failure: "false"
       - name: Calculate test summary
         id: summary
         if: always()