fix(local-ci): harden sandbox startup/seeding and reduce flaky integration failures

This commit groups local CI stability fixes discovered while running make ci on macOS/arm64 and in environments without host-side Vault CLI tools.

Changes included:

1) scripts/up.sh
- Gate the ipmi profile on actual image availability.
- If SBX_IPMI_SIM_IMAGE is neither present locally nor pullable, continue without COMPOSE_PROFILES=ipmi instead of failing up/ci.
- Emit heartbeat + stderr warning to make the fallback explicit.

2) fixtures/vault-seed.sh
- Remove dependency on a host-installed vault binary.
- Add a shell function that executes vault commands inside sandbox-vault via docker exec, forwarding VAULT_ADDR and VAULT_TOKEN.
- Keeps seeding self-contained and consistent with the running stack.

3) tests/bats/cli-smoke.bats
- Same portability fix as seed script: run vault commands inside sandbox-vault instead of requiring host vault CLI.
- Eliminates command-not-found failures in bats smoke tests on fresh machines.

4) compose/bmc-sim.yaml
- Escape HOSTNAME in the openssl subject from  to 18386{HOSTNAME}.
- Prevents docker compose host-side interpolation warning and preserves container runtime expansion.

5) compose/core.yaml
- power-control: set platform: linux/amd64 to avoid manifest/platform resolution failures on arm64 hosts.
- fru-tracker: tune sqlite DSN with _busy_timeout=5000 and _journal_mode=WAL to reduce write-lock contention during reconcile-heavy UC7 runs.

6) tests/integration/uc4_tokensmith_smd_test.go
- Add jwt.WithLeeway(30*time.Second) to token parsing.
- Reduces false negatives caused by minor clock skew where freshly minted tokens can briefly appear 'not valid yet'.

Net effect:
- Sandbox startup is more resilient when optional/local images are missing.
- Vault seed + bats smoke no longer require host vault installation.
- Compose warning noise is removed.
- UC4 claim validation is less flaky across hosts.
- Host architecture and sqlite contention edge cases are handled better for local CI iteration.

Signed-off-by: jvd10 <jay.depasse@hpe.com>

Author: jvd10

compose/bmc-sim.yaml

@@ -41,7 +41,7 @@ x-redfish-bmc: &redfish-bmc
       mkdir -p /tmp/tls
       openssl req -x509 -newkey rsa:2048 -nodes \
         -keyout /tmp/tls/key.pem -out /tmp/tls/cert.pem \
-        -days 365 -subj "/CN=${HOSTNAME}" 2>/dev/null
+        -days 365 -subj "/CN=$${HOSTNAME}" 2>/dev/null
       exec sushy-emulator --fake -i :: -p 443 \
         --ssl-certificate /tmp/tls/cert.pem \
         --ssl-key /tmp/tls/key.pem

compose/core.yaml

@@ -118,7 +118,7 @@ services:
     command:
       - "serve"
       - "--port=27793"
-      - "--database-url=file:/data/fru.db?cache=shared&_fk=1"
+      - "--database-url=file:/data/fru.db?cache=shared&_fk=1&_busy_timeout=5000&_journal_mode=WAL"
     tmpfs:
       - /data:mode=1777
     ports:
@@ -127,6 +127,7 @@ services:
 
   power-control:
     image: ${SBX_POWER_IMAGE:-ghcr.io/openchami/pcs:pr-68}
+    platform: linux/amd64
     container_name: sandbox-power
     # power-control's Vault credstore (HMS hms-securestorage) only authenticates
     # to Vault via the K8s service-account JWT flow — VAULT_TOKEN is ignored.

fixtures/vault-seed.sh

@@ -8,11 +8,19 @@ set -euo pipefail
 VAULT_ADDR="${VAULT_ADDR:-http://127.0.0.1:8200}"
 VAULT_TOKEN="${VAULT_TOKEN:-dev-root-token}"
 CLUSTER="${CLUSTER:-sandbox}"
+CONTAINER="${VAULT_CONTAINER:-sandbox-vault}"
 
 export VAULT_ADDR VAULT_TOKEN
 
 XNAMES=(x0c0s0b0 x0c0s1b0 x0c0s2b0 x0c0s3b0 x0c0s4b0 x0c0s5b0 x0c0s6b0 x0c0s7b0)
 
+vault() {
+  docker exec -i \
+    -e VAULT_ADDR="$VAULT_ADDR" \
+    -e VAULT_TOKEN="$VAULT_TOKEN" \
+    "$CONTAINER" vault "$@"
+}
+
 # --- KV-v2 namespace for cluster-wide secrets (mirrors openchami-operator) ---
 vault secrets enable -path=openchami kv-v2 2>/dev/null || true
 

scripts/up.sh

@@ -28,11 +28,18 @@ fi
 WAIT_PER_TIMEOUT_S=60 bash scripts/wait-for-stack.sh infra
 
 # Gate the ipmi_sim service (compose/bmc-sim.yaml::ipmi-bmc-0 has
-# `profiles: ["ipmi"]`) on SKIP_SIM. With SKIP_SIM=true the build is
-# skipped and the image won't pull; the profile keeps compose from trying.
+# `profiles: ["ipmi"]`) on SKIP_SIM and image availability. If the image
+# is missing and cannot be pulled, skip ipmi instead of hard-failing `up`.
 if [[ -z "${COMPOSE_PROFILES:-}" ]]; then
   if [[ "${SKIP_SIM:-false}" != "true" ]]; then
-    export COMPOSE_PROFILES="ipmi"
+    if docker image inspect "$SBX_IPMI_SIM_IMAGE" >/dev/null 2>&1; then
+      export COMPOSE_PROFILES="ipmi"
+    elif timeout 30 docker pull --quiet "$SBX_IPMI_SIM_IMAGE" >/dev/null 2>&1; then
+      export COMPOSE_PROFILES="ipmi"
+    else
+      bash scripts/heartbeat.sh up-warning "ipmi image unavailable; continuing with SKIP_SIM=true"
+      echo "up: WARN: IPMI image unavailable ($SBX_IPMI_SIM_IMAGE); continuing without ipmi profile" >&2
+    fi
   fi
 fi
 

tests/bats/cli-smoke.bats

@@ -7,6 +7,7 @@ setup() {
   : "${SBX_VAULT_URL:=http://127.0.0.1:8200}"
   : "${SBX_SMD_URL:=http://127.0.0.1:27779}"
   : "${SBX_LOCALSTACK_URL:=http://127.0.0.1:4566}"
+  : "${VAULT_CONTAINER:=sandbox-vault}"
   export VAULT_ADDR="$SBX_VAULT_URL"
   export VAULT_TOKEN="${VAULT_TOKEN:-dev-root-token}"
   export AWS_ACCESS_KEY_ID=test
@@ -15,6 +16,13 @@ setup() {
   export AWS_ENDPOINT_URL="$SBX_LOCALSTACK_URL"
 }
 
+vault() {
+  docker exec -i \
+    -e VAULT_ADDR="$VAULT_ADDR" \
+    -e VAULT_TOKEN="$VAULT_TOKEN" \
+    "$VAULT_CONTAINER" vault "$@"
+}
+
 @test "vault status reports unsealed" {
   run vault status -format=json
   [ "$status" -eq 0 ]

tests/integration/uc4_tokensmith_smd_test.go

@@ -118,7 +118,9 @@ func TestUC4_Tokensmith_SMD(t *testing.T) {
 			"RS256", "RS384", "RS512",
 			"PS256", "PS384", "PS512",
 			"ES256", "ES384", "ES512",
-		}))
+		}),
+		jwt.WithLeeway(30*time.Second),
+	)
 	if err != nil {
 		t.Fatalf("verify JWT against JWKS: %v", err)
 	}