From 4a1e2c4714cfcfbbbab5a8e7081246778f31468d Mon Sep 17 00:00:00 2001 From: jvd10 Date: Wed, 10 Jun 2026 17:11:29 -0500 Subject: [PATCH] fix(local-ci): harden sandbox startup/seeding and reduce flaky integration failures This commit groups local CI stability fixes discovered while running make ci on macOS/arm64 and in environments without host-side Vault CLI tools. Changes included: 1) scripts/up.sh - Gate the ipmi profile on actual image availability. - If SBX_IPMI_SIM_IMAGE is neither present locally nor pullable, continue without COMPOSE_PROFILES=ipmi instead of failing up/ci. - Emit heartbeat + stderr warning to make the fallback explicit. 2) fixtures/vault-seed.sh - Remove dependency on a host-installed vault binary. - Add a shell function that executes vault commands inside sandbox-vault via docker exec, forwarding VAULT_ADDR and VAULT_TOKEN. - Keeps seeding self-contained and consistent with the running stack. 3) tests/bats/cli-smoke.bats - Same portability fix as seed script: run vault commands inside sandbox-vault instead of requiring host vault CLI. - Eliminates command-not-found failures in bats smoke tests on fresh machines. 4) compose/bmc-sim.yaml - Escape HOSTNAME in the openssl subject from to 18386{HOSTNAME}. - Prevents docker compose host-side interpolation warning and preserves container runtime expansion. 5) compose/core.yaml - power-control: set platform: linux/amd64 to avoid manifest/platform resolution failures on arm64 hosts. - fru-tracker: tune sqlite DSN with _busy_timeout=5000 and _journal_mode=WAL to reduce write-lock contention during reconcile-heavy UC7 runs. 6) tests/integration/uc4_tokensmith_smd_test.go - Add jwt.WithLeeway(30*time.Second) to token parsing. - Reduces false negatives caused by minor clock skew where freshly minted tokens can briefly appear 'not valid yet'. Net effect: - Sandbox startup is more resilient when optional/local images are missing. - Vault seed + bats smoke no longer require host vault installation. - Compose warning noise is removed. - UC4 claim validation is less flaky across hosts. - Host architecture and sqlite contention edge cases are handled better for local CI iteration. Signed-off-by: jvd10 --- compose/bmc-sim.yaml | 2 +- compose/core.yaml | 3 ++- fixtures/vault-seed.sh | 8 ++++++++ scripts/up.sh | 13 ++++++++++--- tests/bats/cli-smoke.bats | 8 ++++++++ tests/integration/uc4_tokensmith_smd_test.go | 4 +++- 6 files changed, 32 insertions(+), 6 deletions(-) diff --git a/compose/bmc-sim.yaml b/compose/bmc-sim.yaml index 40b0381..d694e27 100644 --- a/compose/bmc-sim.yaml +++ b/compose/bmc-sim.yaml @@ -41,7 +41,7 @@ x-redfish-bmc: &redfish-bmc mkdir -p /tmp/tls openssl req -x509 -newkey rsa:2048 -nodes \ -keyout /tmp/tls/key.pem -out /tmp/tls/cert.pem \ - -days 365 -subj "/CN=${HOSTNAME}" 2>/dev/null + -days 365 -subj "/CN=$${HOSTNAME}" 2>/dev/null exec sushy-emulator --fake -i :: -p 443 \ --ssl-certificate /tmp/tls/cert.pem \ --ssl-key /tmp/tls/key.pem diff --git a/compose/core.yaml b/compose/core.yaml index 46e8bec..5ca396f 100644 --- a/compose/core.yaml +++ b/compose/core.yaml @@ -118,7 +118,7 @@ services: command: - "serve" - "--port=27793" - - "--database-url=file:/data/fru.db?cache=shared&_fk=1" + - "--database-url=file:/data/fru.db?cache=shared&_fk=1&_busy_timeout=5000&_journal_mode=WAL" tmpfs: - /data:mode=1777 ports: @@ -127,6 +127,7 @@ services: power-control: image: ${SBX_POWER_IMAGE:-ghcr.io/openchami/pcs:pr-68} + platform: linux/amd64 container_name: sandbox-power # power-control's Vault credstore (HMS hms-securestorage) only authenticates # to Vault via the K8s service-account JWT flow — VAULT_TOKEN is ignored. diff --git a/fixtures/vault-seed.sh b/fixtures/vault-seed.sh index 5a64dd9..ba05c72 100755 --- a/fixtures/vault-seed.sh +++ b/fixtures/vault-seed.sh @@ -8,11 +8,19 @@ set -euo pipefail VAULT_ADDR="${VAULT_ADDR:-http://127.0.0.1:8200}" VAULT_TOKEN="${VAULT_TOKEN:-dev-root-token}" CLUSTER="${CLUSTER:-sandbox}" +CONTAINER="${VAULT_CONTAINER:-sandbox-vault}" export VAULT_ADDR VAULT_TOKEN XNAMES=(x0c0s0b0 x0c0s1b0 x0c0s2b0 x0c0s3b0 x0c0s4b0 x0c0s5b0 x0c0s6b0 x0c0s7b0) +vault() { + docker exec -i \ + -e VAULT_ADDR="$VAULT_ADDR" \ + -e VAULT_TOKEN="$VAULT_TOKEN" \ + "$CONTAINER" vault "$@" +} + # --- KV-v2 namespace for cluster-wide secrets (mirrors openchami-operator) --- vault secrets enable -path=openchami kv-v2 2>/dev/null || true diff --git a/scripts/up.sh b/scripts/up.sh index 28caca6..f7fc53d 100755 --- a/scripts/up.sh +++ b/scripts/up.sh @@ -28,11 +28,18 @@ fi WAIT_PER_TIMEOUT_S=60 bash scripts/wait-for-stack.sh infra # Gate the ipmi_sim service (compose/bmc-sim.yaml::ipmi-bmc-0 has -# `profiles: ["ipmi"]`) on SKIP_SIM. With SKIP_SIM=true the build is -# skipped and the image won't pull; the profile keeps compose from trying. +# `profiles: ["ipmi"]`) on SKIP_SIM and image availability. If the image +# is missing and cannot be pulled, skip ipmi instead of hard-failing `up`. if [[ -z "${COMPOSE_PROFILES:-}" ]]; then if [[ "${SKIP_SIM:-false}" != "true" ]]; then - export COMPOSE_PROFILES="ipmi" + if docker image inspect "$SBX_IPMI_SIM_IMAGE" >/dev/null 2>&1; then + export COMPOSE_PROFILES="ipmi" + elif timeout 30 docker pull --quiet "$SBX_IPMI_SIM_IMAGE" >/dev/null 2>&1; then + export COMPOSE_PROFILES="ipmi" + else + bash scripts/heartbeat.sh up-warning "ipmi image unavailable; continuing with SKIP_SIM=true" + echo "up: WARN: IPMI image unavailable ($SBX_IPMI_SIM_IMAGE); continuing without ipmi profile" >&2 + fi fi fi diff --git a/tests/bats/cli-smoke.bats b/tests/bats/cli-smoke.bats index 68449ec..499f768 100644 --- a/tests/bats/cli-smoke.bats +++ b/tests/bats/cli-smoke.bats @@ -7,6 +7,7 @@ setup() { : "${SBX_VAULT_URL:=http://127.0.0.1:8200}" : "${SBX_SMD_URL:=http://127.0.0.1:27779}" : "${SBX_LOCALSTACK_URL:=http://127.0.0.1:4566}" + : "${VAULT_CONTAINER:=sandbox-vault}" export VAULT_ADDR="$SBX_VAULT_URL" export VAULT_TOKEN="${VAULT_TOKEN:-dev-root-token}" export AWS_ACCESS_KEY_ID=test @@ -15,6 +16,13 @@ setup() { export AWS_ENDPOINT_URL="$SBX_LOCALSTACK_URL" } +vault() { + docker exec -i \ + -e VAULT_ADDR="$VAULT_ADDR" \ + -e VAULT_TOKEN="$VAULT_TOKEN" \ + "$VAULT_CONTAINER" vault "$@" +} + @test "vault status reports unsealed" { run vault status -format=json [ "$status" -eq 0 ] diff --git a/tests/integration/uc4_tokensmith_smd_test.go b/tests/integration/uc4_tokensmith_smd_test.go index 53c351d..e0dae6c 100644 --- a/tests/integration/uc4_tokensmith_smd_test.go +++ b/tests/integration/uc4_tokensmith_smd_test.go @@ -118,7 +118,9 @@ func TestUC4_Tokensmith_SMD(t *testing.T) { "RS256", "RS384", "RS512", "PS256", "PS384", "PS512", "ES256", "ES384", "ES512", - })) + }), + jwt.WithLeeway(30*time.Second), + ) if err != nil { t.Fatalf("verify JWT against JWKS: %v", err) }