fix(e2e): unbreak LUKS cli-matrix cells — stdin to cryptsetup, diskful-only replica count (BUG-039) (#152)

* fix(e2e): forward stdin to cryptsetup in LUKS passphrase assert (BUG-039)

assert_luks_passphrase_opens piped the master passphrase into
on_node, but on_node runs kubectl exec without -i, so the pipe was
never forwarded and cryptsetup read an empty key-file ("Nothing to
read on input."). Every kernel-level passphrase assertion therefore
failed on every stand — reported as BUG-039 'LUKS data-plane broken'
— while the satellite had in fact formatted the backing device with
the correct master passphrase (verified live: the operator passphrase
opens the LUKS header once stdin is forwarded).

Add an on_node_stdin helper (kubectl exec -i, same Running-pod
selection) and route the assert through it. Keep cryptsetup stderr
and print it on the failure path — the old 2>/dev/null swallowed the
'Nothing to read on input' tell and masked the root cause.

Co-Authored-By: Claude <noreply@anthropic.com>
Signed-off-by: Andrei Kvapil <kvapss@gmail.com>

* fix(e2e): count diskful replicas only in LUKS clone/resize/snap-restore cells (BUG-039)

The three data-bearing LUKS cells waited for exactly 2 Resource CRDs
after --auto-place=2, but on a 3-worker stand the controller adds
(and flaps) a DISKLESS TIE_BREAKER witness, so the all-CRD count
oscillates 2-3-2 and the equality check times out spuriously with
'did not autoplace 2 replicas'. Count diskful replicas via
linstor_diskful_nodes instead — the convention the sibling
encryption-passphrase-luks-rd and luks-rd-create cells already use.

With the counting fixed, luks-resize-encrypted goes green on a live
stand; luks-clone-encrypted and luks-snapshot-restore-encrypted now
surface the real blocker (cross-node snapshot ship fails in the
clone/restore engine), which is tracked separately as BUG-038.

Co-Authored-By: Claude <noreply@anthropic.com>
Signed-off-by: Andrei Kvapil <kvapss@gmail.com>

---------

Signed-off-by: Andrei Kvapil <kvapss@gmail.com>
Co-authored-by: Claude <noreply@anthropic.com>

Author: kvaps

tests/e2e/cli-matrix/lib.sh

@@ -439,15 +439,24 @@ wait_luks_header_present() {
 # replica of an encrypted RD so a Bug-175-class wire-injection / Bug-
 # 233-class wrong-passphrase regression is caught at the kernel level
 # rather than just at the REST envelope. Non-zero exit on failure.
+#
+# BUG-039: MUST go through on_node_stdin — plain on_node drops stdin
+# (kubectl exec without -i), so cryptsetup read an empty key-file and
+# this assert failed on every stand while the satellite had in fact
+# formatted with the correct master passphrase. The old 2>/dev/null
+# also swallowed cryptsetup's "Nothing to read on input." tell, so we
+# now keep stderr and print it on the failure path for triage.
 assert_luks_passphrase_opens() {
     local node=$1 dev=$2 passphrase=$3
-    # NUL on stdin avoids leaking the passphrase via `ps -ef` argv and
+    # Passphrase on stdin avoids leaking it via `ps -ef` argv and
     # also avoids re-quoting headaches if the passphrase contains shell
     # metachars (the e2e default has `!!` in it, which would trigger
     # bash history expansion inside `bash -c` without the heredoc).
-    if ! printf '%s' "$passphrase" | on_node "$node" \
-            cryptsetup luksOpen --test-passphrase --key-file=- "$dev" 2>/dev/null; then
+    local err
+    if ! err=$(printf '%s' "$passphrase" | on_node_stdin "$node" \
+            cryptsetup luksOpen --test-passphrase --key-file=- "$dev" 2>&1); then
         echo "assert_luks_passphrase_opens: passphrase does NOT open ${node}:${dev}" >&2
+        echo "  cryptsetup said: $err" >&2
         return 1
     fi
     return 0

tests/e2e/cli-matrix/luks-clone-encrypted.sh

@@ -72,13 +72,14 @@ echo ">> [Bug 333] set cluster passphrase + create source encrypted RD"
 "${LCTL[@]}" volume-definition create "$RD_SRC" 64M >/dev/null
 "${LCTL[@]}" resource create --auto-place=2 --storage-pool="$POOL" "$RD_SRC" >/dev/null
 
+# BUG-039: count DISKFUL replicas only. auto-place=2 on a 3-worker
+# stand adds (and flaps) a DISKLESS TIE_BREAKER witness, so a naive
+# all-CRD count oscillates 2→3→2 and an `== 2` equality check times
+# out spuriously. Same convention as encryption-passphrase-luks-rd.
 deadline=$(( $(date +%s) + 60 ))
 placed_src=()
 while (( $(date +%s) < deadline )); do
-    mapfile -t placed_src < <(
-        kubectl get resources.blockstor.cozystack.io --no-headers 2>/dev/null \
-            | awk -v rd="$RD_SRC." '$1 ~ "^"rd {sub(rd, "", $1); print $1}'
-    )
+    mapfile -t placed_src < <(linstor_diskful_nodes "$RD_SRC")
     if (( ${#placed_src[@]} == 2 )); then break; fi
     sleep 2
 done
@@ -113,13 +114,11 @@ rm -f "$err_file"
 "${LCTL[@]}" resource create --auto-place=2 --storage-pool="$POOL" "$RD_DST" \
     >/dev/null 2>&1 || true
 
+# BUG-039: diskful-only count — see the placed_src loop above.
 deadline=$(( $(date +%s) + 120 ))
 placed_dst=()
 while (( $(date +%s) < deadline )); do
-    mapfile -t placed_dst < <(
-        kubectl get resources.blockstor.cozystack.io --no-headers 2>/dev/null \
-            | awk -v rd="$RD_DST." '$1 ~ "^"rd {sub(rd, "", $1); print $1}'
-    )
+    mapfile -t placed_dst < <(linstor_diskful_nodes "$RD_DST")
     if (( ${#placed_dst[@]} == 2 )); then break; fi
     sleep 2
 done

tests/e2e/cli-matrix/luks-resize-encrypted.sh

@@ -73,13 +73,14 @@ echo ">> [Bug 333] linstor rd c $RD -l drbd,luks,storage + vd c 64M + r c --auto
 # Resolve the placed pair so we can wait_uptodate against the actual
 # nodes the autoplacer picked rather than $WORKER_1+$WORKER_2 by
 # convention.
+# BUG-039: count DISKFUL replicas only. auto-place=2 on a 3-worker
+# stand adds (and flaps) a DISKLESS TIE_BREAKER witness, so a naive
+# all-CRD count oscillates 2→3→2 and an `== 2` equality check times
+# out spuriously. Same convention as encryption-passphrase-luks-rd.
 deadline=$(( $(date +%s) + 60 ))
 placed_nodes=()
 while (( $(date +%s) < deadline )); do
-    mapfile -t placed_nodes < <(
-        kubectl get resources.blockstor.cozystack.io --no-headers 2>/dev/null \
-            | awk -v rd="$RD." '$1 ~ "^"rd {sub(rd, "", $1); print $1}'
-    )
+    mapfile -t placed_nodes < <(linstor_diskful_nodes "$RD")
     if (( ${#placed_nodes[@]} == 2 )); then break; fi
     sleep 2
 done

tests/e2e/cli-matrix/luks-snapshot-restore-encrypted.sh

@@ -80,13 +80,14 @@ echo ">> [Bug 333] create source encrypted RD"
 "${LCTL[@]}" resource create --auto-place=2 --storage-pool="$POOL" "$RD_SRC" >/dev/null
 
 # Resolve placed nodes.
+# BUG-039: count DISKFUL replicas only. auto-place=2 on a 3-worker
+# stand adds (and flaps) a DISKLESS TIE_BREAKER witness, so a naive
+# all-CRD count oscillates 2→3→2 and an `== 2` equality check times
+# out spuriously. Same convention as encryption-passphrase-luks-rd.
 deadline=$(( $(date +%s) + 60 ))
 placed_src=()
 while (( $(date +%s) < deadline )); do
-    mapfile -t placed_src < <(
-        kubectl get resources.blockstor.cozystack.io --no-headers 2>/dev/null \
-            | awk -v rd="$RD_SRC." '$1 ~ "^"rd {sub(rd, "", $1); print $1}'
-    )
+    mapfile -t placed_src < <(linstor_diskful_nodes "$RD_SRC")
     if (( ${#placed_src[@]} == 2 )); then break; fi
     sleep 2
 done
@@ -139,13 +140,11 @@ echo ">> [Bug 333] linstor r c $RD_DST --auto-place=2"
 # driven snapshot-restore-cross-node.sh.
 "${LCTL[@]}" resource create --auto-place=2 --storage-pool="$POOL" "$RD_DST" >/dev/null
 
+# BUG-039: diskful-only count — see the placed_src loop above.
 deadline=$(( $(date +%s) + 60 ))
 placed_dst=()
 while (( $(date +%s) < deadline )); do
-    mapfile -t placed_dst < <(
-        kubectl get resources.blockstor.cozystack.io --no-headers 2>/dev/null \
-            | awk -v rd="$RD_DST." '$1 ~ "^"rd {sub(rd, "", $1); print $1}'
-    )
+    mapfile -t placed_dst < <(linstor_diskful_nodes "$RD_DST")
     if (( ${#placed_dst[@]} == 2 )); then break; fi
     sleep 2
 done

tests/e2e/lib.sh

@@ -109,6 +109,35 @@ on_node() {
     kubectl -n "$NS" exec "$pod" -- "$@"
 }
 
+# on_node_stdin runs CMD inside the satellite pod scheduled on NODE
+# with the caller's stdin forwarded into the container (`kubectl
+# exec -i`). Use this — never plain on_node — whenever the remote
+# command reads stdin (e.g. `cryptsetup --key-file=-`).
+#
+# BUG-039 root cause: on_node execs WITHOUT `-i`, so kubectl never
+# forwards the pipe and the remote command sees EOF on fd 0.
+# `printf pass | on_node ... cryptsetup luksOpen --key-file=-` fed
+# cryptsetup an EMPTY key ("Nothing to read on input"), making every
+# kernel-level passphrase assertion fail on every stand regardless of
+# what key the satellite actually formatted with. Kept separate from
+# on_node because an unconditional `-i` would let no-stdin callers
+# (poll loops, `while read` bodies) steal the calling script's stdin.
+on_node_stdin() {
+    local node=$1
+    shift
+    local pod
+    pod=$(kubectl -n "$NS" get pods -l app=blockstor-satellite \
+        --field-selector "spec.nodeName=${node},status.phase=Running" \
+        -o "jsonpath={.items[0].metadata.name}")
+
+    if [[ -z "$pod" ]]; then
+        echo "no Running satellite pod on node $node" >&2
+        return 1
+    fi
+
+    kubectl -n "$NS" exec -i "$pod" -- "$@"
+}
+
 # ---- k8s-native readers (preferred over drbdsetup-status grep) ----
 #
 # Replaces `kubectl exec satellite -- drbdsetup status ... | grep ...`