Fix sshd test flakiness and orphan daemon

- term size test: poll tmux output with retries instead of fixed sleeps,
  re-sending the size query each pass in case the shell was not yet ready
- term close test: scope netstat checks to the test port so unrelated
  host CLOSE_WAIT/TIME_WAIT sockets don't fail the test
- run_all_sshd_tests.sh: validate --match before starting wolfSSHd
- renewcerts.sh: build the per-user cert from a throwaway config copy so
  the tracked renewcerts.cnf is never modified in place

Author: ejohnstown

.gitignore

@@ -70,6 +70,16 @@ apps/wolfsshd/wolfsshd
 apps/wolfsshd/test/test_configuration
 apps/wolfsshd/test/log.txt
 apps/wolfsshd/test/sshd_config_*
+apps/wolfsshd/test/authorized_keys_test
+apps/wolfsshd/test/stdout.txt
+
+# Test-run generated certs/keys and scratch data
+keys/root-*
+keys/*.csr
+keys/renewcerts-*.cnf
+random-test.txt
+random-test-result.txt
+test.dat
 
 # test output
 tests/*.test

apps/wolfsshd/test/run_all_sshd_tests.sh

@@ -61,6 +61,26 @@ done
 TOTAL=0
 SKIPPED=0
 
+# validate the requested test before any setup so a bad name does not leave
+# a wolfSSHd running
+if [[ -n "$MATCH" ]]; then
+    MATCH_FOUND=0
+    for test in "${test_cases[@]}"; do
+        if [[ "$test" == "$MATCH" ]]; then
+            MATCH_FOUND=1
+            break
+        fi
+    done
+    if [[ "$MATCH_FOUND" -eq 0 ]]; then
+        echo "Error: Test '$MATCH' not found."
+        echo "All test cases:"
+        for test in "${test_cases[@]}"; do
+            echo "    $test"
+        done
+        exit 1
+    fi
+fi
+
 # setup
 set -e
 ./create_authorized_test_file.sh
@@ -106,13 +126,8 @@ run_test() {
 
 # Run the tests
 if [[ -n "$MATCH" ]]; then
-    if [[ " ${test_cases[*]} " =~ " $MATCH " ]]; then
-        echo "Running test: $MATCH"
-        run_test "$MATCH"
-    else
-        echo "Error: Test '$MATCH' not found."
-        exit 1
-    fi
+    echo "Running test: $MATCH"
+    run_test "$MATCH"
 
     if [ "$USING_LOCAL_HOST" == 1 ]; then
         printf "Shutting down test wolfSSHd\n"

apps/wolfsshd/test/sshd_term_close_test.sh

@@ -28,7 +28,9 @@ if [ "$WOLFSSHD_PID_COUNT" = "$WOLFSSHD_PID_COUNT_AFTER" ]; then
     exit 1
 fi
 
-netstat -nt | grep ESTABLISHED
+# Only consider sockets for the test port so unrelated host traffic
+# (other CLOSE_WAIT/TIME_WAIT connections) does not skew the result.
+netstat -nt | grep ":$2 " | grep ESTABLISHED
 RESULT=$?
 if [ "$RESULT" != "0" ]; then
     echo "Expecting to find the TCP connection established"
@@ -37,14 +39,14 @@ fi
 
 sleep 2
 
-netstat -nt | grep CLOSE_WAIT
+netstat -nt | grep ":$2 " | grep CLOSE_WAIT
 RESULT=$?
 if [ "$RESULT" = "0" ]; then
     echo "Found close wait and was not expecting it"
     exit 1
 fi
 
-netstat -nt | grep TIME_WAIT
+netstat -nt | grep ":$2 " | grep TIME_WAIT
 RESULT=$?
 if [ "$RESULT" != "0" ]; then
     echo "Did not find timed wait for TCP close down"

apps/wolfsshd/test/sshd_term_size_test.sh

@@ -23,34 +23,63 @@ if [ ${RESULT} = 1 ]; then
     exit 1
 fi
 
+# tear down the tmux session on any exit, so a timeout failure does not
+# leave a stale session that breaks the next run with "duplicate session"
+trap 'tmux kill-session -t test 2>/dev/null || true' EXIT
+
+# Wait until the remote shell produces some output (i.e. a prompt), so the
+# SSH session is known to be up before keys are sent to it. CI runners can
+# take several seconds to get through key exchange and login.
+wait_for_session() {
+    for _ in $(seq 1 10); do
+        if tmux capture-pane -p -t test | grep -q '[^[:space:]]'; then
+            return 0
+        fi
+        sleep 1
+    done
+    echo "Timed out waiting for SSH session output"
+    tmux capture-pane -p -t test
+    return 1
+}
+
+# Ask the remote shell for its size and poll the pane until a line of the
+# form "<columns> <rows>" shows up. The result is left in SIZE_LINE.
+get_size_line() {
+    SIZE_LINE=""
+    for _ in $(seq 1 10); do
+        # Re-send the query each pass in case the shell was not yet ready to
+        # read input on an earlier pass; tail -n 1 below tolerates the extra
+        # numeric line a repeat can produce.
+        tmux send-keys -t test 'echo;echo $COLUMNS $LINES;echo'
+        tmux send-keys -t test 'ENTER'
+        sleep 1
+        SIZE_LINE=$(tmux capture-pane -p -t test | tr -d '\r' | \
+            grep -E '^[0-9]+[[:space:]]+[0-9]+[[:space:]]*$' | tail -n 1)
+        if [ -n "$SIZE_LINE" ]; then
+            return 0
+        fi
+    done
+    echo "Timed out waiting for terminal size output"
+    tmux capture-pane -p -t test
+    return 1
+}
+
 echo "Creating tmux session at $PWD with command :"
 echo "tmux new-session -d -s test \"$TEST_CLIENT -q -t -u $USER -i $PRIVATE_KEY -j $PUBLIC_KEY -h \"$1\" -p \"$2\"\""
 tmux new-session -d -s test "$TEST_CLIENT -q -t -u $USER -i $PRIVATE_KEY -j $PUBLIC_KEY -h \"$1\" -p \"$2\""
 echo "Result of tmux new-session = $?"
 
-# give the command a second to establish SSH connection
-sleep 1
+wait_for_session || exit 1
 
 COL=`tmux display -p -t test '#{pane_width}'`
 ROW=`tmux display -p -t test '#{pane_height}'`
 echo "tmux 'test' session has COL = ${COL} and ROW = ${ROW}"
 
 # get the terminals columns and lines
-tmux send-keys -t test 'echo;echo $COLUMNS $LINES;echo'
-tmux send-keys -t test 'ENTER'
-
-# give the command a second to run
-sleep 1
-
-tmux capture-pane -t test
-RESULT=$(tmux show-buffer | grep '^[0-9]* [0-9]*$')
-tmux show-buffer
+get_size_line || exit 1
+echo "Captured terminal size line: '$SIZE_LINE'"
 
-echo "$RESULT"
-echo ""
-echo ""
-ROW_FOUND=$(echo "$RESULT" | sed -e 's/[0-9]* \([0-9]*\)/\1/')
-COL_FOUND=$(echo "$RESULT" | sed -e 's/\([0-9]*\) [0-9]*/\1/')
+read -r COL_FOUND ROW_FOUND <<< "$SIZE_LINE"
 
 if [ "$COL" != "$COL_FOUND" ]; then
     echo "Col found was $COL_FOUND which does not match expected $COL"
@@ -80,22 +109,13 @@ echo "Starting another session with a smaller window size"
 echo "tmux new-session -d -x 50 -y 10 -s test \"$TEST_CLIENT -q -t -u $USER -i $PRIVATE_KEY -j $PUBLIC_KEY -h \"$1\" -p \"$2\"\""
 tmux new-session -d -x 50 -y 10 -s test "$TEST_CLIENT -q -t -u $USER -i $PRIVATE_KEY -j $PUBLIC_KEY -h \"$1\" -p \"$2\""
 
-# give the command a second to establish SSH connection
-sleep 1
+wait_for_session || exit 1
 
 echo "Sending keys to tmux session for displaying column/rows"
-tmux send-keys -t test 'echo;echo $COLUMNS $LINES;echo'
-tmux send-keys -t test 'ENTER'
-tmux capture-pane -t test
-RESULT=$(tmux show-buffer | grep '^[0-9]* [0-9]*$')
-
-ROW_FOUND=$( echo "$RESULT" | sed -e 's/[0-9]* \([0-9]*\)/\1/' )
-COL_FOUND=$( echo "$RESULT" | sed -e 's/\([0-9]*\) [0-9]*/\1/' )
-
-#remove any newlines, tabs, or returns
-ROW_FOUND=$( tr -d '\n\t\r ' <<<"$ROW_FOUND" )
-COL_FOUND=$( tr -d '\n\t\r ' <<<"$COL_FOUND" )
+get_size_line || exit 1
+echo "Captured terminal size line: '$SIZE_LINE'"
 
+read -r COL_FOUND ROW_FOUND <<< "$SIZE_LINE"
 
 if [ "50" != "$COL_FOUND" ]; then
     echo "Col found was $COL_FOUND which does not match expected 50"

keys/renewcerts.sh

@@ -1,28 +1,40 @@
 touch index.txt
 
+# The tracked renewcerts.cnf uses "fred" as the baseline user. When a
+# different user is requested we work from a throwaway copy with the name
+# substituted in, so the tracked config is never modified in place.
+CONFIG="renewcerts.cnf"
+
 if [ -z "$1" ]; then
     USER_NAME="fred"
 else
-    USER_NAME=$1
-    cp fred-key.der $USER_NAME-key.der
-    cp fred-key.pem $USER_NAME-key.pem
-    sed -i.bak "s/fred/$USER_NAME/g" renewcerts.cnf
+    USER_NAME="$1"
+    # Escape characters that are special in a sed replacement (\, /, &) so a
+    # user name containing them substitutes literally instead of breaking sed.
+    USER_NAME_SED=$(printf '%s' "$USER_NAME" | sed -e 's/[\/&]/\\&/g')
+    cp fred-key.der "$USER_NAME-key.der"
+    cp fred-key.pem "$USER_NAME-key.pem"
+    CONFIG="renewcerts-$USER_NAME.cnf"
+    sed "s/fred/$USER_NAME_SED/g" renewcerts.cnf > "$CONFIG"
 fi
 
 # renew CA
-openssl req -subj '/C=US/ST=Washington/L=Seattle/O=wolfSSL/OU=Development/CN=www.wolfssl.com/emailAddress=ca@example.com' -key ca-key-ecc.pem -text -out ca-cert-ecc.pem -config renewcerts.cnf -new -nodes -x509 -extensions v3_ca -days 3650 -set_serial 6
+openssl req -subj '/C=US/ST=Washington/L=Seattle/O=wolfSSL/OU=Development/CN=www.wolfssl.com/emailAddress=ca@example.com' -key ca-key-ecc.pem -text -out ca-cert-ecc.pem -config "$CONFIG" -new -nodes -x509 -extensions v3_ca -days 3650 -set_serial 6
 openssl x509 -in ca-cert-ecc.pem -outform DER -out ca-cert-ecc.der
 
 # renew user cert
-openssl req -subj "/C=US/ST=WA/L=Seattle/O=wolfSSL Inc/OU=Development/CN=$USER_NAME/emailAddress=fred@example.com" -key $USER_NAME-key.pem -out $USER_NAME-cert.csr -config renewcerts.cnf -new -nodes
+openssl req -subj "/C=US/ST=WA/L=Seattle/O=wolfSSL Inc/OU=Development/CN=$USER_NAME/emailAddress=$USER_NAME@example.com" -key "$USER_NAME-key.pem" -out "$USER_NAME-cert.csr" -config "$CONFIG" -new -nodes
 
-openssl x509 -req -in $USER_NAME-cert.csr -days 3650 -extfile renewcerts.cnf -extensions v3_$USER_NAME -CA ca-cert-ecc.pem -CAkey ca-key-ecc.pem -text -out $USER_NAME-cert.pem -set_serial 7
-openssl x509 -in $USER_NAME-cert.pem -outform DER -out $USER_NAME-cert.der
+openssl x509 -req -in "$USER_NAME-cert.csr" -days 3650 -extfile "$CONFIG" -extensions "v3_$USER_NAME" -CA ca-cert-ecc.pem -CAkey ca-key-ecc.pem -text -out "$USER_NAME-cert.pem" -set_serial 7
+openssl x509 -in "$USER_NAME-cert.pem" -outform DER -out "$USER_NAME-cert.der"
 
 # renew server-cert
-openssl req -subj '/C=US/ST=Washington/L=Seattle/O=Eliptic/OU=ECC/CN=www.wolfssl.com/emailAddress=server@example.com' -key server-key.pem -out server-cert.csr -config renewcerts.cnf -new -nodes
+openssl req -subj '/C=US/ST=Washington/L=Seattle/O=Eliptic/OU=ECC/CN=www.wolfssl.com/emailAddress=server@example.com' -key server-key.pem -out server-cert.csr -config "$CONFIG" -new -nodes
 
-openssl x509 -req -in server-cert.csr -days 3650 -extfile renewcerts.cnf -extensions v3_server -CA ca-cert-ecc.pem -CAkey ca-key-ecc.pem -text -out server-cert.pem -set_serial 8
+openssl x509 -req -in server-cert.csr -days 3650 -extfile "$CONFIG" -extensions v3_server -CA ca-cert-ecc.pem -CAkey ca-key-ecc.pem -text -out server-cert.pem -set_serial 8
 openssl x509 -in server-cert.pem -outform DER -out server-cert.der
 
 rm index.*
+if [ -n "$1" ]; then
+    rm -f "$CONFIG"
+fi