Generalize the pipefail false-negatives fix from #12919 (#12921)

Move output_contains from s3client_test.sh to tests_common.sh and add
output_matches, output_matches_i, output_matches_E variants for regex
and case-insensitive matching. All use here-strings (<<<) instead of
echo|grep pipes, avoiding false negatives from SIGPIPE when pipefail
is enabled.

Convert 26 of 28 echo|grep -q instances across test scripts. Two
chained grep patterns (grep -i | grep -qvi) are left unchanged.

Also make backup_tests_common.sh source tests_common.sh so the helpers
are available in backup test utilities.

Author: saintstack

fdbbackup/tests/backup_tests_common.sh

@@ -22,6 +22,12 @@
 # Shared between s3_backup_test.sh, s3_backup_bulkdump_bulkload.sh, dir_backup_test.sh, etc.
 # These functions work with both S3/blobstore and file-based backup testing
 
+# Source shared test utilities (output_contains, output_matches, etc.)
+_BACKUP_COMMON_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+if [[ -f "${_BACKUP_COMMON_DIR}/../../fdbclient/tests/tests_common.sh" ]]; then
+  source "${_BACKUP_COMMON_DIR}/../../fdbclient/tests/tests_common.sh"
+fi
+
 # Helper function to add base arguments (cluster file and logging)
 # Uses bash nameref (requires bash 4.3+) to modify the array in place
 # $1 name of the array variable to modify (passed by name, not value)
@@ -182,12 +188,12 @@ function run_backup {
     set -e
 
     # Check if backup is restorable (differential state) or completed
-    if echo "${status_output}" | grep -q "is restorable"; then
+    if output_contains "${status_output}" "is restorable"; then
       log "Backup is now restorable after ${elapsed}s"
       break
     fi
 
-    if echo "${status_output}" | grep -q "completed"; then
+    if output_contains "${status_output}" "completed"; then
       log "Backup completed after ${elapsed}s"
       break
     fi
@@ -196,17 +202,17 @@ function run_backup {
     if [[ $((elapsed % 30)) -eq 0 ]]; then
       log "Still waiting for backup to become restorable (${elapsed}s elapsed)..."
       # Show current state for debugging
-      if echo "${status_output}" | grep -q "is restorable"; then
+      if output_contains "${status_output}" "is restorable"; then
         log "  Status: backup is restorable (should have exited loop)"
-      elif echo "${status_output}" | grep -q "in progress to"; then
+      elif output_contains "${status_output}" "in progress to"; then
         log "  Status: backup running, waiting for snapshot to complete"
-      elif echo "${status_output}" | grep -q "just started"; then
+      elif output_contains "${status_output}" "just started"; then
         log "  Status: backup submitted, tasks starting up"
       fi
       # Check snapshot mode for debugging
-      if echo "${status_output}" | grep -q "Snapshot Mode: bulkdump"; then
+      if output_contains "${status_output}" "Snapshot Mode: bulkdump"; then
         log "  Snapshot Mode: bulkdump (using BulkDump for snapshots)"
-      elif echo "${status_output}" | grep -q "Snapshot Mode: both"; then
+      elif output_contains "${status_output}" "Snapshot Mode: both"; then
         log "  Snapshot Mode: both (generating both formats)"
       fi
     fi
@@ -238,7 +244,7 @@ function run_backup {
       set -e
 
       # Check if BulkDump snapshot exists (look for bulkDumpJobId in describe output)
-      if echo "${snapshot_list}" | grep -q "bulkDumpJobId\|,bulk"; then
+      if output_matches "${snapshot_list}" "bulkDumpJobId\|,bulk"; then
         log "BulkDump snapshot found after ${bulkdump_elapsed}s"
         break
       fi
@@ -255,7 +261,7 @@ function run_backup {
   fi
 
   # Check if backup already completed (no need to discontinue)
-  if echo "${status_output}" | grep -q "completed"; then
+  if output_contains "${status_output}" "completed"; then
     log "Backup already completed - no need to discontinue"
     return 0
   fi
@@ -268,7 +274,7 @@ function run_backup {
   set -e
 
   if [[ $stop_exit_code -ne 0 ]]; then
-    if echo "${stop_output}" | grep -q "already discontinued\|not running\|unneeded"; then
+    if output_matches "${stop_output}" "already discontinued\|not running\|unneeded"; then
       log "Backup already completed and finalized - this is success!"
     else
       err "Failed to stop backup: ${stop_output}"
@@ -334,13 +340,13 @@ function run_restore {
     # Check if restore completed
     # Status output contains "State: completed" or "Phase: Complete" when done
     # Also check "No restore" for when restore tag doesn't exist (completed and cleaned up)
-    if echo "${status_output}" | grep -qi "State:.*completed\|Phase:.*Complete\|No restore"; then
+    if output_matches_i "${status_output}" "State:.*completed\|Phase:.*Complete\|No restore"; then
       log "Restore completed after ${elapsed}s"
       return 0
     fi
 
     # Check if restore failed (be specific - "LastError: None" contains "Error" so avoid false positives)
-    if echo "${status_output}" | grep -qi "State:.*aborted"; then
+    if output_matches_i "${status_output}" "State:.*aborted"; then
       err "Restore aborted after ${elapsed}s"
       log "Status output:"
       echo "${status_output}"
@@ -359,7 +365,7 @@ function run_restore {
     if [[ $((elapsed % 30)) -eq 0 ]]; then
       log "Still waiting for restore to complete (${elapsed}s elapsed)..."
       # Show phase info for debugging
-      if echo "${status_output}" | grep -qi "Phase:"; then
+      if output_matches_i "${status_output}" "Phase:"; then
         phase_info=$(echo "${status_output}" | grep -i "Phase:" | head -1)
         log "  ${phase_info}"
       fi
@@ -496,11 +502,11 @@ function run_restore_wait {
     status_output=$("${local_build_dir}"/bin/fdbrestore status -t "${local_tag}" --dest-cluster-file "${local_scratch_dir}/loopback_cluster/fdb.cluster" --log --logdir="${local_scratch_dir}" 2>&1)
     set -e
 
-    if echo "${status_output}" | grep -qi "State:.*completed\|Phase:.*Complete\|No restore"; then
+    if output_matches_i "${status_output}" "State:.*completed\|Phase:.*Complete\|No restore"; then
       return 0
     fi
 
-    if echo "${status_output}" | grep -qi "State:.*aborted"; then
+    if output_matches_i "${status_output}" "State:.*aborted"; then
       return 1
     fi
 

fdbbackup/tests/s3_backup_bulkdump_bulkload.sh

@@ -164,7 +164,7 @@ function run_validate_restore_audit {
       break
     fi
 
-    if echo "${audit_output}" | grep -qE "1221|1230|1010"; then
+    if output_matches_E "${audit_output}" "1221|1230|1010"; then
       log "Transient error detected, retrying in ${retry_delay}s..."
       sleep $retry_delay
       continue
@@ -193,12 +193,12 @@ function run_validate_restore_audit {
     local status_output
     status_output=$("${fdbcli}" -C "${cluster_file}" --exec "get_audit_status validate_restore id ${audit_id}" 2>&1)
 
-    if echo "${status_output}" | grep -q "Phase.*2"; then
+    if output_matches "${status_output}" "Phase.*2"; then
       log "Audit completed successfully after ${elapsed}s"
       return 0
     fi
 
-    if echo "${status_output}" | grep -q "Phase.*[34]"; then
+    if output_matches "${status_output}" "Phase.*[34]"; then
       err "Audit failed with status: ${status_output}"
       return 1
     fi

fdbclient/tests/s3client_test.sh

@@ -35,13 +35,6 @@ function filter_http_debug {
   echo "${output}" | grep -v "Contents of" | grep -v "^$" | grep -v "^[[:space:]]*$" | grep -v "HTTP" | grep -v "^\[.*\]" | grep -v "^Request Header:" | grep -v "^Response Header:" | grep -v "^Response Code:" | grep -v "^Response ContentLen:" | grep -v "^-- RESPONSE CONTENT--" | grep -v "^--------" | grep -v "^<?xml" | grep -v "^<.*>" | grep -v "^'$" | grep -v "^++"
 }
 
-# Check whether captured output contains a literal string. Avoid echo|grep -q because
-# pipefail can turn grep's early success exit into a false negative via SIGPIPE upstream.
-function output_contains {
-  local output="$1"
-  local needle="$2"
-  grep -Fq -- "${needle}" <<<"${output}"
-}
 
 # Run s3client with proper TLS CA file handling
 # This function handles the case where TLS_CA_FILE might be empty
@@ -258,7 +251,7 @@ function test_nonexistent_bucket {
       # 1. The command succeeded (status 0)
       # 2. The output contains the URL header
       # 3. There are no objects listed (no lines after the header)
-      if echo "${output}" | grep -q "Contents of" &&
+      if output_contains "${output}" "Contents of" &&
          [[ $(filter_http_debug "${output}" | wc -l) -eq 0 ]]; then
         # Success - we have the header and no other non-empty lines (excluding HTTP debug lines)
         return 0
@@ -301,7 +294,7 @@ function test_nonexistent_resource {
     local filtered_output
     filtered_output=$(filter_http_debug "${output}")
 
-    if ! (echo "${output}" | grep -q "Contents of" &&
+    if ! (output_contains "${output}" "Contents of" &&
           [[ -z "$(echo "${filtered_output}" | tr -d '[:space:]')" ]]); then
       err "Failed to detect non-existent resource in S3"
       return 1
@@ -315,8 +308,8 @@ function test_nonexistent_resource {
       # 1. The command succeeded (status 0)
       # 2. The output contains the URL header
       # 3. There are no actual object listings (no lines starting with FILE or DIR)
-      if echo "${output}" | grep -q "Contents of" && 
-         ! echo "${output}" | grep -q "^FILE\|^DIR"; then
+      if output_contains "${output}" "Contents of" && 
+         ! output_matches "${output}" "^FILE\|^DIR"; then
         # Success - we have the header and no object listings
         return 0
       else
@@ -375,8 +368,8 @@ function test_empty_bucket {
     local filtered_output
     filtered_output=$(filter_http_debug "${output}")
 
-    if echo "${output}" | grep -q "No objects found" || 
-       (echo "${output}" | grep -q "Contents of" && 
+    if output_contains "${output}" "No objects found" ||
+       (output_contains "${output}" "Contents of" && 
         [[ -z "$(echo "${filtered_output}" | tr -d '[:space:]')" ]]); then
       log "Successfully handled empty bucket listing"
       return 0

fdbclient/tests/tests_common.sh

@@ -96,6 +96,44 @@ function err {
   echo "$(date -Iseconds) ERROR: ${*}" >&2
 }
 
+# Check whether captured output contains a literal string.
+# Avoids echo|grep -q because pipefail can turn grep's early-exit into
+# a false negative via SIGPIPE on the upstream echo.
+# $1 output to search
+# $2 literal string to find
+function output_contains {
+  local output="$1"
+  local needle="$2"
+  grep -Fq -- "${needle}" <<<"${output}"
+}
+
+# Check whether captured output matches a basic regex.
+# $1 output to search
+# $2 regex pattern
+function output_matches {
+  local output="$1"
+  local pattern="$2"
+  grep -q -- "${pattern}" <<<"${output}"
+}
+
+# Case-insensitive regex match.
+# $1 output to search
+# $2 regex pattern
+function output_matches_i {
+  local output="$1"
+  local pattern="$2"
+  grep -qi -- "${pattern}" <<<"${output}"
+}
+
+# Extended regex match.
+# $1 output to search
+# $2 extended regex pattern
+function output_matches_E {
+  local output="$1"
+  local pattern="$2"
+  grep -qE -- "${pattern}" <<<"${output}"
+}
+
 # Check if test data should be preserved (PRESERVE_TEST_DATA=1)
 # If yes, prints preservation message and returns 0 (should skip cleanup)
 # If no, returns 1 (should continue with normal cleanup)