@@ -55,17 +55,25 @@ if [ "${COPY_OFFLOAD}" != "" ]; then
5555 copy_out_method=" offload"
5656fi
5757
58+ # Default to using the lib-copy-offload-tester binary for copy offload tests
59+ if [[ -z " ${COPY_OFFLOAD_TEST_BIN} " ; then
60+ COPY_OFFLOAD_TEST_BIN=lib-copy-offload-tester
61+ fi
62+
5863# Flux command
59- FLUX=" flux run -l -N${N} --wait-event=clean"
64+ FLUX=" flux run -l -N${N} "
65+ FLUX_A=" flux alloc -N${N} "
6066
6167# Use a Flux queue
6268if [ " ${Q} " != " " ; then
6369 FLUX+=" -q ${Q} "
70+ FLUX_A+=" -q ${Q} "
6471fi
6572
6673# Require specific rabbits
6774if [ " ${R} " != " " ; then
6875 FLUX+=" --requires=hosts:${R} "
76+ FLUX_A+=" --requires=hosts:${R} "
6977fi
7078
7179# Read the test file and create a bats test for each entry
@@ -75,6 +83,9 @@ for ((i = 0; i < NUM_TESTS; i++)); do
7583done
7684
7785function setup() {
86+ # Create a test tmpdir at the supplied prefix (e.g. /home/user/nnf/tmp.x8e8f0a/)
87+ export TEST_TMPDIR=$( mktemp -d -p ${TEST_TMPDIR_PREFIX} )
88+
7889 # Make a unique directory to support simulteanous tests
7990 export UUID=$( uuidgen | cut -d' -' )
8091 export DESTDIR=${TESTDIR} /${UUID}
@@ -83,12 +94,115 @@ function setup() {
8394}
8495
8596function teardown() {
97+ if [[ -n " $TEST_TMPDIR " ; then
98+ rm -rf $TEST_TMPDIR
99+ fi
100+
86101 # clean up if it succeeded, otherwise leave it around for inspection (if no other tests run afterwards)
87102 if [[ " ${BATS_TEST_COMPLETED} " -eq 1 ]]; then
88103 rm -rf ${DESTDIR}
89104 fi
105+
90106}
91107
108+ # Create a copy offload file to be used in the test by the flux job. This file is used to
109+ # run the lib-copy-offload-tester program. This file is created in the test tmpdir and
110+ # is passed to the flux job. The file is created with the following contents:
111+ function create_copyoffload_file {
112+ runit_file=" $TEST_TMPDIR /run-it.sh"
113+ touch $runit_file
114+ chmod +x $runit_file
115+ cat << 'EOF ' >> $runit_file
116+ #!/bin/bash
117+ echo "starting test on $(hostname)"
118+ set -e
119+
120+ src=$1
121+ dst=$2
122+ profile=$3
123+
124+ prog=<COPY_OFFLOAD_TEST_BIN>
125+
126+ jobid=$FLUX_JOB_ID
127+ jobuid=$(echo $FLUX_KVS_NAMESPACE | cut -d '-' -f2)
128+ wf=fluxjob-$jobuid
129+
130+ echo "jobid: $jobid"
131+ echo "workflow: $wf"
132+ echo "hostname: $(hostname)"
133+ echo "src: $src"
134+ echo "dst: $dst"
135+ echo "profile: $profile"
136+
137+ if [[ -z "$src" || -z "$dst" || -z "$profile" ]]; then
138+ echo "Error: One or more required variables are not set"
139+ exit 1
140+ fi
141+
142+ # For HPE systems with older versions of flux, fake out the bits that flux will provide on LLNL
143+ # systems. Do this if the env vars are not set for us by flux. This assumes each compute has access
144+ # to k8s.
145+ if [[ -z "$DW_WORKFLOW_TOKEN" || -z "$NNF_CONTAINER_LAUNCHER" || -z "$NNF_CONTAINER_PORTS" ]]; then
146+ echo "Fetching workflow information from kubernetes rather than flux..."
147+
148+ if command -v kubectl >/dev/null 2>&1 && kubectl version --request-timeout=5s >/dev/null 2>&1; then
149+ echo "kubectl is available and can contact the server"
150+ else
151+ echo "kubectl is not available or cannot contact the server"
152+ exit 1
153+ fi
154+
155+ export NNF_CONTAINER_LAUNCHER=$(kubectl get workflow $wf -ojson | jq -rM '.status.env["NNF_CONTAINER_LAUNCHER"]')
156+ export NNF_CONTAINER_PORTS=$(kubectl get workflow $wf -ojson | jq -rM '.status.env["NNF_CONTAINER_PORTS"]')
157+ export DW_WORKFLOW_TOKEN=$(kubectl get secret `kubectl get workflow $wf -ojson | jq -rM .status.workflowToken.secretName` -ojson | jq -rM .data.token | base64 -d)
158+ fi
159+
160+ echo "NNF* env vars:"
161+ env | grep -E "NNF|DW_JOB|WORKFLOW|TOKEN"
162+
163+ # expand the $DW_JOB variable
164+ src=$(eval echo "$src")
165+ echo "src: $src"
166+
167+ export DW_WORKFLOW_NAME=$wf
168+ export DW_WORKFLOW_NAMESPACE=default
169+
170+ # Start a copy offload and capture the ID
171+ args=(
172+ -o
173+ -S $src
174+ -D $dst
175+ -P $profile
176+ )
177+ echo "Running $prog ${args[@]}..."
178+ ID=$($prog "${args[@]}")
179+ prog_status=$?
180+ if [[ $prog_status -ne 0 ]]; then
181+ echo "ERROR: $prog failed with exit code $prog_status"
182+ echo " Command: $prog ${args[@]}"
183+ exit $prog_status
184+ fi
185+ echo "ID: $ID"
186+
187+ # Wait for the copy offload to finish
188+ args=(
189+ -q
190+ -j $ID
191+ -w -1
192+ )
193+ echo "Running $prog ${args[@]}..."
194+ $prog "${args[@]}"
195+ EOF
196+
197+ # replace the prog name
198+ sed -i " s/<COPY_OFFLOAD_TEST_BIN>/$COPY_OFFLOAD_TEST_BIN /g" $runit_file
199+
200+ echo $runit_file
201+ }
202+
203+ # This function runs the copy_in/copy_out test. It takes a single argument, which is the index
204+ # of the test to run. It reads the test parameters from the JSON file and runs the test using
205+ # flux.
92206function test_copy_in_copy_out() {
93207 local idx=$1
94208 local src=$( cat $tests_file | jq -r " .[$idx ].src" )
@@ -104,14 +218,16 @@ function test_copy_in_copy_out() {
104218 # Use copy offload to do the copy_out (copy_in isn't supported)
105219 if [[ " $copy_out_method " == " offload" ; then
106220 # replace the hyphen in src with underscore since it's being used on the compute node rather than in a directive
107- echo " SOURCE: $src "
108221 src=" ${src// -/ _} "
109- echo " AFTER SOURCE: $src "
222+
223+ runit_file=$( create_copyoffload_file)
224+ echo " runit_file: $runit_file "
110225 ${FLUX} --setattr=dw=" \
111- #DW jobdw type=$fs_type capacity=10GiB name=copyout-test \
112- #DW copy_in source=$copy_in_src destination=\$ DW_JOB_copyout-test" \
113- bash -c " hostname && \
114- dm-client-go -source=$src -destination=$dest -profile=$DM_PROFILE "
226+ #DW jobdw type=$fs_type capacity=10GiB name=copyout-test requires=copy-offload \
227+ #DW copy_in source=$copy_in_src destination=\$ DW_JOB_copyout-test \
228+ #DW container name=copyoff-container profile=copy-offload-default \
229+ DW_JOB_my_storage=copyout-test DW_GLOBAL_lus=$GLOBAL_LUSTRE_ROOT " \
230+ $runit_file $src $dest $DM_PROFILE
115231 else
116232 ${FLUX} --setattr=dw=" \
117233 #DW jobdw type=$fs_type capacity=10GiB name=copyout-test \
@@ -127,13 +243,24 @@ function test_copy_in_copy_out() {
127243
128244 # grab the output from ls
129245 local ls_output=$( /bin/ls -l ${expected} )
130- echo " $ls_output " # print it out in case of fail
246+ echo " ls_output: $ls_output " # print it out in case of fail
247+
248+ local actual_count=$( echo " $ls_output " | wc -l)
249+ local expected_count
131250
132- # if lustre, then no index mounts and only 1 file
133251 if [[ " $fs_type " == " lustre" ; then
134- echo " $ls_output " | wc -l | grep 1
135- # otherwise verify the number of lines from `ls -l`
252+ expected_count=1
136253 else
137- echo " $ls_output " | wc -l | grep $N
254+ expected_count=$N
255+ fi
256+
257+ if [[ " $actual_count " -ne " $expected_count " ; then
258+ echo " ERROR: File count mismatch for $expected "
259+ echo " Filesystem type: $fs_type "
260+ echo " Expected count: $expected_count "
261+ echo " Actual count: $actual_count "
262+ echo " ls output:"
263+ echo " $ls_output "
264+ return 1
138265 fi
139266}
0 commit comments