Skip to content

ci: Improve error handling for Python backend model initialization failures (#8303) #8333

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
mc-nv merged 1 commit into from
Aug 7, 2025
Merged

ci: Improve error handling for Python backend model initialization failures (#8303) #8333

Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
59 changes: 59 additions & 0 deletions qa/L0_backend_python/test.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -407,6 +407,65 @@ and shared memory pages after starting triton equals to $current_num_pages \n***
exit 1
fi


# Test model with non-existent model file
# The model.py file is intentionally not created to trigger the failure.
rm -fr ./models
mkdir -p models/non_existent_model/1
cp ../python_models/identity_fp32/config.pbtxt ./models/non_existent_model/config.pbtxt
(cd models/non_existent_model && \
sed -i "s/^name:.*/name: \"non_existent_model\"/" config.pbtxt)

ERROR_MESSAGE_1="Failed to preinitialize Python stub: Python model file not found in '`pwd`/models/non_existent_model/1/model.py'"
ERROR_MESSAGE_2="failed to load 'non_existent_model'"

for test_mode in "default" "auto_config_disabled"; do
if [ "$test_mode" == "default" ]; then
SERVER_LOG_SUFFIX=""
SERVER_ARGS=$BASE_SERVER_ARGS
else
SERVER_LOG_SUFFIX="_auto_config_disabled"
SERVER_ARGS="$BASE_SERVER_ARGS --disable-auto-complete-config"
fi

SERVER_LOG="./non_existent_model_server${SERVER_LOG_SUFFIX}.log"
CLIENT_LOG="./non_existent_model_client${SERVER_LOG_SUFFIX}.log"

prev_num_pages=`get_shm_pages`
run_server
if [ "$SERVER_PID" != "0" ]; then
echo -e "*** FAILED: unexpected success starting $SERVER" >> $CLIENT_LOG
RET=1
kill_server
else
if grep -q "$ERROR_MESSAGE_1" $SERVER_LOG; then
echo -e "Found \"$ERROR_MESSAGE_1\"" >> $CLIENT_LOG
else
echo -e "Not found \"$ERROR_MESSAGE_1\" in $SERVER_LOG" >> $CLIENT_LOG
cat $SERVER_LOG >> $CLIENT_LOG
RET=1
fi

if grep -q "$ERROR_MESSAGE_2" $SERVER_LOG; then
echo -e "Found \"$ERROR_MESSAGE_2\"" >> $CLIENT_LOG
else
echo -e "Not found \"$ERROR_MESSAGE_2\" in $SERVER_LOG" >> $CLIENT_LOG
cat $SERVER_LOG >> $CLIENT_LOG
RET=1
fi
fi

current_num_pages=`get_shm_pages`
if [ $current_num_pages -ne $prev_num_pages ]; then
cat $SERVER_LOG
ls /dev/shm
echo -e "\n***\n*** Test Failed. Shared memory pages were not cleaned properly.
Shared memory pages before starting triton equals to $prev_num_pages
and shared memory pages after starting triton equals to $current_num_pages \n***"
exit 1
fi
done

# Disable env test for Jetson since cloud storage repos are not supported
# Disable ensemble, io and bls tests for Jetson since GPU Tensors are not supported
# Disable variants test for Jetson since already built without GPU Tensor support
Expand Down
Loading