Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion .github/workflows/_accuracy_test.yml
Original file line number Diff line number Diff line change
Expand Up @@ -180,7 +180,7 @@ jobs:
-e TZ="Asia/Shanghai" \
-e "no_proxy=localhost,127.0.0.1,0.0.0.0,bcebos.com,.bcebos.com,bj.bcebos.com,su.bcebos.com,paddle-ci.gz.bcebos.com,apiin.im.baidu.com,baidu-int.com,.baidu.com,aliyun.com,gitee.com,pypi.tuna.tsinghua.edu.cn,.tuna.tsinghua.edu.cn" \
--gpus '"device='"${DEVICES}"'"' ${docker_image} /bin/bash -xc '
python -m pip install --pre paddlepaddle-gpu -i https://www.paddlepaddle.org.cn/packages/nightly/cu126/
python -m pip install paddlepaddle-gpu==3.5.0.dev20260508 -i https://www.paddlepaddle.org.cn/packages/nightly/cu126/

pip config set global.index-url https://mirrors.tuna.tsinghua.edu.cn/pypi/web/simple

Expand Down
2 changes: 1 addition & 1 deletion .github/workflows/_base_test.yml
Original file line number Diff line number Diff line change
Expand Up @@ -213,7 +213,7 @@ jobs:
-e TZ="Asia/Shanghai" \
-e "no_proxy=localhost,127.0.0.1,0.0.0.0,bcebos.com,.bcebos.com,bj.bcebos.com,su.bcebos.com,paddle-ci.gz.bcebos.com,apiin.im.baidu.com,baidu-int.com,.baidu.com,aliyun.com,gitee.com,pypi.tuna.tsinghua.edu.cn,.tuna.tsinghua.edu.cn" \
--gpus '"device='"${DEVICES}"'"' ${docker_image} /bin/bash -xc '
python -m pip install --pre paddlepaddle-gpu -i https://www.paddlepaddle.org.cn/packages/nightly/cu126/
python -m pip install paddlepaddle-gpu==3.5.0.dev20260508 -i https://www.paddlepaddle.org.cn/packages/nightly/cu126/

pip config set global.index-url https://mirrors.tuna.tsinghua.edu.cn/pypi/web/simple

Expand Down
2 changes: 1 addition & 1 deletion .github/workflows/_build_linux.yml
Original file line number Diff line number Diff line change
Expand Up @@ -196,7 +196,7 @@ jobs:
elif [[ "${PADDLEVERSION}" != "" ]];then
python -m pip install paddlepaddle-gpu==${PADDLEVERSION} -i https://www.paddlepaddle.org.cn/packages/stable/cu126/
else
python -m pip install --pre paddlepaddle-gpu -i https://www.paddlepaddle.org.cn/packages/nightly/cu126/
python -m pip install paddlepaddle-gpu==3.5.0.dev20260508 -i https://www.paddlepaddle.org.cn/packages/nightly/cu126/
fi

pip config set global.index-url https://mirrors.tuna.tsinghua.edu.cn/pypi/web/simple
Expand Down
2 changes: 1 addition & 1 deletion .github/workflows/_build_linux_cu129.yml
Original file line number Diff line number Diff line change
Expand Up @@ -183,7 +183,7 @@ jobs:
elif [[ "${PADDLEVERSION}" != "" ]];then
python -m pip install paddlepaddle-gpu==${PADDLEVERSION} -i https://www.paddlepaddle.org.cn/packages/stable/cu129/
else
python -m pip install --pre paddlepaddle-gpu -i https://www.paddlepaddle.org.cn/packages/nightly/cu129/
python -m pip install paddlepaddle-gpu==3.5.0.dev20260508 -i https://www.paddlepaddle.org.cn/packages/nightly/cu129/

Copy link
Copy Markdown

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

❓ 疑问 cu129/cu130 nightly 索引中该日期的包是否存在?

所有 CUDA 版本(cu126/cu129/cu130)和 XPU 均锁定到同一版本 3.5.0.dev20260508,但 nightly 包并不保证每个 CUDA 变体都有相同日期的 build。若 cu129 或 cu130 索引中缺少该日期包,对应流水线将立即失败。

建议在合入前验证:

pip index versions paddlepaddle-gpu -i https://www.paddlepaddle.org.cn/packages/nightly/cu129/ 2>&1 | grep 3.5.0.dev20260508
pip index versions paddlepaddle-gpu -i https://www.paddlepaddle.org.cn/packages/nightly/cu130/ 2>&1 | grep 3.5.0.dev20260508

fi

pip config set global.index-url https://mirrors.tuna.tsinghua.edu.cn/pypi/web/simple
Expand Down
2 changes: 1 addition & 1 deletion .github/workflows/_build_linux_cu130.yml
Original file line number Diff line number Diff line change
Expand Up @@ -183,7 +183,7 @@ jobs:
elif [[ "${PADDLEVERSION}" != "" ]];then
python -m pip install paddlepaddle-gpu==${PADDLEVERSION} -i https://www.paddlepaddle.org.cn/packages/stable/cu130/
else
python -m pip install --pre paddlepaddle-gpu -i https://www.paddlepaddle.org.cn/packages/nightly/cu130/
python -m pip install paddlepaddle-gpu==3.5.0.dev20260508 -i https://www.paddlepaddle.org.cn/packages/nightly/cu130/
fi

pip config set global.index-url https://mirrors.tuna.tsinghua.edu.cn/pypi/web/simple
Expand Down
2 changes: 1 addition & 1 deletion .github/workflows/_build_xpu.yml
Original file line number Diff line number Diff line change
Expand Up @@ -159,7 +159,7 @@ jobs:
python -m pip install paddlepaddle-xpu==${PADDLEVERSION} -i https://www.paddlepaddle.org.cn/packages/stable/xpu-p800/
else
python -m pip uninstall paddlepaddle-xpu fastdeploy-xpu -y
python -m pip install --pre paddlepaddle-xpu -i https://www.paddlepaddle.org.cn/packages/nightly/xpu-p800/
python -m pip install paddlepaddle-xpu==3.5.0.dev20260508 -i https://www.paddlepaddle.org.cn/packages/nightly/xpu-p800/
fi


Expand Down
2 changes: 1 addition & 1 deletion .github/workflows/_golang_router_test.yml
Original file line number Diff line number Diff line change
Expand Up @@ -212,7 +212,7 @@ jobs:
git config --global --add safe.directory /workspace/FastDeploy
cd FastDeploy

python -m pip install --pre paddlepaddle-gpu -i https://www.paddlepaddle.org.cn/packages/nightly/cu126/
python -m pip install paddlepaddle-gpu==3.5.0.dev20260508 -i https://www.paddlepaddle.org.cn/packages/nightly/cu126/
pip config set global.extra-index-url https://mirrors.tuna.tsinghua.edu.cn/pypi/web/simple

python -m pip install -r scripts/unittest_requirement.txt
Expand Down
2 changes: 1 addition & 1 deletion .github/workflows/_gpu_4cards_case_test.yml
Original file line number Diff line number Diff line change
Expand Up @@ -208,7 +208,7 @@ jobs:
cd FastDeploy
git diff origin/${BASE_REF}..HEAD --unified=0 > diff.txt

python -m pip install --pre paddlepaddle-gpu -i https://www.paddlepaddle.org.cn/packages/nightly/cu126/
python -m pip install paddlepaddle-gpu==3.5.0.dev20260508 -i https://www.paddlepaddle.org.cn/packages/nightly/cu126/
pip config set global.extra-index-url https://mirrors.tuna.tsinghua.edu.cn/pypi/web/simple

python -m pip install -r scripts/unittest_requirement.txt
Expand Down
2 changes: 1 addition & 1 deletion .github/workflows/_logprob_test_linux.yml
Original file line number Diff line number Diff line change
Expand Up @@ -189,7 +189,7 @@ jobs:
-e TZ="Asia/Shanghai" \
-e "no_proxy=localhost,127.0.0.1,0.0.0.0,bcebos.com,.bcebos.com,bj.bcebos.com,su.bcebos.com,paddle-ci.gz.bcebos.com,apiin.im.baidu.com,baidu-int.com,.baidu.com,aliyun.com,gitee.com,pypi.tuna.tsinghua.edu.cn,.tuna.tsinghua.edu.cn" \
--gpus '"device='"${DEVICES}"'"' ${docker_image} /bin/bash -xc '
python -m pip install --pre paddlepaddle-gpu -i https://www.paddlepaddle.org.cn/packages/nightly/cu126/
python -m pip install paddlepaddle-gpu==3.5.0.dev20260508 -i https://www.paddlepaddle.org.cn/packages/nightly/cu126/

pip config set global.index-url https://mirrors.tuna.tsinghua.edu.cn/pypi/web/simple

Expand Down
2 changes: 1 addition & 1 deletion .github/workflows/_pre_ce_test.yml
Original file line number Diff line number Diff line change
Expand Up @@ -201,7 +201,7 @@ jobs:
--gpus "\"device=${DEVICES}\"" ${docker_image} /bin/bash -c '
git config --global --add safe.directory /workspace/FastDeploy
cd FastDeploy
python -m pip install --pre paddlepaddle-gpu -i https://www.paddlepaddle.org.cn/packages/nightly/cu126/
python -m pip install paddlepaddle-gpu==3.5.0.dev20260508 -i https://www.paddlepaddle.org.cn/packages/nightly/cu126/
python -m pip install ${fd_wheel_url}
bash scripts/run_pre_ce.sh
'
Expand Down
2 changes: 1 addition & 1 deletion .github/workflows/_stable_test.yml
Original file line number Diff line number Diff line change
Expand Up @@ -193,7 +193,7 @@ jobs:
-e TZ="Asia/Shanghai" \
-e "no_proxy=localhost,127.0.0.1,0.0.0.0,bcebos.com,.bcebos.com,bj.bcebos.com,su.bcebos.com,paddle-ci.gz.bcebos.com,apiin.im.baidu.com,baidu-int.com,.baidu.com,aliyun.com,gitee.com,pypi.tuna.tsinghua.edu.cn,.tuna.tsinghua.edu.cn" \
--gpus '"device='"${DEVICES}"'"' ${docker_image} /bin/bash -xc '
python -m pip install --pre paddlepaddle-gpu -i https://www.paddlepaddle.org.cn/packages/nightly/cu126/
python -m pip install paddlepaddle-gpu==3.5.0.dev20260508 -i https://www.paddlepaddle.org.cn/packages/nightly/cu126/

pip config set global.index-url https://mirrors.tuna.tsinghua.edu.cn/pypi/web/simple

Expand Down
2 changes: 1 addition & 1 deletion .github/workflows/_unit_test_coverage.yml
Original file line number Diff line number Diff line change
Expand Up @@ -224,7 +224,7 @@ jobs:
git config --global --add safe.directory /workspace/FastDeploy
cd FastDeploy
git diff origin/${BASE_REF}..HEAD --unified=0 > diff.txt
python -m pip install --pre paddlepaddle-gpu -i https://www.paddlepaddle.org.cn/packages/nightly/cu126/
python -m pip install paddlepaddle-gpu==3.5.0.dev20260508 -i https://www.paddlepaddle.org.cn/packages/nightly/cu126/
pip config set global.extra-index-url https://mirrors.tuna.tsinghua.edu.cn/pypi/web/simple

python -m pip install -r scripts/unittest_requirement.txt
Expand Down
2 changes: 1 addition & 1 deletion .github/workflows/_xpu_4cards_case_test.yml
Original file line number Diff line number Diff line change
Expand Up @@ -178,7 +178,7 @@ jobs:
python -m pip install paddlepaddle-xpu==${PADDLEVERSION} -i https://www.paddlepaddle.org.cn/packages/stable/xpu-p800/
else
python -m pip uninstall paddlepaddle-xpu fastdeploy-xpu -y
python -m pip install --pre paddlepaddle-xpu -i https://www.paddlepaddle.org.cn/packages/nightly/xpu-p800/
python -m pip install paddlepaddle-xpu==3.5.0.dev20260508 -i https://www.paddlepaddle.org.cn/packages/nightly/xpu-p800/
fi
echo "安装上游任务编译的fastdeploy-xpu..."
python -m pip install ${FASTDEPLOY_WHEEL_URL}
Expand Down
2 changes: 1 addition & 1 deletion .github/workflows/_xpu_8cards_case_test.yml
Original file line number Diff line number Diff line change
Expand Up @@ -167,7 +167,7 @@ jobs:
python -m pip install paddlepaddle-xpu==${PADDLEVERSION} -i https://www.paddlepaddle.org.cn/packages/stable/xpu-p800/
else
python -m pip uninstall paddlepaddle-xpu fastdeploy-xpu -y
python -m pip install --pre paddlepaddle-xpu -i https://www.paddlepaddle.org.cn/packages/nightly/xpu-p800/
python -m pip install paddlepaddle-xpu==3.5.0.dev20260508 -i https://www.paddlepaddle.org.cn/packages/nightly/xpu-p800/
fi
echo "安装上游任务编译的fastdeploy-xpu..."
python -m pip install ${FASTDEPLOY_WHEEL_URL}
Expand Down
34 changes: 1 addition & 33 deletions scripts/CheckPRTemplate.py
Original file line number Diff line number Diff line change
Expand Up @@ -28,7 +28,6 @@
"## Modifications",
"## Usage or Command",
"## Accuracy Tests",
"## Checklist",
]
}
}
Expand Down Expand Up @@ -65,27 +64,6 @@ def check_section_content(body, section_titles):
return results


def parse_checklist(section_content):
"""
Parse a checklist section and return dict of items with checked status.
Example return:
{
'Add at least a tag in the PR title.': False,
'Format your code, run `pre-commit` before commit.': True,
...
}
"""
items = {}
lines = section_content.splitlines()
for line in lines:
match = re.match(r"- \[( |x|X)\] (.+)", line)
if match:
checked = match.group(1).lower() == "x"
item_text = match.group(2).strip()
items[item_text] = checked
return items


def check_pr_template(repo, body):
"""Check whether a PR description follows the expected template."""
body = remove_comments(body)
Expand All @@ -108,21 +86,11 @@ def check_pr_template(repo, body):
else:
messages.append("❌ Missing sections: {}. Please complete them.".format(", ".join(missing)))

# Check Checklist items if present
checklist_content = results.get("## Checklist", "")
if checklist_content:
checklist_items = parse_checklist(checklist_content)
unchecked = [item for item, checked in checklist_items.items() if not checked]
if unchecked:
messages.append("❌ The following checklist items are not completed:")
for item in unchecked:
messages.append(f" - [ ] {item}")

if messages:
messages.append(
"\n💡 **Tips for fixing:**\n"
"1. Each PR must follow the standard FastDeploy PR template.\n"
"2. Ensure every section (Motivation, Modifications, Usage, Accuracy Tests, Checklist) "
"2. Ensure every section (Motivation, Modifications, Usage, Accuracy Tests) "
"is clearly filled with relevant details.\n"
"3. You can refer to the official PR example: "
"https://github.com/PaddlePaddle/FastDeploy/blob/develop/.github/pull_request_template.md\n"
Expand Down
8 changes: 4 additions & 4 deletions scripts/check_approval.sh
Original file line number Diff line number Diff line change
Expand Up @@ -40,7 +40,7 @@ function add_failed(){
}


HAS_CUSTOM_REGISTRER=`git diff -U0 upstream/$BRANCH | grep '^\+' | grep -zoE "PD_BUILD_(STATIC_)?OP" || true`
HAS_CUSTOM_REGISTRER=`git diff --merge-base -U0 upstream/$BRANCH | grep '^\+' | grep -zoE "PD_BUILD_(STATIC_)?OP" || true`
if [ ${HAS_CUSTOM_REGISTRER} ] && [ "${PR_ID}" != "" ]; then
echo_line1="You must have one FastDeploy RD (qingqing01(dangqingqing), Jiang-Jia-Jun(jiangjiajun), heavengate(dengkaipeng)) approval for adding custom op.\n"
echo_line2="You must have one PaddlePaddle RD (jeff41404(gaoxiang), yongqiangma(mayongqiang)) approval for adding custom op.\n"
Expand All @@ -52,7 +52,7 @@ WORKER_OR_CONFIG_LIST=(
"fastdeploy/model_executor/graph_optimization"
)

HAS_WORKER_OR_CONFIG_MODIFY=`git diff upstream/$BRANCH --name-only | grep -E $(printf -- "-e %s " "${WORKER_OR_CONFIG_LIST[@]}") || true`
HAS_WORKER_OR_CONFIG_MODIFY=`git diff --merge-base upstream/$BRANCH --name-only | grep -E $(printf -- "-e %s " "${WORKER_OR_CONFIG_LIST[@]}") || true`
if [ "${HAS_WORKER_OR_CONFIG_MODIFY}" != "" ] && [ "${PR_ID}" != "" ]; then
echo_line1="You must have one FastDeploy RD gongshaotian(gongshaotian) approval for modifing [$(IFS=', '; echo "${WORKER_OR_CONFIG_LIST[*]}")]."
check_approval "$echo_line1" 1 gongshaotian
Expand All @@ -63,15 +63,15 @@ SPECULATIVE_DECODING_LIST=(
"custom_ops/gpu_ops/speculate_decoding"
)

HAS_SPECULATIVE_DECODING_MODIFY=`git diff upstream/$BRANCH --name-only | grep -E $(printf -- "-e %s " "${SPECULATIVE_DECODING_LIST[@]}") || true`
HAS_SPECULATIVE_DECODING_MODIFY=`git diff --merge-base upstream/$BRANCH --name-only | grep -E $(printf -- "-e %s " "${SPECULATIVE_DECODING_LIST[@]}") || true`
if [ "${HAS_SPECULATIVE_DECODING_MODIFY}" != "" ] && [ "${PR_ID}" != "" ]; then
echo_line1="You must have one FastDeploy RD (freeliuzc(liuzichang01), Deleter-D(wangyanpeng04)) approval for modifing [$(IFS=', '; echo "${SPECULATIVE_DECODING_LIST[*]}")]."
check_approval "$echo_line1" 1 freeliuzc Deleter-D
fi

ENV_FILE="fastdeploy/envs.py"

HAS_ENV_MODIFY=$(git diff upstream/$BRANCH --name-only | grep -E "^${ENV_FILE}$" || true)
HAS_ENV_MODIFY=$(git diff --merge-base upstream/$BRANCH --name-only | grep -E "^${ENV_FILE}$" || true)
if [ "${HAS_ENV_MODIFY}" != "" ] && [ "${PR_ID}" != "" ]; then
echo_line1="You must have one FastDeploy RD (Jiang-Jia-Jun(jiangjiajun), yuanlehome(liuyuanle), rainyfly(chenjian26), Wanglongzhi2001(wanglongzhi)) approval for modifying [${ENV_FILE}]."
check_approval "$echo_line1" 1 Jiang-Jia-Jun yuanlehome rainyfly Wanglongzhi2001
Expand Down
49 changes: 30 additions & 19 deletions scripts/coverage_run.sh
Original file line number Diff line number Diff line change
Expand Up @@ -42,30 +42,18 @@ classify_tests() {
fi
fi

# Rule 5: high-risk OOM tests (treat as multi_gpu for sequential execution)
if [[ "$test_file" == "tests/entrypoints/cli/test_main.py" ||
"$test_file" == "tests/entrypoints/cli/test_serve.py" ||
"$test_file" == "tests/operators/test_group_swiglu_with_masked.py" ||
"$test_file" == "tests/operators/test_hybrid_mtp_ngram.py" ||
"$test_file" == "tests/operators/test_moe_top_k_select.py" ||
"$test_file" == "tests/operators/test_noaux_tc.py" ||
"$test_file" == "tests/output/test_get_save_output_v1.py" ||
"$test_file" == "tests/output/test_process_batch_draft_tokens.py" ||
"$test_file" == "tests/output/test_process_batch_output.py" ]]; then
echo "multi_gpu"
return
fi

# ========== Single-GPU tests (no port required, can run in parallel) ==========
echo "single_gpu"
}

# ============================================================
# Run Test With Logging
# Run Test With Logging (with retry for OOM/Kill)
# ============================================================
run_test_with_logging() {
local test_file=$1
local log_prefix=$2
local max_retries=3 # Max retries for OOM/Kill issues
local retry_count=0
local status

echo "Running pytest file: $test_file"
Expand All @@ -81,14 +69,37 @@ run_test_with_logging() {
# Set FD_LOG_DIR to isolate logs for each test
export FD_LOG_DIR="$isolated_log_dir"

# Run test
timeout 600 python -m coverage run -m pytest -c ${PYTEST_INI} "$test_file" -vv -s
status=$?
# Retry loop for OOM/Kill issues (only handle "Killed" / SIGKILL)
while [ $retry_count -le $max_retries ]; do
if [ $retry_count -gt 0 ]; then
echo ""
echo "==================== Retrying (${retry_count}/${max_retries}) ===================="
echo "Previous attempt was Killed, retrying..."
# Clean up before retry
sleep 5 # Wait a bit to let resources be released

Copy link
Copy Markdown

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

🟡 建议 OOM 重试间隔可能不足以释放 GPU 显存

OOM/SIGKILL 触发后,GPU 显存由 CUDA driver 异步清理,在 CI 机器多任务并发时 5 秒往往不够,可能导致下一次重试仍然触发 OOM。

建议将等待时间延长(如 30 秒),或在重试前检测 GPU 显存使用量:

sleep 30  # 给 CUDA driver 更多时间释放显存

fi

# Run test
timeout 600 python -m coverage run -m pytest -c ${PYTEST_INI} "$test_file" -vv -s
status=$?

# Exit code 137 = SIGKILL (Killed / OOM)
if [ "$status" -eq 137 ] && [ $retry_count -lt $max_retries ]; then
retry_count=$((retry_count + 1))
continue
fi

# Break loop on success or non-Kill error or max retries reached
break
done

if [ "$status" -ne 0 ]; then
echo "$test_file" >> "$log_prefix"
echo ""
echo "==================== Test Failed: $test_file ===================="
if [ $retry_count -gt 0 ]; then
echo "Total attempts: $((retry_count + 1))"
fi

# Use isolated log directory for this test
if [ -d "$isolated_log_dir" ]; then
Expand All @@ -108,7 +119,7 @@ run_test_with_logging() {
fi

echo ">>> grep error in ${isolated_log_dir}"
grep -Rni --color=auto "error" "${isolated_log_dir}" || true
grep -Rni --color=auto "error" "${isolated_log_dir}" --exclude="pytest_*_error.log" || true
fi

# print all server logs
Expand Down
2 changes: 1 addition & 1 deletion scripts/run_golang_router.sh
Original file line number Diff line number Diff line change
Expand Up @@ -54,7 +54,7 @@ for test_file in "${test_files[@]}"; do
fi

echo ">>> grep error in ${log_dir}"
grep -Rni --color=auto "error" "${log_dir}" || true
grep -Rni --color=auto "error" "${log_dir}" --exclude="pytest_*_error.log" || true
fi
done

Expand Down
2 changes: 1 addition & 1 deletion scripts/run_gpu_4cards.sh
Original file line number Diff line number Diff line change
Expand Up @@ -44,7 +44,7 @@ for test_file in "${test_files[@]}"; do

if [ -d "${REPO_ROOT}/log" ]; then
echo ">>> grep error in ${REPO_ROOT}/log/"
grep -Rni --color=auto "error" "${REPO_ROOT}/log/" || true
grep -Rni --color=auto "error" "${REPO_ROOT}/log/" --exclude="pytest_*_error.log" || true
else
echo "${REPO_ROOT}/log directory not found"
fi
Expand Down
2 changes: 1 addition & 1 deletion scripts/run_pre_ce.sh
Original file line number Diff line number Diff line change
Expand Up @@ -38,7 +38,7 @@ for subdir in "$run_path"*/; do
if [ $exit_code -ne 0 ]; then
if [ -d "${subdir%/}/log" ]; then
echo ">>> grep error in ${subdir%/}/log/"
grep -Rni --color=auto "error" "${subdir%/}/log/" || true
grep -Rni --color=auto "error" "${subdir%/}/log/" --exclude="pytest_*_error.log" || true
else
echo "${subdir%/}/log directory not found"
fi
Expand Down
Loading
Loading