[Cherry-Pick][CI] Sync dev optimizations to release/online/20260415(#7602) (#7857)

EmmonsCurse · web-flow · commit 55eb3a64af38 · 2026-05-19T21:13:18.000+08:00
diff --git a/.github/workflows/_accuracy_test.yml b/.github/workflows/_accuracy_test.yml
@@ -180,7 +180,7 @@ jobs:
           -e TZ="Asia/Shanghai" \
           -e "no_proxy=localhost,127.0.0.1,0.0.0.0,bcebos.com,.bcebos.com,bj.bcebos.com,su.bcebos.com,paddle-ci.gz.bcebos.com,apiin.im.baidu.com,baidu-int.com,.baidu.com,aliyun.com,gitee.com,pypi.tuna.tsinghua.edu.cn,.tuna.tsinghua.edu.cn" \
           --gpus '"device='"${DEVICES}"'"' ${docker_image} /bin/bash -xc '
-          python -m pip install --pre paddlepaddle-gpu -i https://www.paddlepaddle.org.cn/packages/nightly/cu126/
+          python -m pip install paddlepaddle-gpu==3.5.0.dev20260508 -i https://www.paddlepaddle.org.cn/packages/nightly/cu126/
 
           pip config set global.index-url https://mirrors.tuna.tsinghua.edu.cn/pypi/web/simple
 
diff --git a/.github/workflows/_base_test.yml b/.github/workflows/_base_test.yml
@@ -213,7 +213,7 @@ jobs:
           -e TZ="Asia/Shanghai" \
           -e "no_proxy=localhost,127.0.0.1,0.0.0.0,bcebos.com,.bcebos.com,bj.bcebos.com,su.bcebos.com,paddle-ci.gz.bcebos.com,apiin.im.baidu.com,baidu-int.com,.baidu.com,aliyun.com,gitee.com,pypi.tuna.tsinghua.edu.cn,.tuna.tsinghua.edu.cn" \
           --gpus '"device='"${DEVICES}"'"' ${docker_image} /bin/bash -xc '
-          python -m pip install --pre paddlepaddle-gpu -i https://www.paddlepaddle.org.cn/packages/nightly/cu126/
+          python -m pip install paddlepaddle-gpu==3.5.0.dev20260508 -i https://www.paddlepaddle.org.cn/packages/nightly/cu126/
 
           pip config set global.index-url https://mirrors.tuna.tsinghua.edu.cn/pypi/web/simple
 
diff --git a/.github/workflows/_build_linux.yml b/.github/workflows/_build_linux.yml
@@ -196,7 +196,7 @@ jobs:
             elif [[ "${PADDLEVERSION}" != "" ]];then
               python -m pip install paddlepaddle-gpu==${PADDLEVERSION} -i https://www.paddlepaddle.org.cn/packages/stable/cu126/
             else
-              python -m pip install --pre paddlepaddle-gpu -i https://www.paddlepaddle.org.cn/packages/nightly/cu126/
+              python -m pip install paddlepaddle-gpu==3.5.0.dev20260508 -i https://www.paddlepaddle.org.cn/packages/nightly/cu126/
             fi
 
             pip config set global.index-url https://mirrors.tuna.tsinghua.edu.cn/pypi/web/simple
diff --git a/.github/workflows/_build_linux_cu129.yml b/.github/workflows/_build_linux_cu129.yml
@@ -183,7 +183,7 @@ jobs:
             elif [[ "${PADDLEVERSION}" != "" ]];then
               python -m pip install paddlepaddle-gpu==${PADDLEVERSION} -i https://www.paddlepaddle.org.cn/packages/stable/cu129/
             else
-              python -m pip install --pre paddlepaddle-gpu -i https://www.paddlepaddle.org.cn/packages/nightly/cu129/
+              python -m pip install paddlepaddle-gpu==3.5.0.dev20260508 -i https://www.paddlepaddle.org.cn/packages/nightly/cu129/
             fi
 
             pip config set global.index-url https://mirrors.tuna.tsinghua.edu.cn/pypi/web/simple
diff --git a/.github/workflows/_build_linux_cu130.yml b/.github/workflows/_build_linux_cu130.yml
@@ -183,7 +183,7 @@ jobs:
             elif [[ "${PADDLEVERSION}" != "" ]];then
               python -m pip install paddlepaddle-gpu==${PADDLEVERSION} -i https://www.paddlepaddle.org.cn/packages/stable/cu130/
             else
-              python -m pip install --pre paddlepaddle-gpu -i https://www.paddlepaddle.org.cn/packages/nightly/cu130/
+              python -m pip install paddlepaddle-gpu==3.5.0.dev20260508 -i https://www.paddlepaddle.org.cn/packages/nightly/cu130/
             fi
 
             pip config set global.index-url https://mirrors.tuna.tsinghua.edu.cn/pypi/web/simple
diff --git a/.github/workflows/_build_xpu.yml b/.github/workflows/_build_xpu.yml
@@ -159,7 +159,7 @@ jobs:
               python -m pip install paddlepaddle-xpu==${PADDLEVERSION} -i https://www.paddlepaddle.org.cn/packages/stable/xpu-p800/
             else
               python -m pip uninstall paddlepaddle-xpu fastdeploy-xpu -y
-              python -m pip install --pre paddlepaddle-xpu -i https://www.paddlepaddle.org.cn/packages/nightly/xpu-p800/
+              python -m pip install paddlepaddle-xpu==3.5.0.dev20260508 -i https://www.paddlepaddle.org.cn/packages/nightly/xpu-p800/
             fi
 
 
diff --git a/.github/workflows/_golang_router_test.yml b/.github/workflows/_golang_router_test.yml
@@ -212,7 +212,7 @@ jobs:
           git config --global --add safe.directory /workspace/FastDeploy
           cd FastDeploy
 
-          python -m pip install --pre paddlepaddle-gpu -i https://www.paddlepaddle.org.cn/packages/nightly/cu126/
+          python -m pip install paddlepaddle-gpu==3.5.0.dev20260508 -i https://www.paddlepaddle.org.cn/packages/nightly/cu126/
           pip config set global.extra-index-url https://mirrors.tuna.tsinghua.edu.cn/pypi/web/simple
 
           python -m pip install -r scripts/unittest_requirement.txt
diff --git a/.github/workflows/_gpu_4cards_case_test.yml b/.github/workflows/_gpu_4cards_case_test.yml
@@ -208,7 +208,7 @@ jobs:
           cd FastDeploy
           git diff origin/${BASE_REF}..HEAD --unified=0 > diff.txt
 
-          python -m pip install --pre paddlepaddle-gpu -i https://www.paddlepaddle.org.cn/packages/nightly/cu126/
+          python -m pip install paddlepaddle-gpu==3.5.0.dev20260508 -i https://www.paddlepaddle.org.cn/packages/nightly/cu126/
           pip config set global.extra-index-url https://mirrors.tuna.tsinghua.edu.cn/pypi/web/simple
 
           python -m pip install -r scripts/unittest_requirement.txt
diff --git a/.github/workflows/_logprob_test_linux.yml b/.github/workflows/_logprob_test_linux.yml
@@ -189,7 +189,7 @@ jobs:
           -e TZ="Asia/Shanghai" \
           -e "no_proxy=localhost,127.0.0.1,0.0.0.0,bcebos.com,.bcebos.com,bj.bcebos.com,su.bcebos.com,paddle-ci.gz.bcebos.com,apiin.im.baidu.com,baidu-int.com,.baidu.com,aliyun.com,gitee.com,pypi.tuna.tsinghua.edu.cn,.tuna.tsinghua.edu.cn" \
           --gpus '"device='"${DEVICES}"'"' ${docker_image} /bin/bash -xc '
-          python -m pip install --pre paddlepaddle-gpu -i https://www.paddlepaddle.org.cn/packages/nightly/cu126/
+          python -m pip install paddlepaddle-gpu==3.5.0.dev20260508 -i https://www.paddlepaddle.org.cn/packages/nightly/cu126/
 
           pip config set global.index-url https://mirrors.tuna.tsinghua.edu.cn/pypi/web/simple
 
diff --git a/.github/workflows/_pre_ce_test.yml b/.github/workflows/_pre_ce_test.yml
@@ -201,7 +201,7 @@ jobs:
           --gpus "\"device=${DEVICES}\"" ${docker_image} /bin/bash -c '
           git config --global --add safe.directory /workspace/FastDeploy
           cd FastDeploy
-          python -m pip install --pre paddlepaddle-gpu -i https://www.paddlepaddle.org.cn/packages/nightly/cu126/
+          python -m pip install paddlepaddle-gpu==3.5.0.dev20260508 -i https://www.paddlepaddle.org.cn/packages/nightly/cu126/
           python -m pip install ${fd_wheel_url}
           bash scripts/run_pre_ce.sh
           '
diff --git a/.github/workflows/_stable_test.yml b/.github/workflows/_stable_test.yml
@@ -193,7 +193,7 @@ jobs:
           -e TZ="Asia/Shanghai" \
           -e "no_proxy=localhost,127.0.0.1,0.0.0.0,bcebos.com,.bcebos.com,bj.bcebos.com,su.bcebos.com,paddle-ci.gz.bcebos.com,apiin.im.baidu.com,baidu-int.com,.baidu.com,aliyun.com,gitee.com,pypi.tuna.tsinghua.edu.cn,.tuna.tsinghua.edu.cn" \
           --gpus '"device='"${DEVICES}"'"' ${docker_image} /bin/bash -xc '
-          python -m pip install --pre paddlepaddle-gpu -i https://www.paddlepaddle.org.cn/packages/nightly/cu126/
+          python -m pip install paddlepaddle-gpu==3.5.0.dev20260508 -i https://www.paddlepaddle.org.cn/packages/nightly/cu126/
 
           pip config set global.index-url https://mirrors.tuna.tsinghua.edu.cn/pypi/web/simple
 
diff --git a/.github/workflows/_unit_test_coverage.yml b/.github/workflows/_unit_test_coverage.yml
@@ -224,7 +224,7 @@ jobs:
           git config --global --add safe.directory /workspace/FastDeploy
           cd FastDeploy
           git diff origin/${BASE_REF}..HEAD --unified=0 > diff.txt
-          python -m pip install --pre paddlepaddle-gpu -i https://www.paddlepaddle.org.cn/packages/nightly/cu126/
+          python -m pip install paddlepaddle-gpu==3.5.0.dev20260508 -i https://www.paddlepaddle.org.cn/packages/nightly/cu126/
           pip config set global.extra-index-url https://mirrors.tuna.tsinghua.edu.cn/pypi/web/simple
 
           python -m pip install -r scripts/unittest_requirement.txt
diff --git a/.github/workflows/_xpu_4cards_case_test.yml b/.github/workflows/_xpu_4cards_case_test.yml
@@ -178,7 +178,7 @@ jobs:
             python -m pip install paddlepaddle-xpu==${PADDLEVERSION} -i https://www.paddlepaddle.org.cn/packages/stable/xpu-p800/
           else
             python -m pip uninstall paddlepaddle-xpu fastdeploy-xpu -y
-            python -m pip install --pre paddlepaddle-xpu -i https://www.paddlepaddle.org.cn/packages/nightly/xpu-p800/
+            python -m pip install paddlepaddle-xpu==3.5.0.dev20260508 -i https://www.paddlepaddle.org.cn/packages/nightly/xpu-p800/
           fi
           echo "安装上游任务编译的fastdeploy-xpu..."
           python -m pip install ${FASTDEPLOY_WHEEL_URL}
diff --git a/.github/workflows/_xpu_8cards_case_test.yml b/.github/workflows/_xpu_8cards_case_test.yml
@@ -167,7 +167,7 @@ jobs:
             python -m pip install paddlepaddle-xpu==${PADDLEVERSION} -i https://www.paddlepaddle.org.cn/packages/stable/xpu-p800/
           else
             python -m pip uninstall paddlepaddle-xpu fastdeploy-xpu -y
-            python -m pip install --pre paddlepaddle-xpu -i https://www.paddlepaddle.org.cn/packages/nightly/xpu-p800/
+            python -m pip install paddlepaddle-xpu==3.5.0.dev20260508 -i https://www.paddlepaddle.org.cn/packages/nightly/xpu-p800/
           fi
           echo "安装上游任务编译的fastdeploy-xpu..."
           python -m pip install ${FASTDEPLOY_WHEEL_URL}
diff --git a/scripts/CheckPRTemplate.py b/scripts/CheckPRTemplate.py
@@ -28,7 +28,6 @@
             "## Modifications",
             "## Usage or Command",
             "## Accuracy Tests",
-            "## Checklist",
         ]
     }
 }
@@ -65,27 +64,6 @@ def check_section_content(body, section_titles):
     return results
 
 
-def parse_checklist(section_content):
-    """
-    Parse a checklist section and return dict of items with checked status.
-    Example return:
-    {
-        'Add at least a tag in the PR title.': False,
-        'Format your code, run `pre-commit` before commit.': True,
-        ...
-    }
-    """
-    items = {}
-    lines = section_content.splitlines()
-    for line in lines:
-        match = re.match(r"- \[( |x|X)\] (.+)", line)
-        if match:
-            checked = match.group(1).lower() == "x"
-            item_text = match.group(2).strip()
-            items[item_text] = checked
-    return items
-
-
 def check_pr_template(repo, body):
     """Check whether a PR description follows the expected template."""
     body = remove_comments(body)
@@ -108,21 +86,11 @@ def check_pr_template(repo, body):
         else:
             messages.append("❌ Missing sections: {}. Please complete them.".format(", ".join(missing)))
 
-    # Check Checklist items if present
-    checklist_content = results.get("## Checklist", "")
-    if checklist_content:
-        checklist_items = parse_checklist(checklist_content)
-        unchecked = [item for item, checked in checklist_items.items() if not checked]
-        if unchecked:
-            messages.append("❌ The following checklist items are not completed:")
-            for item in unchecked:
-                messages.append(f"   - [ ] {item}")
-
     if messages:
         messages.append(
             "\n💡 **Tips for fixing:**\n"
             "1. Each PR must follow the standard FastDeploy PR template.\n"
-            "2. Ensure every section (Motivation, Modifications, Usage, Accuracy Tests, Checklist) "
+            "2. Ensure every section (Motivation, Modifications, Usage, Accuracy Tests) "
             "is clearly filled with relevant details.\n"
             "3. You can refer to the official PR example: "
             "https://github.com/PaddlePaddle/FastDeploy/blob/develop/.github/pull_request_template.md\n"
diff --git a/scripts/check_approval.sh b/scripts/check_approval.sh
@@ -40,7 +40,7 @@ function add_failed(){
 }
 
 
-HAS_CUSTOM_REGISTRER=`git diff -U0 upstream/$BRANCH | grep '^\+' | grep -zoE "PD_BUILD_(STATIC_)?OP" || true`
+HAS_CUSTOM_REGISTRER=`git diff --merge-base -U0 upstream/$BRANCH | grep '^\+' | grep -zoE "PD_BUILD_(STATIC_)?OP" || true`
 if [ ${HAS_CUSTOM_REGISTRER} ] && [ "${PR_ID}" != "" ]; then
     echo_line1="You must have one FastDeploy RD (qingqing01(dangqingqing), Jiang-Jia-Jun(jiangjiajun), heavengate(dengkaipeng)) approval for adding custom op.\n"
     echo_line2="You must have one PaddlePaddle RD (jeff41404(gaoxiang), yongqiangma(mayongqiang)) approval for adding custom op.\n"
@@ -52,7 +52,7 @@ WORKER_OR_CONFIG_LIST=(
     "fastdeploy/model_executor/graph_optimization"
 )
 
-HAS_WORKER_OR_CONFIG_MODIFY=`git diff upstream/$BRANCH  --name-only | grep -E $(printf -- "-e %s " "${WORKER_OR_CONFIG_LIST[@]}") || true`
+HAS_WORKER_OR_CONFIG_MODIFY=`git diff --merge-base upstream/$BRANCH  --name-only | grep -E $(printf -- "-e %s " "${WORKER_OR_CONFIG_LIST[@]}") || true`
 if [ "${HAS_WORKER_OR_CONFIG_MODIFY}" != "" ] && [ "${PR_ID}" != "" ]; then
     echo_line1="You must have one FastDeploy RD gongshaotian(gongshaotian) approval for modifing [$(IFS=', '; echo "${WORKER_OR_CONFIG_LIST[*]}")]."
     check_approval "$echo_line1" 1 gongshaotian
@@ -63,15 +63,15 @@ SPECULATIVE_DECODING_LIST=(
     "custom_ops/gpu_ops/speculate_decoding"
 )
 
-HAS_SPECULATIVE_DECODING_MODIFY=`git diff upstream/$BRANCH  --name-only | grep -E $(printf -- "-e %s " "${SPECULATIVE_DECODING_LIST[@]}") || true`
+HAS_SPECULATIVE_DECODING_MODIFY=`git diff --merge-base upstream/$BRANCH  --name-only | grep -E $(printf -- "-e %s " "${SPECULATIVE_DECODING_LIST[@]}") || true`
 if [ "${HAS_SPECULATIVE_DECODING_MODIFY}" != "" ] && [ "${PR_ID}" != "" ]; then
     echo_line1="You must have one FastDeploy RD (freeliuzc(liuzichang01), Deleter-D(wangyanpeng04)) approval for modifing [$(IFS=', '; echo "${SPECULATIVE_DECODING_LIST[*]}")]."
     check_approval "$echo_line1" 1 freeliuzc Deleter-D
 fi
 
 ENV_FILE="fastdeploy/envs.py"
 
-HAS_ENV_MODIFY=$(git diff upstream/$BRANCH --name-only | grep -E "^${ENV_FILE}$" || true)
+HAS_ENV_MODIFY=$(git diff --merge-base upstream/$BRANCH --name-only | grep -E "^${ENV_FILE}$" || true)
 if [ "${HAS_ENV_MODIFY}" != "" ] && [ "${PR_ID}" != "" ]; then
     echo_line1="You must have one FastDeploy RD (Jiang-Jia-Jun(jiangjiajun), yuanlehome(liuyuanle), rainyfly(chenjian26), Wanglongzhi2001(wanglongzhi)) approval for modifying [${ENV_FILE}]."
     check_approval "$echo_line1" 1 Jiang-Jia-Jun yuanlehome rainyfly Wanglongzhi2001
diff --git a/scripts/coverage_run.sh b/scripts/coverage_run.sh
@@ -42,30 +42,18 @@ classify_tests() {
         fi
     fi
 
-    # Rule 5: high-risk OOM tests (treat as multi_gpu for sequential execution)
-    if [[ "$test_file" == "tests/entrypoints/cli/test_main.py" ||
-          "$test_file" == "tests/entrypoints/cli/test_serve.py" ||
-          "$test_file" == "tests/operators/test_group_swiglu_with_masked.py" ||
-          "$test_file" == "tests/operators/test_hybrid_mtp_ngram.py" ||
-          "$test_file" == "tests/operators/test_moe_top_k_select.py" ||
-          "$test_file" == "tests/operators/test_noaux_tc.py" ||
-          "$test_file" == "tests/output/test_get_save_output_v1.py" ||
-          "$test_file" == "tests/output/test_process_batch_draft_tokens.py" ||
-          "$test_file" == "tests/output/test_process_batch_output.py" ]]; then
-        echo "multi_gpu"
-        return
-    fi
-
     # ========== Single-GPU tests (no port required, can run in parallel) ==========
     echo "single_gpu"
 }
 
 # ============================================================
-# Run Test With Logging
+# Run Test With Logging (with retry for OOM/Kill)
 # ============================================================
 run_test_with_logging() {
     local test_file=$1
     local log_prefix=$2
+    local max_retries=3  # Max retries for OOM/Kill issues
+    local retry_count=0
     local status
 
     echo "Running pytest file: $test_file"
@@ -81,14 +69,37 @@ run_test_with_logging() {
     # Set FD_LOG_DIR to isolate logs for each test
     export FD_LOG_DIR="$isolated_log_dir"
 
-    # Run test
-    timeout 600 python -m coverage run -m pytest -c ${PYTEST_INI} "$test_file" -vv -s
-    status=$?
+    # Retry loop for OOM/Kill issues (only handle "Killed" / SIGKILL)
+    while [ $retry_count -le $max_retries ]; do
+        if [ $retry_count -gt 0 ]; then
+            echo ""
+            echo "==================== Retrying (${retry_count}/${max_retries}) ===================="
+            echo "Previous attempt was Killed, retrying..."
+            # Clean up before retry
+            sleep 5  # Wait a bit to let resources be released
+        fi
+
+        # Run test
+        timeout 600 python -m coverage run -m pytest -c ${PYTEST_INI} "$test_file" -vv -s
+        status=$?
+
+        # Exit code 137 = SIGKILL (Killed / OOM)
+        if [ "$status" -eq 137 ] && [ $retry_count -lt $max_retries ]; then
+            retry_count=$((retry_count + 1))
+            continue
+        fi
+
+        # Break loop on success or non-Kill error or max retries reached
+        break
+    done
 
     if [ "$status" -ne 0 ]; then
         echo "$test_file" >> "$log_prefix"
         echo ""
         echo "==================== Test Failed: $test_file ===================="
+        if [ $retry_count -gt 0 ]; then
+            echo "Total attempts: $((retry_count + 1))"
+        fi
 
         # Use isolated log directory for this test
         if [ -d "$isolated_log_dir" ]; then
@@ -108,7 +119,7 @@ run_test_with_logging() {
             fi
 
             echo ">>> grep error in ${isolated_log_dir}"
-            grep -Rni --color=auto "error" "${isolated_log_dir}" || true
+            grep -Rni --color=auto "error" "${isolated_log_dir}" --exclude="pytest_*_error.log" || true
         fi
 
         # print all server logs
diff --git a/scripts/run_golang_router.sh b/scripts/run_golang_router.sh
@@ -54,7 +54,7 @@ for test_file in "${test_files[@]}"; do
                 fi
 
                 echo ">>> grep error in ${log_dir}"
-                grep -Rni --color=auto "error" "${log_dir}" || true
+                grep -Rni --color=auto "error" "${log_dir}" --exclude="pytest_*_error.log" || true
             fi
         done
 
diff --git a/scripts/run_gpu_4cards.sh b/scripts/run_gpu_4cards.sh
@@ -44,7 +44,7 @@ for test_file in "${test_files[@]}"; do
 
         if [ -d "${REPO_ROOT}/log" ]; then
             echo ">>> grep error in ${REPO_ROOT}/log/"
-            grep -Rni --color=auto "error" "${REPO_ROOT}/log/" || true
+            grep -Rni --color=auto "error" "${REPO_ROOT}/log/" --exclude="pytest_*_error.log" || true
         else
             echo "${REPO_ROOT}/log directory not found"
         fi
diff --git a/scripts/run_pre_ce.sh b/scripts/run_pre_ce.sh
@@ -38,7 +38,7 @@ for subdir in "$run_path"*/; do
                 if [ $exit_code -ne 0 ]; then
                     if [ -d "${subdir%/}/log" ]; then
                         echo ">>> grep error in ${subdir%/}/log/"
-                        grep -Rni --color=auto "error" "${subdir%/}/log/" || true
+                        grep -Rni --color=auto "error" "${subdir%/}/log/" --exclude="pytest_*_error.log" || true
                     else
                         echo "${subdir%/}/log directory not found"
                     fi
diff --git a/tests/conftest.py b/tests/conftest.py
diff --git a/tests/e2e/4cards_cases/_test_Qwen3_30b_tp4.py b/tests/e2e/4cards_cases/_test_Qwen3_30b_tp4.py

Original file line number	Diff line number	Diff line change
`@@ -201,7 +201,7 @@ jobs:`
`201`	`201`	`--gpus "\"device=${DEVICES}\"" ${docker_image} /bin/bash -c '`
`202`	`202`	`git config --global --add safe.directory /workspace/FastDeploy`
`203`	`203`	`cd FastDeploy`
`204`		`- python -m pip install --pre paddlepaddle-gpu -i https://www.paddlepaddle.org.cn/packages/nightly/cu126/`
	`204`	`+ python -m pip install paddlepaddle-gpu==3.5.0.dev20260508 -i https://www.paddlepaddle.org.cn/packages/nightly/cu126/`
`205`	`205`	`python -m pip install ${fd_wheel_url}`
`206`	`206`	`bash scripts/run_pre_ce.sh`
`207`	`207`	`'`