fix(tests): skip crashing torch ops on ascend (#614)

* fix(tests): skip crashing torch ops on ascend

* ci: update CI submodule for dynamic device allocation

* fix(tests): skip smooth l1 loss on ascend

* ci: use host-level device leases

* ci: route local runners through leases

* ci: use agent local CI tools

* fix(tests): narrow Ascend torch op crash skips

* fix(tests): skip Ascend soft margin loss crash

* fix(tests): restore Ascend smooth l1 crash skip

* ci: stop queue watchdog after jobs start

* ci: require Ascend device lease before docker run

* ci: remove queued job watchdog checks

* fix(ci): keep iluvatar corex compiler in image

* fix(ci): pass platform to v2 shadow runner

---------

Co-authored-by: zhangyue <zhangyue@localhost.localdomain>

Author: zhangyue207

.ci

@@ -47,7 +47,6 @@ platforms:
       - "--ipc=host"
     volumes:
       - /dev:/dev
-      - /usr/local/corex-4.3.0/bin:/usr/local/corex-4.3.0.20250624/bin:ro
       - /lib/firmware:/lib/firmware
       - /usr/src:/usr/src
       - /lib/modules:/lib/modules

.github/ci_config.yml

@@ -27,10 +27,10 @@ on:
 
 jobs:
   ci:
-    uses: InfiniTensor/ci/.github/workflows/infiniops-ci.yml@c6bf369739f16e759f46fd466f586c51c59f26c3
+    uses: InfiniTensor/ci/.github/workflows/infiniops-ci.yml@45d1046ec42ea73d1a75d5a96eb1a8204d47cbcd
     with:
       config_path: .github/ci_config.yml
-      ci_ref: c6bf369739f16e759f46fd466f586c51c59f26c3
+      ci_ref: 45d1046ec42ea73d1a75d5a96eb1a8204d47cbcd
       max_parallel: 10
       platform: ${{ github.event_name == 'workflow_dispatch' && (inputs.platform == 'all' && 'nvidia,iluvatar,metax,moore,cambricon,ascend' || inputs.platform) || 'nvidia,iluvatar,metax,moore,cambricon,ascend' }}
     secrets: inherit

.github/workflows/ci_test.yml

@@ -28,10 +28,10 @@ on:
 
 jobs:
   ci-v2-shadow:
-    uses: InfiniTensor/ci/.github/workflows/infiniops-ci-v2-shadow.yml@c6bf369739f16e759f46fd466f586c51c59f26c3
+    uses: InfiniTensor/ci/.github/workflows/infiniops-ci-v2-shadow.yml@45d1046ec42ea73d1a75d5a96eb1a8204d47cbcd
     with:
       config_path: .github/ci_config.yml
-      ci_ref: c6bf369739f16e759f46fd466f586c51c59f26c3
+      ci_ref: 45d1046ec42ea73d1a75d5a96eb1a8204d47cbcd
       max_parallel: 10
       platform: ${{ github.event_name == 'workflow_dispatch' && (inputs.platform == 'all' && 'nvidia,iluvatar,metax,moore,cambricon,ascend' || inputs.platform == 'active' && 'nvidia,iluvatar,metax,moore,cambricon,ascend' || inputs.platform) || 'nvidia,iluvatar,metax,moore,cambricon,ascend' }}
     secrets: inherit

.github/workflows/ci_v2_shadow.yml

@@ -208,7 +208,11 @@ def _list_default(aten_type):
 _VENDOR_CRASH_OPS = frozenset(
     {
         ("npu", "mish"),
+        ("npu", "mse_loss"),
+        ("npu", "nonzero"),
         ("npu", "nuclear_norm"),
+        ("npu", "smooth_l1_loss"),
+        ("npu", "soft_margin_loss"),
         ("npu", "_linalg_svd"),
         ("npu", "svd"),
     }