diff --git a/README.md b/README.md index e92eb7d9..b7603bf2 100644 --- a/README.md +++ b/README.md @@ -27,12 +27,15 @@ paddle.concat(tuple(Tensor([31376, 768],"float32"),Tensor([1, 768],"float32"),), ```bash ├── report/ +├── scripts/ +│ └── sync_watch.py ├── test_pipline/ ├── tester/ │ ├── api_config/ │ ├── paddle_to_torch/ │ ├── accuracy.py │ ├── base.py +│ ├── http_server.py │ ├── paddle_cinn_vs_dygraph.py │ └── paddle_only.py ├── tools/ diff --git a/engineV2-README.md b/engineV2-README.md index 6de9eb82..3c4e8ca3 100644 --- a/engineV2-README.md +++ b/engineV2-README.md @@ -87,7 +87,7 @@ | `--show_runtime_status` | bool | 是否实时显示当前的测试进度（默认 True） | | `--random_seed` | int | numpy random的随机种子(默认为0，此时不会显式设置numpy random的seed) | | `--custom_device_vs_gpu` | bool | 启用自定义设备与GPU的精度对比测试模式（默认 False） | -| `--custom_device_vs_gpu_mode` | str | 自定义设备与GPU对比的模式：`upload` 或 `download`（默认 `upload`） | +| `--custom_device_vs_gpu_mode` | str | 自定义设备与GPU对比的模式：`upload`、`download` 或 `http`（默认 `upload`） | | `--bitwise_alignment` | bool | 是否进行诸位对齐对比，开启后所有的api的精度对比都按照atol=0.0,rtol = 0.0的精度对比结果(默认False)| | `--generate_failed_tests` | bool | 是否为失败的测试用例生成可复现的测试文件。开启后，当测试失败时，会在`failed_tests`目录下生成独立的Python测试文件，便于后续复现和调试（默认False）| | `--exit_on_error` | bool | 是否在精度测试出现`paddle_error`或者 `accuracy_error` 错误时立即退出测试进程(exit code 为1)。默认为False，测试进程会继续执行 | @@ -128,58 +128,36 @@ python engineV2.py --accuracy=True --api_config_file="tester/api_config/api_conf ### 自定义设备与 GPU 精度对比测试 -#### 功能说明 +该功能支持跨设备的精度对比测试，提供两种数据传输方式：**BOS 云存储中转**和 **HTTP 直连**。 -`APITestPaddleDeviceVSGPU` 类支持跨设备的精度对比测试，目前主要面向 **GPU 上传 + XPU（或其他设备）下载对比** 这一典型场景。该功能分为两个模式： +#### 方式一：BOS 云存储中转（upload/download 模式） -- **Upload 模式（GPU 侧）**：在 GPU 上执行测试，保存结果到本地，然后上传到 BOS 云存储 -- **Download 模式（XPU/其他设备侧）**：在 XPU 或其他设备上执行测试，从 BOS 下载 GPU 侧的参考数据进行精度对比 +分为两步操作：先在一台机器上 upload，再在另一台机器上 download 对比。 -#### 工作流程 +**工作流程**： -1. **Upload 模式工作流（GPU 侧）**： - - 在 GPU 设备上执行 Paddle API 测试 - - 保存 Forward 输出和 Backward 梯度到本地 PDTensor 文件 - - 文件名依赖随机种子与配置哈希（如 `1210-xxx.pdtensor`） - - 使用 bcecmd 工具将文件上传到 BOS 云存储 +1. **Upload 模式（GPU 侧）**：在 GPU 上执行 Paddle API，保存 Forward 输出和 Backward 梯度为 PDTensor 文件，上传到 BOS +2. **Download 模式（XPU/其他设备侧）**：在目标设备上执行相同 API，从 BOS 下载 GPU 参考数据，进行精度对比 -2. **Download 模式工作流（XPU/其他设备侧）**： - - 在 XPU 或其他设备上执行相同的 Paddle API 测试 - - 使用与 GPU 侧上传时一致的随机种子和配置，构造同名 PDTensor 文件名 - - 从 BOS 云存储下载对应的 GPU 参考数据 - - 对比 Forward 输出和 Backward 梯度，验证与 GPU 的精度一致性 - -#### 配置文件设置 - -首先，编辑 `tester/bos_config.yaml` 配置文件： +**配置文件**：编辑 `tester/bos_config.yaml` ```yaml -# BOS 配置文件 -# 用于自定义设备与 GPU 精度对比测试的云存储配置 - -# BOS 存储路径（如：xly-devops/liujingzong/） bos_path: "xly-devops/liujingzong/" - -# BOS 配置文件路径（bcecmd 使用的配置文件路径） bos_conf_path: "./conf" - -# bcecmd 命令行工具路径 bcecmd_path: "./bcecmd" ``` -#### 命令示例 -**在 GPU 上执行测试并上传结果** +**命令示例**： + ```bash -# 在 GPU 设备上执行，生成1210-xxx.pdtensor 文件并上传到 BOS +# GPU 侧：执行并上传 python engineV2.py --custom_device_vs_gpu=True \ --custom_device_vs_gpu_mode=upload \ --random_seed=1210 \ --api_config_file="./test1.txt" \ --gpu_ids=7 -``` -**在 XPU 上下载 GPU 的参考数据并进行精度对比** -```bash +# XPU 侧：下载并对比 python engineV2.py --custom_device_vs_gpu=True \ --custom_device_vs_gpu_mode=download \ --random_seed=1210 \ @@ -187,6 +165,133 @@ python engineV2.py --custom_device_vs_gpu=True \ --gpu_ids=7 ``` +#### 方式二：HTTP 直连（http 模式） + +只需在本地执行一条命令，自动触发远端执行并拉取结果对比，无需 BOS 服务。 + +**工作流程**： + +1. 在远端机器启动 HTTP 服务器（带多 GPU 进程池） +2. 本地发送 API 配置到远端，远端执行后返回 PDTensor 结果 +3. 本地执行同一 API，与远端结果进行精度对比 + +**第一步：在远端机器启动服务器** + +```bash +cd /path/to/PaddleAPITest +python -m tester.http_server --host 0.0.0.0 --port 8089 --num_gpus=-1 +``` + +服务器参数： + +| 参数 | 默认值 | 说明 | +|---|---|---| +| `--host` | `0.0.0.0` | 监听地址 | +| `--port` | `8089` | 监听端口 | +| `--num_gpus` | `-1` | GPU 数量，`-1` 表示全部 | +| `--num_workers_per_gpu` | `1` | 每张 GPU 的 worker 数 | +| `--required_memory` | `10.0` | 每个 worker 最低显存（GB） | +| `--gpu_ids` | `""` | 指定 GPU，如 `6,7` 或 `0-3` | +| `--timeout` | `1800` | 单个 API 执行超时（秒） | +| `--admin_token` | `""` | 若非空，启用 `/admin/*` 管理接口（见下方"远程代码同步"）| + +可通过健康检查确认服务状态： + +```bash +curl http://<远端IP>:8089/health +# {"status": "ok", "device_type": "gpu", "paddle_version": "3.0.0"} +``` + +**第二步：在本地配置远端地址** + +编辑 `tester/http_config.yaml`： + +```yaml +remote_host: "10.78.119.13" # 远端机器 IP +remote_port: 8089 # 远端服务端口 +timeout: 300 # 单次请求超时（秒） +``` + +**第三步：在本地执行对比** + +```bash +# 单个 API 测试 +python engineV2.py --custom_device_vs_gpu=True \ + --custom_device_vs_gpu_mode=http \ + --random_seed=42 \ + --api_config='paddle.abs(Tensor([2, 3], "float32"))' + +# 批量测试 +python engineV2.py --custom_device_vs_gpu=True \ + --custom_device_vs_gpu_mode=http \ + --random_seed=42 \ + --api_config_file_pattern="tester/api_config/5_accuracy/*.txt" \ + --num_gpus=-1 +``` + +**AMP（自动混合精度）模式**： + +加入 `--test_amp=True` 后，本地设备侧和远端 GPU server 侧会**同步**在 `paddle.amp.auto_cast()` 上下文下执行，确保两端精度对比处于相同的混合精度环境中： + +```bash +python engineV2.py --custom_device_vs_gpu=True \ + --custom_device_vs_gpu_mode=http \ + --test_amp=True \ + --random_seed=42 \ + --api_config_file="tester/api_config/6_accuracy_amp/accuracy_amp.txt" +``` + +`test_amp` 标志会随请求 payload 一并发送到 server，因此无需在 server 启动命令中做任何额外配置。 + +**并发机制**： + +服务端通过三层机制处理并发请求： +- **ThreadingHTTPServer**：每个请求一个线程，并行接收 +- **Semaphore**：限制同时排队+执行的请求数为 `worker数 × 2`，超出则阻塞等待 +- **ProcessPool**：实际执行进程数由 GPU 数和 workers_per_gpu 决定 + +当客户端 worker 多于服务端 worker 时，多余的请求会排队等待，客户端的 `http_timeout` 作为最终兜底——超时后写入 `timeout` 日志，确保不会出现"没跑也没日志"的情况。 + +> 详细的设计原理和异常处理说明见 [docs/http_cross_device_comparison.md](docs/http_cross_device_comparison.md)。 + +#### 远程代码同步（sync_watch） + +在两台机器间 SSH 不通的情况下，可通过 `scripts/sync_watch.py` 将本地代码变更**自动同步**到远端服务器，并触发服务器重启，无需手动操作。 + +**前提：远端启动时带上 `--admin_token`** + +```bash +python -m tester.http_server --host 0.0.0.0 --port 8089 --gpu_ids=6 --admin_token=your_token +``` + +**本地安装依赖（一次性）** + +```bash +pip install watchdog +``` + +**本地启动监听** + +```bash +python scripts/sync_watch.py \ + --host <远端IP> --port 8089 --token your_token +``` + +启动后，每当本地 `.py` 文件被保存，脚本会在约 1.5 秒防抖后： +1. 将变更文件通过 `POST /admin/upload_file` 推送到远端 +2. 调用 `POST /admin/restart` 触发服务器原地重启 +3. 轮询 `/health` 直至服务器重新就绪 + +可选参数： + +| 参数 | 默认值 | 说明 | +|---|---|---| +| `--host` | 必填 | 远端服务器 IP | +| `--port` | `8089` | 远端服务端口 | +| `--token` | 必填 | 与 `--admin_token` 一致 | +| `--watch_dir` | 仓库根目录 | 本地监听目录 | +| `--debounce` | `1.5` | 防抖时间（秒）| + ## 监控方法 执行 `run.sh` 后可通过以下方式监控： diff --git a/engineV2.py b/engineV2.py index 4ac0d58a..9f78de48 100644 --- a/engineV2.py +++ b/engineV2.py @@ -56,6 +56,9 @@ "generate_failed_tests", "bitwise_alignment", "exit_on_error", + "http_host", + "http_port", + "http_timeout", } DEVICE_TYPE = None @@ -675,9 +678,9 @@ def main(): parser.add_argument( "--custom_device_vs_gpu_mode", type=str, - choices=["upload", "download"], + choices=["upload", "download", "http"], default="upload", - help="operation mode for custom_device_vs_gpu: 'upload' or 'download'", + help="operation mode for custom_device_vs_gpu: 'upload', 'download', or 'http'", ) parser.add_argument( "--bitwise_alignment", @@ -697,6 +700,13 @@ def main(): default=False, help="Whether to exit the process when a paddle_error occurs.", ) + parser.add_argument( + "--enable_api_kernel_fallback", + type=parse_bool, + default=False, + help="(device_vs_gpu only) Set FLAGS_enable_api_kernel_fallback=1 on the local device. " + "Has no effect on the remote GPU server.", + ) options = parser.parse_args() print(f"Options: {vars(options)}", flush=True) @@ -733,41 +743,82 @@ def main(): # 处理 custom_device_vs_gpu 模式的配置 bos_config_data = None if options.custom_device_vs_gpu: - # 读取 BOS 配置文件（固定路径：tester/bos_config.yaml） - bos_config_path = Path("tester/bos_config.yaml") - if not bos_config_path.exists(): - print(f"BOS config file not found: {bos_config_path}", flush=True) - return + if options.custom_device_vs_gpu_mode == "http": + # 读取 HTTP 配置文件 + http_config_path = Path("tester/http_config.yaml") + if not http_config_path.exists(): + print(f"HTTP config file not found: {http_config_path}", flush=True) + return - try: - with open(bos_config_path, encoding="utf-8") as f: - bos_config_data = yaml.safe_load(f) + try: + with open(http_config_path, encoding="utf-8") as f: + http_config_data = yaml.safe_load(f) + + if not http_config_data: + print(f"HTTP config file is empty: {http_config_path}", flush=True) + return + + required_keys = ["remote_host", "remote_port"] + missing_keys = [key for key in required_keys if key not in http_config_data] + if missing_keys: + print(f"Missing required keys in HTTP config: {missing_keys}", flush=True) + return + + options.operation_mode = "http" + options.http_host = http_config_data["remote_host"] + options.http_port = http_config_data.get("remote_port", 8089) + options.http_timeout = http_config_data.get("timeout", 300) + # BOS config not needed in HTTP mode + options.bos_path = "" + options.bos_conf_path = "" + options.bcecmd_path = "" - if not bos_config_data: - print(f"BOS config file is empty: {bos_config_path}", flush=True) + except Exception as e: + print(f"Failed to load HTTP config file {http_config_path}: {e}", flush=True) return - - # 验证必需的配置项 - required_keys = ["bos_path", "bos_conf_path", "bcecmd_path"] - missing_keys = [key for key in required_keys if key not in bos_config_data] - if missing_keys: - print(f"Missing required keys in BOS config: {missing_keys}", flush=True) + else: + # 读取 BOS 配置文件（固定路径：tester/bos_config.yaml） + bos_config_path = Path("tester/bos_config.yaml") + if not bos_config_path.exists(): + print(f"BOS config file not found: {bos_config_path}", flush=True) return - # 将配置添加到 options 中，以便传递给测试类 - options.operation_mode = options.custom_device_vs_gpu_mode - options.bos_path = bos_config_data["bos_path"] - options.bos_conf_path = bos_config_data["bos_conf_path"] - options.bcecmd_path = bos_config_data["bcecmd_path"] + try: + with open(bos_config_path, encoding="utf-8") as f: + bos_config_data = yaml.safe_load(f) + + if not bos_config_data: + print(f"BOS config file is empty: {bos_config_path}", flush=True) + return + + # 验证必需的配置项 + required_keys = ["bos_path", "bos_conf_path", "bcecmd_path"] + missing_keys = [key for key in required_keys if key not in bos_config_data] + if missing_keys: + print(f"Missing required keys in BOS config: {missing_keys}", flush=True) + return + + # 将配置添加到 options 中，以便传递给测试类 + options.operation_mode = options.custom_device_vs_gpu_mode + options.bos_path = bos_config_data["bos_path"] + options.bos_conf_path = bos_config_data["bos_conf_path"] + options.bcecmd_path = bos_config_data["bcecmd_path"] - except Exception as e: - print(f"Failed to load BOS config file {bos_config_path}: {e}", flush=True) - return + except Exception as e: + print(f"Failed to load BOS config file {bos_config_path}: {e}", flush=True) + return if options.test_tol and not options.accuracy: print("--test_tol takes effect when --accuracy is True.", flush=True) if options.test_backward and not options.paddle_cinn: print("--test_backward takes effect when --paddle_cinn is True.", flush=True) + if options.custom_device_vs_gpu and options.enable_api_kernel_fallback: + os.environ["FLAGS_enable_api_kernel_fallback"] = "1" + print("[device_vs_gpu] FLAGS_enable_api_kernel_fallback=1 (local only)", flush=True) + if options.gpu_ids and options.gpu_ids != "-1": + first_gpu = options.gpu_ids.split(",")[0].strip() + os.environ["CUDA_VISIBLE_DEVICES"] = first_gpu + print(f"CUDA_VISIBLE_DEVICES={first_gpu}", flush=True) os.environ["USE_CACHED_NUMPY"] = str(options.use_cached_numpy) if options.bitwise_alignment: options.atol = 0.0 @@ -818,22 +869,32 @@ def main(): APITestAccuracy, # default fallback ) - if options.test_cpu: - import paddle - - paddle.device.set_device("cpu") if options.custom_device_vs_gpu: # custom_device_vs_gpu 模式需要传递额外参数 - case = test_class( - api_config, - operation_mode=options.operation_mode, - bos_path=options.bos_path, - bos_conf_path=options.bos_conf_path, - bcecmd_path=options.bcecmd_path, - random_seed=options.random_seed, - atol=options.atol, - rtol=options.rtol, - ) + kwargs = { + "operation_mode": options.operation_mode, + "random_seed": options.random_seed, + "atol": options.atol, + "rtol": options.rtol, + "test_amp": options.test_amp, + } + if options.operation_mode == "http": + kwargs.update( + { + "http_host": options.http_host, + "http_port": options.http_port, + "http_timeout": options.http_timeout, + } + ) + else: + kwargs.update( + { + "bos_path": options.bos_path, + "bos_conf_path": options.bos_conf_path, + "bcecmd_path": options.bcecmd_path, + } + ) + case = test_class(api_config, **kwargs) elif options.accuracy: case = test_class( api_config, diff --git a/scripts/sync_watch.py b/scripts/sync_watch.py new file mode 100644 index 00000000..84a31a5c --- /dev/null +++ b/scripts/sync_watch.py @@ -0,0 +1,293 @@ +"""Local file watcher that syncs .py changes to a remote HTTP server. + +Watches the local PaddleAPITest directory for .py file changes and: +1. Uploads each changed file via POST /admin/upload_file +2. Triggers a server restart via POST /admin/restart +3. Polls /health until the server is back up + +Usage: + pip install watchdog # one-time setup + python scripts/sync_watch.py \\ + --host 10.78.119.13 --port 8089 --token \\ + [--watch_dir /workspace/PaddleAPITest] \\ + [--debounce 1.5] +""" + +from __future__ import annotations + +import argparse +import json +import os +import threading +import time +import urllib.error +import urllib.request +from pathlib import Path + +try: + from watchdog.events import FileSystemEventHandler + from watchdog.observers import Observer +except ImportError: + print("watchdog is required. Install it with: pip install watchdog") + raise + +# Patterns matching paths that should NOT be synced (mirrors .gitignore) +SKIP_PATTERNS = [ + "__pycache__", + ".mypy_cache", + ".vscode", + "test_log", + "api_config/api_config", + "api_config/output", + ".ipynb_checkpoints", + "trace_output", + ".huggingface", +] + + +class SyncHandler(FileSystemEventHandler): + def __init__(self, args: argparse.Namespace, watch_dir: str) -> None: + super().__init__() + self.args = args + self.watch_dir = watch_dir + self._pending: set[str] = set() + self._pending_deletes: set[str] = set() + self._timer: threading.Timer | None = None + self._lock = threading.Lock() + + # ------------------------------------------------------------------ # + # watchdog callbacks # + # ------------------------------------------------------------------ # + + def on_modified(self, event): + self._enqueue(event) + + def on_created(self, event): + self._enqueue(event) + + def on_moved(self, event): + # Editors like Claude Code use atomic writes: write to a temp file then + # rename() to the target. This generates a "moved" event on the dest + # path, not a "modified" event. We enqueue the destination file. + if event.is_directory: + return + dest: str = event.dest_path + if not dest.endswith(".py"): + return + if any(p in dest for p in SKIP_PATTERNS): + return + with self._lock: + self._pending.add(dest) + if self._timer is not None: + self._timer.cancel() + self._timer = threading.Timer(self.args.debounce, self._flush) + self._timer.start() + + def on_deleted(self, event): + if event.is_directory: + return + path: str = event.src_path + if not path.endswith(".py"): + return + if any(p in path for p in SKIP_PATTERNS): + return + with self._lock: + self._pending_deletes.add(path) + # Remove from pending uploads if it was queued + self._pending.discard(path) + if self._timer is not None: + self._timer.cancel() + self._timer = threading.Timer(self.args.debounce, self._flush) + self._timer.start() + + def _enqueue(self, event): + if event.is_directory: + return + path: str = event.src_path + if not path.endswith(".py"): + return + if any(p in path for p in SKIP_PATTERNS): + return + with self._lock: + self._pending.add(path) + if self._timer is not None: + self._timer.cancel() + self._timer = threading.Timer(self.args.debounce, self._flush) + self._timer.start() + + # ------------------------------------------------------------------ # + # sync + restart # + # ------------------------------------------------------------------ # + + def _flush(self): + with self._lock: + paths = list(self._pending) + deletes = list(self._pending_deletes) + self._pending.clear() + self._pending_deletes.clear() + self._timer = None + + deleted = [] + for path in deletes: + if self._delete_file(path): + rel = os.path.relpath(path, self.watch_dir) + deleted.append(rel) + + uploaded = [] + for path in paths: + if self._upload_file(path): + rel = os.path.relpath(path, self.watch_dir) + uploaded.append(rel) + + if deleted: + print(f"[sync] Deleted {len(deleted)} file(s): {', '.join(deleted)}", flush=True) + if uploaded: + print(f"[sync] Uploaded {len(uploaded)} file(s): {', '.join(uploaded)}", flush=True) + if deleted or uploaded: + self._trigger_restart() + + def _delete_file(self, abs_path: str) -> bool: + rel = os.path.relpath(abs_path, self.watch_dir) + body = json.dumps({"path": rel}).encode("utf-8") + req = urllib.request.Request( + f"http://{self.args.host}:{self.args.port}/admin/delete_file", + data=body, + headers={ + "Content-Type": "application/json", + "X-Admin-Token": self.args.token, + }, + method="POST", + ) + try: + with urllib.request.urlopen(req, timeout=15) as resp: + result = json.loads(resp.read().decode()) + if result.get("status") != "ok": + print(f"[sync] Delete failed for {rel}: {result}", flush=True) + return False + return True + except urllib.error.HTTPError as e: + body_text = e.read().decode(errors="replace") + print(f"[sync] Delete HTTP error {e.code} for {rel}: {body_text}", flush=True) + return False + except Exception as e: + print(f"[sync] Delete error for {rel}: {e}", flush=True) + return False + + def _upload_file(self, abs_path: str) -> bool: + rel = os.path.relpath(abs_path, self.watch_dir) + try: + content = Path(abs_path).read_text(encoding="utf-8") + except Exception as e: + print(f"[sync] Read error for {rel}: {e}", flush=True) + return False + + body = json.dumps({"path": rel, "content": content}).encode("utf-8") + req = urllib.request.Request( + f"http://{self.args.host}:{self.args.port}/admin/upload_file", + data=body, + headers={ + "Content-Type": "application/json", + "X-Admin-Token": self.args.token, + }, + method="POST", + ) + try: + with urllib.request.urlopen(req, timeout=15) as resp: + result = json.loads(resp.read().decode()) + if result.get("status") != "ok": + print(f"[sync] Upload failed for {rel}: {result}", flush=True) + return False + return True + except urllib.error.HTTPError as e: + body_text = e.read().decode(errors="replace") + print(f"[sync] Upload HTTP error {e.code} for {rel}: {body_text}", flush=True) + return False + except Exception as e: + print(f"[sync] Upload error for {rel}: {e}", flush=True) + return False + + def _trigger_restart(self): + req = urllib.request.Request( + f"http://{self.args.host}:{self.args.port}/admin/restart", + data=b"{}", + headers={ + "Content-Type": "application/json", + "X-Admin-Token": self.args.token, + }, + method="POST", + ) + try: + with urllib.request.urlopen(req, timeout=10) as resp: + result = json.loads(resp.read().decode()) + print(f"[sync] Restart triggered: {result}", flush=True) + except Exception as e: + print(f"[sync] Restart request failed: {e}", flush=True) + return + + # Poll /health until the server comes back up (max 60 seconds) + health_url = f"http://{self.args.host}:{self.args.port}/health" + print("[sync] Waiting for server to come back up...", flush=True) + for i in range(60): + time.sleep(1) + try: + with urllib.request.urlopen(health_url, timeout=3) as r: + data = json.loads(r.read().decode()) + if data.get("status") == "ok": + print(f"[sync] Server ready (attempt {i + 1}).", flush=True) + return + except Exception: + pass + print("[sync] Warning: server did not come back up within 60 seconds.", flush=True) + + +def main(): + parser = argparse.ArgumentParser( + description="Watch local .py files and sync changes to the remote HTTP server." + ) + parser.add_argument("--host", required=True, help="Remote server host, e.g. 10.78.119.13") + parser.add_argument("--port", type=int, default=8089, help="Remote server port") + parser.add_argument("--token", required=True, help="Admin token (--admin_token on server)") + parser.add_argument( + "--watch_dir", + default=str(Path(__file__).resolve().parent.parent), + help="Local directory to watch (default: repo root)", + ) + parser.add_argument( + "--debounce", + type=float, + default=1.5, + help="Seconds to wait after last change before syncing (default: 1.5)", + ) + args = parser.parse_args() + + watch_dir = os.path.abspath(args.watch_dir) + print(f"[sync] Watching {watch_dir}", flush=True) + print(f"[sync] Remote: http://{args.host}:{args.port}", flush=True) + print(f"[sync] Debounce: {args.debounce}s", flush=True) + + # Verify connectivity before starting the observer + health_url = f"http://{args.host}:{args.port}/health" + try: + with urllib.request.urlopen(health_url, timeout=5) as r: + data = json.loads(r.read().decode()) + print(f"[sync] Server OK: {data}", flush=True) + except Exception as e: + print(f"[sync] Warning: cannot reach server at {health_url}: {e}", flush=True) + + event_handler = SyncHandler(args, watch_dir) + observer = Observer() + observer.schedule(event_handler, watch_dir, recursive=True) + observer.start() + print("[sync] Observer started. Press Ctrl+C to stop.", flush=True) + + try: + while observer.is_alive(): + observer.join(timeout=1) + except KeyboardInterrupt: + print("[sync] Stopping...", flush=True) + observer.stop() + observer.join() + + +if __name__ == "__main__": + main() diff --git a/tester/api_config/config_analyzer.py b/tester/api_config/config_analyzer.py index fb9372d9..dfe48ad4 100644 --- a/tester/api_config/config_analyzer.py +++ b/tester/api_config/config_analyzer.py @@ -405,12 +405,12 @@ def safe_range(low, high): ): if step_val.numpy_tensor.item() > 0: step_val.numpy_tensor = numpy.random.uniform( - 1.0, 5.0, step_config.shape - ).astype(step_config.dtype) + 1.0, 5.0, step_val.shape + ).astype(step_val.dtype) else: step_val.numpy_tensor = numpy.random.uniform( - -5.0, -1.0, step_config.shape - ).astype(step_config.dtype) + -5.0, -1.0, step_val.shape + ).astype(step_val.dtype) elif api_config.api_name in { "paddle.argmax", @@ -1605,10 +1605,13 @@ def get_padding_offset(bsz, max_seq_len, seq_lens_this_time): elif api_config.api_name == "paddle.nn.functional.pad": if self.check_arg(api_config, 1, "pad"): x_shape = self.get_arg(api_config, 0, "x").shape - min_dim_len = min(x_shape) - self.numpy_tensor = self.get_random_numpy_tensor( - shape=self.shape, data_type=self.dtype, min=0, max=min_dim_len - ) + min_dim_len = min(x_shape) if x_shape else 1 + if min_dim_len <= 0: + self.numpy_tensor = numpy.zeros(self.shape, dtype=self.dtype) + else: + self.numpy_tensor = self.get_random_numpy_tensor( + shape=self.shape, data_type=self.dtype, min=0, max=min_dim_len + ) elif api_config.api_name == "paddle.nn.functional.class_center_sample": if self.check_arg(api_config, 0, "label"): num_classes = self.get_arg(api_config, 1, "num_classes") @@ -2234,9 +2237,12 @@ def get_padding_offset(bsz, max_seq_len, seq_lens_this_time): if self.check_arg(api_config, 1, "index"): x = self.get_arg(api_config, 0, "x") dim_size = numpy.prod(x.shape) - self.numpy_tensor = numpy.random.randint(0, dim_size, size=self.shape).astype( - self.dtype - ) + if dim_size <= 0: + self.numpy_tensor = numpy.zeros(self.shape, dtype=self.dtype) + else: + self.numpy_tensor = numpy.random.randint( + 0, dim_size, size=self.shape + ).astype(self.dtype) elif api_config.api_name in {"paddle.Tensor.gather", "paddle.gather"}: if key == "index" or index == 1: @@ -2512,6 +2518,9 @@ def get_base_max(value, dtype_max, default_max=5): # value**(-max) < MAX => (1/value)**max < MAX value = 1 / value ln_value = math.log(value) + if ln_value == 0: + # value == 1: x^1 = x, gradient has no overflow constraint + return default_max # dy/dx = y*ln(value) < MAX, y < MAX => y*max(ln(value), 1) < MAX output_max = dtype_max / max(1, ln_value) value_max = math.log(output_max) / ln_value @@ -2554,7 +2563,12 @@ def get_exponent_max(value, dtype_max, default_max=5): get_max = get_exponent_max default_max = 5 if isinstance(const, (int, float, bool, numpy.number)): - value_max = get_max(const, numpy.finfo(self.dtype).max, default_max) + _dtype_max = ( + numpy.iinfo(self.dtype).max + if "int" in self.dtype + else numpy.finfo(self.dtype).max + ) + value_max = get_max(const, _dtype_max, default_max) if is_base_arg and int(const) != const: # Avoid situations like (-2.3) ^ 0.5 self.numpy_tensor = self.get_random_numpy_tensor( diff --git a/tester/api_config/log_writer.py b/tester/api_config/log_writer.py index 4c989caf..b477cf75 100644 --- a/tester/api_config/log_writer.py +++ b/tester/api_config/log_writer.py @@ -21,6 +21,7 @@ "pass": "api_config_pass", "numpy_error": "api_config_numpy_error", "paddle_error": "api_config_paddle_error", + "remote_error": "api_config_remote_error", "torch_error": "api_config_torch_error", "paddle_to_torch_failed": "api_config_paddle_to_torch_failed", "accuracy_error": "api_config_accuracy_error", @@ -30,6 +31,8 @@ "oom": "api_config_oom", "match_error": "api_config_match_error", "cuda_error": "api_config_cuda_error", + "network_error": "api_config_network_error", + "skip": "api_config_skip", } _is_engineV2 = False @@ -340,14 +343,19 @@ def aggregate_logs(end=False): except Exception as err: print(f"Error reading {log_file}: {err}", flush=True) + # api_configs 现在是 checkpoint 中既不在 pass/error/crash 也不在 + # write_to_log("skip",...) 里的条目（因为 skip 在上方循环中已被减掉）。 + # api_config_skip.txt 里已有 write_to_log("skip",...) 写入的条目， + # 我们只需把差集（未分类的遗漏条目）追加进去，并更新计数。 + skip_file = TEST_LOG_PATH / "api_config_skip.txt" + existing_skip_count = log_counts.get("skip", 0) if api_configs: - log_counts["skip"] = len(api_configs) - skip_file = TEST_LOG_PATH / "api_config_skip.txt" try: - with skip_file.open("w") as f: + with skip_file.open("a") as f: f.writelines(f"{line}\n" for line in sorted(api_configs)) except Exception as err: print(f"Error writing to {skip_file}: {err}", flush=True) + log_counts["skip"] = existing_skip_count + len(api_configs) return log_counts @@ -361,6 +369,7 @@ def print_log_info(all_case, log_counts=None): log_counts.get(log_type, 0) for log_type in [ "paddle_error", + "remote_error", "accuracy_error", "accuracy_diff", "timeout", @@ -372,6 +381,7 @@ def print_log_info(all_case, log_counts=None): skip_case = sum( log_counts.get(log_type, 0) for log_type in [ + "skip", "numpy_error", "torch_error", "paddle_to_torch_failed", diff --git a/tester/base.py b/tester/base.py index f5d710f8..7ba73d6f 100644 --- a/tester/base.py +++ b/tester/base.py @@ -96,19 +96,22 @@ def need_skip(self, paddle_only=False): return True for i in range(len(self.api_config.args)): if isinstance(self.api_config.args[i], TensorConfig): - if self.api_config.args[i].dtype in ["float8_e5m2", "float8_e4m3fn"]: + if not paddle_only and self.api_config.args[i].dtype in [ + "float8_e5m2", + "float8_e4m3fn", + ]: return True elif isinstance(self.api_config.args[i], list) or isinstance( self.api_config.args[i], tuple ): for j in range(len(self.api_config.args[i])): if isinstance(self.api_config.args[i][j], TensorConfig): - if self.api_config.args[i][j].dtype in [ + if not paddle_only and self.api_config.args[i][j].dtype in [ "float8_e5m2", "float8_e4m3fn", ]: return True - elif self.api_config.args[i] in [ + elif not paddle_only and self.api_config.args[i] in [ paddle.base.core.DataType.FLOAT8_E4M3FN, paddle.base.core.DataType.FLOAT8_E5M2, "float8_e5m2", @@ -118,14 +121,17 @@ def need_skip(self, paddle_only=False): for _key, arg_config in self.api_config.kwargs.items(): if isinstance(arg_config, TensorConfig): - if arg_config.dtype in ["float8_e5m2", "float8_e4m3fn"]: + if not paddle_only and arg_config.dtype in ["float8_e5m2", "float8_e4m3fn"]: return True elif isinstance(arg_config, (list, tuple)): for i in range(len(arg_config)): if isinstance(arg_config[i], TensorConfig): - if arg_config[i].dtype in ["float8_e5m2", "float8_e4m3fn"]: + if not paddle_only and arg_config[i].dtype in [ + "float8_e5m2", + "float8_e4m3fn", + ]: return True - elif arg_config in [ + elif not paddle_only and arg_config in [ paddle.base.core.DataType.FLOAT8_E4M3FN, paddle.base.core.DataType.FLOAT8_E5M2, "float8_e5m2", @@ -631,7 +637,11 @@ def gen_paddle_output_and_output_grad(self, outputs): for output in result_outputs: dtype = str(output.dtype).split(".")[-1] if USE_CACHED_NUMPY: - dtype = "float32" if dtype == "bfloat16" else dtype + dtype = ( + "float32" + if dtype in ["bfloat16", "float8_e4m3fn", "float8_e5m2"] + else dtype + ) numpy_tensor = self.get_cached_numpy(dtype, output.shape) else: if "int" in dtype: @@ -639,18 +649,26 @@ def gen_paddle_output_and_output_grad(self, outputs): numpy.random.randint(-65535, 65535, size=output.shape) ).astype(dtype) else: - dtype = "float32" if dtype == "bfloat16" else dtype + dtype = ( + "float32" + if dtype in ["bfloat16", "float8_e4m3fn", "float8_e5m2"] + else dtype + ) numpy_tensor = (numpy.random.random(output.shape) - 0.5).astype(dtype) self.outputs_grad_numpy.append(numpy_tensor) for i, numpy_tensor in enumerate(self.outputs_grad_numpy): dtype = str(result_outputs[i].dtype).split(".")[-1] result_output_grad = paddle.to_tensor( numpy_tensor, - dtype=dtype if dtype != "bfloat16" else "float32", + dtype=dtype + if dtype not in ["bfloat16", "float8_e4m3fn", "float8_e5m2"] + else "float32", ) result_output_grad.stop_gradient = False if dtype == "bfloat16": result_output_grad = paddle.cast(result_output_grad, dtype="bfloat16") + elif dtype in ["float8_e4m3fn", "float8_e5m2"]: + result_output_grad = paddle.cast(result_output_grad, dtype=dtype) result_outputs_grads.append(result_output_grad) return result_outputs, result_outputs_grads diff --git a/tester/device_vs_cpu_config.yaml b/tester/device_vs_cpu_config.yaml new file mode 100644 index 00000000..dbe54a3a --- /dev/null +++ b/tester/device_vs_cpu_config.yaml @@ -0,0 +1,31 @@ +# 自定义设备与 CPU 精度对比测试的容差配置 +# 仅用于 device_vs_cpu 模式（APITestCustomDeviceVSCPU） +# +# 优先级：API+dtype > API default > 全局 dtype > 默认硬编码 1e-2 +# 注：paddle.float == float32，paddle.int == int32，无单独 key，均命中对应条目 + +# 全局 dtype 兜底容差 +dtype_atol_rtol: + # 浮点类型：精度越高要求越严 + float64: [1e-6, 1e-6] # 双精度，最严格 + float32: [1e-5, 1e-5] # 单精度，GPU/CPU 舍入差异典型水平 + float16: [5e-2, 5e-2] # 半精度，尾数 10 位，适当放宽 + bfloat16: [1e-1, 1e-1] # BF16，尾数仅 7 位，最宽松 + # 复数类型：实部/虚部精度跟随对应实数类型 + complex64: [1e-5, 1e-5] # 实部/虚部均为 float32 + complex128: [1e-6, 1e-6] # 实部/虚部均为 float64 + # 整数类型：GPU/CPU 结果应完全一致，精确匹配 + int8: [0, 0] + int16: [0, 0] + int32: [0, 0] + int64: [0, 0] + uint8: [0, 0] + +# API 级细粒度容差（覆盖全局 dtype 兜底） +# atol_rtol: +# paddle.matmul: +# default: [1e-2, 1e-2] +# float16: [8e-2, 8e-2] +# paddle.nn.functional.layer_norm: +# default: [2e-2, 2e-2] +# float16: [1e-1, 1e-1] diff --git a/tester/device_vs_gpu_config.yaml b/tester/device_vs_gpu_config.yaml new file mode 100644 index 00000000..b6540921 --- /dev/null +++ b/tester/device_vs_gpu_config.yaml @@ -0,0 +1,24 @@ +# 自定义设备与 GPU 精度对比测试的容差配置 +# 仅用于 device_vs_gpu 模式（APITestPaddleDeviceVSGPU） +# 与 base_config.yaml 中的 special_accuracy_atol_rtol（用于 Paddle-vs-PyTorch）职责不同 +# +# 优先级：API+dtype > API default > 全局 dtype > 命令行 --atol/--rtol + +# 全局 dtype 兜底容差（激活状态） +dtype_atol_rtol: + float64: [1e-6, 1e-6] + float32: [1e-4, 1e-4] + float16: [1e-2, 1e-2] + bfloat16: [1e-2, 1e-2] + int32: [0, 0 ] + int64: [0, 0 ] + bool: [0, 0 ] + +# API 级细粒度容差（覆盖全局 dtype 兜底） +# atol_rtol: +# paddle.matmul: +# default: [1e-2, 1e-2] +# float16: [8e-2, 8e-2] +# paddle.nn.functional.layer_norm: +# default: [2e-2, 2e-2] +# float16: [1e-1, 1e-1] diff --git a/tester/http_config.yaml b/tester/http_config.yaml new file mode 100644 index 00000000..257cf4cf --- /dev/null +++ b/tester/http_config.yaml @@ -0,0 +1,11 @@ +# HTTP 配置文件 +# 用于自定义设备与 GPU 精度对比测试的 HTTP 传输配置 + +# 远程 HTTP 服务器地址 +remote_host: "10.78.119.13" + +# 远程 HTTP 服务器端口 +remote_port: 8089 + +# HTTP 请求超时（秒） +timeout: 300 diff --git a/tester/http_server.py b/tester/http_server.py new file mode 100644 index 00000000..d2c9cbc6 --- /dev/null +++ b/tester/http_server.py @@ -0,0 +1,560 @@ +"""HTTP server for remote API test execution. + +Runs on the remote machine. Accepts API config strings via HTTP POST, +executes them on the local device (GPU/XPU/etc.) using a multi-GPU +process pool, and returns the pdtensor result bytes. + +Usage: + python -m tester.http_server --host 0.0.0.0 --port 8089 --num_gpus=-1 + python -m tester.http_server --host 0.0.0.0 --port 8089 --gpu_ids=6,7 +""" + +from __future__ import annotations + +import argparse +import gc +import io +import json +import os +import secrets +import signal +import sys +import threading +import time +from concurrent.futures import TimeoutError +from datetime import datetime +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer +from multiprocessing import Lock, Manager, set_start_method +from pathlib import Path + +import numpy as np +from pebble import ProcessExpired, ProcessPool + +# Add project root to path so we can import engineV2 utilities +sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) + +from engineV2 import ( + check_gpu_memory, + detect_device_type, + get_memory_info, + validate_gpu_options, +) +from tester.api_config.log_writer import set_engineV2 + +# Global process pool, set during server startup +_pool = None +_server_timeout = 1800 # default timeout per task +_concurrency_semaphore = None # limits queued + in-flight requests + +# Admin interface: set via --admin_token; empty string means disabled +_admin_token: str = "" +REPO_ROOT = Path(__file__).resolve().parent.parent + + +def init_server_worker(gpu_worker_list, lock, available_gpus, max_workers_per_gpu): + """Initialize a worker process with GPU assignment. + + Similar to engineV2.init_worker_gpu but without test-specific options. + """ + import errno + + my_pid = os.getpid() + + def pid_exists(pid): + try: + os.kill(pid, 0) + return True + except OSError as e: + return e.errno == errno.EPERM + + try: + with lock: + assigned_gpu = -1 + max_available_slots = -1 + for gpu_id in available_gpus: + workers = gpu_worker_list[gpu_id] + workers[:] = [pid for pid in workers if pid_exists(pid)] + available_slots = max_workers_per_gpu[gpu_id] - len(workers) + if available_slots > max_available_slots: + max_available_slots = available_slots + assigned_gpu = gpu_id + + if assigned_gpu == -1: + raise RuntimeError(f"Worker {my_pid} could not be assigned a GPU.") + + gpu_worker_list[assigned_gpu].append(my_pid) + + os.environ["CUDA_VISIBLE_DEVICES"] = str(assigned_gpu) + + # Import paddle/torch AFTER setting CUDA_VISIBLE_DEVICES so that + # the CUDA context is created on the correct device, not GPU 0. + import paddle + import torch + + globals()["torch"] = torch + globals()["paddle"] = paddle + + from tester import APIConfig, APITestPaddleDeviceVSGPU + from tester.paddle_device_vs_gpu import _PaddleSkipError + + globals()["APIConfig"] = APIConfig + globals()["APITestPaddleDeviceVSGPU"] = APITestPaddleDeviceVSGPU + globals()["_PaddleSkipError"] = _PaddleSkipError + + print( + f"{datetime.now()} Server worker PID: {my_pid}, Assigned GPU ID: {assigned_gpu}", + flush=True, + ) + except Exception as e: + print(f"{datetime.now()} Server worker {my_pid} init failed: {e}", flush=True) + raise + + +def run_single_api(api_config_str, random_seed, test_amp=False): + """Execute a single API test and return the pdtensor bytes. + + Runs inside a worker process. Returns bytes on success, raises on failure. + """ + from engineV2 import detect_device_type + + cuda_visible = os.environ.get("CUDA_VISIBLE_DEVICES", "0") + gpu_id = int(cuda_visible.split(",")[0]) + + print( + f"{datetime.now()} GPU {gpu_id} {os.getpid()} server exec: {api_config_str}", + flush=True, + ) + + # Wait for sufficient GPU memory + for _ in range(60): # max ~60 minutes wait + total_memory, used_memory = get_memory_info(gpu_id) + free_memory = total_memory - used_memory + if free_memory >= 2.0: # minimal 2GB required + break + time.sleep(60) + else: + raise RuntimeError(f"GPU {gpu_id} insufficient memory after waiting") + + # Set random seed for reproducibility + np.random.seed(random_seed) + paddle.seed(random_seed) + + api_config = APIConfig(api_config_str) + + # Create a minimal instance to reuse _run_paddle + tester = APITestPaddleDeviceVSGPU( + api_config, + operation_mode="upload", # dummy, we won't call test() + random_seed=random_seed, + test_amp=test_amp, + ) + + device_type = detect_device_type() + try: + output, grads = tester._run_paddle(device_type) + except _PaddleSkipError as e: + # Re-raise as a tagged RuntimeError so pebble can serialize it across + # the process boundary and the handler can detect it. + raise RuntimeError(f"__SKIP__:{e}") from None + + if output is None: + real_error = getattr(tester, "_last_error", None) + raise RuntimeError(real_error if real_error else f"API execution returned None for {api_config_str}") + + # Normalize output: paddle.save does not support named tuples (e.g. + # CummaxRetType, TopKRetType). Convert them to plain tuple recursively. + # Also eagerly evaluate lazy Jacobian/Hessian objects so that paddle.save + # can serialize the result as a plain Tensor. + def _normalize(obj): + if isinstance(obj, paddle.Tensor): + return obj + # Evaluate lazy Jacobian/Hessian objects (Hessian inherits Jacobian) + try: + from paddle.autograd.autograd import Jacobian + if isinstance(obj, Jacobian): + return obj[:] # triggers full evaluation, returns Tensor + except ImportError: + pass + if isinstance(obj, (list, tuple)): + items = [_normalize(x) for x in obj] + return items if isinstance(obj, list) else tuple(items) + return obj + + output = _normalize(output) + + # Serialize to pdtensor bytes + save_data = {"output": output} + if grads is not None: + save_data["grads"] = grads + + buffer = io.BytesIO() + paddle.save(save_data, buffer) + result_bytes = buffer.getvalue() + + # Cleanup + tester.clear_tensor() + del tester, api_config, output, grads, save_data + gc.collect() + torch = globals().get("torch") + if torch is not None: + torch.cuda.empty_cache() + paddle.device.cuda.empty_cache() + + print( + f"{datetime.now()} GPU {gpu_id} {os.getpid()} server done: {api_config_str} ({len(result_bytes)} bytes)", + flush=True, + ) + return result_bytes + + +class APITestHandler(BaseHTTPRequestHandler): + """HTTP request handler for API test execution.""" + + def log_message(self, format, *args): + """Override to use flush for immediate output.""" + print(f"{datetime.now()} {self.client_address[0]} - {format % args}", flush=True) + + def _send_json_response(self, status_code, data): + body = json.dumps(data, ensure_ascii=False).encode("utf-8") + self.send_response(status_code) + self.send_header("Content-Type", "application/json") + self.send_header("Content-Length", str(len(body))) + self.end_headers() + self.wfile.write(body) + + def _send_bytes_response(self, data, device_type="unknown"): + self.send_response(200) + self.send_header("Content-Type", "application/octet-stream") + self.send_header("Content-Length", str(len(data))) + self.send_header("X-Device-Type", device_type) + self.end_headers() + self.wfile.write(data) + + def do_GET(self): + if self.path == "/health": + device_type = detect_device_type() + self._send_json_response( + 200, + { + "status": "ok", + "device_type": device_type, + "paddle_version": __import__("paddle").__version__, + }, + ) + else: + self._send_json_response(404, {"error": "not_found"}) + + def do_POST(self): + if self.path == "/run_api_test": + self._handle_run_api_test() + elif self.path == "/admin/upload_file": + self._handle_admin_upload_file() + elif self.path == "/admin/delete_file": + self._handle_admin_delete_file() + elif self.path == "/admin/restart": + self._handle_admin_restart() + else: + self._send_json_response(404, {"error": "not_found"}) + + def _check_admin_token(self) -> bool: + if not _admin_token: + self._send_json_response(503, {"error": "admin_disabled"}) + return False + provided = self.headers.get("X-Admin-Token", "") + if not secrets.compare_digest(provided, _admin_token): + self._send_json_response(403, {"error": "forbidden"}) + return False + return True + + def _handle_run_api_test(self): + # Parse request body + try: + content_length = int(self.headers.get("Content-Length", 0)) + body = self.rfile.read(content_length) + data = json.loads(body.decode("utf-8")) + except Exception as e: + self._send_json_response( + 400, + { + "error": "bad_request", + "detail": f"Invalid JSON: {e}", + }, + ) + return + + api_config_str = data.get("api_config") + random_seed = data.get("random_seed", 0) + test_amp = data.get("test_amp", False) + + if not api_config_str: + self._send_json_response( + 400, + { + "error": "bad_request", + "detail": "Missing field: api_config", + }, + ) + return + + # Submit to process pool + global _pool, _server_timeout, _concurrency_semaphore + if _pool is None: + self._send_json_response( + 503, + { + "error": "service_unavailable", + "detail": "Process pool not initialized", + "api_config": api_config_str, + }, + ) + return + + # Block until a slot opens. The client's http_timeout is the ultimate + # backstop — if we wait too long here the client will close the + # connection and this thread will get a BrokenPipeError on write, + # which is caught by the outer except. + if _concurrency_semaphore is not None: + _concurrency_semaphore.acquire() + + try: + future = _pool.schedule( + run_single_api, + [api_config_str, random_seed, test_amp], + timeout=_server_timeout, + ) + result_bytes = future.result() + device_type = detect_device_type() + self._send_bytes_response(result_bytes, device_type) + + except TimeoutError: + self._send_json_response( + 504, + { + "error": "timeout", + "detail": f"Execution timed out after {_server_timeout}s", + "api_config": api_config_str, + }, + ) + + except ProcessExpired as e: + if e.exitcode == 99: + error_type = "cuda_error" + elif e.exitcode == 98: + error_type = "oom" + else: + error_type = "crash" + self._send_json_response( + 500, + { + "error": error_type, + "detail": str(e), + "api_config": api_config_str, + }, + ) + + except Exception as e: + detail = str(e) + if detail.startswith("__SKIP__:"): + self._send_json_response( + 422, + { + "error": "skip", + "detail": detail[len("__SKIP__:") :], + "api_config": api_config_str, + }, + ) + else: + self._send_json_response( + 500, + { + "error": "remote_error", + "detail": detail, + "api_config": api_config_str, + }, + ) + + finally: + if _concurrency_semaphore is not None: + _concurrency_semaphore.release() + + def _handle_admin_upload_file(self): + if not self._check_admin_token(): + return + try: + content_length = int(self.headers.get("Content-Length", 0)) + body = self.rfile.read(content_length) + data = json.loads(body.decode("utf-8")) + rel_path = data["path"] + content = data["content"] + except Exception as e: + self._send_json_response(400, {"error": "bad_request", "detail": str(e)}) + return + # Security: reject path traversal + target = (REPO_ROOT / rel_path).resolve() + if not str(target).startswith(str(REPO_ROOT)): + self._send_json_response(403, {"error": "forbidden", "detail": "path traversal"}) + return + try: + target.parent.mkdir(parents=True, exist_ok=True) + target.write_text(content, encoding="utf-8") + self._send_json_response(200, {"status": "ok", "path": rel_path}) + except Exception as e: + self._send_json_response(500, {"status": "error", "detail": str(e)}) + + def _handle_admin_delete_file(self): + if not self._check_admin_token(): + return + try: + content_length = int(self.headers.get("Content-Length", 0)) + body = self.rfile.read(content_length) + data = json.loads(body.decode("utf-8")) + rel_path = data["path"] + except Exception as e: + self._send_json_response(400, {"error": "bad_request", "detail": str(e)}) + return + # Security: reject path traversal + target = (REPO_ROOT / rel_path).resolve() + if not str(target).startswith(str(REPO_ROOT)): + self._send_json_response(403, {"error": "forbidden", "detail": "path traversal"}) + return + try: + if target.exists(): + target.unlink() + # Also remove the corresponding .pyc from __pycache__ if it exists + pyc = target.parent / "__pycache__" / (target.stem + ".cpython-310.pyc") + if pyc.exists(): + pyc.unlink() + self._send_json_response(200, {"status": "ok", "path": rel_path}) + except Exception as e: + self._send_json_response(500, {"status": "error", "detail": str(e)}) + + def _handle_admin_restart(self): + if not self._check_admin_token(): + return + self._send_json_response(200, {"status": "restarting"}) + + def do_restart(): + time.sleep(0.5) + os.execv(sys.executable, [sys.executable] + sys.argv) + + threading.Thread(target=do_restart, daemon=True).start() + + +def parse_bool(value): + if isinstance(value, str): + value = value.lower() + if value in ["true", "1", "yes", "y"]: + return True + elif value in ["false", "0", "no", "n"]: + return False + raise ValueError(f"Invalid boolean value: {value}") + + +def main(): + set_start_method("spawn") + + parser = argparse.ArgumentParser(description="HTTP server for remote API test execution") + parser.add_argument("--host", type=str, default="0.0.0.0", help="Server bind address") + parser.add_argument("--port", type=int, default=8089, help="Server port") + parser.add_argument( + "--num_gpus", type=int, default=-1, help="Number of GPUs to use, -1 for all" + ) + parser.add_argument("--num_workers_per_gpu", type=int, default=1, help="Workers per GPU") + parser.add_argument( + "--required_memory", type=float, default=10.0, help="Required memory per worker in GB" + ) + parser.add_argument("--gpu_ids", type=str, default="", help="GPU IDs (e.g., '0,1,2' or '0-3')") + parser.add_argument("--timeout", type=int, default=1800, help="Per-task timeout in seconds") + parser.add_argument( + "--admin_token", + type=str, + default="", + help="If non-empty, enables /admin/upload_file and /admin/restart endpoints " + "protected by X-Admin-Token header", + ) + + args = parser.parse_args() + + global _server_timeout, _admin_token + _server_timeout = args.timeout + _admin_token = args.admin_token + + # Detect device + device_type = detect_device_type() + print(f"Detected device type: {device_type}", flush=True) + + # Build a minimal options namespace for validate_gpu_options + gpu_options = argparse.Namespace( + num_gpus=args.num_gpus, + num_workers_per_gpu=args.num_workers_per_gpu, + required_memory=args.required_memory, + gpu_ids=args.gpu_ids, + ) + + gpu_ids = validate_gpu_options(gpu_options) + available_gpus, max_workers_per_gpu = check_gpu_memory( + gpu_ids, gpu_options.num_workers_per_gpu, gpu_options.required_memory + ) + + if not available_gpus: + print( + f"No GPUs with sufficient memory. Required: {gpu_options.required_memory} GB.", + flush=True, + ) + sys.exit(1) + + total_workers = sum(max_workers_per_gpu.values()) + print( + f"Using {len(available_gpus)} GPU(s) with workers: {max_workers_per_gpu}. " + f"Total workers: {total_workers}.", + flush=True, + ) + + # Limit concurrent requests to 2x workers — enough to keep the pool saturated + # without unbounded queuing. Excess requests get 503 immediately. + global _concurrency_semaphore + _concurrency_semaphore = threading.Semaphore(total_workers * 2) + + # Initialize process pool + manager = Manager() + gpu_worker_list = manager.dict({gid: manager.list() for gid in available_gpus}) + lock = Lock() + + global _pool + _pool = ProcessPool( + max_workers=total_workers, + initializer=init_server_worker, + initargs=[gpu_worker_list, lock, available_gpus, max_workers_per_gpu], + ) + + # Start HTTP server + server = ThreadingHTTPServer((args.host, args.port), APITestHandler) + + def shutdown_handler(*_): + print(f"\n{datetime.now()} Shutting down...", flush=True) + if _pool is not None: + try: + if _pool.active: + _pool.stop() + _pool.join(timeout=5) + except Exception: + pass + os._exit(0) + + signal.signal(signal.SIGINT, shutdown_handler) + signal.signal(signal.SIGTERM, shutdown_handler) + + print( + f"{datetime.now()} HTTP server listening on {args.host}:{args.port} " + f"(device: {device_type}, workers: {total_workers})", + flush=True, + ) + + try: + server.serve_forever() + except KeyboardInterrupt: + shutdown_handler() + + +if __name__ == "__main__": + main() diff --git a/tester/paddle_device_vs_cpu.py b/tester/paddle_device_vs_cpu.py index 82d36132..d64c3454 100644 --- a/tester/paddle_device_vs_cpu.py +++ b/tester/paddle_device_vs_cpu.py @@ -1,11 +1,27 @@ from __future__ import annotations +import os + import paddle import torch +import yaml from .api_config.log_writer import write_to_log from .base import APITestBase +_DEVICE_VS_CPU_CONFIG_PATH = os.path.join( + os.path.dirname(os.path.abspath(__file__)), "device_vs_cpu_config.yaml" +) +_device_vs_cpu_atol_rtol: dict = {} +_device_vs_cpu_dtype_atol_rtol: dict = {} + +if os.path.exists(_DEVICE_VS_CPU_CONFIG_PATH): + with open(_DEVICE_VS_CPU_CONFIG_PATH, encoding="utf-8") as _f: + _cfg = yaml.safe_load(_f) or {} + _device_vs_cpu_atol_rtol = _cfg.get("atol_rtol", {}) + _device_vs_cpu_dtype_atol_rtol = _cfg.get("dtype_atol_rtol", {}) + del _cfg, _f + class APITestCustomDeviceVSCPU(APITestBase): def __init__(self, api_config, **kwargs): @@ -103,8 +119,25 @@ def run_on_device(self, device_type, device_id=0): ) return None, None + def _resolve_atol_rtol(self, dtype_str: str) -> tuple[float, float]: + """按三级优先级解析容差：API+dtype > API default > 全局 dtype > 默认 1e-2。""" + api_name = self.api_config.api_name + if api_name in _device_vs_cpu_atol_rtol: + api_cfg = _device_vs_cpu_atol_rtol[api_name] + if dtype_str in api_cfg: + return tuple(float(x) for x in api_cfg[dtype_str]) + if "default" in api_cfg: + return tuple(float(x) for x in api_cfg["default"]) + if dtype_str in _device_vs_cpu_dtype_atol_rtol: + return tuple(float(x) for x in _device_vs_cpu_dtype_atol_rtol[dtype_str]) + return 1e-2, 1e-2 + def _compare_single_tensor(self, cpu_tensor, custom_tensor, tensor_name=""): try: + # 在 cast 之前提取原始 dtype + dtype_str = str(cpu_tensor.dtype).split(".")[-1] + atol, rtol = self._resolve_atol_rtol(dtype_str) + # bfloat16 if cpu_tensor.dtype == paddle.bfloat16: cpu_tensor = paddle.cast(cpu_tensor, dtype="float32") @@ -114,12 +147,14 @@ def _compare_single_tensor(self, cpu_tensor, custom_tensor, tensor_name=""): # Convert CustomDevice tensor to CPU custom_tensor_cpu = custom_tensor.cpu() - cpu_torch = torch.from_numpy(cpu_tensor.numpy()) - custom_torch = torch.from_numpy(custom_tensor_cpu.numpy()) + # .copy() 确保得到可写副本，避免 requires_grad tensor 的只读内存 + # 导致 torch.testing.assert_close 内部抛 unexpected exception + cpu_torch = torch.from_numpy(cpu_tensor.numpy().copy()) + custom_torch = torch.from_numpy(custom_tensor_cpu.numpy().copy()) # 使用 torch.testing.assert_close 来替代 numpy.testing.assert_allclose torch.testing.assert_close( - cpu_torch, custom_torch, rtol=1e-2, atol=1e-2, equal_nan=True + cpu_torch, custom_torch, rtol=rtol, atol=atol, equal_nan=True ) return True @@ -231,11 +266,13 @@ def test(self): print("[Skip]", flush=True) return - # 2. Determine target device: prioritize XPU, fallback to CustomDevice + # 2. Determine target device: prioritize XPU, fallback to CustomDevice, then GPU if self.check_xpu_available(): target_device, device_id = "xpu", self.xpu_device_id elif self.check_custom_device_available(): target_device, device_id = self.custom_device_type, self.custom_device_id + elif paddle.is_compiled_with_cuda() and paddle.device.cuda.device_count() > 0: + target_device, device_id = "gpu", 0 else: print("[no available device]", self.api_config.config, flush=True) write_to_log("crash", self.api_config.config) @@ -367,6 +404,9 @@ def test(self): elif self.check_custom_device_available(): target_device = self.custom_device_type device_id = self.custom_device_id + elif paddle.is_compiled_with_cuda() and paddle.device.cuda.device_count() > 0: + target_device = "gpu" + device_id = 0 else: target_device = "cpu" device_id = 0 diff --git a/tester/paddle_device_vs_gpu.py b/tester/paddle_device_vs_gpu.py index 3694e163..8edfe426 100644 --- a/tester/paddle_device_vs_gpu.py +++ b/tester/paddle_device_vs_gpu.py @@ -2,15 +2,37 @@ import hashlib import json +import os import subprocess import tempfile from pathlib import Path import numpy as np import paddle +import yaml +from .api_config.config_analyzer import TensorConfig from .api_config.log_writer import write_to_log from .paddle_device_vs_cpu import APITestCustomDeviceVSCPU +from .special_compare import SkipComparison, get_backward_compare, get_forward_compare + + +class _PaddleSkipError(Exception): + """Raised by _run_paddle when need_skip(paddle_only=True) is True.""" + + +_DEVICE_VS_GPU_CONFIG_PATH = os.path.join( + os.path.dirname(os.path.abspath(__file__)), "device_vs_gpu_config.yaml" +) +_device_vs_gpu_atol_rtol: dict = {} +_device_vs_gpu_dtype_atol_rtol: dict = {} + +if os.path.exists(_DEVICE_VS_GPU_CONFIG_PATH): + with open(_DEVICE_VS_GPU_CONFIG_PATH, encoding="utf-8") as _f: + _cfg = yaml.safe_load(_f) or {} + _device_vs_gpu_atol_rtol = _cfg.get("atol_rtol", {}) + _device_vs_gpu_dtype_atol_rtol = _cfg.get("dtype_atol_rtol", {}) + del _cfg, _f class APITestPaddleDeviceVSGPU(APITestCustomDeviceVSCPU): @@ -27,6 +49,11 @@ def __init__(self, api_config, **kwargs): self.bcecmd_path = Path(kwargs.get("bcecmd_path", "./bcecmd")).resolve() self.bos_conf_path = kwargs.get("bos_conf_path", "./conf") + # HTTP 模式参数 + self.http_host = kwargs.get("http_host", "") + self.http_port = kwargs.get("http_port", 8089) + self.http_timeout = kwargs.get("http_timeout", 300) + # 设置随机种子确保一致性 if self.random_seed != 0: np.random.seed(self.random_seed) @@ -50,6 +77,122 @@ def _get_local_device_type(self): return detect_device_type() + def _has_float8_dtype(self): + """Return True if any arg/kwarg tensor uses a float8 dtype.""" + + def _check(cfg): + if isinstance(cfg, TensorConfig): + return cfg.dtype in self._FLOAT8_DTYPES + if isinstance(cfg, (list, tuple)): + return any(_check(c) for c in cfg) + if isinstance(cfg, slice): + return False + return cfg in self._FLOAT8_DTYPES + + return any(_check(c) for c in self.api_config.args) or any( + _check(c) for c in self.api_config.kwargs.values() + ) + + def _has_complex128(self): + """Return True if the config will result in complex128 computation on XPU. + + XPU does not support complex128 in cast_kernel, tensor memory allocation, + or gradient accumulation. Two cases trigger complex128: + 1. A tensor explicitly has dtype complex128. + 2. A Python complex scalar is combined with a float64 tensor — Paddle + promotes the result to complex128, which XPU cannot handle. + (complex scalar + float32/bfloat16/int* tensor promotes to complex64, + which XPU does support.) + """ + all_args = list(self.api_config.args) + list(self.api_config.kwargs.values()) + + def _has_complex128_dtype(cfg): + if isinstance(cfg, TensorConfig): + return cfg.dtype == "complex128" + if isinstance(cfg, (list, tuple)): + return any(_has_complex128_dtype(c) for c in cfg) + if isinstance(cfg, str): + return cfg == "complex128" + try: + import paddle + if isinstance(cfg, paddle.base.core.DataType): + return str(cfg) == "paddle.complex128" + except Exception: + pass + return False + + def _has_float64_tensor(cfg): + if isinstance(cfg, TensorConfig): + return cfg.dtype == "float64" + if isinstance(cfg, (list, tuple)): + return any(_has_float64_tensor(c) for c in cfg) + return False + + def _has_complex_scalar(cfg): + if isinstance(cfg, complex): + return True + if isinstance(cfg, (list, tuple)): + return any(_has_complex_scalar(c) for c in cfg) + return False + + # Case 1: explicit complex128 dtype anywhere + if any(_has_complex128_dtype(c) for c in all_args): + return True + + # Case 2: complex scalar + float64 tensor → promotes to complex128 + if any(_has_complex_scalar(c) for c in all_args) and any( + _has_float64_tensor(c) for c in all_args + ): + return True + + return False + + def need_skip(self, paddle_only=False): + # Device vs GPU compares Paddle on XPU against Paddle on GPU — no Torch + # involved. All conditions in base.need_skip() are Torch-specific (sparse, + # prod multi-axis, torch_error_skip, float8 dtype), so we do NOT call + # super() here. + # XPU cannot create float8 tensors via the float32->cast path. + if self._get_local_device_type() == "xpu" and self._has_float8_dtype(): + return True + # XPU does not support sparse kernels. + # Also covers sparse-related Tensor methods that don't carry "sparse" in their name. + _SPARSE_APIS = {"paddle.Tensor.coalesce", "paddle.Tensor.is_coalesced"} + if self._get_local_device_type() == "xpu" and ( + "sparse" in self.api_config.api_name + or self.api_config.api_name in _SPARSE_APIS + ): + return True + # XPU does not support complex128 (cast_kernel, memory allocation, grad accumulation). + if self._get_local_device_type() == "xpu" and self._has_complex128(): + return True + return False + + def need_check_grad(self): + if not super().need_check_grad(): + return False + # dropout 推理模式（training=False）在 XPU 上不注册 GradOp，强行触发 backward 会报 + # (InvalidArgument) GradOp is only callable when is_test is false + # 仅在 XPU 本地设备时跳过，其他设备/模式不影响。 + # 签名：dropout(x, p, axis, training, ...) training 在 args[3] + # fused_dropout_add(x, y, p, training, ...) training 在 args[3] + _DROPOUT_INFERENCE_APIS = { + "paddle.nn.functional.dropout", + "paddle.incubate.nn.functional.fused_dropout_add", + } + if ( + self._get_local_device_type() == "xpu" + and self.api_config.api_name in _DROPOUT_INFERENCE_APIS + ): + training = self.api_config.kwargs.get("training", None) + if training is None and len(self.api_config.args) > 3: + training = self.api_config.args[3] + if training is None: + training = True # 默认值 + if not training: + return False + return True + def _get_filename(self): """生成PDTensor文件名（不再包含设备前缀，只依赖随机种子和配置哈希）""" return f"{self.random_seed}-{self._get_config_hash()}.pdtensor" @@ -137,8 +280,75 @@ def _download_from_bos(self, filename): print(f"[download] Download failed: {e}", flush=True) return None + _FLOAT8_DTYPES = frozenset(["float8_e4m3fn", "float8_e5m2"]) + + @staticmethod + def _normalize_output(obj): + """Recursively normalize output for comparison. + + Eagerly evaluates lazy Jacobian/Hessian objects so that the result is + a plain Tensor (or nested list/tuple of Tensors) that paddle.save and + the comparison logic can handle. + """ + if isinstance(obj, paddle.Tensor): + return obj + try: + from paddle.autograd.autograd import Jacobian + if isinstance(obj, Jacobian): + return obj[:] # triggers full evaluation, returns Tensor + except ImportError: + pass + if isinstance(obj, (list, tuple)): + items = [APITestPaddleDeviceVSGPU._normalize_output(x) for x in obj] + return items if isinstance(obj, list) else tuple(items) + return obj + + def _fill_float8_paddle_inputs(self): + """Create float8 paddle tensors for args that get_paddle_tensor left as None. + + config_analyzer.get_paddle_tensor() returns None for float8 dtypes + because the generic torch_vs_paddle mode doesn't support them. In + device_vs_gpu mode paddle does support float8, so we patch up the + None entries here without touching any shared code. + """ + + def make_tensor(cfg: TensorConfig) -> paddle.Tensor: + # Generate float32 numpy data (numpy has no float8 dtype), + # create a paddle float32 tensor, then cast to float8. + numpy_data = (np.random.random(cfg.shape) - 0.5).astype("float32") + t = paddle.to_tensor(numpy_data, dtype="float32", place=cfg.place) + t = paddle.cast(t, dtype=cfg.dtype) + t.stop_gradient = False + return t + + def fix_list(args: list, cfgs: list): + for i, (arg, cfg) in enumerate(zip(args, cfgs)): + if ( + arg is None + and isinstance(cfg, TensorConfig) + and cfg.dtype in self._FLOAT8_DTYPES + ): + args[i] = make_tensor(cfg) + elif isinstance(arg, list) and isinstance(cfg, list): + fix_list(arg, cfg) + + fix_list(self.paddle_args, self.paddle_args_config) + + for key, val in list(self.paddle_kwargs.items()): + cfg = self.paddle_kwargs_config.get(key) + if val is None and isinstance(cfg, TensorConfig) and cfg.dtype in self._FLOAT8_DTYPES: + self.paddle_kwargs[key] = make_tensor(cfg) + def _run_paddle(self, device_type: str): """在指定设备上运行 Paddle（统一 GPU / XPU / 自定义设备逻辑）。""" + # Called directly by http_server (bypasses test()), so check paddle-only + # skips here (e.g. sparse APIs). float8 is intentionally NOT skipped + # because paddle supports it; need_skip(paddle_only=True) excludes the + # torch-incompatible float8 check. + if self.need_skip(paddle_only=True): + print(f"[skip] {self.api_config.config}", flush=True) + raise _PaddleSkipError(f"API not supported on this device: {self.api_config.config}") + try: paddle_device_type = device_type if device_type == "gpu": @@ -166,7 +376,24 @@ def _run_paddle(self, device_type: str): print("gen_paddle_input failed", flush=True) return None, None - paddle_output = self.paddle_api(*tuple(self.paddle_args), **self.paddle_kwargs) + # device_vs_gpu supports float8 natively; gen_paddle_input() leaves + # None for float8 tensors (shared guard in config_analyzer). Fix them + # up here so that only this mode is affected. + self._fill_float8_paddle_inputs() + + if self.test_amp: + with paddle.amp.auto_cast(): + paddle_output = self.paddle_api(*tuple(self.paddle_args), **self.paddle_kwargs) + else: + paddle_output = self.paddle_api(*tuple(self.paddle_args), **self.paddle_kwargs) + + # 原地操作返回 None，用被修改的 tensor 替代，使后续对比逻辑不变 + if paddle_output is None: + api_name = self.api_config.api_name + if api_name == "paddle.Tensor.__setitem__": + paddle_output = self.paddle_args[0] + elif api_name == "paddle.nn.utils.vector_to_parameters": + paddle_output = self.paddle_args[1] paddle_grads = None if self.need_check_grad(): @@ -181,6 +408,25 @@ def _run_paddle(self, device_type: str): grad_outputs=result_outputs_grads, allow_unused=True, ) + # sparse=True ops (e.g. embedding) produce SelectedRows gradients. + # SelectedRows is NOT detected by is_sparse() / is_dense() — both + # return False. Calling to_dense() on SelectedRows triggers a + # Segfault in Paddle's C++ layer. Detection: a Tensor that is + # neither dense nor any sparse variant is a SelectedRows. + # Conversion: numpy() works correctly on SelectedRows; rebuild as + # a plain dense Tensor so paddle.save() can serialize it. + if paddle_grads is not None: + paddle_grads = [ + paddle.to_tensor(g.numpy()) + if g is not None + and isinstance(g, paddle.Tensor) + and not g.is_dense() + and not g.is_sparse() + and not g.is_sparse_coo() + and not g.is_sparse_csr() + else g + for g in paddle_grads + ] return paddle_output, paddle_grads @@ -190,8 +436,60 @@ def _run_paddle(self, device_type: str): flush=True, ) write_to_log("paddle_error", self.api_config.config) + self._last_error = str(e) return None, None + def _resolve_atol_rtol(self, dtype_str: str) -> tuple[float, float]: + """按三级优先级解析容差：API+dtype > API default > 全局 dtype > 命令行值。""" + api_name = self.api_config.api_name + if api_name in _device_vs_gpu_atol_rtol: + api_cfg = _device_vs_gpu_atol_rtol[api_name] + if dtype_str in api_cfg: + return tuple(float(v) for v in api_cfg[dtype_str]) + if "default" in api_cfg: + return tuple(float(v) for v in api_cfg["default"]) + if dtype_str in _device_vs_gpu_dtype_atol_rtol: + return tuple(float(v) for v in _device_vs_gpu_dtype_atol_rtol[dtype_str]) + return self.atol, self.rtol + + def _print_diff(self, label, local_np, remote_np): + """打印两个 numpy 数组之间的实际最大绝对误差和最大相对误差，返回 (max_abs, max_rel)""" + a = local_np.astype(np.float64) + b = remote_np.astype(np.float64) + abs_diff = np.abs(a - b) + if abs_diff.size == 0: + print( + f"[compare] {label} max_abs_diff=0 (empty tensor), max_rel_diff=0 (empty tensor)", + flush=True, + ) + return 0.0, 0.0 + max_abs = float(np.nanmax(abs_diff)) + denom = np.abs(b) + with np.errstate(invalid="ignore", divide="ignore"): + rel_diff = np.where(denom == 0, abs_diff, abs_diff / denom) + max_rel = float(np.nanmax(rel_diff)) + print( + f"[compare] {label} max_abs_diff={max_abs:.6g}, max_rel_diff={max_rel:.6g}", flush=True + ) + return max_abs, max_rel + + def _assert_close(self, local_np, remote_np, atol, rtol): + """手动实现 allclose 语义，避免 np.testing.assert_allclose 在 float16/bfloat16 下 + 因内部格式化引发 'Unknown format code g for object of type str' 的 bug。 + 判定条件：|actual - desired| <= atol + rtol * |desired|（element-wise，NaN==NaN 视为相等） + """ + a = local_np.astype(np.float64) + b = remote_np.astype(np.float64) + nan_equal = np.isnan(a) == np.isnan(b) + abs_diff = np.abs(a - b) + tol = atol + rtol * np.abs(b) + mismatch = ~nan_equal | ((~np.isnan(a)) & (abs_diff > tol)) + if np.any(mismatch): + max_abs = float(np.nanmax(abs_diff)) + raise AssertionError( + f"Arrays not close: max_abs_diff={max_abs:.6g}, atol={atol:.6g}, rtol={rtol:.6g}" + ) + def _compare_with_downloaded(self, local_output, local_grads, downloaded_tensor): """与下载的结果进行对比""" try: @@ -203,17 +501,19 @@ def _compare_with_downloaded(self, local_output, local_grads, downloaded_tensor) # 对比Forward输出（直接使用Paddle对比） try: - if isinstance(local_output, paddle.Tensor) and isinstance( + forward_fn = get_forward_compare(self.api_config.api_name) + if forward_fn is not None: + forward_fn(local_output, remote_output, self.api_config, tester=self) + elif isinstance(local_output, paddle.Tensor) and isinstance( remote_output, paddle.Tensor ): # 使用Paddle的对比方法 - np.testing.assert_allclose( - local_output.numpy(), - remote_output.numpy(), - atol=self.atol, - rtol=self.rtol, - equal_nan=True, - ) + dtype_str = str(local_output.dtype).split(".")[-1] + atol, rtol = self._resolve_atol_rtol(dtype_str) + local_np = local_output.numpy() + remote_np = remote_output.numpy() + self._print_diff("Forward", local_np, remote_np) + self._assert_close(local_np, remote_np, atol, rtol) elif isinstance(local_output, (list, tuple)) and isinstance( remote_output, (list, tuple) ): @@ -222,13 +522,12 @@ def _compare_with_downloaded(self, local_output, local_grads, downloaded_tensor) if isinstance(local_item, paddle.Tensor) and isinstance( remote_item, paddle.Tensor ): - np.testing.assert_allclose( - local_item.numpy(), - remote_item.numpy(), - atol=self.atol, - rtol=self.rtol, - equal_nan=True, - ) + dtype_str = str(local_item.dtype).split(".")[-1] + atol, rtol = self._resolve_atol_rtol(dtype_str) + local_np = local_item.numpy() + remote_np = remote_item.numpy() + self._print_diff(f"Forward output[{i}]", local_np, remote_np) + self._assert_close(local_np, remote_np, atol, rtol) print( f"[compare] Forward output[{i}] comparison passed", flush=True, @@ -257,6 +556,10 @@ def _compare_with_downloaded(self, local_output, local_grads, downloaded_tensor) f"[compare] Forward accuracy check passed for {self.api_config.config}", flush=True, ) + except SkipComparison as e: + print(f"[compare] Forward skipped for {self.api_config.config}: {e}", flush=True) + write_to_log("skip", self.api_config.config) + return True except Exception as e: print( f"[compare] Forward accuracy check failed for {self.api_config.config}, error: {e}", @@ -270,7 +573,10 @@ def _compare_with_downloaded(self, local_output, local_grads, downloaded_tensor) remote_grads = remote_data["grads"] try: - if isinstance(local_grads, (list, tuple)) and isinstance( + backward_fn = get_backward_compare(self.api_config.api_name) + if backward_fn is not None: + backward_fn(local_grads, remote_grads, self.api_config, tester=self) + elif isinstance(local_grads, (list, tuple)) and isinstance( remote_grads, (list, tuple) ): for i, (local_grad, remote_grad) in enumerate( @@ -279,13 +585,12 @@ def _compare_with_downloaded(self, local_output, local_grads, downloaded_tensor) if isinstance(local_grad, paddle.Tensor) and isinstance( remote_grad, paddle.Tensor ): - np.testing.assert_allclose( - local_grad.numpy(), - remote_grad.numpy(), - atol=self.atol, - rtol=self.rtol, - equal_nan=True, - ) + dtype_str = str(local_grad.dtype).split(".")[-1] + atol, rtol = self._resolve_atol_rtol(dtype_str) + local_np = local_grad.numpy() + remote_np = remote_grad.numpy() + self._print_diff(f"Backward gradient[{i}]", local_np, remote_np) + self._assert_close(local_np, remote_np, atol, rtol) print( f"[compare] Backward gradient[{i}] comparison passed", flush=True, @@ -293,23 +598,28 @@ def _compare_with_downloaded(self, local_output, local_grads, downloaded_tensor) elif isinstance(local_grads, paddle.Tensor) and isinstance( remote_grads, paddle.Tensor ): - np.testing.assert_allclose( - local_grads.numpy(), - remote_grads.numpy(), - atol=self.atol, - rtol=self.rtol, - equal_nan=True, - ) + dtype_str = str(local_grads.dtype).split(".")[-1] + atol, rtol = self._resolve_atol_rtol(dtype_str) + local_np = local_grads.numpy() + remote_np = remote_grads.numpy() + self._print_diff("Backward", local_np, remote_np) + self._assert_close(local_np, remote_np, atol, rtol) print( f"[compare] Backward gradient check passed for {self.api_config.config}", flush=True, ) + except SkipComparison as e: + print( + f"[compare] Backward skipped for {self.api_config.config}: {e}", + flush=True, + ) except Exception as e: print( f"[compare] Backward gradient check failed for {self.api_config.config}, error: {e}", flush=True, ) + write_to_log("accuracy_error", self.api_config.config) return False print( @@ -333,9 +643,11 @@ def test(self): self._test_upload_mode() elif self.operation_mode == "download": self._test_download_mode() + elif self.operation_mode == "http": + self._test_http_mode() else: print( - "[error] operation_mode 不能为空，请指定 --operation_mode=upload 或 download", + "[error] operation_mode 不能为空，请指定 --operation_mode=upload 或 download 或 http", flush=True, ) return @@ -399,3 +711,119 @@ def _test_download_mode(self): f"[download] Download mode completed for {self.api_config.config}", flush=True, ) + + def _request_remote_execution(self): + """发送API配置到远程HTTP服务器执行并返回 (bytes, None) 或 (None, error_info)""" + import json + import urllib.error + import urllib.request + + url = f"http://{self.http_host}:{self.http_port}/run_api_test" + payload = json.dumps( + { + "api_config": self.api_config.config, + "random_seed": self.random_seed, + "test_amp": self.test_amp, + } + ).encode("utf-8") + + req = urllib.request.Request( + url, + data=payload, + headers={"Content-Type": "application/json"}, + method="POST", + ) + try: + with urllib.request.urlopen(req, timeout=self.http_timeout) as resp: + if resp.status == 200: + return resp.read(), None + body = resp.read().decode("utf-8", errors="replace") + print(f"[http] Remote returned status {resp.status}: {body}", flush=True) + return None, {"error": "unknown", "detail": body} + except urllib.error.HTTPError as e: + try: + body = e.read().decode("utf-8", errors="replace") + error_info = json.loads(body) + except Exception: + error_info = {"error": "unknown", "detail": str(e)} + status = e.code + error_type = error_info.get("error", "unknown") + detail = error_info.get("detail", "") + print( + f"[http] HTTP error {status} ({error_type}): {detail}", + flush=True, + ) + return None, error_info + except urllib.error.URLError as e: + print(f"[http] Network error: {e.reason}", flush=True) + return None, {"error": "network_error", "detail": str(e.reason)} + except Exception as e: + print(f"[http] Request failed: {e}", flush=True) + return None, {"error": "network_error", "detail": str(e)} + + def _save_bytes_to_temp(self, data_bytes): + """将接收到的字节数据保存到临时文件""" + temp_dir = tempfile.gettempdir() + filename = f"http_{self._get_filename()}" + local_path = Path(temp_dir) / filename + local_path.write_bytes(data_bytes) + return local_path + + def _test_http_mode(self): + """HTTP模式：发送API配置到远程服务器，获取结果并本地对比""" + # Skip before sending HTTP request (e.g. float8 on XPU) + if self.need_skip(paddle_only=True): + write_to_log("skip", self.api_config.config) + return + + print(f"[http] Starting HTTP mode for {self.api_config.config}", flush=True) + + # 1. 发送到远端执行 + result_bytes, error_info = self._request_remote_execution() + + if result_bytes is None: + # 根据错误类型写对应日志 + if error_info: + error_type = error_info.get("error", "unknown") + if error_type in ("remote_error", "cuda_error", "oom", "crash", "timeout"): + write_to_log(error_type, self.api_config.config) + elif error_type == "skip": + write_to_log("skip", self.api_config.config) + elif error_type == "network_error": + write_to_log("network_error", self.api_config.config) + else: + print( + f"[http] Unknown remote error for {self.api_config.config}", + flush=True, + ) + return + + # 2. 保存远端结果到临时文件 + downloaded_tensor_path = self._save_bytes_to_temp(result_bytes) + + # 3. 在本地设备上执行（设置与服务端相同的随机种子，保证输入数据一致） + np.random.seed(self.random_seed) + paddle.seed(self.random_seed) + local_device_type = self._get_local_device_type() + local_output, local_grads = self._run_paddle(local_device_type) + + if local_output is None: + print( + f"[http] Local execution failed for {self.api_config.config}", + flush=True, + ) + downloaded_tensor_path.unlink(missing_ok=True) + return + + # Normalize lazy Jacobian/Hessian objects on the local side + local_output = self._normalize_output(local_output) + if local_grads is not None: + local_grads = self._normalize_output(local_grads) + + # 4. 用现有对比逻辑进行比较 + self._compare_with_downloaded(local_output, local_grads, downloaded_tensor_path) + + # 5. 清理临时文件 + downloaded_tensor_path.unlink(missing_ok=True) + + print(f"[http] HTTP mode completed for {self.api_config.config}", flush=True) diff --git a/tester/special_compare/__init__.py b/tester/special_compare/__init__.py new file mode 100644 index 00000000..5e9137d4 --- /dev/null +++ b/tester/special_compare/__init__.py @@ -0,0 +1,147 @@ +""" +device_vs_gpu 特殊对比注册表。 + +每个 API 的特殊对比逻辑放在本目录下独立的文件中（如 argsort.py）。 + +添加新 API 只需： + 1. 在本目录新建一个文件，如 topk.py + 2. 在文件中编写对比函数并用装饰器注册 + 3. 不需要修改任何其他文件，系统会自动发现新文件 + +对比函数签名： + compare_forward(local_output, remote_output, api_config, tester) -> None + compare_backward(local_grads, remote_grads, api_config, tester) -> None + 失败时抛出 AssertionError（与 np.testing.assert_allclose 行为一致），成功返回 None。 + +参数说明： + local_output / local_grads: XPU 侧的输出/梯度（paddle.Tensor 或 list[Tensor]） + remote_output / remote_grads: GPU 侧的输出/梯度（从 BOS 下载并 paddle.load 还原） + api_config: APIConfig 实例，提供 api_name / args / kwargs + tester: APITestPaddleDeviceVSGPU 实例，提供： + - tester.paddle_args[0] 原始输入张量（XPU 侧，与 GPU 侧同数据） + - tester.atol / tester.rtol 命令行配置的精度容差 +""" + +from __future__ import annotations + +import importlib +import pkgutil +from pathlib import Path + +_FORWARD_REGISTRY: dict[str, callable] = {} +_BACKWARD_REGISTRY: dict[str, callable] = {} + + +class SkipComparison(Exception): + """ + 在特殊对比函数中抛出此异常，表示跳过本次对比，结果记为 skip。 + + 用法： + @register_forward("paddle.some_api") + def compare_some_api(local_output, remote_output, api_config, tester): + raise SkipComparison("XPU 暂不支持该 API 的精度对比") + + 或直接使用快捷装饰器： + @register_skip("paddle.some_api", "paddle.Tensor.some_api") + def _skip_some_api(): ... # 函数体不重要，永远不会被执行 + """ + + +def register_forward(*api_names: str): + """ + 装饰器：注册一个前向（Forward output）特殊对比函数。 + + 可同时注册多个 API 别名到同一个函数： + @register_forward("paddle.argsort", "paddle.Tensor.argsort") + def compare_argsort_forward(local_output, remote_output, api_config, tester): + ... + """ + def decorator(fn): + for name in api_names: + if name in _FORWARD_REGISTRY: + raise ValueError( + f"register_forward: '{name}' 已注册到 {_FORWARD_REGISTRY[name].__name__!r}，" + "请勿重复注册" + ) + _FORWARD_REGISTRY[name] = fn + return fn + return decorator + + +def register_backward(*api_names: str): + """ + 装饰器：注册一个反向（Backward gradient）特殊对比函数。 + + 可同时注册多个 API 别名到同一个函数： + @register_backward("paddle.some_api", "paddle.Tensor.some_api") + def compare_some_api_backward(local_grads, remote_grads, api_config, tester): + ... + """ + def decorator(fn): + for name in api_names: + if name in _BACKWARD_REGISTRY: + raise ValueError( + f"register_backward: '{name}' 已注册到 {_BACKWARD_REGISTRY[name].__name__!r}，" + "请勿重复注册" + ) + _BACKWARD_REGISTRY[name] = fn + return fn + return decorator + + +def get_forward_compare(api_name: str): + """ + 返回 api_name 已注册的前向对比函数，未注册则返回 None。 + 调用方应检查返回值，为 None 时回退到默认对比逻辑。 + """ + return _FORWARD_REGISTRY.get(api_name, None) + + +def get_backward_compare(api_name: str): + """ + 返回 api_name 已注册的反向对比函数，未注册则返回 None。 + 调用方应检查返回值，为 None 时回退到默认对比逻辑。 + """ + return _BACKWARD_REGISTRY.get(api_name, None) + + +def register_skip(*api_names: str): + """ + 快捷装饰器：将 API 的 forward 和 backward 对比均标记为跳过。 + 等价于同时注册 forward 和 backward 函数，两者都抛出 SkipComparison。 + + 用法： + @register_skip("paddle.some_api", "paddle.Tensor.some_api") + def _(): ... # 函数体不会被执行，名字不重要 + """ + def decorator(fn): + def _skip(*, reason=""): + raise SkipComparison(reason or f"{api_names[0]} 已注册为跳过对比") + + for name in api_names: + if name in _FORWARD_REGISTRY: + raise ValueError( + f"register_skip: '{name}' 已注册到 {_FORWARD_REGISTRY[name].__name__!r}" + ) + if name in _BACKWARD_REGISTRY: + raise ValueError( + f"register_skip: '{name}' 已注册到 {_BACKWARD_REGISTRY[name].__name__!r}" + ) + + def _skip_fn(*args, _name=name, **kwargs): + raise SkipComparison(f"{_name} 已注册为跳过对比") + + _FORWARD_REGISTRY[name] = _skip_fn + _BACKWARD_REGISTRY[name] = _skip_fn + return fn + return decorator + + +# --------------------------------------------------------------------------- +# 自动发现并导入本目录下所有子模块 +# 每个子模块文件顶层的 @register_forward / @register_backward 装饰器会在 import +# 时立即执行，完成注册。新增 API 文件只需放入本目录，无需修改此文件。 +# --------------------------------------------------------------------------- +_pkg_dir = Path(__file__).parent +for _mod_info in pkgutil.iter_modules([str(_pkg_dir)]): + importlib.import_module(f".{_mod_info.name}", package=__name__) diff --git a/tester/special_compare/argsort.py b/tester/special_compare/argsort.py new file mode 100644 index 00000000..ff03c2c4 --- /dev/null +++ b/tester/special_compare/argsort.py @@ -0,0 +1,82 @@ +""" +paddle.argsort / paddle.Tensor.argsort 的特殊对比逻辑。 + +问题背景 +-------- +argsort 在输入存在相同值（ties）时，XPU 和 GPU 打破平局的方式不同，会产生 +合法但不同的排序索引，导致逐元素对比误判为 accuracy_error。 + +对比策略 +-------- +不直接比较索引，而是用两边的索引分别 gather 原始输入的值，再对比 gather 后的值。 +如果两边的排序都是合法的，gather 后的值应完全一致（bitwise），因为是对同一数组的 +两种等价排列。 +""" + +import numpy as np + +from . import register_forward + + +@register_forward("paddle.argsort", "paddle.Tensor.argsort") +def compare_argsort_forward(local_output, remote_output, api_config, tester): + """ + argsort 前向特殊对比：gather 原始输入值后比较，而非直接比较索引。 + + tester.paddle_args[0] 是排序前的输入张量，在 _run_paddle 执行后仍保留在实例上。 + 由于两侧使用相同 random_seed 生成输入，GPU 侧与 XPU 侧输入数据完全相同， + 因此可用 XPU 侧的输入张量作为两边 gather 的数据源。 + """ + # Bug 1 fix: 先从 paddle_args 取，若为空再 fallback 到 paddle_kwargs["x"] + if tester.paddle_args: + input_tensor = tester.paddle_args[0] + elif "x" in tester.paddle_kwargs: + input_tensor = tester.paddle_kwargs["x"] + else: + raise AssertionError( + "compare_argsort_forward: paddle_args 为空且 paddle_kwargs 中没有 'x'，无法获取原始输入张量" + ) + + input_np = input_tensor.cpu().numpy() + + # 提取 axis 参数 + # paddle.argsort 签名：argsort(x, axis=-1, descending=False, stable=False) + axis = api_config.kwargs.get("axis", None) + if axis is None and len(api_config.args) > 1 and isinstance(api_config.args[1], int): + axis = api_config.args[1] + if axis is None: + axis = -1 + if isinstance(axis, int) and axis < 0: + axis = axis + input_np.ndim + + local_indices_np = local_output.numpy() # int64，形状与输入相同 + remote_indices_np = remote_output.numpy() # int64，形状与输入相同 + + # Bug 2 fix: 0-dim tensor（ndim==0）时 axis 无意义，直接比较索引本身 + if input_np.ndim == 0: + np.testing.assert_array_equal( + local_indices_np, + remote_indices_np, + err_msg=( + f"argsort 特殊对比失败（0-dim tensor，索引不一致）：{api_config.config}" + ), + ) + return + + # 用各自的索引 gather 原始输入值 + local_vals = np.take_along_axis(input_np, local_indices_np, axis=axis) + remote_vals = np.take_along_axis(input_np, remote_indices_np, axis=axis) + + # 两边都是对同一数组的合法排序，gather 后的值应 bitwise 完全相同 + # （gather 本身只是索引查找，不涉及浮点运算，无需容差） + np.testing.assert_array_equal( + local_vals, + remote_vals, + err_msg=( + f"argsort 特殊对比失败（gather values 不一致）：{api_config.config}\n" + "这说明两侧排序结果不是同一输入的等价排列，可能存在真实精度问题。" + ), + ) + + +# argsort 输出为整数索引，不涉及浮点梯度，无需注册 backward 特殊对比。 diff --git a/tester/special_compare/grid_sample.py b/tester/special_compare/grid_sample.py new file mode 100644 index 00000000..51a251a1 --- /dev/null +++ b/tester/special_compare/grid_sample.py @@ -0,0 +1,186 @@ +""" +paddle.nn.functional.grid_sample 的特殊对比逻辑。 + +问题背景 +-------- +grid_sample 在 mode='nearest' 时，网格坐标落在两个像素边界正中间（tie）时， +XPU 和 GPU 的取整方向可能不同，选取不同的源像素。 + +观测到的失败案例（fuzzy_run）： + Case 1: mode='nearest', Tensor([1, 4, 280, 350]), Tensor([1, 280, 350, 2]) + forward max_abs_diff=0（两侧选择了相同的像素，前向一致）， + backward gradient[0] max_abs_diff=0.381912（梯度回流到不同源像素位置）。 + + Case 2: mode 未指定（默认 bilinear）, Tensor([100, 1, 176, 176]), Tensor([100, 1, 12544, 2]) + forward max_abs_diff=0， + backward gradient[1] max_abs_diff=0.000631332（大张量 bilinear backward + 的浮点累积顺序差异，超出 dtype 级 atol=1e-4，但在命令行默认 atol=1e-2 内）。 + +对比策略 +-------- +Forward： + - 任何 mode：使用与默认逻辑相同的容差（tester._resolve_atol_rtol）， + 显式注册仅为保持对称性与可审查性。 + - 若 mode='nearest' 但前向出现超出容差的差异，说明两端采样了不同值的像素， + 这仍属于 tie-breaking 合法行为（两端都选择合法近邻），接受此类差异。 + 对 nearest mode 使用 tester.atol / tester.rtol（命令行级，较宽松）。 + +Backward： + - mode='nearest'：tie-breaking 只改变梯度回流的落点，不改变梯度总量。 + 对展平后的梯度排序后比较，使用 tester.atol / tester.rtol。 + - mode!='nearest'：大张量 bilinear/bicubic backward 存在浮点加法顺序差异， + 使用 tester.atol / tester.rtol（命令行级容差，通常为 1e-2）。 + 注意：这比 device_vs_gpu_config.yaml 中 dtype 级 float32=[1e-4, 1e-4] 更宽松， + 是为了容纳大张量（如 12544 个网格点）反向传播的累积误差。 +""" + +from __future__ import annotations + +import numpy as np +import paddle + +from . import register_backward, register_forward + + +def _get_mode(api_config) -> str: + """ + 从 api_config.args / api_config.kwargs 中提取 mode 参数值。 + + paddle.nn.functional.grid_sample 签名： + grid_sample(x, grid, mode='bilinear', padding_mode='zeros', + align_corners=True, name=None) + mode 在位置参数中的索引为 2（0-based，x=0, grid=1, mode=2）。 + """ + if "mode" in api_config.kwargs: + return str(api_config.kwargs["mode"]) + if len(api_config.args) > 2: + return str(api_config.args[2]) + return "bilinear" + + +@register_forward("paddle.nn.functional.grid_sample") +def compare_grid_sample_forward(local_output, remote_output, api_config, tester): + """ + grid_sample 前向特殊对比。 + + - mode != 'nearest'：使用 _resolve_atol_rtol 的标准容差（与默认逻辑相同）。 + - mode == 'nearest'：使用命令行级宽松容差（tester.atol / tester.rtol）， + 因为 tie-breaking 可能导致两端采样不同值的源像素。 + """ + mode = _get_mode(api_config) + + if isinstance(local_output, paddle.Tensor) and isinstance(remote_output, paddle.Tensor): + local_np = local_output.numpy() + remote_np = remote_output.numpy() + tester._print_diff("Forward", local_np, remote_np) + if mode == "nearest": + atol = tester.atol + rtol = tester.rtol + else: + dtype_str = str(local_output.dtype).split(".")[-1] + atol, rtol = tester._resolve_atol_rtol(dtype_str) + tester._assert_close(local_np, remote_np, atol, rtol) + elif isinstance(local_output, (list, tuple)) and isinstance(remote_output, (list, tuple)): + for i, (local_item, remote_item) in enumerate(zip(local_output, remote_output)): + if isinstance(local_item, paddle.Tensor) and isinstance(remote_item, paddle.Tensor): + local_np = local_item.numpy() + remote_np = remote_item.numpy() + tester._print_diff(f"Forward output[{i}]", local_np, remote_np) + if mode == "nearest": + atol = tester.atol + rtol = tester.rtol + else: + dtype_str = str(local_item.dtype).split(".")[-1] + atol, rtol = tester._resolve_atol_rtol(dtype_str) + tester._assert_close(local_np, remote_np, atol, rtol) + print(f"[compare] Forward output[{i}] comparison passed", flush=True) + else: + local_np = ( + local_output.numpy() + if isinstance(local_output, paddle.Tensor) + else np.array(local_output) + ) + remote_np = ( + remote_output.numpy() + if isinstance(remote_output, paddle.Tensor) + else np.array(remote_output) + ) + np.testing.assert_allclose( + local_np, remote_np, atol=tester.atol, rtol=tester.rtol, equal_nan=True + ) + + +@register_backward("paddle.nn.functional.grid_sample") +def compare_grid_sample_backward(local_grads, remote_grads, api_config, tester): + """ + grid_sample 反向特殊对比。 + + grid_sample 有两个可导输入，对应两组梯度： + gradient[0]：对输入图像张量 x 的梯度（形状与 x 相同） + gradient[1]：对网格张量 grid 的梯度（形状与 grid 相同） + + mode='nearest' 策略： + tie-breaking 只改变梯度回流位置，不改变总梯度量。将两端梯度展平后排序， + 再用 tester.atol / tester.rtol 比较，以接受等价的不同分配。 + + mode!='nearest' 策略： + bilinear/bicubic 反向在大张量上存在浮点加法顺序差异（atomic add 顺序不同）， + 使用 tester.atol / tester.rtol（命令行级，默认 1e-2）直接比较， + 以容纳大张量累积误差（实测 max_abs_diff=0.000631332）。 + """ + mode = _get_mode(api_config) + atol = tester.atol + rtol = tester.rtol + + def _compare_single_grad(local_grad, remote_grad, idx_label: str): + if not ( + isinstance(local_grad, paddle.Tensor) and isinstance(remote_grad, paddle.Tensor) + ): + return + + local_np = local_grad.numpy() + remote_np = remote_grad.numpy() + + if mode == "nearest": + # 排序后对比：tie-breaking 只改变梯度落点，排序后应一致 + local_sorted = np.sort(local_np.flatten()) + remote_sorted = np.sort(remote_np.flatten()) + tol = atol + rtol * np.abs(remote_sorted) + max_abs_diff = float(np.max(np.abs(local_sorted - remote_sorted))) + print( + f"[compare] Backward gradient[{idx_label}] (nearest, sorted) " + f"max_abs_diff={max_abs_diff:.6g}", + flush=True, + ) + if np.any(np.abs(local_sorted - remote_sorted) > tol): + raise AssertionError( + f"grid_sample 反向对比失败（nearest 模式，排序后梯度不一致）：" + f"{api_config.config}\n" + f"grad[{idx_label}] sorted max_abs_diff={max_abs_diff:.6g}, " + f"atol={atol:.6g}, rtol={rtol:.6g}" + ) + print( + f"[compare] Backward gradient[{idx_label}] comparison passed" + f" (nearest sorted, atol={atol:.3g})", + flush=True, + ) + else: + # bilinear/bicubic：直接比较，使用命令行级容差 + tester._print_diff(f"Backward gradient[{idx_label}]", local_np, remote_np) + tester._assert_close(local_np, remote_np, atol, rtol) + print( + f"[compare] Backward gradient[{idx_label}] comparison passed" + f" (mode={mode}, atol={atol:.3g})", + flush=True, + ) + + if isinstance(local_grads, (list, tuple)) and isinstance(remote_grads, (list, tuple)): + for i, (lg, rg) in enumerate(zip(local_grads, remote_grads)): + _compare_single_grad(lg, rg, str(i)) + elif isinstance(local_grads, paddle.Tensor) and isinstance(remote_grads, paddle.Tensor): + _compare_single_grad(local_grads, remote_grads, "0") + else: + raise AssertionError( + f"compare_grid_sample_backward: unexpected grad type " + f"local={type(local_grads)}, remote={type(remote_grads)}" + ) diff --git a/tester/special_compare/linalg_sign.py b/tester/special_compare/linalg_sign.py new file mode 100644 index 00000000..e0d7b0ef --- /dev/null +++ b/tester/special_compare/linalg_sign.py @@ -0,0 +1,172 @@ +""" +特征向量 / 奇异向量符号歧义的特殊对比逻辑。 + +问题背景 +-------- +对于对称矩阵的特征向量 v，若 A @ v = λ·v，则 -v 也满足同样的方程。 +因此特征向量的符号是任意的（v 和 -v 均合法）。 +类似地，对于奇异值分解 A = U @ diag(S) @ Vh，若将 U 的某列乘以 -1， +只需同时将 Vh 对应行也乘以 -1，分解结果保持不变。 + +XPU 和 GPU 独立选择各自的符号约定，导致逐元素比较时差异高达 2.0（diff = v - (-v) = 2v）， +即使两侧结果都完全正确。 + +涉及 API +-------- +- paddle.linalg.eigh(x, UPLO) → (eigenvalues, eigenvectors) + eigenvalues：实数标量，符号确定，正常比较。 + eigenvectors：列向量符号任意，对比 |V| 而非 V。 + +- paddle.linalg.svd(x, full_matrices) → (U, S, Vh) + S：奇异值，非负且唯一，正常比较。 + U：列向量符号任意，对比 |U|。 + Vh：行向量符号任意，对比 |Vh|。 + Backward：SVD 反向梯度依赖 U/Vh 的符号约定，XPU/GPU 符号不同时梯度不同。 + → 反向梯度跳过比较（skip）。 + +- paddle.linalg.svd_lowrank(x, q, niter, M) → (U, S, Vh) + svd_lowrank 使用随机化投影算法，输出本身是不确定的近似值。 + 奇异值、U、Vh 在 XPU/GPU 上使用独立的随机数，结果无法对比。 + → 无条件跳过。 +""" + +from __future__ import annotations + +import numpy as np + +from . import SkipComparison, register_backward, register_forward, register_skip + + +def _to_np(t) -> np.ndarray: + """paddle.Tensor → numpy array，复数转 complex128，实数转 float64。""" + arr = t.numpy() + if np.issubdtype(arr.dtype, np.complexfloating): + return arr.astype(np.complex128) + return arr.astype(np.float64) + + +def _assert_allclose_abs(local_t, remote_t, atol: float, rtol: float, label: str, config_str: str): + """比较 |local_t| 与 |remote_t|，容差 atol/rtol。""" + local_np = np.abs(_to_np(local_t)) + remote_np = np.abs(_to_np(remote_t)) + try: + np.testing.assert_allclose(local_np, remote_np, atol=atol, rtol=rtol) + except AssertionError as e: + raise AssertionError( + f"{label} |absolute value| 比较失败（atol={atol}, rtol={rtol}）：" + f"{config_str}\n{e}" + ) from None + + +def _assert_allclose(local_t, remote_t, atol: float, rtol: float, label: str, config_str: str): + """直接比较 local_t 与 remote_t，容差 atol/rtol。""" + local_np = _to_np(local_t) + remote_np = _to_np(remote_t) + try: + np.testing.assert_allclose(local_np, remote_np, atol=atol, rtol=rtol) + except AssertionError as e: + raise AssertionError( + f"{label} 精度比较失败（atol={atol}, rtol={rtol}）：" + f"{config_str}\n{e}" + ) from None + + +# --------------------------------------------------------------------------- +# paddle.linalg.svd_lowrank → 无条件跳过（随机化算法） +# --------------------------------------------------------------------------- + +@register_skip("paddle.linalg.svd_lowrank") +def _skip_svd_lowrank(): + ... + + +# --------------------------------------------------------------------------- +# paddle.linalg.eigh → (eigenvalues, eigenvectors) +# --------------------------------------------------------------------------- + +@register_forward("paddle.linalg.eigh") +def compare_eigh_forward(local_output, remote_output, api_config, tester): + """ + eigh 前向特殊对比。 + + 输出格式：(eigenvalues: Tensor, eigenvectors: Tensor) + - eigenvalues：实数，符号确定，正常比较。 + - eigenvectors：符号任意，对比 |V|。 + """ + if not isinstance(local_output, (list, tuple)) or len(local_output) < 2: + dtype_str = str(local_output.dtype).split(".")[-1] + atol, rtol = tester._resolve_atol_rtol(dtype_str) + _assert_allclose(local_output, remote_output, atol, rtol, + "eigh single output", api_config.config) + return + + eigenvalues_local, eigenvectors_local = local_output[0], local_output[1] + eigenvalues_remote, eigenvectors_remote = remote_output[0], remote_output[1] + + dtype_str = str(eigenvalues_local.dtype).split(".")[-1] + atol, rtol = tester._resolve_atol_rtol(dtype_str) + + # 1. 特征值：直接比较 + _assert_allclose(eigenvalues_local, eigenvalues_remote, atol, rtol, + "eigh eigenvalues", api_config.config) + + # 2. 特征向量：|V| 比较（消除符号歧义） + _assert_allclose_abs(eigenvectors_local, eigenvectors_remote, atol, rtol, + "eigh eigenvectors", api_config.config) + + +# --------------------------------------------------------------------------- +# paddle.linalg.svd → (U, S, Vh) +# --------------------------------------------------------------------------- + +@register_forward("paddle.linalg.svd") +def compare_svd_forward(local_output, remote_output, api_config, tester): + """ + svd 前向特殊对比。 + + 输出格式：(U: Tensor, S: Tensor, Vh: Tensor) + - S：奇异值，非负且唯一，正常比较。 + - U：列向量符号任意，对比 |U|。 + - Vh：行向量符号任意，对比 |Vh|。 + """ + if not isinstance(local_output, (list, tuple)) or len(local_output) < 3: + dtype_str = str(local_output.dtype).split(".")[-1] + atol, rtol = tester._resolve_atol_rtol(dtype_str) + _assert_allclose(local_output, remote_output, atol, rtol, + "svd single output", api_config.config) + return + + U_local, S_local, Vh_local = local_output[0], local_output[1], local_output[2] + U_remote, S_remote, Vh_remote = remote_output[0], remote_output[1], remote_output[2] + + dtype_str = str(S_local.dtype).split(".")[-1] + atol, rtol = tester._resolve_atol_rtol(dtype_str) + + # 1. 奇异值：直接比较（非负，无歧义） + _assert_allclose(S_local, S_remote, atol, rtol, + "svd singular values S", api_config.config) + + # 2. U：|U| 比较 + _assert_allclose_abs(U_local, U_remote, atol, rtol, + "svd U", api_config.config) + + # 3. Vh：|Vh| 比较 + _assert_allclose_abs(Vh_local, Vh_remote, atol, rtol, + "svd Vh", api_config.config) + + +@register_backward("paddle.linalg.svd") +def compare_svd_backward(local_grads, remote_grads, api_config, tester): + """ + svd 反向特殊对比。 + + SVD 的反向梯度 ∂L/∂x 依赖前向的 U 和 Vh 的具体符号约定。 + 当 XPU 与 GPU 对某列/行选取了相反的符号时，该列对应的梯度分量方向也相反， + 导致 max_abs_diff ≈ 2 * |梯度值|，属于合法的符号歧义传播，而非真实 Bug。 + + 处理策略：跳过反向梯度比较（与 sort/topk 的 tie-breaking backward 策略一致）。 + """ + raise SkipComparison( + "svd backward gradient depends on sign convention of U/Vh; " + "XPU and GPU may choose opposite signs for tied singular vectors — skip." + ) diff --git a/tester/special_compare/max_pool.py b/tester/special_compare/max_pool.py new file mode 100644 index 00000000..cbbfd830 --- /dev/null +++ b/tester/special_compare/max_pool.py @@ -0,0 +1,108 @@ +""" +paddle.nn.functional.max_pool1d / max_pool2d 的特殊对比逻辑。 + +问题背景 +-------- +max pooling 在 pooling window 内存在相同最大值（tie）时，XPU 和 GPU 选择不同位置 +作为"赢家"。前向输出（池化值本身）完全相同，但反向梯度流向不同位置，导致 +backward max_rel_diff=2.0，被误判为 accuracy_error。 + +对比策略 +-------- +Forward： + - 当 return_mask=True 时，输出是 (out, mask)；只比较 out[0]（池化值）。 + - 当 return_mask=False 时，输出是普通 Tensor，直接 bitwise 比较。 + - 使用 np.testing.assert_array_equal（无需容差，两边池化值应完全一致）。 + +Backward： + - local_grads / remote_grads 是只含一个元素的 list，元素形状与输入相同。 + - Tie-breaking 只改变梯度落点，不改变梯度总量。对于每个 pooling window， + XPU 和 GPU 分配的梯度总量应该相同（只是落在不同的 tied 位置上）。 + - 策略：将展平后的梯度排序后进行对比。当所有 tied 元素接收到来自输出侧 + 相同的梯度时，排序后的梯度向量在两边应该完全一致。 + - 使用 tester.atol / tester.rtol 作为容差。 +""" + +import numpy as np + +from . import register_backward, register_forward + + +@register_forward( + "paddle.nn.functional.max_pool1d", + "paddle.nn.functional.max_pool2d", +) +def compare_max_pool_forward(local_output, remote_output, api_config, tester): + """ + max_pool1d / max_pool2d 前向特殊对比。 + + 当 return_mask=True 时，输出为 (out, mask)；forward 值 out[0] 应 bitwise 相同。 + 当 return_mask=False 时，输出为普通 Tensor；同样应 bitwise 相同。 + mask 不做对比（tie-breaking 导致两端 mask 不同是预期行为）。 + """ + # 提取 pooled 值（忽略 mask） + if isinstance(local_output, (tuple, list)): + local_out = local_output[0] + else: + local_out = local_output + + if isinstance(remote_output, (tuple, list)): + remote_out = remote_output[0] + else: + remote_out = remote_output + + local_np = local_out.numpy() + remote_np = remote_out.numpy() + + np.testing.assert_array_equal( + local_np, + remote_np, + err_msg=( + f"max_pool 前向对比失败（池化值不一致，非 tie-breaking 问题）：" + f"{api_config.config}" + ), + ) + + +@register_backward( + "paddle.nn.functional.max_pool1d", + "paddle.nn.functional.max_pool2d", +) +def compare_max_pool_backward(local_grads, remote_grads, api_config, tester): + """ + max_pool1d / max_pool2d 反向特殊对比。 + + Tie-breaking 只影响梯度落点，排序后两端梯度应一致。 + local_grads / remote_grads 均为 list[Tensor]，每个 Tensor 形状与输入相同。 + """ + # 支持 list/tuple 和单个 Tensor + if isinstance(local_grads, (list, tuple)): + local_grad_list = list(local_grads) + else: + local_grad_list = [local_grads] + + if isinstance(remote_grads, (list, tuple)): + remote_grad_list = list(remote_grads) + else: + remote_grad_list = [remote_grads] + + for i, (local_g, remote_g) in enumerate(zip(local_grad_list, remote_grad_list)): + local_np = local_g.numpy() + remote_np = remote_g.numpy() + + # 排序后对比：tie-breaking 只改变梯度落点，排好序后应一致 + local_sorted = np.sort(local_np.flatten()) + remote_sorted = np.sort(remote_np.flatten()) + + atol = tester.atol + rtol = tester.rtol + + max_abs_diff = np.max(np.abs(local_sorted - remote_sorted)) + tol = atol + rtol * np.abs(remote_sorted) + if np.any(np.abs(local_sorted - remote_sorted) > tol): + raise AssertionError( + f"max_pool 反向对比失败（排序后梯度不一致，非 tie-breaking 问题）：" + f"{api_config.config}\n" + f"grad[{i}] sorted max_abs_diff={max_abs_diff:.6g}, " + f"atol={atol:.6g}, rtol={rtol:.6g}" + ) diff --git a/tester/special_compare/nondeterministic.py b/tester/special_compare/nondeterministic.py new file mode 100644 index 00000000..76d3f5d2 --- /dev/null +++ b/tester/special_compare/nondeterministic.py @@ -0,0 +1,22 @@ +""" +非确定性 API 的跳过注册。 + +以下 API 的输出在两端之间天然不一致，不应进行精度对比： + +- paddle.empty / paddle.Tensor.empty_like：分配未初始化内存，两端垃圾值不同。 +- paddle.empty_like：同上。 +- paddle.multinomial：随机采样，结果天然不确定。 +""" + +from . import register_skip + + +@register_skip( + "paddle.empty", + "paddle.empty_like", + "paddle.Tensor.empty_like", + "paddle.multinomial", + "paddle.nn.functional.rrelu", +) +def _skip(): + ... diff --git a/tester/special_compare/random_ops.py b/tester/special_compare/random_ops.py new file mode 100644 index 00000000..674480f3 --- /dev/null +++ b/tester/special_compare/random_ops.py @@ -0,0 +1,279 @@ +""" +随机/非确定性算子的跳过逻辑。 + +包含两类处理策略： + +1. 无条件跳过（unconditional skip） + 以下 API 在 XPU 与 GPU 之间使用各自独立的随机数生成器，输出天然不一致， + 无论参数如何均应跳过精度对比： + + - paddle.normal / paddle.standard_normal / paddle.log_normal + - paddle.poisson / paddle.bernoulli / paddle.standard_gamma / paddle.binomial + - paddle.Tensor.normal_ / paddle.Tensor.exponential_ / paddle.Tensor.cauchy_ + - paddle.Tensor.geometric_ / paddle.Tensor.log_normal_ / paddle.Tensor.bernoulli_ + - paddle.nn.functional.gumbel_softmax + - paddle.geometric.sample_neighbors + - paddle.nn.functional.fractional_max_pool2d / fractional_max_pool3d + +2. 条件跳过（conditional skip） + 仅当 training=True 时输出随机，training=False 时完全确定： + + - paddle.nn.functional.dropout + - paddle.nn.functional.dropout2d + - paddle.nn.functional.dropout3d + - paddle.nn.functional.alpha_dropout + + 对于 paddle.incubate.nn.functional.fused_dropout_add，仅当 + training=True 且 p > 0 时跳过。 +""" + +from __future__ import annotations + +import numpy as np + +from . import SkipComparison, register_backward, register_forward, register_skip + +# --------------------------------------------------------------------------- +# 1. 无条件跳过 +# --------------------------------------------------------------------------- + + +@register_skip( + "paddle.normal", + "paddle.standard_normal", + "paddle.log_normal", + "paddle.poisson", + "paddle.bernoulli", + "paddle.standard_gamma", + "paddle.binomial", + "paddle.Tensor.normal_", + "paddle.Tensor.exponential_", + "paddle.Tensor.cauchy_", + "paddle.Tensor.geometric_", + "paddle.Tensor.log_normal_", + "paddle.Tensor.bernoulli_", + "paddle.nn.functional.gumbel_softmax", + "paddle.geometric.sample_neighbors", + "paddle.nn.functional.fractional_max_pool2d", + "paddle.nn.functional.fractional_max_pool3d", +) +def _skip_random_ops(): + ... + + +# --------------------------------------------------------------------------- +# 2. 条件跳过辅助函数 +# --------------------------------------------------------------------------- + + +def _get_training(api_config, training_arg_index: int) -> bool: + """ + 从 api_config 中读取 training 参数的布尔值。 + + 优先从 kwargs 中读取；若不存在则尝试 args[training_arg_index]； + 若两者均无则按默认值 True 处理（所有 dropout 变体的默认值均为 True）。 + + 参数 + ---- + api_config : APIConfig 实例 + training_arg_index: training 在位置参数列表中的 0-base 下标 + """ + if "training" in api_config.kwargs: + return bool(api_config.kwargs["training"]) + if len(api_config.args) > training_arg_index: + return bool(api_config.args[training_arg_index]) + # 默认 True + return True + + +def _get_p(api_config, p_arg_index: int) -> float: + """ + 从 api_config 中读取 p 参数的浮点值。 + + 若 p 是 Tensor（TensorConfig 实例），无法静态判断大小，按 p > 0 处理（保守策略）。 + 若两者均无则按默认值 0.5 处理。 + """ + p_val = api_config.kwargs.get("p", None) + if p_val is None and len(api_config.args) > p_arg_index: + p_val = api_config.args[p_arg_index] + if p_val is None: + return 0.5 # 默认值 + # 若 p 是 TensorConfig，无法静态判断，保守地视为 > 0 + try: + return float(p_val) + except (TypeError, ValueError): + return 0.5 # 保守策略：视为 0.5 > 0 + + +def _compare_tensors(local_output, remote_output, api_config, tester): + """ + 对两个 Tensor（或 Tensor 列表/元组）执行 assert_allclose 对比。 + """ + def _to_np(t): + if hasattr(t, "numpy"): + return t.numpy() + return np.array(t) + + if isinstance(local_output, (list, tuple)): + if len(local_output) != len(remote_output): + raise AssertionError( + f"输出数量不一致: local={len(local_output)}, remote={len(remote_output)}, " + f"config={api_config.config}" + ) + for i, (l, r) in enumerate(zip(local_output, remote_output)): + np.testing.assert_allclose( + _to_np(l), + _to_np(r), + atol=tester.atol, + rtol=tester.rtol, + err_msg=f"输出[{i}] 不一致: {api_config.config}", + ) + else: + np.testing.assert_allclose( + _to_np(local_output), + _to_np(remote_output), + atol=tester.atol, + rtol=tester.rtol, + err_msg=f"输出不一致: {api_config.config}", + ) + + +# --------------------------------------------------------------------------- +# 3. paddle.nn.functional.dropout +# 签名: dropout(x, p=0.5, axis=None, training=True, inplace=False, +# mode='upscale_in_train', name=None) +# training 在位置参数列表中的 0-base 下标 = 3 +# --------------------------------------------------------------------------- + +_DROPOUT_TRAINING_IDX = 3 + + +@register_forward( + "paddle.nn.functional.dropout", +) +def _compare_dropout_forward(local_output, remote_output, api_config, tester): + if _get_training(api_config, _DROPOUT_TRAINING_IDX): + raise SkipComparison( + "paddle.nn.functional.dropout forward: training=True 时随机掩码不可跨设备对比" + ) + _compare_tensors(local_output, remote_output, api_config, tester) + + +@register_backward( + "paddle.nn.functional.dropout", +) +def _compare_dropout_backward(local_grads, remote_grads, api_config, tester): + if _get_training(api_config, _DROPOUT_TRAINING_IDX): + raise SkipComparison( + "paddle.nn.functional.dropout backward: training=True 时随机掩码不可跨设备对比" + ) + _compare_tensors(local_grads, remote_grads, api_config, tester) + + +# --------------------------------------------------------------------------- +# 4. paddle.nn.functional.dropout2d / dropout3d +# 签名: dropout2d(x, p=0.5, training=True, data_format='NCHW', name=None) +# dropout3d(x, p=0.5, training=True, data_format='NCDHW', name=None) +# training 位于 0-base 下标 = 2 +# --------------------------------------------------------------------------- + +_DROPOUT2D3D_TRAINING_IDX = 2 + + +@register_forward( + "paddle.nn.functional.dropout2d", + "paddle.nn.functional.dropout3d", +) +def _compare_dropout2d3d_forward(local_output, remote_output, api_config, tester): + if _get_training(api_config, _DROPOUT2D3D_TRAINING_IDX): + raise SkipComparison( + f"{api_config.api_name} forward: training=True 时随机掩码不可跨设备对比" + ) + _compare_tensors(local_output, remote_output, api_config, tester) + + +@register_backward( + "paddle.nn.functional.dropout2d", + "paddle.nn.functional.dropout3d", +) +def _compare_dropout2d3d_backward(local_grads, remote_grads, api_config, tester): + if _get_training(api_config, _DROPOUT2D3D_TRAINING_IDX): + raise SkipComparison( + f"{api_config.api_name} backward: training=True 时随机掩码不可跨设备对比" + ) + _compare_tensors(local_grads, remote_grads, api_config, tester) + + +# --------------------------------------------------------------------------- +# 5. paddle.nn.functional.alpha_dropout +# 签名: alpha_dropout(x, p=0.5, training=True, name=None) +# training 位于 0-base 下标 = 2 +# --------------------------------------------------------------------------- + +_ALPHA_DROPOUT_TRAINING_IDX = 2 + + +@register_forward( + "paddle.nn.functional.alpha_dropout", +) +def _compare_alpha_dropout_forward(local_output, remote_output, api_config, tester): + if _get_training(api_config, _ALPHA_DROPOUT_TRAINING_IDX): + raise SkipComparison( + "paddle.nn.functional.alpha_dropout forward: training=True 时随机掩码不可跨设备对比" + ) + _compare_tensors(local_output, remote_output, api_config, tester) + + +@register_backward( + "paddle.nn.functional.alpha_dropout", +) +def _compare_alpha_dropout_backward(local_grads, remote_grads, api_config, tester): + if _get_training(api_config, _ALPHA_DROPOUT_TRAINING_IDX): + raise SkipComparison( + "paddle.nn.functional.alpha_dropout backward: training=True 时随机掩码不可跨设备对比" + ) + _compare_tensors(local_grads, remote_grads, api_config, tester) + + +# --------------------------------------------------------------------------- +# 6. paddle.incubate.nn.functional.fused_dropout_add +# 签名: fused_dropout_add(x, y, p=0.5, training=True, mode=..., name=None) +# training 位于 0-base 下标 = 3 +# p 位于 0-base 下标 = 2 +# 仅当 training=True 且 p > 0 时跳过。 +# --------------------------------------------------------------------------- + +_FUSED_DA_TRAINING_IDX = 3 +_FUSED_DA_P_IDX = 2 + + +def _fused_dropout_add_is_random(api_config) -> bool: + training = _get_training(api_config, _FUSED_DA_TRAINING_IDX) + if not training: + return False + p = _get_p(api_config, _FUSED_DA_P_IDX) + return p > 0.0 + + +@register_forward( + "paddle.incubate.nn.functional.fused_dropout_add", +) +def _compare_fused_dropout_add_forward(local_output, remote_output, api_config, tester): + if _fused_dropout_add_is_random(api_config): + raise SkipComparison( + "paddle.incubate.nn.functional.fused_dropout_add forward: " + "training=True 且 p>0 时随机掩码不可跨设备对比" + ) + _compare_tensors(local_output, remote_output, api_config, tester) + + +@register_backward( + "paddle.incubate.nn.functional.fused_dropout_add", +) +def _compare_fused_dropout_add_backward(local_grads, remote_grads, api_config, tester): + if _fused_dropout_add_is_random(api_config): + raise SkipComparison( + "paddle.incubate.nn.functional.fused_dropout_add backward: " + "training=True 且 p>0 时随机掩码不可跨设备对比" + ) + _compare_tensors(local_grads, remote_grads, api_config, tester) diff --git a/tester/special_compare/reduce_max_min.py b/tester/special_compare/reduce_max_min.py new file mode 100644 index 00000000..60b154b5 --- /dev/null +++ b/tester/special_compare/reduce_max_min.py @@ -0,0 +1,242 @@ +""" +paddle.max / paddle.Tensor.max / paddle.Tensor.min 的特殊对比逻辑。 + +问题背景 +-------- +paddle.max(x, axis=...) 和 paddle.Tensor.max(axis=...) 对指定轴做 reduce_max， +返回一个值 Tensor（不是 (values, indices) 元组）。 + +当输入中存在多个相同的最大/最小值（ties）时，XPU 和 GPU 对反传梯度的处理方式 +不同： + - GPU (CUDA)：将 output_grad / k 均匀分配到所有 k 个 tied 位置 + - XPU：将 output_grad 广播到所有 tied 位置（每个都获得完整梯度） + +两种做法都是合法的次梯度（subgradient），不影响最优化的正确性，但数值不同， +会导致直接比较误报。 + +对比策略 +-------- +Forward： + - paddle.max 始终返回单个 Tensor（values），无平局歧义，用默认逻辑对比。 + +Backward： + - 基于前向输出 max_val 和原始输入 x，找到所有 tied 位置（x == max_val）。 + - 验证两侧梯度的非零值都在 tied 位置上（不在 tied 位置的非零梯度 = 真实 bug）。 + - 不比较梯度的具体数值（不同实现对 tied 位置给出不同数值，均合法）。 +""" + +from __future__ import annotations + +import numpy as np + +from . import register_backward + +# --------------------------------------------------------------------------- +# 辅助函数 +# --------------------------------------------------------------------------- + +_DYNAMIC_AXIS = object() # sentinel: axis 是动态 Tensor，无法静态确定 + + +def _get_axis(api_config): + """从 api_config 中解析 axis 参数（返回原始值，含 int/list/tuple/None）。 + + 支持的配置格式： + paddle.max(Tensor(...), axis=1, ...) → kwargs["axis"] = 1 + paddle.max(Tensor(...), 1, ...) → args[1] = 1 + paddle.Tensor.max(Tensor(...), -2, ...) → args[1] = -2 + paddle.Tensor.max(Tensor(...), axis=1, ...) → kwargs["axis"] = 1 + paddle.max(Tensor(...), axis=None, ...) → None（全局 reduce） + paddle.max(Tensor(...), Tensor([2],"int64"), ...) → _DYNAMIC_AXIS（无法静态分析） + """ + # 检查 kwargs["axis"] 是否是 TensorConfig（动态 axis） + from tester.api_config.config_analyzer import TensorConfig + + if "axis" in api_config.kwargs: + axis = api_config.kwargs["axis"] + if isinstance(axis, TensorConfig): + return _DYNAMIC_AXIS + return axis + + if len(api_config.args) > 1: + # args[0] is x (or self for Tensor methods), args[1] might be axis + candidate = api_config.args[1] + if isinstance(candidate, (int, list, tuple)): + return candidate + if isinstance(candidate, TensorConfig): + return _DYNAMIC_AXIS + + return None # 全局 reduce + + +def _normalize_axis(axis, ndim: int): + """将 axis 统一为非负整数列表。 + + 返回： + - list[int] → 有效的 reduce 轴列表 + - None → 全局 reduce（axis=None） + - _DYNAMIC_AXIS → 动态 axis（无法静态分析，应跳过比较） + """ + if axis is _DYNAMIC_AXIS: + return _DYNAMIC_AXIS + if axis is None: + return list(range(ndim)) # 全局 reduce + if isinstance(axis, int): + axes = [axis] + elif isinstance(axis, (list, tuple)): + # 过滤掉 axis 列表中含 TensorConfig 的元素（动态 axis 无法静态分析） + int_axes = [] + for a in axis: + if isinstance(a, int): + int_axes.append(a) + else: + # Contains non-int → fall back to dynamic + return _DYNAMIC_AXIS + axes = int_axes + else: + return _DYNAMIC_AXIS + return [a % ndim if ndim > 0 else a for a in axes] + + +def _check_grads_at_tied_positions( + local_np: np.ndarray, + remote_np: np.ndarray, + input_np: np.ndarray, + reduce_axes: list[int], + atol: float, + label: str, + api_config_str: str, + is_min: bool = False, +): + """ + 验证两侧的梯度非零值都落在 tied（最大/最小值）位置上。 + + 策略： + 1. 沿 reduce_axes 计算 input_np 的 max/min，得到 extreme_vals（keepdim=True）。 + 2. 计算 tied_mask = (input_np == extreme_vals)，形状同 input_np。 + 3. 检查 local_np 和 remote_np 中所有 "显著非零" 的位置是否都在 tied_mask 内。 + "显著非零" = |grad| > atol（避免浮点噪声干扰）。 + """ + # 计算 extreme_vals + if is_min: + extreme_vals = np.min(input_np, axis=tuple(reduce_axes), keepdims=True) + else: + extreme_vals = np.max(input_np, axis=tuple(reduce_axes), keepdims=True) + + tied_mask = (input_np == extreme_vals) # bool array, same shape as input_np + + # 找出显著非零的梯度位置 + # 使用 atol 作为"显著"阈值，避免 float16 舍入误差产生的小噪声 + threshold = max(atol, 1e-7) + + local_nonzero = np.abs(local_np.astype(np.float64)) > threshold + remote_nonzero = np.abs(remote_np.astype(np.float64)) > threshold + + # 非零梯度不在 tied 位置 = 真实 bug + local_invalid = local_nonzero & ~tied_mask + remote_invalid = remote_nonzero & ~tied_mask + + if np.any(local_invalid): + bad_val = float(np.max(np.abs(local_np[local_invalid].astype(np.float64)))) + count = int(np.sum(local_invalid)) + raise AssertionError( + f"{label} [local/XPU] 梯度非零位置不在 tied 区域（count={count}, " + f"max_abs={bad_val:.6g}）：{api_config_str}\n" + "这表明 XPU 将梯度传到了非最大值位置，可能存在真实 bug。" + ) + + if np.any(remote_invalid): + bad_val = float(np.max(np.abs(remote_np[remote_invalid].astype(np.float64)))) + count = int(np.sum(remote_invalid)) + raise AssertionError( + f"{label} [remote/GPU] 梯度非零位置不在 tied 区域（count={count}, " + f"max_abs={bad_val:.6g}）：{api_config_str}\n" + "这表明 GPU 将梯度传到了非最大值位置，可能存在真实 bug。" + ) + + +# --------------------------------------------------------------------------- +# Backward 特殊对比 +# --------------------------------------------------------------------------- + +@register_backward("paddle.max", "paddle.Tensor.max", "paddle.Tensor.min") +def compare_reduce_max_min_backward(local_grads, remote_grads, api_config, tester): + """ + paddle.max / Tensor.max / Tensor.min 反向特殊对比。 + + 验证两侧梯度的非零值都在 tied 位置（最大/最小值所在位置）， + 不比较具体数值（不同平台对 tied 位置的分配策略不同，但都合法）。 + """ + is_min = "min" in api_config.api_name + + # 获取输入张量 + if tester.paddle_args: + input_tensor = tester.paddle_args[0] + elif "x" in tester.paddle_kwargs: + input_tensor = tester.paddle_kwargs["x"] + else: + raise AssertionError( + "compare_reduce_max_min_backward: 无法获取原始输入张量" + ) + + input_np = input_tensor.numpy() + ndim = input_np.ndim + + # 解析 axis + raw_axis = _get_axis(api_config) + reduce_axes = _normalize_axis(raw_axis, ndim) + + # local_grads / remote_grads 是 list[Tensor | None] + if not isinstance(local_grads, (list, tuple)): + local_grads = [local_grads] + if not isinstance(remote_grads, (list, tuple)): + remote_grads = [remote_grads] + + import paddle as _paddle + + for i, (local_g, remote_g) in enumerate(zip(local_grads, remote_grads)): + if local_g is None and remote_g is None: + continue + if local_g is None or remote_g is None: + raise AssertionError( + f"reduce_max_min backward gradient[{i}] 一侧为 None，另一侧不为 None：{api_config.config}" + ) + if not isinstance(local_g, _paddle.Tensor): + continue + + local_np = local_g.numpy() + remote_np = remote_g.numpy() + + dtype_str = str(local_g.dtype).split(".")[-1] + atol, _ = tester._resolve_atol_rtol(dtype_str) + + if reduce_axes is _DYNAMIC_AXIS: + # 动态 axis（TensorConfig），无法静态分析，跳过梯度比较 + print( + f"[compare] reduce_max_min backward gradient[{i}] 动态 axis，跳过比较：{api_config.config}", + flush=True, + ) + continue + + if local_np.ndim == 0 or not reduce_axes: + # 0-dim 或无 reduce 轴：直接比较（无 tied 问题） + a = local_np.astype(np.float64) + b = remote_np.astype(np.float64) + abs_diff = np.abs(a - b) + if np.any(abs_diff > atol): + max_abs = float(np.nanmax(abs_diff)) + raise AssertionError( + f"reduce_max_min backward gradient[{i}] 直接比较失败 " + f"(max_abs_diff={max_abs:.6g})：{api_config.config}" + ) + else: + _check_grads_at_tied_positions( + local_np=local_np, + remote_np=remote_np, + input_np=input_np, + reduce_axes=reduce_axes, + atol=atol, + label=f"Backward gradient[{i}]", + api_config_str=api_config.config, + is_min=is_min, + ) diff --git a/tester/special_compare/roi_align.py b/tester/special_compare/roi_align.py new file mode 100644 index 00000000..91757fb2 --- /dev/null +++ b/tester/special_compare/roi_align.py @@ -0,0 +1,166 @@ +""" +paddle.vision.ops.roi_align 的特殊对比逻辑。 + +问题背景 +-------- +roi_align 在 aligned=True 时，backward 阶段多个 RoI 可能重叠，梯度累积回同一 +输入像素时使用原子加（atomic add）。XPU 与 GPU 的原子加执行顺序不同，导致浮点 +加法顺序不同，产生舍入误差，表现为梯度值在对应位置略有差异。 + +观测到的失败案例（fuzzy_run）： + - Tensor([3, 3, 8, 6], float64), aligned=True: backward max_abs_diff=0.0955747 + - Tensor([1, 1024, 40, 60], float32), aligned=True: backward max_abs_diff=0.000724792 + +前向输出（bilinear interpolation 的 gather 阶段）是确定性的，两端完全一致。 +只有 backward（scatter-add/atomic-add 阶段）才会有差异。 + +对比策略 +-------- +前向：使用与默认逻辑相同的容差（tester.atol / tester.rtol），显式写出以便审查。 + +反向： + - 当 aligned=True 时，atomic-add 顺序不确定，放宽容差： + float32：atol = max(tester.atol, 1e-3) + float64：atol = max(tester.atol, 0.15) + 其他精度（float16 等）：atol = max(tester.atol, 1e-2) + 同时 rtol 沿用 tester.rtol（不放宽相对误差，只放宽绝对误差）。 + - 当 aligned=False 时，无 atomic-add 不确定性，使用默认容差。 + - 若 aligned 参数无法从 api_config 确定，保守地按 aligned=True 处理。 +""" + +from __future__ import annotations + +import numpy as np +import paddle + +from . import register_forward, register_backward + + +def _is_aligned(api_config) -> bool: + """ + 从 api_config.args / api_config.kwargs 中提取 aligned 参数值。 + + paddle.vision.ops.roi_align 签名： + roi_align(x, boxes, boxes_num, output_size, spatial_scale=1.0, + sampling_ratio=-1, aligned=True, name=None) + positional index of aligned: 6 (0-based) + """ + # 优先从 kwargs 中读取 + if "aligned" in api_config.kwargs: + return bool(api_config.kwargs["aligned"]) + + # 从 positional args 中读取（index 6） + if len(api_config.args) > 6: + return bool(api_config.args[6]) + + # aligned 默认值为 True + return True + + +def _resolve_backward_atol(dtype_str: str, base_atol: float) -> float: + """ + 根据数据类型返回放宽后的 backward atol。 + + 两个已知失败案例的实测差异量级： + float64: max_abs_diff=0.0955747 -> 使用 0.15 覆盖 + float32: max_abs_diff=0.000724792 -> 使用 1e-3 覆盖 + """ + if dtype_str == "float64": + return max(base_atol, 0.15) + elif dtype_str == "float32": + return max(base_atol, 1e-3) + else: + # float16 / bfloat16 等精度更低，放宽到 1e-2 + return max(base_atol, 1e-2) + + +@register_forward("paddle.vision.ops.roi_align") +def compare_roi_align_forward(local_output, remote_output, api_config, tester): + """ + roi_align 前向特殊对比：与默认逻辑完全相同的容差，显式注册以便日后审查。 + + 前向是确定性的 bilinear interpolation gather，XPU/GPU 结果应在默认容差内一致。 + 若此处出现失败，说明存在真实的前向精度问题，不应进一步放宽。 + """ + if isinstance(local_output, paddle.Tensor) and isinstance(remote_output, paddle.Tensor): + dtype_str = str(local_output.dtype).split(".")[-1] + atol, rtol = tester._resolve_atol_rtol(dtype_str) + local_np = local_output.numpy() + remote_np = remote_output.numpy() + tester._print_diff("Forward", local_np, remote_np) + tester._assert_close(local_np, remote_np, atol, rtol) + elif isinstance(local_output, (list, tuple)) and isinstance(remote_output, (list, tuple)): + for i, (local_item, remote_item) in enumerate(zip(local_output, remote_output)): + if isinstance(local_item, paddle.Tensor) and isinstance(remote_item, paddle.Tensor): + dtype_str = str(local_item.dtype).split(".")[-1] + atol, rtol = tester._resolve_atol_rtol(dtype_str) + local_np = local_item.numpy() + remote_np = remote_item.numpy() + tester._print_diff(f"Forward output[{i}]", local_np, remote_np) + tester._assert_close(local_np, remote_np, atol, rtol) + else: + local_np = ( + local_output.numpy() + if isinstance(local_output, paddle.Tensor) + else np.array(local_output) + ) + remote_np = ( + remote_output.numpy() + if isinstance(remote_output, paddle.Tensor) + else np.array(remote_output) + ) + np.testing.assert_allclose( + local_np, + remote_np, + atol=tester.atol, + rtol=tester.rtol, + equal_nan=True, + ) + + +@register_backward("paddle.vision.ops.roi_align") +def compare_roi_align_backward(local_grads, remote_grads, api_config, tester): + """ + roi_align 反向特殊对比：当 aligned=True 时放宽 atol，容纳 atomic-add 顺序差异。 + + local_grads / remote_grads 是梯度列表，gradient[0] 是对输入图像张量 x 的梯度。 + boxes 和 boxes_num 不可导，所以通常只有 gradient[0] 有值。 + + 放宽策略： + - aligned=False：无 atomic-add 不确定性，使用默认容差。 + - aligned=True：按 dtype 放宽 atol（见 _resolve_backward_atol），rtol 不变。 + """ + aligned = _is_aligned(api_config) + + def _compare_single_grad(local_grad, remote_grad, idx_label: str): + if not ( + isinstance(local_grad, paddle.Tensor) and isinstance(remote_grad, paddle.Tensor) + ): + return + dtype_str = str(local_grad.dtype).split(".")[-1] + base_atol, rtol = tester._resolve_atol_rtol(dtype_str) + if aligned: + atol = _resolve_backward_atol(dtype_str, base_atol) + else: + atol = base_atol + local_np = local_grad.numpy() + remote_np = remote_grad.numpy() + tester._print_diff(f"Backward gradient[{idx_label}]", local_np, remote_np) + tester._assert_close(local_np, remote_np, atol, rtol) + print( + f"[compare] Backward gradient[{idx_label}] comparison passed" + + (f" (relaxed atol={atol:.3g} for aligned=True)" if aligned else ""), + flush=True, + ) + + if isinstance(local_grads, (list, tuple)) and isinstance(remote_grads, (list, tuple)): + for i, (lg, rg) in enumerate(zip(local_grads, remote_grads)): + _compare_single_grad(lg, rg, str(i)) + elif isinstance(local_grads, paddle.Tensor) and isinstance(remote_grads, paddle.Tensor): + _compare_single_grad(local_grads, remote_grads, "0") + else: + # 兜底：不应到达此分支，但以防万一 + raise AssertionError( + f"compare_roi_align_backward: unexpected grad type " + f"local={type(local_grads)}, remote={type(remote_grads)}" + ) diff --git a/tester/special_compare/sort.py b/tester/special_compare/sort.py new file mode 100644 index 00000000..10808aa0 --- /dev/null +++ b/tester/special_compare/sort.py @@ -0,0 +1,149 @@ +""" +paddle.sort / paddle.Tensor.sort 的特殊对比逻辑。 + +问题背景 +-------- +sort 在输入存在相同值（ties）时，XPU 和 GPU 打破平局的方式不同，会产生 +合法但不同的排序索引。前向输出（排序后的值本身）两边完全一致。但反向时， +梯度通过 sort indices 散射回输入位置：tied 位置的索引不同，导致梯度落在 +不同位置，被误判为 accuracy_error。 + +API 说明 +-------- +paddle.sort(x, axis=-1, ...) 返回单个 Tensor（排序后的值），不返回索引。 +paddle.Tensor.sort 是同一函数的 Tensor 方法形式。 + +对比策略 +-------- +Forward： + - 两边均为排序后的值 Tensor，应 bitwise 完全相同。 + - 使用 np.testing.assert_array_equal（无容差）。 + +Backward： + - local_grads / remote_grads 是 list[Tensor]，每个 Tensor 形状与输入相同。 + - Tie-breaking 只改变梯度落点，不改变梯度值集合。 + - 策略：将梯度沿 sort axis 排序后对比。 + 对于每个 "fiber"（沿 axis 切出的一维切片），排序后的梯度值集合在两边应一致。 + - 若输入为 0-dim tensor（ndim==0），直接对比，无需排序。 +""" + +import numpy as np + +from . import register_backward, register_forward + + +@register_forward("paddle.sort", "paddle.Tensor.sort") +def compare_sort_forward(local_output, remote_output, api_config, tester): + """ + sort 前向特殊对比：直接比较排序后的值（bitwise）。 + + paddle.sort 返回排序后的值 Tensor，两边输入相同，排序值应完全一致。 + """ + # Handle tuple/list output defensively (in case of future API changes) + if isinstance(local_output, (tuple, list)): + local_vals = local_output[0] + else: + local_vals = local_output + + if isinstance(remote_output, (tuple, list)): + remote_vals = remote_output[0] + else: + remote_vals = remote_output + + local_np = local_vals.numpy() + remote_np = remote_vals.numpy() + + np.testing.assert_array_equal( + local_np, + remote_np, + err_msg=( + f"sort 前向对比失败（排序值不一致，可能存在真实精度问题）：" + f"{api_config.config}" + ), + ) + + +@register_backward("paddle.sort", "paddle.Tensor.sort") +def compare_sort_backward(local_grads, remote_grads, api_config, tester): + """ + sort 反向特殊对比：沿 sort axis 排序梯度后对比。 + + Tie-breaking 只影响梯度落点，不改变梯度值集合。对每个 fiber 排序后， + 两边应拥有完全相同的梯度值集合。 + """ + # 获取原始输入张量（用于 ndim 信息） + if tester.paddle_args: + input_tensor = tester.paddle_args[0] + elif "x" in tester.paddle_kwargs: + input_tensor = tester.paddle_kwargs["x"] + else: + raise AssertionError( + "compare_sort_backward: paddle_args 为空且 paddle_kwargs 中没有 'x'，" + "无法获取原始输入张量" + ) + + input_ndim = input_tensor.numpy().ndim + + # 提取 axis 参数 + # paddle.sort 签名：sort(x, axis=-1, descending=False, stable=False) + axis = api_config.kwargs.get("axis", None) + if axis is None and len(api_config.args) > 1 and isinstance(api_config.args[1], int): + axis = api_config.args[1] + if axis is None: + axis = -1 + if isinstance(axis, int) and axis < 0: + axis = axis + input_ndim + + # 标准化 local_grads / remote_grads 为 list + if isinstance(local_grads, (list, tuple)): + local_grad_list = list(local_grads) + else: + local_grad_list = [local_grads] + + if isinstance(remote_grads, (list, tuple)): + remote_grad_list = list(remote_grads) + else: + remote_grad_list = [remote_grads] + + for i, (local_g, remote_g) in enumerate(zip(local_grad_list, remote_grad_list)): + if local_g is None and remote_g is None: + continue + if local_g is None or remote_g is None: + raise AssertionError( + f"sort 反向对比失败：grad[{i}] 一侧为 None，另一侧非 None；" + f"{api_config.config}" + ) + + local_np = local_g.numpy() + remote_np = remote_g.numpy() + + # 0-dim tensor：直接对比，无需排序 + if local_np.ndim == 0: + atol = tester.atol + rtol = tester.rtol + if not np.allclose(local_np, remote_np, atol=atol, rtol=rtol, equal_nan=True): + raise AssertionError( + f"sort 反向对比失败（0-dim gradient 不一致）：" + f"{api_config.config}" + ) + continue + + # 沿 sort axis 排序梯度后对比 + # np.sort 对每个 fiber（axis 方向的切片）独立排序 + local_sorted = np.sort(local_np, axis=axis) + remote_sorted = np.sort(remote_np, axis=axis) + + atol = tester.atol + rtol = tester.rtol + + abs_diff = np.abs(local_sorted.astype(np.float64) - remote_sorted.astype(np.float64)) + max_abs_diff = float(np.nanmax(abs_diff)) if abs_diff.size > 0 else 0.0 + tol = atol + rtol * np.abs(remote_sorted.astype(np.float64)) + + if abs_diff.size > 0 and np.any(abs_diff > tol): + raise AssertionError( + f"sort 反向对比失败（排序后梯度不一致，非 tie-breaking 问题）：" + f"{api_config.config}\n" + f"grad[{i}] sorted max_abs_diff={max_abs_diff:.6g}, " + f"atol={atol:.6g}, rtol={rtol:.6g}" + ) diff --git a/tester/special_compare/topk.py b/tester/special_compare/topk.py new file mode 100644 index 00000000..c3fab75d --- /dev/null +++ b/tester/special_compare/topk.py @@ -0,0 +1,174 @@ +""" +paddle.topk / paddle.Tensor.topk 的特殊对比逻辑。 + +问题背景 +-------- +topk 在输入存在相同值（ties）时，XPU 和 GPU 打破平局的方式不同，会导致： +1. 两侧 values 输出中相同 slot 存放的是不同（但等价）的元素——尤其在 sorted=False 时。 +2. 两侧 indices 输出永远可能不同（tie 时合法索引不唯一）。 + +对比策略 +-------- +**Forward**： + - 对两侧的 values 输出沿 topk 轴排序（升序），再逐元素比较。 + 排序后，两侧的合法 top-k 集合应当完全相同（bitwise），因为值来自 + 同一输入张量，只是顺序/选取位置不同。 + - indices 输出完全忽略（tie 时合法索引不唯一）。 + +**Backward**： + - topk 的反向梯度是将 upstream_grad scatter 回 x 的对应位置。 + 当 tie 导致两侧选取不同索引时，梯度也会 scatter 到不同位置， + 逐元素对比会失败。 + - 解决方法：对两侧输入梯度沿 topk 轴排序后比较。 + 如果两侧 top-k 集合相同（Forward 已验证），则排序后的梯度也应相同。 +""" + +from __future__ import annotations + +import numpy as np + +from . import register_backward, register_forward + + +def _get_topk_axis(api_config, input_ndim: int) -> int: + """从 api_config 中提取 topk 的 axis/dim 参数，归一化为非负整数。 + + paddle.topk 签名：topk(x, k, axis=None, largest=True, sorted=True, ...) + - axis=None 时默认取最后一轴（等价于 axis=-1） + - 也接受 dim 作为别名（Tensor.topk / torch-style alias） + + 对 paddle.Tensor.topk，args 的位置参数顺序为： + args[0]=x, args[1]=k, args[2]=axis, args[3]=largest, args[4]=sorted + """ + kwargs = api_config.kwargs + args = api_config.args + + # 优先从 kwargs 中读取（两个可能的名称：axis / dim） + axis = kwargs.get("axis", None) + if axis is None: + axis = kwargs.get("dim", None) + + # 其次从位置参数中读取（args[2] 对应 axis，适用于 Tensor.topk 的 positional 形式） + if axis is None and len(args) > 2: + candidate = args[2] + if isinstance(candidate, int): + axis = candidate + + # 默认最后一轴 + if axis is None: + axis = -1 + + # 将负数轴转换为非负数 + if isinstance(axis, int) and axis < 0: + axis = axis + input_ndim + + return int(axis) + + +@register_forward("paddle.topk", "paddle.Tensor.topk") +def compare_topk_forward(local_output, remote_output, api_config, tester): + """ + topk 前向特殊对比：对两侧的 values 输出沿 topk 轴排序后比较，忽略 indices。 + + local_output / remote_output 均为 TopKRetType（长度为 2 的具名元组）， + 其中 [0] 为 values，[1] 为 indices。 + """ + local_vals = local_output[0] + remote_vals = remote_output[0] + + local_np = local_vals.numpy() + remote_np = remote_vals.numpy() + + # 0-dim 或 empty tensor：直接比较值（无 tie 问题） + if local_np.ndim == 0 or local_np.size == 0: + np.testing.assert_array_equal( + local_np, + remote_np, + err_msg=( + f"topk 特殊对比失败（0-dim/empty tensor）：{api_config.config}" + ), + ) + return + + axis = _get_topk_axis(api_config, local_np.ndim) + + # 沿 topk 轴排序（升序），消除 tie-breaking 和 sorted=False 引入的顺序差异 + local_sorted = np.sort(local_np, axis=axis) + remote_sorted = np.sort(remote_np, axis=axis) + + # values 来自同一输入，bitwise 应完全一致（gather 不涉及浮点运算） + np.testing.assert_array_equal( + local_sorted, + remote_sorted, + err_msg=( + f"topk 特殊对比失败（排序后的 values 不一致）：{api_config.config}\n" + "这说明两侧的 top-k 集合不同，可能存在真实精度问题（非 tie-breaking 引起）。" + ), + ) + + +@register_backward("paddle.topk", "paddle.Tensor.topk") +def compare_topk_backward(local_grads, remote_grads, api_config, tester): + """ + topk 反向特殊对比：对两侧输入梯度沿 topk 轴排序后比较。 + + topk 的反向梯度是将 upstream_grad scatter 回 x 的对应位置。 + Tie-breaking 导致两侧 scatter 位置不同，但排序后应相同。 + + local_grads / remote_grads 是 list[Tensor]，通常只有一个元素（x 的梯度）。 + """ + # 统一为列表方便逐一处理 + if isinstance(local_grads, (list, tuple)): + local_list = list(local_grads) + remote_list = list(remote_grads) + else: + local_list = [local_grads] + remote_list = [remote_grads] + + for i, (local_g, remote_g) in enumerate(zip(local_list, remote_list)): + # 允许其中一侧梯度为 None（unused input） + if local_g is None and remote_g is None: + continue + if local_g is None or remote_g is None: + # 一侧有梯度一侧没有，属于真实差异 + raise AssertionError( + f"topk backward 特殊对比失败（梯度[{i}] 一侧为 None，另一侧不为 None）：" + f"{api_config.config}" + ) + + local_np = local_g.numpy() + remote_np = remote_g.numpy() + + # 0-dim 或 empty：直接数值对比 + if local_np.ndim == 0 or local_np.size == 0: + atol, rtol = tester.atol, tester.rtol + np.testing.assert_allclose( + local_np, + remote_np, + atol=atol, + rtol=rtol, + equal_nan=True, + err_msg=( + f"topk backward 特殊对比失败（梯度[{i}]，0-dim/empty）：{api_config.config}" + ), + ) + continue + + axis = _get_topk_axis(api_config, local_np.ndim) + + # 沿 topk 轴排序：scatter 位置不同但值集合相同时，排序后应一致 + local_sorted = np.sort(local_np, axis=axis) + remote_sorted = np.sort(remote_np, axis=axis) + + atol, rtol = tester.atol, tester.rtol + np.testing.assert_allclose( + local_sorted, + remote_sorted, + atol=atol, + rtol=rtol, + equal_nan=True, + err_msg=( + f"topk backward 特殊对比失败（梯度[{i}] 排序后不一致）：{api_config.config}\n" + "这说明两侧 scatter 的梯度值集合不同，可能存在真实精度问题。" + ), + )