diff --git a/engineV2.py b/engineV2.py index 805d4604..8a0b3420 100644 --- a/engineV2.py +++ b/engineV2.py @@ -66,6 +66,11 @@ _MEM_SNAPSHOT = None # dict: gpu_id -> (total_gb, used_gb) _MEM_SNAPSHOT_TS = 0.0 _MEM_SNAPSHOT_TTL = 2.0 # seconds — snapshot cache ttl +FATAL_CUDA_EXIT_CODE = 99 +FATAL_OOM_EXIT_CODE = 98 +FATAL_TORCH_EXIT_CODE = 97 +MEMORY_WAIT_SECONDS = 10 +MEMORY_WAIT_LOG_INTERVAL = 60 def cleanup(pool): @@ -402,7 +407,7 @@ def check_gpu_memory(gpu_ids, num_workers_per_gpu, required_memory): # required else min(max_workers, num_workers_per_gpu) ) except pynvml.NVMLError as e: - print(f"[WARNING] Failed to check GPU {gpu_id}: {e!s}", flush=True) + print(f"[warn] Failed to check GPU {gpu_id}: {e!s}", flush=True) continue return available_gpus, max_workers_per_gpu @@ -517,6 +522,7 @@ def run_test_case(api_config_str, options): flush=True, ) + last_memory_log_time = 0 while True: total_memory, used_memory = get_memory_info(gpu_id) free_memory = total_memory - used_memory @@ -524,20 +530,32 @@ def run_test_case(api_config_str, options): if free_memory >= options.required_memory: break + now = time.time() + if now - last_memory_log_time >= MEMORY_WAIT_LOG_INTERVAL: + print( + f"{datetime.now()} device {gpu_id} Free: {free_memory:.1f} GB, " + f"Required: {options.required_memory:.1f} GB. ", + "Waiting for available memory...", + flush=True, + ) + last_memory_log_time = now + time.sleep(MEMORY_WAIT_SECONDS) + + if options.show_runtime_status: + total_memory, used_memory_before = get_memory_info(gpu_id) print( - f"{datetime.now()} device {gpu_id} Free: {free_memory:.1f} GB, " - f"Required: {options.required_memory:.1f} GB. ", - "Waiting for available memory...", + f"{datetime.now()} GPU {gpu_id} memory before: used={used_memory_before:.1f} GB, " + f"free={total_memory - used_memory_before:.1f} GB", flush=True, ) - time.sleep(60) api_config = None case = None try: api_config = APIConfig(api_config_str) except Exception as err: - print(f"[config parse error] {api_config_str} {err!s}", flush=True) + print(f"[config_parse] {api_config_str} {err!s}", flush=True) + write_terminal_log("config_parse", api_config_str) return option_to_class = { @@ -560,16 +578,54 @@ def run_test_case(api_config_str, options): case = test_class(api_config, **kwargs) try: case.test() + if has_terminal_log(api_config_str): + write_checkpoint(api_config_str) except Exception as err: - # if fatal error happens, subprocess need to exit with non-zero status - if "CUDA error" in str(err) or "memory corruption" in str(err): - os._exit(99) - if "CUDA out of memory" in str(err) or "Out of memory error" in str(err): - os._exit(98) - if "AssertionError" in str(err) or "Tensor-likes are not equal" in str(err): - os._exit(1) + err_msg = str(err).lower() + terminal_log_type = get_terminal_log_type(api_config_str) + oom_markers = ( + "cuda out of memory", + "out of memory error", + "resourceexhaustederror", + "out of memory", + "outofmemoryerror", + "cannot allocate memory", + "std::bad_alloc", + "bad allocation", + "memoryerror", + "cublas_status_alloc_failed", + ) + cuda_markers = ( + "cuda error", + "memory corruption", + "illegal memory access", + "invalid configuration argument", + "invalid resource handle", + ) + exit_code = None + if any(marker in err_msg for marker in oom_markers): + exit_code = FATAL_OOM_EXIT_CODE + elif terminal_log_type == "torch_error" and any( + marker in err_msg for marker in cuda_markers + ): + exit_code = FATAL_TORCH_EXIT_CODE + elif any(marker in err_msg for marker in cuda_markers): + exit_code = FATAL_CUDA_EXIT_CODE + if exit_code is not None: + if has_terminal_log(api_config_str): + write_checkpoint(api_config_str) + try: + close_process_files() + finally: + try: + restore_stdio() + finally: + os._exit(exit_code) + if has_terminal_log(api_config_str): + write_checkpoint(api_config_str) + return # if not fatal error, subprocess will be alive and report error - print(f"[test error] {api_config_str}: {err}", flush=True) + print(f"[error] {api_config_str}: {err}", flush=True) raise finally: del test_class, api_config, case @@ -584,6 +640,19 @@ def run_test_case(api_config_str, options): ) and not getattr(options, "use_gpu_cache_mode", False): torch.cuda.empty_cache() paddle.device.cuda.empty_cache() + if options.show_runtime_status: + try: + total_memory, used_memory_after = get_memory_info(gpu_id) + print( + f"{datetime.now()} GPU {gpu_id} memory after cleanup: used={used_memory_after:.1f} GB, " + f"free={total_memory - used_memory_after:.1f} GB", + flush=True, + ) + except Exception as err: + print( + f"{datetime.now()} Failed to read GPU {gpu_id} memory after cleanup: {err}", + flush=True, + ) def main(): @@ -927,7 +996,7 @@ def main(): try: api_config = APIConfig(options.api_config) except Exception as err: - print(f"[config parse error] {options.api_config} {err!s}", flush=True) + print(f"[config_parse] {options.api_config} {err!s}", flush=True) return option_to_class = { @@ -987,7 +1056,7 @@ def main(): or "Error Message Summary" in str(err) ): exit(1) - print(f"[test error] {options.api_config}: {err}", flush=True) + print(f"[error] {options.api_config}: {err}", flush=True) finally: case.clear_tensor() del case @@ -1020,6 +1089,11 @@ def main(): return config_files = [options.api_config_file] + # set log_writer before resume/checkpoint handling + if options.log_dir: + set_test_log_path(options.log_dir) + set_engineV2() + # when engineV2 was interrupted, resume from .tmp dir aggregate_logs(cleanup=True) removed_stale_logs = cleanup_uncheckpointed_result_logs() @@ -1078,11 +1152,6 @@ def main(): if options.test_cpu: print(f"Using {cpu_count()} CPU(s) for paddle in CPU mode.", flush=True) - # set log_writer - if options.log_dir: - set_test_log_path(options.log_dir) - set_engineV2() - # initialize process pool manager = Manager() gpu_worker_list = manager.dict({gpu_id: manager.list() for gpu_id in available_gpus}) @@ -1132,23 +1201,30 @@ def cleanup_handler(*args): if options.show_runtime_status or tested_case % 10000 == 0: print(f"[info] Test case succeeded for {config}", flush=True) except TimeoutError as err: - write_to_log("timeout", config) + write_terminal_log("timeout", config) print( - f"[error] Test case timed out for {config}: {err}", + f"[timeout] {config}: {err}", flush=True, ) except ProcessExpired as err: - # we have caught 99 and 98 error in test class, so we only print info here - # when any cuda error and oom happen, subprocess will crash too, - # these case has been classified to oom and cuda_error and won't be classified to crash - if err.exitcode == 99: + # CUDA, OOM, and Torch fatal errors may also expire the subprocess; + # classify them by dedicated terminal log types instead of falling through to crash. + if err.exitcode == FATAL_CUDA_EXIT_CODE: + write_terminal_log("paddle_cuda", config) + print( + f"[paddle_cuda] {config}: {err}", + flush=True, + ) + elif err.exitcode == FATAL_OOM_EXIT_CODE: + write_terminal_log("oom", config) print( - f"[error] CUDA error for {config}: {err}", + f"[oom] {config}: {err}", flush=True, ) - elif err.exitcode == 98: + elif err.exitcode == FATAL_TORCH_EXIT_CODE: + write_terminal_log("torch_error", config) print( - f"[error] CUDA out of memory for {config}", + f"[torch_error] {config}: {err}", flush=True, ) elif err.exitcode in (-signal.SIGKILL, -signal.SIGTERM): @@ -1159,12 +1235,13 @@ def cleanup_handler(*args): flush=True, ) else: - write_to_log("crash", config) + write_terminal_log("paddle_crash", config) print( - f"[fatal] Worker crashed for {config}: {err}", + f"[paddle_crash] {config}: {err}", flush=True, ) except Exception as err: + checkpoint_ready = False print( f"[warn] Test case failed for {config}: {err}", flush=True, @@ -1176,7 +1253,6 @@ def cleanup_handler(*args): f"[{tested_case}/{all_case}] Testing {config}", flush=True, ) - write_to_log("checkpoint", config) aggregate_logs() pool.close() pool.join() diff --git a/tester/accuracy.py b/tester/accuracy.py index bd5ef30a..98539dd5 100644 --- a/tester/accuracy.py +++ b/tester/accuracy.py @@ -9,7 +9,7 @@ import yaml from .api_config.log_writer import write_to_log -from .base import CUDA_ERROR, CUDA_OOM, APITestBase +from .base import APITestBase from .paddle_to_torch import get_converter # from func_timeout import func_set_timeout @@ -89,45 +89,48 @@ def _reset_random_state(self, seed: int = 42): # @func_set_timeout(600) def test(self): if self.need_skip(): - print("[Skip]", flush=True) + print(f"[skip] {self.api_config.config}", flush=True) + write_to_log("skip", self.api_config.config) return if not self.ana_api_info(): print("ana_api_info failed", flush=True) + write_to_log("config_parse", self.api_config.config) return try: convert_result = self.converter.convert(self.api_config.api_name) except Exception as e: print( - f"[paddle_to_torch] Conversion failed for {self.api_config.config}: {e!s}", + f"[config_convert] Conversion failed for {self.api_config.config}: {e!s}", flush=True, ) - write_to_log("paddle_to_torch_failed", self.api_config.config) + write_to_log("config_convert", self.api_config.config) return if not convert_result.is_supported: print( - f"[paddle_to_torch] Unsupported API {self.api_config.api_name}: {convert_result.error_message}", + f"[config_convert] Unsupported API {self.api_config.api_name}: {convert_result.error_message}", flush=True, ) - write_to_log("paddle_to_torch_failed", self.api_config.config) + write_to_log("config_convert", self.api_config.config) return if not convert_result.code or not convert_result.code.is_valid(): print( - f"[paddle_to_torch] No code generated for {self.api_config.api_name}", + f"[config_convert] No code generated for {self.api_config.api_name}", flush=True, ) - write_to_log("paddle_to_torch_failed", self.api_config.config) + write_to_log("config_convert", self.api_config.config) return try: if not self.gen_numpy_input(): print("gen_numpy_input failed") + write_to_log("config_input", self.api_config.config) return except Exception as err: - print(f"[numpy error] {self.api_config.config}\n{err!s}") + print(f"[config_input] {self.api_config.config}\n{err!s}") traceback.print_exc() - write_to_log("numpy_error", self.api_config.config) + write_to_log("config_input", self.api_config.config) return try: @@ -135,6 +138,7 @@ def test(self): torch.set_default_device(device) if not self.gen_torch_input(): print("gen_torch_input failed", flush=True) + write_to_log("torch_error", self.api_config.config) return # Reseed before executing torch, so that random APIs @@ -199,12 +203,9 @@ def test(self): paddle.base.core.eager._for_test_check_cuda_error() except Exception as err: - print(f"[torch error] {self.api_config.config}\n{err!s}", flush=True) traceback.print_exc() - write_to_log("torch_error", self.api_config.config) - if any(cuda_err in str(err) for cuda_err in CUDA_ERROR) or any( - cuda_err in str(err) for cuda_err in CUDA_OOM - ): + _, fatal = self.report_runtime_error(err, "torch_error", "torch_forward") + if fatal: raise return @@ -227,28 +228,17 @@ def test(self): del inputs_list, result_outputs, result_outputs_grads except Exception as err: if str(err).startswith("Too large tensor to get cached numpy: "): - print(f"[numpy error] {self.api_config.config}\n{err!s}") - write_to_log("numpy_error", self.api_config.config) + print(f"[config_input] {self.api_config.config}\n{err!s}") + write_to_log("config_input", self.api_config.config) return - # some torch backward error can be tolerable, so we catch cuda error here - if any(cuda_err in str(err) for cuda_err in CUDA_ERROR) or any( - cuda_err in str(err) for cuda_err in CUDA_OOM - ): - print( - f"[torch error] backward {self.api_config.config}\n{err!s}", - flush=True, - ) - write_to_log("torch_error", self.api_config.config) + _, fatal = self.report_runtime_error(err, "torch_error", "torch_backward") + if fatal: raise - print(str(err), flush=True) + return try: paddle.base.core.eager._for_test_check_cuda_error() except Exception as err: - print( - f"[torch error] backward {self.api_config.config}\n{err!s}", - flush=True, - ) - write_to_log("torch_error", self.api_config.config) + self.report_runtime_error(err, "torch_error", "torch_backward_cuda_check") raise else: del self.torch_args, self.torch_kwargs @@ -282,6 +272,7 @@ def process_torch_outputs(obj): try: if not self.gen_paddle_input(): print("gen_paddle_input failed") + write_to_log("paddle_error", self.api_config.config) return # Reseed before executing paddle so that random APIs @@ -315,29 +306,17 @@ def process_torch_outputs(obj): else next(iter(self.paddle_kwargs.values())) ) except Exception as err: - if self.should_ignore_paddle_error(str(err)): - print(f"[Pass] {self.api_config.config}", flush=True) - write_to_log("pass", self.api_config.config) - return - if any(cuda_err in str(err) for cuda_err in CUDA_ERROR): - print(f"[cuda error] {self.api_config.config}\n{err!s}", flush=True) - write_to_log("cuda_error", self.api_config.config) - raise - if any(cuda_err in str(err) for cuda_err in CUDA_OOM): - print(f"[oom] {self.api_config.config}\n{err!s}", flush=True) - write_to_log("oom", self.api_config.config) - raise - print(f"[paddle error] {self.api_config.config}\n{err!s}", flush=True) - write_to_log("paddle_error", self.api_config.config) - if self.exit_on_error: + log_type, fatal = self.report_runtime_error( + err, "paddle_error", "paddle_forward", allow_ignore_paddle=True + ) + if fatal or (self.exit_on_error and log_type == "paddle_error"): raise return try: paddle.base.core.eager._for_test_check_cuda_error() except Exception as err: - print(f"[cuda error] {self.api_config.config}\n{err!s}", flush=True) - write_to_log("cuda_error", self.api_config.config) + self.report_runtime_error(err, "paddle_cuda", "paddle_forward_cuda_check") raise paddle_output, torch_output = process_output(self.api_config, paddle_output, torch_output) @@ -355,17 +334,12 @@ def compare_paddle_and_torch(paddle_tensor, torch_tensor, idx=0) -> bool: paddle_tensor, torch_tensor, atol=self.get_atol(), rtol=self.get_rtol() ) except Exception as err: - if self.is_backward: - print( - f"[accuracy error] backward at {idx} {self.api_config.config}\n{err!s}", - flush=True, - ) - else: - print( - f"[accuracy error] at {idx} {self.api_config.config}\n{err!s}", - flush=True, - ) - write_to_log("accuracy_error", self.api_config.config) + phase = "backward" if self.is_backward else "forward" + print( + f"[paddle_accuracy] phase={phase} idx={idx} {self.api_config.config}\n{err!s}", + flush=True, + ) + write_to_log("paddle_accuracy", self.api_config.config) if self.exit_on_error: raise return False @@ -385,10 +359,10 @@ def compare_paddle_and_torch(paddle_tensor, torch_tensor, idx=0) -> bool: ) except Exception as err: print( - f"[not compare] {self.api_config.config}\n{err!s}", + f"[paddle_accuracy] reason=not_compare {self.api_config.config}\n{err!s}", flush=True, ) - write_to_log("accuracy_error", self.api_config.config) + write_to_log("paddle_accuracy", self.api_config.config) return elif isinstance(torch_output, (torch.return_types.max, torch.return_types.min)): torch_output = torch_output.values @@ -396,30 +370,30 @@ def compare_paddle_and_torch(paddle_tensor, torch_tensor, idx=0) -> bool: return else: print( - f"[not compare] {self.api_config.config}\n" + f"[paddle_accuracy] reason=not_compare {self.api_config.config}\n" f"torch is {type(torch_output)} but paddle is {type(paddle_output)}", flush=True, ) - write_to_log("accuracy_error", self.api_config.config) + write_to_log("paddle_accuracy", self.api_config.config) return elif isinstance(paddle_output, (list, tuple)): if not isinstance(torch_output, (list, tuple)): print( - f"[not compare] {self.api_config.config}\n" + f"[paddle_accuracy] reason=not_compare {self.api_config.config}\n" f"torch is {type(torch_output)} but paddle is {type(paddle_output)}", flush=True, ) - write_to_log("accuracy_error", self.api_config.config) + write_to_log("paddle_accuracy", self.api_config.config) return paddle_output = list(paddle_output) torch_output = list(torch_output) if len(paddle_output) != len(torch_output): print( - f"[not compare] {self.api_config.config}\n" + f"[paddle_accuracy] reason=not_compare {self.api_config.config}\n" f"torch len is {len(torch_output)} but paddle len is {len(paddle_output)}", flush=True, ) - write_to_log("accuracy_error", self.api_config.config) + write_to_log("paddle_accuracy", self.api_config.config) return for i, (paddle_item, torch_item) in enumerate( zip(paddle_output, torch_output, strict=False) @@ -443,11 +417,11 @@ def compare_paddle_and_torch(paddle_tensor, torch_tensor, idx=0) -> bool: return else: print( - f"[not compare] at {i} {self.api_config.config}\n" + f"[paddle_accuracy] reason=not_compare idx={i} {self.api_config.config}\n" f"torch is {type(torch_item)} but paddle is {type(paddle_item)}", flush=True, ) - write_to_log("accuracy_error", self.api_config.config) + write_to_log("paddle_accuracy", self.api_config.config) return elif ( paddle_item is None @@ -462,11 +436,11 @@ def compare_paddle_and_torch(paddle_tensor, torch_tensor, idx=0) -> bool: torch_item, torch.Tensor ): print( - f"[not compare] at {i} {self.api_config.config}\n" + f"[paddle_accuracy] reason=not_compare idx={i} {self.api_config.config}\n" f"torch is {type(torch_item)} but paddle is {type(paddle_item)}", flush=True, ) - write_to_log("accuracy_error", self.api_config.config) + write_to_log("paddle_accuracy", self.api_config.config) return else: if not compare_paddle_and_torch(paddle_item, torch_item, i): @@ -494,46 +468,22 @@ def compare_paddle_and_torch(paddle_tensor, torch_tensor, idx=0) -> bool: except Exception as err: if str(err).startswith("Too large tensor to get cached numpy: "): print( - f"[numpy error] backward {self.api_config.config}\n{err!s}", + f"[config_input] phase=backward {self.api_config.config}\n{err!s}", flush=True, ) - write_to_log("numpy_error", self.api_config.config) - return - if self.should_ignore_paddle_error(str(err)): - print(f"[Pass] {self.api_config.config}", flush=True) - write_to_log("pass", self.api_config.config) + write_to_log("config_input", self.api_config.config) return - if any(cuda_err in str(err) for cuda_err in CUDA_ERROR): - print( - f"[cuda error] backward {self.api_config.config}\n{err!s}", - flush=True, - ) - write_to_log("cuda_error", self.api_config.config) - raise - if any(cuda_err in str(err) for cuda_err in CUDA_OOM): - print( - f"[oom] backward {self.api_config.config}\n{err!s}", - flush=True, - ) - write_to_log("oom", self.api_config.config) - raise - print( - f"[paddle error] backward {self.api_config.config}\n{err!s}", - flush=True, + log_type, fatal = self.report_runtime_error( + err, "paddle_error", "paddle_backward", allow_ignore_paddle=True ) - if self.exit_on_error: + if fatal or (self.exit_on_error and log_type == "paddle_error"): raise - write_to_log("paddle_error", self.api_config.config) return try: paddle.base.core.eager._for_test_check_cuda_error() except Exception as err: - print( - f"[cuda error] backward {self.api_config.config}\n{err!s}", - flush=True, - ) - write_to_log("cuda_error", self.api_config.config) + self.report_runtime_error(err, "paddle_cuda", "paddle_backward_cuda_check") raise paddle_out_grads, torch_out_grads = process_grad_output( @@ -547,30 +497,30 @@ def compare_paddle_and_torch(paddle_tensor, torch_tensor, idx=0) -> bool: return else: print( - f"[not compare] backward {self.api_config.config}\n" + f"[paddle_accuracy] reason=not_compare phase=backward {self.api_config.config}\n" f"torch is {type(torch_out_grads)} but paddle is {type(paddle_out_grads)}", flush=True, ) - write_to_log("accuracy_error", self.api_config.config) + write_to_log("paddle_accuracy", self.api_config.config) return elif isinstance(paddle_out_grads, (list, tuple)): if not isinstance(torch_out_grads, (list, tuple)): print( - f"[not compare] backward {self.api_config.config}\n" + f"[paddle_accuracy] reason=not_compare phase=backward {self.api_config.config}\n" f"torch is {type(torch_out_grads)} but paddle is {type(paddle_out_grads)}", flush=True, ) - write_to_log("accuracy_error", self.api_config.config) + write_to_log("paddle_accuracy", self.api_config.config) return paddle_out_grads = list(paddle_out_grads) torch_out_grads = list(torch_out_grads) if len(paddle_out_grads) != len(torch_out_grads): print( - f"[not compare] backward {self.api_config.config}\n" + f"[paddle_accuracy] reason=not_compare phase=backward {self.api_config.config}\n" f"torch len is {len(torch_out_grads)} but paddle len is {len(paddle_out_grads)}", flush=True, ) - write_to_log("accuracy_error", self.api_config.config) + write_to_log("paddle_accuracy", self.api_config.config) return for i, (paddle_item, torch_item) in enumerate( zip(paddle_out_grads, torch_out_grads, strict=False) @@ -596,17 +546,17 @@ def compare_paddle_and_torch(paddle_tensor, torch_tensor, idx=0) -> bool: torch_item, torch.Tensor ): print( - f"[not compare] backward at {i} {self.api_config.config}\n" + f"[paddle_accuracy] reason=not_compare phase=backward idx={i} {self.api_config.config}\n" f"torch is {type(torch_item)} but paddle is {type(paddle_item)}", flush=True, ) - write_to_log("accuracy_error", self.api_config.config) + write_to_log("paddle_accuracy", self.api_config.config) return else: if not compare_paddle_and_torch(paddle_item, torch_item, i): return - print(f"[Pass] {self.api_config.config}", flush=True) + print(f"[pass] {self.api_config.config}", flush=True) write_to_log("pass", self.api_config.config) diff --git a/tester/accuracy_stable.py b/tester/accuracy_stable.py index b1713709..1c18bf06 100644 --- a/tester/accuracy_stable.py +++ b/tester/accuracy_stable.py @@ -7,7 +7,7 @@ import torch from .accuracy import process_grad_output, process_output -from .api_config.log_writer import log_accuracy_stable, write_to_log +from .api_config.log_writer import has_terminal_log, log_accuracy_stable, write_to_log from .base import CUDA_ERROR, CUDA_OOM, APITestBase from .paddle_to_torch import get_converter @@ -44,45 +44,48 @@ def _reset_random_state(self, seed: int = 42): def test(self): if self.need_skip(): - print("[Skip]", flush=True) + print(f"[skip] {self.api_config.config}", flush=True) + write_to_log("skip", self.api_config.config) return if not self.ana_api_info(): print("ana_api_info failed", flush=True) + write_to_log("config_parse", self.api_config.config) return try: convert_result = self.converter.convert(self.api_config.api_name) except Exception as e: print( - f"[paddle_to_torch] Conversion failed for {self.api_config.config}: {e!s}", + f"[config_convert] Conversion failed for {self.api_config.config}: {e!s}", flush=True, ) - write_to_log("paddle_to_torch_failed", self.api_config.config) + write_to_log("config_convert", self.api_config.config) return if not convert_result.is_supported: print( - f"[paddle_to_torch] Unsupported API {self.api_config.api_name}: {convert_result.error_message}", + f"[config_convert] Unsupported API {self.api_config.api_name}: {convert_result.error_message}", flush=True, ) - write_to_log("paddle_to_torch_failed", self.api_config.config) + write_to_log("config_convert", self.api_config.config) return if not convert_result.code or not convert_result.code.is_valid(): print( - f"[paddle_to_torch] No code generated for {self.api_config.api_name}", + f"[config_convert] No code generated for {self.api_config.api_name}", flush=True, ) - write_to_log("paddle_to_torch_failed", self.api_config.config) + write_to_log("config_convert", self.api_config.config) return try: if not self.gen_numpy_input(): print("gen_numpy_input failed") + write_to_log("config_input", self.api_config.config) return except Exception as err: - print("[numpy error]", self.api_config.config, "\n", str(err)) + print("[config_input]", self.api_config.config, "\n", str(err)) traceback.print_exc() - write_to_log("numpy_error", self.api_config.config) + write_to_log("config_input", self.api_config.config) return torch_output_pair = [] @@ -137,8 +140,9 @@ def test(self): self.compare(paddle_output_pair[0], paddle_output_pair[1], "P1P2") self.compare(paddle_grad_pair[0], paddle_grad_pair[1], "P1P2B") - print(f"[Pass] {self.api_config.config}", flush=True) - write_to_log("pass", self.api_config.config) + if not has_terminal_log(self.api_config.config): + print(f"[pass] {self.api_config.config}", flush=True) + write_to_log("pass", self.api_config.config) def get_torch_output(self, convert_result): # ======== run torch forward ========: @@ -176,12 +180,14 @@ def get_torch_output(self, convert_result): paddle.base.core.eager._for_test_check_cuda_error() except Exception as err: err_str = str(err) - print(f"[torch error] {self.api_config.config}\n{err_str}", flush=True) + if any(cuda_err in err_str for cuda_err in CUDA_OOM): + print(f"[oom] {self.api_config.config}\n{err_str}", flush=True) + write_to_log("oom", self.api_config.config) + raise + print(f"[torch_error] {self.api_config.config}\n{err_str}", flush=True) traceback.print_exc() write_to_log("torch_error", self.api_config.config) - if any(cuda_err in err_str for cuda_err in CUDA_ERROR) or any( - cuda_err in err_str for cuda_err in CUDA_OOM - ): + if any(cuda_err in err_str for cuda_err in CUDA_ERROR): raise return None, None, None @@ -205,17 +211,21 @@ def get_torch_output(self, convert_result): err_str = str(err) if err_str.startswith("Too large tensor to get cached numpy: "): print( - f"[numpy error] {self.api_config.config}\n{err_str}", + f"[config_input] {self.api_config.config}\n{err_str}", flush=True, ) - write_to_log("numpy_error", self.api_config.config) + write_to_log("config_input", self.api_config.config) return None, None, None - # some torch backward error can be tolerable, so we catch cuda error here - if any(cuda_err in err_str for cuda_err in CUDA_ERROR) or any( - cuda_err in err_str for cuda_err in CUDA_OOM - ): + if any(cuda_err in err_str for cuda_err in CUDA_OOM): + print( + f"[oom] phase=backward {self.api_config.config}\n{err_str}", + flush=True, + ) + write_to_log("oom", self.api_config.config) + raise + if any(cuda_err in err_str for cuda_err in CUDA_ERROR): print( - f"[torch error] backward {self.api_config.config}\n{err_str}", + f"[torch_error] phase=backward {self.api_config.config}\n{err_str}", flush=True, ) write_to_log("torch_error", self.api_config.config) @@ -227,7 +237,7 @@ def get_torch_output(self, convert_result): except Exception as err: err_str = str(err) print( - f"[torch error] backward {self.api_config.config}\n{err_str}", + f"[torch_error] phase=backward {self.api_config.config}\n{err_str}", flush=True, ) traceback.print_exc() @@ -295,18 +305,18 @@ def get_paddle_output(self, torch_grad_success): except Exception as err: err_str = str(err) if self.should_ignore_paddle_error(err_str): - print(f"[Pass] {self.api_config.config}", flush=True) + print(f"[pass] {self.api_config.config}", flush=True) write_to_log("pass", self.api_config.config) return None, None if any(cuda_err in err_str for cuda_err in CUDA_ERROR): - print(f"[cuda error] {self.api_config.config}\n{err_str}", flush=True) - write_to_log("cuda_error", self.api_config.config) + print(f"[paddle_cuda] {self.api_config.config}\n{err_str}", flush=True) + write_to_log("paddle_cuda", self.api_config.config) raise if any(cuda_err in err_str for cuda_err in CUDA_OOM): print(f"[oom] {self.api_config.config}\n{err_str}", flush=True) write_to_log("oom", self.api_config.config) raise - print(f"[paddle error] {self.api_config.config}\n{err_str}", flush=True) + print(f"[paddle_error] {self.api_config.config}\n{err_str}", flush=True) traceback.print_exc() write_to_log("paddle_error", self.api_config.config) return None, None @@ -314,8 +324,8 @@ def get_paddle_output(self, torch_grad_success): try: paddle.base.core.eager._for_test_check_cuda_error() except Exception as err: - print(f"[cuda error] {self.api_config.config}\n{err!s}", flush=True) - write_to_log("cuda_error", self.api_config.config) + print(f"[paddle_cuda] {self.api_config.config}\n{err!s}", flush=True) + write_to_log("paddle_cuda", self.api_config.config) raise # ======== run paddle backward ======== @@ -337,30 +347,30 @@ def get_paddle_output(self, torch_grad_success): err_str = str(err) if err_str.startswith("Too large tensor to get cached numpy: "): print( - f"[numpy error] {self.api_config.config}\n{err_str}", + f"[config_input] {self.api_config.config}\n{err_str}", flush=True, ) - write_to_log("numpy_error", self.api_config.config) + write_to_log("config_input", self.api_config.config) return None, None if self.should_ignore_paddle_error(err_str): - print(f"[Pass] {self.api_config.config}", flush=True) + print(f"[pass] {self.api_config.config}", flush=True) write_to_log("pass", self.api_config.config) return None, None if any(cuda_err in err_str for cuda_err in CUDA_ERROR): print( - f"[cuda error] backward {self.api_config.config}\n{err_str}", + f"[paddle_cuda] phase=backward {self.api_config.config}\n{err_str}", ) - write_to_log("cuda_error", self.api_config.config) + write_to_log("paddle_cuda", self.api_config.config) raise if any(cuda_err in err_str for cuda_err in CUDA_OOM): print( - f"[oom] backward {self.api_config.config}\n{err_str}", + f"[oom] phase=backward {self.api_config.config}\n{err_str}", flush=True, ) write_to_log("oom", self.api_config.config) raise print( - f"[paddle error] backward {self.api_config.config}\n{err_str}", + f"[paddle_error] phase=backward {self.api_config.config}\n{err_str}", flush=True, ) traceback.print_exc() @@ -371,10 +381,10 @@ def get_paddle_output(self, torch_grad_success): paddle.base.core.eager._for_test_check_cuda_error() except Exception as err: print( - f"[cuda error] backward {self.api_config.config}\n{err!s}", + f"[paddle_cuda] phase=backward {self.api_config.config}\n{err!s}", flush=True, ) - write_to_log("cuda_error", self.api_config.config) + write_to_log("paddle_cuda", self.api_config.config) raise def process_paddle_outputs(obj): @@ -393,36 +403,36 @@ def compare(self, input1, input2, comp): self.assert_accuracy(input1, input2, comp) except Exception as err: print( - f"[{comp}] [accuracy error] {self.api_config.config}\n{err!s}", + f"[paddle_accuracy] comp={comp} {self.api_config.config}\n{err!s}", flush=True, ) - write_to_log("accuracy_error", self.api_config.config) + write_to_log("paddle_accuracy", self.api_config.config) return else: print( - f"[{comp}] [accuracy error] {self.api_config.config}\n[not compare],", + f"[paddle_accuracy] comp={comp} {self.api_config.config}\nreason=not_compare,", f"{type(input1)} / {type(input2)}", flush=True, ) - write_to_log("accuracy_error", self.api_config.config) + write_to_log("paddle_accuracy", self.api_config.config) return elif isinstance(input1, (list, tuple)): if not isinstance(input2, (list, tuple)): print( - f"[{comp}] [accuracy error] {self.api_config.config}\n[not compare],", + f"[paddle_accuracy] comp={comp} {self.api_config.config}\nreason=not_compare,", f"{type(input1)} / {type(input2)}", flush=True, ) - write_to_log("accuracy_error", self.api_config.config) + write_to_log("paddle_accuracy", self.api_config.config) return if len(input1) != len(input2): print( - f"[{comp}] [accuracy error] {self.api_config.config}\n[not compare],", + f"[paddle_accuracy] comp={comp} {self.api_config.config}\nreason=not_compare,", f"{type(input1)} : {len(input1)} /", f"{type(input2)} : {len(input2)}", flush=True, ) - write_to_log("accuracy_error", self.api_config.config) + write_to_log("paddle_accuracy", self.api_config.config) return for idx, (item1, item2) in enumerate(zip(input1, input2, strict=False)): if isinstance(item1, (paddle.Tensor, torch.Tensor)) and isinstance( @@ -432,10 +442,10 @@ def compare(self, input1, input2, comp): self.assert_accuracy(item1, item2, comp, idx) except Exception as err: print( - f"[{comp}] [accuracy error] {self.api_config.config}\n{err!s}", + f"[paddle_accuracy] comp={comp} {self.api_config.config}\n{err!s}", flush=True, ) - write_to_log("accuracy_error", self.api_config.config) + write_to_log("paddle_accuracy", self.api_config.config) return elif not isinstance(item1, (paddle.Tensor, torch.Tensor)) and not isinstance( item2, (paddle.Tensor, torch.Tensor) @@ -444,28 +454,28 @@ def compare(self, input1, input2, comp): self.assert_accuracy(torch.tensor(item1), torch.tensor(item2), comp, idx) except Exception as err: print( - f"[{comp}] [accuracy error] {self.api_config.config}\n{err!s}", + f"[paddle_accuracy] comp={comp} {self.api_config.config}\n{err!s}", flush=True, ) - write_to_log("accuracy_error", self.api_config.config) + write_to_log("paddle_accuracy", self.api_config.config) return else: print( - f"[{comp}] [accuracy error] {self.api_config.config}\n[not compare]", + f"[paddle_accuracy] comp={comp} {self.api_config.config}\nreason=not_compare", f"{type(item1)} / {type(item2)}", flush=True, ) - write_to_log("accuracy_error", self.api_config.config) + write_to_log("paddle_accuracy", self.api_config.config) return else: try: self.assert_accuracy(torch.tensor(input1), torch.tensor(input2), comp) except Exception as err: print( - f"[{comp}] [accuracy error] {self.api_config.config}\n{err!s}", + f"[paddle_accuracy] comp={comp} {self.api_config.config}\n{err!s}", flush=True, ) - write_to_log("accuracy_error", self.api_config.config) + write_to_log("paddle_accuracy", self.api_config.config) return def assert_accuracy(self, tensor1, tensor2, comp, idx=0): @@ -557,6 +567,6 @@ def error_msg(msg): dtype, comp, ) - write_to_log("accuracy_diff", config) + write_to_log("paddle_bitwise", config) else: raise diff --git a/tester/api_config/log_writer.py b/tester/api_config/log_writer.py index 754c4f4f..1ef7dc8d 100644 --- a/tester/api_config/log_writer.py +++ b/tester/api_config/log_writer.py @@ -20,23 +20,37 @@ LOG_PREFIXES = { "checkpoint": "checkpoint", "pass": "api_config_pass", - "numpy_error": "api_config_numpy_error", + "skip": "api_config_skip", "paddle_error": "api_config_paddle_error", - "torch_error": "api_config_torch_error", - "paddle_to_torch_failed": "api_config_paddle_to_torch_failed", - "accuracy_error": "api_config_accuracy_error", - "accuracy_diff": "api_config_accuracy_diff", - "timeout": "api_config_timeout", - "crash": "api_config_crash", + "paddle_accuracy": "api_config_paddle_accuracy", + "paddle_bitwise": "api_config_paddle_bitwise", + "paddle_cuda": "api_config_paddle_cuda", + "paddle_crash": "api_config_paddle_crash", "oom": "api_config_oom", - "match_error": "api_config_match_error", - "cuda_error": "api_config_cuda_error", + "timeout": "api_config_timeout", + "torch_error": "api_config_torch_error", + "config_input": "api_config_config_input", + "config_parse": "api_config_config_parse", + "config_convert": "api_config_config_convert", } +LOG_ALIASES = { + "numpy_error": "config_input", + "match_error": "config_parse", + "paddle_to_torch_failed": "config_convert", + "accuracy_error": "paddle_accuracy", + "accuracy_diff": "paddle_bitwise", + "cuda_error": "paddle_cuda", + "crash": "paddle_crash", +} + +TERMINAL_LOG_TYPES = frozenset(LOG_PREFIXES) - {"checkpoint"} + _is_engineV2 = False _process_file_handlers = {} _aggregated_offsets = {} +_process_terminal_configs = {} # Command line arguments configuration # Used in engine.py @@ -103,8 +117,28 @@ def close_process_files(): _process_file_handlers = {} +def has_terminal_log(line): + return line.strip() in _process_terminal_configs + + +def get_terminal_log_type(line): + return _process_terminal_configs.get(line.strip()) + + +def write_checkpoint(line): + line = line.strip() + write_to_log("checkpoint", line) + _process_terminal_configs.pop(line, None) + + +def write_terminal_log(log_type, line): + write_to_log(log_type, line) + write_checkpoint(line) + + def get_log_file(log_type: str): """获取指定日志类型和PID对应的日志文件路径""" + log_type = LOG_ALIASES.get(log_type, log_type) if log_type not in LOG_PREFIXES: raise ValueError(f"Invalid log type: {log_type}") prefix = LOG_PREFIXES[log_type] @@ -118,21 +152,28 @@ def get_log_file(log_type: str): def write_to_log(log_type, line): """添加单条日志到当前进程的日志文件""" + log_type = LOG_ALIASES.get(log_type, log_type) line = line.strip() if not line: return + terminal_log_type = _process_terminal_configs.get(line) + if _is_engineV2 and log_type == "pass" and terminal_log_type not in (None, "pass"): + return file_path = get_log_file(log_type) try: if file_path not in _process_file_handlers: _process_file_handlers[file_path] = file_path.open("a", buffering=1) handler = _process_file_handlers[file_path] handler.write(line + "\n") + if log_type in TERMINAL_LOG_TYPES and _is_engineV2: + _process_terminal_configs[line] = log_type except Exception as err: print(f"Error writing to {file_path}: {err}", flush=True) def read_log(log_type): """读取文件所有行，返回集合""" + log_type = LOG_ALIASES.get(log_type, log_type) if log_type not in LOG_PREFIXES: raise ValueError(f"Invalid log type: {log_type}") cfg = get_cfg() @@ -452,13 +493,13 @@ def aggregate_logs(end=False, cleanup=False): print(f"Error reading {log_file}: {err}", flush=True) if api_configs: - log_counts["skip"] = len(api_configs) - skip_file = TEST_LOG_PATH / "api_config_skip.txt" + log_counts["incomplete"] = len(api_configs) + incomplete_file = TEST_LOG_PATH / "api_config_incomplete.txt" try: - with skip_file.open("w") as f: + with incomplete_file.open("w") as f: f.writelines(f"{line}\n" for line in sorted(api_configs)) except Exception as err: - print(f"Error writing to {skip_file}: {err}", flush=True) + print(f"Error writing to {incomplete_file}: {err}", flush=True) return log_counts @@ -468,27 +509,27 @@ def print_log_info(all_case, log_counts=None): log_counts = {} test_case = log_counts.get("checkpoint", 0) pass_case = log_counts.get("pass", 0) - fail_case = sum( + paddle_issue_case = sum( log_counts.get(log_type, 0) for log_type in [ "paddle_error", - "accuracy_error", - "accuracy_diff", - "timeout", - "crash", - "oom", - "cuda_error", + "paddle_accuracy", + "paddle_bitwise", + "paddle_cuda", + "paddle_crash", ] ) - skip_case = sum( + retest_case = sum(log_counts.get(log_type, 0) for log_type in ["oom", "timeout"]) + framework_blocked_case = sum( log_counts.get(log_type, 0) for log_type in [ - "numpy_error", "torch_error", - "paddle_to_torch_failed", - "match_error", + "config_input", + "config_parse", + "config_convert", ] ) + skip_case = log_counts.get("skip", 0) # 打印统计信息 print("\n" + "=" * 50) @@ -497,7 +538,9 @@ def print_log_info(all_case, log_counts=None): print(f"{'Total cases':<30}: {all_case}") print(f"{'Tested cases':<30}: {test_case}") print(f"{'Passed cases':<30}: {pass_case}") - print(f"{'Failed cases':<30}: {fail_case}") + print(f"{'Paddle issue cases':<30}: {paddle_issue_case}") + print(f"{'Retest cases':<30}: {retest_case}") + print(f"{'Framework blocked cases':<30}: {framework_blocked_case}") print(f"{'Skipped cases':<30}: {skip_case}") if log_counts: print("-" * 50) @@ -563,7 +606,7 @@ def log_accuracy_tolerance(error_msg, api, config, dtype, is_backward=False): """ output_file = TMP_LOG_PATH / f"tol_{os.getpid()}.csv" mode = "backward" if is_backward else "forward" - print(f"[{mode}] {config}\n{error_msg}", flush=True) + print(f"mode={mode} {config}\n{error_msg}", flush=True) if error_msg == "Identical": max_abs_diff = 0.0 @@ -611,7 +654,7 @@ def log_accuracy_tolerance(error_msg, api, config, dtype, is_backward=False): def log_accuracy_stable(error_msg, api, config, dtype, comp): output_file = TMP_LOG_PATH / f"stable_{os.getpid()}.csv" - print(f"[{comp}] {config}\n{error_msg}", flush=True) + print(f"comp={comp} {config}\n{error_msg}", flush=True) if error_msg == "Identical": max_abs_diff = 0.0 diff --git a/tester/base.py b/tester/base.py index cc4fc89b..5621c800 100644 --- a/tester/base.py +++ b/tester/base.py @@ -46,10 +46,34 @@ [ "CUDA out of memory", "Out of memory error", + "ResourceExhaustedError", + "out of memory", + "OutOfMemoryError", ] ) +def classify_runtime_error(error_msg): + error_msg_lower = error_msg.lower() + oom_markers = tuple(marker.lower() for marker in CUDA_OOM) + ( + "cannot allocate memory", + "std::bad_alloc", + "bad allocation", + "memoryerror", + "cublas_status_alloc_failed", + ) + if any(marker in error_msg_lower for marker in oom_markers): + return "oom", True + cuda_markers = tuple(marker.lower() for marker in CUDA_ERROR) + ( + "illegal memory access", + "invalid configuration argument", + "invalid resource handle", + ) + if any(marker in error_msg_lower for marker in cuda_markers): + return "paddle_cuda", True + return None, False + + def get_arg(api_config, arg_pos, arg_name, default=None): if 0 <= arg_pos < len(api_config.args): return api_config.args[arg_pos] @@ -181,6 +205,25 @@ def __init__(self, api_config): torch.set_num_threads(8) torch.set_printoptions(threshold=100, linewidth=120) + def report_runtime_error(self, err, default_log_type, phase="", allow_ignore_paddle=False): + err_msg = str(err) + log_type, fatal = classify_runtime_error(err_msg) + if log_type is None and allow_ignore_paddle and self.should_ignore_paddle_error(err_msg): + print(f"[pass] {self.api_config.config}", flush=True) + write_to_log("pass", self.api_config.config) + return "pass", False + if log_type is None: + log_type = default_log_type + elif default_log_type == "torch_error" and log_type != "oom": + log_type = "torch_error" + phase_text = f" phase={phase}" if phase else "" + print( + f"[{log_type}]{phase_text} {self.api_config.config}\n{err_msg}", + flush=True, + ) + write_to_log(log_type, self.api_config.config) + return log_type, fatal + def need_skip(self, paddle_only=False): # not support if "sparse" in self.api_config.api_name: diff --git a/tester/paddle_only.py b/tester/paddle_only.py index 085600c4..5302838a 100644 --- a/tester/paddle_only.py +++ b/tester/paddle_only.py @@ -3,7 +3,7 @@ import paddle from .api_config.log_writer import write_to_log -from .base import CUDA_ERROR, CUDA_OOM, APITestBase +from .base import APITestBase # from func_timeout import func_set_timeout @@ -16,25 +16,29 @@ def __init__(self, api_config, **kwargs): # @func_set_timeout(600) def test(self): if self.need_skip(paddle_only=True): - print(f"[Skip] {self.api_config.config}", flush=True) + print(f"[skip] {self.api_config.config}", flush=True) + write_to_log("skip", self.api_config.config) return if not self.ana_paddle_api_info(): print("ana_paddle_api_info failed", flush=True) + write_to_log("config_parse", self.api_config.config) return try: if not self.gen_numpy_input(): print("gen_numpy_input failed", flush=True) + write_to_log("config_input", self.api_config.config) return except Exception as err: - print(f"[numpy error] {self.api_config.config}\n{err!s}", flush=True) - write_to_log("numpy_error", self.api_config.config) + print(f"[config_input] {self.api_config.config}\n{err!s}", flush=True) + write_to_log("config_input", self.api_config.config) return try: if not self.gen_paddle_input(): print("gen_paddle_input failed", flush=True) + write_to_log("paddle_error", self.api_config.config) return if self.test_amp: with paddle.amp.auto_cast(): @@ -61,20 +65,11 @@ def test(self): paddle_output = None result_outputs = None result_outputs_grads = None - if self.should_ignore_paddle_error(str(err)): - print(f"[Pass] {self.api_config.config}", flush=True) - write_to_log("pass", self.api_config.config) - return - if any(cuda_err in str(err) for cuda_err in CUDA_ERROR): - print(f"[cuda error] {self.api_config.config}\n{err!s}", flush=True) - write_to_log("cuda_error", self.api_config.config) - raise - if any(cuda_err in str(err) for cuda_err in CUDA_OOM): - print(f"[oom] {self.api_config.config}\n{err!s}", flush=True) - write_to_log("oom", self.api_config.config) + _, fatal = self.report_runtime_error( + err, "paddle_error", "paddle_only", allow_ignore_paddle=True + ) + if fatal: raise - print(f"[paddle error] {self.api_config.config}\n{err!s}", flush=True) - write_to_log("paddle_error", self.api_config.config) return try: @@ -83,12 +78,11 @@ def test(self): paddle_output = None result_outputs = None result_outputs_grads = None - print(f"[cuda error] {self.api_config.config}\n{err!s}", flush=True) - write_to_log("cuda_error", self.api_config.config) + self.report_runtime_error(err, "paddle_cuda", "paddle_only_cuda_check") raise paddle_output = None result_outputs = None result_outputs_grads = None - print(f"[Pass] {self.api_config.config}", flush=True) + print(f"[pass] {self.api_config.config}", flush=True) write_to_log("pass", self.api_config.config) diff --git a/tools/error_stat/error_stat.py b/tools/error_stat/error_stat.py index 512bb602..e441a165 100644 --- a/tools/error_stat/error_stat.py +++ b/tools/error_stat/error_stat.py @@ -20,10 +20,10 @@ "Out of memory error", "[Skip]", "[config parse error]", - "[match error]", - "[numpy error]", - "[paddle_to_torch]", - "[torch error]", + "[config_parse]", + "[config_input]", + "[config_convert]", + "[torch_error]", "output type diff error", "Too large tensor to get cached numpy", "There is no grad op for inputs:", @@ -35,17 +35,18 @@ LOG_PREFIXES = { "checkpoint": "checkpoint", "pass": "api_config_pass", - "crash": "api_config_crash", + "skip": "api_config_skip", + "paddle_error": "api_config_paddle_error", + "paddle_accuracy": "api_config_paddle_accuracy", + "paddle_bitwise": "api_config_paddle_bitwise", + "paddle_cuda": "api_config_paddle_cuda", + "paddle_crash": "api_config_paddle_crash", "oom": "api_config_oom", "timeout": "api_config_timeout", - "paddle_error": "api_config_paddle_error", - "accuracy_error": "api_config_accuracy_error", - "accuracy_diff": "api_config_accuracy_diff", "torch_error": "api_config_torch_error", - "paddle_to_torch_failed": "api_config_paddle_to_torch_failed", - "match_error": "api_config_match_error", - "numpy_error": "api_config_numpy_error", - "cuda_error": "api_config_cuda_error", + "config_input": "api_config_config_input", + "config_parse": "api_config_config_parse", + "config_convert": "api_config_config_convert", } diff --git a/tools/remove_retest_configs.py b/tools/remove_retest_configs.py index f2d8da1c..0a526f38 100644 --- a/tools/remove_retest_configs.py +++ b/tools/remove_retest_configs.py @@ -11,18 +11,18 @@ LOG_PREFIXES = { "checkpoint": "checkpoint", "pass": "api_config_pass", - "crash": "api_config_crash", + "skip": "api_config_skip", + "paddle_error": "api_config_paddle_error", + "paddle_accuracy": "api_config_paddle_accuracy", + "paddle_bitwise": "api_config_paddle_bitwise", + "paddle_cuda": "api_config_paddle_cuda", + "paddle_crash": "api_config_paddle_crash", "oom": "api_config_oom", "timeout": "api_config_timeout", - "paddle_error": "api_config_paddle_error", - "accuracy_error": "api_config_accuracy_error", - "accuracy_diff": "api_config_accuracy_diff", "torch_error": "api_config_torch_error", - "paddle_to_torch_failed": "api_config_paddle_to_torch_failed", - "match_error": "api_config_match_error", - "numpy_error": "api_config_numpy_error", - "cuda_error": "api_config_cuda_error", - "skip": "api_config_skip", + "config_input": "api_config_config_input", + "config_parse": "api_config_config_parse", + "config_convert": "api_config_config_convert", } DEFAULT_REMOVE_TYPES = ["timeout", "oom", "skip"] @@ -124,18 +124,19 @@ def parse_args(argv=None): python %(prog)s --path tester/api_config/test_log # 指定测试日志路径 python %(prog)s --remove timeout oom skip # 指定需要移除的配置 支持移除的配置集合: - pass - api_config_pass - numpy_error - api_config_numpy_error - paddle_error - api_config_paddle_error - torch_error - api_config_torch_error - paddle_to_torch_failed - api_config_paddle_to_torch_failed - accuracy_error - api_config_accuracy_error - accuracy_diff - api_config_accuracy_diff - timeout - api_config_timeout - crash - api_config_crash - oom - api_config_oom - match_error - api_config_match_error - skip - api_config_skip + pass - api_config_pass + skip - api_config_skip + paddle_error - api_config_paddle_error + paddle_accuracy - api_config_paddle_accuracy + paddle_bitwise - api_config_paddle_bitwise + paddle_cuda - api_config_paddle_cuda + paddle_crash - api_config_paddle_crash + oom - api_config_oom + timeout - api_config_timeout + torch_error - api_config_torch_error + config_input - api_config_config_input + config_parse - api_config_config_parse + config_convert - api_config_config_convert """, ) parser.add_argument(