fix: output result file to folder && add error info in log

Author: baominghelly

infinimetrics/common/constants.py

@@ -22,6 +22,21 @@ class AcceleratorType(str, Enum):
     GENERIC = "generic"
 
 
+class TestCategory(str, Enum):
+    """Test category classifications - single source of truth for test type categorization"""
+
+    HARDWARE = "hardware"
+    OPERATOR = "operator"
+    INFER = "infer"
+    COMM = "comm"
+    TRAIN = "train"
+
+
+# Valid test categories (derived from TestCategory enum)
+# Used for testcase prefix matching and output directory organization
+TEST_CATEGORIES = {cat.value: cat.value for cat in TestCategory}
+
+
 # Common execution defaults
 DEFAULT_WARMUP_ITERATIONS = 10
 DEFAULT_MEASURED_ITERATIONS = 100

infinimetrics/dispatcher.py

@@ -9,16 +9,18 @@
 
 from infinimetrics.adapter import BaseAdapter
 from infinimetrics.executor import Executor, TestResult
+from infinimetrics.common.constants import TestCategory
 
 logger = logging.getLogger(__name__)
 
 # Adapter registry: maps (test_type, framework) -> adapter factory
+# test_type must use TestCategory enum (not string literals)
 _ADAPTER_REGISTRY = {
-    ("operator", "infinicore"): lambda: _create_infinicore_adapter(),
-    ("hardware", "cudaunified"): lambda: _create_hardware_adapter(),
-    ("comm", "nccltest"): lambda: _create_nccltests_adapter(),
-    ("infer", "infinilm"): lambda: _create_inference_adapter(),
-    ("infer", "vllm"): lambda: _create_inference_adapter(),
+    (TestCategory.OPERATOR, "infinicore"): lambda: _create_infinicore_adapter(),
+    (TestCategory.HARDWARE, "cudaunified"): lambda: _create_hardware_adapter(),
+    (TestCategory.COMM, "nccltest"): lambda: _create_nccltests_adapter(),
+    (TestCategory.INFER, "infinilm"): lambda: _create_inference_adapter(),
+    (TestCategory.INFER, "vllm"): lambda: _create_inference_adapter(),
 }
 
 
@@ -99,9 +101,9 @@ def dispatch(self, inputs: Any) -> Dict[str, Any]:
 
         for test_input in valid_test_inputs:
             testcase = test_input["testcase"]
-            test_type, framework = self._parse_testcase(testcase)
 
             try:
+                test_type, framework = self._parse_testcase(testcase)
                 adapter = self._create_adapter(test_type, framework)
                 valid_executions.append((test_input, adapter))
                 logger.debug(f"Validated {testcase} - adapter ready")
@@ -168,14 +170,23 @@ def _parse_testcase(self, testcase: str) -> tuple[str, str]:
 
         if len(parts) < 2:
             logger.warning(f"Invalid testcase format: {testcase}, using defaults")
-            return "operator", "infinicore"
+            return TestCategory.OPERATOR.value, "infinicore"
+
+        # First part is test_type - validate against TestCategory enum
+        test_type_str = parts[0].lower()
+        try:
+            test_type = TestCategory(test_type_str)
+        except ValueError:
+            valid_types = ", ".join(c.value for c in TestCategory)
+            raise ValueError(
+                f"Invalid test_type '{test_type_str}' in '{testcase}'. "
+                f"Must be one of: {valid_types}"
+            )
 
-        # First part is test_type
-        test_type = parts[0].lower()
         # Second part is framework
         framework = parts[1].lower()
 
-        return test_type, framework
+        return test_type.value, framework
 
     def _aggregate_results(self, results: List[Dict[str, Any]]) -> Dict[str, Any]:
         """Aggregate results from executors."""

infinimetrics/executor.py

@@ -15,7 +15,7 @@
 from infinimetrics.adapter import BaseAdapter
 from infinimetrics.input import TestInput
 from infinimetrics.utils.path_utils import sanitize_filename
-from infinimetrics.common.constants import ErrorCode
+from infinimetrics.common.constants import ErrorCode, TEST_CATEGORIES
 
 
 logger = logging.getLogger(__name__)
@@ -664,17 +664,27 @@ def _save_result(self, result: Dict[str, Any]) -> str:
         if not run_id:
             run_id = self.payload.get("run_id") or self.run_id
 
+        # 3) Determine output subdirectory based on testcase
+        testcase = self.payload.get("testcase", "")
+
+        # Extract category from testcase (e.g., "hardware.cudaUnified.Comprehensive" -> "hardware")
+        category = "other"  # default fallback
+        for prefix, subdir in TEST_CATEGORIES.items():
+            if testcase.startswith(prefix):
+                category = subdir
+                break
+
         if run_id:
             safe_run_id = sanitize_filename(run_id)
 
-            # Put final JSON next to timeseries csv files under infer/
-            infer_dir = self.output_dir / "infer"
-            infer_dir.mkdir(parents=True, exist_ok=True)
+            # Put results in category-specific subdirectory
+            category_dir = self.output_dir / category
+            category_dir.mkdir(parents=True, exist_ok=True)
 
             filename = f"{safe_run_id}_results.json"
-            output_file = infer_dir / filename
+            output_file = category_dir / filename
         else:
-            # Final fallback to old naming
+            # Final fallback to old naming (in root output_dir)
             safe_name = self.testcase.replace(".", "_").replace("/", "_")
             filename = f"{safe_name}_{timestamp}_results.json"
             output_file = self.output_dir / filename

scripts/common/install_deps.sh

@@ -70,11 +70,16 @@ check_cuda() {
 # Check InfiniCore
 check_infinicore() {
     echo -n "  InfiniCore... "
-    if check_python_package infinicore; then
+
+    # Try to import and capture error details
+    error_msg=$(python -c "import infinicore" 2>&1)
+    if [ $? -eq 0 ]; then
         echo -e "${GREEN}[OK]${NC}"
         return 0
     else
-        echo -e "${YELLOW}[WARNING]${NC} not installed"
+        echo -e "${YELLOW}[WARNING]${NC}"
+        echo "    Failed to import infinicore:"
+        echo "    ${error_msg}" | head -n 15 | sed 's/^/    /'
         return 1
     fi
 }
@@ -164,11 +169,20 @@ install_infinicore() {
     echo -e "${BLUE}Verifying installation...${NC}"
     echo "INFINI_ROOT: $INFINI_ROOT"
     echo "LD_LIBRARY_PATH: $LD_LIBRARY_PATH"
-    if python -c "import infinicore" 2>/dev/null; then
+
+    error_msg=$(python -c "import infinicore" 2>&1)
+    if [ $? -eq 0 ]; then
         echo -e "${GREEN}[OK] InfiniCore installation completed successfully!${NC}"
         return 0
     else
         echo -e "${RED}[ERROR] InfiniCore installation verification failed${NC}"
+        echo "    Import error:"
+        echo "    ${error_msg}" | head -n 15 | sed 's/^/    /'
+        echo ""
+        echo "    Possible causes:"
+        echo "    - Dependencies not installed (run: python scripts/install.py --nv-gpu=y)"
+        echo "    - Library not built (run: xmake build _infinicore)"
+        echo "    - LD_LIBRARY_PATH not set correctly"
         return 1
     fi
 }