fix: skip truncation for endpoints submissions instead of targeting stale results.json (#2631)

The endpoints accuracy artifact was renamed from results.json (with a
responses field) to accuracy/accuracy_results.json in
mlcommons/endpoints#400, which also dropped the responses field
entirely (per-sample text now lives in events.jsonl, not part of the
submission). truncate_accuracy_log.py still looked for the old
results.json/responses shape and would only log "missing" for every
endpoints submission. Detect the endpoints layout via the scenario-root
config.yaml and skip truncation, and remove the now-dead
PERFORMANCE_ENDPOINTS_DIR/ACCURACY_ENDPOINTS_DIR constants left over
from the loader.py refactor to ENDPOINTS_SCENARIO_DIR.

Co-authored-by: Claude Sonnet 5 <noreply@anthropic.com>

Author: anandhu-eng

tools/submission/submission_checker/constants.py

@@ -2036,14 +2036,6 @@
     "deepseek-r1"
 ]
 
-PERFORMANCE_ENDPOINTS_DIR = {
-    "v5.0": "{division}/{submitter}/results/{system}/{benchmark}/{scenario}/performance/run_1/",
-    "v5.1": "{division}/{submitter}/results/{system}/{benchmark}/{scenario}/performance/run_1/",
-    "v6.0": "{division}/{submitter}/results/{system}/{benchmark}/{scenario}/performance/run_1/",
-    "v6.1": "{division}/{submitter}/results/{system}/{benchmark}/{scenario}/performance/run_1/",
-    "default": "{division}/{submitter}/results/{system}/{benchmark}/{scenario}/performance/run_1/",
-}
-
 ENDPOINTS_SCENARIO_DIR = {
     "v5.0": "{division}/{submitter}/results/{system}/{benchmark}/{scenario}/",
     "v5.1": "{division}/{submitter}/results/{system}/{benchmark}/{scenario}/",
@@ -2076,15 +2068,6 @@
     "default": "{division}/{submitter}/results/{system}/{benchmark}/{scenario}/accuracy/mlperf_log_accuracy.json",
 }
 
-ACCURACY_ENDPOINTS_DIR = {
-    "v5.0": "{division}/{submitter}/results/{system}/{benchmark}/{scenario}/accuracy/",
-    "v5.1": "{division}/{submitter}/results/{system}/{benchmark}/{scenario}/accuracy/",
-    "v6.0": "{division}/{submitter}/results/{system}/{benchmark}/{scenario}/accuracy/",
-    "v6.1": "{division}/{submitter}/results/{system}/{benchmark}/{scenario}/accuracy/",
-    "default": "{division}/{submitter}/results/{system}/{benchmark}/{scenario}/accuracy/",
-}
-
-
 POWER_DIR_PATH = {
     "v5.0": "{division}/{submitter}/results/{system}/{benchmark}/{scenario}/performance/power",
     "v5.1": "{division}/{submitter}/results/{system}/{benchmark}/{scenario}/performance/power",

tools/submission/truncate_accuracy_log.py

@@ -18,7 +18,6 @@
 
 MAX_ACCURACY_LOG_SIZE = 10 * 1024
 VIEWABLE_SIZE = 4096
-RESPONSES_LIMIT = 10 * 1024  # cap on truncated responses bytes
 
 HELP_TEXT = """
 You can run this tool in 2 ways:
@@ -124,84 +123,18 @@ def copy_submission_dir(src, dst, filter_submitter):
             )
 
 
-def _truncate_endpoints_results(results_path, acc_path, backup):
-    """Truncate the ``responses`` field in an endpoints accuracy results.json.
+def _is_endpoints_submission(scenario_dir):
+    """Detect the endpoints submission layout via its scenario-root config.yaml.
 
-    Mirrors the behaviour of truncate_accuracy_log.py for traditional
-    submissions: backs up the original file (when --backup is given),
-    computes a SHA-256 hash of the original, appends ``hash=<hex>`` to
-    accuracy.txt, then writes back a copy with responses capped at
-    RESPONSES_LIMIT bytes.
+    Endpoints submissions have no mlperf_log_accuracy.json; the accuracy
+    artifact they produce (accuracy/accuracy_results.json) carries no large
+    per-sample payload to truncate, so callers should skip truncation
+    entirely rather than look for a file to shrink.
     """
-    try:
-        with open(results_path, "r", encoding="utf-8") as f:
-            original_bytes = f.read().encode()
-        data = json.loads(original_bytes)
-    except Exception as exc:
-        log.error("Could not read %s: %s", results_path, exc)
-        return
-
-    # Back up before any modification.
-    if backup:
-        backup_dir = os.path.join(backup, acc_path)
-        os.makedirs(backup_dir, exist_ok=True)
-        dst = os.path.join(backup_dir, "results.json")
-        if os.path.exists(dst):
-            log.error(
-                "not processing %s because %s already exists",
-                results_path,
-                dst,
-            )
-            return
-        shutil.copy(results_path, dst)
-
-    # Hash the original file for audit purposes (mirrors accuracy.txt hash
-    # written for traditional mlperf_log_accuracy.json truncation).
-    hash_val = hashlib.sha256(original_bytes).hexdigest()
-    acc_txt = os.path.join(acc_path, "accuracy.txt")
-    with open(acc_txt, "a", encoding="utf-8") as f:
-        f.write("\nhash={0}\n".format(hash_val))
-
-    # Truncate the responses collection so the file stays small.
-    responses = data.get("responses")
-    if responses is None or (isinstance(
-            responses, (list, dict)) and not responses):
-        log.info("%s has no responses to truncate", results_path)
-        return
-
-    if isinstance(responses, list):
-        total = 2  # "[]"
-        idx = 0
-        for i, r in enumerate(responses):
-            total += len(json.dumps(r).encode()) + (2 if i > 0 else 0)
-            if total > RESPONSES_LIMIT:
-                break
-            idx = i + 1
-        data["responses"] = responses[:idx]
-    elif isinstance(responses, dict):
-        total = 2  # "{}"
-        kept = {}
-        for i, (k, v) in enumerate(responses.items()):
-            entry = len(json.dumps(k).encode()) + \
-                len(json.dumps(v).encode()) + 2
-            if i > 0:
-                entry += 2
-            if total + entry > RESPONSES_LIMIT:
-                break
-            total += entry
-            kept[k] = v
-        data["responses"] = kept
-    else:
-        log.error(
-            "%s: responses has unexpected type %s; skipping truncation",
-            results_path,
-            type(responses).__name__,
-        )
-        return
-
-    with open(results_path, "w", encoding="utf-8") as f:
-        json.dump(data, f, indent=2)
-    log.info("%s responses truncated", results_path)
+    return any(
+        os.path.exists(os.path.join(scenario_dir, name))
+        for name in ("config.yaml", "config.yml")
+    )
 
 
 def truncate_results_dir(filter_submitter, backup, scenarios_to_skip):
@@ -264,14 +197,10 @@ def truncate_results_dir(filter_submitter, backup, scenarios_to_skip):
                                 acc_txt = os.path.join(
                                     acc_path, "accuracy.txt")
                                 if not os.path.exists(acc_log):
-                                    # Endpoints submissions have results.json
-                                    # instead of mlperf_log_accuracy.json
-                                    endpoints_results = os.path.join(
-                                        acc_path, "results.json"
-                                    )
-                                    if os.path.exists(endpoints_results):
-                                        _truncate_endpoints_results(
-                                            endpoints_results, acc_path, backup
+                                    if _is_endpoints_submission(name):
+                                        log.info(
+                                            "%s is an endpoints submission; nothing to truncate",
+                                            name,
                                         )
                                     else:
                                         log.error("%s missing", acc_log)