fix(hipdnn): wire golden harness into TOML tolerance + skip chain (#8738)

JIRA ID: ALMIOPEN-1969

## Summary

Extract shared `TomlGuards.hpp` and wire the non-golden and golden GPU
harnesses through it so TOML-based tolerance overrides and test skips
work uniformly. Fix `toleranceForNodeAttributes` missing SDPA cases.

## Motivation

Tolerance-override and test-skip logic was duplicated in the non-golden
harness and absent from the golden GPU harness
(`IntegrationGraphGoldenReferenceVerificationHarness`). TOML
`[[test_skips]]` and `[[tolerance_overrides]]` entries were silently
ignored for bundle tests. Additionally, `toleranceForNodeAttributes` had
no `SdpaAttributes`/`SdpaBackwardAttributes` cases, so SDPA bundles fell
through to the 1e-3 default instead of using
`tol::sdpa::getToleranceFwd<T>()`.

## Technical Details

- **New `TomlGuards.hpp`** -- three shared free functions:
`currentTestName()`, `applyTomlToleranceOverride()`, `checkTomlSkip()`.
- **Non-golden harness** -- replaced inline skip/override blocks with
`TomlGuards` calls (net -25 lines). `checkTomlSkip()` is inlined in
`SetUp()` so `GTEST_SKIP()` returns from `SetUp()` directly.
- **Golden GPU harness** -- added `checkTomlSkip()` inline in `SetUp()`
before `applyMetadataGuards()`. TOML tolerance override resolved once
before the per-tensor comparison loop in `compareEach()`, applied
per-tensor after `resolveTolerances()`.
- **SDPA tolerance** -- added `SdpaAttributes`/`SdpaBackwardAttributes`
cases to `toleranceForNodeAttributes<T>()`.

### Why `TestGoldenReferenceCpu` is intentionally excluded

`TestGoldenReferenceCpu` validates the CPU reference executor against
golden data — it does not run the GPU engine. The TOML `[[test_skips]]`
and `[[tolerance_overrides]]` are engine-specific (per-arch,
per-engine). Applying them here would mask CPU reference regressions: if
the oracle drifts from golden data, every GPU engine test that falls
back to the CPU reference (auto mode) silently compares against a broken
oracle and passes green. The CPU reference is deterministic and must
always run with its hardcoded per-op tolerances.

## Test Plan

- [x] Existing `TestTestConfig`, `TestTestSettings`,
`TestVerificationModePaths`, `TestGoldenVerificationHarness`,
`TestSynthesisTracker`, `TestSynthesizeInputs` all pass
- [x] Pre-commit hooks clean (clang-format, cmake-lint)
- [x] Bundle tests still gated behind `--allow-bundles`

## Test Result

All unit tests pass.

## Submission Checklist

- [x] Follows [contributing
guidelines](https://github.com/ROCm/ROCm/blob/develop/CONTRIBUTING.md#pull-requests)
- [x] No new warnings
- [x] All existing tests pass
cat: cd: No such file or directory
cat: /home/AMD/bghimire/worktrees/rocm-libraries-wt-1969: Is a directory

---------

Co-authored-by: Claude Opus 4.6 <noreply@anthropic.com>

Author: bghimireamd

dnn-providers/integration-tests/src/harness/IntegrationGraphVerificationHarness.hpp

@@ -30,6 +30,7 @@
 #include "harness/SharedHandle.hpp"
 #include "harness/SupportMatrixCollector.hpp"
 #include "harness/TestConfig.hpp"
+#include "harness/TomlGuards.hpp"
 
 namespace hipdnn_integration_tests
 {
@@ -58,15 +59,9 @@ class IntegrationGraphVerificationHarness : public ::testing::TestWithParam<Test
         ASSERT_EQ(hipInit(0), hipSuccess);
         ASSERT_EQ(hipGetDevice(&_deviceId), hipSuccess);
 
-        // Check for any engine specific test skips
-        if(auto* info = ::testing::UnitTest::GetInstance()->current_test_info(); info != nullptr)
+        if(auto reason = checkTomlSkip(currentTestName()))
         {
-            const std::string testName = std::string(info->test_suite_name()) + "." + info->name();
-            if(auto skipReason = TestConfig::get().findSkipForTest(testName))
-            {
-                GTEST_SKIP() << "[arch " << TestConfig::get().getCurrentArch() << "] "
-                             << *skipReason;
-            }
+            GTEST_SKIP() << "[arch " << TestConfig::get().getCurrentArch() << "] " << *reason;
         }
     }
 
@@ -254,25 +249,9 @@ class IntegrationGraphVerificationHarness : public ::testing::TestWithParam<Test
                            float absoluteTolerance,
                            float relativeTolerance)
     {
-        // Check for per-test tolerance override from TOML config
         float finalAtol = absoluteTolerance;
         float finalRtol = relativeTolerance;
-
-        auto* testInfo = ::testing::UnitTest::GetInstance()->current_test_info();
-        if(testInfo != nullptr)
-        {
-            std::string testName
-                = std::string(testInfo->test_suite_name()) + "." + testInfo->name();
-            auto override = TestConfig::get().findToleranceOverride(testName);
-            if(override.has_value())
-            {
-                finalAtol = override->atol;
-                finalRtol = override->rtol;
-                HIPDNN_PLUGIN_LOG_INFO("Tolerance override applied for " << testName
-                                                                         << ": atol=" << finalAtol
-                                                                         << " rtol=" << finalRtol);
-            }
-        }
+        applyTomlToleranceOverride(currentTestName(), finalAtol, finalRtol);
 
         // Since the graph can infer properties + Ids, we defer validator registration until right
         // before validation in verifyGraph

dnn-providers/integration-tests/src/harness/TomlGuards.hpp

@@ -0,0 +1,55 @@
+// Copyright © Advanced Micro Devices, Inc., or its affiliates.
+// SPDX-License-Identifier: MIT
+
+#pragma once
+
+#include <optional>
+#include <string>
+
+#include <gtest/gtest.h>
+
+#include <hipdnn_plugin_sdk/PluginLogging.hpp>
+
+#include "harness/TestConfig.hpp"
+
+namespace hipdnn_integration_tests
+{
+
+inline std::string currentTestName()
+{
+    auto* info = ::testing::UnitTest::GetInstance()->current_test_info();
+    if(info == nullptr)
+    {
+        return {};
+    }
+    return std::string(info->test_suite_name()) + "." + info->name();
+}
+
+inline bool applyTomlToleranceOverride(const std::string& testName, float& atol, float& rtol)
+{
+    if(testName.empty())
+    {
+        return false;
+    }
+    auto ovr = TestConfig::get().findToleranceOverride(testName);
+    if(!ovr)
+    {
+        return false;
+    }
+    atol = ovr->atol;
+    rtol = ovr->rtol;
+    HIPDNN_PLUGIN_LOG_INFO("Tolerance override applied for " << testName << ": atol=" << atol
+                                                             << " rtol=" << rtol);
+    return true;
+}
+
+inline std::optional<std::string> checkTomlSkip(const std::string& testName)
+{
+    if(testName.empty())
+    {
+        return std::nullopt;
+    }
+    return TestConfig::get().findSkipForTest(testName);
+}
+
+} // namespace hipdnn_integration_tests

dnn-providers/integration-tests/src/harness/golden/IntegrationGraphGoldenReferenceVerificationHarness.cpp

@@ -24,6 +24,7 @@
 #include "harness/ReferenceCapabilityError.hpp"
 #include "harness/SharedHandle.hpp"
 #include "harness/TestConfig.hpp"
+#include "harness/TomlGuards.hpp"
 #include "harness/golden/UnverifiableBundleReport.hpp"
 #include "harness/golden/input_init/SynthesizeInputs.hpp"
 #include "harness/gpu_graph_executor/GpuReferenceGraphExecutor.hpp"
@@ -490,6 +491,14 @@ void IntegrationGraphGoldenReferenceVerificationHarness::compareEach(OutputTenso
     auto wrapper = _bundle->graphWrapper();
     const auto& tensorAttrMap = wrapper.getTensorMap();
 
+    const auto tomlOverride = TestConfig::get().findToleranceOverride(currentTestName());
+    if(tomlOverride)
+    {
+        HIPDNN_PLUGIN_LOG_INFO("Tolerance override applied for " << currentTestName()
+                                                                 << ": atol=" << tomlOverride->atol
+                                                                 << " rtol=" << tomlOverride->rtol);
+    }
+
     for(const int64_t uid : _bundle->outputTensorUids)
     {
         auto& actualTensor = *engineOutputs.at(uid);
@@ -502,6 +511,12 @@ void IntegrationGraphGoldenReferenceVerificationHarness::compareEach(OutputTenso
         float rtol = 0.0f;
         resolveTolerances(wrapper, dataType, atol, rtol);
 
+        if(tomlOverride)
+        {
+            atol = tomlOverride->atol;
+            rtol = tomlOverride->rtol;
+        }
+
         compareOutputTensor(uid, *attrs, dataType, expectedTensor, actualTensor, atol, rtol);
     }
 }
@@ -672,6 +687,10 @@ float IntegrationGraphGoldenReferenceVerificationHarness::toleranceForNodeAttrib
         return tol::pointwise::getTolerance<T>();
     case NA::LayernormAttributes:
         return tol::layernorm::getTolerance<T>();
+    case NA::SdpaAttributes:
+    case NA::SdpaBackwardAttributes:
+        // No backward golden tests yet; share forward tolerance until data exists
+        return tol::sdpa::getToleranceFwd<T>();
     default:
         return 1e-3f;
     }

dnn-providers/integration-tests/src/harness/golden/IntegrationGraphGoldenReferenceVerificationHarness.hpp

@@ -21,6 +21,7 @@
 
 #include "harness/IReferenceGraphExecutor.hpp"
 #include "harness/TestConfig.hpp"
+#include "harness/TomlGuards.hpp"
 #include "harness/golden/IntegrationTestBundle.hpp"
 
 namespace hipdnn_integration_tests::golden
@@ -53,6 +54,16 @@ using OutputTensors
 //     are mark*Modified().
 //   * Virtual (inter-node) tensors are allocated internally by each executor; the
 //     variant packs we build carry only real (input + output) tensors.
+//
+// TODO(ALMIOPEN-1969 follow-up): Unify graph-init with the non-golden harness.
+//   Stage 1 — Route non-golden ops whose initializeBundle() is plain randomize
+//             (conv, matmul, BN-inference, reduction, rmsnorm-fwd, layernorm,
+//             pointwise) through the synthesis switch. Zero behavioral change.
+//   Stage 2 — Migrate structured recipes one op at a time: copy the exact
+//             ranges/seeds/derivation from each non-golden subclass override
+//             into the corresponding fill function, using fillComputed/tensorAt
+//             for derived inputs. Delete each override once its fill fn works.
+//   Stage 3 — Both harnesses share one init pipeline via SynthesisTracker.
 class IntegrationGraphGoldenReferenceVerificationHarness : public ::testing::Test
 {
 public:
@@ -81,6 +92,11 @@ class IntegrationGraphGoldenReferenceVerificationHarness : public ::testing::Tes
             GTEST_SKIP() << "No bundle set";
         }
 
+        if(auto reason = checkTomlSkip(currentTestName()))
+        {
+            GTEST_SKIP() << "[arch " << TestConfig::get().getCurrentArch() << "] " << *reason;
+        }
+
         applyMetadataGuards();
     }
 
@@ -244,6 +260,9 @@ class IntegrationGraphGoldenReferenceVerificationHarness : public ::testing::Tes
     static std::string dataTypeName(hipdnn_flatbuffers_sdk::data_objects::DataType dataType);
 
     // ── tolerances ──────────────────────────────────────────────────────
+    // Two-level lookup: per-operation default from TestTolerances.hpp,
+    // then TOML per-engine override (if a [[tolerance_overrides]] filter
+    // matches the current gtest name).
     static void
         resolveTolerances(const hipdnn_flatbuffers_sdk::flatbuffer_utilities::GraphWrapper& wrapper,
                           hipdnn_flatbuffers_sdk::data_objects::DataType dataType,

dnn-providers/integration-tests/tests/CMakeLists.txt

@@ -12,6 +12,7 @@ add_executable(hipdnn_integration_tests_unit_tests
     TestSupportMatrixCollector.cpp
     TestTestConfig.cpp
     TestTestSettings.cpp
+    TestTomlGuards.cpp
     TestGpuReferenceGraphExecutor.cpp
     TestGpuConvolutionFwdSignatureKey.cpp
     TestGpuConvolutionFwdPlan.cpp

dnn-providers/integration-tests/tests/TestTomlGuards.cpp

@@ -0,0 +1,70 @@
+// Copyright © Advanced Micro Devices, Inc., or its affiliates.
+// SPDX-License-Identifier: MIT
+
+#include <gtest/gtest.h>
+
+#include "harness/TomlGuards.hpp"
+
+using hipdnn_integration_tests::applyTomlToleranceOverride;
+using hipdnn_integration_tests::checkTomlSkip;
+using hipdnn_integration_tests::currentTestName;
+
+// NOLINTBEGIN(readability-identifier-naming) -- gtest macro-generated names
+
+// ---------------------------------------------------------------------------
+// currentTestName — pure gtest, no TestConfig dependency
+// ---------------------------------------------------------------------------
+
+TEST(TestTomlGuards, NameReturnsExpectedFormat)
+{
+    const auto name = currentTestName();
+    EXPECT_EQ(name, "TestTomlGuards.NameReturnsExpectedFormat");
+}
+
+TEST(TestTomlGuards, NameContainsDot)
+{
+    const auto name = currentTestName();
+    EXPECT_NE(name.find('.'), std::string::npos);
+}
+
+// ---------------------------------------------------------------------------
+// checkTomlSkip / applyTomlToleranceOverride — empty-name early-return path
+// ---------------------------------------------------------------------------
+
+TEST(TestTomlGuards, CheckTomlSkipReturnsNulloptForEmptyName)
+{
+    EXPECT_EQ(checkTomlSkip(""), std::nullopt);
+}
+
+TEST(TestTomlGuards, ApplyTomlToleranceOverrideReturnsFalseForEmptyName)
+{
+    float atol = 1.0f;
+    float rtol = 1.0f;
+    EXPECT_FALSE(applyTomlToleranceOverride("", atol, rtol));
+    EXPECT_FLOAT_EQ(atol, 1.0f);
+    EXPECT_FLOAT_EQ(rtol, 1.0f);
+}
+
+// ---------------------------------------------------------------------------
+// checkTomlSkip / applyTomlToleranceOverride — no TOML loaded
+//
+// TestConfig is initialized (by TestConfigInitialized in TestTestConfig.cpp,
+// same binary) without a settings file, so findSkipForTest / findToleranceOverride
+// return nullopt for any test name.
+// ---------------------------------------------------------------------------
+
+TEST(TestTomlGuards, CheckTomlSkipReturnsNulloptWhenNoSettings)
+{
+    EXPECT_EQ(checkTomlSkip("SomeTest.Name"), std::nullopt);
+}
+
+TEST(TestTomlGuards, ApplyTomlToleranceOverrideReturnsFalseWhenNoSettings)
+{
+    float atol = 1.0f;
+    float rtol = 1.0f;
+    EXPECT_FALSE(applyTomlToleranceOverride("SomeTest.Name", atol, rtol));
+    EXPECT_FLOAT_EQ(atol, 1.0f);
+    EXPECT_FLOAT_EQ(rtol, 1.0f);
+}
+
+// NOLINTEND(readability-identifier-naming)