Add per-instance InstanceFlags YAML field and front-CCW test (#1287)

Depends on #1245, #1286

## Summary
Introduces `AccelerationStructureInstanceFlags` with bit values that
intentionally match `D3D12_RAYTRACING_INSTANCE_FLAGS`,
`VkGeometryInstanceFlagBitsKHR`, and
`MTLAccelerationStructureInstanceOptions` so the value passes straight
through each backend's instance fill with a single `static_cast`. YAML
exposes the field as a bitset list (e.g. `InstanceFlags:
[TriangleCullDisable, ForceOpaque]`).

The covering test (`Feature/InlineRT/instance-flags.test`) sets
`TriangleFrontCounterclockwise` on one of two instances of the same
single-triangle BLAS and verifies `CommittedTriangleFrontFace()` flips —
same world-space ray, same vertices, only the per-instance flag changes.

Part of the inline-RT test coverage epic
(#1258).

## Test plan

Local on an NVIDIA RTX 3060:
- [x] Linux Vulkan (native `offloader`)
- [ ] Linux D3D12 (Wine + vkd3d-proton + cross-compiled `offloader.exe`)
- [ ] Windows Vulkan (native `offloader.exe`)
- [ ] Windows D3D12 (native `offloader.exe`)

CI (RT-capable runners):
- [x] windows-nvidia D3D12 (`RaytracingTier 1.2`)
- [ ] windows-intel VK (`VK_KHR_ray_tracing_pipeline`)
- [x] macOS Metal (`supportsRaytracing`)

Co-authored-by: Claude Opus 4.7 (1M context) <noreply@anthropic.com>

Author: MarijnS95

include/API/AccelerationStructure.h

@@ -14,13 +14,26 @@
 #include "API/Resources.h"
 
 #include "llvm/ADT/ArrayRef.h"
+#include "llvm/ADT/BitmaskEnum.h"
 #include "llvm/ADT/SmallVector.h"
 #include "llvm/Support/Error.h"
 
 #include <cstdint>
 #include <variant>
 
 namespace offloadtest {
+LLVM_ENABLE_BITMASK_ENUMS_IN_NAMESPACE();
+
+// Bit values match D3D12_RAYTRACING_INSTANCE_FLAGS, VkGeometryInstanceFlagBits-
+// KHR, and MTLAccelerationStructureInstanceOptions so backends can pass the
+// value through unchanged.
+enum AccelerationStructureInstanceFlags : uint32_t {
+  InstanceFlagNone = 0,
+  TriangleCullDisable = 1 << 0,
+  TriangleFrontCounterclockwise = 1 << 1,
+  ForceOpaque = 1 << 2,
+  ForceNonOpaque = 1 << 3,
+};
 
 struct AccelerationStructureSizes {
   uint64_t ResultDataMaxSizeInBytes = 0;
@@ -57,6 +70,7 @@ struct AccelerationStructureInstance {
   uint8_t InstanceMask = 0xFF;
   // 24-bit; high bits are truncated by the backend to match DXR's bitfield.
   uint32_t InstanceContributionToHitGroupIndex = 0;
+  AccelerationStructureInstanceFlags Flags = InstanceFlagNone;
   AccelerationStructure *BLAS = nullptr;
 };
 
@@ -178,4 +192,9 @@ class AccelerationStructure {
 
 } // namespace offloadtest
 
+namespace llvm {
+LLVM_DECLARE_ENUM_AS_BITMASK(::offloadtest::AccelerationStructureInstanceFlags,
+                             ::offloadtest::ForceNonOpaque);
+} // namespace llvm
+
 #endif // OFFLOADTEST_API_ACCELERATIONSTRUCTURE_H

include/Support/Pipeline.h

@@ -13,6 +13,7 @@
 #ifndef OFFLOADTEST_SUPPORT_PIPELINE_H
 #define OFFLOADTEST_SUPPORT_PIPELINE_H
 
+#include "API/AccelerationStructure.h"
 #include "API/Enums.h"
 #include "API/Resources.h"
 #include "llvm/ADT/SmallVector.h"
@@ -574,6 +575,7 @@ struct InstanceDesc {
   uint32_t InstanceID = 0;
   uint8_t InstanceMask = 0xFF;
   uint32_t InstanceContributionToHitGroupIndex = 0;
+  AccelerationStructureInstanceFlags Flags = InstanceFlagNone;
 };
 
 struct TLASDesc {
@@ -850,6 +852,19 @@ template <> struct MappingTraits<offloadtest::ShaderBindingTableDesc> {
   static void mapping(IO &I, offloadtest::ShaderBindingTableDesc &S);
 };
 
+template <>
+struct ScalarBitSetTraits<offloadtest::AccelerationStructureInstanceFlags> {
+  static void bitset(IO &I,
+                     offloadtest::AccelerationStructureInstanceFlags &V) {
+#define BIT_CASE(Val) I.bitSetCase(V, #Val, offloadtest::Val)
+    BIT_CASE(TriangleCullDisable);
+    BIT_CASE(TriangleFrontCounterclockwise);
+    BIT_CASE(ForceOpaque);
+    BIT_CASE(ForceNonOpaque);
+#undef BIT_CASE
+  }
+};
+
 template <> struct ScalarEnumerationTraits<offloadtest::Rule> {
   static void enumeration(IO &I, offloadtest::Rule &V) {
 #define ENUM_CASE(Val) I.enumCase(V, #Val, offloadtest::Rule::Val)

lib/API/DX/Device.cpp

@@ -3454,7 +3454,9 @@ llvm::Error DXComputeEncoder::batchBuildAS(llvm::ArrayRef<ASBuildItem> Items) {
         NI.InstanceMask = Inst.InstanceMask;
         NI.InstanceContributionToHitGroupIndex =
             Inst.InstanceContributionToHitGroupIndex & 0xFFFFFFu;
-        NI.Flags = D3D12_RAYTRACING_INSTANCE_FLAG_NONE;
+        // Bits in AccelerationStructureInstanceFlags match
+        // D3D12_RAYTRACING_INSTANCE_FLAGS by design.
+        NI.Flags = static_cast<D3D12_RAYTRACING_INSTANCE_FLAGS>(Inst.Flags);
         auto *BLASPtr = llvm::cast<DXAccelerationStructure>(Inst.BLAS);
         NI.AccelerationStructure = BLASPtr->getGPUVirtualAddress();
         Native.push_back(NI);

lib/API/Device.cpp

@@ -250,6 +250,7 @@ llvm::Error offloadtest::buildPipelineAccelerationStructures(
       Inst.InstanceMask = I.InstanceMask;
       Inst.InstanceContributionToHitGroupIndex =
           I.InstanceContributionToHitGroupIndex;
+      Inst.Flags = I.Flags;
       Inst.BLAS = It->second;
       Req.Instances.push_back(Inst);
     }

lib/API/MTL/MTLDevice.cpp

@@ -3249,7 +3249,10 @@ llvm::Error MTLComputeEncoder::batchBuildAS(llvm::ArrayRef<ASBuildItem> Items) {
         for (int Row = 0; Row < 3; ++Row)
           for (int Col = 0; Col < 4; ++Col)
             D.transformationMatrix.columns[Col][Row] = Src.Transform[Row][Col];
-        D.options = MTL::AccelerationStructureInstanceOptionNone;
+        // Bits in AccelerationStructureInstanceFlags match
+        // MTLAccelerationStructureInstanceOptions by design.
+        D.options =
+            static_cast<MTL::AccelerationStructureInstanceOptions>(Src.Flags);
         D.mask = Src.InstanceMask;
         D.intersectionFunctionTableOffset = 0;
         D.accelerationStructureIndex = InstanceASIdx[I];

lib/API/VK/Device.cpp

@@ -4901,7 +4901,9 @@ llvm::Error VKComputeEncoder::batchBuildAS(llvm::ArrayRef<ASBuildItem> Items) {
         NI.mask = Inst.InstanceMask;
         NI.instanceShaderBindingTableRecordOffset =
             Inst.InstanceContributionToHitGroupIndex & 0xFFFFFFu;
-        NI.flags = 0;
+        // Bits in AccelerationStructureInstanceFlags match
+        // VkGeometryInstanceFlagBitsKHR by design.
+        NI.flags = static_cast<VkGeometryInstanceFlagsKHR>(Inst.Flags);
         auto *BLASPtr = llvm::cast<VulkanAccelerationStructure>(Inst.BLAS);
         NI.accelerationStructureReference = BLASPtr->getDeviceAddress();
         Native.push_back(NI);

lib/Support/Pipeline.cpp

@@ -674,6 +674,7 @@ void MappingTraits<offloadtest::InstanceDesc>::mapping(
   D.InstanceMask = static_cast<uint8_t>(Mask);
   I.mapOptional("InstanceContributionToHitGroupIndex",
                 D.InstanceContributionToHitGroupIndex, 0u);
+  I.mapOptional("InstanceFlags", D.Flags, offloadtest::InstanceFlagNone);
 }
 
 void MappingTraits<offloadtest::TLASDesc>::mapping(IO &I,

test/Feature/InlineRT/instance-flags.test

@@ -0,0 +1,89 @@
+#--- source.hlsl
+
+[[vk::binding(0, 0)]] RaytracingAccelerationStructure Scene : register(t0);
+[[vk::binding(1, 0)]] RWStructuredBuffer<uint> Output : register(u0);
+
+[numthreads(2,1,1)]
+void main(uint3 tid : SV_DispatchThreadID) {
+  // The triangle vertices wind CW from +z under the DX / VK / MTL ray-
+  // traversal convention. Two instances tiled along x: instance 0 has no
+  // flags (front-facing), instance 1 has TriangleFrontCounterclockwise
+  // (winding interpretation flips → back-facing).
+  RayDesc Ray;
+  Ray.Origin = float3((float(tid.x) - 0.5) * 4.0, 0, 1);
+  Ray.Direction = float3(0, 0, -1);
+  Ray.TMin = 0.0;
+  Ray.TMax = 100.0;
+  RayQuery<RAY_FLAG_NONE> Q;
+  Q.TraceRayInline(Scene, RAY_FLAG_NONE, 0xFF, Ray);
+  Q.Proceed();
+  Output[tid.x] = Q.CommittedStatus() == COMMITTED_TRIANGLE_HIT
+                      ? (uint)Q.CommittedTriangleFrontFace()
+                      : 0xFFFFFFFF;
+}
+//--- pipeline.yaml
+---
+Shaders:
+  - Stage: Compute
+    Entry: main
+Buffers:
+  - Name: Vertices
+    Format: Float32
+    Stride: 12
+    Data: [ 0.0, 1.0, 0.0, -1.0, -1.0, 0.0, 1.0, -1.0, 0.0 ]
+  - Name: Output
+    Format: UInt32
+    Stride: 4
+    FillSize: 8
+  - Name: Expected
+    Format: UInt32
+    Stride: 4
+    # Instance 0: default winding → front face (1)
+    # Instance 1: TriangleFrontCounterclockwise flips → back face (0)
+    Data: [ 1, 0 ]
+AccelerationStructures:
+  BLAS:
+    - Name: TriangleBLAS
+      Triangles:
+        - VertexBuffer: Vertices
+          VertexFormat: RGB32Float
+          VertexStride: 12
+          VertexCount: 3
+  TLAS:
+    - Name: Scene
+      Instances:
+        - BLAS: TriangleBLAS
+          Transform: [1, 0, 0, -2,  0, 1, 0, 0,  0, 0, 1, 0]
+        - BLAS: TriangleBLAS
+          Transform: [1, 0, 0, 2,  0, 1, 0, 0,  0, 0, 1, 0]
+          InstanceFlags: [TriangleFrontCounterclockwise]
+DescriptorSets:
+  - Resources:
+    - Name: Scene
+      Kind: AccelerationStructure
+      DirectXBinding:
+        Register: 0
+        Space: 0
+      VulkanBinding:
+        Binding: 0
+    - Name: Output
+      Kind: RWStructuredBuffer
+      DirectXBinding:
+        Register: 0
+        Space: 0
+      VulkanBinding:
+        Binding: 1
+Results:
+  - Result: InstanceFlags
+    Rule: BufferExact
+    Actual: Output
+    Expected: Expected
+...
+#--- end
+
+# REQUIRES: acceleration-structure
+# XFAIL: Clang
+
+# RUN: split-file %s %t
+# RUN: %dxc_target -T cs_6_5 -Fo %t.o %t/source.hlsl
+# RUN: %offloader %t/pipeline.yaml %t.o