[clang] Don't optimize out no-op atomics in kernel mode (llvm#193562)

The no-op atomics like InterlockedAnd(addr, (UINT32)-1) don't modify
the underlying value, however kernel code depends on these accesses
to touch the pool page virtual address and intentionally trigger a page
fault during page migration. This patch also fixes an LLVM issue where
idempotent volatile atomics were incorrectly lowered into memory fences.

Author: eleviant

clang/lib/CodeGen/CGBuiltin.cpp

@@ -315,6 +315,9 @@ Value *MakeBinaryAtomicValue(
 
   llvm::Value *Result =
       CGF.Builder.CreateAtomicRMW(Kind, DestAddr, Val, Ordering);
+  // Consider atomics to be volatile in MS kernel mode.
+  if (CGF.CGM.getLangOpts().Kernel)
+    cast<llvm::AtomicRMWInst>(Result)->setVolatile(true);
   return EmitFromInt(CGF, Result, T, ValueType);
 }
 

clang/test/CodeGen/AArch64/mskernel-interlocked.c

@@ -0,0 +1,11 @@
+// Check that we don't fold no-op andl to memory barrier
+// REQUIRES: aarch64-registered-target
+// RUN: %clang_cc1 -fms-kernel -fms-extensions -Wno-implicit-function-declaration  -triple aarch64-pc-win32 -O2 -S -o - %s | FileCheck %s --check-prefix=ARM64
+
+// ARM64:      ldaxr
+// ARM64-NEXT: stlxr
+// ARM64-NEXT: cbnz
+
+void access_via_interlocked(long volatile* addr) {
+    _InterlockedAnd(addr, (long)-1);
+}

clang/test/CodeGen/X86/mskernel-interlocked.c

@@ -0,0 +1,10 @@
+// Check that we don't fold no-op andl to memory barrier
+// REQUIRES: x86-registered-target
+// RUN: %clang_cc1 -fms-kernel -fms-extensions -Wno-implicit-function-declaration  -triple x86_64-pc-win32 -O2 -S -o - %s | FileCheck %s --check-prefix=X86
+
+// X86:      lock andl $-1, (%rcx)
+// X86-NEXT: retq
+
+void access_via_interlocked(long volatile* addr) {
+    _InterlockedAnd(addr, (long)-1);
+}

llvm/lib/CodeGen/AtomicExpandPass.cpp

@@ -1710,6 +1710,8 @@ bool AtomicExpandImpl::expandAtomicCmpXchg(AtomicCmpXchgInst *CI) {
 }
 
 bool AtomicExpandImpl::isIdempotentRMW(AtomicRMWInst *RMWI) {
+  if (RMWI->isVolatile())
+    return false;
   // TODO: Add floating point support.
   auto C = dyn_cast<ConstantInt>(RMWI->getValOperand());
   if (!C)

llvm/test/CodeGen/X86/volatile-atomicrmw.ll

@@ -0,0 +1,13 @@
+; RUN: opt -S -passes='require<libcall-lowering-info>,expand-ir-insts,atomic-expand' %s -o - | FileCheck %s
+
+; volatile atomicrmw shouldn't be converted to a fence
+; CHECK:  %0 = atomicrmw volatile and ptr %addr, i32 -1 seq_cst
+
+target triple = "x86_64-pc-windows-msvc"
+
+define dso_local void @access_via_interlocked(ptr noundef %addr) {
+entry:
+  %0 = atomicrmw volatile and ptr %addr, i32 -1 seq_cst, align 4
+  ret void
+}
+