[CIR] Implement static-local-tls lowering (llvm#194059)

thread_local variables at function scope work very much like
static-locals, except with slightly different lowering from
cir-lowering-prepare. This patch implements that lowering. Global tls
variables are left to a later patch.

One decision I made here was that LocalInitOp lost its
'static-local'-ness, and assumes it is always static-local. Global TLS
is probably just going to use Global directly, so we don't need to to
permit it.

I DID leave it in the printing, as it makes it more clear what is
happening/for symmetry with get_global/global.

---------

Co-authored-by: Henrich Lauko <henrich.lau@gmail.com>

Author: erichkeane

clang/include/clang/CIR/Dialect/IR/CIROps.td

@@ -3819,9 +3819,7 @@ def CIR_FuncOp : CIR_Op<"func", [
 //===----------------------------------------------------------------------===//
 
 def CIR_LocalInitOp : CIR_Op<"local_init", [
-  DeclareOpInterfaceMethods<SymbolUserOpInterface>, NoRegionArguments,
-  ParentOneOf<["cir::FuncOp"]>,
-  HasAtMostOneOfAttrs<["tls", "static_local"]>
+  DeclareOpInterfaceMethods<SymbolUserOpInterface>, NoRegionArguments
 ]> {
   let summary = "initialize a static or thread local object";
   let description = [{
@@ -3830,8 +3828,13 @@ def CIR_LocalInitOp : CIR_Op<"local_init", [
     variable. This will be handled during lowering-prepare to include the
     guard variables correctly for the variable.
 
+    This operation may also represent a static local thread local variable,
+    which would be indicated by the 'tls' flag.
+
     Example:
     ```
+    // Note: despite this always being static_local, we print it anyway to be
+    // visually consistent with get_global, and as a 'counter' to 'tls'.
     cir.local_init static_local @GlobalName ctor {
       %4 = cir.get_global static_local @GlobalName : !cir.ptr<!rec_CtorDtor>
       %5 = cir.call @_Z5get_iv() : () -> !s32i
@@ -3846,8 +3849,7 @@ def CIR_LocalInitOp : CIR_Op<"local_init", [
   }];
   let arguments = (ins
     FlatSymbolRefAttr : $globalName,
-    UnitAttr : $tls,
-    UnitAttr : $static_local
+    UnitProp : $tls
   );
 
   let regions = (region
@@ -3856,8 +3858,7 @@ def CIR_LocalInitOp : CIR_Op<"local_init", [
   );
 
   let assemblyFormat = [{
-    (`thread_local` $tls^)?
-    (`static_local` $static_local^)?
+    (`thread_local` $tls^) : (`static_local`)?
     $globalName attr-dict
     (`ctor` $ctorRegion^)?
     (`dtor` $dtorRegion^)?
@@ -3873,6 +3874,7 @@ def CIR_LocalInitOp : CIR_Op<"local_init", [
 }
 }];
 
+  let hasVerifier = 1;
   let hasLLVMLowering = false;
 }
 

clang/lib/CIR/CodeGen/CIRGenCXX.cpp

@@ -155,8 +155,6 @@ static void emitDeclDestroy(CIRGenFunction &cgf, const VarDecl *vd,
   // directly.
   cir::FuncOp fnOp;
   if (record && (canRegisterDestructor || cgm.getCodeGenOpts().CXAAtExit)) {
-    if (vd->getTLSKind())
-      cgm.errorNYI(vd->getSourceRange(), "TLS destructor");
     assert(!record->hasTrivialDestructor());
     assert(!cir::MissingFeatures::openCL());
     CXXDestructorDecl *dtor = record->getDestructor();
@@ -343,16 +341,11 @@ void CIRGenModule::emitCXXGlobalVarDeclInit(const VarDecl *varDecl,
 void CIRGenModule::emitCXXStaticLocalVarDeclInit(const VarDecl *varDecl,
                                                  cir::GlobalOp addr,
                                                  bool performInit) {
-  assert(varDecl->isStaticLocal() ||
-         varDecl->getTLSKind() != VarDecl::TLS_None);
+  assert(varDecl->isStaticLocal());
 
-  if (varDecl->getTLSKind() != VarDecl::TLS_None)
-    errorNYI(varDecl->getSourceRange(),
-             "TLS not implemented for static-local init");
-
-  auto initOp = cir::LocalInitOp::create(
-      builder, addr->getLoc(), addr.getSymNameAttr(),
-      varDecl->getTLSKind() != VarDecl::TLS_None, varDecl->isStaticLocal());
+  auto initOp =
+      cir::LocalInitOp::create(builder, addr->getLoc(), addr.getSymNameAttr(),
+                               varDecl->getTLSKind() != VarDecl::TLS_None);
 
   emitCXXSpecialVarDeclInit(varDecl, addr, performInit, initOp.getCtorRegion(),
                             initOp.getDtorRegion());

clang/lib/CIR/CodeGen/CIRGenDecl.cpp

@@ -459,7 +459,7 @@ CIRGenModule::getOrCreateStaticVarDecl(const VarDecl &d,
     gv.setComdat(true);
 
   if (d.getTLSKind())
-    errorNYI(d.getSourceRange(), "getOrCreateStaticVarDecl: TLS");
+    setTLSMode(gv, d);
 
   setGVProperties(gv, &d);
 
@@ -642,7 +642,8 @@ void CIRGenFunction::emitStaticVarDecl(const VarDecl &d,
   cir::GlobalOp globalOp = cgm.getOrCreateStaticVarDecl(d, linkage);
   // TODO(cir): we should have a way to represent global ops as values without
   // having to emit a get global op. Sometimes these emissions are not used.
-  mlir::Value addr = builder.createGetGlobal(globalOp);
+  mlir::Value addr =
+      builder.createGetGlobal(globalOp, d.getTLSKind() != VarDecl::TLS_None);
   auto getAddrOp = addr.getDefiningOp<cir::GetGlobalOp>();
   assert(getAddrOp && "expected cir::GetGlobalOp");
 

clang/lib/CIR/CodeGen/CIRGenItaniumCXXABI.cpp

@@ -1763,11 +1763,6 @@ void CIRGenItaniumCXXABI::registerGlobalDtor(const VarDecl *vd,
   if (vd->isNoDestroy(cgm.getASTContext()))
     return;
 
-  if (vd->getTLSKind()) {
-    cgm.errorNYI(vd->getSourceRange(), "registerGlobalDtor: TLS");
-    return;
-  }
-
   // HLSL doesn't support atexit.
   if (cgm.getLangOpts().HLSL) {
     cgm.errorNYI(vd->getSourceRange(), "registerGlobalDtor: HLSL");

clang/lib/CIR/Dialect/IR/CIRDialect.cpp

@@ -375,6 +375,12 @@ LogicalResult cir::BreakOp::verify() {
 // LocalInitOp
 //===----------------------------------------------------------------------===//
 
+LogicalResult cir::LocalInitOp::verify() {
+  if (!getOperation()->getParentOfType<FuncOp>())
+    return emitOpError("must be inside of a 'cir.func'");
+  return success();
+}
+
 LogicalResult
 cir::LocalInitOp::verifySymbolUses(SymbolTableCollection &symbolTable) {
   cir::GlobalOp global = getReferencedGlobal(symbolTable);
@@ -385,10 +391,7 @@ cir::LocalInitOp::verifySymbolUses(SymbolTableCollection &symbolTable) {
   if (getTls() && !global.getTlsModel())
     return emitOpError("access to global not marked thread local");
 
-  bool isStaticLocal = getStaticLocal();
-  bool globalIsStaticLocal = global.getStaticLocalGuard().has_value();
-
-  if (isStaticLocal != globalIsStaticLocal)
+  if (!global.getStaticLocalGuard().has_value())
     return emitOpError("static_local attribute mismatch");
 
   return success();

clang/lib/CIR/Dialect/Transforms/LoweringPrepare.cpp

@@ -183,6 +183,7 @@ struct LoweringPreparePass
       guard.setInitialValueAttr(cir::IntAttr::get(guardTy, 0));
       guard.setDSOLocal(globalOp.getDsoLocal());
       guard.setAlignment(guardAlignment.getAsAlign().value());
+      guard.setTlsModel(globalOp.getTlsModel());
 
       // The ABI says: "It is suggested that it be emitted in the same COMDAT
       // group as the associated data object." In practice, this doesn't work
@@ -241,7 +242,7 @@ struct LoweringPreparePass
 
   void emitGlobalGuardedDtorRegion(CIRBaseBuilderTy &builder,
                                    cir::GlobalOp global,
-                                   mlir::Region &dtorRegion,
+                                   mlir::Region &dtorRegion, bool tls,
                                    mlir::Block &entryBB) {
     // Create a variable that binds the atexit to this shared object.
     builder.setInsertionPointToStart(&mlirModule.getBodyRegion().front());
@@ -266,6 +267,11 @@ struct LoweringPreparePass
         builder.getVoidFnTy({voidFnPtrTy, voidPtrTy, handlePtrTy});
 
     llvm::StringLiteral nameAtExit = "__cxa_atexit";
+    if (tls)
+      nameAtExit = astCtx->getTargetInfo().getTriple().isOSDarwin()
+                       ? llvm::StringLiteral("_tlv_atexit")
+                       : llvm::StringLiteral("__cxa_thread_atexit");
+
     cir::FuncOp fnAtExit = buildRuntimeFunction(builder, nameAtExit,
                                                 global.getLoc(), fnAtExitType);
 
@@ -317,6 +323,28 @@ struct LoweringPreparePass
     // initialization so that recursive accesses during initialization do not
     // restart initialization.
 
+    auto emitBody = [&]() {
+      // Emit the initializer and add a global destructor if appropriate.
+      mlir::Block *insertBlock = builder.getInsertionBlock();
+      if (!ctorRegion.empty()) {
+        assert(ctorRegion.hasOneBlock() && "Enforced by MaxSizedRegion<1>");
+
+        mlir::Block &block = ctorRegion.front();
+        insertBlock->getOperations().splice(
+            insertBlock->end(), block.getOperations(), block.begin(),
+            std::prev(block.end()));
+      }
+
+      if (!dtorRegion.empty()) {
+        assert(dtorRegion.hasOneBlock() && "Enforced by MaxSizedRegion<1>");
+
+        emitGlobalGuardedDtorRegion(builder, globalOp, dtorRegion, !threadsafe,
+                                    *insertBlock);
+      }
+      builder.setInsertionPointToEnd(insertBlock);
+      ctorRegion.getBlocks().clear();
+    };
+
     // Variables used when coping with thread-safe statics and exceptions.
     if (threadsafe) {
       // Call __cxa_guard_acquire.
@@ -341,26 +369,7 @@ struct LoweringPreparePass
       // OG: CGF.EHStack.pushCleanup<CallGuardAbort>(EHCleanup, guard);
       assert(!cir::MissingFeatures::guardAbortOnException());
 
-      // Emit the initializer and add a global destructor if appropriate.
-      mlir::Block *insertBlock = builder.getInsertionBlock();
-      if (!ctorRegion.empty()) {
-        assert(ctorRegion.hasOneBlock() && "Enforced by MaxSizedRegion<1>");
-
-        mlir::Block &block = ctorRegion.front();
-        insertBlock->getOperations().splice(
-            insertBlock->end(), block.getOperations(), block.begin(),
-            std::prev(block.end()));
-      }
-
-      if (!dtorRegion.empty()) {
-        assert(dtorRegion.hasOneBlock() && "Enforced by MaxSizedRegion<1>");
-
-        emitGlobalGuardedDtorRegion(builder, globalOp, dtorRegion,
-                                    *insertBlock);
-      }
-
-      builder.setInsertionPointToEnd(insertBlock);
-      ctorRegion.getBlocks().clear();
+      emitBody();
 
       // Pop the guard-abort cleanup if we pushed one.
       // OG: CGF.PopCleanupBlock();
@@ -380,11 +389,13 @@ struct LoweringPreparePass
       globalOp->emitError("NYI: non-threadsafe init for non-local variables");
       return;
     } else {
+      emitBody();
       // For local variables, store 1 into the first byte of the guard variable
       // after the object initialization completes so that initialization is
       // retried if initialization is interrupted by an exception.
-      globalOp->emitError("NYI: non-threadsafe init for local variables");
-      return;
+      builder.createStore(
+          loc, builder.getConstantInt(loc, guardPtrTy.getPointee(), 1),
+          guardPtr);
     }
 
     builder.createYield(loc); // Outermost IfOp
@@ -1063,7 +1074,8 @@ LoweringPreparePass::buildCXXGlobalVarDeclInitFunc(cir::GlobalOp op) {
     assert(!cir::MissingFeatures::astVarDeclInterface());
     assert(!cir::MissingFeatures::opGlobalThreadLocal());
 
-    emitGlobalGuardedDtorRegion(builder, op, dtorRegion, *entryBB);
+    emitGlobalGuardedDtorRegion(builder, op, dtorRegion,
+                                op.getTlsModel().has_value(), *entryBB);
   }
 
   // Replace cir.yield with cir.return
@@ -1159,31 +1171,27 @@ void LoweringPreparePass::handleStaticLocal(cir::GlobalOp globalOp,
                     (varDecl.isLocalVarDecl() || nonTemplateInline) &&
                     !varDecl.getTLSKind();
 
-  // TLS variables need special handling - the guard must also be thread-local.
-  if (varDecl.getTLSKind()) {
-    globalOp->emitError("NYI: guarded initialization for thread-local statics");
-    return;
-  }
-
   // If we have a global variable with internal linkage and thread-safe statics
   // are disabled, we can just let the guard variable be of type i8.
   bool useInt8GuardVariable = !threadsafe && globalOp.hasInternalLinkage();
-  if (useInt8GuardVariable) {
-    globalOp->emitError("NYI: int8 guard variables for non-threadsafe statics");
-    return;
-  }
-
+  cir::CIRDataLayout dataLayout(mlirModule);
+  cir::IntType guardTy;
+  clang::CharUnits guardAlignment;
   // Guard variables are 64 bits in the generic ABI and size width on ARM
   // (i.e. 32-bit on AArch32, 64-bit on AArch64).
-  if (useARMGuardVarABI()) {
+  if (useInt8GuardVariable) {
+    guardTy = cir::IntType::get(&getContext(), 8, /*isSigned=*/true);
+    guardAlignment = clang::CharUnits::One();
+  } else if (useARMGuardVarABI()) {
     globalOp->emitError("NYI: ARM-style guard variables for static locals");
     return;
+  } else {
+    guardTy = cir::IntType::get(&getContext(), 64, /*isSigned=*/true);
+    guardAlignment =
+        clang::CharUnits::fromQuantity(dataLayout.getABITypeAlign(guardTy));
   }
-  cir::IntType guardTy =
-      cir::IntType::get(&getContext(), 64, /*isSigned=*/true);
-  cir::CIRDataLayout dataLayout(mlirModule);
-  clang::CharUnits guardAlignment =
-      clang::CharUnits::fromQuantity(dataLayout.getABITypeAlign(guardTy));
+  assert(guardTy && guardAlignment.getQuantity() != 0);
+
   auto guardPtrTy = cir::PointerType::get(guardTy);
 
   // Create the guard variable if we don't already have it.
@@ -1195,7 +1203,7 @@ void LoweringPreparePass::handleStaticLocal(cir::GlobalOp globalOp,
     return;
   }
 
-  mlir::Value guardPtr = builder.createGetGlobal(guard, /*threadLocal*/ false);
+  mlir::Value guardPtr = builder.createGetGlobal(guard, localInitOp.getTls());
 
   // Test whether the variable has completed initialization.
   //
@@ -1292,10 +1300,6 @@ void LoweringPreparePass::handleStaticLocal(cir::GlobalOp globalOp,
 
 void LoweringPreparePass::lowerLocalInitOp(
     cir::LocalInitOp initOp, mlir::SymbolTableCollection &symbolTables) {
-  if (!initOp.getStaticLocal()) {
-    initOp->emitError("NYI: Non-static-local in lower-init-local op");
-    return;
-  }
 
   // If we don't actually need to initialize anything anymore, we're done here.
   if (initOp.getCtorRegion().empty() && initOp.getDtorRegion().empty()) {