Implement -femit-local-var-lifetime which adds local (stack) variable… (#4395)

Implement -femit-local-var-lifetime which adds local (stack) variable lifetime annotation to LLVM IR, which enables sharing stack space for variables whose lifetimes do not overlap.
Resolves issue #2227

This is not enabled by default yet, to prevent miscompilation due to bugs (should be enabled in future for optimization levels > 0, and when sanitizers are enabled).

CHANGELOG.md

@@ -2,6 +2,7 @@
 
 #### Big news
 - Frontend, druntime and Phobos are at version [2.103.1](https://dlang.org/changelog/2.103.0.html), incl. new command-line option `-verror-supplements`. (#4345)
+- New commandline option `-femit-local-var-lifetime` that enables variable lifetime (scope) annotation to LLVM IR codegen. Lifetime annotation enables stack memory reuse for local variables with non-overlapping scope. (#4395)
 
 #### Platform support
 

gen/funcgenstate.cpp

@@ -100,8 +100,8 @@ llvm::BasicBlock *SwitchCaseTargets::getOrCreate(Statement *stmt,
 }
 
 FuncGenState::FuncGenState(IrFunction &irFunc, IRState &irs)
-    : irFunc(irFunc), scopes(irs), jumpTargets(scopes), switchTargets(),
-      irs(irs) {}
+    : irFunc(irFunc), scopes(irs), localVariableLifetimeAnnotator(irs),
+      jumpTargets(scopes), switchTargets(), irs(irs) {}
 
 LLCallBasePtr FuncGenState::callOrInvoke(llvm::Value *callee,
                                          llvm::FunctionType *calleeType,

gen/funcgenstate.h

@@ -17,6 +17,7 @@
 #include "gen/irstate.h"
 #include "gen/pgo_ASTbased.h"
 #include "gen/trycatchfinally.h"
+#include "gen/variable_lifetime.h"
 #include "llvm/ADT/DenseMap.h"
 #include <vector>
 
@@ -176,6 +177,8 @@ public:
 
   TryCatchFinallyScopes scopes;
 
+  LocalVariableLifetimeAnnotator localVariableLifetimeAnnotator;
+
   JumpTargets jumpTargets;
 
   // PGO information

gen/llvmhelpers.cpp

@@ -919,19 +919,29 @@ void DtoVarDeclaration(VarDeclaration *vd) {
     Type *type = isSpecialRefVar(vd) ? vd->type->pointerTo() : vd->type;
 
     llvm::Value *allocainst;
+    bool isRealAlloca = false;
     LLType *lltype = DtoType(type); // void for noreturn
     if (lltype->isVoidTy() || gDataLayout->getTypeSizeInBits(lltype) == 0) {
       allocainst = llvm::ConstantPointerNull::get(getPtrToType(lltype));
     } else if (type != vd->type) {
       allocainst = DtoAlloca(type, vd->toChars());
+      isRealAlloca = true;
     } else {
       allocainst = DtoAlloca(vd, vd->toChars());
+      isRealAlloca = true;
     }
 
     irLocal->value = allocainst;
 
     if (!lltype->isVoidTy())
       gIR->DBuilder.EmitLocalVariable(allocainst, vd);
+
+    // Lifetime annotation is only valid on alloca.
+    if (isRealAlloca) {
+      // The lifetime of a stack variable starts from the point it is declared
+      gIR->funcGen().localVariableLifetimeAnnotator.addLocalVariable(
+          allocainst, DtoConstUlong(type->size()));
+    }
   }
 
   IF_LOG Logger::cout() << "llvm value for decl: " << *getIrLocal(vd)->value

gen/llvmhelpers.h

@@ -294,3 +294,6 @@ DValue *makeVarDValue(Type *type, VarDeclaration *vd,
 bool toInPlaceConstruction(DLValue *lhs, Expression *rhs);
 
 std::string llvmTypeToString(LLType *type);
+
+void emitLifetimeStart(llvm::Value *size, llvm::Value *addr);
+void emitLifetimeEnd(llvm::Value *size, llvm::Value *addr);

gen/optimizer.cpp

@@ -204,7 +204,9 @@ static void legacyAddGarbageCollect2StackPass(const PassManagerBuilder &builder,
 
 static void legacyAddAddressSanitizerPasses(const PassManagerBuilder &Builder,
                                             PassManagerBase &PM) {
-  PM.add(createAddressSanitizerFunctionPass());
+  PM.add(createAddressSanitizerFunctionPass(/*CompileKernel = */ false,
+                                            /*Recover = */ false,
+                                            /*UseAfterScope = */ true));
   PM.add(createModuleAddressSanitizerLegacyPassPass());
 }
 

gen/statements.cpp

@@ -397,6 +397,9 @@ public:
     // start a dwarf lexical block
     irs->DBuilder.EmitBlockStart(stmt->loc);
     emitCoverageLinecountInc(stmt->loc);
+    // Open a new scope for the optional condition variable (`if (auto i = ...)`)
+    irs->funcGen().localVariableLifetimeAnnotator.pushScope();
+
 
     // This is a (dirty) hack to get codegen time conditional
     // compilation, on account of the fact that we are trying
@@ -482,6 +485,9 @@ public:
 
     // rewrite the scope
     irs->ir->SetInsertPoint(endbb);
+    // Close the scope for the optional condition variable. This is suboptimal,
+    // because the condition variable is not in scope in the else block.
+    irs->funcGen().localVariableLifetimeAnnotator.popScope();
   }
 
   //////////////////////////////////////////////////////////////////////////
@@ -494,9 +500,11 @@ public:
     PGO.setCurrentStmt(stmt);
 
     if (stmt->statement) {
+      irs->funcGen().localVariableLifetimeAnnotator.pushScope();
       irs->DBuilder.EmitBlockStart(stmt->statement->loc);
       stmt->statement->accept(this);
       irs->DBuilder.EmitBlockEnd();
+      irs->funcGen().localVariableLifetimeAnnotator.popScope();
     }
   }
 
@@ -636,6 +644,7 @@ public:
 
     // start new dwarf lexical block
     irs->DBuilder.EmitBlockStart(stmt->loc);
+    irs->funcGen().localVariableLifetimeAnnotator.pushScope();
 
     // create for blocks
     llvm::BasicBlock *forbb = irs->insertBB("forcond");
@@ -717,6 +726,7 @@ public:
     irs->ir->SetInsertPoint(endbb);
 
     // end the dwarf lexical block
+    irs->funcGen().localVariableLifetimeAnnotator.popScope();
     irs->DBuilder.EmitBlockEnd();
   }
 

gen/variable_lifetime.cpp

@@ -0,0 +1,97 @@
+//===-- gen/variable_lifetime.cpp - -----------------------------*- C++ -*-===//
+//
+//                         LDC – the LLVM D compiler
+//
+// This file is distributed under the BSD-style LDC license. See the LICENSE
+// file for details.
+//
+//===----------------------------------------------------------------------===//
+//
+// Codegen for local variable lifetime: llvm.lifetime.start abd
+// llvm.lifetime.end.
+//
+//===----------------------------------------------------------------------===//
+
+#include "gen/variable_lifetime.h"
+
+#include "driver/cl_options.h"
+#include "gen/irstate.h"
+
+#include <vector>
+#include <utility>
+
+// TODO: make this option depend on -O and -fsanitize settings.
+static llvm::cl::opt<bool> fEmitLocalVarLifetime(
+    "femit-local-var-lifetime",
+    llvm::cl::desc(
+        "Emit local variable lifetime, enabling more optimizations."),
+    llvm::cl::Hidden, llvm::cl::ZeroOrMore);
+
+LocalVariableLifetimeAnnotator::LocalVariableLifetimeAnnotator(IRState &irs)
+    : irs(irs) {
+  allocaType =
+      llvm::Type::getInt8Ty(irs.context())
+          ->getPointerTo(irs.module.getDataLayout().getAllocaAddrSpace());
+}
+
+void LocalVariableLifetimeAnnotator::pushScope() { scopes.emplace_back(); }
+
+void LocalVariableLifetimeAnnotator::addLocalVariable(llvm::Value *address,
+                                                      llvm::Value *size) {
+  assert(address);
+  assert(size);
+
+  if (!fEmitLocalVarLifetime)
+    return;
+
+  if (scopes.empty())
+    return;
+
+  // Push to scopes
+  scopes.back().variables.emplace_back(size, address);
+
+  // Emit lifetime start
+  address = irs.ir->CreateBitCast(address, allocaType);
+  irs.CreateCallOrInvoke(getLLVMLifetimeStartFn(), {size, address}, "",
+                         true /*nothrow*/);
+}
+
+// Emits end-of-lifetime annotation for all variables in current scope.
+void LocalVariableLifetimeAnnotator::popScope() {
+  if (scopes.empty())
+    return;
+
+  for (const auto &var : scopes.back().variables) {
+    auto size = var.first;
+    auto address = var.second;
+
+    address = irs.ir->CreateBitCast(address, allocaType);
+    assert(address);
+
+    irs.CreateCallOrInvoke(getLLVMLifetimeEndFn(), {size, address}, "",
+                           true /*nothrow*/);
+  }
+  scopes.pop_back();
+}
+
+/// Lazily declare the @llvm.lifetime.start intrinsic.
+llvm::Function *LocalVariableLifetimeAnnotator::getLLVMLifetimeStartFn() {
+  if (lifetimeStartFunction)
+    return lifetimeStartFunction;
+
+  lifetimeStartFunction = llvm::Intrinsic::getDeclaration(
+      &irs.module, llvm::Intrinsic::lifetime_start, allocaType);
+  assert(lifetimeStartFunction);
+  return lifetimeStartFunction;
+}
+
+/// Lazily declare the @llvm.lifetime.end intrinsic.
+llvm::Function *LocalVariableLifetimeAnnotator::getLLVMLifetimeEndFn() {
+  if (lifetimeEndFunction)
+    return lifetimeEndFunction;
+
+  lifetimeEndFunction = llvm::Intrinsic::getDeclaration(
+      &irs.module, llvm::Intrinsic::lifetime_end, allocaType);
+  assert(lifetimeEndFunction);
+  return lifetimeEndFunction;
+}

gen/variable_lifetime.h

@@ -0,0 +1,56 @@
+//===-- gen/variable_lifetime.h - -------------------------------*- C++ -*-===//
+//
+//                         LDC – the LLVM D compiler
+//
+// This file is distributed under the BSD-style LDC license. See the LICENSE
+// file for details.
+//
+//===----------------------------------------------------------------------===//
+//
+// Codegen for local variable lifetime: llvm.lifetime.start abd
+// llvm.lifetime.end.
+//
+//===----------------------------------------------------------------------===//
+
+#pragma once
+
+#include <vector>
+#include <utility>
+
+namespace llvm {
+class Function;
+class Type;
+class Value;
+}
+struct IRState;
+
+struct LocalVariableLifetimeAnnotator {
+  struct LocalVariableScope {
+    std::vector<std::pair<llvm::Value *, llvm::Value *>> variables;
+  };
+  /// Stack of scopes, each scope can have multiple variables.
+  std::vector<LocalVariableScope> scopes;
+
+  /// Cache the llvm types and intrinsics used for codegen.
+  llvm::Function *lifetimeStartFunction = nullptr;
+  llvm::Function *lifetimeEndFunction = nullptr;
+  llvm::Type *allocaType = nullptr;
+
+  llvm::Function *getLLVMLifetimeStartFn();
+  llvm::Function *getLLVMLifetimeEndFn();
+
+  IRState &irs;
+
+public:
+  LocalVariableLifetimeAnnotator(IRState &irs);
+
+  /// Opens a new scope.
+  void pushScope();
+
+  /// Closes current scope and emits end-of-lifetime annotation for all
+  /// variables in current scope.
+  void popScope();
+
+  /// Register a new local variable for lifetime annotation.
+  void addLocalVariable(llvm::Value *address, llvm::Value *size);
+};

runtime/CMakeLists.txt

@@ -38,7 +38,7 @@ set(BUILD_SHARED_LIBS     AUTO                                CACHE STRING "Whet
 set(D_FLAGS               -w;-de;-preview=dip1000;-preview=dtorfields;-preview=fieldwise CACHE STRING "Runtime D compiler flags, separated by ';'")
 set(D_EXTRA_FLAGS         ""                                  CACHE STRING "Runtime extra D compiler flags, separated by ';'")
 set(D_FLAGS_DEBUG         -g;-link-defaultlib-debug;-d-debug  CACHE STRING "Runtime D compiler flags (debug libraries), separated by ';'")
-set(D_FLAGS_RELEASE       -O3;-release                        CACHE STRING "Runtime D compiler flags (release libraries), separated by ';'")
+set(D_FLAGS_RELEASE       -O3;-release;-femit-local-var-lifetime  CACHE STRING "Runtime D compiler flags (release libraries), separated by ';'")
 set(COMPILE_ALL_D_FILES_AT_ONCE ON                            CACHE BOOL   "Compile all D files for the runtime libs in a single command line instead of separately. Disabling this is useful for many CPU cores and/or iterative development.")
 set(RT_ARCHIVE_WITH_LDC   ON                                  CACHE STRING "Whether to archive the static runtime libs via LDC instead of CMake archiver")
 set(RT_CFLAGS             ""                                  CACHE STRING "Runtime extra C compiler flags, separated by ' '")

runtime/phobos

@@ -1 +1 @@
-Subproject commit 6c83b490f7d6c66bf430e5249dae608848d3ac2c
+Subproject commit f4961356a33849f24557a77bdc386eff852bb9f5

tests/codegen/lifetime_local_variables.d

@@ -0,0 +1,122 @@
+// RUN: %ldc -femit-local-var-lifetime -c -output-ll -of=%t.ll %s && FileCheck %s < %t.ll
+
+extern(C): // disable mangling for easier matching
+
+void opaque(byte* i);
+
+// CHECK-LABEL: define void @foo_array_foo()
+void foo_array_foo() {
+    // CHECK: alloca [400 x i8]
+    // CHECK: alloca [800 x i8]
+    {
+        // CHECK: call void @llvm.lifetime.start.p0i8(i64 immarg 400
+        byte[400] arr = void;
+        // CHECK: call void @opaque
+        opaque(&arr[0]);
+        // CHECK: call void @llvm.lifetime.end.p0i8(i64 immarg 400
+    }
+
+    {
+        // CHECK: call void @llvm.lifetime.start.p0i8(i64 immarg 800
+        byte[800] arr = void;
+        // CHECK: call void @opaque
+        opaque(&arr[0]);
+        // CHECK: call void @llvm.lifetime.end.p0i8(i64 immarg 800
+    }
+
+    // CHECK-LABEL: ret void
+}
+
+// CHECK-LABEL: define void @foo_forloop_foo()
+void foo_forloop_foo() {
+    byte i;
+    // CHECK: call void @opaque
+    // This call should appear before lifetime start of while-loop variable.
+    opaque(&i);
+    for (byte[13] d; d[0] < 2; d[0]++) {
+        // CHECK: call void @llvm.lifetime.start.p0i8(i64 immarg 13
+        // Lifetime should start before initializing the variable
+        // CHECK: call void @llvm.memset.p0i8.i{{.*}}13
+        // CHECK: call void @llvm.lifetime.start.p0i8(i64 immarg 44
+        byte[44] arr = void;
+        // CHECK: call void @opaque
+        opaque(&arr[0]);
+        // CHECK: call void @llvm.lifetime.end.p0i8(i64 immarg 44
+        // CHECK: endfor:
+        // CHECK: call void @llvm.lifetime.end.p0i8(i64 immarg 13
+    }
+
+    // CHECK-LABEL: ret void
+}
+
+// CHECK-LABEL: define void @foo_whileloop_foo()
+void foo_whileloop_foo() {
+    byte i;
+    // CHECK: call void @opaque
+    // This call should appear before lifetime start of while-loop variable.
+    opaque(&i);
+    while (ulong d = 131) {
+        // CHECK: call void @llvm.lifetime.start.p0i8(i64 immarg 8
+        // Lifetime should start before initializing the variable
+        // CHECK: store i64 131
+        // CHECK: call void @llvm.lifetime.start.p0i8(i64 immarg 33
+        byte[33] arr = void;
+        // CHECK: call void @opaque
+        opaque(&arr[0]);
+        // CHECK: call void @llvm.lifetime.end.p0i8(i64 immarg 33
+        // CHECK: call void @llvm.lifetime.end.p0i8(i64 immarg 8
+    }
+
+    // CHECK-LABEL: ret void
+}
+
+// CHECK-LABEL: define void @foo_if_foo()
+void foo_if_foo() {
+    byte i;
+    // CHECK: call void @opaque
+    // This call should appear before lifetime start of if-statement condition variable.
+    opaque(&i);
+    // CHECK: call void @llvm.lifetime.start.p0i8(i64 immarg 8
+    // Lifetime should start before initializing the variable
+    // CHECK: store i64 565
+    if (ulong d = 565) {
+        // CHECK: call void @llvm.lifetime.start.p0i8(i64 immarg 72
+        byte[72] arr = void;
+        // CHECK: call void @opaque
+        opaque(&arr[0]);
+        // CHECK: call void @llvm.lifetime.end.p0i8(i64 immarg 72
+    } else {
+        // d is out of scope here.
+        // CHECK: call void @llvm.lifetime.start.p0i8(i64 immarg 51
+        byte[51] arr = void;
+        // CHECK: call void @opaque
+        opaque(&arr[0]);
+        // CHECK: call void @llvm.lifetime.end.p0i8(i64 immarg 51
+    }
+    // CHECK: endif:
+    // CHECK: call void @llvm.lifetime.end.p0i8(i64 immarg 8
+
+    // CHECK-LABEL: ret void
+}
+
+struct S {
+    byte[123] a;
+    ~this() {
+        opaque(&a[1]);
+    }
+}
+
+void opaque_S(S* i);
+
+// CHECK-LABEL: define void @foo_struct_foo()
+void foo_struct_foo() {
+    {
+        // CHECK: call void @llvm.lifetime.start.p0i8(i64 immarg 123
+        S s;
+        // CHECK: invoke void @opaque_S
+        opaque_S(&s);
+    }
+
+    // CHECK: call void @llvm.lifetime.end.p0i8(i64 immarg 123
+    // CHECK-NEXT: ret void
+}

tests/sanitizers/asan_use_after_scope.d

@@ -0,0 +1,24 @@
+// REQUIRES: ASan
+
+// RUN: %ldc -femit-local-var-lifetime -g -fsanitize=address %s -of=%t%exe
+// RUN: not %t%exe 2>&1 | FileCheck %s
+
+// CHECK: ERROR: AddressSanitizer: stack-use-after-scope
+// CHECK-NEXT: WRITE of size 4
+
+void useAfterScope() {
+  int* p;
+  {
+    int x = 0;
+    p = &x;  // cannot statically disallow this because
+    *p = 1;  // this is a valid use of things
+  }
+// CHECK-NEXT: #0 {{.*}} in {{.*}}asan_use_after_scope.d:[[@LINE+1]]
+  *p = 5;  // but then this can happen... stack use after scope bug!
+}
+
+void main()
+{
+// CHECK-NEXT: #1 {{.*}} in {{.*}}asan_use_after_scope.d:[[@LINE+1]]
+    useAfterScope();
+}

tests/sanitizers/asan_use_after_scope_if.d

@@ -0,0 +1,23 @@
+// REQUIRES: ASan
+
+// RUN: %ldc -femit-local-var-lifetime -g -fsanitize=address %s -of=%t%exe
+// RUN: not %t%exe 2>&1 | FileCheck %s
+
+// CHECK: ERROR: AddressSanitizer: stack-use-after-scope
+// CHECK-NEXT: WRITE of size 4
+
+void useAfterScope(int xparam) {
+  int* p;
+  if (int x = xparam) {
+    p = &x;  // cannot statically disallow this because
+    *p = 1;  // this is a valid use of things
+  }
+// CHECK-NEXT: #0 {{.*}} in {{.*}}asan_use_after_scope_if.d:[[@LINE+1]]
+  *p = 5;  // but then this can happen... stack use after scope bug!
+}
+
+void main()
+{
+// CHECK-NEXT: #1 {{.*}} in {{.*}}asan_use_after_scope_if.d:[[@LINE+1]]
+    useAfterScope(1);
+}