Implement cross-module inlining (with and without pragma).

ddmd/func.d

@@ -2484,8 +2484,16 @@ else
             TemplateInstance spec = isSpeculative();
             uint olderrs = global.errors;
             uint oldgag = global.gag;
+version (IN_LLVM)
+{
+            if (global.gag && !spec && !global.gaggedForInlining)
+                global.gag = 0;
+}
+else
+{
             if (global.gag && !spec)
                 global.gag = 0;
+}
             semantic3(_scope);
             global.gag = oldgag;
 

ddmd/globals.d

@@ -246,6 +246,8 @@ struct Global
         const(char)* s_ext;
         const(char)* ldc_version;
         const(char)* llvm_version;
+
+        bool gaggedForInlining; // Set for functionSemantic3 for external inlining canditates
     }
     const(char)* lib_ext;
     const(char)* dll_ext;

ddmd/globals.h

@@ -242,6 +242,8 @@ struct Global
     const char *s_ext;
     const char *ldc_version;
     const char *llvm_version;
+
+    bool gaggedForInlining; // Set for functionSemantic3 for external inlining canditates
 #endif
     const char *lib_ext;
     const char *dll_ext;

gen/declarations.cpp

@@ -338,7 +338,10 @@ public:
         fprintf(global.stdmsg, "%s: %s is thread local\n", p, decl->toChars());
       }
       // Check if we are defining or just declaring the global in this module.
-      if (!(decl->storage_class & STCextern)) {
+      // If we reach here during codegen of an available_externally function,
+      // new variable declarations should stay external and therefore must not
+      // have an initializer.
+      if (!(decl->storage_class & STCextern) && !decl->inNonRoot()) {
         // Build the initializer. Might use this->ir.irGlobal->value!
         LLConstant *initVal =
             DtoConstInitializer(decl->loc, decl->type, decl->_init);

gen/functions.cpp

@@ -42,6 +42,123 @@
 #include "llvm/IR/CFG.h"
 #include <iostream>
 
+////////////////////////////////////////////////////////////////////////////////
+// Function inlining
+// TODO: move to its own file?
+namespace {
+
+// Use a heuristic to determine if it could make sense to inline this fdecl.
+// Note: isInlineCandidate is called _before_ full semantic analysis of fdecl.
+bool isInlineCandidate(FuncDeclaration &fdecl) {
+  if (fdecl.inlining == PINLINEalways)
+    return true;
+
+  return true;
+}
+
+/// Check whether the frontend knows that the function is already defined
+/// in some other module (see DMD's FuncDeclaration::toObjFile).
+bool alreadyOrWillBeDefined(FuncDeclaration &fdecl) {
+  for (FuncDeclaration *f = &fdecl; f;) {
+    if (!f->isInstantiated() && f->inNonRoot()) {
+      return false;
+    }
+    if (f->isNested()) {
+      f = f->toParent2()->isFuncDeclaration();
+    } else {
+      break;
+    }
+  }
+  return true;
+}
+
+/// If true: define this function with externally_available linkage, for
+/// inlining potential.
+/// If true: full semantic analysis done on this fdecl, ready for codegen.
+bool defineAsExternallyAvailable(FuncDeclaration &fdecl) {
+  IF_LOG Logger::println("Enter defineAsExternallyAvailable");
+  LOG_SCOPE
+
+  // Try to do cheap checks first.
+
+  if (fdecl.neverInline || fdecl.inlining == PINLINEnever)
+    return false;
+
+  // pragma(inline, true) functions will be inlined even at -O0
+  if (!willInline() && (fdecl.inlining != PINLINEalways))
+    return false;
+
+  if (fdecl.isUnitTestDeclaration())
+    return false;
+  if (fdecl.isFuncAliasDeclaration())
+    return false;
+  if (!fdecl.fbody)
+    return false;
+
+  // Disable inlining functions from object.d because of TypeInfo related issue
+  if (fdecl.getModule()->ident == Id::object)
+    return false;
+
+  if (alreadyOrWillBeDefined(fdecl))
+    return false;
+
+  // Weak-linkage functions can not be inlined.
+  if (hasWeakUDA(&fdecl))
+    return false;
+
+  if (!isInlineCandidate(fdecl))
+    return false;
+
+  IF_LOG Logger::println("Potential inlining candidate");
+
+  // If semantic analysis is already complete, the function will be codegenned
+  // elsewhere.
+  if (fdecl.semanticRun >= PASSsemantic3)
+    return false;
+
+  {
+    IF_LOG Logger::println("Do semantic analysis");
+    LOG_SCOPE
+
+    // The inlining is aggressive and may give semantic errors that are forward
+    // referencing errors. Simply avoid those cases for inlining.
+    uint errors = global.startGagging();
+    global.gaggedForInlining = true;
+
+    bool semantic_error = false;
+    if (fdecl.functionSemantic3()) {
+      Module::runDeferredSemantic3();
+    } else {
+      IF_LOG Logger::println("Failed functionSemantic3.");
+      semantic_error = true;
+    }
+
+    global.gaggedForInlining = false;
+    if (global.endGagging(errors) || semantic_error) {
+      IF_LOG Logger::println("Errors occured during semantic analysis.");
+      return false;
+    }
+    assert(fdecl.semanticRun >= PASSsemantic3done);
+  }
+
+  // For naked functions (inline assembly), we emit the assembly directly as
+  // globals in the text section. Emitting them during this inline pass will
+  // therefore result in multiple definitions. Solution: don't try to inline
+  // them.
+  // These naked functions don't appear to be inlined anyway, so it is pointless
+  // at this moment to try.
+  // FuncDeclaration::naked is set by the AsmParser during semantic analysis.
+  if (fdecl.naked) {
+    IF_LOG Logger::println("Naked asm functions cannot be inlined.");
+    return false;
+  }
+
+  IF_LOG Logger::println("defineAsExternallyAvailable? Yes.");
+  return true;
+}
+}
+////////////////////////////////////////////////////////////////////////////////
+
 llvm::FunctionType *DtoFunctionType(Type *type, IrFuncTy &irFty, Type *thistype,
                                     Type *nesttype, bool isMain, bool isCtor,
                                     bool isIntrinsic, bool hasSel) {
@@ -457,6 +574,15 @@ void DtoDeclareFunction(FuncDeclaration *fdecl) {
     fatal();
   }
 
+  // Check if fdecl should be defined too for cross-module inlining.
+  // If true, semantic is fully done for fdecl which is needed for some code
+  // below (e.g. code that uses fdecl->vthis).
+  bool defineAtEnd = defineAsExternallyAvailable(*fdecl);
+  if (defineAtEnd) {
+    IF_LOG Logger::println(
+        "Function is an externally_available inline candidate.");
+  }
+
   // get TypeFunction*
   Type *t = fdecl->type->toBasetype();
   TypeFunction *f = static_cast<TypeFunction *>(t);
@@ -620,6 +746,13 @@ void DtoDeclareFunction(FuncDeclaration *fdecl) {
     irParam->arg = arg;
     irParam->value = &(*iarg);
   }
+
+  // Now that this function is declared, also define it if needed.
+  if (defineAtEnd) {
+    IF_LOG Logger::println(
+        "Function is an externally_available inline candidate: define it now.");
+    DtoDefineFunction(fdecl, true);
+  }
 }
 
 ////////////////////////////////////////////////////////////////////////////////
@@ -719,12 +852,22 @@ void defineParameters(IrFuncTy &irFty, VarDeclarations &parameters) {
 
 } // anonymous namespace
 
-void DtoDefineFunction(FuncDeclaration *fd) {
+void DtoDefineFunction(FuncDeclaration *fd, bool availableExternally) {
   IF_LOG Logger::println("DtoDefineFunction(%s): %s", fd->toPrettyChars(),
                          fd->loc.toChars());
   LOG_SCOPE;
+  if (availableExternally) {
+    IF_LOG Logger::println("availableExternally = true");
+  }
 
   if (fd->ir->isDefined()) {
+    if (!availableExternally &&
+        (getIrFunc(fd)->func->getLinkage() ==
+         llvm::GlobalValue::AvailableExternallyLinkage)) {
+      // Fix linkage
+      const auto lwc = lowerFuncLinkage(fd);
+      setLinkage(lwc, getIrFunc(fd)->func);
+    }
     return;
   }
 
@@ -748,6 +891,7 @@ void DtoDefineFunction(FuncDeclaration *fd) {
           fd->toPrettyChars());
     fatal();
   }
+  assert(fd->semanticRun >= PASSsemantic3done);
 
   DtoResolveFunction(fd);
 
@@ -767,6 +911,7 @@ void DtoDefineFunction(FuncDeclaration *fd) {
     return;
   }
 
+  if (!availableExternally) {
     // Check whether the frontend knows that the function is already defined
     // in some other module (see DMD's FuncDeclaration::toObjFile).
     for (FuncDeclaration *f = fd; f;) {
@@ -783,6 +928,7 @@ void DtoDefineFunction(FuncDeclaration *fd) {
         break;
       }
     }
+  }
 
   DtoDeclareFunction(fd);
   assert(fd->ir->isDeclared());
@@ -867,7 +1013,15 @@ void DtoDefineFunction(FuncDeclaration *fd) {
   gIR->functions.push_back(irFunc);
 
   const auto lwc = lowerFuncLinkage(fd);
+  if (availableExternally) {
+    func->setLinkage(llvm::GlobalValue::AvailableExternallyLinkage);
+    // Assert that we are not overriding a linkage type that disallows inlining
+    assert(lwc.first != llvm::GlobalValue::WeakAnyLinkage &&
+           lwc.first != llvm::GlobalValue::ExternalWeakLinkage &&
+           lwc.first != llvm::GlobalValue::LinkOnceAnyLinkage);
+  } else {
     setLinkage(lwc, func);
+  }
 
   // On x86_64, always set 'uwtable' for System V ABI compatibility.
   // TODO: Find a better place for this.

gen/functions.h

@@ -36,7 +36,7 @@ llvm::FunctionType *DtoFunctionType(FuncDeclaration *fdecl);
 
 void DtoResolveFunction(FuncDeclaration *fdecl);
 void DtoDeclareFunction(FuncDeclaration *fdecl);
-void DtoDefineFunction(FuncDeclaration *fd);
+void DtoDefineFunction(FuncDeclaration *fd, bool availableExternally = false);
 
 void DtoDefineNakedFunction(FuncDeclaration *fd);
 void emitABIReturnAsmStmt(IRAsmBlock *asmblock, Loc &loc,

tests/codegen/inlining_imports.d

@@ -0,0 +1,42 @@
+// Test inlining of imported functions
+
+// RUN: %ldc %s -I%S -c -output-ll -release -O3 -of=%t.O3.ll && FileCheck %s --check-prefix OPT3 < %t.O3.ll
+
+import inputs.inlinables;
+
+extern (C): // simplify mangling for easier matching
+
+// Simple functions for reference.
+int foo()
+{
+    return goo();
+}
+
+int goo()
+{
+    return 1;
+}
+
+// OPT3-LABEL: define{{.*}} @call_easily_inlinable(
+int call_easily_inlinable(int i)
+{
+    // OPT3-NOT: call {{.*}} @easily_inlinable(
+    return easily_inlinable(i);
+    // OPT3: ret i32 2
+}
+
+// OPT3-LABEL: define{{.*}} @call_class_function(
+int call_class_function(A a)
+{
+    // OPT3-NOT: call
+    return a.final_class_function();
+    // OPT3: ret i32 12345
+}
+
+// OPT3-LABEL: define{{.*}} @call_weak_function(
+int call_weak_function()
+{
+    // OPT3: call
+    return weak_function();
+    // OPT3-NOT: 654
+}

tests/codegen/inlining_imports_pragma.d

@@ -0,0 +1,42 @@
+// Test inlining of functions marked with pragma(inline) in an imported module
+
+// O0 and O3 should behave the same for these tests with explicit inlining directives by the user.
+
+// RUN: %ldc %s -I%S -c -output-ll -O0 -of=%t.O0.ll && FileCheck %s --check-prefix OPTNONE < %t.O0.ll
+// RUN: %ldc %s -I%S -c -output-ll -O3 -of=%t.O3.ll && FileCheck %s --check-prefix OPT3 < %t.O3.ll
+
+import inputs.inlinables;
+
+extern (C): // simplify mangling for easier matching
+
+// OPTNONE-LABEL: define{{.*}} @call_never_inline(
+// OPT3-LABEL: define{{.*}} @call_never_inline(
+int call_never_inline()
+{
+    // OPTNONE: call {{.*}} @never_inline()
+    // OPT3: call {{.*}} @never_inline()
+    return never_inline();
+}
+// OPTNONE-DAG: declare {{.*}} @never_inline()
+
+// OPTNONE-LABEL: define{{.*}} @call_always_inline(
+// OPT3-LABEL: define{{.*}} @call_always_inline(
+int call_always_inline()
+{
+    // OPTNONE-NOT: call {{.*}} @always_inline()
+    // OPT3-NOT: call {{.*}} @always_inline()
+    return always_inline();
+    // OPTNONE: ret
+    // OPT3: ret
+}
+
+// OPTNONE-LABEL: define{{.*}} @call_inline_chain(
+// OPT3-LABEL: define{{.*}} @call_inline_chain(
+int call_inline_chain()
+{
+    // OPTNONE-NOT: call
+    // OPT3-NOT: call
+    return always_inline_chain0();
+    // OPTNONE: ret
+    // OPT3: ret
+}

tests/codegen/inlining_leakdefinitions.d

@@ -0,0 +1,65 @@
+// Test that inlining does not leak definitions without marking them as available_externally
+// "Leaking" = symbols definitions in .o file that shouldn't be declarations instead (undefined symbols).
+
+// RUN: %ldc %s -I%S -c -output-ll -release                  -O3 -of=%t.O3.ll && FileCheck %s --check-prefix OPT3 < %t.O3.ll
+// RUN: %ldc %s -I%S -c -output-ll -release -enable-inlining -O0 -of=%t.O0.ll && FileCheck %s --check-prefix OPT0 < %t.O0.ll
+// RUN: %ldc -I%S -enable-inlining %S/inputs/inlinables.d -run %s
+// RUN: %ldc -I%S -O3 %S/inputs/inlinables.d -run %s
+
+import inputs.inlinables;
+
+extern (C): // simplify mangling for easier matching
+
+// Inlined naked asm func could end up as global symbols, definitely bad!
+// (would give multiple definition linker error)
+// OPT0-NOT: module asm {{.*}}.globl{{.*}}_naked_asm_func
+// OPT3-NOT: module asm {{.*}}.globl{{.*}}_naked_asm_func
+
+// Check that the global variables that are added due to "available_externally
+// inlining" do not have initializers, i.e. they are declared only and not definined.
+// OPT3-DAG: @module_variable = external thread_local global i32, align
+// OPT3-DAG: @{{.*}}write_function_static_variableUiZ15static_func_vari = external thread_local global i32, align
+
+// OPT0-LABEL: define{{.*}} @call_class_function(
+// OPT3-LABEL: define{{.*}} @call_class_function(
+int call_class_function(A a)
+{
+    // There should be only one call to "virtual_func".
+    // OPT3: call
+    // OPT3-NOT: call
+    return a.final_func();
+    // There should be a return from an LLVM variable (not a direct value)
+    // OPT0: ret i32 %
+    // OPT3: ret i32 %
+}
+
+// OPT0-LABEL: define{{.*}} @dont_leak_module_variables(
+// OPT3-LABEL: define{{.*}} @dont_leak_module_variables(
+void dont_leak_module_variables()
+{
+    write_module_variable(987);
+    write_function_static_variable(167);
+    get_typeid_A();
+    // OPT0: ret void
+    // OPT3: ret void
+}
+
+// OPT0-LABEL: define{{.*}} @asm_func(
+// OPT3-LABEL: define{{.*}} @asm_func(
+void asm_func()
+{
+    naked_asm_func();
+    // OPT0: ret void
+    // OPT3: ret void
+}
+
+// OPT0-LABEL: define{{.*}} @main(
+// OPT3-LABEL: define{{.*}} @main(
+int main()
+{
+    dont_leak_module_variables();
+
+    return 0;
+    // OPT0: ret i32 0
+    // OPT3: ret i32 0
+}

tests/codegen/inlining_stdlib.d

@@ -0,0 +1,34 @@
+// Test inlining of some standard library functions
+
+// Test also that the tested functions are indeed not inlined at -O0 (basically verifying that we are testing something real)
+// RUN: %ldc %s -c -output-ll -release -O0 -of=%t.O0.ll && FileCheck %s --check-prefix OPT0 < %t.O0.ll
+// RUN: %ldc %s -c -output-ll -release -O3 -of=%t.O3.ll && FileCheck %s --check-prefix OPT3 < %t.O3.ll
+
+extern (C): // simplify mangling for easier matching
+
+// OPT0-LABEL: define{{.*}} @foo(
+// OPT3-LABEL: define{{.*}} @foo(
+int foo(size_t i)
+{
+    import core.bitop;
+    // OPT0: call i32 @{{.*}}core5bitop3bsf
+    // OPT3: call {{.*}} @llvm.cttz
+    return bsf(i);
+    // OPT0: ret
+    // OPT3: ret
+}
+// OPT0: declare {{.*}}core5bitop3bsf
+
+// OPT0-LABEL: define{{.*}} @ggg(
+// OPT3-LABEL: define{{.*}} @ggg(
+char[] ggg(char* str)
+{
+    import std.string;
+    // OPT0: call {{.*}} @{{.*}}std6string11fromStringz
+    // OPT3: call {{.*}}strlen
+    return fromStringz(str);
+    // OPT0: ret
+    // OPT3: ret
+}
+// OPT0: declare {{.*}}std6string11fromStringz
+// OPT3: declare {{.*}}strlen

tests/codegen/inputs/inlinables.d

@@ -0,0 +1,103 @@
+module inputs.inlinables;
+
+import ldc.attributes;
+
+extern (C): // simplify mangling for easier function name matching
+
+int easily_inlinable(int i)
+{
+    if (i > 0)
+        return easily_inlinable(i - 1);
+    return 2;
+}
+
+pragma(inline, false) int never_inline()
+{
+    return 1;
+}
+
+@weak int external()
+{
+    return 1;
+}
+
+pragma(inline, true) int always_inline()
+{
+    int a;
+    foreach (i; 1 .. 10)
+    {
+        foreach (ii; 1 .. 10)
+        {
+            foreach (iii; 1 .. 10)
+            {
+                a += i * external();
+            }
+        }
+    }
+    return a;
+}
+
+pragma(inline, true) int always_inline_chain0()
+{
+    return always_inline_chain1();
+}
+
+pragma(inline, true) int always_inline_chain1()
+{
+    return always_inline_chain2();
+}
+
+pragma(inline, true) int always_inline_chain2()
+{
+    return 345;
+}
+
+class A
+{
+    int virtual_func()
+    {
+        return 12345;
+    }
+
+    pragma(inline, true) final int final_func()
+    {
+        return virtual_func();
+    }
+
+    final int final_class_function()
+    {
+        return 12345;
+    }
+}
+
+// Weak-linkage functions can not be inlined.
+@weak int weak_function()
+{
+    return 654;
+}
+
+int module_variable = 666;
+pragma(inline, true) void write_module_variable(int i)
+{
+    module_variable = i;
+}
+
+pragma(inline, true) void write_function_static_variable(int i)
+{
+    static int static_func_var = 5;
+    static_func_var = i;
+}
+
+pragma(inline, true) auto get_typeid_A()
+{
+    return typeid(A);
+}
+
+pragma(inline, true) extern (C) void naked_asm_func()
+{
+    asm pure nothrow @nogc
+    {
+        naked;
+        nop;
+    }
+}