Merge pull request #1577 from JohanEngelen/inlining

Implement cross-module inlining

ddmd/func.d

@@ -2484,8 +2484,16 @@ else
             TemplateInstance spec = isSpeculative();
             uint olderrs = global.errors;
             uint oldgag = global.gag;
+version (IN_LLVM)
+{
+            if (global.gag && !spec && !global.gaggedForInlining)
+                global.gag = 0;
+}
+else
+{
             if (global.gag && !spec)
                 global.gag = 0;
+}
             semantic3(_scope);
             global.gag = oldgag;
 

ddmd/globals.d

@@ -246,6 +246,8 @@ struct Global
         const(char)* s_ext;
         const(char)* ldc_version;
         const(char)* llvm_version;
+
+        bool gaggedForInlining; // Set for functionSemantic3 for external inlining candidates
     }
     const(char)* lib_ext;
     const(char)* dll_ext;

ddmd/globals.h

@@ -242,6 +242,8 @@ struct Global
     const char *s_ext;
     const char *ldc_version;
     const char *llvm_version;
+
+    bool gaggedForInlining; // Set for functionSemantic3 for external inlining candidates
 #endif
     const char *lib_ext;
     const char *dll_ext;

driver/main.cpp

@@ -1393,7 +1393,15 @@ int cppmain(int argc, char **argv) {
   if (global.params.obj && !modules.empty()) {
     ldc::CodeGenerator cg(getGlobalContext(), singleObj);
 
-    for (unsigned i = 0; i < modules.dim; i++) {
+    // When inlining is enabled, we are calling semantic3 on function
+    // declarations, which may _add_ members to the first module in the modules
+    // array. These added functions must be codegenned, because these functions
+    // may be "alwaysinline" and linker problems arise otherwise with templates
+    // that have __FILE__ as parameters (which must be `pragma(inline, true);`)
+    // Therefore, codegen is done in reverse order with members[0] last, to make
+    // sure these functions (added to members[0] by members[x>0]) are
+    // codegenned.
+    for (d_size_t i = modules.dim; i-- > 0;) {
       Module *const m = modules[i];
       if (global.params.verbose) {
         fprintf(global.stdmsg, "code      %s\n", m->toChars());

gen/declarations.cpp

@@ -338,7 +338,10 @@ public:
         fprintf(global.stdmsg, "%s: %s is thread local\n", p, decl->toChars());
       }
       // Check if we are defining or just declaring the global in this module.
-      if (!(decl->storage_class & STCextern)) {
+      // If we reach here during codegen of an available_externally function,
+      // new variable declarations should stay external and therefore must not
+      // have an initializer.
+      if (!(decl->storage_class & STCextern) && !decl->inNonRoot()) {
         // Build the initializer. Might use this->ir.irGlobal->value!
         LLConstant *initVal =
             DtoConstInitializer(decl->loc, decl->type, decl->_init);
@@ -430,10 +433,19 @@ public:
       return;
     }
 
-    // FIXME: This is #673 all over again.
+    // Force codegen if this is a templated function with pragma(inline, true).
-    if (!decl->needsCodegen()) {
+    if ((decl->members->dim == 1) &&
-      Logger::println("Does not need codegen, skipping.");
+        ((*decl->members)[0]->isFuncDeclaration()) &&
-      return;
+        ((*decl->members)[0]->isFuncDeclaration()->inlining == PINLINEalways)) {
+      Logger::println("needsCodegen() == false, but function is marked with "
+                      "pragma(inline, true), so it really does need "
+                      "codegen.");
+    } else {
+      // FIXME: This is #673 all over again.
+      if (!decl->needsCodegen()) {
+        Logger::println("Does not need codegen, skipping.");
+        return;
+      }
     }
 
     for (auto &m : *decl->members) {

gen/function-inlining.cpp

@@ -0,0 +1,196 @@
+//===-- function-inlining.cpp ---------------------------------------------===//
+//
+//                         LDC – the LLVM D compiler
+//
+// This file is distributed under the BSD-style LDC license. See the LICENSE
+// file for details.
+//
+//===----------------------------------------------------------------------===//
+
+#include "gen/function-inlining.h"
+
+#include "declaration.h"
+#include "globals.h"
+#include "id.h"
+#include "module.h"
+#include "statement.h"
+#include "template.h"
+#include "gen/logger.h"
+#include "gen/optimizer.h"
+#include "gen/recursivevisitor.h"
+#include "gen/uda.h"
+
+namespace {
+
+/// An ASTVisitor that checks whether the number of statements is larger than a
+/// certain number.
+struct MoreThanXStatements : public StoppableVisitor {
+  /// Are there more or fewer statements than `threshold`?.
+  unsigned threshold;
+  /// The statement count.
+  unsigned count;
+
+  MoreThanXStatements(unsigned X) : threshold(X), count(0) {}
+
+  using StoppableVisitor::visit;
+
+  void visit(Statement *stmt) override {
+    count++;
+    if (count > threshold)
+      stop = true;
+  }
+  void visit(Expression *exp) override {}
+  void visit(Declaration *decl) override {}
+  void visit(Initializer *init) override {}
+  void visit(Dsymbol *) override {}
+};
+
+// Use a heuristic to determine if it could make sense to inline this fdecl.
+// Note: isInlineCandidate is called _before_ semantic3 analysis of fdecl.
+bool isInlineCandidate(FuncDeclaration &fdecl) {
+  // Giving maximum inlining potential to LLVM should be possible, but we
+  // restrict it to save some compile time.
+  // return true;
+
+  // TODO: make the heuristic more sophisticated?
+  // In the end, LLVM will make the decision whether to _actually_ inline.
+  // The statement count threshold is completely arbitrary. Also, all
+  // statements are weighed the same.
+
+  unsigned statementThreshold = 10;
+  MoreThanXStatements statementCounter(statementThreshold);
+  RecursiveWalker walker(&statementCounter, false);
+  fdecl.fbody->accept(&walker);
+
+  IF_LOG Logger::println("Contains %u statements or more (threshold = %u).",
+                         statementCounter.count, statementThreshold);
+  return statementCounter.count <= statementThreshold;
+}
+
+} // end anonymous namespace
+
+bool alreadyOrWillBeDefined(FuncDeclaration &fdecl) {
+  for (FuncDeclaration *f = &fdecl; f;) {
+    if (!f->isInstantiated() && f->inNonRoot()) {
+      return false;
+    }
+    if (f->isNested()) {
+      f = f->toParent2()->isFuncDeclaration();
+    } else {
+      break;
+    }
+  }
+  return true;
+}
+
+bool defineAsExternallyAvailable(FuncDeclaration &fdecl) {
+  IF_LOG Logger::println("Enter defineAsExternallyAvailable");
+  LOG_SCOPE
+
+#if LDC_LLVM_VER < 307
+  // Pre-3.7, cross-module inlining is disabled completely.
+  // See the commandline flag definition for more details.
+  IF_LOG Logger::println("LLVM < 3.7: Cross-module inlining disabled.");
+  return false;
+#endif
+
+  // Implementation note: try to do cheap checks first.
+
+  if (fdecl.neverInline || fdecl.inlining == PINLINEnever) {
+    IF_LOG Logger::println("pragma(inline, false) specified");
+    return false;
+  }
+
+  // pragma(inline, true) functions will be inlined even at -O0
+  if (fdecl.inlining == PINLINEalways) {
+    IF_LOG Logger::println(
+        "pragma(inline, true) specified, overrides cmdline flags");
+  } else if (!willCrossModuleInline()) {
+    IF_LOG Logger::println("Commandline flags indicate no inlining");
+    return false;
+  }
+
+  if (fdecl.isUnitTestDeclaration()) {
+    IF_LOG Logger::println("isUnitTestDeclaration() == true");
+    return false;
+  }
+  if (fdecl.isFuncAliasDeclaration()) {
+    IF_LOG Logger::println("isFuncAliasDeclaration() == true");
+    return false;
+  }
+  if (!fdecl.fbody) {
+    IF_LOG Logger::println("No function body available for inlining");
+    return false;
+  }
+
+  // TODO: Fix inlining functions from object.d. Currently errors because of
+  // TypeInfo type-mismatch issue (TypeInfo classes get special treatment by the
+  // compiler). To start working on it: comment-out this check and druntime will
+  // fail to compile.
+  if (fdecl.getModule()->ident == Id::object) {
+    IF_LOG Logger::println("Inlining of object.d functions is disabled");
+    return false;
+  }
+
+  if (fdecl.semanticRun >= PASSsemantic3) {
+    // If semantic analysis has come this far, the function will be defined
+    // elsewhere and should not get the available_externally attribute from
+    // here.
+    // TODO: This check prevents inlining of nested functions.
+    IF_LOG Logger::println("Semantic analysis already completed");
+    return false;
+  }
+
+  if (alreadyOrWillBeDefined(fdecl)) {
+    // This check is needed because of ICEs happening because of unclear issues
+    // upon changing the codegen order without this check.
+    IF_LOG Logger::println("Function will be defined later.");
+    return false;
+  }
+
+  // Weak-linkage functions can not be inlined.
+  if (hasWeakUDA(&fdecl)) {
+    IF_LOG Logger::println("@weak functions cannot be inlined.");
+    return false;
+  }
+
+  if (fdecl.inlining != PINLINEalways && !isInlineCandidate(fdecl))
+    return false;
+
+  IF_LOG Logger::println("Potential inlining candidate");
+
+  {
+    IF_LOG Logger::println("Do semantic analysis");
+    LOG_SCOPE
+
+    // The inlining is aggressive and may give semantic errors that are
+    // forward referencing errors. Simply avoid those cases for inlining.
+    unsigned errors = global.startGagging();
+    global.gaggedForInlining = true;
+
+    bool semantic_error = false;
+    if (fdecl.functionSemantic3()) {
+      Module::runDeferredSemantic3();
+    } else {
+      IF_LOG Logger::println("Failed functionSemantic3.");
+      semantic_error = true;
+    }
+
+    global.gaggedForInlining = false;
+    if (global.endGagging(errors) || semantic_error) {
+      IF_LOG Logger::println("Errors occured during semantic analysis.");
+      return false;
+    }
+    assert(fdecl.semanticRun >= PASSsemantic3done);
+  }
+
+  // FuncDeclaration::naked is set by the AsmParser during semantic3 analysis,
+  // and so this check can only be done at this late point.
+  if (fdecl.naked) {
+    IF_LOG Logger::println("Naked asm functions cannot be inlined.");
+    return false;
+  }
+
+  IF_LOG Logger::println("defineAsExternallyAvailable? Yes.");
+  return true;
+}

gen/function-inlining.h

@@ -0,0 +1,29 @@
+//===-- gen/function-inlining.h ---------------------------------*- C++ -*-===//
+//
+//                         LDC – the LLVM D compiler
+//
+// This file is distributed under the BSD-style LDC license. See the LICENSE
+// file for details.
+//
+//===----------------------------------------------------------------------===//
+//
+// Determines whether a function is fit for inlining or not.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LDC_GEN_FUNCTION_INLINING_H
+#define LDC_GEN_FUNCTION_INLINING_H
+
+class FuncDeclaration;
+
+/// Check whether the frontend knows that the function is already defined
+/// in some other module (see DMD's FuncDeclaration::toObjFile).
+bool alreadyOrWillBeDefined(FuncDeclaration &fdecl);
+
+/// Returns whether `fdecl` should be emitted with externally_available
+/// linkage to make it available for inlining.
+///
+/// If true, `semantic3` will have been run on the declaration.
+bool defineAsExternallyAvailable(FuncDeclaration &fdecl);
+
+#endif

gen/functions.cpp

@@ -22,6 +22,7 @@
 #include "gen/arrays.h"
 #include "gen/classes.h"
 #include "gen/dvalue.h"
+#include "gen/function-inlining.h"
 #include "gen/inlineir.h"
 #include "gen/irstate.h"
 #include "gen/linkage.h"
@@ -457,6 +458,15 @@ void DtoDeclareFunction(FuncDeclaration *fdecl) {
     fatal();
   }
 
+  // Check if fdecl should be defined too for cross-module inlining.
+  // If true, semantic is fully done for fdecl which is needed for some code
+  // below (e.g. code that uses fdecl->vthis).
+  const bool defineAtEnd = defineAsExternallyAvailable(*fdecl);
+  if (defineAtEnd) {
+    IF_LOG Logger::println(
+        "Function is an externally_available inline candidate.");
+  }
+
   // get TypeFunction*
   Type *t = fdecl->type->toBasetype();
   TypeFunction *f = static_cast<TypeFunction *>(t);
@@ -527,8 +537,15 @@ void DtoDeclareFunction(FuncDeclaration *fdecl) {
     gIR->mainFunc = func;
   }
 
+  // Set inlining attribute
   if (fdecl->neverInline) {
     irFunc->setNeverInline();
+  } else {
+    if (fdecl->inlining == PINLINEalways) {
+      irFunc->setAlwaysInline();
+    } else if (fdecl->inlining == PINLINEnever) {
+      irFunc->setNeverInline();
+    }
   }
 
   if (fdecl->llvmInternal == LLVMglobal_crt_ctor ||
@@ -613,6 +630,13 @@ void DtoDeclareFunction(FuncDeclaration *fdecl) {
     irParam->arg = arg;
     irParam->value = &(*iarg);
   }
+
+  // Now that this function is declared, also define it if needed.
+  if (defineAtEnd) {
+    IF_LOG Logger::println(
+        "Function is an externally_available inline candidate: define it now.");
+    DtoDefineFunction(fdecl, true);
+  }
 }
 
 ////////////////////////////////////////////////////////////////////////////////
@@ -712,12 +736,22 @@ void defineParameters(IrFuncTy &irFty, VarDeclarations &parameters) {
 
 } // anonymous namespace
 
-void DtoDefineFunction(FuncDeclaration *fd) {
+void DtoDefineFunction(FuncDeclaration *fd, bool linkageAvailableExternally) {
   IF_LOG Logger::println("DtoDefineFunction(%s): %s", fd->toPrettyChars(),
                          fd->loc.toChars());
   LOG_SCOPE;
+  if (linkageAvailableExternally) {
+    IF_LOG Logger::println("linkageAvailableExternally = true");
+  }
 
   if (fd->ir->isDefined()) {
+    if (!linkageAvailableExternally &&
+        (getIrFunc(fd)->func->getLinkage() ==
+         llvm::GlobalValue::AvailableExternallyLinkage)) {
+      // Fix linkage
+      const auto lwc = lowerFuncLinkage(fd);
+      setLinkage(lwc, getIrFunc(fd)->func);
+    }
     return;
   }
 
@@ -760,21 +794,10 @@ void DtoDefineFunction(FuncDeclaration *fd) {
     return;
   }
 
-  // Check whether the frontend knows that the function is already defined
+  if (!linkageAvailableExternally && !alreadyOrWillBeDefined(*fd)) {
-  // in some other module (see DMD's FuncDeclaration::toObjFile).
+    IF_LOG Logger::println("Skipping '%s'.", fd->toPrettyChars());
-  for (FuncDeclaration *f = fd; f;) {
+    fd->ir->setDefined();
-    if (!f->isInstantiated() && f->inNonRoot()) {
+    return;
-      IF_LOG Logger::println("Skipping '%s'.", fd->toPrettyChars());
-      // TODO: Emit as available_externally for inlining purposes instead
-      // (see #673).
-      fd->ir->setDefined();
-      return;
-    }
-    if (f->isNested()) {
-      f = f->toParent2()->isFuncDeclaration();
-    } else {
-      break;
-    }
   }
 
   DtoDeclareFunction(fd);
@@ -860,7 +883,15 @@ void DtoDefineFunction(FuncDeclaration *fd) {
   gIR->functions.push_back(irFunc);
 
   const auto lwc = lowerFuncLinkage(fd);
-  setLinkage(lwc, func);
+  if (linkageAvailableExternally) {
+    func->setLinkage(llvm::GlobalValue::AvailableExternallyLinkage);
+    // Assert that we are not overriding a linkage type that disallows inlining
+    assert(lwc.first != llvm::GlobalValue::WeakAnyLinkage &&
+           lwc.first != llvm::GlobalValue::ExternalWeakLinkage &&
+           lwc.first != llvm::GlobalValue::LinkOnceAnyLinkage);
+  } else {
+    setLinkage(lwc, func);
+  }
 
   // On x86_64, always set 'uwtable' for System V ABI compatibility.
   // TODO: Find a better place for this.
@@ -889,7 +920,7 @@ void DtoDefineFunction(FuncDeclaration *fd) {
   // assert(gIR->scopes.empty());
   gIR->scopes.push_back(IRScope(beginbb));
 
-  // Set the FastMath options for this function scope.
+// Set the FastMath options for this function scope.
 #if LDC_LLVM_VER >= 308
   gIR->scopes.back().builder.setFastMathFlags(irFunc->FMF);
 #else

gen/functions.h

@@ -36,7 +36,7 @@ llvm::FunctionType *DtoFunctionType(FuncDeclaration *fdecl);
 
 void DtoResolveFunction(FuncDeclaration *fdecl);
 void DtoDeclareFunction(FuncDeclaration *fdecl);
-void DtoDefineFunction(FuncDeclaration *fd);
+void DtoDefineFunction(FuncDeclaration *fd, bool linkageAvailableExternally = false);
 
 void DtoDefineNakedFunction(FuncDeclaration *fd);
 void emitABIReturnAsmStmt(IRAsmBlock *asmblock, Loc &loc,

gen/optimizer.cpp

@@ -90,6 +90,14 @@ static cl::opt<cl::boolOrDefault, false, opts::FlagParser<cl::boolOrDefault>>
         cl::desc("Enable function inlining (default in -O2 and higher)"),
         cl::ZeroOrMore);
 
+static llvm::cl::opt<llvm::cl::boolOrDefault, false,
+                     opts::FlagParser<llvm::cl::boolOrDefault>>
+enableCrossModuleInlining(
+    "cross-module-inlining",
+    llvm::cl::desc("Enable cross-module function inlining (default enabled "
+                   "with inlining) (LLVM >= 3.7)"),
+    llvm::cl::ZeroOrMore, llvm::cl::Hidden);
+
 static cl::opt<bool> unitAtATime("unit-at-a-time", cl::desc("Enable basic IPO"),
                                  cl::init(true));
 
@@ -131,6 +139,19 @@ bool willInline() {
          (enableInlining == cl::BOU_UNSET && optLevel() > 1);
 }
 
+bool willCrossModuleInline() {
+#if LDC_LLVM_VER >= 307
+  return enableCrossModuleInlining == llvm::cl::BOU_TRUE ||
+         (enableCrossModuleInlining == llvm::cl::BOU_UNSET && willInline());
+#else
+// Cross-module inlining is disabled for <3.7 because we don't emit symbols in
+// COMDAT any groups pre-LLVM3.7. With cross-module inlining enabled, without
+// COMDAT any there are multiple-def linker errors when linking druntime.
+// See supportsCOMDAT().
+  return false;
+#endif
+}
+
 bool isOptimizationEnabled() { return optimizeLevel != 0; }
 
 llvm::CodeGenOpt::Level codeGenOptLevel() {

gen/optimizer.h

@@ -41,6 +41,8 @@ bool ldc_optimize_module(llvm::Module *m);
 // Returns whether the normal, full inlining pass will be run.
 bool willInline();
 
+bool willCrossModuleInline();
+
 bool isOptimizationEnabled();
 
 llvm::CodeGenOpt::Level codeGenOptLevel();

gen/recursivevisitor.h

@@ -29,7 +29,10 @@
 #ifndef LDC_GEN_RECURSIVEVISITOR_H
 #define LDC_GEN_RECURSIVEVISITOR_H
 
-#include "visitor.h"
+#include "ddmd/declaration.h"
+#include "ddmd/init.h"
+#include "ddmd/statement.h"
+#include "ddmd/visitor.h"
 
 class RecursiveVisitor : public Visitor {
 public:
@@ -240,40 +243,44 @@ public:
 ///////////////////////////////////////////////////////////////////////////////
 
 /// A recursive AST walker, that walks both Statements and Expressions
-/// The recursion stops at a depth where the visitor sets stop to true. Then,
+/// The recursion stops at a depth where the visitor sets stop to true.
-/// the visitor's stop is reset to false and traversal continues at the next
+/// If `continueAfterStop` is true, the visitor's stop is reset to false and
-/// node in the hierarchy at the same level as where stop was set.
+/// traversal continues at the next node in the hierarchy at the same level as
+/// where stop was set.
 class RecursiveWalker : public Visitor {
 public:
   StoppableVisitor *v;
+  bool continueAfterStop;
 
-  RecursiveWalker(StoppableVisitor *visitor) : v(visitor) {}
+  RecursiveWalker(StoppableVisitor *visitor, bool _continueAfterStop = true)
+      : v(visitor), continueAfterStop(_continueAfterStop) {}
 
   template <class T> bool recurse(T *stmt) {
     if (stmt) {
       stmt->accept(this);
     }
-    return false;
+    return v->stop;
   }
 
   template <class T> bool recurse(Array<T> *stmts) {
     if (stmts) {
       for (auto s : *stmts) {
-        recurse(s);
+        if (recurse(s))
+          break;
       }
     }
-    return false;
+    return v->stop;
   }
 
   template <class T> bool call_visitor(T *obj) {
     obj->accept(v);
-    if (v->stop) {
+    if (v->stop && continueAfterStop) {
       // Reset stop to false, so that traversion continues at neighboring node
       // in the tree.
       v->stop = false;
       return true;
     }
-    return false;
+    return v->stop;
   }
 
   using Visitor::visit;
@@ -326,9 +333,7 @@ public:
   }
 
   void visit(OnScopeStatement *stmt) override {
-    stmt->error("Internal Compiler Error: OnScopeStatement should have been "
+    call_visitor(stmt) || recurse(stmt->statement);
-                "lowered by frontend.");
-    fatal();
   }
 
   void visit(ThrowStatement *stmt) override {

runtime/phobos

@@ -1 +1 @@
-Subproject commit 3c50c24a8e778fdc9d4a36f8e0a949403cbe6bf9
+Subproject commit 33835f7996e555b9e24800b3e9c26686faa3cba7

tests/codegen/inlining_disablecross.d

@@ -0,0 +1,23 @@
+// Test disabling/enabling of cross-module inlining
+
+// REQUIRES: atleast_llvm307
+
+// RUN: %ldc %s -I%S -c -output-ll  -enable-cross-module-inlining -O0 -of=%t.ENA.ll && FileCheck %s --check-prefix ENABLED  < %t.ENA.ll
+// RUN: %ldc %s -I%S -c -output-ll -disable-cross-module-inlining -O3 -of=%t.DIS.ll && FileCheck %s --check-prefix DISABLED < %t.DIS.ll
+
+import inputs.inlinables;
+
+extern (C): // simplify mangling for easier matching
+
+// DISABLED-LABEL: define{{.*}} @call_easily_inlinable(
+// ENABLED-LABEL: define{{.*}} @call_easily_inlinable(
+int call_easily_inlinable(int i)
+{
+    // DISABLED: call {{.*}} @easily_inlinable(
+    return easily_inlinable(i);
+    // DISABLED: ret
+    // ENABLED: ret
+}
+
+// ENABLED-DAG: define {{.*}} @easily_inlinable(
+// DISABLED-DAG: declare {{.*}} @easily_inlinable(

tests/codegen/inlining_imports.d

@@ -0,0 +1,44 @@
+// Test inlining of imported functions
+
+// REQUIRES: atleast_llvm307
+
+// RUN: %ldc %s -I%S -c -output-ll -release -O3 -of=%t.O3.ll && FileCheck %s --check-prefix OPT3 < %t.O3.ll
+
+import inputs.inlinables;
+
+extern (C): // simplify mangling for easier matching
+
+// Simple functions for reference.
+int foo()
+{
+    return goo();
+}
+
+int goo()
+{
+    return 1;
+}
+
+// OPT3-LABEL: define{{.*}} @call_easily_inlinable(
+int call_easily_inlinable(int i)
+{
+    // OPT3-NOT: call {{.*}} @easily_inlinable(
+    return easily_inlinable(i);
+    // OPT3: ret i32 2
+}
+
+// OPT3-LABEL: define{{.*}} @call_class_function(
+int call_class_function(A a)
+{
+    // OPT3-NOT: call
+    return a.final_class_function();
+    // OPT3: ret i32 12345
+}
+
+// OPT3-LABEL: define{{.*}} @call_weak_function(
+int call_weak_function()
+{
+    // OPT3: call
+    return weak_function();
+    // OPT3-NOT: 654
+}

tests/codegen/inlining_imports_pragma.d

@@ -0,0 +1,44 @@
+// Test inlining of functions marked with pragma(inline) in an imported module
+
+// O0 and O3 should behave the same for these tests with explicit inlining directives by the user.
+
+// REQUIRES: atleast_llvm307
+
+// RUN: %ldc %s -I%S -c -output-ll -O0 -of=%t.O0.ll && FileCheck %s --check-prefix OPTNONE < %t.O0.ll
+// RUN: %ldc %s -I%S -c -output-ll -O3 -of=%t.O3.ll && FileCheck %s --check-prefix OPT3 < %t.O3.ll
+
+import inputs.inlinables;
+
+extern (C): // simplify mangling for easier matching
+
+// OPTNONE-LABEL: define{{.*}} @call_never_inline(
+// OPT3-LABEL: define{{.*}} @call_never_inline(
+int call_never_inline()
+{
+    // OPTNONE: call {{.*}} @never_inline()
+    // OPT3: call {{.*}} @never_inline()
+    return never_inline();
+}
+// OPTNONE-DAG: declare {{.*}} @never_inline()
+
+// OPTNONE-LABEL: define{{.*}} @call_always_inline(
+// OPT3-LABEL: define{{.*}} @call_always_inline(
+int call_always_inline()
+{
+    // OPTNONE-NOT: call {{.*}} @always_inline()
+    // OPT3-NOT: call {{.*}} @always_inline()
+    return always_inline();
+    // OPTNONE: ret
+    // OPT3: ret
+}
+
+// OPTNONE-LABEL: define{{.*}} @call_inline_chain(
+// OPT3-LABEL: define{{.*}} @call_inline_chain(
+int call_inline_chain()
+{
+    // OPTNONE-NOT: call
+    // OPT3-NOT: call
+    return always_inline_chain0();
+    // OPTNONE: ret
+    // OPT3: ret
+}

tests/codegen/inlining_leakdefinitions.d

@@ -0,0 +1,68 @@
+// Test that inlining does not leak definitions without marking them as available_externally
+// "Leaking" = symbols definitions in .o file that shouldn't be declarations instead (undefined symbols).
+
+// REQUIRES: atleast_llvm307
+
+// RUN: %ldc %s -I%S -c -output-ll -release                  -O3 -of=%t.O3.ll && FileCheck %s --check-prefix OPT3 < %t.O3.ll
+// RUN: %ldc %s -I%S -c -output-ll -release -enable-inlining -O0 -of=%t.O0.ll && FileCheck %s --check-prefix OPT0 < %t.O0.ll
+// RUN: %ldc -I%S -enable-inlining %S/inputs/inlinables.d -run %s
+// RUN: %ldc -I%S -O3 %S/inputs/inlinables.d -run %s
+
+import inputs.inlinables;
+
+extern (C): // simplify mangling for easier matching
+
+// Inlined naked asm func could end up as global symbols, definitely bad!
+// (would give multiple definition linker error)
+// OPT0-NOT: module asm {{.*}}.globl{{.*}}_naked_asm_func
+// OPT3-NOT: module asm {{.*}}.globl{{.*}}_naked_asm_func
+
+// Check that the global variables that are added due to "available_externally
+// inlining" do not have initializers, i.e. they are declared only and not definined.
+
+// OPT3-DAG: @module_variable = external thread_local{{.*}} global i32, align
+// OPT3-DAG: @{{.*}}write_function_static_variableUiZ15static_func_vari = external thread_local{{.*}} global i32, align
+
+// OPT0-LABEL: define{{.*}} @call_class_function(
+// OPT3-LABEL: define{{.*}} @call_class_function(
+int call_class_function(A a)
+{
+    // There should be only one call to "virtual_func".
+    // OPT3: call
+    // OPT3-NOT: call
+    return a.final_func();
+    // There should be a return from an LLVM variable (not a direct value)
+    // OPT0: ret i32 %
+    // OPT3: ret i32 %
+}
+
+// OPT0-LABEL: define{{.*}} @dont_leak_module_variables(
+// OPT3-LABEL: define{{.*}} @dont_leak_module_variables(
+void dont_leak_module_variables()
+{
+    write_module_variable(987);
+    write_function_static_variable(167);
+    get_typeid_A();
+    // OPT0: ret void
+    // OPT3: ret void
+}
+
+// OPT0-LABEL: define{{.*}} @asm_func(
+// OPT3-LABEL: define{{.*}} @asm_func(
+void asm_func()
+{
+    naked_asm_func();
+    // OPT0: ret void
+    // OPT3: ret void
+}
+
+// OPT0-LABEL: define{{.*}} @main(
+// OPT3-LABEL: define{{.*}} @main(
+int main()
+{
+    dont_leak_module_variables();
+
+    return 0;
+    // OPT0: ret i32 0
+    // OPT3: ret i32 0
+}

tests/codegen/inlining_pragma.d

@@ -0,0 +1,67 @@
+// Test inlining of functions marked with pragma(inline)
+
+// RUN: %ldc %s -I%S -c -output-ll -O0 -of=%t.O0.ll && FileCheck %s --check-prefix OPTNONE < %t.O0.ll
+// RUN: %ldc %s -I%S -c -output-ll -O3 -of=%t.O3.ll && FileCheck %s --check-prefix OPT3 < %t.O3.ll
+
+extern (C): // simplify mangling for easier matching
+
+int dummy;
+
+// OPTNONE-LABEL: define{{.*}} @never_inline
+// OPTNONE-SAME: #[[NEVER:[0-9]+]]
+pragma(inline, false) int never_inline()
+{
+    dummy = 111;
+    return 222;
+}
+
+int external();
+
+// OPTNONE-LABEL: define{{.*}} @always_inline
+// OPTNONE-SAME: #[[ALWAYS:[0-9]+]]
+pragma(inline, true) int always_inline()
+{
+    int a;
+    foreach (i; 1 .. 10)
+    {
+        foreach (ii; 1 .. 10)
+        {
+            foreach (iii; 1 .. 10)
+            {
+                a += i * external();
+            }
+        }
+    }
+    dummy = 444;
+    return a;
+}
+
+// OPTNONE-LABEL: define{{.*}} @foo
+// OPTNONE-SAME: #[[FOO:[0-9]+]]
+int foo()
+{
+    return 333;
+}
+
+// OPT3-LABEL: define{{.*}} @call_always_inline
+int call_always_inline()
+{
+    // OPT3-NOT: call {{.*}} @always_inline()
+    // OPT3: ret
+    return always_inline();
+}
+
+// OPT3-LABEL: define{{.*}} @call_never_inline
+int call_never_inline()
+{
+    // OPT3: call {{.*}} @never_inline()
+    // OPT3: ret
+    return never_inline();
+}
+
+// OPTNONE-NOT: attributes #[[FOO]] ={{.*}} alwaysinline
+// OPTNONE-NOT: attributes #[[FOO]] ={{.*}} noinline
+// OPTNONE-NOT: attributes #[[NEVER]] ={{.*}} alwaysinline
+// OPTNONE-NOT: attributes #[[ALWAYS]] ={{.*}} noinline
+// OPTNONE-DAG: attributes #[[NEVER]] ={{.*}} noinline
+// OPTNONE-DAG: attributes #[[ALWAYS]] ={{.*}} alwaysinline

tests/codegen/inlining_staticvar.d

@@ -0,0 +1,71 @@
+// Test cross-module inlining involving static variables
+
+// REQUIRES: atleast_llvm307
+
+// RUN: %ldc %s -I%S -c -output-ll                  -O3 -of=%t.O3.ll && FileCheck %s --check-prefix OPT3 < %t.O3.ll
+// RUN: %ldc %s -I%S -c -output-ll -enable-inlining -O0 -of=%t.O0.ll && FileCheck %s --check-prefix OPT0 < %t.O0.ll
+// RUN: %ldc -I%S -enable-inlining %S/inputs/inlinables_staticvar.d -run %s
+// RUN: %ldc -I%S -O3              %S/inputs/inlinables_staticvar.d -run %s
+
+import inputs.inlinables_staticvar;
+
+import ldc.attributes;
+
+extern (C): // simplify mangling for easier matching
+
+// Functions are intentionally split and @weak to thwart LLVM constant folding.
+
+void checkModuleScope_1() @weak
+{
+    addToModuleScopeInline(7);
+}
+void checkModuleScope_2() @weak
+{
+    addToModuleScopeOutline(101);
+    assert(equalModuleScope(7+101));
+}
+
+void checkInsideFunc_1() @weak
+{
+    assert(addAndCheckInsideFunc(0, 7));
+}
+void checkInsideFunc_2() @weak
+{
+    assert(addAndCheckInsideFuncIndirect(7, 101));
+    assert(addAndCheckInsideFunc(7+101, 9));
+}
+
+void checkInsideNestedFunc_1() @weak
+{
+    assert(addAndCheckInsideNestedFunc(0, 7));
+}
+void checkInsideNestedFunc_2() @weak
+{
+    assert(addAndCheckInsideNestedFuncIndirect(7, 101));
+    assert(addAndCheckInsideNestedFunc(7+101, 9));
+}
+
+void checkNestedStruct_1() @weak
+{
+    assert(addAndCheckNestedStruct(0, 7));
+}
+void checkNestedStruct_2() @weak
+{
+    assert(addAndCheckNestedStructIndirect(7, 101));
+    assert(addAndCheckNestedStruct(7+101, 9));
+}
+
+// OPT0-LABEL: define{{.*}} @_Dmain(
+// OPT3-LABEL: define{{.*}} @_Dmain(
+extern(D)
+void main()
+{
+    checkModuleScope_1();
+    checkModuleScope_2();
+    checkInsideFunc_1();
+    checkInsideFunc_2();
+    checkInsideNestedFunc_1();
+    checkInsideNestedFunc_2();
+    checkNestedStruct_1();
+    checkNestedStruct_2();
+}

tests/codegen/inlining_stdlib.d

@@ -0,0 +1,36 @@
+// Test inlining of some standard library functions
+
+// REQUIRES: atleast_llvm307
+
+// Test also that the tested functions are indeed not inlined at -O0 (basically verifying that we are testing something real)
+// RUN: %ldc %s -c -output-ll -release -O0 -of=%t.O0.ll && FileCheck %s --check-prefix OPT0 < %t.O0.ll
+// RUN: %ldc %s -c -output-ll -release -O3 -of=%t.O3.ll && FileCheck %s --check-prefix OPT3 < %t.O3.ll
+
+extern (C): // simplify mangling for easier matching
+
+// OPT0-LABEL: define{{.*}} @foo(
+// OPT3-LABEL: define{{.*}} @foo(
+int foo(size_t i)
+{
+    import core.bitop;
+    // OPT0: call {{.*}} @{{.*}}core5bitop3bsf
+    // OPT3: call {{.*}} @llvm.cttz
+    return bsf(i);
+    // OPT0: ret
+    // OPT3: ret
+}
+// OPT0: declare {{.*}}core5bitop3bsf
+
+// OPT0-LABEL: define{{.*}} @ggg(
+// OPT3-LABEL: define{{.*}} @ggg(
+char[] ggg(char* str)
+{
+    import std.string;
+    // OPT0: call {{.*}} @{{.*}}std6string11fromStringz
+    // OPT3: call {{.*}}strlen
+    return fromStringz(str);
+    // OPT0: ret
+    // OPT3: ret
+}
+// OPT0: declare {{.*}}std6string11fromStringz
+// OPT3: declare {{.*}}strlen

tests/codegen/inlining_templates.d

@@ -0,0 +1,45 @@
+// Test inlining of templates
+// Templates that would otherwise not be codegenned, _should_ be codegenned for inlining when pragma(inline, true) is specified.
+
+// REQUIRES: atleast_llvm307
+
+// RUN: %ldc %s -I%S -c -output-ll -release -enable-inlining -O0 -of=%t.O0.ll && FileCheck %s < %t.O0.ll
+
+// RUN: %ldc -singleobj %S/inputs/inlinables.d %s -I%S -c -output-ll -release -enable-inlining -O0 -of=%t.singleobj.O0.ll && FileCheck %s < %t.singleobj.O0.ll
+
+// Test linking too.
+// Separate compilation
+//   RUN: %ldc -c -enable-inlining %S/inputs/inlinables.d -of=%t.inlinables%obj \
+//   RUN: && %ldc -I%S -enable-inlining %t.inlinables%obj %s -of=%t%exe
+// Singleobj compilation
+//   RUN: %ldc -I%S -enable-inlining -singleobj %S/inputs/inlinables.d %s -of=%t2%exe
+
+import inputs.inlinables;
+import std.stdio;
+import std.exception;
+
+int foo(int i)
+{
+    return call_template_foo(i);
+}
+
+// stdio.File.flush contains a call to errnoException, which contains __FILE__ as default template parameter.
+// Make sure the symbol is inlined/defined and not declared (which will lead to linker errors if the location
+// of the stdlib is different from where LDC was built from)
+void ggg(ref File f)
+{
+    f.flush();
+}
+
+void main()
+{
+}
+
+// CHECK-NOT: declare{{.*}}D6inputs10inlinables20__T12template_fooTiZ12template_foo
+// CHECK-NOT: declare{{.*}}D3std9exception{{[0-9]+}}__T12errnoEnforce
+
+// CHECK-DAG: define{{.*}}D6inputs10inlinables20__T12template_fooTiZ12template_foo{{.*}}) #[[ATTR1:[0-9]+]]
+// CHECK-DAG: define{{.*}}D3std9exception{{[0-9]+}}__T12errnoEnforce{{.*}}) #[[ATTR2:[0-9]+]]
+
+// CHECK-DAG: attributes #[[ATTR1]] ={{.*}} alwaysinline
+// CHECK-DAG: attributes #[[ATTR2]] ={{.*}} alwaysinline

tests/codegen/inputs/inlinables.d

@@ -0,0 +1,113 @@
+module inputs.inlinables;
+
+import ldc.attributes;
+
+extern (C): // simplify mangling for easier function name matching
+
+int easily_inlinable(int i)
+{
+    if (i > 0)
+        return easily_inlinable(i - 1);
+    return 2;
+}
+
+pragma(inline, false) int never_inline()
+{
+    return 1;
+}
+
+@weak int external()
+{
+    return 1;
+}
+
+pragma(inline, true) int always_inline()
+{
+    int a;
+    foreach (i; 1 .. 10)
+    {
+        foreach (ii; 1 .. 10)
+        {
+            foreach (iii; 1 .. 10)
+            {
+                a += i * external();
+            }
+        }
+    }
+    return a;
+}
+
+pragma(inline, true) int always_inline_chain0()
+{
+    return always_inline_chain1();
+}
+
+pragma(inline, true) int always_inline_chain1()
+{
+    return always_inline_chain2();
+}
+
+pragma(inline, true) int always_inline_chain2()
+{
+    return 345;
+}
+
+class A
+{
+    int virtual_func()
+    {
+        return 12345;
+    }
+
+    pragma(inline, true) final int final_func()
+    {
+        return virtual_func();
+    }
+
+    final int final_class_function()
+    {
+        return 12345;
+    }
+}
+
+// Weak-linkage functions can not be inlined.
+@weak int weak_function()
+{
+    return 654;
+}
+
+int module_variable = 666;
+pragma(inline, true) void write_module_variable(int i)
+{
+    module_variable = i;
+}
+
+pragma(inline, true) void write_function_static_variable(int i)
+{
+    static int static_func_var = 5;
+    static_func_var = i;
+}
+
+pragma(inline, true) auto get_typeid_A()
+{
+    return typeid(A);
+}
+
+pragma(inline, true) extern (C) void naked_asm_func()
+{
+    asm pure nothrow @nogc
+    {
+        naked;
+        nop;
+    }
+}
+
+pragma(inline, true) int call_template_foo(int i)
+{
+    return template_foo(i);
+}
+
+pragma(inline, true) T template_foo(T)(T i)
+{
+    return i;
+}

tests/codegen/inputs/inlinables_staticvar.d

@@ -0,0 +1,85 @@
+module inputs.inlinables_staticvar;
+
+/+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++/
+
+private int atModuleScope;
+
+pragma(inline, true) void addToModuleScopeInline(int i)
+{
+    atModuleScope += i;
+}
+
+pragma(inline, false) void addToModuleScopeOutline(int i)
+{
+    atModuleScope += i;
+}
+
+pragma(inline, false) bool equalModuleScope(int i)
+{
+    return atModuleScope == i;
+}
+
+/+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++/
+
+pragma(inline, true) bool addAndCheckInsideFunc(int checkbefore, int increment)
+{
+    static int insideFunc;
+
+    if (insideFunc != checkbefore)
+        return false;
+
+    insideFunc += increment;
+    return true;
+}
+
+pragma(inline, false) bool addAndCheckInsideFuncIndirect(int checkbefore, int increment)
+{
+    return addAndCheckInsideFunc(checkbefore, increment);
+}
+
+/+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++/
+
+pragma(inline, true) bool addAndCheckInsideNestedFunc(int checkbefore, int increment)
+{
+    pragma(inline, true)
+    bool addCheckNested(int checkbefore, int increment)
+    {
+        static int insideFunc;
+
+        if (insideFunc != checkbefore)
+            return false;
+
+        insideFunc += increment;
+        return true;
+    }
+
+    return addCheckNested(checkbefore, increment);
+}
+
+pragma(inline, false) bool addAndCheckInsideNestedFuncIndirect(int checkbefore, int increment)
+{
+    return addAndCheckInsideNestedFunc(checkbefore, increment);
+}
+
+/+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++/
+
+pragma(inline, true) bool addAndCheckNestedStruct(int checkbefore, int increment)
+{
+    struct NestedStruct
+    {
+        static int structValue;
+    }
+
+    if (NestedStruct.structValue != checkbefore)
+        return false;
+
+    NestedStruct.structValue += increment;
+    return true;
+}
+
+pragma(inline, false) bool addAndCheckNestedStructIndirect(int checkbefore, int increment)
+{
+    return addAndCheckNestedStruct(checkbefore, increment);
+}
+
+/+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++/