MSVC: Port Rainer's upstream extern(C++) method ABI fixes

From https://github.com/dlang/dmd/pull/8330.

dmd/semantic3.d

@@ -602,7 +602,7 @@ private extern(C++) final class Semantic3Visitor : Visitor
                     }
                 }
 
-                if (!funcdecl.inferRetType && !Target.isReturnOnStack(f))
+                if (!funcdecl.inferRetType && !Target.isReturnOnStack(f, funcdecl.needThis()))
                     funcdecl.nrvo_can = 0;
 
                 bool inferRef = (f.isref && (funcdecl.storage_class & STC.auto_));
@@ -656,7 +656,7 @@ private extern(C++) final class Semantic3Visitor : Visitor
                     if (funcdecl.storage_class & STC.auto_)
                         funcdecl.storage_class &= ~STC.auto_;
                 }
-                if (!Target.isReturnOnStack(f))
+                if (!Target.isReturnOnStack(f, funcdecl.needThis()))
                     funcdecl.nrvo_can = 0;
 
                 if (funcdecl.fbody.isErrorStatement())

dmd/target.d

@@ -21,6 +21,7 @@ import dmd.dmodule;
 import dmd.dstruct;
 import dmd.dsymbol;
 import dmd.expression;
+import dmd.func;
 import dmd.globals;
 import dmd.id;
 import dmd.identifier;
@@ -653,18 +654,21 @@ struct Target
     }
 
     /**
+     * Determine return style of function - whether in registers or
+     * through a hidden pointer to the caller's stack.
      * Params:
      *   tf = function type to check
+     *   needsThis = true if the function type is for a non-static member function
      * Returns:
      *   true if return value from function is on the stack
      */
   version(IN_LLVM)
   {
-    extern (C++) static bool isReturnOnStack(TypeFunction tf);
+    extern (C++) static bool isReturnOnStack(TypeFunction tf, bool needsThis);
   }
   else
   {
-    extern (C++) static bool isReturnOnStack(TypeFunction tf)
+    extern (C++) static bool isReturnOnStack(TypeFunction tf, bool needsThis)
     {
         if (tf.isref)
         {
@@ -691,6 +695,8 @@ struct Target
                 StructDeclaration sd = (cast(TypeStruct)tns).sym;
                 if (sd.ident == Id.__c_long_double)
                     return false;
+                if (tf.linkage == LINK.cpp && needsThis)
+                    return true;
                 if (!sd.isPOD() || sz > 8)
                     return true;
                 if (sd.fields.dim == 0)
@@ -708,6 +714,8 @@ struct Target
                 StructDeclaration sd = (cast(TypeStruct)tb).sym;
                 if (sd.ident == Id.__c_long_double)
                     return false;
+                if (tf.linkage == LINK.cpp && needsThis)
+                    return true;
             }
         }
 

dmd/target.h

@@ -99,7 +99,7 @@ struct Target
     static Type *cppParameterType(Parameter *p);
     static LINK systemLinkage();
     static TypeTuple *toArgTypes(Type *t);
-    static bool isReturnOnStack(TypeFunction *tf);
+    static bool isReturnOnStack(TypeFunction *tf, bool needsThis);
     static d_uns64 parameterSize(const Loc& loc, Type *t);
 };
 

dmd/traits.d

@@ -1074,7 +1074,7 @@ extern (C++) Expression semanticTraits(TraitsExp e, Scope* sc)
             return new ErrorExp();
         }
 
-        bool value = Target.isReturnOnStack(tf);
+        bool value = Target.isReturnOnStack(tf, fd && fd.needThis());
         return new IntegerExp(e.loc, value, Type.tbool);
     }
     if (e.ident == Id.getFunctionVariadicStyle)

gen/abi-aarch64.cpp

@@ -22,7 +22,7 @@ struct AArch64TargetABI : TargetABI {
   CompositeToArray64 compositeToArray64;
   IntegerRewrite integerRewrite;
 
-  bool returnInArg(TypeFunction *tf) override {
+  bool returnInArg(TypeFunction *tf, bool) override {
     if (tf->isref) {
       return false;
     }

gen/abi-arm.cpp

@@ -25,7 +25,7 @@ struct ArmTargetABI : TargetABI {
   CompositeToArray64 compositeToArray64;
   IntegerRewrite integerRewrite;
 
-  bool returnInArg(TypeFunction *tf) override {
+  bool returnInArg(TypeFunction *tf, bool) override {
     // AAPCS 5.4 wants composites > 4-bytes returned by arg except for
     // Homogeneous Aggregates of up-to 4 float types (6.1.2.1) - an HFA.
     // TODO: see if Tsarray should be candidate for HFA.

gen/abi-mips64.cpp

@@ -25,7 +25,7 @@ struct MIPS64TargetABI : TargetABI {
 
   explicit MIPS64TargetABI(const bool Is64Bit) : Is64Bit(Is64Bit) {}
 
-  bool returnInArg(TypeFunction *tf) override {
+  bool returnInArg(TypeFunction *tf, bool) override {
     if (tf->isref) {
       return false;
     }

gen/abi-nvptx.cpp

@@ -35,7 +35,7 @@ struct NVPTXTargetABI : TargetABI {
         rewriteArgument(fty, *arg);
     }
   }
-  bool returnInArg(TypeFunction *tf) override {
+  bool returnInArg(TypeFunction *tf, bool) override {
     return !tf->isref && DtoIsInMemoryOnly(tf->next);
   }
   void rewriteArgument(IrFuncTy &fty, IrFuncTyArg &arg) override {

gen/abi-ppc.cpp

@@ -35,7 +35,7 @@ struct PPCTargetABI : TargetABI {
 
   explicit PPCTargetABI(const bool Is64Bit) : Is64Bit(Is64Bit) {}
 
-  bool returnInArg(TypeFunction *tf) override {
+  bool returnInArg(TypeFunction *tf, bool) override {
     if (tf->isref) {
       return false;
     }

gen/abi-ppc64le.cpp

@@ -28,7 +28,7 @@ struct PPC64LETargetABI : TargetABI {
 
   explicit PPC64LETargetABI() : hfaToArray(8) {}
 
-  bool returnInArg(TypeFunction *tf) override {
+  bool returnInArg(TypeFunction *tf, bool) override {
     if (tf->isref) {
       return false;
     }

gen/abi-spirv.cpp

@@ -35,7 +35,7 @@ struct SPIRVTargetABI : TargetABI {
         rewriteArgument(fty, *arg);
     }
   }
-  bool returnInArg(TypeFunction *tf) override {
+  bool returnInArg(TypeFunction *tf, bool) override {
     return !tf->isref && DtoIsInMemoryOnly(tf->next);
   }
   void rewriteArgument(IrFuncTy &fty, IrFuncTyArg &arg) override {

gen/abi-win64.cpp

@@ -104,16 +104,23 @@ public:
     return name;
   }
 
-  bool returnInArg(TypeFunction *tf) override {
+  bool returnInArg(TypeFunction *tf, bool needsThis) override {
     if (tf->isref)
       return false;
 
+    Type *rt = tf->next->toBasetype();
+
+    // for non-static member functions, MSVC++ enforces sret for all structs
+    if (isMSVC && tf->linkage == LINKcpp && needsThis && rt->ty == Tstruct &&
+        !isMagicCppStruct(rt)) {
+      return true;
+    }
+
     // * all POD types of a power-of-2 size <= 8 bytes (incl. 2x32-bit cfloat)
     //   are returned in a register (RAX, or XMM0 for single float/ifloat/
     //   double/idouble)
     // * 80-bit real/ireal are returned on the x87 stack
     // * all other types are returned via sret
-    Type *rt = tf->next->toBasetype();
     return passPointerToHiddenCopy(rt, /*isReturnValue=*/true, tf->linkage);
   }
 

gen/abi-x86-64.cpp

@@ -219,7 +219,7 @@ struct X86_64TargetABI : TargetABI {
   ImplicitByvalRewrite byvalRewrite;
   IndirectByvalRewrite indirectByvalRewrite;
 
-  bool returnInArg(TypeFunction *tf) override;
+  bool returnInArg(TypeFunction *tf, bool needsThis) override;
 
   bool passByVal(TypeFunction *tf, Type *t) override;
 
@@ -248,7 +248,7 @@ private:
 // The public getter for abi.cpp
 TargetABI *getX86_64TargetABI() { return new X86_64TargetABI; }
 
-bool X86_64TargetABI::returnInArg(TypeFunction *tf) {
+bool X86_64TargetABI::returnInArg(TypeFunction *tf, bool) {
   if (tf->isref) {
     return false;
   }

gen/abi-x86.cpp

@@ -45,7 +45,7 @@ struct X86TargetABI : TargetABI {
     case LINKobjc:
       return llvm::CallingConv::C;
     case LINKcpp:
-      return isMSVC && fdecl && fdecl->isThis()
+      return isMSVC && fdecl && fdecl->needThis()
                  ? llvm::CallingConv::X86_ThisCall
                  : llvm::CallingConv::C;
     case LINKd:
@@ -83,7 +83,7 @@ struct X86TargetABI : TargetABI {
     return name;
   }
 
-  bool returnInArg(TypeFunction *tf) override {
+  bool returnInArg(TypeFunction *tf, bool needsThis) override {
     if (tf->isref)
       return false;
 
@@ -109,8 +109,15 @@ struct X86TargetABI : TargetABI {
     if (!externD && !returnStructsInRegs)
       return true;
 
+    const bool isMSVCpp = isMSVC && tf->linkage == LINKcpp;
+
+    // for non-static member functions, MSVC++ enforces sret for all structs
+    if (isMSVCpp && needsThis && rt->ty == Tstruct) {
+      return true;
+    }
+
     // force sret for non-POD structs
-    const bool excludeStructsWithCtor = (isMSVC && tf->linkage == LINKcpp);
+    const bool excludeStructsWithCtor = isMSVCpp;
     if (!isPOD(rt, excludeStructsWithCtor))
       return true;
 

gen/abi.cpp

@@ -297,7 +297,7 @@ const char *TargetABI::objcMsgSendFunc(Type *ret, IrFuncTy &fty) {
 
 // Some reasonable defaults for when we don't know what ABI to use.
 struct UnknownTargetABI : TargetABI {
-  bool returnInArg(TypeFunction *tf) override {
+  bool returnInArg(TypeFunction *tf, bool) override {
     if (tf->isref) {
       return false;
     }
@@ -361,7 +361,7 @@ TargetABI *TargetABI::getTarget() {
 struct IntrinsicABI : TargetABI {
   RemoveStructPadding remove_padding;
 
-  bool returnInArg(TypeFunction *tf) override { return false; }
+  bool returnInArg(TypeFunction *, bool) override { return false; }
 
   bool passByVal(TypeFunction *, Type *t) override { return false; }
 

gen/abi.h

@@ -120,13 +120,15 @@ struct TargetABI {
   }
 
   /// Returns true if the D function uses sret (struct return).
+  /// `needsThis` is true if the function type is for a non-static member
+  /// function.
   ///
   /// A LL sret function doesn't really return a struct (in fact, it returns
   /// void); it merely just sets a struct which has been pre-allocated by the
   /// caller.
   /// The address is passed as additional function parameter using the StructRet
   /// attribute.
-  virtual bool returnInArg(TypeFunction *tf) = 0;
+  virtual bool returnInArg(TypeFunction *tf, bool needsThis) = 0;
 
   /// Returns true if the D type is passed using the LLVM ByVal attribute.
   ///

gen/functions.cpp

@@ -57,8 +57,7 @@ static bool isMainFunction(FuncDeclaration *fd) {
 }
 
 llvm::FunctionType *DtoFunctionType(Type *type, IrFuncTy &irFty, Type *thistype,
-                                    Type *nesttype, bool isMain, bool isCtor,
-                                    bool isIntrinsic, bool hasSel) {
+                                    Type *nesttype, FuncDeclaration *fd) {
   IF_LOG Logger::println("DtoFunctionType(%s)", type->toChars());
   LOG_SCOPE
 
@@ -73,7 +72,7 @@ llvm::FunctionType *DtoFunctionType(Type *type, IrFuncTy &irFty, Type *thistype,
     return irFty.funcType;
   }
 
-  TargetABI *abi = (isIntrinsic ? TargetABI::getIntrinsic() : gABI);
+  TargetABI *abi = fd && DtoIsIntrinsic(fd) ? TargetABI::getIntrinsic() : gABI;
 
   // Do not modify irFty yet; this function may be called recursively if any
   // of the argument types refer to this type.
@@ -82,6 +81,7 @@ llvm::FunctionType *DtoFunctionType(Type *type, IrFuncTy &irFty, Type *thistype,
   // The index of the next argument on the LLVM level.
   unsigned nextLLArgIdx = 0;
 
+  const bool isMain = fd && isMainFunction(fd);
   if (isMain) {
     // D and C main functions always return i32, even if declared as returning
     // void.
@@ -91,7 +91,7 @@ llvm::FunctionType *DtoFunctionType(Type *type, IrFuncTy &irFty, Type *thistype,
     const bool byref = f->isref && rt->toBasetype()->ty != Tvoid;
     AttrBuilder attrs;
 
-    if (abi->returnInArg(f)) {
+    if (abi->returnInArg(f, fd && fd->needThis())) {
       // sret return
       newIrFty.arg_sret = new IrFuncTyArg(
           rt, true,
@@ -112,7 +112,7 @@ llvm::FunctionType *DtoFunctionType(Type *type, IrFuncTy &irFty, Type *thistype,
     // Add the this pointer for member functions
     AttrBuilder attrs;
     attrs.add(LLAttribute::NonNull);
-    if (isCtor) {
+    if (fd && fd->isCtorDeclaration()) {
       attrs.add(LLAttribute::Returned);
     }
     newIrFty.arg_this =
@@ -126,7 +126,15 @@ llvm::FunctionType *DtoFunctionType(Type *type, IrFuncTy &irFty, Type *thistype,
     ++nextLLArgIdx;
   }
 
-  if (hasSel) {
+  bool hasObjCSelector = false;
+  if (fd && fd->linkage == LINKobjc && thistype) {
+    if (fd->selector) {
+      hasObjCSelector = true;
+    } else if (fd->parent->isClassDeclaration()) {
+      fd->error("Objective-C `@selector` is missing");
+    }
+  }
+  if (hasObjCSelector) {
     // TODO: make arg_objcselector to match dmd type
     newIrFty.arg_objcSelector = new IrFuncTyArg(Type::tvoidptr, false);
     ++nextLLArgIdx;
@@ -272,7 +280,6 @@ llvm::FunctionType *DtoFunctionType(FuncDeclaration *fdecl) {
   }
 
   Type *dthis = nullptr, *dnest = nullptr;
-  bool hasSel = false;
 
   if (fdecl->ident == Id::ensure || fdecl->ident == Id::require) {
     FuncDeclaration *p = fdecl->parent->isFuncDeclaration();
@@ -299,18 +306,8 @@ llvm::FunctionType *DtoFunctionType(FuncDeclaration *fdecl) {
     dnest = Type::tvoid->pointerTo();
   }
 
-  if (fdecl->linkage == LINKobjc && dthis) {
-    if (fdecl->selector) {
-      hasSel = true;
-    } else if (fdecl->parent->isClassDeclaration()) {
-      fdecl->error("Objective-C `@selector` is missing");
-    }
-  }
-
-  LLFunctionType *functype =
-      DtoFunctionType(fdecl->type, getIrFunc(fdecl, true)->irFty, dthis, dnest,
-                      isMainFunction(fdecl), fdecl->isCtorDeclaration(),
-                      DtoIsIntrinsic(fdecl), hasSel);
+  LLFunctionType *functype = DtoFunctionType(
+      fdecl->type, getIrFunc(fdecl, true)->irFty, dthis, dnest, fdecl);
 
   return functype;
 }

gen/functions.h

@@ -28,10 +28,8 @@ class FunctionType;
 }
 
 llvm::FunctionType *DtoFunctionType(Type *t, IrFuncTy &irFty, Type *thistype,
-                                    Type *nesttype, bool isMain = false,
-                                    bool isCtor = false,
-                                    bool isIntrinsic = false,
-                                    bool hasSel = false);
+                                    Type *nesttype,
+                                    FuncDeclaration *fd = nullptr);
 llvm::FunctionType *DtoFunctionType(FuncDeclaration *fdecl);
 
 void DtoResolveFunction(FuncDeclaration *fdecl);

gen/target.cpp

@@ -268,6 +268,6 @@ Expression *Target::paintAsType(Expression *e, Type *type) {
  */
 void Target::loadModule(Module *m) {}
 
-bool Target::isReturnOnStack(TypeFunction *tf) {
-  return gABI->returnInArg(tf);
+bool Target::isReturnOnStack(TypeFunction *tf, bool needsThis) {
+  return gABI->returnInArg(tf, needsThis);
 }

gen/tollvm.cpp

@@ -44,7 +44,8 @@ bool DtoIsInMemoryOnly(Type *type) {
 bool DtoIsReturnInArg(CallExp *ce) {
   Type *t = ce->e1->type->toBasetype();
   if (t->ty == Tfunction && (!ce->f || !DtoIsIntrinsic(ce->f))) {
-    return gABI->returnInArg(static_cast<TypeFunction *>(t));
+    return gABI->returnInArg(static_cast<TypeFunction *>(t),
+                             ce->f && ce->f->needThis());
   }
   return false;
 }

tests/d2/dmd-testsuite

@@ -1 +1 @@
-Subproject commit b0b28297a8befba66640c7b4462ecdc12c40c318
+Subproject commit e1e05efb40fa122cae53faa6d8dff25b098c9203