Enable cross-static-lib generation for LLVM 3.9+

By directly integrating LLVM's `llvm-lib.exe` driver to generate static
libs for MSVC targets and a stripped-down version of the `llvm-ar` tool
for the other targets.

Introduce command-line option `-archiver=<file>` to allow the user to
specify an external archiver to be invoked.

CMakeLists.txt

@@ -23,7 +23,7 @@ endif()
 #
 
 find_package(LLVM 3.5 REQUIRED
-    all-targets analysis asmparser asmprinter bitreader bitwriter codegen core debuginfocodeview debuginfodwarf debuginfomsf debuginfopdb globalisel instcombine ipa ipo instrumentation irreader linker lto mc mcdisassembler mcparser objcarcopts object option profiledata scalaropts selectiondag support tablegen target transformutils vectorize ${EXTRA_LLVM_MODULES})
+    all-targets analysis asmparser asmprinter bitreader bitwriter codegen core debuginfocodeview debuginfodwarf debuginfomsf debuginfopdb globalisel instcombine ipa ipo instrumentation irreader libdriver linker lto mc mcdisassembler mcparser objcarcopts object option profiledata scalaropts selectiondag support tablegen target transformutils vectorize ${EXTRA_LLVM_MODULES})
 math(EXPR LDC_LLVM_VER ${LLVM_VERSION_MAJOR}*100+${LLVM_VERSION_MINOR})
 # Remove LLVMTableGen library from list of libraries
 string(REGEX MATCH "^-.*LLVMTableGen[^;]*;|;-.*LLVMTableGen[^;]*" LLVM_TABLEGEN_LIBRARY "${LLVM_LIBRARIES}")
@@ -347,6 +347,7 @@ set(DRV_SRC
     driver/targetmachine.cpp
     driver/toobj.cpp
     driver/tool.cpp
+    driver/archiver.cpp
     driver/linker.cpp
     driver/main.cpp
     ${CMAKE_BINARY_DIR}/driver/ldc-version.cpp
@@ -359,6 +360,7 @@ set(DRV_HDR
     driver/configfile.h
     driver/exe_path.h
     driver/ldc-version.h
+    driver/archiver.h
     driver/linker.h
     driver/targetmachine.h
     driver/toobj.h

cmake/Modules/FindLLVM.cmake

@@ -105,6 +105,10 @@ if ((WIN32 AND NOT(MINGW OR CYGWIN)) OR NOT LLVM_CONFIG)
             # Versions below 4.0 do not support component debuginfomsf
             list(REMOVE_ITEM LLVM_FIND_COMPONENTS "debuginfomsf" index)
         endif()
+        if(${LLVM_VERSION_STRING} MATCHES "^3\\.[0-6][\\.0-9A-Za-z]*")
+            # Versions below 3.7 do not support component libdriver
+            list(REMOVE_ITEM LLVM_FIND_COMPONENTS "libdriver" index)
+        endif()
 
         llvm_map_components_to_libnames(tmplibs ${LLVM_FIND_COMPONENTS})
         if(MSVC)
@@ -198,6 +202,10 @@ else()
         # Versions below 4.0 do not support component debuginfomsf
         list(REMOVE_ITEM LLVM_FIND_COMPONENTS "debuginfomsf" index)
     endif()
+    if(${LLVM_VERSION_STRING} MATCHES "^3\\.[0-6][\\.0-9A-Za-z]*")
+        # Versions below 3.7 do not support component libdriver
+        list(REMOVE_ITEM LLVM_FIND_COMPONENTS "libdriver" index)
+    endif()
 
     llvm_set(LDFLAGS ldflags)
     if(NOT ${LLVM_VERSION_STRING} MATCHES "^3\\.[0-4][\\.0-9A-Za-z]*")

driver/archiver.cpp

@@ -0,0 +1,270 @@
+//===-- archiver.cpp ------------------------------------------------------===//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LLVM LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+//
+// Builds up (relatively) standard unix archive files (.a) containing LLVM
+// bitcode or other files.
+//
+//===----------------------------------------------------------------------===//
+
+#if LDC_LLVM_VER >= 309
+
+#include "driver/archiver.h"
+
+#include "llvm/ADT/Triple.h"
+#include "llvm/LibDriver/LibDriver.h"
+#include "llvm/Object/Archive.h"
+#include "llvm/Object/ArchiveWriter.h"
+#include "llvm/Object/MachO.h"
+#include "llvm/Object/ObjectFile.h"
+#include "llvm/Support/Errc.h"
+#include "llvm/Support/Path.h"
+#include "llvm/Support/raw_ostream.h"
+
+#include <cstring>
+
+using namespace llvm;
+
+/* Unlike the llvm-lib driver, llvm-ar is not available as library; it's
+ * unfortunately a separate tool.
+ * The following is a stripped-down version of LLVM's
+ * `tools/llvm-ar/llvm-ar.cpp` (based on early LLVM 5.0), as LDC only needs
+ * support for `llvm-ar rcs <archive name> <member> ...`.
+ * It also makes sure the process isn't simply exited whenever a problem arises.
+ */
+namespace {
+
+StringRef ArchiveName;
+std::vector<const char *> Members;
+
+bool Symtab = true;
+bool Deterministic = true;
+bool Thin = false;
+
+void fail(Twine Error) { outs() << "llvm-ar: " << Error << ".\n"; }
+
+void fail(std::error_code EC, std::string Context = {}) {
+  if (Context.empty())
+    fail(EC.message());
+  else
+    fail(Context + ": " + EC.message());
+}
+
+void fail(Error E, std::string Context = {}) {
+  if (!Context.empty())
+    Context += ": ";
+
+  handleAllErrors(std::move(E), [&](const ErrorInfoBase &EIB) {
+    if (Context.empty())
+      fail(EIB.message());
+    else
+      fail(Context + EIB.message());
+  });
+}
+
+int addMember(std::vector<NewArchiveMember> &Members, StringRef FileName,
+              int Pos = -1) {
+  Expected<NewArchiveMember> NMOrErr =
+      NewArchiveMember::getFile(FileName, Deterministic);
+  if (auto Error = NMOrErr.takeError()) {
+    fail(std::move(Error), FileName);
+    return 1;
+  }
+  if (Pos == -1)
+    Members.push_back(std::move(*NMOrErr));
+  else
+    Members[Pos] = std::move(*NMOrErr);
+  return 0;
+}
+
+int addMember(std::vector<NewArchiveMember> &Members,
+              const object::Archive::Child &M, int Pos = -1) {
+  if (Thin && !M.getParent()->isThin()) {
+    fail("Cannot convert a regular archive to a thin one");
+    return 1;
+  }
+  Expected<NewArchiveMember> NMOrErr =
+      NewArchiveMember::getOldMember(M, Deterministic);
+  if (auto Error = NMOrErr.takeError()) {
+    fail(std::move(Error));
+    return 1;
+  }
+  if (Pos == -1)
+    Members.push_back(std::move(*NMOrErr));
+  else
+    Members[Pos] = std::move(*NMOrErr);
+  return 0;
+}
+
+int computeNewArchiveMembers(object::Archive *OldArchive,
+                             std::vector<NewArchiveMember> &Ret) {
+  if (OldArchive) {
+    Error Err = Error::success();
+    for (auto &Child : OldArchive->children(Err)) {
+#if LDC_LLVM_VER < 400
+      auto NameOrErr = Child.getName();
+      if (auto Error = NameOrErr.getError()) {
+#else
+      Expected<StringRef> NameOrErr = Child.getName();
+      if (auto Error = NameOrErr.takeError()) {
+#endif
+        fail(std::move(Error));
+        return 1;
+      }
+      StringRef Name = NameOrErr.get();
+
+      auto MemberI = find_if(Members, [Name](StringRef Path) {
+        return Name == sys::path::filename(Path);
+      });
+
+      if (MemberI == Members.end()) {
+        if (int Status = addMember(Ret, Child))
+          return Status;
+      } else {
+        if (int Status = addMember(Ret, *MemberI))
+          return Status;
+        Members.erase(MemberI);
+      }
+    }
+    if (Err) {
+      fail(std::move(Err));
+      return 1;
+    }
+  }
+
+  const int InsertPos = Ret.size();
+  for (unsigned I = 0; I != Members.size(); ++I)
+    Ret.insert(Ret.begin() + InsertPos, NewArchiveMember());
+  int Pos = InsertPos;
+  for (auto &Member : Members) {
+    if (int Status = addMember(Ret, Member, Pos))
+      return Status;
+    ++Pos;
+  }
+
+  return 0;
+}
+
+object::Archive::Kind getDefaultForHost() {
+  return Triple(sys::getProcessTriple()).isOSDarwin()
+#if LDC_LLVM_VER >= 500
+             ? object::Archive::K_DARWIN
+#else
+             ? object::Archive::K_BSD
+#endif
+             : object::Archive::K_GNU;
+}
+
+object::Archive::Kind getKindFromMember(const NewArchiveMember &Member) {
+  auto OptionalObject =
+      object::ObjectFile::createObjectFile(Member.Buf->getMemBufferRef());
+
+  if (OptionalObject) {
+    return isa<object::MachOObjectFile>(**OptionalObject)
+#if LDC_LLVM_VER >= 500
+               ? object::Archive::K_DARWIN
+#else
+               ? object::Archive::K_BSD
+#endif
+               : object::Archive::K_GNU;
+  }
+
+  // squelch the error in case we had a non-object file
+  consumeError(OptionalObject.takeError());
+  return getDefaultForHost();
+}
+
+int performWriteOperation(object::Archive *OldArchive,
+                          std::unique_ptr<MemoryBuffer> OldArchiveBuf) {
+  std::vector<NewArchiveMember> NewMembers;
+  if (int Status = computeNewArchiveMembers(OldArchive, NewMembers))
+    return Status;
+
+  object::Archive::Kind Kind;
+  if (Thin)
+    Kind = object::Archive::K_GNU;
+  else if (OldArchive)
+    Kind = OldArchive->kind();
+  else
+    Kind = getKindFromMember(NewMembers.front());
+
+  const auto Result =
+      writeArchive(ArchiveName, NewMembers, Symtab, Kind, Deterministic, Thin,
+                   std::move(OldArchiveBuf));
+
+  if (Result.second) {
+    fail(Result.second, Result.first);
+    return 1;
+  }
+
+  return 0;
+}
+
+int performWriteOperation() {
+  // Create or open the archive object.
+  auto Buf = MemoryBuffer::getFile(ArchiveName, -1, false);
+  std::error_code EC = Buf.getError();
+  if (EC && EC != errc::no_such_file_or_directory) {
+    fail("error opening '" + ArchiveName + "': " + EC.message() + "!");
+    return 1;
+  }
+
+  if (!EC) {
+    Error Err = Error::success();
+    object::Archive Archive(Buf.get()->getMemBufferRef(), Err);
+    EC = errorToErrorCode(std::move(Err));
+    if (EC) {
+      fail(
+          EC,
+          ("error loading '" + ArchiveName + "': " + EC.message() + "!").str());
+      return 1;
+    }
+    return performWriteOperation(&Archive, std::move(Buf.get()));
+  }
+
+  assert(EC == errc::no_such_file_or_directory);
+
+  return performWriteOperation(nullptr, nullptr);
+}
+
+} // anonymous namespace
+
+////////////////////////////////////////////////////////////////////////////////
+
+namespace ldc {
+
+int ar(ArrayRef<const char *> args) {
+  if (args.size() < 4 || strcmp(args[0], "llvm-ar") != 0 ||
+      strcmp(args[1], "rcs") != 0) {
+    llvm_unreachable(
+        "Expected archiver command line: llvm-ar rcs <archive file> "
+        "<object file> ...");
+    return -1;
+  }
+
+  ArchiveName = args[2];
+
+  auto membersSlice = args.slice(3);
+  Members.clear();
+  Members.insert(Members.end(), membersSlice.begin(), membersSlice.end());
+
+  return performWriteOperation();
+}
+
+int lib(ArrayRef<const char *> args) {
+  if (args.size() < 1 || strcmp(args[0], "llvm-lib.exe") != 0) {
+    llvm_unreachable("Expected archiver command line: llvm-lib.exe ...");
+    return -1;
+  }
+
+  return libDriverMain(args);
+}
+
+} // namespace ldc
+
+#endif // LDC_LLVM_VER >= 309

driver/archiver.h

@@ -0,0 +1,27 @@
+//===-- driver/archiver.h - Creating static libs via LLVM--------*- C++ -*-===//
+//
+//                         LDC – the LLVM D compiler
+//
+// This file is distributed under the BSD-style LDC license. See the LICENSE
+// file for details.
+//
+//===----------------------------------------------------------------------===//
+//
+// Provides an interface to LLVM built-in static lib generation via llvm-lib
+// (MSVC targets) or llvm-ar (all other targets).
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LDC_DRIVER_ARCHIVER_H
+#define LDC_DRIVER_ARCHIVER_H
+
+#if LDC_LLVM_VER >= 309
+#include "llvm/ADT/ArrayRef.h"
+
+namespace ldc {
+int ar(llvm::ArrayRef<const char *> args);
+int lib(llvm::ArrayRef<const char *> args);
+}
+#endif // LDC_LLVM_VER >= 309
+
+#endif // !LDC_DRIVER_ARCHIVER_H

driver/linker.cpp

@@ -11,6 +11,7 @@
 #include "mars.h"
 #include "module.h"
 #include "root.h"
+#include "driver/archiver.h"
 #include "driver/cl_options.h"
 #include "driver/exe_path.h"
 #include "driver/tool.h"
@@ -51,6 +52,11 @@ static llvm::cl::opt<std::string>
                               "LLVMgold.so (Unixes) or libLTO.dylib (Darwin))"),
                llvm::cl::value_desc("file"), llvm::cl::ZeroOrMore);
 
+static llvm::cl::opt<std::string>
+    externalArchiver("archiver",
+                     llvm::cl::desc("External static library archiver"),
+                     llvm::cl::value_desc("file"), llvm::cl::ZeroOrMore);
+
 //////////////////////////////////////////////////////////////////////////////
 
 static void CreateDirectoryOnDisk(llvm::StringRef fileName) {
@@ -815,8 +821,21 @@ int createStaticLibrary() {
   const bool isTargetMSVC =
       global.params.targetTriple->isWindowsMSVCEnvironment();
 
+#if LDC_LLVM_VER >= 309
+  const bool useInternalArchiver = externalArchiver.empty();
+#else
+  const bool useInternalArchiver = false;
+#endif
+
   // find archiver
-  std::string tool(isTargetMSVC ? "lib.exe" : getArchiver());
+  std::string tool;
+  if (useInternalArchiver) {
+    tool = isTargetMSVC ? "llvm-lib.exe" : "llvm-ar";
+  } else if (!externalArchiver.empty()) {
+    tool = externalArchiver;
+  } else {
+    tool = isTargetMSVC ? "lib.exe" : getArchiver();
+  }
 
   // build arguments
   std::vector<std::string> args;
@@ -869,13 +888,35 @@ int createStaticLibrary() {
   // create path to the library
   CreateDirectoryOnDisk(libName);
 
-  // try to call archiver
-  int exitCode;
-  if (isTargetMSVC) {
-    exitCode = executeMsvcToolAndWait(tool, args, global.params.verbose);
-  } else {
-    exitCode = executeToolAndWait(tool, args, global.params.verbose);
+#if LDC_LLVM_VER >= 309
+  if (useInternalArchiver) {
+    std::vector<const char *> fullArgs;
+    fullArgs.reserve(1 + args.size());
+    fullArgs.push_back(tool.c_str());
+    for (const auto &arg : args)
+      fullArgs.push_back(arg.c_str());
+
+    if (global.params.verbose) {
+      for (auto arg : fullArgs) {
+        fprintf(global.stdmsg, "%s ", arg);
       }
+      fprintf(global.stdmsg, "\n");
+      fflush(global.stdmsg);
+    }
+
+    const int exitCode = isTargetMSVC ? ldc::lib(fullArgs) : ldc::ar(fullArgs);
+    if (exitCode)
+      error(Loc(), "%s failed with status: %d", tool.c_str(), exitCode);
+
+    return exitCode;
+  }
+#endif
+
+  // try to call archiver
+  const int exitCode =
+      isTargetMSVC ? executeMsvcToolAndWait(tool, args, global.params.verbose)
+                   : executeToolAndWait(tool, args, global.params.verbose);
+
   return exitCode;
 }