Merge topic 'clang-ipo-support'

dca9c33abc Tests: Remove old IPO test
c856d4556b bindexplib: supporting llvm bitcode formats using llvm-nm
079b8e2916 Clang: prefer lld-link over link.exe
6e3655db2c Clang: add LTO support for GNU-command line clang on windows

Acked-by: Kitware Robot <kwrobot@kitware.com>
Merge-request: !3527

Modules/CMakeFindBinUtils.cmake

@@ -69,7 +69,12 @@ if(("x${CMAKE_${_CMAKE_PROCESSING_LANGUAGE}_SIMULATE_ID}" STREQUAL "xMSVC" AND
    OR (CMAKE_GENERATOR MATCHES "Visual Studio"
        AND NOT CMAKE_VS_PLATFORM_NAME STREQUAL "Tegra-Android"))
 
-  find_program(CMAKE_LINKER NAMES link HINTS ${_CMAKE_TOOLCHAIN_LOCATION})
+  if("x${CMAKE_${_CMAKE_PROCESSING_LANGUAGE}_COMPILER_ID}" STREQUAL "xClang")
+    find_program(CMAKE_NM NAMES ${_CMAKE_TOOLCHAIN_PREFIX}nm llvm-nm HINTS ${_CMAKE_TOOLCHAIN_LOCATION})
+    set(_CMAKE_ADDITIONAL_LINKER_NAMES "lld-link")
+  endif()
+
+  find_program(CMAKE_LINKER NAMES ${_CMAKE_ADDITIONAL_LINKER_NAMES} link HINTS ${_CMAKE_TOOLCHAIN_LOCATION})
   find_program(CMAKE_MT     NAMES mt   HINTS ${_CMAKE_TOOLCHAIN_LOCATION})
 
   list(APPEND _CMAKE_TOOL_VARS LINKER MT)
@@ -115,6 +120,17 @@ else()
 
   list(APPEND _CMAKE_TOOL_VARS AR RANLIB STRIP LINKER NM OBJDUMP OBJCOPY READELF DLLTOOL ADDR2LINE)
 
+
+  unset(_CMAKE_ADDITIONAL_AR_NAMES)
+  unset(_CMAKE_ADDITIONAL_RANLIB_NAMES)
+  unset(_CMAKE_ADDITIONAL_STRIP_NAMES)
+  unset(_CMAKE_ADDITIONAL_LINKER_NAMES)
+  unset(_CMAKE_ADDITIONAL_NM_NAMES)
+  unset(_CMAKE_ADDITIONAL_OBJDUMP_NAMES)
+  unset(_CMAKE_ADDITIONAL_OBJCOPY_NAMES)
+  unset(_CMAKE_ADDITIONAL_READELF_NAMES)
+  unset(_CMAKE_ADDITIONAL_DLLTOOL_NAMES)
+  unset(_CMAKE_ADDITIONAL_ADDR2LINE_NAMES)
 endif()
 
 if(CMAKE_PLATFORM_HAS_INSTALLNAME)

Modules/Platform/Windows-Clang.cmake

@@ -47,15 +47,22 @@ macro(__windows_compiler_clang_gnu lang)
   set(CMAKE_${lang}_USE_RESPONSE_FILE_FOR_LIBRARIES 1)
   set(CMAKE_${lang}_USE_RESPONSE_FILE_FOR_INCLUDES 1)
 
+  set(CMAKE_${lang}_COMPILE_OPTIONS_IPO "-flto")
+  set(_CMAKE_${lang}_IPO_SUPPORTED_BY_CMAKE YES)
+  set(_CMAKE_${lang}_IPO_MAY_BE_SUPPORTED_BY_COMPILER YES)
+  set(CMAKE_${lang}_ARCHIVE_CREATE_IPO "<CMAKE_AR> cr <TARGET> <LINK_FLAGS> <OBJECTS>")
+  set(CMAKE_${lang}_ARCHIVE_APPEND_IPO "<CMAKE_AR> r <TARGET> <LINK_FLAGS> <OBJECTS>")
+  set(CMAKE_${lang}_ARCHIVE_FINISH_IPO "<CMAKE_RANLIB> <TARGET>")
+
   # Create archiving rules to support large object file lists for static libraries.
   set(CMAKE_${lang}_ARCHIVE_CREATE "<CMAKE_AR> qc <TARGET> <LINK_FLAGS> <OBJECTS>")
   set(CMAKE_${lang}_ARCHIVE_APPEND "<CMAKE_AR> q  <TARGET> <LINK_FLAGS> <OBJECTS>")
   set(CMAKE_${lang}_ARCHIVE_FINISH "<CMAKE_RANLIB> <TARGET>")
   set(CMAKE_${lang}_CREATE_SHARED_LIBRARY
-    "<CMAKE_${lang}_COMPILER> -nostartfiles -nostdlib <CMAKE_SHARED_LIBRARY_${lang}_FLAGS> <LANGUAGE_COMPILE_FLAGS> <LINK_FLAGS> <CMAKE_SHARED_LIBRARY_CREATE_${lang}_FLAGS> -o <TARGET> ${CMAKE_GNULD_IMAGE_VERSION} -Xlinker /implib:<TARGET_IMPLIB> -Xlinker /pdb:<TARGET_PDB> -Xlinker /version:<TARGET_VERSION_MAJOR>.<TARGET_VERSION_MINOR> <OBJECTS> <LINK_LIBRARIES>")
+    "<CMAKE_${lang}_COMPILER> -fuse-ld=lld-link -nostartfiles -nostdlib <CMAKE_SHARED_LIBRARY_${lang}_FLAGS> <LANGUAGE_COMPILE_FLAGS> <LINK_FLAGS> <CMAKE_SHARED_LIBRARY_CREATE_${lang}_FLAGS> -o <TARGET> ${CMAKE_GNULD_IMAGE_VERSION} -Xlinker /implib:<TARGET_IMPLIB> -Xlinker /pdb:<TARGET_PDB> -Xlinker /version:<TARGET_VERSION_MAJOR>.<TARGET_VERSION_MINOR> <OBJECTS> <LINK_LIBRARIES>")
   set(CMAKE_${lang}_CREATE_SHARED_MODULE ${CMAKE_${lang}_CREATE_SHARED_LIBRARY})
   set(CMAKE_${lang}_LINK_EXECUTABLE
-    "<CMAKE_${lang}_COMPILER> -nostartfiles -nostdlib <FLAGS> <CMAKE_${lang}_LINK_FLAGS> <LINK_FLAGS> <OBJECTS>  -o <TARGET> -Xlinker /implib:<TARGET_IMPLIB> -Xlinker /pdb:<TARGET_PDB> -Xlinker /version:<TARGET_VERSION_MAJOR>.<TARGET_VERSION_MINOR> ${CMAKE_GNULD_IMAGE_VERSION} <LINK_LIBRARIES>")
+    "<CMAKE_${lang}_COMPILER> -fuse-ld=lld-link -nostartfiles -nostdlib <FLAGS> <CMAKE_${lang}_LINK_FLAGS> <LINK_FLAGS> <OBJECTS>  -o <TARGET> -Xlinker /implib:<TARGET_IMPLIB> -Xlinker /pdb:<TARGET_PDB> -Xlinker /version:<TARGET_VERSION_MAJOR>.<TARGET_VERSION_MINOR> ${CMAKE_GNULD_IMAGE_VERSION} <LINK_LIBRARIES>")
 
   set(CMAKE_${lang}_COMPILE_OPTIONS_MSVC_RUNTIME_LIBRARY_MultiThreaded         -Xclang -flto-visibility-public-std -D_MT -Xclang --dependent-lib=libcmt)
   set(CMAKE_${lang}_COMPILE_OPTIONS_MSVC_RUNTIME_LIBRARY_MultiThreadedDLL      -D_DLL -D_MT -Xclang --dependent-lib=msvcrt)

Source/bindexplib.cxx

@@ -64,9 +64,12 @@
  */
 #include "bindexplib.h"
 
+#include "cmSystemTools.h"
 #include "cmsys/Encoding.hxx"
 #include "cmsys/FStream.hxx"
 #include <iostream>
+#include <sstream>
+#include <vector>
 #include <windows.h>
 
 #ifndef IMAGE_FILE_MACHINE_ARM
@@ -301,7 +304,63 @@ private:
   bool IsI386;
 };
 
-bool DumpFile(const char* filename, std::set<std::string>& symbols,
+bool DumpFileWithLlvmNm(std::string const& nmPath, const char* filename,
+                        std::set<std::string>& symbols,
+                        std::set<std::string>& dataSymbols)
+{
+  std::string output;
+  // break up command line into a vector
+  std::vector<std::string> command;
+  command.push_back(nmPath);
+  command.push_back("--no-weak");
+  command.push_back("--defined-only");
+  command.push_back("--format=posix");
+  command.push_back(filename);
+
+  // run the command
+  int exit_code = 0;
+  cmSystemTools::RunSingleCommand(command, &output, &output, &exit_code, "",
+                                  cmSystemTools::OUTPUT_NONE);
+
+  if (exit_code != 0) {
+    fprintf(stderr, "llvm-nm returned an error: %s\n", output.c_str());
+    return false;
+  }
+
+  std::istringstream ss(output);
+  std::string line;
+  while (std::getline(ss, line)) {
+    if (line.empty()) { // last line
+      continue;
+    }
+    size_t sym_end = line.find(" ");
+    if (sym_end == std::string::npos) {
+      fprintf(stderr, "Couldn't parse llvm-nm output line: %s\n",
+              line.c_str());
+      return false;
+    }
+    if (line.size() < sym_end + 1) {
+      fprintf(stderr, "Couldn't parse llvm-nm output line: %s\n",
+              line.c_str());
+      return false;
+    }
+    const std::string sym = line.substr(0, sym_end);
+    const char sym_type = line[sym_end + 1];
+    switch (sym_type) {
+      case 'D':
+        dataSymbols.insert(sym);
+        break;
+      case 'T':
+        symbols.insert(sym);
+        break;
+    }
+  }
+
+  return true;
+}
+
+bool DumpFile(std::string const& nmPath, const char* filename,
+              std::set<std::string>& symbols,
               std::set<std::string>& dataSymbols)
 {
   HANDLE hFile;
@@ -356,16 +415,26 @@ bool DumpFile(const char* filename, std::set<std::string>& symbols,
         (imageHeader->Machine == IMAGE_FILE_MACHINE_I386));
       symbolDumper.DumpObjFile();
     } else {
-      // check for /bigobj format
+      // check for /bigobj and llvm LTO format
       cmANON_OBJECT_HEADER_BIGOBJ* h =
         (cmANON_OBJECT_HEADER_BIGOBJ*)lpFileBase;
       if (h->Sig1 == 0x0 && h->Sig2 == 0xffff) {
+        // bigobj
         DumpSymbols<cmANON_OBJECT_HEADER_BIGOBJ, cmIMAGE_SYMBOL_EX>
           symbolDumper((cmANON_OBJECT_HEADER_BIGOBJ*)lpFileBase, symbols,
                        dataSymbols, (h->Machine == IMAGE_FILE_MACHINE_I386));
         symbolDumper.DumpObjFile();
+      } else if (
+        // BCexCODE - llvm bitcode
+        (h->Sig1 == 0x4342 && h->Sig2 == 0xDEC0) ||
+        // 0x0B17C0DE - llvm bitcode BC wrapper
+        (h->Sig1 == 0x0B17 && h->Sig2 == 0xC0DE)) {
+
+        return DumpFileWithLlvmNm(nmPath, filename, symbols, dataSymbols);
+
       } else {
-        printf("unrecognized file format in '%s'\n", filename);
+        printf("unrecognized file format in '%s, %u'\n", filename,
+               imageHeader->Machine);
         return false;
       }
     }
@@ -378,7 +447,7 @@ bool DumpFile(const char* filename, std::set<std::string>& symbols,
 
 bool bindexplib::AddObjectFile(const char* filename)
 {
-  return DumpFile(filename, this->Symbols, this->DataSymbols);
+  return DumpFile(NmPath, filename, this->Symbols, this->DataSymbols);
 }
 
 bool bindexplib::AddDefinitionFile(const char* filename)
@@ -419,3 +488,8 @@ void bindexplib::WriteFile(FILE* file)
     fprintf(file, "\t%s\n", s.c_str());
   }
 }
+
+void bindexplib::SetNmPath(std::string const& nm)
+{
+  NmPath = nm;
+}

Source/bindexplib.h

@@ -12,13 +12,16 @@
 class bindexplib
 {
 public:
-  bindexplib() {}
+  bindexplib() { NmPath = "nm"; }
   bool AddDefinitionFile(const char* filename);
   bool AddObjectFile(const char* filename);
   void WriteFile(FILE* file);
 
+  void SetNmPath(std::string const& nm);
+
 private:
   std::set<std::string> Symbols;
   std::set<std::string> DataSymbols;
+  std::string NmPath;
 };
 #endif

Source/cmMakefileTargetGenerator.cxx

@@ -15,6 +15,7 @@
 #include "cmGeneratorExpression.h"
 #include "cmGeneratorTarget.h"
 #include "cmGlobalUnixMakefileGenerator3.h"
+#include "cmLocalCommonGenerator.h"
 #include "cmLocalUnixMakefileGenerator3.h"
 #include "cmMakefile.h"
 #include "cmMakefileExecutableTargetGenerator.h"
@@ -1756,6 +1757,12 @@ void cmMakefileTargetGenerator::GenDefFile(
     this->LocalGenerator->MaybeConvertToRelativePath(
       this->LocalGenerator->GetCurrentBinaryDirectory(), objlist_file),
     cmOutputConverter::SHELL);
+  const char* nm_executable = this->Makefile->GetDefinition("CMAKE_NM");
+  if (nm_executable && *nm_executable) {
+    cmd += " --nm=";
+    cmd += this->LocalCommonGenerator->ConvertToOutputFormat(
+      nm_executable, cmOutputConverter::SHELL);
+  }
   real_link_commands.insert(real_link_commands.begin(), cmd);
   // create a list of obj files for the -E __create_def to read
   cmGeneratedFileStream fout(objlist_file);

Source/cmNinjaNormalTargetGenerator.cxx

@@ -20,6 +20,7 @@
 #include "cmGlobalNinjaGenerator.h"
 #include "cmLinkLineComputer.h"
 #include "cmLinkLineDeviceComputer.h"
+#include "cmLocalCommonGenerator.h"
 #include "cmLocalGenerator.h"
 #include "cmLocalNinjaGenerator.h"
 #include "cmMakefile.h"
@@ -976,6 +977,13 @@ void cmNinjaNormalTargetGenerator::WriteLinkStatement()
     std::string obj_list_file = mdi->DefFile + ".objs";
     cmd += this->GetLocalGenerator()->ConvertToOutputFormat(
       obj_list_file, cmOutputConverter::SHELL);
+
+    const char* nm_executable = GetMakefile()->GetDefinition("CMAKE_NM");
+    if (nm_executable && *nm_executable) {
+      cmd += " --nm=";
+      cmd += this->LocalCommonGenerator->ConvertToOutputFormat(
+        nm_executable, cmOutputConverter::SHELL);
+    }
     preLinkCmdLines.push_back(std::move(cmd));
 
     // create a list of obj files for the -E __create_def to read

Source/cmcmd.cxx

@@ -558,8 +558,8 @@ int cmcmd::ExecuteCMakeCommand(std::vector<std::string> const& args)
 #if defined(_WIN32) && !defined(CMAKE_BOOTSTRAP)
     else if (args[1] == "__create_def") {
       if (args.size() < 4) {
-        std::cerr
-          << "__create_def Usage: -E __create_def outfile.def objlistfile\n";
+        std::cerr << "__create_def Usage: -E __create_def outfile.def "
+                     "objlistfile [-nm=nm-path]\n";
         return 1;
       }
       FILE* fout = cmsys::SystemTools::Fopen(args[2].c_str(), "w+");
@@ -576,6 +576,15 @@ int cmcmd::ExecuteCMakeCommand(std::vector<std::string> const& args)
       }
       std::string file;
       bindexplib deffile;
+      if (args.size() >= 5) {
+        auto a = args[4];
+        if (cmHasLiteralPrefix(a, "--nm=")) {
+          deffile.SetNmPath(a.substr(5));
+          std::cerr << a.substr(5) << "\n";
+        } else {
+          std::cerr << "unknown argument: " << a << "\n";
+        }
+      }
       while (cmSystemTools::GetLineFromStream(fin, file)) {
         std::string const& ext = cmSystemTools::GetFilenameLastExtension(file);
         if (cmSystemTools::LowerCase(ext) == ".def") {

Tests/CMakeLists.txt

@@ -416,7 +416,6 @@ if(BUILD_TESTING)
   ADD_TEST_MACRO(COnly COnly)
   ADD_TEST_MACRO(CxxOnly CxxOnly)
   ADD_TEST_MACRO(CxxSubdirC CxxSubdirC)
-  ADD_TEST_MACRO(IPO COnly/COnly)
   ADD_TEST_MACRO(OutDir runtime/OutDir)
   ADD_TEST_MACRO(OutName exe.OutName.exe)
   ADD_TEST_MACRO(ObjectLibrary UseCshared)

Tests/IPO/CMakeLists.txt

@@ -1,7 +0,0 @@
-cmake_minimum_required (VERSION 2.8)
-project(IPO NONE)
-
-set_property(DIRECTORY PROPERTY INTERPROCEDURAL_OPTIMIZATION 1)
-
-add_subdirectory(../COnly COnly)
-add_subdirectory(../CxxOnly CxxOnly)

Tests/Module/CheckIPOSupported-C/CMakeLists.txt

@@ -13,8 +13,18 @@ elseif(CMake_TEST_IPO_WORKS_C)
 endif()
 
 add_library(foo foo.c)
+if(NOT CYGWIN AND (NOT WIN32 OR "x${CMAKE_C_COMPILER_ID}" STREQUAL "xClang"))
+  add_library(bar SHARED bar.c)
+  if(WIN32)
+    # Bindexplib for clang supports LTO objects
+    set_target_properties(bar PROPERTIES WINDOWS_EXPORT_ALL_SYMBOLS ON)
+  endif()
+else()
+  # TODO: bindexplib doesn't support exporting IPO symbols with other compilers on Windows
+  add_library(bar STATIC bar.c)
+endif()
 add_executable(CheckIPOSupported-C main.c)
-target_link_libraries(CheckIPOSupported-C PUBLIC foo)
+target_link_libraries(CheckIPOSupported-C PUBLIC foo bar)
 
 enable_testing()
 add_test(NAME CheckIPOSupported-C COMMAND CheckIPOSupported-C)

Tests/Module/CheckIPOSupported-C/bar.c

@@ -0,0 +1,4 @@
+int bar()
+{
+  return 0x42;
+}

Tests/Module/CheckIPOSupported-C/main.c

@@ -1,8 +1,9 @@
 int foo();
+int bar();
 
 int main()
 {
-  if (foo() == 0) {
+  if (foo() != bar()) {
     return 1;
   }
   return 0;

Tests/Module/CheckIPOSupported-CXX/CMakeLists.txt

@@ -12,9 +12,20 @@ elseif(CMake_TEST_IPO_WORKS_CXX)
   message(FATAL_ERROR "IPO expected to work, but the check failed:\n  ${ipo_output}")
 endif()
 
-add_library(foo foo.cpp)
+
+add_library(foo STATIC foo.cpp)
+if(NOT CYGWIN AND (NOT WIN32 OR "x${CMAKE_CXX_COMPILER_ID}" STREQUAL "Clang"))
+  add_library(bar SHARED bar.cpp)
+  if(WIN32)
+    # Bindexplib for clang supports LTO objects
+    set_target_properties(bar PROPERTIES WINDOWS_EXPORT_ALL_SYMBOLS ON)
+  endif()
+else()
+  # TODO: bindexplib doesn't support exporting IPO symbols with other compilers on Windows
+  add_library(bar STATIC bar.cpp)
+endif()
 add_executable(CheckIPOSupported-CXX main.cpp)
-target_link_libraries(CheckIPOSupported-CXX PUBLIC foo)
+target_link_libraries(CheckIPOSupported-CXX PUBLIC foo bar)
 
 enable_testing()
 add_test(NAME CheckIPOSupported-CXX COMMAND CheckIPOSupported-CXX)

Tests/Module/CheckIPOSupported-CXX/bar.cpp

@@ -0,0 +1,4 @@
+int bar()
+{
+  return 0x42;
+}

Tests/Module/CheckIPOSupported-CXX/main.cpp

@@ -1,8 +1,9 @@
 int foo();
+int bar();
 
 int main()
 {
-  if (foo() == 0) {
+  if (foo() != bar()) {
     return 1;
   }
   return 0;