From 7114c141e27532df709fcae1266bb4ce9b6e850c Mon Sep 17 00:00:00 2001 From: Cristian Adam Date: Fri, 26 Jul 2019 16:08:32 +0200 Subject: [PATCH] Unity build: Add support for Ninja and Makefile generators --- Help/manual/cmake-properties.7.rst | 5 ++ Help/manual/cmake-variables.7.rst | 2 + Help/prop_sf/SKIP_UNITY_BUILD_INCLUSION.rst | 7 ++ Help/prop_tgt/UNITY_BUILD.rst | 55 ++++++++++++ Help/prop_tgt/UNITY_BUILD_BATCH_SIZE.rst | 13 +++ .../UNITY_BUILD_CODE_AFTER_INCLUDE.rst | 8 ++ .../UNITY_BUILD_CODE_BEFORE_INCLUDE.rst | 8 ++ Help/release/dev/unity-build.rst | 6 ++ Help/variable/CMAKE_UNITY_BUILD.rst | 6 ++ .../variable/CMAKE_UNITY_BUILD_BATCH_SIZE.rst | 6 ++ Source/cmLocalGenerator.cxx | 85 +++++++++++++++++++ Source/cmLocalGenerator.h | 1 + .../cmMakefileExecutableTargetGenerator.cxx | 1 + Source/cmMakefileLibraryTargetGenerator.cxx | 1 + Source/cmMakefileUtilityTargetGenerator.cxx | 1 + Source/cmNinjaNormalTargetGenerator.cxx | 1 + Source/cmTarget.cxx | 2 + 17 files changed, 208 insertions(+) create mode 100644 Help/prop_sf/SKIP_UNITY_BUILD_INCLUSION.rst create mode 100644 Help/prop_tgt/UNITY_BUILD.rst create mode 100644 Help/prop_tgt/UNITY_BUILD_BATCH_SIZE.rst create mode 100644 Help/prop_tgt/UNITY_BUILD_CODE_AFTER_INCLUDE.rst create mode 100644 Help/prop_tgt/UNITY_BUILD_CODE_BEFORE_INCLUDE.rst create mode 100644 Help/release/dev/unity-build.rst create mode 100644 Help/variable/CMAKE_UNITY_BUILD.rst create mode 100644 Help/variable/CMAKE_UNITY_BUILD_BATCH_SIZE.rst diff --git a/Help/manual/cmake-properties.7.rst b/Help/manual/cmake-properties.7.rst index 25d6b247da..19afb7d57d 100644 --- a/Help/manual/cmake-properties.7.rst +++ b/Help/manual/cmake-properties.7.rst @@ -323,6 +323,10 @@ Properties on Targets /prop_tgt/Swift_MODULE_DIRECTORY /prop_tgt/Swift_MODULE_NAME /prop_tgt/TYPE + /prop_tgt/UNITY_BUILD + /prop_tgt/UNITY_BUILD_BATCH_SIZE + /prop_tgt/UNITY_BUILD_CODE_AFTER_INCLUDE + /prop_tgt/UNITY_BUILD_CODE_BEFORE_INCLUDE /prop_tgt/VERSION /prop_tgt/VISIBILITY_INLINES_HIDDEN /prop_tgt/VS_CONFIGURATION_TYPE @@ -450,6 +454,7 @@ Properties on Source Files /prop_sf/SKIP_AUTORCC /prop_sf/SKIP_AUTOUIC /prop_sf/SKIP_PRECOMPILE_HEADERS + /prop_sf/SKIP_UNITY_BUILD_INCLUSION /prop_sf/Swift_DEPENDENCIES_FILE /prop_sf/Swift_DIAGNOSTICS_FILE /prop_sf/SYMBOLIC diff --git a/Help/manual/cmake-variables.7.rst b/Help/manual/cmake-variables.7.rst index 0ae1ebe1de..739e4b5ffd 100644 --- a/Help/manual/cmake-variables.7.rst +++ b/Help/manual/cmake-variables.7.rst @@ -431,6 +431,8 @@ Variables that Control the Build /variable/CMAKE_TRY_COMPILE_CONFIGURATION /variable/CMAKE_TRY_COMPILE_PLATFORM_VARIABLES /variable/CMAKE_TRY_COMPILE_TARGET_TYPE + /variable/CMAKE_UNITY_BUILD + /variable/CMAKE_UNITY_BUILD_BATCH_SIZE /variable/CMAKE_USE_RELATIVE_PATHS /variable/CMAKE_VISIBILITY_INLINES_HIDDEN /variable/CMAKE_VS_GLOBALS diff --git a/Help/prop_sf/SKIP_UNITY_BUILD_INCLUSION.rst b/Help/prop_sf/SKIP_UNITY_BUILD_INCLUSION.rst new file mode 100644 index 0000000000..53f3970256 --- /dev/null +++ b/Help/prop_sf/SKIP_UNITY_BUILD_INCLUSION.rst @@ -0,0 +1,7 @@ +SKIP_UNITY_BUILD_INCLUSION +-------------------------- + +Is this source file skipped by :prop_tgt:`UNITY_BUILD` feature. + +This property helps with "ODR (One definition rule)" problems +that one would run into when using an :prop_tgt:`UNITY_BUILD`. diff --git a/Help/prop_tgt/UNITY_BUILD.rst b/Help/prop_tgt/UNITY_BUILD.rst new file mode 100644 index 0000000000..d326ee2466 --- /dev/null +++ b/Help/prop_tgt/UNITY_BUILD.rst @@ -0,0 +1,55 @@ +UNITY_BUILD +----------- + +Should the target source files be processed into batches for +faster compilation. This feature is known as "Unity build", +or "Jumbo build". + +The `C` and `CXX` source files are grouped separately. + +This property is initialized by the value of the +:variable:`CMAKE_UNITY_BUILD` variable if it is set when +a target is created. + +.. note :: + + It's not recommended to directly set :prop_tgt:`UNITY_BUILD` + to `ON`, but to instead set :variable:`CMAKE_UNITY_BUILD` from + the command line. However, it IS recommended to set + :prop_tgt:`UNITY_BUILD` to `OFF` if you need to ensure that a + target doesn't get a unity build. + +The batch size can be specified by setting +:prop_tgt:`UNITY_BUILD_BATCH_SIZE`. + +The batching of source files is done by adding new sources files +wich will `#include` the source files, and exclude them from +building by setting :prop_sf:`HEADER_FILE_ONLY` to `ON`. + + +ODR (One definition rule) errors +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +Since multiple source files are included into one source file, +it can lead to ODR errors. This section contains properites +which help fixing these errors. + +The source files marked by :prop_sf:`GENERATED` will be skipped +from unity build. This applies also for the source files marked +with :prop_sf:`SKIP_UNITY_BUILD_INCLUSION`. + +The source files that have :prop_sf:`COMPILE_OPTIONS`, +:prop_sf:`COMPILE_DEFINITIONS`, :prop_sf:`COMPILE_FLAGS`, or +:prop_sf:`INCLUDE_DIRECTORIES` will also be skipped. + +With the :prop_tgt:`UNITY_BUILD_CODE_BEFORE_INCLUDE` and +:prop_tgt:`UNITY_BUILD_CODE_AFTER_INCLUDE` one can specify code +to be injected in the unity source file before and after every +`#include` statement. + +.. note :: + + The order of source files defined in the `CMakeLists.txt` will + be preserved into the generated unity source files. This can + be used to manually enforce a specific grouping based on the + :prop_tgt:`UNITY_BUILD_BATCH_SIZE`. diff --git a/Help/prop_tgt/UNITY_BUILD_BATCH_SIZE.rst b/Help/prop_tgt/UNITY_BUILD_BATCH_SIZE.rst new file mode 100644 index 0000000000..24266894b9 --- /dev/null +++ b/Help/prop_tgt/UNITY_BUILD_BATCH_SIZE.rst @@ -0,0 +1,13 @@ +UNITY_BUILD_BATCH_SIZE +---------------------- + +Specifies how many source code files will be included into a +:prop_tgt:`UNITY_BUILD` source file. + +If the property is not set, CMake will use the value provided +by :variable:`CMAKE_UNITY_BUILD_BATCH_SIZE`. + +By setting it to value `0` the generated unity source file will +contain all the source files that would be otherwise be split +into multiple batches. It is not recommended to do so, since it +would affect performance. diff --git a/Help/prop_tgt/UNITY_BUILD_CODE_AFTER_INCLUDE.rst b/Help/prop_tgt/UNITY_BUILD_CODE_AFTER_INCLUDE.rst new file mode 100644 index 0000000000..779528902f --- /dev/null +++ b/Help/prop_tgt/UNITY_BUILD_CODE_AFTER_INCLUDE.rst @@ -0,0 +1,8 @@ +UNITY_BUILD_CODE_AFTER_INCLUDE +------------------------------ + +Code snippet which is included verbatim by the :prop_tgt:`UNITY_BUILD` +feature just after the `#include` statement of the targeted source +files. + +This could be something like `#undef NOMINMAX`. diff --git a/Help/prop_tgt/UNITY_BUILD_CODE_BEFORE_INCLUDE.rst b/Help/prop_tgt/UNITY_BUILD_CODE_BEFORE_INCLUDE.rst new file mode 100644 index 0000000000..f33546328a --- /dev/null +++ b/Help/prop_tgt/UNITY_BUILD_CODE_BEFORE_INCLUDE.rst @@ -0,0 +1,8 @@ +UNITY_BUILD_CODE_BEFORE_INCLUDE +------------------------------- + +Code snippet which is included verbatim by the :prop_tgt:`UNITY_BUILD` +feature just before the `#include` statement of the targeted source +files. + +This could be something like `#define NOMINMAX`. diff --git a/Help/release/dev/unity-build.rst b/Help/release/dev/unity-build.rst new file mode 100644 index 0000000000..293a3754d2 --- /dev/null +++ b/Help/release/dev/unity-build.rst @@ -0,0 +1,6 @@ +Unity build +----------- + +* The :prop_tgt:`UNITY_BUILD` target property was added to tell + generators to batch include source files for faster compilation + times. diff --git a/Help/variable/CMAKE_UNITY_BUILD.rst b/Help/variable/CMAKE_UNITY_BUILD.rst new file mode 100644 index 0000000000..3096954c20 --- /dev/null +++ b/Help/variable/CMAKE_UNITY_BUILD.rst @@ -0,0 +1,6 @@ +CMAKE_UNITY_BUILD +----------------- + +Default value for :prop_tgt:`UNITY_BUILD` of targets. + +By default ``CMAKE_UNITY_BUILD`` is ``OFF``. diff --git a/Help/variable/CMAKE_UNITY_BUILD_BATCH_SIZE.rst b/Help/variable/CMAKE_UNITY_BUILD_BATCH_SIZE.rst new file mode 100644 index 0000000000..3ab2344caa --- /dev/null +++ b/Help/variable/CMAKE_UNITY_BUILD_BATCH_SIZE.rst @@ -0,0 +1,6 @@ +CMAKE_UNITY_BUILD_BATCH_SIZE +---------------------------- + +Default value for :prop_tgt:`UNITY_BUILD_BATCH_SIZE` of targets. + +By default ``CMAKE_UNITY_BUILD_BATCH_SIZE`` is set to ``8``. diff --git a/Source/cmLocalGenerator.cxx b/Source/cmLocalGenerator.cxx index 7177694068..6fade3e5bc 100644 --- a/Source/cmLocalGenerator.cxx +++ b/Source/cmLocalGenerator.cxx @@ -38,6 +38,7 @@ #include #include +#include #include #include #include @@ -2202,6 +2203,90 @@ void cmLocalGenerator::AddPchDependencies(cmGeneratorTarget* target, } } +void cmLocalGenerator::AddUnityBuild(cmGeneratorTarget* target, + const std::string& config) +{ + if (!target->GetPropertyAsBool("UNITY_BUILD")) { + return; + } + + const std::string buildType = cmSystemTools::UpperCase(config); + + std::string filename_base = + cmStrCat(this->GetCurrentBinaryDirectory(), "/CMakeFiles/", + target->GetName(), ".dir/Unity/"); + + std::vector sources; + target->GetSourceFiles(sources, buildType); + + auto batchSizeString = target->GetProperty("UNITY_BUILD_BATCH_SIZE"); + const size_t unityBatchSize = + static_cast(std::atoi(batchSizeString)); + + auto beforeInclude = target->GetProperty("UNITY_BUILD_CODE_BEFORE_INCLUDE"); + auto afterInclude = target->GetProperty("UNITY_BUILD_CODE_AFTER_INCLUDE"); + + for (std::string lang : { "C", "CXX" }) { + std::vector filtered_sources; + std::copy_if(sources.begin(), sources.end(), + std::back_inserter(filtered_sources), [&](cmSourceFile* sf) { + return sf->GetLanguage() == lang && + !sf->GetPropertyAsBool("SKIP_UNITY_BUILD_INCLUSION") && + !sf->GetPropertyAsBool("GENERATED") && + !sf->GetProperty("COMPILE_OPTIONS") && + !sf->GetProperty("COMPILE_DEFINITIONS") && + !sf->GetProperty("COMPILE_FLAGS") && + !sf->GetProperty("INCLUDE_DIRECTORIES"); + }); + + size_t batchSize = unityBatchSize; + if (unityBatchSize == 0) { + batchSize = filtered_sources.size(); + } + + for (size_t itemsLeft = filtered_sources.size(), chunk = batchSize, + batch = 0; + itemsLeft > 0; itemsLeft -= chunk, ++batch) { + + chunk = std::min(itemsLeft, batchSize); + + std::string filename = cmStrCat(filename_base, "unity_", batch, + (lang == "C") ? ".c" : ".cxx"); + + const std::string filename_tmp = cmStrCat(filename, ".tmp"); + { + size_t begin = batch * batchSize; + size_t end = begin + chunk; + + cmGeneratedFileStream file( + filename_tmp, false, + this->GetGlobalGenerator()->GetMakefileEncoding()); + file << "/* generated by CMake */\n\n"; + + for (; begin != end; ++begin) { + cmSourceFile* sf = filtered_sources[begin]; + + sf->SetProperty("HEADER_FILE_ONLY", "ON"); + + if (beforeInclude) { + file << beforeInclude << "\n"; + } + + file << "#include \"" << sf->GetFullPath() << "\"\n"; + + if (afterInclude) { + file << afterInclude << "\n"; + } + } + } + cmSystemTools::CopyFileIfDifferent(filename_tmp, filename); + cmSystemTools::RemoveFile(filename_tmp); + + target->AddSource(filename, true); + } + } +} + void cmLocalGenerator::AppendIPOLinkerFlags(std::string& flags, cmGeneratorTarget* target, const std::string& config, diff --git a/Source/cmLocalGenerator.h b/Source/cmLocalGenerator.h index f63fe0f353..515ffae8b8 100644 --- a/Source/cmLocalGenerator.h +++ b/Source/cmLocalGenerator.h @@ -126,6 +126,7 @@ public: const std::string& rawFlag) const; void AddPchDependencies(cmGeneratorTarget* target, const std::string& config); + void AddUnityBuild(cmGeneratorTarget* target, const std::string& config); void AppendIPOLinkerFlags(std::string& flags, cmGeneratorTarget* target, const std::string& config, const std::string& lang); diff --git a/Source/cmMakefileExecutableTargetGenerator.cxx b/Source/cmMakefileExecutableTargetGenerator.cxx index bebd5c439a..002addffae 100644 --- a/Source/cmMakefileExecutableTargetGenerator.cxx +++ b/Source/cmMakefileExecutableTargetGenerator.cxx @@ -41,6 +41,7 @@ cmMakefileExecutableTargetGenerator::cmMakefileExecutableTargetGenerator( cm::make_unique(target, this->ConfigName); this->OSXBundleGenerator->SetMacContentFolders(&this->MacContentFolders); + this->LocalGenerator->AddUnityBuild(target, this->ConfigName); this->LocalGenerator->AddPchDependencies(target, this->ConfigName); } diff --git a/Source/cmMakefileLibraryTargetGenerator.cxx b/Source/cmMakefileLibraryTargetGenerator.cxx index 424440211f..d603dac1f3 100644 --- a/Source/cmMakefileLibraryTargetGenerator.cxx +++ b/Source/cmMakefileLibraryTargetGenerator.cxx @@ -43,6 +43,7 @@ cmMakefileLibraryTargetGenerator::cmMakefileLibraryTargetGenerator( cm::make_unique(target, this->ConfigName); this->OSXBundleGenerator->SetMacContentFolders(&this->MacContentFolders); + this->LocalGenerator->AddUnityBuild(target, this->ConfigName); this->LocalGenerator->AddPchDependencies(target, this->ConfigName); } diff --git a/Source/cmMakefileUtilityTargetGenerator.cxx b/Source/cmMakefileUtilityTargetGenerator.cxx index 556191f2b8..d4045b392e 100644 --- a/Source/cmMakefileUtilityTargetGenerator.cxx +++ b/Source/cmMakefileUtilityTargetGenerator.cxx @@ -26,6 +26,7 @@ cmMakefileUtilityTargetGenerator::cmMakefileUtilityTargetGenerator( cm::make_unique(target, this->ConfigName); this->OSXBundleGenerator->SetMacContentFolders(&this->MacContentFolders); + this->LocalGenerator->AddUnityBuild(target, this->ConfigName); this->LocalGenerator->AddPchDependencies(target, this->ConfigName); } diff --git a/Source/cmNinjaNormalTargetGenerator.cxx b/Source/cmNinjaNormalTargetGenerator.cxx index 0e3aa3ab6f..a4e1d61552 100644 --- a/Source/cmNinjaNormalTargetGenerator.cxx +++ b/Source/cmNinjaNormalTargetGenerator.cxx @@ -60,6 +60,7 @@ cmNinjaNormalTargetGenerator::cmNinjaNormalTargetGenerator( cm::make_unique(target, this->GetConfigName()); this->OSXBundleGenerator->SetMacContentFolders(&this->MacContentFolders); + GetLocalGenerator()->AddUnityBuild(target, this->GetConfigName()); GetLocalGenerator()->AddPchDependencies(target, this->GetConfigName()); } diff --git a/Source/cmTarget.cxx b/Source/cmTarget.cxx index 2a09b4332b..10ea7dd569 100644 --- a/Source/cmTarget.cxx +++ b/Source/cmTarget.cxx @@ -352,6 +352,8 @@ cmTarget::cmTarget(std::string const& name, cmStateEnums::TargetType type, initProp("Swift_MODULE_DIRECTORY"); initProp("VS_JUST_MY_CODE_DEBUGGING"); initProp("DISABLE_PRECOMPILE_HEADERS"); + initProp("UNITY_BUILD"); + initPropValue("UNITY_BUILD_BATCH_SIZE", "8"); #ifdef __APPLE__ if (this->GetGlobalGenerator()->IsXcode()) { initProp("XCODE_GENERATE_SCHEME");