From 20fe27cda19708f4bbc92b081d3d5eaaaee015ae Mon Sep 17 00:00:00 2001 From: David Cole Date: Mon, 26 Oct 2009 16:09:27 -0400 Subject: [PATCH] Add test to enable detecting differences between generation N and N+1 of a cmake generated build tree. Theoretically, each generation run with exactly the same arguments should be identical to the previous generation... Practically, there are differences. This is a tool to help us measure those differences. For now, the test always succeeds. Eventually, we may add the code to make it fail when there are differences. --- Tests/CMakeLists.txt | 15 ++ .../RunCMake.cmake | 165 ++++++++++++++++++ 2 files changed, 180 insertions(+) create mode 100644 Tests/CMakeTestMultipleConfigures/RunCMake.cmake diff --git a/Tests/CMakeLists.txt b/Tests/CMakeLists.txt index c3bf856206..dd39df0901 100644 --- a/Tests/CMakeLists.txt +++ b/Tests/CMakeLists.txt @@ -535,6 +535,21 @@ ${CMake_BINARY_DIR}/bin/cmake -DVERSION=CVS -P ${CMake_SOURCE_DIR}/Utilities/Rel "${CMake_BINARY_DIR}/Tests/CMakeTestAllGenerators") ENDIF(CTEST_RUN_CMakeTestAllGenerators) + if(NOT DEFINED CTEST_RUN_CMakeTestMultipleConfigures) + set(CTEST_RUN_CMakeTestMultipleConfigures ON) + endif(NOT DEFINED CTEST_RUN_CMakeTestMultipleConfigures) + + IF(CTEST_RUN_CMakeTestMultipleConfigures) + ADD_TEST(CMakeTestMultipleConfigures ${CMAKE_CMAKE_COMMAND} + -D dir=${CMake_BINARY_DIR}/Tests/CMakeTestMultipleConfigures + -D gen=${CMAKE_TEST_GENERATOR} + -D CMake_SOURCE_DIR=${CMake_SOURCE_DIR} + -P ${CMake_SOURCE_DIR}/Tests/CMakeTestMultipleConfigures/RunCMake.cmake + ) + LIST(APPEND TEST_BUILD_DIRS + "${CMake_BINARY_DIR}/Tests/CMakeTestMultipleConfigures") + ENDIF(CTEST_RUN_CMakeTestMultipleConfigures) + ADD_TEST(LoadedCommandOneConfig ${CMAKE_CTEST_COMMAND} --build-and-test "${CMake_SOURCE_DIR}/Tests/LoadCommandOneConfig" diff --git a/Tests/CMakeTestMultipleConfigures/RunCMake.cmake b/Tests/CMakeTestMultipleConfigures/RunCMake.cmake new file mode 100644 index 0000000000..19391d7f18 --- /dev/null +++ b/Tests/CMakeTestMultipleConfigures/RunCMake.cmake @@ -0,0 +1,165 @@ +if(NOT DEFINED CMake_SOURCE_DIR) + message(FATAL_ERROR "CMake_SOURCE_DIR not defined") +endif() + +if(NOT DEFINED dir) + message(FATAL_ERROR "dir not defined") +endif() + +if(NOT DEFINED gen) + message(FATAL_ERROR "gen not defined") +endif() + +# Call cmake once to get a baseline/reference output build tree: "Build". +# Then call cmake N more times, each time making a copy of the entire +# build tree after cmake is done configuring/generating. At the end, +# analyze the diffs in the generated build trees. Expect no diffs. +# +message(STATUS "CTEST_FULL_OUTPUT (Avoid ctest truncation of output)") + +set(N 7) + +# First setup source and binary trees: +# +execute_process(COMMAND ${CMAKE_COMMAND} -E remove_directory + ${dir}/Source +) + +execute_process(COMMAND ${CMAKE_COMMAND} -E remove_directory + ${dir}/Build +) + +execute_process(COMMAND ${CMAKE_COMMAND} -E copy_directory + ${CMake_SOURCE_DIR}/Tests/CTestTest/SmallAndFast + ${dir}/Source +) + +execute_process(COMMAND ${CMAKE_COMMAND} -E make_directory + ${dir}/Build +) + +# Patch SmallAndFast to build a .cxx executable too: +# +execute_process(COMMAND ${CMAKE_COMMAND} -E copy + ${dir}/Source/echoargs.c + ${dir}/Source/echoargs.cxx +) +file(APPEND "${dir}/Source/CMakeLists.txt" "\nadd_executable(echoargsCXX echoargs.cxx)\n") + +# Loop N times, saving a copy of the configured/generated build tree each time: +# +foreach(i RANGE 1 ${N}) + # Equivalent sequence of shell commands: + # + message(STATUS "${i}: cd Build && cmake -G \"${gen}\" ../Source && cd .. && cp -r Build b${i}") + + # Run cmake: + # + execute_process(COMMAND ${CMAKE_COMMAND} -G ${gen} ../Source + RESULT_VARIABLE result + OUTPUT_VARIABLE stdout + ERROR_VARIABLE stderr + WORKING_DIRECTORY ${dir}/Build + ) + + message(STATUS "result='${result}'") + message(STATUS "stdout='${stdout}'") + message(STATUS "stderr='${stderr}'") + message(STATUS "") + + # Save this iteration of the Build directory: + # + execute_process(COMMAND ${CMAKE_COMMAND} -E remove_directory + ${dir}/b${i} + ) + execute_process(COMMAND ${CMAKE_COMMAND} -E copy_directory + ${dir}/Build + ${dir}/b${i} + RESULT_VARIABLE result + OUTPUT_VARIABLE stdout + ERROR_VARIABLE stderr + ) + + message(STATUS "result='${result}'") + message(STATUS "stdout='${stdout}'") + message(STATUS "stderr='${stderr}'") + message(STATUS "") +endforeach() + + +# Function to analyze diffs between two directories. +# Set DIFF_EXECUTABLE before calling if 'diff' is available. +# +function(analyze_directory_diffs d1 d2 diff_count_var) + set(diffs 0) + + message(STATUS "Analyzing directory diffs between:") + message(STATUS " d1='${d1}'") + message(STATUS " d2='${d2}'") + + if(NOT "${d1}" STREQUAL "" AND NOT "${d2}" STREQUAL "") + message(STATUS "info: analyzing directories") + + file(GLOB_RECURSE files1 RELATIVE "${d1}" "${d1}/*") + file(GLOB_RECURSE files2 RELATIVE "${d2}" "${d2}/*") + + if("${files1}" STREQUAL "${files2}") + message(STATUS "info: file lists the same") + #message(STATUS " files='${files1}'") + + foreach(f ${files1}) + execute_process(COMMAND ${CMAKE_COMMAND} -E compare_files + ${d1}/${f} + ${d2}/${f} + RESULT_VARIABLE result + OUTPUT_VARIABLE stdout + ERROR_VARIABLE stderr + ) + if(result STREQUAL 0) + #message(STATUS "info: file '${f}' the same") + else() + math(EXPR diffs "${diffs} + 1") + message(STATUS "warning: file '${f}' differs from d1 to d2") + file(READ "${d1}/${f}" f1contents) + message(STATUS "contents of file '${d1}/${f}' +[===[${f1contents}]===]") + file(READ "${d2}/${f}" f2contents) + message(STATUS "contents of file '${d2}/${f}' +[===[${f2contents}]===]") + if(DIFF_EXECUTABLE) + message(STATUS "diff of files '${d1}/${f}' '${d2}/${f}'") + message(STATUS "[====[") + execute_process(COMMAND ${DIFF_EXECUTABLE} "${d1}/${f}" "${d2}/${f}") + message(STATUS "]====]") + endif() + endif() + endforeach() + else() + math(EXPR diffs "${diffs} + 1") + message(STATUS "warning: file *lists* differ - some files exist in d1/not-d2 or not-d1/d2...") + message(STATUS " files1='${files1}'") + message(STATUS " files2='${files2}'") + endif() + endif() + + set(${diff_count_var} ${diffs} PARENT_SCOPE) +endfunction(analyze_directory_diffs) + + +# Analyze diffs between b1:b2, b2:b3, b3:b4, b4:b5 ... bN-1:bN. +# Expect no diffs. +# +find_program(DIFF_EXECUTABLE diff) +set(total_diffs 0) + +foreach(i RANGE 2 ${N}) + math(EXPR prev "${i} - 1") + set(count 0) + analyze_directory_diffs(${dir}/b${prev} ${dir}/b${i} count) + message(STATUS "diff count='${count}'") + message(STATUS "") + math(EXPR total_diffs "${total_diffs} + ${count}") +endforeach() + +message(STATUS "CMAKE_COMMAND='${CMAKE_COMMAND}'") +message(STATUS "total_diffs='${total_diffs}'")