mirror of
https://github.com/reactos/CMake.git
synced 2025-02-21 20:40:44 +00:00
Merge topic 'ExternalData-multiple-hashes'
33a9aaa8 ExternalData: Add support for multiple hash algorithms
This commit is contained in:
Brad King
CMake Topic Stage
commit
c61b6f7f37
7
Help/release/dev/ExternalData-multiple-hashes.rst
Normal file
7
Help/release/dev/ExternalData-multiple-hashes.rst
Normal file
@ -0,0 +1,7 @@
|
||||
ExternalData-multiple-hashes
|
||||
----------------------------
|
||||
|
||||
* The :module:`ExternalData` module learned to support multiple
|
||||
content links for one data file using different hashes, e.g.
|
||||
``img.png.sha256`` and ``img.png.sha1``. This allows objects
|
||||
to be fetched from sources indexed by different hash algorithms.
|
||||
@ -201,6 +201,11 @@ For example, the argument ``DATA{img.png}`` may be satisfied by either a
|
||||
real ``img.png`` file in the current source directory or a ``img.png.md5``
|
||||
file containing its MD5 sum.
|
||||
|
||||
Multiple content links of the same name with different hash algorithms
|
||||
are supported (e.g. ``img.png.sha256`` and ``img.png.sha1``) so long as
|
||||
they all correspond to the same real file. This allows objects to be
|
||||
fetched from sources indexed by different hash algorithms.
|
||||
|
||||
Referencing File Series
|
||||
"""""""""""""""""""""""
|
||||
|
||||
@ -429,8 +434,10 @@ function(ExternalData_add_target target)
|
||||
string(REPLACE "|" ";" tuple "${entry}")
|
||||
list(GET tuple 0 file)
|
||||
list(GET tuple 1 name)
|
||||
list(GET tuple 2 ext)
|
||||
set(stamp "${ext}-stamp")
|
||||
list(GET tuple 2 exts)
|
||||
string(REPLACE "+" ";" exts_list "${exts}")
|
||||
list(GET exts_list 0 first_ext)
|
||||
set(stamp "-hash-stamp")
|
||||
if(NOT DEFINED "_ExternalData_FILE_${file}")
|
||||
set("_ExternalData_FILE_${file}" 1)
|
||||
get_property(added DIRECTORY PROPERTY "_ExternalData_FILE_${file}")
|
||||
@ -446,12 +453,12 @@ function(ExternalData_add_target target)
|
||||
OUTPUT "${file}${stamp}" "${file}"
|
||||
# Run the data fetch/update script.
|
||||
COMMAND ${CMAKE_COMMAND} -Drelative_top=${CMAKE_BINARY_DIR}
|
||||
-Dfile=${file} -Dname=${name} -Dext=${ext}
|
||||
-Dfile=${file} -Dname=${name} -Dexts=${exts}
|
||||
-DExternalData_ACTION=fetch
|
||||
-DExternalData_CONFIG=${config}
|
||||
-P ${_ExternalData_SELF}
|
||||
# Update whenever the object hash changes.
|
||||
MAIN_DEPENDENCY "${name}${ext}"
|
||||
MAIN_DEPENDENCY "${name}${first_ext}"
|
||||
)
|
||||
endif()
|
||||
list(APPEND files "${file}${stamp}")
|
||||
@ -798,6 +805,7 @@ function(_ExternalData_arg_find_files glob pattern regex)
|
||||
cmake_policy(SET CMP0009 NEW)
|
||||
file(${glob} globbed RELATIVE "${top_src}" "${top_src}/${pattern}*")
|
||||
cmake_policy(POP)
|
||||
set(externals_count -1)
|
||||
foreach(entry IN LISTS globbed)
|
||||
if("x${entry}" MATCHES "^x(.*)(\\.(${_ExternalData_REGEX_EXT}))$")
|
||||
set(relname "${CMAKE_MATCH_1}")
|
||||
@ -817,7 +825,11 @@ function(_ExternalData_arg_find_files glob pattern regex)
|
||||
set(name "${top_src}/${relname}")
|
||||
set(file "${top_bin}/${relname}")
|
||||
if(alg)
|
||||
list(APPEND external "${file}|${name}|${alg}")
|
||||
if(NOT "${external_${externals_count}_file_name}" STREQUAL "${file}|${name}")
|
||||
math(EXPR externals_count "${externals_count} + 1")
|
||||
set(external_${externals_count}_file_name "${file}|${name}")
|
||||
endif()
|
||||
list(APPEND external_${externals_count}_algs "${alg}")
|
||||
elseif(ExternalData_LINK_CONTENT)
|
||||
_ExternalData_link_content("${name}" alg)
|
||||
list(APPEND external "${file}|${name}|${alg}")
|
||||
@ -830,6 +842,14 @@ function(_ExternalData_arg_find_files glob pattern regex)
|
||||
endif()
|
||||
endif()
|
||||
endforeach()
|
||||
if(${externals_count} GREATER -1)
|
||||
foreach(ii RANGE ${externals_count})
|
||||
string(REPLACE ";" "+" algs_delim "${external_${ii}_algs}")
|
||||
list(APPEND external "${external_${ii}_file_name}|${algs_delim}")
|
||||
unset(external_${ii}_algs)
|
||||
unset(external_${ii}_file_name)
|
||||
endforeach()
|
||||
endif()
|
||||
set(external "${external}" PARENT_SCOPE)
|
||||
set(internal "${internal}" PARENT_SCOPE)
|
||||
set(have_original "${have_original}" PARENT_SCOPE)
|
||||
@ -947,13 +967,28 @@ function(_ExternalData_custom_fetch key loc file err_var msg_var)
|
||||
set("${msg_var}" "${msg}" PARENT_SCOPE)
|
||||
endfunction()
|
||||
|
||||
function(_ExternalData_download_object name hash algo var_obj)
|
||||
function(_ExternalData_get_from_object_store hash algo var_obj var_success)
|
||||
# Search all object stores for an existing object.
|
||||
foreach(dir ${ExternalData_OBJECT_STORES})
|
||||
set(obj "${dir}/${algo}/${hash}")
|
||||
if(EXISTS "${obj}")
|
||||
message(STATUS "Found object: \"${obj}\"")
|
||||
set("${var_obj}" "${obj}" PARENT_SCOPE)
|
||||
set("${var_success}" 1 PARENT_SCOPE)
|
||||
return()
|
||||
endif()
|
||||
endforeach()
|
||||
endfunction()
|
||||
|
||||
function(_ExternalData_download_object name hash algo var_obj var_success var_errorMsg)
|
||||
# Search all object stores for an existing object.
|
||||
set(success 1)
|
||||
foreach(dir ${ExternalData_OBJECT_STORES})
|
||||
set(obj "${dir}/${algo}/${hash}")
|
||||
if(EXISTS "${obj}")
|
||||
message(STATUS "Found object: \"${obj}\"")
|
||||
set("${var_obj}" "${obj}" PARENT_SCOPE)
|
||||
set("${var_success}" "${success}" PARENT_SCOPE)
|
||||
return()
|
||||
endif()
|
||||
endforeach()
|
||||
@ -1008,6 +1043,7 @@ function(_ExternalData_download_object name hash algo var_obj)
|
||||
get_filename_component(dir "${name}" PATH)
|
||||
set(staged "${dir}/.ExternalData_${algo}_${hash}")
|
||||
|
||||
set(success 1)
|
||||
if(found)
|
||||
file(RENAME "${tmp}" "${obj}")
|
||||
message(STATUS "Downloaded object: \"${obj}\"")
|
||||
@ -1018,38 +1054,74 @@ function(_ExternalData_download_object name hash algo var_obj)
|
||||
if(NOT tried)
|
||||
set(tried "\n (No ExternalData_URL_TEMPLATES given)")
|
||||
endif()
|
||||
message(FATAL_ERROR "Object ${algo}=${hash} not found at:${tried}")
|
||||
set(success 0)
|
||||
set("${var_errorMsg}" "Object ${algo}=${hash} not found at:${tried}" PARENT_SCOPE)
|
||||
endif()
|
||||
|
||||
set("${var_obj}" "${obj}" PARENT_SCOPE)
|
||||
set("${var_success}" "${success}" PARENT_SCOPE)
|
||||
endfunction()
|
||||
|
||||
if("${ExternalData_ACTION}" STREQUAL "fetch")
|
||||
foreach(v ExternalData_OBJECT_STORES file name ext)
|
||||
foreach(v ExternalData_OBJECT_STORES file name exts)
|
||||
if(NOT DEFINED "${v}")
|
||||
message(FATAL_ERROR "No \"-D${v}=\" value provided!")
|
||||
endif()
|
||||
endforeach()
|
||||
|
||||
file(READ "${name}${ext}" hash)
|
||||
string(STRIP "${hash}" hash)
|
||||
string(REPLACE "+" ";" exts_list "${exts}")
|
||||
set(succeeded 0)
|
||||
set(errorMsg "")
|
||||
set(hash_list )
|
||||
set(algo_list )
|
||||
set(hash )
|
||||
set(algo )
|
||||
foreach(ext ${exts_list})
|
||||
file(READ "${name}${ext}" hash)
|
||||
string(STRIP "${hash}" hash)
|
||||
|
||||
if("${ext}" MATCHES "^\\.(${_ExternalData_REGEX_EXT})$")
|
||||
string(TOUPPER "${CMAKE_MATCH_1}" algo)
|
||||
else()
|
||||
message(FATAL_ERROR "Unknown hash algorithm extension \"${ext}\"")
|
||||
if("${ext}" MATCHES "^\\.(${_ExternalData_REGEX_EXT})$")
|
||||
string(TOUPPER "${CMAKE_MATCH_1}" algo)
|
||||
else()
|
||||
message(FATAL_ERROR "Unknown hash algorithm extension \"${ext}\"")
|
||||
endif()
|
||||
|
||||
list(APPEND hash_list ${hash})
|
||||
list(APPEND algo_list ${algo})
|
||||
endforeach()
|
||||
|
||||
list(LENGTH exts_list num_extensions)
|
||||
math(EXPR exts_range "${num_extensions} - 1")
|
||||
foreach(ii RANGE 0 ${exts_range})
|
||||
list(GET hash_list ${ii} hash)
|
||||
list(GET algo_list ${ii} algo)
|
||||
_ExternalData_get_from_object_store("${hash}" "${algo}" obj succeeded)
|
||||
if(succeeded)
|
||||
break()
|
||||
endif()
|
||||
endforeach()
|
||||
if(NOT succeeded)
|
||||
foreach(ii RANGE 0 ${exts_range})
|
||||
list(GET hash_list ${ii} hash)
|
||||
list(GET algo_list ${ii} algo)
|
||||
_ExternalData_download_object("${name}" "${hash}" "${algo}"
|
||||
obj succeeded algoErrorMsg)
|
||||
set(errorMsg "${errorMsg}\n${algoErrorMsg}")
|
||||
if(succeeded)
|
||||
break()
|
||||
endif()
|
||||
endforeach()
|
||||
endif()
|
||||
if(NOT succeeded)
|
||||
message(FATAL_ERROR "${errorMsg}")
|
||||
endif()
|
||||
|
||||
_ExternalData_download_object("${name}" "${hash}" "${algo}" obj)
|
||||
|
||||
# Check if file already corresponds to the object.
|
||||
set(stamp "${ext}-stamp")
|
||||
set(stamp "-hash-stamp")
|
||||
set(file_up_to_date 0)
|
||||
if(EXISTS "${file}" AND EXISTS "${file}${stamp}")
|
||||
file(READ "${file}${stamp}" f_hash)
|
||||
string(STRIP "${f_hash}" f_hash)
|
||||
if("${f_hash}" STREQUAL "${hash}")
|
||||
#message(STATUS "File already corresponds to object")
|
||||
set(file_up_to_date 1)
|
||||
endif()
|
||||
endif()
|
||||
|
||||
@ -45,6 +45,8 @@ ExternalData_Add_Test(Data1
|
||||
-D Meta=DATA{MetaTop.dat,REGEX:Meta[ABC].dat}
|
||||
-D Directory=DATA{Directory/,A.dat,REGEX:[BC].dat}
|
||||
-D DirRecurse=DATA{DirRecurse/,RECURSE:,A.dat,REGEX:[BC].dat}
|
||||
-D MultipleAlgorithmNoSHA1=DATA{MultipleAlgorithmNoSHA1.dat}
|
||||
-D MultipleAlgorithmNoMD5=DATA{MultipleAlgorithmNoMD5.dat}
|
||||
-D "Semicolons=DATA{Data.dat}\\;DATA{Data.dat}"
|
||||
-P ${CMAKE_CURRENT_SOURCE_DIR}/Data1Check.cmake
|
||||
)
|
||||
|
||||
1
Tests/Module/ExternalData/MultipleAlgorithmNoMD5.dat.md5
Normal file
1
Tests/Module/ExternalData/MultipleAlgorithmNoMD5.dat.md5
Normal file
@ -0,0 +1 @@
|
||||
29848e54a4d0343f138ab14419b863de
|
||||
@ -0,0 +1 @@
|
||||
2af59a7022024974f3b8521b7ed8137c996a79f1
|
||||
@ -0,0 +1 @@
|
||||
08cfcf221f76ace7b906b312284e73d7
|
||||
@ -0,0 +1 @@
|
||||
223b134e6e3a9bf34aa7531c009d97cff6b0d8a3
|
||||
Loading…
x
Reference in New Issue
Block a user