diff --git a/emba b/emba index 3acc633a..0e1f4e82 100755 --- a/emba +++ b/emba @@ -172,7 +172,7 @@ run_modules() fi local MOD_FIN=0 if ( file "${MODULE_FILE}" | grep -q "shell script" ) && ! [[ "${MODULE_FILE}" =~ \ |\' ]] ; then - if [[ "${MODULE_GROUP^^}" == "P" ]]; then + if [[ "${MODULE_GROUP^^}" == "P" ]] || [[ "${MODULE_GROUP^^}" == "D" ]]; then # we are able to enable/disable threading on module basis in the the pre-checker modules with the header: # export PRE_THREAD_ENA=1/0 # shellcheck source=/dev/null @@ -295,7 +295,7 @@ run_modules() fi local MOD_FIN=0 if ( file "${MODULE_FILE}" | grep -q "shell script" ) && ! [[ "${MODULE_FILE}" =~ \ |\' ]] ; then - if [[ "${MODULE_GROUP^^}" == "P" ]]; then + if [[ "${MODULE_GROUP^^}" == "P" ]] || [[ "${MODULE_GROUP^^}" == "D" ]]; then # we are able to enable/disable threading on module basis in the the pre-checker modules with the header: # export PRE_THREAD_ENA=1/0 # shellcheck source=/dev/null @@ -881,9 +881,6 @@ main() { ####################################################################################### if [[ "${DIFF_MODE}" -eq 1 ]] ; then - # Diff mode is unthreaded - export THREADED=0 - print_ln "no_log" if [[ -d "${LOG_DIR}" ]]; then print_output "[!] Diff mode started on ""$(date)" "main" diff --git a/helpers/helpers_emba_dependency_check.sh b/helpers/helpers_emba_dependency_check.sh index 95307d18..7a155894 100755 --- a/helpers/helpers_emba_dependency_check.sh +++ b/helpers/helpers_emba_dependency_check.sh @@ -205,7 +205,6 @@ dependency_check() ####################################################################################### print_output "[*] Network connection:" "no_log" - if [[ "${CONTAINER_NUMBER}" -ne 1 ]]; then print_output " Internet connection - \\c" "no_log" @@ -254,6 +253,7 @@ dependency_check() -H "Authorization: Bearer ${OPENAI_API_KEY}" \ -d @"${CONFIG_DIR}/gpt_template.json" --write-out "%{http_code}" -o /tmp/chatgpt-test.json) + echo "API request finished ... $HTTP_CODE_" if [[ "${HTTP_CODE_}" -eq 200 ]] ; then echo -e "${GREEN}""ok""${NC}" rm /tmp/chatgpt-test.json @@ -285,7 +285,7 @@ dependency_check() fi # Note: we are running into issues in the case where the key can't be verified, but GPT is not enabled at all # In such a case we will fail the check without the need of GPT - DEP_ERROR=1 + # DEP_ERROR=1 fi if grep -q "Testing phase ended" "${LOG_DIR}"/"${MAIN_LOG_FILE}" 2>/dev/null; then print_output " Testing phase ended - \\c" "no_log" diff --git a/helpers/helpers_emba_html_generator.sh b/helpers/helpers_emba_html_generator.sh index a0243eb6..334b608e 100755 --- a/helpers/helpers_emba_html_generator.sh +++ b/helpers/helpers_emba_html_generator.sh @@ -91,6 +91,7 @@ add_link_tags() { if ( grep -a -q -E '\[REF\]' "${LINK_FILE}" ) ; then readarray -t REF_LINKS_L_NUMBER < <(grep -a -n -E '\[REF\].*' "${LINK_FILE}" | cut -d':' -f1 ) for REF_LINK_NUMBER in "${REF_LINKS_L_NUMBER[@]}" ; do + DEPTH="." REF_LINK="$(sed "${REF_LINK_NUMBER}""q;d" "${LINK_FILE}" | cut -c12- | cut -d'<' -f1 || true)" URL_REGEX='(www.|https?|ftp|file):\/\/' if [[ -f "$(echo "${REF_LINK}" | cut -d"#" -f1)" ]] ; then diff --git a/modules/D02_firmware_diffing_bin_details.sh b/modules/D02_firmware_diffing_bin_details.sh index 27dd89ed..27b46265 100755 --- a/modules/D02_firmware_diffing_bin_details.sh +++ b/modules/D02_firmware_diffing_bin_details.sh @@ -18,6 +18,9 @@ # This module is doing some basic checks on the firmware. It uses mainly # the functionality from the p02 module but on both firmware images. +export PRE_THREAD_ENA=0 + + D02_firmware_diffing_bin_details() { module_log_init "${FUNCNAME[0]}" diff --git a/modules/D05_firmware_diffing_extractor.sh b/modules/D05_firmware_diffing_extractor.sh index 3ffe3ca9..f9932f2f 100755 --- a/modules/D05_firmware_diffing_extractor.sh +++ b/modules/D05_firmware_diffing_extractor.sh @@ -19,6 +19,8 @@ # unblob extraction from module p55. There is currently no deep extraction # mode supported. +export PRE_THREAD_ENA=0 + D05_firmware_diffing_extractor() { module_log_init "${FUNCNAME[0]}" diff --git a/modules/D10_firmware_diffing.sh b/modules/D10_firmware_diffing.sh index 27e03010..be17fc0b 100755 --- a/modules/D10_firmware_diffing.sh +++ b/modules/D10_firmware_diffing.sh @@ -21,11 +21,15 @@ D10_firmware_diffing() { module_log_init "${FUNCNAME[0]}" - module_title "Firmware diff analysis" - local NEG_LOG=0 + module_title "Diff analysis of two firmware images" + # we only look at files ranking lt 95 in ssdeep - probably we need to adjust this in the future local SSDEEP_MIN_RANK=95 + # local THREADED=1 + # local MAX_MOD_THREADS=10 + local NEG_LOG=0 + if ! command -v ssdeep > /dev/null ; then print_output "[-] Missing ssdeep installation" module_end_log "${FUNCNAME[0]}" 0 @@ -74,157 +78,253 @@ D10_firmware_diffing() { # lets iterate through the files from firmware 1 and check if they are available in firmware 2 # if they are also available we are going to check the diffs with ssdeep for FW_FILE1 in "${FW_FILES1[@]}"; do - FW_FILE1="${OUTPUT_DIR_UNBLOB1}""${FW_FILE1#.}" - - FW_FILE_NAME1=$(basename "${FW_FILE1}") - # From extraction process we often get a huge amount of files called "gzip.uncompressed" - # Currently we just skip them. In the future we probably need to respect the directory name right before: - # /lib/modules/4.19.163/kernel/net/netfilter/xt_LOG.ko.gz_extract/gzip.uncompressed - # -> the name that we need to take care of is xt_LOG.ko.gz - if [[ "${FW_FILE_NAME1}" == "gzip.uncompressed" ]]; then - continue - fi - # print_output "[*] Testing $FW_FILE1" - - # find the file in OUTPUT_DIR_UNBLOB2 - mapfile -t FW_FILES1in2 < <(find "${OUTPUT_DIR_UNBLOB2}" -type f -name "${FW_FILE_NAME1}" -exec md5sum {} \; 2>/dev/null | sort -u -k1,1 | cut -d\ -f3) - - if [[ "${#FW_FILES1in2[@]}" -ne 0 ]]; then - for FW_FILE2 in "${FW_FILES1in2[@]}"; do - - MD5_FW_FILE1=$(md5sum "${FW_FILE1}" | awk '{print $1}') - MD5_FW_FILE2=$(md5sum "${FW_FILE2}" | awk '{print $1}') - if [[ "${MD5_FW_FILE1}" != "${MD5_FW_FILE2}" ]]; then - SSDEEP_OUT=$(ssdeep -d -s "${FW_FILE1}" "${FW_FILE2}") - [[ -z "${SSDEEP_OUT}" ]] && continue - # extract the ssdeep ranking from the output - SSDEEP_RANK=$(echo "${SSDEEP_OUT}" | rev | awk '{print $1}' | rev | tr -d '(' | tr -d ')') - NEG_LOG=1 - - LOG_FILE_SUB_NAME="diff_report_${FW_FILE_NAME1}.txt" - LOG_PATH_MODULE_SUB="${LOG_PATH_MODULE}"/"${LOG_FILE_SUB_NAME/.txt}" - LOG_FILE_DETAILS="${LOG_PATH_MODULE_SUB}/${LOG_FILE_SUB_NAME}" - - ! [[ -d "${LOG_PATH_MODULE_SUB}" ]] && mkdir "${LOG_PATH_MODULE_SUB}" - - if [[ "${SSDEEP_RANK}" -lt "${SSDEEP_MIN_RANK}" ]]; then - print_ln "no_log" - if [[ "$(file "${FW_FILE1}")" == *"text"* ]]; then - print_output "[+] Found modified ASCII file ${ORANGE}${FW_FILE_NAME1}${GREEN} in 2nd firmware directory - Ranking ${ORANGE}${SSDEEP_RANK}${NC}" "" "${LOG_FILE_DETAILS}" - write_log "[+] Found modified ASCII file ${ORANGE}${FW_FILE_NAME1}${GREEN} in 2nd firmware directory - Ranking ${ORANGE}${SSDEEP_RANK}${NC}" "${LOG_FILE_DETAILS}" - else - print_output "[+] Found modified binary file ${ORANGE}${FW_FILE_NAME1}${GREEN} in 2nd firmware directory - Ranking ${ORANGE}${SSDEEP_RANK}${NC}" "" "${LOG_FILE_DETAILS}" - write_log "[+] Found modified binary file ${ORANGE}${FW_FILE_NAME1}${GREEN} in 2nd firmware directory - Ranking ${ORANGE}${SSDEEP_RANK}${NC}" "${LOG_FILE_DETAILS}" - fi - write_log "" "${LOG_FILE_DETAILS}" - write_log "[*] Base firmware file details to compare with:" "${LOG_FILE_DETAILS}" - write_log "$(indent "$(orange "$(print_path "${FW_FILE1}")")")" "${LOG_FILE_DETAILS}" - write_log "$(indent "$(orange "$(md5sum "${FW_FILE1}")")")" "${LOG_FILE_DETAILS}" - write_log "$(indent "$(orange "$(file "${FW_FILE1}")")")" "${LOG_FILE_DETAILS}" - write_log "" "${LOG_FILE_DETAILS}" - write_log "[*] Second firmware file details:" "${LOG_FILE_DETAILS}" - write_log "$(indent "$(orange "$(print_path "${FW_FILE2}")")")" "${LOG_FILE_DETAILS}" - write_log "$(indent "$(orange "$(md5sum "${FW_FILE2}")")")" "${LOG_FILE_DETAILS}" - write_log "$(indent "$(orange "$(file "${FW_FILE2}")")")" "${LOG_FILE_DETAILS}" - - if [[ "$(file "${FW_FILE1}")" == *"text"* ]]; then - sub_module_title "Diff for clear text file ${FW_FILE_NAME1}" "${LOG_FILE_DETAILS}" - # clear text handling - just colordiffing - diff -yb --color=always --suppress-common-lines "${FW_FILE1}" "${FW_FILE2}" > "${LOG_PATH_MODULE_SUB}"/colordiff_"${FW_FILE_NAME1}".txt || true - if [[ -f "${LOG_PATH_MODULE_SUB}"/colordiff_"${FW_FILE_NAME1}".txt ]]; then - print_output "[*] Diffing results from clear text file ${ORANGE}${FW_FILE_NAME1}${NC} logged to ${ORANGE}${LOG_PATH_MODULE_SUB}/colordiff_${FW_FILE_NAME1}.txt${NC}" "no_log" - write_log "" "${LOG_FILE_DETAILS}" - write_log "[*] Diffing results from clear text file ${ORANGE}${FW_FILE_NAME1}${NC}" "${LOG_FILE_DETAILS}" - write_log "" "${LOG_FILE_DETAILS}" - cat "${LOG_PATH_MODULE_SUB}"/colordiff_"$(basename "${FW_FILE1}")".txt >> "${LOG_FILE_DETAILS}" - fi - else - sub_module_title "Diff for binary file ${FW_FILE_NAME1}" "${LOG_FILE_DETAILS}" - # binary handling - colordiffing the hex dump and some radare2 diffing - diff -yb --color=always --suppress-common-lines <(xxd "${FW_FILE1}") <(xxd "${FW_FILE2}") > "${LOG_PATH_MODULE_SUB}"/colordiff_"${FW_FILE_NAME1}".txt || true - if [[ -f "${LOG_PATH_MODULE_SUB}"/colordiff_"${FW_FILE_NAME1}".txt ]]; then - print_output "[*] Diffing results from binary file ${ORANGE}${FW_FILE_NAME1}${NC} logged to ${ORANGE}${LOG_PATH_MODULE_SUB}/colordiff_${FW_FILE_NAME1}.txt${NC}" "no_log" - write_log "" "${LOG_FILE_DETAILS}" - # we only link to the binary colordiff in the web report: - # on the cli output we do not see the complete diff - write_log "[*] Diffing results from binary file ${ORANGE}${FW_FILE_NAME1}${NC}" "${LOG_FILE_DETAILS}" - write_link "${LOG_PATH_MODULE_SUB}/colordiff_${FW_FILE_NAME1}.txt" "${LOG_FILE_DETAILS}" - write_log "" "${LOG_FILE_DETAILS}" - fi - write_log "" "${LOG_FILE_DETAILS}" - - # radare2: - sub_module_title "Non matching functions for binary file ${FW_FILE_NAME1}" "${LOG_FILE_DETAILS}" - write_log "" "${LOG_FILE_DETAILS}" - write_log "[*] Non matching functions in ${ORANGE}$(basename "${FW_FILE1}")${NC}:" "${LOG_FILE_DETAILS}" - print_output "[*] Non matching functions in ${ORANGE}$(basename "${FW_FILE1}")${NC} logged to ${ORANGE}${LOG_PATH_MODULE_SUB}/r2_diff_fct_${FW_FILE_NAME1}_${FW_FILE_NAME1}.txt${NC}" "no_log" - # get the functions which are different with radiff2: - radiff2 -AC "${FW_FILE1}" "${FW_FILE2}" 2>/dev/null | grep UNMATCH > "${LOG_PATH_MODULE_SUB}"/r2_diff_fct_"${FW_FILE_NAME1}"_"${FW_FILE_NAME1}".txt || true - if [[ -f "${LOG_PATH_MODULE_SUB}"/r2_diff_fct_"${FW_FILE_NAME1}"_"${FW_FILE_NAME1}".txt ]]; then - cat "${LOG_PATH_MODULE_SUB}"/r2_diff_fct_"${FW_FILE_NAME1}"_"${FW_FILE_NAME1}".txt >> "${LOG_FILE_DETAILS}" - mapfile -t UNMATCHED_FCTs < <(awk '{print $1}' "${LOG_PATH_MODULE_SUB}"/r2_diff_fct_"${FW_FILE_NAME1}"_"${FW_FILE_NAME1}".txt | sort -u) - fi - write_log "" "${LOG_FILE_DETAILS}" - - sub_module_title "R2 diff for binary file ${FW_FILE_NAME1}" "${LOG_FILE_DETAILS}" - - # let's do a diff on the complete radare2 output: - # shellcheck disable=SC2016 - r2 -e io.cache=true -e scr.color=false -A -q -c 'pd $s' "${FW_FILE1}" 2>/dev/null > "${LOG_PATH_MODULE_SUB}"/r2_disasm_"${FW_FILE_NAME1}"_dir1.txt - # shellcheck disable=SC2016 - r2 -e io.cache=true -e scr.color=false -A -q -c 'pd $s' "${FW_FILE2}" 2>/dev/null > "${LOG_PATH_MODULE_SUB}"/r2_disasm_"$(basename "${FW_FILE2}")"_dir2.txt - diff -yb --color=always --suppress-common-lines "${LOG_PATH_MODULE_SUB}"/r2_disasm_"$(basename "${FW_FILE1}")"_dir1.txt "${LOG_PATH_MODULE_SUB}"/r2_disasm_"$(basename "${FW_FILE2}")"_dir2.txt 2>/dev/null > "${LOG_PATH_MODULE_SUB}"/colordiff_radare2_disasm_"$(basename "${FW_FILE1}")".txt || true - if [[ -f "${LOG_PATH_MODULE_SUB}"/colordiff_radare2_disasm_"${FW_FILE_NAME1}".txt ]]; then - print_output "[*] Radare2 binary diffing results from binary file ${ORANGE}${FW_FILE_NAME1}${NC} logged to ${ORANGE}${LOG_FILE_DETAILS}${NC}." "no_log" - write_log "[*] Radare2 binary diffing results from binary file ${ORANGE}${FW_FILE_NAME1}${NC}" "${LOG_FILE_DETAILS}" - write_link "${LOG_PATH_MODULE_SUB}"/colordiff_radare2_disasm_"${FW_FILE_NAME1}".txt "${LOG_FILE_DETAILS}" - write_log "" "${LOG_FILE_DETAILS}" - fi - - # now we check diff all the functions with differences and generate a xdot and png picture - # see also https://book.rada.re/tools/radiff2/binary_diffing.html - ! [[ -d "${LOG_PATH_MODULE}"/r2_fct_graphing/ ]] && mkdir "${LOG_PATH_MODULE}"/r2_fct_graphing/ - write_log "[*] Radare2 binary function diff for ${ORANGE}${FW_FILE_NAME1}${NC}:" "${LOG_FILE_DETAILS}" - - for FCT in "${UNMATCHED_FCTs[@]}"; do - write_log "" "${LOG_FILE_DETAILS}" - radiff2 -e bin.cache=true -md -g "${FCT}" "${FW_FILE2}" "${FW_FILE1}" 2>/dev/null > "${LOG_PATH_MODULE}"/r2_fct_graphing/r2_fct_graph_"${FW_FILE_NAME1}"_"${FCT}".xdot - # we only print the graph if the log file was generated and has content and it has multiple addresses (0x) included - if [[ -s "${LOG_PATH_MODULE}"/r2_fct_graphing/r2_fct_graph_"${FW_FILE_NAME1}"_"${FCT}".xdot ]]; then - if [[ "$(grep -c "0x" "${LOG_PATH_MODULE}"/r2_fct_graphing/r2_fct_graph_"${FW_FILE_NAME1}"_"${FCT}".xdot 2>/dev/null)" -gt 1 ]]; then - print_output "[*] Generating png for function ${FCT} of binary ${FW_FILE_NAME1}" "no_log" - dot -Tpng "${LOG_PATH_MODULE}"/r2_fct_graphing/r2_fct_graph_"${FW_FILE_NAME1}"_"${FCT}".xdot 2>/dev/null > "${LOG_PATH_MODULE}"/r2_fct_graphing/r2_fct_graph_"${FW_FILE_NAME1}"_"${FCT}".png || true - fi - fi - if [[ -s "${LOG_PATH_MODULE}/r2_fct_graphing/r2_fct_graph_${FW_FILE_NAME1}_${FCT}.png" ]]; then - write_log "[*] Radare2 binary function diff for function ${ORANGE}${FCT}${NC} in binary ${ORANGE}${FW_FILE_NAME1}${NC}" "${LOG_FILE_DETAILS}" - write_link "${LOG_PATH_MODULE}/r2_fct_graphing/r2_fct_graph_${FW_FILE_NAME1}_${FCT}.png" "${LOG_FILE_DETAILS}" - else - write_log "[-] No function graph available for ${ORANGE}${FCT}${NC}" "${LOG_FILE_DETAILS}" - fi - done - fi - write_log "" "${LOG_FILE_DETAILS}" - fi - fi - done + NEG_LOG=1 + if [[ "${THREADED}" -eq 1 ]]; then + analyse_fw_files "${FW_FILE1}" & + local TMP_PID="$!" + store_kill_pids "${TMP_PID}" + WAIT_PIDS_D10+=( "${TMP_PID}" ) + max_pids_protection "${MAX_MOD_THREADS}" "${WAIT_PIDS_D10[@]}" else - # deleted files in 2nd firmware directory - if [[ -f "${FW_FILE1}" ]]; then - if file "${FW_FILE1}" | grep -q ASCII; then - cp "${FW_FILE1}" "${LOG_PATH_MODULE}"/"${FW_FILE_NAME1}".log - print_output "[+] Firmware ASCII file ${ORANGE}${FW_FILE_NAME1}${GREEN} is deleted in ${ORANGE}${OUTPUT_DIR_UNBLOB2}${GREEN}." - else - print_output "[+] Firmware binary file ${ORANGE}${FW_FILE_NAME1}${GREEN} is deleted in ${ORANGE}${OUTPUT_DIR_UNBLOB2}${GREEN}." - fi - [[ -f "${LOG_PATH_MODULE}"/"${FW_FILE_NAME1}".log ]] && write_link "${LOG_PATH_MODULE}/${FW_FILE_NAME1}.log" - fi + # echo "Testing ${FW_FILE1}" + analyse_fw_files "${FW_FILE1}" fi done + [[ "$THREADED" -eq 1 ]] && wait_for_pid "${WAIT_PIDS_D10[@]}" check_for_new_files - module_end_log "${FUNCNAME[0]}" "$NEG_LOG" + module_end_log "${FUNCNAME[0]}" "${NEG_LOG}" +} + +analyse_fw_files() { + FW_FILE1="${1:-}" + + # as we searched directly in the extracted path, we need to adjust the file path with the unblob extraction directory + FW_FILE1="${OUTPUT_DIR_UNBLOB1}""${FW_FILE1#.}" + FW_FILE_NAME1=$(basename "${FW_FILE1}") + # From extraction process we often get a huge amount of files called "gzip.uncompressed" + # Currently we just skip them. In the future we probably need to respect the directory name right before: + # /lib/modules/4.19.163/kernel/net/netfilter/xt_LOG.ko.gz_extract/gzip.uncompressed + # -> the name that we need to take care of is xt_LOG.ko.gz + if [[ "${FW_FILE_NAME1}" == "gzip.uncompressed" ]]; then + return + fi + # print_output "[*] Testing $FW_FILE1" + + # find the file in OUTPUT_DIR_UNBLOB2 + mapfile -t FW_FILES1in2 < <(find "${OUTPUT_DIR_UNBLOB2}" -type f -name "${FW_FILE_NAME1}" -exec md5sum {} \; 2>/dev/null | sort -u -k1,1 | cut -d\ -f3) + + if [[ "${#FW_FILES1in2[@]}" -ne 0 ]]; then + for FW_FILE2 in "${FW_FILES1in2[@]}"; do + + MD5_FW_FILE1=$(md5sum "${FW_FILE1}" | awk '{print $1}') + MD5_FW_FILE2=$(md5sum "${FW_FILE2}" | awk '{print $1}') + FW_FILE_NAME2=$(basename "${FW_FILE2}") + + if [[ "${MD5_FW_FILE1}" != "${MD5_FW_FILE2}" ]]; then + # fuzzy hash diffing here: + SSDEEP_OUT=$(ssdeep -d -s "${FW_FILE1}" "${FW_FILE2}") + [[ -z "${SSDEEP_OUT}" ]] && return + # extract the ssdeep ranking from the output + SSDEEP_RANK=$(echo "${SSDEEP_OUT}" | rev | awk '{print $1}' | rev | tr -d '(' | tr -d ')') + NEG_LOG=1 + + LOG_FILE_SUB_NAME="diff_report_${FW_FILE_NAME1}.txt" + LOG_PATH_MODULE_SUB="${LOG_PATH_MODULE}"/"${LOG_FILE_SUB_NAME/.txt}" + LOG_FILE_DETAILS="${LOG_PATH_MODULE_SUB}/${LOG_FILE_SUB_NAME}" + + ! [[ -d "${LOG_PATH_MODULE_SUB}" ]] && mkdir "${LOG_PATH_MODULE_SUB}" + + if [[ "${SSDEEP_RANK}" -lt "${SSDEEP_MIN_RANK}" ]]; then + print_ln "no_log" + if [[ "$(file "${FW_FILE1}")" == *"text"* ]]; then + print_output "[+] Found modified ASCII file ${ORANGE}${FW_FILE_NAME1}${GREEN} in 2nd firmware directory - Ranking ${ORANGE}${SSDEEP_RANK}${NC}" "" "${LOG_FILE_DETAILS}" + write_log "[+] Found modified ASCII file ${ORANGE}${FW_FILE_NAME1}${GREEN} in 2nd firmware directory - Ranking ${ORANGE}${SSDEEP_RANK}${NC}" "${LOG_FILE_DETAILS}" + else + print_output "[+] Found modified binary file ${ORANGE}${FW_FILE_NAME1}${GREEN} in 2nd firmware directory - Ranking ${ORANGE}${SSDEEP_RANK}${NC}" "" "${LOG_FILE_DETAILS}" + write_log "[+] Found modified binary file ${ORANGE}${FW_FILE_NAME1}${GREEN} in 2nd firmware directory - Ranking ${ORANGE}${SSDEEP_RANK}${NC}" "${LOG_FILE_DETAILS}" + fi + + write_log "" "${LOG_FILE_DETAILS}" + write_log "[*] Base firmware file details to compare with:" "${LOG_FILE_DETAILS}" + write_log "$(indent "$(orange "$(print_path "${FW_FILE1}")")")" "${LOG_FILE_DETAILS}" + write_log "$(indent "$(orange "$(md5sum "${FW_FILE1}")")")" "${LOG_FILE_DETAILS}" + write_log "$(indent "$(orange "$(file "${FW_FILE1}")")")" "${LOG_FILE_DETAILS}" + write_log "" "${LOG_FILE_DETAILS}" + write_log "[*] Second firmware file details:" "${LOG_FILE_DETAILS}" + write_log "$(indent "$(orange "$(print_path "${FW_FILE2}")")")" "${LOG_FILE_DETAILS}" + write_log "$(indent "$(orange "$(md5sum "${FW_FILE2}")")")" "${LOG_FILE_DETAILS}" + write_log "$(indent "$(orange "$(file "${FW_FILE2}")")")" "${LOG_FILE_DETAILS}" + + if [[ "$(file "${FW_FILE1}")" == *"text"* ]]; then + sub_module_title "Diff for clear text file ${FW_FILE_NAME1}" "${LOG_FILE_DETAILS}" + # clear text handling - just colordiffing + # colored diff output not possible in -y mode to file output -> change the diffs with sed + diff -yb --color=always --suppress-common-lines "${FW_FILE1}" "${FW_FILE2}" | sed 's/.*[[:blank:]]|[[:blank:]].*/\x1b[32m&\x1b[0m/' > "${LOG_PATH_MODULE_SUB}"/colordiff_"${FW_FILE_NAME1}".txt || true + if [[ -f "${LOG_PATH_MODULE_SUB}"/colordiff_"${FW_FILE_NAME1}".txt ]]; then + print_output "[*] Diffing results from clear text file ${ORANGE}${FW_FILE_NAME1}${NC} logged to ${ORANGE}${LOG_PATH_MODULE_SUB}/colordiff_${FW_FILE_NAME1}.txt${NC}" "no_log" + write_log "" "${LOG_FILE_DETAILS}" + write_log "[*] Diffing results from clear text file ${ORANGE}${FW_FILE_NAME1}${NC}" "${LOG_FILE_DETAILS}" + write_log "" "${LOG_FILE_DETAILS}" + cat "${LOG_PATH_MODULE_SUB}"/colordiff_"$(basename "${FW_FILE1}")".txt >> "${LOG_FILE_DETAILS}" + fi + else + sub_module_title "Diff for binary file ${FW_FILE_NAME1}" "${LOG_FILE_DETAILS}" + # binary handling - colordiffing the hex dump and some radare2 diffing + diff -yb --color=always --suppress-common-lines <(xxd "${FW_FILE1}") <(xxd "${FW_FILE2}") > "${LOG_PATH_MODULE_SUB}"/colordiff_"${FW_FILE_NAME1}".txt || true + if [[ -f "${LOG_PATH_MODULE_SUB}"/colordiff_"${FW_FILE_NAME1}".txt ]]; then + print_output "[*] Diffing results from binary file ${ORANGE}${FW_FILE_NAME1}${NC} logged to ${ORANGE}${LOG_PATH_MODULE_SUB}/colordiff_${FW_FILE_NAME1}.txt${NC}" "no_log" + write_log "" "${LOG_FILE_DETAILS}" + # we only link to the binary colordiff in the web report: + # on the cli output we do not see the complete diff + write_log "[*] Diffing results from binary file ${ORANGE}${FW_FILE_NAME1}${NC}" "${LOG_FILE_DETAILS}" + write_link "${LOG_PATH_MODULE_SUB}/colordiff_${FW_FILE_NAME1}.txt" "${LOG_FILE_DETAILS}" + write_log "" "${LOG_FILE_DETAILS}" + fi + write_log "" "${LOG_FILE_DETAILS}" + + # radare2: + # get the functions which are different with radiff2: + radiff2 -AC "${FW_FILE1}" "${FW_FILE2}" 2>/dev/null | grep UNMATCH > "${LOG_PATH_MODULE_SUB}"/r2_diff_fct_"${FW_FILE_NAME1}"_"${FW_FILE_NAME1}".txt || true + + if [[ -s "${LOG_PATH_MODULE_SUB}"/r2_diff_fct_"${FW_FILE_NAME1}"_"${FW_FILE_NAME1}".txt ]]; then + sub_module_title "Non matching functions for binary file ${FW_FILE_NAME1}" "${LOG_FILE_DETAILS}" + cat "${LOG_PATH_MODULE_SUB}"/r2_diff_fct_"${FW_FILE_NAME1}"_"${FW_FILE_NAME1}".txt + write_log "" "${LOG_FILE_DETAILS}" + print_ln "no_log" + write_log "[*] Non matching functions in ${ORANGE}$(basename "${FW_FILE1}")${NC}:" "${LOG_FILE_DETAILS}" + print_output "[*] Non matching functions in ${ORANGE}$(basename "${FW_FILE1}")${NC} logged to ${ORANGE}${LOG_PATH_MODULE_SUB}/r2_diff_fct_${FW_FILE_NAME1}_${FW_FILE_NAME1}.txt${NC}" "no_log" + cat "${LOG_PATH_MODULE_SUB}"/r2_diff_fct_"${FW_FILE_NAME1}"_"${FW_FILE_NAME1}".txt >> "${LOG_FILE_DETAILS}" + mapfile -t UNMATCHED_FCTs < <(awk '{print $1}' "${LOG_PATH_MODULE_SUB}"/r2_diff_fct_"${FW_FILE_NAME1}"_"${FW_FILE_NAME1}".txt | sort -u) + else + write_log "[-] No function diff available for ${ORANGE}${FW_FILE_NAME1}${NC}" "${LOG_FILE_DETAILS}" + fi + write_log "" "${LOG_FILE_DETAILS}" + + # let's do a diff on the complete radare2 output: + # create disassembly from file in first directory: + # shellcheck disable=SC2016 + r2 -e io.cache=true -e scr.color=false -A -q -c 'pd $s' "${FW_FILE1}" 2>/dev/null > "${LOG_PATH_MODULE_SUB}"/r2_disasm_"${FW_FILE_NAME1}"_dir1.txt + # create disassembly from file in second directory: + # shellcheck disable=SC2016 + r2 -e io.cache=true -e scr.color=false -A -q -c 'pd $s' "${FW_FILE2}" 2>/dev/null > "${LOG_PATH_MODULE_SUB}"/r2_disasm_"$(basename "${FW_FILE2}")"_dir2.txt + # create diff of both disassemblies: + diff -yb --color=always --suppress-common-lines "${LOG_PATH_MODULE_SUB}"/r2_disasm_"$(basename "${FW_FILE1}")"_dir1.txt "${LOG_PATH_MODULE_SUB}"/r2_disasm_"$(basename "${FW_FILE2}")"_dir2.txt 2>/dev/null > "${LOG_PATH_MODULE_SUB}"/colordiff_radare2_disasm_"$(basename "${FW_FILE1}")".txt || true + + if [[ -s "${LOG_PATH_MODULE_SUB}"/colordiff_radare2_disasm_"${FW_FILE_NAME1}".txt ]]; then + sub_module_title "R2 diff for binary file ${FW_FILE_NAME1}" "${LOG_FILE_DETAILS}" + print_output "[*] Radare2 binary diffing results from binary file ${ORANGE}${FW_FILE_NAME1}${NC} logged to ${ORANGE}${LOG_FILE_DETAILS}${NC}." "no_log" + write_log "[*] Radare2 binary diffing results from binary file ${ORANGE}${FW_FILE_NAME1}${NC}" "${LOG_FILE_DETAILS}" + write_link "${LOG_PATH_MODULE_SUB}"/colordiff_radare2_disasm_"${FW_FILE_NAME1}".txt "${LOG_FILE_DETAILS}" + write_log "" "${LOG_FILE_DETAILS}" + fi + + # now we check diff all the functions with differences and generate a xdot and png picture + # see also https://book.rada.re/tools/radiff2/binary_diffing.html + ! [[ -d "${LOG_PATH_MODULE}"/r2_fct_graphing/ ]] && mkdir "${LOG_PATH_MODULE}"/r2_fct_graphing/ + write_log "[*] Radare2 binary function diff for ${ORANGE}${FW_FILE_NAME1}${NC}:" "${LOG_FILE_DETAILS}" + + # walk through all changed functions: + for FCT in "${UNMATCHED_FCTs[@]}"; do + # Threading currently mangles the links + if [[ "$THREADED" -eq 2 ]]; then + analyse_bin_fct "${FCT}" & + local TMP_PID="$!" + store_kill_pids "$TMP_PID" + WAIT_PIDS_D10+=( "$TMP_PID" ) + max_pids_protection "$MAX_MOD_THREADS" "${WAIT_PIDS_D10[@]}" + else + analyse_bin_fct "${FCT}" + fi + done + [[ "$THREADED" -eq 1 ]] && wait_for_pid "${WAIT_PIDS_D10[@]}" + fi + write_log "" "${LOG_FILE_DETAILS}" + fi + fi + done + else + # deleted files in 2nd firmware directory + if [[ -f "${FW_FILE1}" ]]; then + if file "${FW_FILE1}" | grep -q ASCII; then + cp "${FW_FILE1}" "${LOG_PATH_MODULE}"/"${FW_FILE_NAME1}".log + print_output "[+] Firmware ASCII file ${ORANGE}${FW_FILE_NAME1}${GREEN} is deleted in ${ORANGE}${OUTPUT_DIR_UNBLOB2}${GREEN}." + else + print_output "[+] Firmware binary file ${ORANGE}${FW_FILE_NAME1}${GREEN} is deleted in ${ORANGE}${OUTPUT_DIR_UNBLOB2}${GREEN}." + fi + if [[ -f "${LOG_PATH_MODULE}"/"${FW_FILE_NAME1}".log ]]; then + write_link "${LOG_PATH_MODULE}/${FW_FILE_NAME1}.log" + fi + fi + fi +} + + +analyse_bin_fct() { + local FCT="${1:-}" + + write_log "" "${LOG_FILE_DETAILS}" + radiff2 -e bin.cache=true -md -g "${FCT}" "${FW_FILE2}" "${FW_FILE1}" 2>/dev/null > "${LOG_PATH_MODULE}"/r2_fct_graphing/r2_fct_graph_"${FW_FILE_NAME1}"_"${FCT}".xdot + + # we only print the graph if the log file was generated and has content and it has multiple addresses (0x) included + if [[ -s "${LOG_PATH_MODULE}"/r2_fct_graphing/r2_fct_graph_"${FW_FILE_NAME1}"_"${FCT}".xdot ]]; then + if [[ "$(grep -c "0x" "${LOG_PATH_MODULE}"/r2_fct_graphing/r2_fct_graph_"${FW_FILE_NAME1}"_"${FCT}".xdot 2>/dev/null)" -gt 1 ]]; then + print_output "[*] Generating diff image for function ${ORANGE}${FCT}${NC} of binary ${ORANGE}${FW_FILE_NAME1}${NC}" "no_log" + dot -Tpng "${LOG_PATH_MODULE}"/r2_fct_graphing/r2_fct_graph_"${FW_FILE_NAME1}"_"${FCT}".xdot 2>/dev/null > "${LOG_PATH_MODULE}"/r2_fct_graphing/r2_fct_graph_"${FW_FILE_NAME1}"_"${FCT}".png || true + + print_output "[*] Generating disasm for function ${ORANGE}${FCT}${NC} of binary ${ORANGE}${FW_FILE_NAME1}${NC}" "no_log" + # now we need to generate the disassembly of the current function of both files to include it in the report for further manual tear-down + write_log "[*] Disassembly function ${ORANGE}${FCT}${NC} of ${ORANGE}${FW_FILE_NAME1}${NC} in ${ORANGE}first${NC} firmware directory" "${LOG_PATH_MODULE_SUB}"/r2_disasm_"${FW_FILE_NAME1}"_"${FCT}"_dir1.txt + write_log "" "${LOG_PATH_MODULE_SUB}"/r2_disasm_"${FW_FILE_NAME1}"_"${FCT}"_dir1.txt + r2 -e io.cache=true -e scr.color=false -A -q -c 'pdf @ '"${FCT}" "${FW_FILE1}" 2>/dev/null >> "${LOG_PATH_MODULE_SUB}"/r2_disasm_"${FW_FILE_NAME1}"_"${FCT}"_dir1.txt || true + + write_log "[*] Disassembly function ${ORANGE}${FCT}${NC} of ${ORANGE}${FW_FILE_NAME2}${NC} in ${ORANGE}second${NC} firmware directory" "${LOG_PATH_MODULE_SUB}"/r2_disasm_"${FW_FILE_NAME2}"_"${FCT}"_dir2.txt + write_log "" "${LOG_PATH_MODULE_SUB}"/r2_disasm_"${FW_FILE_NAME2}"_"${FCT}"_dir2.txt + r2 -e io.cache=true -e scr.color=false -A -q -c 'pdf @ '"${FCT}" "${FW_FILE2}" 2>/dev/null >> "${LOG_PATH_MODULE_SUB}"/r2_disasm_"${FW_FILE_NAME2}"_"${FCT}"_dir2.txt || true + fi + fi + + if [[ -s "${LOG_PATH_MODULE}/r2_fct_graphing/r2_fct_graph_${FW_FILE_NAME1}_${FCT}.png" ]]; then + write_log "" "${LOG_FILE_DETAILS}" + write_log "[*] Radare2 binary function diff for function ${ORANGE}${FCT}${NC} in binary ${ORANGE}${FW_FILE_NAME1}${NC}" "${LOG_FILE_DETAILS}" + write_link "${LOG_PATH_MODULE}/r2_fct_graphing/r2_fct_graph_${FW_FILE_NAME1}_${FCT}.png" "${LOG_FILE_DETAILS}" + + if [[ -s "${LOG_PATH_MODULE_SUB}"/r2_disasm_"${FW_FILE_NAME1}"_"${FCT}"_dir1.txt ]]; then + write_log "$(indent "Disassembly function ${ORANGE}${FCT}${NC} of ${ORANGE}${FW_FILE_NAME1}${NC} in ${ORANGE}first${NC} firmware directory")" "${LOG_FILE_DETAILS}" + write_link "${LOG_PATH_MODULE_SUB}"/r2_disasm_"${FW_FILE_NAME1}"_"${FCT}"_dir1.txt "${LOG_FILE_DETAILS}" + fi + + if [[ -s "${LOG_PATH_MODULE_SUB}"/r2_disasm_"${FW_FILE_NAME2}"_"${FCT}"_dir2.txt ]]; then + write_log "$(indent "Disassembly function ${ORANGE}${FCT}${NC} of ${ORANGE}${FW_FILE_NAME2} in ${ORANGE}second${NC} firmware directory")" "${LOG_FILE_DETAILS}" + write_link "${LOG_PATH_MODULE_SUB}"/r2_disasm_"${FW_FILE_NAME2}"_"${FCT}"_dir2.txt "${LOG_FILE_DETAILS}" + fi + + if [[ -s "${LOG_PATH_MODULE_SUB}"/r2_disasm_"${FW_FILE_NAME1}"_"${FCT}"_dir1.txt ]] && [[ -s "${LOG_PATH_MODULE_SUB}"/r2_disasm_"${FW_FILE_NAME2}"_"${FCT}"_dir2.txt ]]; then + # letz diff it + # extract only valid disassembly code: + grep "0x" "${LOG_PATH_MODULE_SUB}"/r2_disasm_"${FW_FILE_NAME1}"_"${FCT}"_dir1.txt | grep -v "; arg \|; var " | grep -v -E " XREF(S)? from " \ + | sed 's/[^0x]*0x/0x/' | cut -d\ -f2- | sed 's/^[[:blank:]]*//' > "${LOG_PATH_MODULE_SUB}"/r2_disasm_"${FW_FILE_NAME1}"_"${FCT}"_mod_dir1.txt || true + grep "0x" "${LOG_PATH_MODULE_SUB}"/r2_disasm_"${FW_FILE_NAME2}"_"${FCT}"_dir2.txt | grep -v "; arg \|; var " | grep -v -E " XREF(S)? from " \ + | sed 's/[^0x]*0x/0x/' | cut -d\ -f2- | sed 's/^[[:blank:]]*//' > "${LOG_PATH_MODULE_SUB}"/r2_disasm_"${FW_FILE_NAME2}"_"${FCT}"_mod_dir2.txt || true + + # if we have both disassemblies we can diff them: + if [[ -s "${LOG_PATH_MODULE_SUB}"/r2_disasm_"${FW_FILE_NAME1}"_"${FCT}"_mod_dir1.txt ]] && [[ -s "${LOG_PATH_MODULE_SUB}"/r2_disasm_"${FW_FILE_NAME2}"_"${FCT}"_mod_dir2.txt ]]; then + write_log "[*] Function diff for function ${ORANGE}${FCT}${NC} - file ${ORANGE}${FW_FILE_NAME2}${NC}" "${LOG_PATH_MODULE_SUB}"/r2_disasm_"${FW_FILE_NAME1}"_"${FCT}"_diff.txt + write_log "" "${LOG_PATH_MODULE_SUB}"/r2_disasm_"${FW_FILE_NAME1}"_"${FCT}"_diff.txt + # colorize diff changes orange (see the sed part): + diff -yb --color=always "${LOG_PATH_MODULE_SUB}"/r2_disasm_"${FW_FILE_NAME1}"_"${FCT}"_mod_dir1.txt "${LOG_PATH_MODULE_SUB}"/r2_disasm_"${FW_FILE_NAME2}"_"${FCT}"_mod_dir2.txt \ + | sed 's/.*[[:blank:]]|[[:blank:]].*/\x1b[33m&\x1b[0m/' >> "${LOG_PATH_MODULE_SUB}"/r2_disasm_"${FW_FILE_NAME1}"_"${FCT}"_diff.txt || true + fi + + if [[ -s "${LOG_PATH_MODULE_SUB}"/r2_disasm_"${FW_FILE_NAME2}"_"${FCT}"_diff.txt ]]; then + write_log "$(indent "Diffing disassembly function ${ORANGE}${FCT}${NC} of ${ORANGE}${FW_FILE_NAME2}${NC}")" "${LOG_FILE_DETAILS}" + write_link "${LOG_PATH_MODULE_SUB}"/r2_disasm_"${FW_FILE_NAME2}"_"${FCT}"_diff.txt "${LOG_FILE_DETAILS}" + fi + fi + # else + # write_log "[-] No function graph available for ${ORANGE}${FCT}${NC}" "${LOG_FILE_DETAILS}" + fi } check_for_new_files() { @@ -249,9 +349,6 @@ check_for_new_files() { print_output "[+] Firmware binary file ${ORANGE}${FW_FILE_NAME2}${GREEN} is a new file in ${ORANGE}${OUTPUT_DIR_UNBLOB2}${GREEN}." fi fi - fi done - - }