e-m-b-a/emba
1
0
Fork 0
mirror of https://github.com/e-m-b-a/emba.git synced 2026-10-12 02:13:21 +00:00
Code Issues Packages Projects Releases Wiki Activity

improve diff mode

Browse Source
This commit is contained in:
m-1-k-3
2023-11-06 12:23:26 +01:00
parent 2186a8c5f3
commit 8271f4df30
6 changed files with 257 additions and 157 deletions
Download Patch File Download Diff File Expand all files Collapse all files
emba
+2 -5
View File
@@ -172,7 +172,7 @@ run_modules()
fi
local MOD_FIN=0
if ( file "${MODULE_FILE}" | grep -q "shell script" ) && ! [[ "${MODULE_FILE}" =~ \ |\' ]] ; then
if [[ "${MODULE_GROUP^^}" == "P" ]]; then
if [[ "${MODULE_GROUP^^}" == "P" ]] || [[ "${MODULE_GROUP^^}" == "D" ]]; then
# we are able to enable/disable threading on module basis in the the pre-checker modules with the header:
# export PRE_THREAD_ENA=1/0
# shellcheck source=/dev/null
@@ -295,7 +295,7 @@ run_modules()
fi
local MOD_FIN=0
if ( file "${MODULE_FILE}" | grep -q "shell script" ) && ! [[ "${MODULE_FILE}" =~ \ |\' ]] ; then
if [[ "${MODULE_GROUP^^}" == "P" ]]; then
if [[ "${MODULE_GROUP^^}" == "P" ]] || [[ "${MODULE_GROUP^^}" == "D" ]]; then
# we are able to enable/disable threading on module basis in the the pre-checker modules with the header:
# export PRE_THREAD_ENA=1/0
# shellcheck source=/dev/null
@@ -881,9 +881,6 @@ main() {
#######################################################################################
if [[ "${DIFF_MODE}" -eq 1 ]] ; then
# Diff mode is unthreaded
export THREADED=0
print_ln "no_log"
if [[ -d "${LOG_DIR}" ]]; then
print_output "[!] Diff mode started on ""$(date)" "main"
helpers/helpers_emba_dependency_check.sh
+2 -2
View File
@@ -205,7 +205,6 @@ dependency_check()
#######################################################################################
print_output "[*] Network connection:" "no_log"
if [[ "${CONTAINER_NUMBER}" -ne 1 ]]; then
print_output " Internet connection - \\c" "no_log"
@@ -254,6 +253,7 @@ dependency_check()
-H "Authorization: Bearer ${OPENAI_API_KEY}" \
-d @"${CONFIG_DIR}/gpt_template.json" --write-out "%{http_code}" -o /tmp/chatgpt-test.json)
echo "API request finished ... $HTTP_CODE_"
if [[ "${HTTP_CODE_}" -eq 200 ]] ; then
echo -e "${GREEN}""ok""${NC}"
rm /tmp/chatgpt-test.json
@@ -285,7 +285,7 @@ dependency_check()
fi
# Note: we are running into issues in the case where the key can't be verified, but GPT is not enabled at all
# In such a case we will fail the check without the need of GPT
DEP_ERROR=1
# DEP_ERROR=1
fi
if grep -q "Testing phase ended" "${LOG_DIR}"/"${MAIN_LOG_FILE}" 2>/dev/null; then
print_output " Testing phase ended - \\c" "no_log"
helpers/helpers_emba_html_generator.sh
+1
View File
@@ -91,6 +91,7 @@ add_link_tags() {
if ( grep -a -q -E '\[REF\]' "${LINK_FILE}" ) ; then
readarray -t REF_LINKS_L_NUMBER < <(grep -a -n -E '\[REF\].*' "${LINK_FILE}" | cut -d':' -f1 )
for REF_LINK_NUMBER in "${REF_LINKS_L_NUMBER[@]}" ; do
DEPTH="."
REF_LINK="$(sed "${REF_LINK_NUMBER}""q;d" "${LINK_FILE}" | cut -c12- | cut -d'<' -f1 || true)"
URL_REGEX='(www.|https?|ftp|file):\/\/'
if [[ -f "$(echo "${REF_LINK}" | cut -d"#" -f1)" ]] ; then
modules/D02_firmware_diffing_bin_details.sh
+3
View File
@@ -18,6 +18,9 @@
# This module is doing some basic checks on the firmware. It uses mainly
# the functionality from the p02 module but on both firmware images.
export PRE_THREAD_ENA=0
D02_firmware_diffing_bin_details() {
module_log_init "${FUNCNAME[0]}"
modules/D05_firmware_diffing_extractor.sh
+2
View File
@@ -19,6 +19,8 @@
# unblob extraction from module p55. There is currently no deep extraction
# mode supported.
export PRE_THREAD_ENA=0
D05_firmware_diffing_extractor() {
module_log_init "${FUNCNAME[0]}"
modules/D10_firmware_diffing.sh
+247 -150
View File
@@ -21,11 +21,15 @@
D10_firmware_diffing() {
module_log_init "${FUNCNAME[0]}"
module_title "Firmware diff analysis"
local NEG_LOG=0
module_title "Diff analysis of two firmware images"
# we only look at files ranking lt 95 in ssdeep - probably we need to adjust this in the future
local SSDEEP_MIN_RANK=95
# local THREADED=1
# local MAX_MOD_THREADS=10
local NEG_LOG=0
if ! command -v ssdeep > /dev/null ; then
print_output "[-] Missing ssdeep installation"
module_end_log "${FUNCNAME[0]}" 0
@@ -74,157 +78,253 @@ D10_firmware_diffing() {
# lets iterate through the files from firmware 1 and check if they are available in firmware 2
# if they are also available we are going to check the diffs with ssdeep
for FW_FILE1 in "${FW_FILES1[@]}"; do
FW_FILE1="${OUTPUT_DIR_UNBLOB1}""${FW_FILE1#.}"
FW_FILE_NAME1=$(basename "${FW_FILE1}")
# From extraction process we often get a huge amount of files called "gzip.uncompressed"
# Currently we just skip them. In the future we probably need to respect the directory name right before:
# /lib/modules/4.19.163/kernel/net/netfilter/xt_LOG.ko.gz_extract/gzip.uncompressed
# -> the name that we need to take care of is xt_LOG.ko.gz
if [[ "${FW_FILE_NAME1}" == "gzip.uncompressed" ]]; then
continue
fi
# print_output "[*] Testing $FW_FILE1"
# find the file in OUTPUT_DIR_UNBLOB2
mapfile -t FW_FILES1in2 < <(find "${OUTPUT_DIR_UNBLOB2}" -type f -name "${FW_FILE_NAME1}" -exec md5sum {} \; 2>/dev/null | sort -u -k1,1 | cut -d\ -f3)
if [[ "${#FW_FILES1in2[@]}" -ne 0 ]]; then
for FW_FILE2 in "${FW_FILES1in2[@]}"; do
MD5_FW_FILE1=$(md5sum "${FW_FILE1}" | awk '{print $1}')
MD5_FW_FILE2=$(md5sum "${FW_FILE2}" | awk '{print $1}')
if [[ "${MD5_FW_FILE1}" != "${MD5_FW_FILE2}" ]]; then
SSDEEP_OUT=$(ssdeep -d -s "${FW_FILE1}" "${FW_FILE2}")
[[ -z "${SSDEEP_OUT}" ]] && continue
# extract the ssdeep ranking from the output
SSDEEP_RANK=$(echo "${SSDEEP_OUT}" | rev | awk '{print $1}' | rev | tr -d '(' | tr -d ')')
NEG_LOG=1
LOG_FILE_SUB_NAME="diff_report_${FW_FILE_NAME1}.txt"
LOG_PATH_MODULE_SUB="${LOG_PATH_MODULE}"/"${LOG_FILE_SUB_NAME/.txt}"
LOG_FILE_DETAILS="${LOG_PATH_MODULE_SUB}/${LOG_FILE_SUB_NAME}"
! [[ -d "${LOG_PATH_MODULE_SUB}" ]] && mkdir "${LOG_PATH_MODULE_SUB}"
if [[ "${SSDEEP_RANK}" -lt "${SSDEEP_MIN_RANK}" ]]; then
print_ln "no_log"
if [[ "$(file "${FW_FILE1}")" == *"text"* ]]; then
print_output "[+] Found modified ASCII file ${ORANGE}${FW_FILE_NAME1}${GREEN} in 2nd firmware directory - Ranking ${ORANGE}${SSDEEP_RANK}${NC}" "" "${LOG_FILE_DETAILS}"
write_log "[+] Found modified ASCII file ${ORANGE}${FW_FILE_NAME1}${GREEN} in 2nd firmware directory - Ranking ${ORANGE}${SSDEEP_RANK}${NC}" "${LOG_FILE_DETAILS}"
else
print_output "[+] Found modified binary file ${ORANGE}${FW_FILE_NAME1}${GREEN} in 2nd firmware directory - Ranking ${ORANGE}${SSDEEP_RANK}${NC}" "" "${LOG_FILE_DETAILS}"
write_log "[+] Found modified binary file ${ORANGE}${FW_FILE_NAME1}${GREEN} in 2nd firmware directory - Ranking ${ORANGE}${SSDEEP_RANK}${NC}" "${LOG_FILE_DETAILS}"
fi
write_log "" "${LOG_FILE_DETAILS}"
write_log "[*] Base firmware file details to compare with:" "${LOG_FILE_DETAILS}"
write_log "$(indent "$(orange "$(print_path "${FW_FILE1}")")")" "${LOG_FILE_DETAILS}"
write_log "$(indent "$(orange "$(md5sum "${FW_FILE1}")")")" "${LOG_FILE_DETAILS}"
write_log "$(indent "$(orange "$(file "${FW_FILE1}")")")" "${LOG_FILE_DETAILS}"
write_log "" "${LOG_FILE_DETAILS}"
write_log "[*] Second firmware file details:" "${LOG_FILE_DETAILS}"
write_log "$(indent "$(orange "$(print_path "${FW_FILE2}")")")" "${LOG_FILE_DETAILS}"
write_log "$(indent "$(orange "$(md5sum "${FW_FILE2}")")")" "${LOG_FILE_DETAILS}"
write_log "$(indent "$(orange "$(file "${FW_FILE2}")")")" "${LOG_FILE_DETAILS}"
if [[ "$(file "${FW_FILE1}")" == *"text"* ]]; then
sub_module_title "Diff for clear text file ${FW_FILE_NAME1}" "${LOG_FILE_DETAILS}"
# clear text handling - just colordiffing
diff -yb --color=always --suppress-common-lines "${FW_FILE1}" "${FW_FILE2}" > "${LOG_PATH_MODULE_SUB}"/colordiff_"${FW_FILE_NAME1}".txt || true
if [[ -f "${LOG_PATH_MODULE_SUB}"/colordiff_"${FW_FILE_NAME1}".txt ]]; then
print_output "[*] Diffing results from clear text file ${ORANGE}${FW_FILE_NAME1}${NC} logged to ${ORANGE}${LOG_PATH_MODULE_SUB}/colordiff_${FW_FILE_NAME1}.txt${NC}" "no_log"
write_log "" "${LOG_FILE_DETAILS}"
write_log "[*] Diffing results from clear text file ${ORANGE}${FW_FILE_NAME1}${NC}" "${LOG_FILE_DETAILS}"
write_log "" "${LOG_FILE_DETAILS}"
cat "${LOG_PATH_MODULE_SUB}"/colordiff_"$(basename "${FW_FILE1}")".txt >> "${LOG_FILE_DETAILS}"
fi
else
sub_module_title "Diff for binary file ${FW_FILE_NAME1}" "${LOG_FILE_DETAILS}"
# binary handling - colordiffing the hex dump and some radare2 diffing
diff -yb --color=always --suppress-common-lines <(xxd "${FW_FILE1}") <(xxd "${FW_FILE2}") > "${LOG_PATH_MODULE_SUB}"/colordiff_"${FW_FILE_NAME1}".txt || true
if [[ -f "${LOG_PATH_MODULE_SUB}"/colordiff_"${FW_FILE_NAME1}".txt ]]; then
print_output "[*] Diffing results from binary file ${ORANGE}${FW_FILE_NAME1}${NC} logged to ${ORANGE}${LOG_PATH_MODULE_SUB}/colordiff_${FW_FILE_NAME1}.txt${NC}" "no_log"
write_log "" "${LOG_FILE_DETAILS}"
# we only link to the binary colordiff in the web report:
# on the cli output we do not see the complete diff
write_log "[*] Diffing results from binary file ${ORANGE}${FW_FILE_NAME1}${NC}" "${LOG_FILE_DETAILS}"
write_link "${LOG_PATH_MODULE_SUB}/colordiff_${FW_FILE_NAME1}.txt" "${LOG_FILE_DETAILS}"
write_log "" "${LOG_FILE_DETAILS}"
fi
write_log "" "${LOG_FILE_DETAILS}"
# radare2:
sub_module_title "Non matching functions for binary file ${FW_FILE_NAME1}" "${LOG_FILE_DETAILS}"
write_log "" "${LOG_FILE_DETAILS}"
write_log "[*] Non matching functions in ${ORANGE}$(basename "${FW_FILE1}")${NC}:" "${LOG_FILE_DETAILS}"
print_output "[*] Non matching functions in ${ORANGE}$(basename "${FW_FILE1}")${NC} logged to ${ORANGE}${LOG_PATH_MODULE_SUB}/r2_diff_fct_${FW_FILE_NAME1}_${FW_FILE_NAME1}.txt${NC}" "no_log"
# get the functions which are different with radiff2:
radiff2 -AC "${FW_FILE1}" "${FW_FILE2}" 2>/dev/null | grep UNMATCH > "${LOG_PATH_MODULE_SUB}"/r2_diff_fct_"${FW_FILE_NAME1}"_"${FW_FILE_NAME1}".txt || true
if [[ -f "${LOG_PATH_MODULE_SUB}"/r2_diff_fct_"${FW_FILE_NAME1}"_"${FW_FILE_NAME1}".txt ]]; then
cat "${LOG_PATH_MODULE_SUB}"/r2_diff_fct_"${FW_FILE_NAME1}"_"${FW_FILE_NAME1}".txt >> "${LOG_FILE_DETAILS}"
mapfile -t UNMATCHED_FCTs < <(awk '{print $1}' "${LOG_PATH_MODULE_SUB}"/r2_diff_fct_"${FW_FILE_NAME1}"_"${FW_FILE_NAME1}".txt | sort -u)
fi
write_log "" "${LOG_FILE_DETAILS}"
sub_module_title "R2 diff for binary file ${FW_FILE_NAME1}" "${LOG_FILE_DETAILS}"
# let's do a diff on the complete radare2 output:
# shellcheck disable=SC2016
r2 -e io.cache=true -e scr.color=false -A -q -c 'pd $s' "${FW_FILE1}" 2>/dev/null > "${LOG_PATH_MODULE_SUB}"/r2_disasm_"${FW_FILE_NAME1}"_dir1.txt
# shellcheck disable=SC2016
r2 -e io.cache=true -e scr.color=false -A -q -c 'pd $s' "${FW_FILE2}" 2>/dev/null > "${LOG_PATH_MODULE_SUB}"/r2_disasm_"$(basename "${FW_FILE2}")"_dir2.txt
diff -yb --color=always --suppress-common-lines "${LOG_PATH_MODULE_SUB}"/r2_disasm_"$(basename "${FW_FILE1}")"_dir1.txt "${LOG_PATH_MODULE_SUB}"/r2_disasm_"$(basename "${FW_FILE2}")"_dir2.txt 2>/dev/null > "${LOG_PATH_MODULE_SUB}"/colordiff_radare2_disasm_"$(basename "${FW_FILE1}")".txt || true
if [[ -f "${LOG_PATH_MODULE_SUB}"/colordiff_radare2_disasm_"${FW_FILE_NAME1}".txt ]]; then
print_output "[*] Radare2 binary diffing results from binary file ${ORANGE}${FW_FILE_NAME1}${NC} logged to ${ORANGE}${LOG_FILE_DETAILS}${NC}." "no_log"
write_log "[*] Radare2 binary diffing results from binary file ${ORANGE}${FW_FILE_NAME1}${NC}" "${LOG_FILE_DETAILS}"
write_link "${LOG_PATH_MODULE_SUB}"/colordiff_radare2_disasm_"${FW_FILE_NAME1}".txt "${LOG_FILE_DETAILS}"
write_log "" "${LOG_FILE_DETAILS}"
fi
# now we check diff all the functions with differences and generate a xdot and png picture
# see also https://book.rada.re/tools/radiff2/binary_diffing.html
! [[ -d "${LOG_PATH_MODULE}"/r2_fct_graphing/ ]] && mkdir "${LOG_PATH_MODULE}"/r2_fct_graphing/
write_log "[*] Radare2 binary function diff for ${ORANGE}${FW_FILE_NAME1}${NC}:" "${LOG_FILE_DETAILS}"
for FCT in "${UNMATCHED_FCTs[@]}"; do
write_log "" "${LOG_FILE_DETAILS}"
radiff2 -e bin.cache=true -md -g "${FCT}" "${FW_FILE2}" "${FW_FILE1}" 2>/dev/null > "${LOG_PATH_MODULE}"/r2_fct_graphing/r2_fct_graph_"${FW_FILE_NAME1}"_"${FCT}".xdot
# we only print the graph if the log file was generated and has content and it has multiple addresses (0x) included
if [[ -s "${LOG_PATH_MODULE}"/r2_fct_graphing/r2_fct_graph_"${FW_FILE_NAME1}"_"${FCT}".xdot ]]; then
if [[ "$(grep -c "0x" "${LOG_PATH_MODULE}"/r2_fct_graphing/r2_fct_graph_"${FW_FILE_NAME1}"_"${FCT}".xdot 2>/dev/null)" -gt 1 ]]; then
print_output "[*] Generating png for function ${FCT} of binary ${FW_FILE_NAME1}" "no_log"
dot -Tpng "${LOG_PATH_MODULE}"/r2_fct_graphing/r2_fct_graph_"${FW_FILE_NAME1}"_"${FCT}".xdot 2>/dev/null > "${LOG_PATH_MODULE}"/r2_fct_graphing/r2_fct_graph_"${FW_FILE_NAME1}"_"${FCT}".png || true
fi
fi
if [[ -s "${LOG_PATH_MODULE}/r2_fct_graphing/r2_fct_graph_${FW_FILE_NAME1}_${FCT}.png" ]]; then
write_log "[*] Radare2 binary function diff for function ${ORANGE}${FCT}${NC} in binary ${ORANGE}${FW_FILE_NAME1}${NC}" "${LOG_FILE_DETAILS}"
write_link "${LOG_PATH_MODULE}/r2_fct_graphing/r2_fct_graph_${FW_FILE_NAME1}_${FCT}.png" "${LOG_FILE_DETAILS}"
else
write_log "[-] No function graph available for ${ORANGE}${FCT}${NC}" "${LOG_FILE_DETAILS}"
fi
done
fi
write_log "" "${LOG_FILE_DETAILS}"
fi
fi
done
NEG_LOG=1
if [[ "${THREADED}" -eq 1 ]]; then
analyse_fw_files "${FW_FILE1}" &
local TMP_PID="$!"
store_kill_pids "${TMP_PID}"
WAIT_PIDS_D10+=( "${TMP_PID}" )
max_pids_protection "${MAX_MOD_THREADS}" "${WAIT_PIDS_D10[@]}"
else
# deleted files in 2nd firmware directory
if [[ -f "${FW_FILE1}" ]]; then
if file "${FW_FILE1}" | grep -q ASCII; then
cp "${FW_FILE1}" "${LOG_PATH_MODULE}"/"${FW_FILE_NAME1}".log
print_output "[+] Firmware ASCII file ${ORANGE}${FW_FILE_NAME1}${GREEN} is deleted in ${ORANGE}${OUTPUT_DIR_UNBLOB2}${GREEN}."
else
print_output "[+] Firmware binary file ${ORANGE}${FW_FILE_NAME1}${GREEN} is deleted in ${ORANGE}${OUTPUT_DIR_UNBLOB2}${GREEN}."
fi
[[ -f "${LOG_PATH_MODULE}"/"${FW_FILE_NAME1}".log ]] && write_link "${LOG_PATH_MODULE}/${FW_FILE_NAME1}.log"
fi
# echo "Testing ${FW_FILE1}"
analyse_fw_files "${FW_FILE1}"
fi
done
[[ "$THREADED" -eq 1 ]] && wait_for_pid "${WAIT_PIDS_D10[@]}"
check_for_new_files
module_end_log "${FUNCNAME[0]}" "$NEG_LOG"
module_end_log "${FUNCNAME[0]}" "${NEG_LOG}"
}
analyse_fw_files() {
FW_FILE1="${1:-}"
# as we searched directly in the extracted path, we need to adjust the file path with the unblob extraction directory
FW_FILE1="${OUTPUT_DIR_UNBLOB1}""${FW_FILE1#.}"
FW_FILE_NAME1=$(basename "${FW_FILE1}")
# From extraction process we often get a huge amount of files called "gzip.uncompressed"
# Currently we just skip them. In the future we probably need to respect the directory name right before:
# /lib/modules/4.19.163/kernel/net/netfilter/xt_LOG.ko.gz_extract/gzip.uncompressed
# -> the name that we need to take care of is xt_LOG.ko.gz
if [[ "${FW_FILE_NAME1}" == "gzip.uncompressed" ]]; then
return
fi
# print_output "[*] Testing $FW_FILE1"
# find the file in OUTPUT_DIR_UNBLOB2
mapfile -t FW_FILES1in2 < <(find "${OUTPUT_DIR_UNBLOB2}" -type f -name "${FW_FILE_NAME1}" -exec md5sum {} \; 2>/dev/null | sort -u -k1,1 | cut -d\ -f3)
if [[ "${#FW_FILES1in2[@]}" -ne 0 ]]; then
for FW_FILE2 in "${FW_FILES1in2[@]}"; do
MD5_FW_FILE1=$(md5sum "${FW_FILE1}" | awk '{print $1}')
MD5_FW_FILE2=$(md5sum "${FW_FILE2}" | awk '{print $1}')
FW_FILE_NAME2=$(basename "${FW_FILE2}")
if [[ "${MD5_FW_FILE1}" != "${MD5_FW_FILE2}" ]]; then
# fuzzy hash diffing here:
SSDEEP_OUT=$(ssdeep -d -s "${FW_FILE1}" "${FW_FILE2}")
[[ -z "${SSDEEP_OUT}" ]] && return
# extract the ssdeep ranking from the output
SSDEEP_RANK=$(echo "${SSDEEP_OUT}" | rev | awk '{print $1}' | rev | tr -d '(' | tr -d ')')
NEG_LOG=1
LOG_FILE_SUB_NAME="diff_report_${FW_FILE_NAME1}.txt"
LOG_PATH_MODULE_SUB="${LOG_PATH_MODULE}"/"${LOG_FILE_SUB_NAME/.txt}"
LOG_FILE_DETAILS="${LOG_PATH_MODULE_SUB}/${LOG_FILE_SUB_NAME}"
! [[ -d "${LOG_PATH_MODULE_SUB}" ]] && mkdir "${LOG_PATH_MODULE_SUB}"
if [[ "${SSDEEP_RANK}" -lt "${SSDEEP_MIN_RANK}" ]]; then
print_ln "no_log"
if [[ "$(file "${FW_FILE1}")" == *"text"* ]]; then
print_output "[+] Found modified ASCII file ${ORANGE}${FW_FILE_NAME1}${GREEN} in 2nd firmware directory - Ranking ${ORANGE}${SSDEEP_RANK}${NC}" "" "${LOG_FILE_DETAILS}"
write_log "[+] Found modified ASCII file ${ORANGE}${FW_FILE_NAME1}${GREEN} in 2nd firmware directory - Ranking ${ORANGE}${SSDEEP_RANK}${NC}" "${LOG_FILE_DETAILS}"
else
print_output "[+] Found modified binary file ${ORANGE}${FW_FILE_NAME1}${GREEN} in 2nd firmware directory - Ranking ${ORANGE}${SSDEEP_RANK}${NC}" "" "${LOG_FILE_DETAILS}"
write_log "[+] Found modified binary file ${ORANGE}${FW_FILE_NAME1}${GREEN} in 2nd firmware directory - Ranking ${ORANGE}${SSDEEP_RANK}${NC}" "${LOG_FILE_DETAILS}"
fi
write_log "" "${LOG_FILE_DETAILS}"
write_log "[*] Base firmware file details to compare with:" "${LOG_FILE_DETAILS}"
write_log "$(indent "$(orange "$(print_path "${FW_FILE1}")")")" "${LOG_FILE_DETAILS}"
write_log "$(indent "$(orange "$(md5sum "${FW_FILE1}")")")" "${LOG_FILE_DETAILS}"
write_log "$(indent "$(orange "$(file "${FW_FILE1}")")")" "${LOG_FILE_DETAILS}"
write_log "" "${LOG_FILE_DETAILS}"
write_log "[*] Second firmware file details:" "${LOG_FILE_DETAILS}"
write_log "$(indent "$(orange "$(print_path "${FW_FILE2}")")")" "${LOG_FILE_DETAILS}"
write_log "$(indent "$(orange "$(md5sum "${FW_FILE2}")")")" "${LOG_FILE_DETAILS}"
write_log "$(indent "$(orange "$(file "${FW_FILE2}")")")" "${LOG_FILE_DETAILS}"
if [[ "$(file "${FW_FILE1}")" == *"text"* ]]; then
sub_module_title "Diff for clear text file ${FW_FILE_NAME1}" "${LOG_FILE_DETAILS}"
# clear text handling - just colordiffing
# colored diff output not possible in -y mode to file output -> change the diffs with sed
diff -yb --color=always --suppress-common-lines "${FW_FILE1}" "${FW_FILE2}" | sed 's/.*[[:blank:]]|[[:blank:]].*/\x1b[32m&\x1b[0m/' > "${LOG_PATH_MODULE_SUB}"/colordiff_"${FW_FILE_NAME1}".txt || true
if [[ -f "${LOG_PATH_MODULE_SUB}"/colordiff_"${FW_FILE_NAME1}".txt ]]; then
print_output "[*] Diffing results from clear text file ${ORANGE}${FW_FILE_NAME1}${NC} logged to ${ORANGE}${LOG_PATH_MODULE_SUB}/colordiff_${FW_FILE_NAME1}.txt${NC}" "no_log"
write_log "" "${LOG_FILE_DETAILS}"
write_log "[*] Diffing results from clear text file ${ORANGE}${FW_FILE_NAME1}${NC}" "${LOG_FILE_DETAILS}"
write_log "" "${LOG_FILE_DETAILS}"
cat "${LOG_PATH_MODULE_SUB}"/colordiff_"$(basename "${FW_FILE1}")".txt >> "${LOG_FILE_DETAILS}"
fi
else
sub_module_title "Diff for binary file ${FW_FILE_NAME1}" "${LOG_FILE_DETAILS}"
# binary handling - colordiffing the hex dump and some radare2 diffing
diff -yb --color=always --suppress-common-lines <(xxd "${FW_FILE1}") <(xxd "${FW_FILE2}") > "${LOG_PATH_MODULE_SUB}"/colordiff_"${FW_FILE_NAME1}".txt || true
if [[ -f "${LOG_PATH_MODULE_SUB}"/colordiff_"${FW_FILE_NAME1}".txt ]]; then
print_output "[*] Diffing results from binary file ${ORANGE}${FW_FILE_NAME1}${NC} logged to ${ORANGE}${LOG_PATH_MODULE_SUB}/colordiff_${FW_FILE_NAME1}.txt${NC}" "no_log"
write_log "" "${LOG_FILE_DETAILS}"
# we only link to the binary colordiff in the web report:
# on the cli output we do not see the complete diff
write_log "[*] Diffing results from binary file ${ORANGE}${FW_FILE_NAME1}${NC}" "${LOG_FILE_DETAILS}"
write_link "${LOG_PATH_MODULE_SUB}/colordiff_${FW_FILE_NAME1}.txt" "${LOG_FILE_DETAILS}"
write_log "" "${LOG_FILE_DETAILS}"
fi
write_log "" "${LOG_FILE_DETAILS}"
# radare2:
# get the functions which are different with radiff2:
radiff2 -AC "${FW_FILE1}" "${FW_FILE2}" 2>/dev/null | grep UNMATCH > "${LOG_PATH_MODULE_SUB}"/r2_diff_fct_"${FW_FILE_NAME1}"_"${FW_FILE_NAME1}".txt || true
if [[ -s "${LOG_PATH_MODULE_SUB}"/r2_diff_fct_"${FW_FILE_NAME1}"_"${FW_FILE_NAME1}".txt ]]; then
sub_module_title "Non matching functions for binary file ${FW_FILE_NAME1}" "${LOG_FILE_DETAILS}"
cat "${LOG_PATH_MODULE_SUB}"/r2_diff_fct_"${FW_FILE_NAME1}"_"${FW_FILE_NAME1}".txt
write_log "" "${LOG_FILE_DETAILS}"
print_ln "no_log"
write_log "[*] Non matching functions in ${ORANGE}$(basename "${FW_FILE1}")${NC}:" "${LOG_FILE_DETAILS}"
print_output "[*] Non matching functions in ${ORANGE}$(basename "${FW_FILE1}")${NC} logged to ${ORANGE}${LOG_PATH_MODULE_SUB}/r2_diff_fct_${FW_FILE_NAME1}_${FW_FILE_NAME1}.txt${NC}" "no_log"
cat "${LOG_PATH_MODULE_SUB}"/r2_diff_fct_"${FW_FILE_NAME1}"_"${FW_FILE_NAME1}".txt >> "${LOG_FILE_DETAILS}"
mapfile -t UNMATCHED_FCTs < <(awk '{print $1}' "${LOG_PATH_MODULE_SUB}"/r2_diff_fct_"${FW_FILE_NAME1}"_"${FW_FILE_NAME1}".txt | sort -u)
else
write_log "[-] No function diff available for ${ORANGE}${FW_FILE_NAME1}${NC}" "${LOG_FILE_DETAILS}"
fi
write_log "" "${LOG_FILE_DETAILS}"
# let's do a diff on the complete radare2 output:
# create disassembly from file in first directory:
# shellcheck disable=SC2016
r2 -e io.cache=true -e scr.color=false -A -q -c 'pd $s' "${FW_FILE1}" 2>/dev/null > "${LOG_PATH_MODULE_SUB}"/r2_disasm_"${FW_FILE_NAME1}"_dir1.txt
# create disassembly from file in second directory:
# shellcheck disable=SC2016
r2 -e io.cache=true -e scr.color=false -A -q -c 'pd $s' "${FW_FILE2}" 2>/dev/null > "${LOG_PATH_MODULE_SUB}"/r2_disasm_"$(basename "${FW_FILE2}")"_dir2.txt
# create diff of both disassemblies:
diff -yb --color=always --suppress-common-lines "${LOG_PATH_MODULE_SUB}"/r2_disasm_"$(basename "${FW_FILE1}")"_dir1.txt "${LOG_PATH_MODULE_SUB}"/r2_disasm_"$(basename "${FW_FILE2}")"_dir2.txt 2>/dev/null > "${LOG_PATH_MODULE_SUB}"/colordiff_radare2_disasm_"$(basename "${FW_FILE1}")".txt || true
if [[ -s "${LOG_PATH_MODULE_SUB}"/colordiff_radare2_disasm_"${FW_FILE_NAME1}".txt ]]; then
sub_module_title "R2 diff for binary file ${FW_FILE_NAME1}" "${LOG_FILE_DETAILS}"
print_output "[*] Radare2 binary diffing results from binary file ${ORANGE}${FW_FILE_NAME1}${NC} logged to ${ORANGE}${LOG_FILE_DETAILS}${NC}." "no_log"
write_log "[*] Radare2 binary diffing results from binary file ${ORANGE}${FW_FILE_NAME1}${NC}" "${LOG_FILE_DETAILS}"
write_link "${LOG_PATH_MODULE_SUB}"/colordiff_radare2_disasm_"${FW_FILE_NAME1}".txt "${LOG_FILE_DETAILS}"
write_log "" "${LOG_FILE_DETAILS}"
fi
# now we check diff all the functions with differences and generate a xdot and png picture
# see also https://book.rada.re/tools/radiff2/binary_diffing.html
! [[ -d "${LOG_PATH_MODULE}"/r2_fct_graphing/ ]] && mkdir "${LOG_PATH_MODULE}"/r2_fct_graphing/
write_log "[*] Radare2 binary function diff for ${ORANGE}${FW_FILE_NAME1}${NC}:" "${LOG_FILE_DETAILS}"
# walk through all changed functions:
for FCT in "${UNMATCHED_FCTs[@]}"; do
# Threading currently mangles the links
if [[ "$THREADED" -eq 2 ]]; then
analyse_bin_fct "${FCT}" &
local TMP_PID="$!"
store_kill_pids "$TMP_PID"
WAIT_PIDS_D10+=( "$TMP_PID" )
max_pids_protection "$MAX_MOD_THREADS" "${WAIT_PIDS_D10[@]}"
else
analyse_bin_fct "${FCT}"
fi
done
[[ "$THREADED" -eq 1 ]] && wait_for_pid "${WAIT_PIDS_D10[@]}"
fi
write_log "" "${LOG_FILE_DETAILS}"
fi
fi
done
else
# deleted files in 2nd firmware directory
if [[ -f "${FW_FILE1}" ]]; then
if file "${FW_FILE1}" | grep -q ASCII; then
cp "${FW_FILE1}" "${LOG_PATH_MODULE}"/"${FW_FILE_NAME1}".log
print_output "[+] Firmware ASCII file ${ORANGE}${FW_FILE_NAME1}${GREEN} is deleted in ${ORANGE}${OUTPUT_DIR_UNBLOB2}${GREEN}."
else
print_output "[+] Firmware binary file ${ORANGE}${FW_FILE_NAME1}${GREEN} is deleted in ${ORANGE}${OUTPUT_DIR_UNBLOB2}${GREEN}."
fi
if [[ -f "${LOG_PATH_MODULE}"/"${FW_FILE_NAME1}".log ]]; then
write_link "${LOG_PATH_MODULE}/${FW_FILE_NAME1}.log"
fi
fi
fi
}
analyse_bin_fct() {
local FCT="${1:-}"
write_log "" "${LOG_FILE_DETAILS}"
radiff2 -e bin.cache=true -md -g "${FCT}" "${FW_FILE2}" "${FW_FILE1}" 2>/dev/null > "${LOG_PATH_MODULE}"/r2_fct_graphing/r2_fct_graph_"${FW_FILE_NAME1}"_"${FCT}".xdot
# we only print the graph if the log file was generated and has content and it has multiple addresses (0x) included
if [[ -s "${LOG_PATH_MODULE}"/r2_fct_graphing/r2_fct_graph_"${FW_FILE_NAME1}"_"${FCT}".xdot ]]; then
if [[ "$(grep -c "0x" "${LOG_PATH_MODULE}"/r2_fct_graphing/r2_fct_graph_"${FW_FILE_NAME1}"_"${FCT}".xdot 2>/dev/null)" -gt 1 ]]; then
print_output "[*] Generating diff image for function ${ORANGE}${FCT}${NC} of binary ${ORANGE}${FW_FILE_NAME1}${NC}" "no_log"
dot -Tpng "${LOG_PATH_MODULE}"/r2_fct_graphing/r2_fct_graph_"${FW_FILE_NAME1}"_"${FCT}".xdot 2>/dev/null > "${LOG_PATH_MODULE}"/r2_fct_graphing/r2_fct_graph_"${FW_FILE_NAME1}"_"${FCT}".png || true
print_output "[*] Generating disasm for function ${ORANGE}${FCT}${NC} of binary ${ORANGE}${FW_FILE_NAME1}${NC}" "no_log"
# now we need to generate the disassembly of the current function of both files to include it in the report for further manual tear-down
write_log "[*] Disassembly function ${ORANGE}${FCT}${NC} of ${ORANGE}${FW_FILE_NAME1}${NC} in ${ORANGE}first${NC} firmware directory" "${LOG_PATH_MODULE_SUB}"/r2_disasm_"${FW_FILE_NAME1}"_"${FCT}"_dir1.txt
write_log "" "${LOG_PATH_MODULE_SUB}"/r2_disasm_"${FW_FILE_NAME1}"_"${FCT}"_dir1.txt
r2 -e io.cache=true -e scr.color=false -A -q -c 'pdf @ '"${FCT}" "${FW_FILE1}" 2>/dev/null >> "${LOG_PATH_MODULE_SUB}"/r2_disasm_"${FW_FILE_NAME1}"_"${FCT}"_dir1.txt || true
write_log "[*] Disassembly function ${ORANGE}${FCT}${NC} of ${ORANGE}${FW_FILE_NAME2}${NC} in ${ORANGE}second${NC} firmware directory" "${LOG_PATH_MODULE_SUB}"/r2_disasm_"${FW_FILE_NAME2}"_"${FCT}"_dir2.txt
write_log "" "${LOG_PATH_MODULE_SUB}"/r2_disasm_"${FW_FILE_NAME2}"_"${FCT}"_dir2.txt
r2 -e io.cache=true -e scr.color=false -A -q -c 'pdf @ '"${FCT}" "${FW_FILE2}" 2>/dev/null >> "${LOG_PATH_MODULE_SUB}"/r2_disasm_"${FW_FILE_NAME2}"_"${FCT}"_dir2.txt || true
fi
fi
if [[ -s "${LOG_PATH_MODULE}/r2_fct_graphing/r2_fct_graph_${FW_FILE_NAME1}_${FCT}.png" ]]; then
write_log "" "${LOG_FILE_DETAILS}"
write_log "[*] Radare2 binary function diff for function ${ORANGE}${FCT}${NC} in binary ${ORANGE}${FW_FILE_NAME1}${NC}" "${LOG_FILE_DETAILS}"
write_link "${LOG_PATH_MODULE}/r2_fct_graphing/r2_fct_graph_${FW_FILE_NAME1}_${FCT}.png" "${LOG_FILE_DETAILS}"
if [[ -s "${LOG_PATH_MODULE_SUB}"/r2_disasm_"${FW_FILE_NAME1}"_"${FCT}"_dir1.txt ]]; then
write_log "$(indent "Disassembly function ${ORANGE}${FCT}${NC} of ${ORANGE}${FW_FILE_NAME1}${NC} in ${ORANGE}first${NC} firmware directory")" "${LOG_FILE_DETAILS}"
write_link "${LOG_PATH_MODULE_SUB}"/r2_disasm_"${FW_FILE_NAME1}"_"${FCT}"_dir1.txt "${LOG_FILE_DETAILS}"
fi
if [[ -s "${LOG_PATH_MODULE_SUB}"/r2_disasm_"${FW_FILE_NAME2}"_"${FCT}"_dir2.txt ]]; then
write_log "$(indent "Disassembly function ${ORANGE}${FCT}${NC} of ${ORANGE}${FW_FILE_NAME2} in ${ORANGE}second${NC} firmware directory")" "${LOG_FILE_DETAILS}"
write_link "${LOG_PATH_MODULE_SUB}"/r2_disasm_"${FW_FILE_NAME2}"_"${FCT}"_dir2.txt "${LOG_FILE_DETAILS}"
fi
if [[ -s "${LOG_PATH_MODULE_SUB}"/r2_disasm_"${FW_FILE_NAME1}"_"${FCT}"_dir1.txt ]] && [[ -s "${LOG_PATH_MODULE_SUB}"/r2_disasm_"${FW_FILE_NAME2}"_"${FCT}"_dir2.txt ]]; then
# letz diff it
# extract only valid disassembly code:
grep "0x" "${LOG_PATH_MODULE_SUB}"/r2_disasm_"${FW_FILE_NAME1}"_"${FCT}"_dir1.txt | grep -v "; arg \|; var " | grep -v -E " XREF(S)? from " \
| sed 's/[^0x]*0x/0x/' | cut -d\ -f2- | sed 's/^[[:blank:]]*//' > "${LOG_PATH_MODULE_SUB}"/r2_disasm_"${FW_FILE_NAME1}"_"${FCT}"_mod_dir1.txt || true
grep "0x" "${LOG_PATH_MODULE_SUB}"/r2_disasm_"${FW_FILE_NAME2}"_"${FCT}"_dir2.txt | grep -v "; arg \|; var " | grep -v -E " XREF(S)? from " \
| sed 's/[^0x]*0x/0x/' | cut -d\ -f2- | sed 's/^[[:blank:]]*//' > "${LOG_PATH_MODULE_SUB}"/r2_disasm_"${FW_FILE_NAME2}"_"${FCT}"_mod_dir2.txt || true
# if we have both disassemblies we can diff them:
if [[ -s "${LOG_PATH_MODULE_SUB}"/r2_disasm_"${FW_FILE_NAME1}"_"${FCT}"_mod_dir1.txt ]] && [[ -s "${LOG_PATH_MODULE_SUB}"/r2_disasm_"${FW_FILE_NAME2}"_"${FCT}"_mod_dir2.txt ]]; then
write_log "[*] Function diff for function ${ORANGE}${FCT}${NC} - file ${ORANGE}${FW_FILE_NAME2}${NC}" "${LOG_PATH_MODULE_SUB}"/r2_disasm_"${FW_FILE_NAME1}"_"${FCT}"_diff.txt
write_log "" "${LOG_PATH_MODULE_SUB}"/r2_disasm_"${FW_FILE_NAME1}"_"${FCT}"_diff.txt
# colorize diff changes orange (see the sed part):
diff -yb --color=always "${LOG_PATH_MODULE_SUB}"/r2_disasm_"${FW_FILE_NAME1}"_"${FCT}"_mod_dir1.txt "${LOG_PATH_MODULE_SUB}"/r2_disasm_"${FW_FILE_NAME2}"_"${FCT}"_mod_dir2.txt \
| sed 's/.*[[:blank:]]|[[:blank:]].*/\x1b[33m&\x1b[0m/' >> "${LOG_PATH_MODULE_SUB}"/r2_disasm_"${FW_FILE_NAME1}"_"${FCT}"_diff.txt || true
fi
if [[ -s "${LOG_PATH_MODULE_SUB}"/r2_disasm_"${FW_FILE_NAME2}"_"${FCT}"_diff.txt ]]; then
write_log "$(indent "Diffing disassembly function ${ORANGE}${FCT}${NC} of ${ORANGE}${FW_FILE_NAME2}${NC}")" "${LOG_FILE_DETAILS}"
write_link "${LOG_PATH_MODULE_SUB}"/r2_disasm_"${FW_FILE_NAME2}"_"${FCT}"_diff.txt "${LOG_FILE_DETAILS}"
fi
fi
# else
# write_log "[-] No function graph available for ${ORANGE}${FCT}${NC}" "${LOG_FILE_DETAILS}"
fi
}
check_for_new_files() {
@@ -249,9 +349,6 @@ check_for_new_files() {
print_output "[+] Firmware binary file ${ORANGE}${FW_FILE_NAME2}${GREEN} is a new file in ${ORANGE}${OUTPUT_DIR_UNBLOB2}${GREEN}."
fi
fi
fi
done
}