e-m-b-a/emba
1
0
Fork 0
mirror of https://github.com/e-m-b-a/emba.git synced 2026-10-11 22:09:54 +00:00
Code Issues Packages Projects Releases Wiki Activity

rework AI integration

Browse Source
This commit is contained in:
Michael Messner
2026-05-21 13:57:43 +02:00
parent 14652b4080
commit ecc908d62a
24 changed files with 1073 additions and 364 deletions
Download Patch File Download Diff File Expand all files Collapse all files
config/ai_config.env.template
+8
View File
@@ -0,0 +1,8 @@
LOCAL_AI_IP="192.168.111.1"
# EMBA checks if this model is available for chatting
# if this model is not available EMBA can't proceed
LOCAL_AI_MODEL="Qwen2.5-Coder-7B-Instruct-GGUF"
# if EMBA will finish earlier with the testing phase she will wait for Q03 until AI requests are done or the timeperiod is over
AI_MIN_RUNTIME="12h"
# maximal code size for AI analysis
AI_MAX_CHARS_TO_ANALYSE=5000
config/gpt_config.env.template
-3
View File
@@ -1,3 +0,0 @@
# OPENAI_API_KEY=""
# GPT_QUESTION="For the following code I need you to tell me how an attacker could exploit it and point out all vulnerabilities."
# MAXMIUM_GPT_PRIO=3
helpers/helpers_emba_defaults.sh
+6 -1
View File
@@ -91,7 +91,7 @@ set_defaults() {
export SHELLCHECK=1
export GPT_OPTION=0 # 0 -> off 1-> unpaid plan 2 -> no rate-limit
export AI_OPTION=0 # 0 -> off, 1-> unpaid plan, 2 -> no rate-limit, 3 -> LocalAI Server
export GPT_QUESTION="For the following code I need you to tell me how an attacker could exploit it and point out all vulnerabilities:"
export MINIMUM_GPT_PRIO=1 # everything above this value gets checked
@@ -175,6 +175,9 @@ set_defaults() {
export SVG_BUILD_TIMEOUT="12h"
export SVG_FILE="${LOG_PATH_MODULE}/EMBA-dependency-map.svg"
DEPENDENCY_TRACK_ENABLED=0
OPENAI_API_KEY=""
# we can enable/disable the s08 submodules with the following array configuration
# -> just comment the submodule that should not be used
# usually this should be done via a scan-profile
@@ -209,6 +212,7 @@ set_log_paths() {
export P02_LOG="${LOG_DIR}/p02_firmware_bin_file_check.txt"
export P02_LOG_DIR="${P02_LOG/\.txt/\/}"
export P02_CSV_LOG="${CSV_DIR}/p02_firmware_bin_file_check.csv"
export P99_LOG="${LOG_DIR}/p99_prepare_analyzer.txt"
export P99_CSV_LOG="${CSV_DIR}/p99_prepare_analyzer.csv"
export P55_LOG="${LOG_DIR}/p55_unblob_extractor.txt"
export P60_LOG="${LOG_DIR}/p60_deep_extractor.txt"
@@ -270,6 +274,7 @@ set_log_paths() {
export S118_LOG="${LOG_DIR}/s118_busybox_verifier.txt"
export S118_LOG_DIR="${S118_LOG/\.txt/\/}"
export Q02_LOG="${LOG_DIR}/q02_openai_question.txt"
export Q03_LOG="${LOG_DIR}/q03_localai_connector.txt"
export L10_LOG="${LOG_DIR}/l10_system_emulator.txt"
export L10_SYS_EMU_RESULTS="${LOG_DIR}/emulator_online_results.log"
export L15_LOG="${LOG_DIR}/l15_emulated_checks_init.txt"
helpers/helpers_emba_dependency_check.sh
+6 -6
View File
@@ -297,7 +297,7 @@ dependency_check() {
check_nvd_db "${lNVD_GITHUB_HASH}"
fi
else
echo -e "${RED}""not ok""${NC}"
echo -e "${RED}not ok${NC}"
print_output "[!] Warning: EMBA has NO internet connection!" "no_log"
print_output "[!] Warning: Update checks and multiple EMBA modules are disabled!" "no_log"
print_output "[!] Warning: GPT (Q02), kernel verification (S26) and further online modules are disabled!" "no_log"
@@ -306,14 +306,14 @@ dependency_check() {
# running into this in Quest container and on host, but not on isolated EMBA container (as it is CONTAINER_NUMBER 1):
if [[ "${CONTAINER_NUMBER}" -ne 1 ]]; then
if [[ -f "${CONFIG_DIR}/gpt_config.env" ]]; then
if grep -v -q "#" "${CONFIG_DIR}/gpt_config.env"; then
# readin gpt_config.env
if [[ -f "${CONFIG_DIR}/ai_config.env" ]]; then
if grep -v -q "#" "${CONFIG_DIR}/ai_config.env"; then
# readin ai_config.env
while read -r LINE; do
if [[ "${LINE}" == *'='* ]] && [[ "${LINE}" != '#'* ]]; then
export "$(echo "${LINE}" | xargs)"
fi
done <"${CONFIG_DIR}/gpt_config.env"
done <"${CONFIG_DIR}/ai_config.env"
fi
fi
@@ -344,7 +344,7 @@ dependency_check() {
print_output "$(indent "ChatGPT-API key not set - ${ORANGE}see https://github.com/e-m-b-a/emba/wiki/AI-supported-firmware-analysis for more information${NC}")" "no_log"
# The following if clause is currently not working! We have not loaded the profile in this stage
# TODO: Find a workaround!
if [[ "${GPT_OPTION}" -eq 1 ]]; then
if [[ "${AI_OPTION}" -eq 1 ]]; then
DEP_ERROR=1
fi
else
helpers/helpers_emba_helpers.sh
+38 -1
View File
@@ -42,6 +42,43 @@ run_web_reporter_mod_name() {
fi
}
# simple markdown to EMBA log parser
# Parameter 1: markdown file
# Parameter 2: EMBA log file to create and log to
# Parameter 3: How many # symbols are used for the first headline -> module_title
parse_markdown_to_emba_txt() {
local lMD_LOG="${1:-}"
local lEMBA_TXT_LOG="${2:-}"
# how many # signs are used for the first headline
local lHEADLINE_STARTER="${3:-1}"
[[ -z "${lEMBA_TXT_LOG}" ]] && lEMBA_TXT_LOG="${lMD_LOG//\.md/\.txt}"
[[ ! -f "${lEMBA_TXT_LOG}" ]] && touch "${lEMBA_TXT_LOG}"
local lMD_LINE=""
local lSTORED_IFS="${IFS}"
IFS=''
while read -r lMD_LINE; do
if [[ "${lMD_LINE}" =~ ^#{$((lHEADLINE_STARTER+1))}.* ]]; then
# other headlines -> sub-module-titles
# remove the # signs
lMD_LINE="${lMD_LINE//#}"
lMD_LINE="${lMD_LINE#\ }"
sub_module_title "${lMD_LINE}" "${lEMBA_TXT_LOG}"
elif [[ "${lMD_LINE}" =~ ^#{${lHEADLINE_STARTER}}\ .* ]]; then
# main headline
# remove the # signs
lMD_LINE="${lMD_LINE:${lHEADLINE_STARTER}}"
lMD_LINE="${lMD_LINE#\ }"
module_title "${lMD_LINE}" "${lEMBA_TXT_LOG}"
else
lMD_LINE=${lMD_LINE//\*/}
write_log "${lMD_LINE}" "${lEMBA_TXT_LOG}"
fi
done < "${lMD_LOG}"
IFS="${lSTORED_IFS}"
}
wait_for_pid() {
local lWAIT_PIDS_ARR=("$@")
local lPID=""
@@ -54,7 +91,7 @@ wait_for_pid() {
continue
fi
while [[ -e /proc/"${lPID}" ]]; do
print_debug "[*] wait pid protection - running pid: ${lPID}" "no_log"
# print_debug "[*] wait pid protection - running pid: ${lPID}" "no_log"
print_dot
# if S115 is running we have to kill old qemu processes
if [[ -f "${LOG_DIR}/${MAIN_LOG_FILE}" ]] && [[ $(grep -i -c S115_ "${LOG_DIR}/${MAIN_LOG_FILE}") -gt 0 && -v QRUNTIME ]]; then
helpers/helpers_emba_html_generator.sh
+15 -7
View File
@@ -113,18 +113,24 @@ add_link_tags() {
lREF_ANCHOR="$(echo "${lREF_LINK}" | cut -d"#" -f2 || true)"
lREF_LINK="$(echo "${lREF_LINK}" | cut -d"#" -f1 || true)"
fi
print_debug "[*] Generate info_file - REF link ${lREF_LINK} / BACK_LINK ${lBACK_LINK}" "no_log"
# in some cases we link to files from different modules. Now we check if the html file is already
# in some cases we link to files from different modules. Now we check if the html report file is already
# generated and we can directly use the already available html report file
lORIG_SRC_MODULE_DIR=$(dirname "${lREF_LINK}")
lORIG_SRC_MODULE_DIR=${lORIG_SRC_MODULE_DIR//*\//}
lORIG_SRC_MODULE_DIR=$(echo "${lREF_LINK}" | tr '/' '\n' | grep -E "^[psqfl][0-9]{2,3}_.*")
print_debug "[*] REF link ${lREF_LINK} / BACK_LINK ${lBACK_LINK} / lORIG_SRC_MODULE_DIR: ${lORIG_SRC_MODULE_DIR} / lLINK_FILE: ${lLINK_FILE}" "no_log"
if [[ -f "${ABS_HTML_PATH%/}/${lORIG_SRC_MODULE_DIR}/${lMD5_REF_LINK}" ]]; then
print_debug "[*] Found already generated log file in ${ABS_HTML_PATH%/}/${lORIG_SRC_MODULE_DIR}/${lMD5_REF_LINK}" "no_log"
# link to the already available report file without generating a new one:
lBACK_LINK_NEW="./${lORIG_SRC_MODULE_DIR}/${lMD5_REF_LINK}"
if [[ "$(basename ${lLINK_FILE})" =~ ^(d|p|l|s|q|f){1}[0-9]{2,3}_.*$ ]]; then
# link to the already available report file without generating a new one:
# we need to link to the original module directory of the file if we are in a main module log file
lBACK_LINK_NEW="${lORIG_SRC_MODULE_DIR}/${lMD5_REF_LINK}"
else
# if we are in our own sub-module log directory we do not need to include the module directory
lBACK_LINK_NEW="${lMD5_REF_LINK}"
fi
else
# generate reference file if it not already available
print_debug "[*] Generating info file lREF_LINK: ${lREF_LINK} / lBACK_LINK: ${lBACK_LINK}"
generate_info_file "${lREF_LINK}" "${lBACK_LINK}" &
lWAIT_PIDS_WR+=("$!")
@@ -141,6 +147,7 @@ add_link_tags() {
else
lHTML_LINK="$(echo "${REFERENCE_LINK}" | sed -e "s@LINK@${DEPTH}/${lBACK_LINK_NEW}@g" || true)"
fi
print_debug "[*] Generating ${lBACK_LINK_NEW} / lBACK_LINK_NEW: ${lBACK_LINK_NEW} / lREF_LINK: ${lREF_LINK} / DEPTH: ${DEPTH} / lHTML_LINK: ${lHTML_LINK}"
lLINE_NUMBER_INFO_PREV="$((lREF_LINK_NUMBER - 1))"
while [[ ("$(sed "${lLINE_NUMBER_INFO_PREV}""q;d" "${lLINK_FILE}")" == "${P_START}${SPAN_END}${P_END}") || ("$(sed "${lLINE_NUMBER_INFO_PREV}""q;d" "${lLINK_FILE}")" == "${BR}") ]]; do
lLINE_NUMBER_INFO_PREV=$((lLINE_NUMBER_INFO_PREV - 1))
@@ -570,7 +577,8 @@ generate_info_file() {
sed -i -e "s:^:${P_START}: ; s:$:${P_END}:" "${lTMP_INFO_FILE}" || true
# add html tags for style
add_color_tags "${lTMP_INFO_FILE}"
sed -i -e "s:[=]{65}:${HR_DOUBLE}:g ; s:^[-]{65}$:${HR_MONO}:g" "${lTMP_INFO_FILE}" || true
sed -i -E "s:[=]{65}:${HR_DOUBLE}:g" "${lTMP_INFO_FILE}" || true
sed -i -E "s:[-]{65}:${HR_MONO}:g" "${lTMP_INFO_FILE}" || true
# add link tags to links/generate info files and link to them and write line to tmp file
add_link_tags "${lTMP_INFO_FILE}" "${lINFO_HTML_FILE}"
helpers/helpers_emba_print.sh
+14 -28
View File
@@ -447,9 +447,12 @@ write_link() {
# The copy_and_link_file is used to copy files from the filesystem to the web-report and
# automatically link it
# This is usually used after a print_output "[*] asdf"
# 3rd parameter is optional log file to write this link into
copy_and_link_file() {
local lSRC_FILE="${1:-}"
local lDST_FILE="${2:-}"
# optional
local lLOG_FILE="${3:-}"
if ! [[ -d "$(dirname "${lDST_FILE}")" ]]; then
mkdir -p "$(dirname "${lDST_FILE}")" || true
@@ -460,30 +463,13 @@ copy_and_link_file() {
cp "${lSRC_FILE}" "${lDST_FILE}" 2>/dev/null || true
fi
fi
if [[ -f "${lDST_FILE}" ]]; then
if [[ -f "${lDST_FILE}" && -n "${lLOG_FILE}" ]]; then
write_link "${lDST_FILE}" "${lLOG_FILE}"
elif [[ -f "${lDST_FILE}" ]]; then
write_link "${lDST_FILE}"
fi
}
# we add an entry like
# [LOV] local_link
# this entry is later replaced from web reporter with the
# correct call to JS function
write_local_overlay_link() {
if [[ ${HTML} -eq 1 ]]; then
local lLINK="${1:-}"
lLINK="$(format_log "[LOV] ""${lLINK}" 1)"
local lLOG_FILE_ALT="${2:-}"
if [[ "${lLOG_FILE_ALT}" != "no_log" ]] && [[ "${lLOG_FILE_ALT}" != "main" ]]; then
if [[ -f "${lLOG_FILE_ALT}" ]]; then
echo -e "${lLINK}" | tee -a "${lLOG_FILE_ALT}" >/dev/null
else
echo -e "${lLINK}" | tee -a "${LOG_FILE}" >/dev/null
fi
fi
fi
}
write_anchor() {
if [[ ${HTML} -eq 1 ]]; then
local lANCHOR="${1:-}"
@@ -993,32 +979,32 @@ print_notification() {
# writes inputs into csv for chatgpt
# Args: "${GPT_INPUT_FILE_}" "${GPT_ANCHOR_}" "${GPT_PRIO_}" "${GPT_QUESTION_}" "${GPT_OUTPUT_FILE_}" "cost=${GPT_TOKENS_}" "${GPT_RESPONSE_}"
write_csv_gpt() {
write_csv_AI() {
local lCSV_ITEMS_ARR=("$@")
if ! [[ -d "${CSV_DIR}" ]]; then
print_output "[-] WARNING: CSV directory ${ORANGE}${CSV_DIR}${NC} not found"
return
fi
printf '%s;' "${lCSV_ITEMS_ARR[@]}" >>"${CSV_DIR}/q02_openai_question.csv" || true
printf '\n' >>"${CSV_DIR}/q02_openai_question.csv" || true
printf '%s;' "${lCSV_ITEMS_ARR[@]}" >>"${CSV_DIR}/ai_question.csv" || true
printf '\n' >>"${CSV_DIR}/ai_question.csv" || true
}
# writes inputs into tmp csv for chatgpt
# Args: "${GPT_INPUT_FILE_}" "${GPT_ANCHOR_}" "${GPT_PRIO_}" "${GPT_QUESTION_}" "${GPT_OUTPUT_FILE_}" "cost=${GPT_TOKENS_}" "${GPT_RESPONSE_}"
write_csv_gpt_tmp() {
write_csv_AI_tmp() {
local lCSV_ITEMS_ARR=("$@")
if ! [[ -d "${CSV_DIR}" ]]; then
print_output "[-] WARNING: CSV directory ${ORANGE}${CSV_DIR}${NC} not found"
return
fi
printf '%s;' "${lCSV_ITEMS_ARR[@]}" >>"${CSV_DIR}/q02_openai_question.csv.tmp" || true
printf '\n' >>"${CSV_DIR}/q02_openai_question.csv.tmp" || true
printf '%s;' "${lCSV_ITEMS_ARR[@]}" >>"${CSV_DIR}/ai_question.csv.tmp" || true
printf '\n' >>"${CSV_DIR}/ai_question.csv.tmp" || true
}
write_anchor_gpt() {
write_anchor_AI() {
if [[ ${HTML} -eq 1 ]]; then
local lLINK="${1:-}"
lLINK="$(format_log "[ASK_GPT] ""${lLINK}" 1)"
lLINK="$(format_log "[ASK_AI] ""${lLINK}" 1)"
local lLOG_FILE_ALT="${2:-}"
if [[ "${lLOG_FILE_ALT}" != "no_log" ]] && [[ "${lLOG_FILE_ALT}" != "main" ]]; then
if [[ -f "${lLOG_FILE_ALT}" ]]; then
helpers/style.css
+5
View File
@@ -81,6 +81,11 @@ a.f {
padding-right: 18px !important;
}
.double {
border: 0;
border-top: 5px double #9f9f9f;
}
hr.double {
border: 0;
border-top: 5px double #9f9f9f;
modules/F05_qs_resolver.sh
+104 -131
View File
@@ -18,90 +18,87 @@
F05_qs_resolver() {
module_log_init "${FUNCNAME[0]}"
module_title "GPT Resolver"
module_title "AI Resolver"
if [[ "${GPT_OPTION}" -gt 0 ]]; then
# wait for completion or 1m
if grep -q "Q02_openai_question starting" "${LOG_DIR}"/"${MAIN_LOG_FILE}"; then
print_output "[*] Waiting for the GPT testing module to stop ... " "no_log"
grep -q "Q02_openai_question finished" "${LOG_DIR}"/"${MAIN_LOG_FILE}" || sleep 1m
print_output "[*] GPT testing module stopped ... " "no_log"
if [[ "${AI_OPTION}" -gt 0 ]]; then
if grep -q "Q02_openai_question starting" "${MAIN_LOG}"; then
print_output "[*] Waiting for the GPT analysis module to stop ... " "no_log"
while ! grep -q "Q02_openai_question finished\|Quest container finished" "${MAIN_LOG}"; do
print_output "[*] Waiting for Q02 module"
sleep 5
done
print_output "[*] GPT testing module finished ... " "no_log"
fi
if grep -q "Q03_localai_connector starting" "${MAIN_LOG}"; then
print_output "[*] Waiting for the LocalAI analysis module to stop ... " "no_log"
while ! grep -q "Q03_localai_connector finished\|Quest container finished" "${MAIN_LOG}"; do
print_output "[*] Waiting for Q03 module"
sleep 5
done
print_output "[*] LocalAI testing module finished ... " "no_log"
fi
# local _GPT_INPUT_FILE_=""
local lGPT_ANCHOR_=""
local lAI_ANCHOR=""
local l_GPT_PRIO_=3
local lGPT_QUESTION_=""
local lGPT_RESPONSE_=""
local lAI_RESPONSE_FILE=""
local lGPT_TOKENS_=0
local lGPT_OUTPUT_FILE_=""
local lGPT_OUTPUT_FILE_HTML_ARR_=()
local lGPT_REVERSE_LINK_=""
local lWAIT_PIDS_F05_ARR=()
if [[ -f "${CSV_DIR}/q02_openai_question.csv" ]]; then
if [[ -f "${CSV_DIR}/ai_question.csv" ]]; then
while IFS=";" read -r COL1_ COL2_ COL3_ COL4_ COL5_ COL6_ COL7_; do
lGPT_INPUT_FILE_="${COL1_}"
lGPT_ANCHOR_="${COL2_}"
lAI_ANCHOR="${COL2_}"
l_GPT_PRIO_="${COL3_}"
lGPT_QUESTION_="${COL4_}"
lGPT_OUTPUT_FILE_="${COL5_}"
lGPT_TOKENS_="${COL6_//cost\=/}"
lGPT_RESPONSE_="${COL7_//\"/}"
# file with AI response:
lAI_RESPONSE_FILE="${COL7_}"
print_output "[*] GPT resolver - testing ${ORANGE}${CSV_DIR}/q02_openai_question.csv${NC}"
print_output "[*] AI resolver - testing ${ORANGE}${CSV_DIR}/ai_question.csv${NC}"
if [[ "${THREADED}" -eq 1 ]]; then
gpt_resolver_csv "${lGPT_INPUT_FILE_}" "${lGPT_ANCHOR_}" "${l_GPT_PRIO_}" "${lGPT_QUESTION_}" "${lGPT_OUTPUT_FILE_}" "${lGPT_TOKENS_}" "${lGPT_RESPONSE_}" &
local lTMP_PID="$!"
store_kill_pids "${lTMP_PID}"
lWAIT_PIDS_F05_ARR+=("${lTMP_PID}")
# max_pids_protection "${MAX_MOD_THREADS}" "${lWAIT_PIDS_F05_ARR[@]}"
else
gpt_resolver_csv "${lGPT_INPUT_FILE_}" "${lGPT_ANCHOR_}" "${l_GPT_PRIO_}" "${lGPT_QUESTION_}" "${lGPT_OUTPUT_FILE_}" "${lGPT_TOKENS_}" "${lGPT_RESPONSE_}"
fi
done <"${CSV_DIR}/q02_openai_question.csv"
[[ "${THREADED}" -eq 1 ]] && wait_for_pid "${lWAIT_PIDS_F05_ARR[@]}"
gpt_resolver_csv "${lGPT_INPUT_FILE_}" "${lAI_ANCHOR}" "${l_GPT_PRIO_}" "${lGPT_QUESTION_}" "${lGPT_OUTPUT_FILE_}" "${lGPT_TOKENS_}" "${lAI_RESPONSE_FILE}" &
done <"${CSV_DIR}/ai_question.csv"
wait_for_pid "${lWAIT_PIDS_F05_ARR[@]}"
fi
if [[ -f "${CSV_DIR}/q02_openai_question.csv.tmp" ]]; then
if [[ -f "${CSV_DIR}/ai_question.csv.tmp" ]]; then
while IFS=";" read -r COL1_ COL2_ COL3_ COL4_ COL5_ COL6_ COL7_; do
local lGPT_INPUT_FILE_="${COL1_}"
local lGPT_ANCHOR_="${COL2_}"
local lAI_ANCHOR="${COL2_}"
local l_GPT_PRIO_="${COL3_}"
local lGPT_QUESTION_="${COL4_}"
local lGPT_OUTPUT_FILE_="${COL5_}"
local lGPT_TOKENS_="${COL6_//cost\=/}"
local lGPT_RESPONSE_="${COL7_//\"/}"
print_output "[*] Trying to resolve ${ORANGE}Anchor ${lGPT_ANCHOR_}${NC} in ${ORANGE}Output_file ${lGPT_OUTPUT_FILE_}${NC}."
print_output "[*] Trying to resolve ${ORANGE}Anchor ${lAI_ANCHOR}${NC} in ${ORANGE}Output_file ${lGPT_OUTPUT_FILE_}${NC}."
if [[ "${THREADED}" -eq 1 ]]; then
gpt_resolver_csv_tmp "${lGPT_INPUT_FILE_}" "${lGPT_ANCHOR_}" "${l_GPT_PRIO_}" "${lGPT_QUESTION_}" "${lGPT_OUTPUT_FILE_}" "${lGPT_TOKENS_}" "${lGPT_RESPONSE_}" &
local lTMP_PID="$!"
store_kill_pids "${lTMP_PID}"
lWAIT_PIDS_F05_ARR+=("${lTMP_PID}")
# max_pids_protection "${MAX_MOD_THREADS}" "${lWAIT_PIDS_F05_ARR[@]}"
else
gpt_resolver_csv_tmp "${lGPT_INPUT_FILE_}" "${lGPT_ANCHOR_}" "${l_GPT_PRIO_}" "${lGPT_QUESTION_}" "${lGPT_OUTPUT_FILE_}" "${lGPT_TOKENS_}" "${lGPT_RESPONSE_}"
fi
done <"${CSV_DIR}/q02_openai_question.csv.tmp"
[[ "${THREADED}" -eq 1 ]] && wait_for_pid "${lWAIT_PIDS_F05_ARR[@]}"
gpt_resolver_csv_tmp "${lGPT_INPUT_FILE_}" "${lAI_ANCHOR}" "${l_GPT_PRIO_}" "${lGPT_QUESTION_}" "${lGPT_OUTPUT_FILE_}" "${lGPT_TOKENS_}" "${lGPT_RESPONSE_}" &
local lTMP_PID="$!"
lWAIT_PIDS_F05_ARR+=("${lTMP_PID}")
# max_pids_protection "${MAX_MOD_THREADS}" "${lWAIT_PIDS_F05_ARR[@]}"
done <"${CSV_DIR}/ai_question.csv.tmp"
wait_for_pid "${lWAIT_PIDS_F05_ARR[@]}"
fi
fi
if [[ -d "${HTML_PATH}" ]]; then
# lets do a final cleanup to get rid of all the ASK_GPT entries:
find "${HTML_PATH}" -type f -name "*.html" -exec sed -i '/ASK_GPT/d' {} \;
fi
#if [[ -d "${HTML_PATH}" ]]; then
# lets do a final cleanup to get rid of all the ASK_AI entries:
# find "${HTML_PATH}" -type f -name "*.html" -exec sed -i '/ASK_AI/d' {} \;
#fi
# lets do a final cleanup to get rid of all the ASK_GPT entries:
find "${LOG_DIR}" -maxdepth 1 -type f -name "*.txt" -exec sed -i '/ASK_GPT/d' {} \;
# lets do a final cleanup to get rid of all the ASK_AI entries:
#find "${LOG_DIR}" -maxdepth 1 -type f -name "*.txt" -exec sed -i '/ASK_AI/d' {} \;
# do not create a web reporter page
module_end_log "${FUNCNAME[0]}" 0
@@ -109,69 +106,58 @@ F05_qs_resolver() {
gpt_resolver_csv() {
local lGPT_INPUT_FILE_="${1:-}"
local lGPT_ANCHOR_="${2:-}"
local lAI_ANCHOR="${2:-}"
local l_GPT_PRIO_="${3:-}"
local lGPT_QUESTION_="${4:-}"
local lGPT_OUTPUT_FILE_="${5:-}"
local lGPT_TOKENS_="${6:-}"
local lGPT_RESPONSE_="${7:-}"
local lAI_RESPONSE_FILE="${7:-}"
local lGPT_OUTPUT_FILE_NAME=""
local lGPT_OUTPUT_FILE_NAME_bak=""
local lHTML_FILE_=""
local lHTML_FILE_X=""
print_output "[*] Trying to resolve ${ORANGE}Anchor ${lGPT_ANCHOR_}${NC} in ${ORANGE}Output_file ${lGPT_OUTPUT_FILE_}${NC}."
print_output "[*] Trying to resolve ${ORANGE}Anchor ${lAI_ANCHOR}${NC}."
if [[ ${lGPT_TOKENS_} -ne 0 ]]; then
if ! [ -f "${lGPT_OUTPUT_FILE_}" ]; then
print_output "[-] Something went wrong with the Output file ${lGPT_OUTPUT_FILE_}"
if [[ -z ${lGPT_OUTPUT_FILE_} ]]; then
print_output " there is no file name for anchor: ${lGPT_ANCHOR_}"
# replace anchor in html-report with link to response
local lMD5_OF_AI_RESPONSE_FILE=""
lMD5_OF_AI_RESPONSE_FILE=$(md5sum "${lAI_RESPONSE_FILE}" | awk '{print $1}')
lAI_RESPONSE_FILE=$(find "${HTML_PATH}" -iname "${lMD5_OF_AI_RESPONSE_FILE}.html" 2>/dev/null | sort -u | head -1)
lAI_RESPONSE_FILE_NAME=$(basename "${lAI_RESPONSE_FILE}")
print_output "[*] Testing ${lAI_RESPONSE_FILE} with md5sum of ${lMD5_OF_AI_RESPONSE_FILE}" "no_log"
readarray -t lGPT_OUTPUT_FILE_HTML_ARR_ < <(grep -r -l "\[ASK_AI\]\ ${lAI_ANCHOR}" "${HTML_PATH}" 2>/dev/null || true)
for lHTML_FILE_ in "${lGPT_OUTPUT_FILE_HTML_ARR_[@]}"; do
print_output "[*] Testing ${lHTML_FILE_} for anchor ${lAI_ANCHOR}" "no_log"
# should point back to q02-submodule with name "${lGPT_INPUT_FILE_}"
lGPT_REVERSE_LINK_="$(tr "[:upper:]" "[:lower:]" <<<"${lGPT_INPUT_FILE_}" | sed -e "s@[^a-zA-Z0-9]@@g")"
# shellcheck disable=SC2001
lHTML_FILE_X=$(echo "${lHTML_FILE_}" | sed 's#'"${HTML_PATH}"'##')
print_output "[*] Linking AI results ${ORANGE}${lGPT_REVERSE_LINK_}${NC} into ${ORANGE}${lHTML_FILE_X}${NC}" "no_log"
local lDEPTH="\.\.\/"
if [[ "${AI_OPTION}" -eq 3 ]]; then
if [[ -f "${lAI_RESPONSE_FILE}" ]]; then
print_output "[*] Replacing Anchor ${lAI_ANCHOR} with link to ${lAI_RESPONSE_FILE_NAME} - ${lAI_RESPONSE_FILE} in ${lHTML_FILE_}"
sed -i "s/\[ASK_AI\]\ ${lAI_ANCHOR}/\ \ \ \ \<a class\=\"reference\" href\=\"${lDEPTH}q03\_localai\_connector\/${lAI_RESPONSE_FILE_NAME}\" title\=\"${lAI_RESPONSE_FILE_NAME}\"\ \>\<span\ class=\"green\"\>[+] LocalAI results are available\<\/span\>\<\/a\>\n/1" "${lHTML_FILE_}"
else
# link to localAI module results - q03_localai_connector.html
sed -i "s/\[ASK_AI\]\ ${lAI_ANCHOR}/\ \ \ \ \<a class\=\"reference\" href\=\"${lDEPTH}q03\_localai\_connector\.html\#aianalysisfor${lGPT_REVERSE_LINK_}\" title\=\"${lGPT_REVERSE_LINK_}\"\ \>\<span\ class=\"green\"\>[+] LocalAI results are available\<\/span\>\<\/a\>\n/1" "${lHTML_FILE_}"
fi
else
# openai mode - deprected and probably not working anymore
# Todo: remove OpenAI module environment from EMBA
sed -i "s/\[ASK_AI\]\ ${lAI_ANCHOR}/\ \ \ \ \<a class\=\"reference\" href\=\"${lDEPTH}q02\_openai\_question\.html\#aianalysisfor${lGPT_REVERSE_LINK_}\" title\=\"${lGPT_REVERSE_LINK_}\"\ \>\<span\ class=\"green\"\>[+] OpenAI results are available\<\/span\>\<\/a\>\n/1" "${lHTML_FILE_}"
fi
else
sed -i "s/${lGPT_ANCHOR_}/Q\: ${lGPT_QUESTION_}\nA\: /1" "${lGPT_OUTPUT_FILE_}"
# grep "${lGPT_ANCHOR_}" "${CSV_DIR}/q02_openai_question.csv" | cut -d";" -f7 >> "${lGPT_OUTPUT_FILE_}"
printf '%q\n' "${lGPT_RESPONSE_//\\/}" >>"${lGPT_OUTPUT_FILE_}"
# replace anchor in html-report with link to response
if [[ "${lGPT_OUTPUT_FILE_}" == *".log" ]]; then
lGPT_OUTPUT_FILE_NAME="$(basename "${lGPT_OUTPUT_FILE_//\.log/}.html")"
elif [[ "${lGPT_OUTPUT_FILE_}" == *".txt" ]]; then
lGPT_OUTPUT_FILE_NAME="$(basename "${lGPT_OUTPUT_FILE_//\.txt/}.html")"
elif [[ "${lGPT_OUTPUT_FILE_}" == *".c" ]]; then
lGPT_OUTPUT_FILE_NAME="$(basename "${lGPT_OUTPUT_FILE_//\.c/}.html")"
fi
readarray -t lGPT_OUTPUT_FILE_HTML_ARR_ < <(find "${HTML_PATH}" -iname "${lGPT_OUTPUT_FILE_NAME}" 2>/dev/null)
# the following search is because of inconsistency in file names.
# Todo: check this and fix it to only use the rules above
lGPT_OUTPUT_FILE_NAME_bak="$(basename "${lGPT_OUTPUT_FILE_//\./}.html")"
readarray -t GPT_OUTPUT_FILE_HTML_ARR_bak < <(find "${HTML_PATH}" -iname "${lGPT_OUTPUT_FILE_NAME_bak}" 2>/dev/null)
lGPT_OUTPUT_FILE_HTML_ARR_+=("${GPT_OUTPUT_FILE_HTML_ARR_bak[@]}")
for lHTML_FILE_ in "${lGPT_OUTPUT_FILE_HTML_ARR_[@]}"; do
# should point back to q02-submodule with name "${lGPT_INPUT_FILE_}"
lGPT_REVERSE_LINK_="$(tr "[:upper:]" "[:lower:]" <<<"${lGPT_INPUT_FILE_}" | sed -e "s@[^a-zA-Z0-9]@@g")"
# we need to find the depth which we need to link to the file
# shellcheck disable=SC2001
lHTML_FILE_X=$(echo "${lHTML_FILE_}" | sed 's#'"${HTML_PATH}"'##')
print_output "[*] Linking GPT results ${ORANGE}${lGPT_REVERSE_LINK_}${NC} into ${ORANGE}${lHTML_FILE_X}${NC}" "no_log"
depth_cnt="${lHTML_FILE_X//[^\/]/}"
depth_cnt="$(("${#depth_cnt}" - 1))"
local lDEPTH="\.\.\/"
local lmyDEPTH=""
lmyDEPTH=$(printf "%${depth_cnt}s")
lDEPTH="${lmyDEPTH// /${lDEPTH}}"
sed -i "s/\[ASK_GPT\]\ ${lGPT_ANCHOR_}/\ \ \ \ \<a class\=\"reference\" href\=\"${lDEPTH}q02\_openai\_question\.html\#aianalysisfor${lGPT_REVERSE_LINK_}\" title\=\"${lGPT_REVERSE_LINK_}\"\ \>\<span\ class=\"green\"\>[+] OpenAI results are available\<\/span\>\<\/a\>\n/1" "${lHTML_FILE_}"
done
fi
done
fi
}
gpt_resolver_csv_tmp() {
local lGPT_INPUT_FILE_="${1:-}"
local lGPT_ANCHOR_="${2:-}"
local lAI_ANCHOR="${2:-}"
local l_GPT_PRIO_="${3:-}"
local lGPT_QUESTION_="${4:-}"
local lGPT_OUTPUT_FILE_="${5:-}"
@@ -181,44 +167,31 @@ gpt_resolver_csv_tmp() {
local lHTML_FILE_=""
local lHTML_FILE_X=""
print_output "[*] Trying to resolve ${ORANGE}Anchor ${lGPT_ANCHOR_}${NC} in ${ORANGE}output_file ${lGPT_OUTPUT_FILE_}${NC}."
print_output "[*] Trying to resolve ${ORANGE}Anchor ${lAI_ANCHOR}${NC}."
if ! [ -f "${lGPT_OUTPUT_FILE_}" ]; then
print_output "[-] Something went wrong with the Output file ${lGPT_OUTPUT_FILE_}"
if [[ -z ${lGPT_OUTPUT_FILE_} ]]; then
print_output " there is no file name for anchor: ${lGPT_ANCHOR_}"
print_output "[*] AI module didn't check ${lGPT_INPUT_FILE_}, linking to the GPT module page instead"
local lMD5_OF_ORIG_SRC_FILE=""
lMD5_OF_ORIG_SRC_FILE=$(md5sum "${lGPT_OUTPUT_FILE_}" | awk '{print $1}')
readarray -t lGPT_OUTPUT_FILE_HTML_ARR_ < <(grep -r -l "\[ASK_AI\]\ ${lAI_ANCHOR}" "${HTML_PATH}" 2>/dev/null || true)
for lHTML_FILE_ in "${lGPT_OUTPUT_FILE_HTML_ARR_[@]}"; do
# should point back to q02-submodule with name "${lGPT_INPUT_FILE_}"
lGPT_REVERSE_LINK_="$(tr "[:upper:]" "[:lower:]" <<<"${lGPT_INPUT_FILE_}" | sed -e "s@[^a-zA-Z0-9]@@g")"
# we need to find the depth which we need to link to the file
# shellcheck disable=SC2001
lHTML_FILE_X=$(echo "${lHTML_FILE_}" | sed 's#'"${HTML_PATH}"'##')
print_output "[*] Linking AI results ${ORANGE}${lGPT_REVERSE_LINK_}${NC} into ${ORANGE}${lHTML_FILE_X}${NC}" "no_log"
depth_cnt="${lHTML_FILE_X//[^\/]/}"
depth_cnt="$(("${#depth_cnt}" - 1))"
local lDEPTH="\.\.\/"
if [[ "${AI_OPTION}" -eq 3 ]]; then
sed -i "s/\[ASK_AI\]\ ${lAI_ANCHOR}/\ \ \ \ \<a class\=\"reference\" href\=\"${lDEPTH}q03\_localai\_connector\.html\" title\=\"${lGPT_REVERSE_LINK_}\"\ \>\<span\ class=\"orange\"\>[*] LocalAI module did not finish\<\/span\>\<\/a\>\n/1" "${lHTML_FILE_}"
else
sed -i "s/\[ASK_AI\]\ ${lAI_ANCHOR}/\ \ \ \ \<a class\=\"reference\" href\=\"${lDEPTH}q02\_openai\_question\.html\" title\=\"${lGPT_REVERSE_LINK_}\"\ \>\<span\ class=\"orange\"\>[*] OpenAI module did not finish\<\/span\>\<\/a\>\n/1" "${lHTML_FILE_}"
fi
else
print_output "[*] Q02 didn't check ${lGPT_INPUT_FILE_}, linking to the GPT module page instead"
# sed -i "s/${lGPT_ANCHOR_}/Check did not finish!/1" "${lGPT_OUTPUT_FILE_}"
if [[ "${lGPT_OUTPUT_FILE_}" == *".log" ]]; then
lGPT_OUTPUT_FILE_NAME="$(basename "${lGPT_OUTPUT_FILE_//\.log/}.html")"
elif [[ "${lGPT_OUTPUT_FILE_}" == *".txt" ]]; then
lGPT_OUTPUT_FILE_NAME="$(basename "${lGPT_OUTPUT_FILE_//\.txt/}.html")"
elif [[ "${lGPT_OUTPUT_FILE_}" == *".c" ]]; then
lGPT_OUTPUT_FILE_NAME="$(basename "${lGPT_OUTPUT_FILE_//\.c/}.html")"
fi
readarray -t lGPT_OUTPUT_FILE_HTML_ARR_ < <(find "${HTML_PATH}" -iname "${lGPT_OUTPUT_FILE_NAME}" 2>/dev/null)
for lHTML_FILE_ in "${lGPT_OUTPUT_FILE_HTML_ARR_[@]}"; do
# should point back to q02-submodule with name "${lGPT_INPUT_FILE_}"
lGPT_REVERSE_LINK_="$(tr "[:upper:]" "[:lower:]" <<<"${lGPT_INPUT_FILE_}" | sed -e "s@[^a-zA-Z0-9]@@g")"
# we need to find the depth which we need to link to the file
# shellcheck disable=SC2001
lHTML_FILE_X=$(echo "${lHTML_FILE_}" | sed 's#'"${HTML_PATH}"'##')
print_output "[*] Linking GPT results ${ORANGE}${lGPT_REVERSE_LINK_}${NC} into ${ORANGE}${lHTML_FILE_X}${NC}" "no_log"
depth_cnt="${lHTML_FILE_X//[^\/]/}"
depth_cnt="$(("${#depth_cnt}" - 1))"
local lDEPTH="\.\.\/"
local lmyDEPTH=""
lmyDEPTH=$(printf "%${depth_cnt}s")
lDEPTH="${lmyDEPTH// /${lDEPTH}}"
sed -i "s/\[ASK_GPT\]\ ${lGPT_ANCHOR_}/\ \ \ \ \<a class\=\"reference\" href\=\"${lDEPTH}q02\_openai\_question\.html\" title\=\"${lGPT_REVERSE_LINK_}\"\ \>\<span\ class=\"orange\"\>[*] OpenAI module did not finish\<\/span\>\<\/a\>\n/1" "${lHTML_FILE_}"
done
fi
done
}
modules/F50_base_aggregator.sh
+19 -10
View File
@@ -96,14 +96,6 @@ output_overview() {
print_output "[+] EMBA version: ""${ORANGE}""${lSBOM_TOOL_VERS}""${NC}"
write_csv_log "EMBA_version" "${lSBOM_TOOL_VERS}" "NA" "NA" "NA" "NA" "NA" "NA" "NA"
if [[ -f "${Q02_LOG}" ]] && [[ "${GPT_OPTION}" -gt 0 ]]; then
lGPT_RESULTS_CNT=$(grep -c "AI analysis for" "${Q02_LOG}" || true)
if [[ "${lGPT_RESULTS_CNT}" -gt 0 ]]; then
print_output "[+] EMBA AI analysis discovered ${ORANGE}${lGPT_RESULTS_CNT}${GREEN} results."
write_link "q02"
fi
fi
if [[ -n "${ARCH}" ]] && [[ "${ARCH}" != "NA" ]]; then
if [[ -n "${D_END:-"NA"}" ]]; then
write_csv_log "architecture_verified" "${ARCH}" "${D_END}" "NA" "NA" "NA" "NA" "NA" "NA"
@@ -263,10 +255,27 @@ output_details() {
fi
fi
if [[ -f "${Q02_LOG}" ]] && [[ "${AI_OPTION}" -gt 0 ]]; then
lGPT_RESULTS_CNT=$(grep -c "AI analysis for" "${Q02_LOG}" || true)
if [[ "${lGPT_RESULTS_CNT}" -gt 0 ]]; then
print_output "[+] AI analysis identified ${ORANGE}${lGPT_RESULTS_CNT}${GREEN} results via ChatGPT."
write_link "q02"
write_csv_log "AI results" "${lGPT_RESULTS_CNT}" "ChatGPT" "NA" "NA" "NA" "NA" "NA" "NA"
fi
fi
if [[ -f "${Q03_LOG}" ]] && [[ "${AI_OPTION}" -gt 0 ]]; then
lAI_RESULTS_CNT=$(grep -c "AI analysis results for" "${Q03_LOG}" || true)
if [[ "${lAI_RESULTS_CNT}" -gt 0 ]]; then
print_output "[+] AI analysis discovered ${ORANGE}${lAI_RESULTS_CNT}${GREEN} results via LocalAI."
write_link "q03"
write_csv_log "AI results" "${lAI_RESULTS_CNT}" "LocalAI" "NA" "NA" "NA" "NA" "NA" "NA"
fi
fi
if [[ "${lGPT_RESULTS_CNT:-0}" -gt 0 ]]; then
print_output "[+] EMBA AI tests identified ${ORANGE}${lGPT_RESULTS_CNT}${GREEN} results via ChatGPT."
write_link "q02"
write_csv_log "AI results" "${lGPT_RESULTS_CNT}" "NA" "NA" "NA" "NA" "NA" "NA" "NA"
fi
if [[ "${BOOTED:-0}" -gt 0 ]] || [[ "${IP_ADDR_CNT:-0}" -gt 0 ]]; then
modules/Q02_openai_question.sh
+14 -14
View File
@@ -19,7 +19,7 @@
Q02_openai_question() {
module_log_init "${FUNCNAME[0]}"
export CHATGPT_RESULT_CNT=0
if [[ "${GPT_OPTION}" -gt 0 ]] && [[ -n "${OPENAI_API_KEY}" ]]; then
if [[ "${AI_OPTION}" -gt 0 ]] && [[ -n "${OPENAI_API_KEY}" ]]; then
# Prints title to CLI and into log
module_title "AI analysis via OpenAI"
pre_module_reporter "${FUNCNAME[0]}"
@@ -34,13 +34,13 @@ Q02_openai_question() {
done
if [[ -f "${LOG_DIR}"/"${MAIN_LOG_FILE}" ]]; then
while ! [[ -f "${CSV_DIR}/q02_openai_question.csv.tmp" ]]; do
while ! [[ -f "${CSV_DIR}/ai_question.csv.tmp" ]]; do
sleep 3
done
fi
export GTP_CHECKED_ARR=()
while ! grep -q "Testing phase ended" "${LOG_DIR}"/"${MAIN_LOG_FILE}"; do
while ! grep -q "Reporting phase started" "${LOG_DIR}"/"${MAIN_LOG_FILE}" && [[ "${lMODULE_RUNTIME}" -le "${LOCAL_AI_MIN_RUNTIME}" ]]; do
if [[ "${CHATGPT_RESULT_CNT}" -ge 0 ]]; then
ask_chatgpt
fi
@@ -79,7 +79,7 @@ ask_chatgpt() {
# generating Array for GPT requests - sorting according the prio in field 3
# this array gets regenerated on every round
readarray -t Q02_OPENAI_QUESTIONS < <(sort -k 3 -t ';' -r "${CSV_DIR}/q02_openai_question.csv.tmp")
readarray -t Q02_OPENAI_QUESTIONS < <(sort -k 3 -t ';' -r "${CSV_DIR}/ai_question.csv.tmp")
for ((lELE_INDEX = 0; lELE_INDEX < "${#Q02_OPENAI_QUESTIONS[@]}"; lELE_INDEX++)); do
local lELEM="${Q02_OPENAI_QUESTIONS["${lELE_INDEX}"]}"
@@ -197,7 +197,7 @@ ask_chatgpt() {
lCNT=$((lCNT + 1))
local lTEMP_VAR="$(("${lCNT}" % 100))"
(("${lTEMP_VAR}" == 0)) && print_output "[*] Rate limit handling ... sleep mode - ${lCNT}" "no_log"
if grep -q "Testing phase ended" "${LOG_DIR}"/"${MAIN_LOG_FILE}"; then
if grep -q "Reporting phase started" "${LOG_DIR}"/"${MAIN_LOG_FILE}" && [[ "${lMODULE_RUNTIME}" -ge "${LOCAL_AI_MIN_RUNTIME}" ]]; then
break 2
fi
sleep 1
@@ -211,10 +211,10 @@ ask_chatgpt() {
fi
cat "${TMP_DIR}/${lGPT_INPUT_FILE_mod}_response.json" >>"${lGPT_FILE_DIR}/openai_server_errors.log"
readarray -t Q02_OPENAI_QUESTIONS < <(sort -k 3 -t ';' -r "${CSV_DIR}/q02_openai_question.csv.tmp")
readarray -t Q02_OPENAI_QUESTIONS < <(sort -k 3 -t ';' -r "${CSV_DIR}/ai_question.csv.tmp")
# reset the array index to start again with the highest rated entry
lELE_INDEX=0
if grep -q "Testing phase ended" "${LOG_DIR}"/"${MAIN_LOG_FILE}"; then
if grep -q "Reporting phase started" "${LOG_DIR}"/"${MAIN_LOG_FILE}" && [[ "${lMODULE_RUNTIME}" -ge "${LOCAL_AI_MIN_RUNTIME}" ]]; then
return
fi
sleep 30s
@@ -235,7 +235,7 @@ ask_chatgpt() {
if [[ ${lGPT_TOKENS} -ne 0 ]]; then
GTP_CHECKED_ARR+=("${lSCRIPT_PATH_TMP}")
# write new into done csv
write_csv_gpt "${lGPT_INPUT_FILE}" "${lGPT_ANCHOR}" "${lGPT_PRIO}" "${lGPT_QUESTION}" "${lGPT_OUTPUT_FILE}" "cost=${lGPT_TOKENS}" "'${lGPT_RESPONSE_CLEANED//\'/}'"
write_csv_AI "${lGPT_INPUT_FILE}" "${lGPT_ANCHOR}" "${lGPT_PRIO}" "${lGPT_QUESTION}" "${lGPT_OUTPUT_FILE}" "cost=${lGPT_TOKENS}" "'${lGPT_RESPONSE_CLEANED//\'/}'"
# we store the answers in dedicated files for further interlinking within the report
if ! [[ -d "${LOG_PATH_MODULE}"/gpt_answers ]]; then
@@ -275,26 +275,26 @@ ask_chatgpt() {
fi
fi
if grep -q "Testing phase ended" "${LOG_DIR}"/"${MAIN_LOG_FILE}"; then
if grep -q "Reporting phase started" "${LOG_DIR}"/"${MAIN_LOG_FILE}" && [[ "${lMODULE_RUNTIME}" -ge "${LOCAL_AI_MIN_RUNTIME}" ]]; then
break
fi
if [[ "${GPT_OPTION}" -ne 2 ]]; then
if [[ "${AI_OPTION}" -ne 2 ]]; then
sleep 20s
fi
# reload q02 results:
print_output "[*] Regenerate analysis array ..." "no_log"
readarray -t Q02_OPENAI_QUESTIONS < <(sort -k 3 -t ';' -r "${CSV_DIR}/q02_openai_question.csv.tmp")
readarray -t Q02_OPENAI_QUESTIONS < <(sort -k 3 -t ';' -r "${CSV_DIR}/ai_question.csv.tmp")
# reset the array index to start again with the highest rated entry
lELE_INDEX=0
done
if [[ -f "${CSV_DIR}/q02_openai_question.csv" ]]; then
if [[ -f "${CSV_DIR}/ai_question.csv" ]]; then
local lGPT_ENTRY_LINE=""
while read -r lGPT_ENTRY_LINE; do
lGPT_ANCHOR="$(echo "${lGPT_ENTRY_LINE}" | cut -d ';' -f2)"
sed -i "/${lGPT_ANCHOR}/d" "${CSV_DIR}/q02_openai_question.csv.tmp"
done <"${CSV_DIR}/q02_openai_question.csv"
sed -i "/${lGPT_ANCHOR}/d" "${CSV_DIR}/ai_question.csv.tmp"
done <"${CSV_DIR}/ai_question.csv"
fi
}
modules/Q03_localai_connector.sh
Executable
+592
View File
@@ -0,0 +1,592 @@
#!/bin/bash -p
# EMBA - EMBEDDED LINUX ANALYZER
#
# Copyright 2020-2026 Siemens Energy AG
#
# EMBA comes with ABSOLUTELY NO WARRANTY. This is free software, and you are
# welcome to redistribute it under the terms of the GNU General Public License.
# See LICENSE file for usage of this software.
#
# EMBA is licensed under GPLv3
# SPDX-License-Identifier: GPL-3.0-only
#
# Author(s): Benedikt Kuehne, Michael Messner
# Description: LocalAI questioning module for container #2
# Note: Important requirement for Q-modules is the self termination when a certain phase ends
# This module has an additional minimum runtime mechanism. Ensure the F05 module is respecting
# the minimum runtime and waits for the module to finish.
# set DEBUG to 1 to enable further debug messages
: "${DEBUG:=0}"
export DEBUG
Q03_localai_connector() {
module_log_init "${FUNCNAME[0]}"
export AI_RESULT_CNT=0
if [[ "${AI_OPTION}" -eq 3 ]]; then
module_title "AI analysis via LocalAI"
pre_module_reporter "${FUNCNAME[0]}"
export AI_RESULT_CNT=0
export AI_ERROR_CNT=0
local lMAX_AI_ERROR_CNT=10
local lMODEL_LOCALAI=""
lMODEL_LOCALAI=$(identify_ai_model)
if [[ -z "${lMODEL_LOCALAI}" ]]; then
print_output "[-] No model identified - is the AI responsive? Exit now."
module_end_log "${FUNCNAME[0]}" 0
return
fi
print_output "[*] Identified AI model: ${lMODEL_LOCALAI}"
if [[ "${lMODEL_LOCALAI}" != *"${LOCAL_AI_MODEL}"* ]]; then
print_output "[-] Configured model ${LOCAL_AI_MODEL} does not match the identified model ${lMODEL_LOCALAI} - exit now"
module_end_log "${FUNCNAME[0]}" 0
return
fi
# we wait until there arer entries in the question csv
while ! [[ -f "${MAIN_LOG}" ]]; do
if ! [[ -d "${LOG_DIR}" ]]; then
# this usually happens if we automate analysis and remove the logging directory while this module was not finished at all
return
fi
sleep 10
done
if [[ -f "${MAIN_LOG}" ]]; then
while ! [[ -f "${CSV_DIR}/ai_question.csv.tmp" ]]; do
sleep 3
done
fi
export SECONDS=0
if [[ "${AI_MIN_RUNTIME}" == *"h" ]]; then
AI_MIN_RUNTIME=${AI_MIN_RUNTIME//h/}
AI_MIN_RUNTIME=$((AI_MIN_RUNTIME*3600))
fi
if [[ "${AI_MIN_RUNTIME}" == *"m" ]]; then
AI_MIN_RUNTIME=${AI_MIN_RUNTIME//m/}
AI_MIN_RUNTIME=$((AI_MIN_RUNTIME*60))
fi
if [[ "${AI_MIN_RUNTIME}" == *"d" ]]; then
AI_MIN_RUNTIME=${AI_MIN_RUNTIME//d/}
AI_MIN_RUNTIME=$((AI_MIN_RUNTIME*24*3600))
fi
export GTP_CHECKED_ARR=()
local lMODULE_RUNTIME=${SECONDS}
adjust_minimum_runtime "${lMODULE_RUNTIME}"
# check for runtime
while [[ "${lMODULE_RUNTIME}" -lt "${AI_MIN_RUNTIME}" ]]; do
if [[ "${AI_ERROR_CNT}" -lt "${lMAX_AI_ERROR_CNT}" ]]; then
ask_localai "${lMODEL_LOCALAI}"
else
# exit if too many errors happened
break
fi
local lMODULE_RUNTIME=${SECONDS}
adjust_minimum_runtime "${lMODULE_RUNTIME}"
done
fi
module_end_log "${FUNCNAME[0]}" "${AI_RESULT_CNT}"
}
adjust_minimum_runtime() {
local lMODULE_RUNTIME="${1:-}"
# if the minimum runtime is not set we adjust it to something bigger then the module runtime
# with this we can start and the later rules to end this module will jump in
if [[ -z "${AI_MIN_RUNTIME}" ]]; then
AI_MIN_RUNTIME=$((lMODULE_RUNTIME+10))
return
fi
# to enusure the AI module is running at least until the reporting phase started we tweak the minimum
# runtime to a higher value as the module runtime as long as the reporting phase is not started
if ! grep -q "Reporting phase started" "${MAIN_LOG}" && [[ "${lMODULE_RUNTIME}" -ge "${AI_MIN_RUNTIME}" ]]; then
AI_MIN_RUNTIME=$((lMODULE_RUNTIME+10))
fi
}
# looks through the modules and finds localAI questions inside the csv
ask_localai() {
local lMODEL_LOCALAI="${1:-}"
local lAI_FILE_DIR="${LOG_PATH_MODULE}/ai_files"
export AI_PROMPT_DIR="${LOG_PATH_MODULE}/ai_prompt"
local lAI_PRIO=3
# default vars
local lHTTP_CODE=200
local lORIGIN_MODULE=""
local lELE_INDEX=0
local lAI_ANCHOR=""
local lAI_INPUT_FILE=""
local lAI_INPUT_FILE_mod=""
local lAI_SOURCE_FILE=""
local lSCRIPT_PATH_TMP=""
print_output "[*] Checking scripts with LocalAI that have a priority of ${ORANGE}${MINIMUM_GPT_PRIO}${NC} or lower" "no_log"
if ! [[ -d "${lAI_FILE_DIR}" ]]; then
mkdir "${lAI_FILE_DIR}"
fi
if ! [[ -d "${AI_PROMPT_DIR}" ]]; then
mkdir "${AI_PROMPT_DIR}"
fi
# generating Array for AI requests - sorting according the prio in field 3
# this array gets regenerated on every round
regenerate_ai_todo_list
for ((lELE_INDEX = 0; lELE_INDEX < "${#Q03_LOCALAI_QUESTIONS[@]}"; lELE_INDEX++)); do
local lELEM="${Q03_LOCALAI_QUESTIONS["${lELE_INDEX}"]}"
lSCRIPT_PATH_TMP="$(echo "${lELEM}" | cut -d\; -f1)"
# as we always start with the highest rated entry, we need to check if this entry was already tested:
if [[ " ${GTP_CHECKED_ARR[*]} " =~ ${lSCRIPT_PATH_TMP} ]]; then
print_output "[*] AI - Already tested ${lSCRIPT_PATH_TMP}" "no_log"
# lets test the next entry
continue
fi
lAI_ANCHOR="$(echo "${lELEM}" | cut -d\; -f2)"
lAI_PRIO="$(echo "${lELEM}" | cut -d\; -f3)"
lAI_SOURCE_FILE="$(echo "${lELEM}" | cut -d\; -f5)"
lAI_INPUT_FILE="$(basename "${lSCRIPT_PATH_TMP}")"
lAI_INPUT_FILE_mod="${lAI_INPUT_FILE//\./}"
# in case we have nothing we are going to move on
[[ -z "${lSCRIPT_PATH_TMP}" ]] && continue
if [[ "${lSCRIPT_PATH_TMP}" == *"s16_ghidra_decompile_checks"* ]]; then
print_output "[*] Ghidra decompiled code found ${lSCRIPT_PATH_TMP}" "no_log"
print_output "[*] AI-Assisted analysis of binary function ${ORANGE}${lSCRIPT_PATH_TMP}${NC}" "no_log"
elif [[ "${lSCRIPT_PATH_TMP}" == *"s28_java_check"* ]]; then
print_output "[*] Decompiled Java code found ${lSCRIPT_PATH_TMP}" "no_log"
print_output "[*] AI-Assisted analysis of Java binary ${ORANGE}${lSCRIPT_PATH_TMP}${NC}" "no_log"
elif [[ "${lSCRIPT_PATH_TMP}" == *"s20_shell_check"* ]]; then
print_output "[*] Shell code found ${lSCRIPT_PATH_TMP}" "no_log"
print_output "[*] AI-Assisted analysis of shell script ${ORANGE}${lSCRIPT_PATH_TMP}${NC}" "no_log"
elif [[ "${lSCRIPT_PATH_TMP}" == *"s21_python_check"* ]]; then
print_output "[*] Python code found ${lSCRIPT_PATH_TMP}" "no_log"
print_output "[*] AI-Assisted analysis of python script ${ORANGE}${lSCRIPT_PATH_TMP}${NC}" "no_log"
elif [[ "${lSCRIPT_PATH_TMP}" == *"s22_php_check"* ]]; then
print_output "[*] PHP code found ${lSCRIPT_PATH_TMP}" "no_log"
print_output "[*] AI-Assisted analysis of PHP script ${ORANGE}${lSCRIPT_PATH_TMP}${NC}" "no_log"
elif [[ "${lSCRIPT_PATH_TMP}" == *"s23_lua_check"* ]]; then
print_output "[*] Lua code found ${lSCRIPT_PATH_TMP}" "no_log"
print_output "[*] AI-Assisted analysis of lua script ${ORANGE}${lSCRIPT_PATH_TMP}${NC}" "no_log"
elif [[ "${lSCRIPT_PATH_TMP}" == *"s27_perl_check"* ]]; then
print_output "[*] Perl code found ${lSCRIPT_PATH_TMP}" "no_log"
print_output "[*] AI-Assisted analysis of perl script ${ORANGE}${lSCRIPT_PATH_TMP}${NC}" "no_log"
else
# this should not happen. If we land here we have changed something in the module area
# or we have a new module that is not handled correctly
print_output "[*] Identification of ${ORANGE}${lSCRIPT_PATH_TMP} / ${lAI_INPUT_FILE}${NC} inside ${ORANGE}${LOG_DIR}/firmware${NC}" "no_log"
if [[ "${lSCRIPT_PATH_TMP}" == ".""${LOG_DIR}"* ]]; then
print_output "[*] Warning: System path is not stripped with the root directory - we try to fix it now" "no_log"
# remove the '.'
lSCRIPT_PATH_TMP="${lSCRIPT_PATH_TMP:1}"
# remove the LOG_DIR
# shellcheck disable=SC2001
lSCRIPT_PATH_TMP="$(echo "${lSCRIPT_PATH_TMP}" | sed 's#'"${LOG_DIR}"'##')"
print_output "[*] Stripped path ${lSCRIPT_PATH_TMP}" "no_log"
fi
# dirty fix - Todo: use array in future
lSCRIPT_PATH_TMP="$(find "${LOG_DIR}/firmware" -wholename "*${lSCRIPT_PATH_TMP}" | head -1)"
# in case we have nothing we are going to move on
! [[ -f "${lSCRIPT_PATH_TMP}" ]] && continue
print_output "[*] AI-Assisted analysis of script ${ORANGE}${lSCRIPT_PATH_TMP}${NC}" "no_log"
fi
print_output "[*] Current priority for testing is ${lAI_PRIO}" "no_log"
if [[ ${lAI_PRIO} -ge ${MINIMUM_GPT_PRIO} ]] && [[ "${lSCRIPT_PATH_TMP}" != '' ]]; then
if [[ -f "${lSCRIPT_PATH_TMP}" ]]; then
sub_module_title "AI analysis for ${lAI_INPUT_FILE//\.log}"
local lBINARY_NAME="NA"
local lFUNCTION_NAME="NA"
# Starting with special attention modules
if [[ "${lSCRIPT_PATH_TMP}" == *"s16_ghidra_decompile_checks"* ]]; then
lBINARY_NAME="$(echo "${lSCRIPT_PATH_TMP}" | rev | cut -d '/' -f2 | rev | sed 's/haruspex_//')"
lFUNCTION_NAME=$(extract_s16_fct_name "${lSCRIPT_PATH_TMP}")
print_output "[*] AI-Assisted analysis for function ${ORANGE}${lFUNCTION_NAME}${NC} of decompiled binary ${ORANGE}${lBINARY_NAME}${NC}" "" "${lSCRIPT_PATH_TMP}"
print_output "$(indent "$(orange "$(print_path "${lSCRIPT_PATH_TMP}")")")"
generate_prompt_binary "${lSCRIPT_PATH_TMP}"
elif [[ "${lSCRIPT_PATH_TMP}" == *"s28_java_check"* ]]; then
# we start with a path like /logs/s28_java_check/java_decompile/i.class_16708/i.java
# -> the last are we use as function name
lFUNCTION_NAME="${lSCRIPT_PATH_TMP//*\/}"
# -> the middle area is the binary name
lBINARY_NAME="${lSCRIPT_PATH_TMP%\/*}"
lBINARY_NAME="${lBINARY_NAME//*\/}"
lBINARY_NAME="${lBINARY_NAME//_*}"
print_output "[*] AI-Assisted analysis for decompiled Java binary function ${ORANGE}${lFUNCTION_NAME}${NC} of ${ORANGE}${lBINARY_NAME}${NC}" "" "${lSCRIPT_PATH_TMP}"
# as we are using lSCRIPT_PATH_TMP in the prompt generator we get a prompt file like prompt_localai_NeoterisStatic.java_Java.txt which is the current lFUNCTION_NAME
# later on we can't find the prompt file as we look for the lBINARY_NAME. This means we rewrite the lBINARY_NAME with the lFUNCTION_NAME
lBINARY_NAME="${lFUNCTION_NAME//\.log}"
# we need to redefine the lFUNCTION_NAME now as the type of analysed binary
# Todo: rename and cleanup variable names
lFUNCTION_NAME="Java"
print_output "$(indent "$(orange "$(print_path "${lSCRIPT_PATH_TMP}")")")"
generate_prompt_script "${lSCRIPT_PATH_TMP}" "${lFUNCTION_NAME}"
else
# for all the default modules
lBINARY_NAME=$(basename "${lSCRIPT_PATH_TMP}")
lBINARY_NAME="${lBINARY_NAME//*\/}"
lBINARY_NAME="${lBINARY_NAME//\.log}"
if [[ "${lAI_SOURCE_FILE}" == *"s22_php_check"* ]]; then
lFUNCTION_NAME="PHP"
elif [[ "${lAI_SOURCE_FILE}" == *"s20_shell_check"* ]]; then
lFUNCTION_NAME="shell"
elif [[ "${lAI_SOURCE_FILE}" == *"s21_python_check"* ]]; then
lFUNCTION_NAME="python"
elif [[ "${lAI_SOURCE_FILE}" == *"s23_lua_check"* ]]; then
lFUNCTION_NAME="lua"
elif [[ "${lAI_SOURCE_FILE}" == *"s27_perl_check"* ]]; then
lFUNCTION_NAME="perl"
else
# fallback
lBINARY_NAME=$(basename "${lSCRIPT_PATH_TMP}")
lFUNCTION_NAME="unknown"
fi
print_output "[*] AI-Assisted analysis for ${ORANGE}${lFUNCTION_NAME}${NC} script ${ORANGE}${lBINARY_NAME}${NC}" "" "${lSCRIPT_PATH_TMP}"
print_output "$(indent "$(orange "$(print_path "${lSCRIPT_PATH_TMP}")")")"
generate_prompt_script "${lSCRIPT_PATH_TMP}" "${lFUNCTION_NAME}"
fi
if [[ ! -f "${AI_PROMPT_DIR}/prompt_localai_${lBINARY_NAME}_${lFUNCTION_NAME}.txt" ]]; then
print_output "[-] No prompt generated for ${AI_PROMPT_DIR}/prompt_localai_${lBINARY_NAME}_${lFUNCTION_NAME}.txt ... next one"
continue
fi
lPROMPT=$(cat "${AI_PROMPT_DIR}/prompt_localai_${lBINARY_NAME}_${lFUNCTION_NAME}.txt")
local lAI_LOG_FILE="${lAI_FILE_DIR}/ai_response_${lBINARY_NAME}_${lFUNCTION_NAME}.md"
local lWC_M_FILE=""
local lWC_L_FILE=""
local lTOKENS_PRE=""
# lWC_M_FILE=$(echo "${lPROMPT}" | wc -m)
lWC_M_FILE="${#lPROMPT}"
lWC_L_FILE=$(echo "${lPROMPT}" | wc -l)
# just a very rough estimation
lTOKENS_PRE=$((lWC_M_FILE/2))
print_output "[*] Testing prompt" "" "${AI_PROMPT_DIR}/prompt_localai_${lBINARY_NAME}_${lFUNCTION_NAME}.txt"
print_output "${ORANGE}${lPROMPT}${NC}" "no_log"
print_output "[*] Testing code with ${ORANGE}${lWC_L_FILE}${NC} lines and ${ORANGE}${lWC_M_FILE}${NC} characters"
print_output "[*] Testing will need at least ${ORANGE}${lTOKENS_PRE}${NC} tokens\n"
local lJSON_PAYLOAD=""
local lstart_time=""
local lTOKENS_POST=""
local lfinished_time=""
local lruntime=""
lJSON_PAYLOAD=$(jq -n \
--arg model "${lMODEL_LOCALAI}" \
--arg prompt "${lPROMPT}" \
'{model: $model, messages: [{role: "user", content: $prompt}], temperature: 0.2}')
lstart_time=$(date +%s)
lHTTP_CODE=$(curl --connect-timeout 10 --max-time 600 -s http://"${LOCAL_AI_IP}":8080/v1/chat/completions \
-H "Content-Type: application/json" \
-d "${lJSON_PAYLOAD}" -o "${lAI_LOG_FILE}.json" --write-out "%{http_code}" || true)
if [[ -f "${lAI_LOG_FILE}.json" ]]; then
# reformat the output to also have the token usage avaialble
jq -r '. | "\(.choices[].message.content)\n\nTokens used: \(.usage.total_tokens)"' "${lAI_LOG_FILE}.json" > "${lAI_LOG_FILE}" || print_error "[-] Q03 - AI parsing error for ${lAI_LOG_FILE}.json"
fi
if [[ ! -f "${lAI_LOG_FILE}" ]] || [[ ! -s "${lAI_LOG_FILE}" ]]; then
print_output "[-] WARNING: No AI respoonse for function ${lFUNCTION_NAME}"
continue
fi
# our currenty AI output starts the headlines with level 3 headlines ###
parse_markdown_to_emba_txt "${lAI_LOG_FILE}" "${lAI_LOG_FILE//\.md/\.txt}" 3
# just for the user output on cli:
cat "${lAI_LOG_FILE//\.md/\.txt}"
lTOKENS_POST=$(grep "Tokens used:" "${lAI_LOG_FILE}" | cut -d ':' -f2)
lTOKENS_POST=${lTOKENS_POST//\ /}
lfinished_time=$(date +%s)
lruntime=$((lfinished_time-lstart_time))
if [[ "${lTOKENS_POST}" -gt 0 && "${lruntime}" -gt 0 ]]; then
print_output "[*] AI took ${ORANGE}${lruntime}${NC} seconds and ${ORANGE}${lTOKENS_POST}${NC} tokens to complete this request."
print_output "[*] Tokens/sec: ${ORANGE}~$((lTOKENS_POST/lruntime))${NC}" || true
fi
if [[ "${lHTTP_CODE}" -ne 200 ]]; then
print_output "[-] Something went wrong with the LocalAI requests"
# track the errors. If we get too many we are going to stop AI requests
AI_ERROR_CNT=$((AI_ERROR_CNT+1))
if [[ -f "${lAI_LOG_FILE}" ]]; then
print_output "[-] ERROR response: $(cat "${lAI_LOG_FILE}")"
fi
# if the testing is over we can just return now
local lMODULE_RUNTIME=${SECONDS}
adjust_minimum_runtime "${lMODULE_RUNTIME}"
if [[ "${lMODULE_RUNTIME}" -gt "${AI_MIN_RUNTIME}" ]]; then
return
fi
continue
fi
if ! [[ -f "${lAI_LOG_FILE}" ]]; then
# catches: (56) Recv failure: Connection reset by peer
print_output "[-] Something went wrong with the LocalAI request for ${lAI_INPUT_FILE}"
AI_ERROR_CNT=$((AI_ERROR_CNT+1))
continue
fi
if [[ ${lTOKENS_POST} -ne 0 ]]; then
GTP_CHECKED_ARR+=("${lSCRIPT_PATH_TMP}")
# write results into done csv
# lAI_INPUT_FILE - lAI_ANCHOR - Priority - used Prompt file - source file to test - cost - ai final log file with results
write_csv_AI "${lAI_INPUT_FILE}" "${lAI_ANCHOR}" "${lAI_PRIO}" "${AI_PROMPT_DIR}/prompt_localai_${lBINARY_NAME}_${lFUNCTION_NAME}.txt" "${lAI_SOURCE_FILE}" "cost=${lTOKENS_POST}" "${lAI_LOG_FILE//\.md/\.txt}"
# add proper module link
if [[ "${lAI_SOURCE_FILE}" == *'/csv_logs/'* ]]; then
# if we have a csv_logs path we need to adjust the cut
lORIGIN_MODULE="$(echo "${lAI_SOURCE_FILE}" | cut -d / -f4 | cut -d_ -f1)"
elif [[ "${lAI_SOURCE_FILE}" == '/logs/'* ]]; then
lORIGIN_MODULE="$(echo "${lAI_SOURCE_FILE}" | grep -E "[splfq][0-9]{2,3}_.*" -o | cut -d '_' -f1)"
else
lORIGIN_MODULE="$(basename "$(dirname "${lAI_SOURCE_FILE}")" | cut -d_ -f1)"
fi
print_ln
print_output "[+] Further analysis results for ${ORANGE}${lAI_INPUT_FILE_mod}${GREEN} available in module ${ORANGE}${lORIGIN_MODULE}${NC}" "" "${lAI_SOURCE_FILE}"
if [[ -f "${lAI_LOG_FILE}" ]]; then
print_output "[+] AI analysis results for ${ORANGE}${lAI_INPUT_FILE_mod}${GREEN} available${NC}" "" "${lAI_LOG_FILE//\.md/\.txt}"
local lSECURITY_RATING=""
lSECURITY_RATING=$(grep -h "Security rating: " "${lAI_LOG_FILE}" | tr '.' '\n' | grep -o -E "[0-9]+/10" || true)
if [[ -n "${lSECURITY_RATING}" ]]; then
print_output "[+] AI security rating for ${ORANGE}${lAI_INPUT_FILE_mod}${GREEN}: ${ORANGE}${lSECURITY_RATING}${NC}" "" "${lAI_LOG_FILE//\.md/\.txt}"
fi
fi
print_ln
((AI_RESULT_CNT += 1))
fi
else
print_output "[-] Couldn't find ${ORANGE}$(print_path "${lSCRIPT_PATH_TMP}")${NC}"
fi
fi
# exit conditions
local lMODULE_RUNTIME=${SECONDS}
adjust_minimum_runtime "${lMODULE_RUNTIME}"
print_output "[*] Current AI module runtime: ${lMODULE_RUNTIME} / min module runtime ${AI_MIN_RUNTIME}" "no_log"
# if our module runtime is longer as our configured min runtime:
if [[ "${lMODULE_RUNTIME}" -gt "${AI_MIN_RUNTIME}" ]]; then
break
fi
regenerate_ai_todo_list
# reset the array index to start again with the highest rated entry
lELE_INDEX=-1
done
}
regenerate_ai_todo_list() {
print_output "[*] Regenerate analysis array ..." "no_log"
if [[ -f "${CSV_DIR}/ai_question.csv" ]]; then
local lGPT_ENTRY_LINE=""
while read -r lGPT_ENTRY_LINE; do
lAI_ANCHOR="$(echo "${lGPT_ENTRY_LINE}" | cut -d ';' -f2)"
sed -i "/${lAI_ANCHOR}/d" "${CSV_DIR}/ai_question.csv.tmp"
done <"${CSV_DIR}/ai_question.csv"
fi
# reload and resort current state of questions:
readarray -t Q03_LOCALAI_QUESTIONS < <(sort -k 3 -t ';' -r "${CSV_DIR}/ai_question.csv.tmp")
}
identify_ai_model() {
local lMODEL_LOCALAI=""
local lCNT=1
while [[ -z "${lMODEL_LOCALAI}" ]]; do
lMODEL_LOCALAI=$(curl --connect-timeout 10 --max-time 30 http://"${LOCAL_AI_IP}":8080/v1/models 2>/dev/null | jq -r .data[].id || true)
[[ -z "${lMODEL_LOCALAI}" ]] && sleep 5
lCNT=$((lCNT+1))
[[ "${lCNT}" -gt 5 ]] && break
done
echo "${lMODEL_LOCALAI}"
}
extract_s16_fct_name() {
local lFUNCTION_NAME="${1:-}"
# extract the function name from the beginning of the first part of the file
# e.g.: bool fct_name(asdf)
local lBINARY_NAME=""
lBINARY_NAME="$(echo "${lSCRIPT_PATH_TMP}" | rev | cut -d '/' -f2 | rev | sed 's/haruspex_//')"
lFUNCTION_NAME="$(strip_color_codes "$(head "${lSCRIPT_PATH_TMP}")")"
lFUNCTION_NAME="$(echo "${lFUNCTION_NAME}" | grep "^bool\|^uint\|^ulong\|^long\|^int\|^undefined\|^void\|^char\|^ssize\|^size" | grep "(.*" || true)"
if [[ -z "${lFUNCTION_NAME}" ]]; then
# e.g.: fct_name(asdf)
lFUNCTION_NAME="$(strip_color_codes "$(head "${lSCRIPT_PATH_TMP}")")"
lFUNCTION_NAME="$(echo "${lFUNCTION_NAME}" | grep "^[a-zA-Z0-9_-]*(.*" || true)"
fi
if [[ -z "${lFUNCTION_NAME}" ]]; then
lFUNCTION_NAME="unknown_${lBINARY_NAME:-NA}"
fi
lFUNCTION_NAME="${lFUNCTION_NAME//\(*}"
lFUNCTION_NAME="${lFUNCTION_NAME//* }"
echo "${lFUNCTION_NAME}"
}
generate_prompt_script() {
local lSCRIPT_PATH_TMP="${1:-}"
local lSCRIPT_TYPE="${2:-unknown}"
local lCODE_CONTENT_tmp=""
local lCODE_CONTENT=""
local lPROMPT=""
local lBINARY_NAME=""
lBINARY_NAME=$(basename "${lSCRIPT_PATH_TMP}")
lBINARY_NAME="${lBINARY_NAME//*\/}"
lBINARY_NAME="${lBINARY_NAME//\.log}"
print_output "[*] Build prompt for ${lSCRIPT_PATH_TMP} - lSCRIPT_NAME: ${lBINARY_NAME} - script type: ${lSCRIPT_TYPE}" "no_log"
# Read the file content and remove the ASK_AI marker
lCODE_CONTENT_tmp=$(grep -v "ASK_AI" "${lSCRIPT_PATH_TMP}")
# remove REF entries for HTML links
lCODE_CONTENT_tmp=$(sed '/^\[REF\] .*/d' <<< "${lCODE_CONTENT_tmp}")
# remove semgrep identifiers
lCODE_CONTENT_tmp=$(sed 's/\/\/possible issue identified -.*//' <<< "${lCODE_CONTENT_tmp}")
# remove color codes
lCODE_CONTENT_tmp=$(strip_color_codes "${lCODE_CONTENT_tmp}")
# remove shell comments and empty lines:
lCODE_CONTENT=$(sed '/^[[:blank:]]*#/d;s/[[:blank:]]*#.*//' <<< "${lCODE_CONTENT_tmp}")
lPROMPT="You are an expert software architect. Analyze the provided source code."
lPROMPT+=" The following source code is from a Linux based firmware ${lSCRIPT_TYPE} script called ${lBINARY_NAME}."
lPROMPT+=" Give a short summary of the functionality of the provided code"
lPROMPT+=" Improve the readability of the original source code inline with comments on the functionality. Perform an indepth security and vulnerability analysis of the provided source code. Use all available information for your security and vulnerability analysis. Include security notes inline in the original source code for security relevant areas and for identified vulnerabilities inline to the improved source code. Finally, give a security rating between 1 and 10 points, while 1 point is very insecure and 10 is a good security rating."
lPROMPT+=" Structure your report the following way:"
lPROMPT+=" 1.) Summary of ${lSCRIPT_TYPE} script ${lBINARY_NAME}"
lPROMPT+=" 2.) Optimized source code of ${lSCRIPT_TYPE} script ${lBINARY_NAME} with comments"
lPROMPT+=" 3.) Security overview with identified vulnerabilities"
lPROMPT+=" 4.) Final security rating in the form \"Security rating: <your rating>/10.\""
if [[ "${#lCODE_CONTENT}" -gt "${AI_MAX_CHARS_TO_ANALYSE}" ]]; then
print_output "[*] Limit code of ${lBINARY_NAME}/${lSCRIPT_TYPE} for analysis to ${AI_MAX_CHARS_TO_ANALYSE} characters" "no_log"
fi
lPROMPT="${lPROMPT}\nHere is the code:\n\n${lCODE_CONTENT:0:${AI_MAX_CHARS_TO_ANALYSE}}\n"
echo -n "${lPROMPT}" | iconv -f ISO-8859-1 -t UTF-8 > "${AI_PROMPT_DIR}/prompt_localai_${lBINARY_NAME}_${lSCRIPT_TYPE}.txt"
}
generate_prompt_binary() {
local lSCRIPT_PATH_TMP="${1:-}"
local lARCH=""
local lEND=""
local lKERNELV=""
local lRELRO=""
local lCANARIES=""
local lNX=""
local lSYMBOLS=""
local lCODE_CONTENT_tmp=""
local lCODE_CONTENT=""
local lPROMPT=""
local lBINARY_NAME=""
local lFUNCTION_NAME=""
lBINARY_NAME="$(echo "${lSCRIPT_PATH_TMP}" | rev | cut -d '/' -f2 | rev | sed 's/haruspex_//')"
lFUNCTION_NAME=$(extract_s16_fct_name "${lSCRIPT_PATH_TMP}")
print_output "[*] Build prompt for ${lSCRIPT_PATH_TMP} - lBINARY_NAME: ${lBINARY_NAME} - lFUNCTION_NAME: ${lFUNCTION_NAME}" "no_log"
# extract further binary details from the current EMBA report
if [[ -f "${P99_LOG}" ]]; then
lARCH="$(grep Statistics "${P99_LOG}" | cut -d ':' -f2)"
lEND="$(grep Statistics "${P99_LOG}" | cut -d ':' -f3)"
fi
if [[ -f "${S24_CSV_LOG}" ]]; then
lKERNELV=$(tail -n +2 "${S24_CSV_LOG}" | cut -d ';' -f2 | sort -u | head -1)
fi
if [[ -f "${S12_CSV_LOG}" ]]; then
lRELRO=$(grep "/${lBINARY_NAME}$" "${S12_CSV_LOG}" | cut -d ';' -f1 | sort -u | head -1)
lCANARIES=$(grep "/${lBINARY_NAME}$" "${S12_CSV_LOG}" | cut -d ';' -f2 | sort -u | head -1)
lNX=$(grep "/${lBINARY_NAME}$" "${S12_CSV_LOG}" | cut -d ';' -f3 | sort -u | head -1)
lSYMBOLS=$(grep "/${lBINARY_NAME}$" "${S12_CSV_LOG}" | cut -d ';' -f7 | sort -u | head -1)
fi
# Read the file content and remove the ASK_AI marker
lCODE_CONTENT_tmp=$(grep -v "ASK_AI" "${lSCRIPT_PATH_TMP}")
# remove REF entries for HTML links
lCODE_CONTENT_tmp=$(sed '/^\[REF\] .*/d' <<< "${lCODE_CONTENT_tmp}")
# remove semgrep identifiers
lCODE_CONTENT_tmp=$(sed 's/\/\/possible issue identified -.*//' <<< "${lCODE_CONTENT_tmp}")
# remove color codes
lCODE_CONTENT=$(strip_color_codes "${lCODE_CONTENT_tmp}")
lPROMPT="You are an expert software architect. Analyze the provided source code."
lPROMPT+=" The following source code is from a Linux based firmware binary called ${lBINARY_NAME}."
lPROMPT+=" It was automatically generated with the reverse engineering framework Ghidra."
if [[ -n "${lARCH}" ]]; then
lPROMPT+=" The architecture of the firmware is ${lARCH}."
if [[ -n "${lEND}" ]]; then
lPROMPT+=" The endianess of the firmware is ${lEND}."
fi
fi
if [[ -n "${lKERNELV}" ]]; then
lPROMPT+=" The original device was running a Linux kernel in version ${lKERNELV}."
fi
if [[ -n "${lRELRO}" ]]; then
lPROMPT+=" The original binary has the following RELRO security setting: ${lRELRO}."
fi
if [[ -n "${lCANARIES}" ]]; then
lPROMPT+=" The original binary has the following Stack Canaries security setting: ${lCANARIES}."
fi
if [[ -n "${lNX}" ]]; then
lPROMPT+=" The original binary has the following NX security setting: ${lNX}."
fi
if [[ -n "${lSYMBOLS}" ]]; then
if [[ "${lSYMBOLS}" == "No Symbols" ]]; then
lPROMPT+=" The original binary has no symbols included."
else
lPROMPT+=" The original binary has symbols included."
fi
fi
lPROMPT+=" Give a short summary of the functionality of the provided code with the functionname ${lFUNCTION_NAME}."
lPROMPT+=" Improve the readability of the original source code inline with comments on the functionality and useful variable names. Perform an indepth security and vulnerability analysis of the provided source code. Use all available information for your security and vulnerability analysis. Include security notes inline in the original source code for security relevant areas and for identified vulnerabilities inline to the improved source code. The variable names of the provided source code should be always renamed for better readability to support manual verification and further manual analysis. Finally, give a security rating between 1 and 10 points, while 1 point is very insecure and 10 is a good security rating."
lPROMPT+=" Structure your report the following way:"
lPROMPT+=" 1.) Summary of function ${lFUNCTION_NAME}"
lPROMPT+=" 2.) Optimized source code of function ${lFUNCTION_NAME} with renamed variables and comments"
lPROMPT+=" 3.) Security overview with identified vulnerabilities"
lPROMPT+=" 4.) Final security rating in the form \"Security rating: <your rating>/10.\""
if [[ "${#lCODE_CONTENT}" -gt "${AI_MAX_CHARS_TO_ANALYSE}" ]]; then
print_output "[*] Limit code of ${lBINARY_NAME}/${lFUNCTION_NAME} for analysis to ${AI_MAX_CHARS_TO_ANALYSE} characters"
fi
# limit the code to analyse to 5000 characters
lPROMPT="${lPROMPT}\nHere is the code:\n\n${lCODE_CONTENT:0:${AI_MAX_CHARS_TO_ANALYSE}}\n"
# print_output "[*] Wrting prompt for ${lSCRIPT_PATH_TMP} - ${lPROMPT} to ${AI_PROMPT_DIR}/prompt_localai_${lBINARY_NAME}_${lFUNCTION_NAME}.txt"
echo -n "${lPROMPT}" | iconv -f ISO-8859-1 -t UTF-8 > "${AI_PROMPT_DIR}/prompt_localai_${lBINARY_NAME}_${lFUNCTION_NAME}.txt"
}
modules/S15_radare_decompile_checks.sh
+3 -3
View File
@@ -217,14 +217,14 @@ radare_decomp_print_top10_statistics() {
if [[ -f "${lR2_BIN_LOG_PATH}" ]]; then
write_link "${lR2_BIN_LOG_PATH}"
# FIXME
if [[ "${GPT_OPTION}" -gt 0 ]]; then
if [[ "${AI_OPTION}" -gt 0 ]]; then
print_output "[*] Asking OpenAI chatbot about ${lR2_BIN_LOG_PATH}"
lGPT_ANCHOR_="$(openssl rand -hex 8)"
# "${GPT_INPUT_FILE_}" "${lGPT_ANCHOR_}" "${GPT_PRIO_}" "${GPT_QUESTION_}" "${GPT_OUTPUT_FILE_}" "cost=$GPT_TOKENS_" "${GPT_RESPONSE_}"
write_csv_gpt_tmp "${lR2_BIN_LOG_PATH}" "${lGPT_ANCHOR_}" "${lGPT_PRIO}" "Can you give me a side by side desciption of the following code in a table, where on the left is the code and on the right the desciption. And please use proper spacing and | to make it terminal friendly:" "${lR2_BIN_LOG_PATH}" "" ""
write_csv_AI_tmp "${lR2_BIN_LOG_PATH}" "${lGPT_ANCHOR_}" "${lGPT_PRIO}" "Can you give me a side by side desciption of the following code in a table, where on the left is the code and on the right the desciption. And please use proper spacing and | to make it terminal friendly:" "${lR2_BIN_LOG_PATH}" "" ""
# add ChatGPT link
printf '%s\n\n' "" >>"${lR2_BIN_LOG_PATH}"
write_anchor_gpt "${lGPT_ANCHOR_}" "${lR2_BIN_LOG_PATH}"
write_anchor_AI "${lGPT_ANCHOR_}" "${lR2_BIN_LOG_PATH}"
fi
fi
done
modules/S16_ghidra_decompile_checks.sh
+25 -9
View File
@@ -18,10 +18,9 @@
# The generated source code is further analysed with semgrep and the rules provided by 0xdea
# (https://github.com/0xdea/semgrep-rules)
# set DEBUG to 1 to enable further print statements
export DEBUG=0
S16_ghidra_decompile_checks() {
# export MAX_EXT_CHECK_BINS=3
module_log_init "${FUNCNAME[0]}"
module_title "Check decompiled binary source code for vulnerabilities"
pre_module_reporter "${FUNCNAME[0]}"
@@ -47,6 +46,9 @@ S16_ghidra_decompile_checks() {
local lNAME=""
local lBINS_CHECKED_ARR=()
# the blacklist elements are handled as regex in the form *pattern*
local lBIN_BLACKLIST_ARR=("libc")
if [[ "${FULL_TEST}" -ne 1 ]]; then
# we need to wait in default mode for the results of S13 and S14
module_wait "S13_weak_func_check"
@@ -64,6 +66,16 @@ S16_ghidra_decompile_checks() {
for lBIN_TO_CHECK in "${lBINARIES_ARR[@]}"; do
lNAME=$(basename "${lBIN_TO_CHECK}" 2>/dev/null)
if [[ "${FULL_TEST}" -ne 1 ]]; then
# first of all we check our binary name against our blacklist
# the blacklist are handled as regex in the form *pattern*
for lBIN_BLACKLIST in "${lBIN_BLACKLIST_ARR[@]}"; do
if [[ "${lNAME}" == *"${lBIN_BLACKLIST}"* ]]; then
continue 2
fi
done
fi
if [[ -f "${BASE_LINUX_FILES}" ]]; then
# if we have the base linux config file we only test non known Linux binaries
# with this we do not waste too much time on open source Linux stuff
@@ -379,14 +391,18 @@ s16_semgrep_logger() {
# GPT integration
lGPT_ANCHOR="$(openssl rand -hex 8)"
if [[ -f "${BASE_LINUX_FILES}" ]]; then
# if we have the base linux config file we are checking it:
if ! grep -E -q "^${lNAME}$" "${BASE_LINUX_FILES}" 2>/dev/null; then
lGPT_PRIO=$((lGPT_PRIO + 1))
fi
# if we have some semgrep results for this function we increase our priority
if grep -q "possible issue identified - semgrep" "${lC_SRC_FCT}"; then
local lSEMGREP_CNT=0
lSEMGREP_CNT=$(grep -c "possible issue identified - semgrep" "${lC_SRC_FCT}" || echo 0)
lGPT_PRIO=$((lGPT_PRIO + lSEMGREP_CNT))
fi
write_csv_gpt_tmp "${LOG_PATH_MODULE}/haruspex_${lNAME}/${lHARUSPEX_FILE_NAME}" "${lGPT_ANCHOR}" "${lGPT_PRIO}" "${GPT_QUESTION}" "${LOG_PATH_MODULE}/haruspex_${lNAME}/${lHARUSPEX_FILE_NAME}" "" ""
write_anchor_gpt "${lGPT_ANCHOR}" "${LOG_PATH_MODULE}"/haruspex_"${lNAME}"/"${lHARUSPEX_FILE_NAME}"
# if it is not a default (known) linux file we increase the priority
if ! grep -E -q "^${lNAME}$" "${BASE_LINUX_FILES}" 2>/dev/null; then
lGPT_PRIO=$((lGPT_PRIO + 1))
fi
write_csv_AI_tmp "${LOG_PATH_MODULE}/haruspex_${lNAME}/${lHARUSPEX_FILE_NAME}" "${lGPT_ANCHOR}" "${lGPT_PRIO}" "${GPT_QUESTION}" "${LOG_PATH_MODULE}/haruspex_${lNAME}/${lHARUSPEX_FILE_NAME}" "" ""
write_anchor_AI "${lGPT_ANCHOR}" "${LOG_PATH_MODULE}"/haruspex_"${lNAME}"/"${lHARUSPEX_FILE_NAME}"
if [[ -f "${lSEMGREPLOG_TMP}" ]]; then
cat "${lSEMGREPLOG_TMP}" >>"${lSEMGREPLOG_TXT}"
modules/S19_apk_check.sh
+1 -1
View File
@@ -80,7 +80,7 @@ apk_checker_helper() {
lAPK_NAME=$(basename -s .apk "${lAPK_FILE}")
print_output "[*] Testing Android apk with APKHunt - ${ORANGE}$(print_path "${lAPK_FILE}")${NC}"
go run "${EXT_DIR}"/APKHunt/apkhunt.go -p "${lAPK_FILE}" -l 2>&1 | tee -a "${LOG_PATH_MODULE}/APKHunt-${lAPK_NAME}.txt"
go run "${EXT_DIR}"/APKHunt/apkhunt.go -p "${lAPK_FILE}" -l 2>&1 | tee -a "${LOG_PATH_MODULE}/APKHunt-${lAPK_NAME}.txt" || print_error "[-] Failed to analyze apk file ${lAPK_FILE} with apkhunt"
if [[ -f "${LOG_PATH_MODULE}/APKHunt-${lAPK_NAME}.txt" ]]; then
lAPK_ISSUES=$(grep -c -E "^[0-9]+:" "${LOG_PATH_MODULE}/APKHunt-${lAPK_NAME}.txt" || true)
modules/S20_shell_check.sh
+29 -14
View File
@@ -115,8 +115,9 @@ S20_shell_check() {
s20_eval_script_check() {
local lSH_SCRIPTS_ARR=("${@}")
local lSH_SCRIPT=""
local lGPT_PRIO_=3
local lGPT_ANCHOR_=""
# we rate eval usages higher compared to other shellcheck issues
local lGPT_PRIO_=4
local lAI_ANCHOR_=""
local lEVAL_RESULTS=0
local lSH_SCRIPT_NAME=""
@@ -133,19 +134,19 @@ s20_eval_script_check() {
sed -i -r "s/.*eval\ .*/\x1b[32m&\x1b[0m/" "${lSHELL_LOG}"
print_output "[+] Found ${ORANGE}eval${GREEN} usage in ${ORANGE}${lSH_SCRIPT_NAME}${NC}" "" "${lSHELL_LOG}"
if [[ "${GPT_OPTION}" -gt 0 ]]; then
lGPT_ANCHOR_="$(openssl rand -hex 8)"
if [[ "${AI_OPTION}" -gt 0 ]]; then
lAI_ANCHOR_="$(openssl rand -hex 8)"
if [[ -f "${BASE_LINUX_FILES}" ]]; then
# if we have the base linux config file we are checking it:
if ! grep -E -q "^${lSH_SCRIPT_NAME}$" "${BASE_LINUX_FILES}" 2>/dev/null; then
lGPT_PRIO_=$((lGPT_PRIO_ + 1))
fi
fi
# "${GPT_INPUT_FILE_}" "${lGPT_ANCHOR_}" "GPT-Prio-$lGPT_PRIO_" "${GPT_QUESTION_}" "${GPT_OUTPUT_FILE_}" "cost=$GPT_TOKENS_" "${GPT_RESPONSE_}"
write_csv_gpt_tmp "$(cut_path "${lSH_SCRIPT/;*/}")" "${lGPT_ANCHOR_}" "${lGPT_PRIO_}" "${GPT_QUESTION}" "${lSHELL_LOG}" "" ""
# "${GPT_INPUT_FILE_}" "${lAI_ANCHOR_}" "GPT-Prio-$lGPT_PRIO_" "${GPT_QUESTION_}" "${GPT_OUTPUT_FILE_}" "cost=$GPT_TOKENS_" "${GPT_RESPONSE_}"
write_csv_AI_tmp "$(cut_path "${lSH_SCRIPT/;*/}")" "${lAI_ANCHOR_}" "${lGPT_PRIO_}" "${GPT_QUESTION}" "${lSHELL_LOG}" "" ""
# add ChatGPT link
printf '%s\n\n' "" >>"${lSHELL_LOG}"
write_anchor_gpt "${lGPT_ANCHOR_}" "${lSHELL_LOG}"
write_anchor_AI "${lAI_ANCHOR_}" "${lSHELL_LOG}"
fi
fi
done
@@ -173,11 +174,19 @@ s20_reporter() {
local lSH_SCRIPT_="${2:-}"
local lSHELL_LOG="${3:-}"
local lGPT_PRIO_=2
local lGPT_ANCHOR_=""
local lAI_ANCHOR_=""
local lSH_NAME=""
lSH_NAME=$(basename "${lSH_SCRIPT_}" 2>/dev/null | sed -e 's/:/_/g')
local lS20_SOURCE_DIR="${LOG_PATH_MODULE}/sh_sources"
if [[ "${lVULNS}" -ne 0 ]]; then
[[ ! -d "${lS20_SOURCE_DIR}" ]] && mkdir -p "${lS20_SOURCE_DIR}"
# safe the sources and link it in the lSHELL_LOG
write_log "" "${lSHELL_LOG}"
write_log "[*] Source file ${ORANGE}${lSH_NAME}${NC}" "${lSHELL_LOG}"
copy_and_link_file "${lSH_SCRIPT_}" "${lS20_SOURCE_DIR}/${lSH_NAME}.log" "${lSHELL_LOG}"
# check if this is common linux file:
local lCOMMON_FILES_FOUND=""
local lCFF=""
@@ -198,15 +207,21 @@ s20_reporter() {
else
print_output "[+] Found ${ORANGE}${lVULNS} issues${GREEN} in script ${lCOMMON_FILES_FOUND}:${NC} $(print_path "${lSH_SCRIPT_}")" "" "${lSHELL_LOG}"
fi
# scripts with eval usage are higher rated
if grep -q "eval" "${lS20_SOURCE_DIR}/${lSH_NAME}.log"; then
lGPT_PRIO_=$((lGPT_PRIO_ + 2))
fi
write_csv_log "$(print_path "${lSH_SCRIPT_}")" "${lVULNS}" "${lCFF}" "NA"
if [[ "${GPT_OPTION}" -gt 0 ]]; then
lGPT_ANCHOR_="$(openssl rand -hex 8)"
# "${GPT_INPUT_FILE_}" "${lGPT_ANCHOR_}" "GPT-Prio-$lGPT_PRIO_" "${GPT_QUESTION_}" "${GPT_OUTPUT_FILE_}" "cost=$GPT_TOKENS_" "${GPT_RESPONSE_}"
write_csv_gpt_tmp "$(cut_path "${lSH_SCRIPT_}")" "${lGPT_ANCHOR_}" "${lGPT_PRIO_}" "${GPT_QUESTION}" "${lSHELL_LOG}" "" ""
# add ChatGPT link
if [[ "${AI_OPTION}" -gt 0 ]]; then
lAI_ANCHOR_="$(openssl rand -hex 8)"
# "${GPT_INPUT_FILE_}" "${lAI_ANCHOR_}" "GPT-Prio-$lGPT_PRIO_" "${GPT_QUESTION_}" "${GPT_OUTPUT_FILE_}" "cost=$GPT_TOKENS_" "${GPT_RESPONSE_}"
write_csv_AI_tmp "${lS20_SOURCE_DIR}/${lSH_NAME}.log" "${lAI_ANCHOR_}" "${lGPT_PRIO_}" "${GPT_QUESTION}" "${lSHELL_LOG}" "" ""
# add AI link
printf '%s\n\n' "" >>"${lSHELL_LOG}"
write_anchor_gpt "${lGPT_ANCHOR_}" "${lSHELL_LOG}"
write_anchor_AI "${lAI_ANCHOR_}" "${lSHELL_LOG}"
printf '%s\n\n' "" >>"${lS20_SOURCE_DIR}/${lSH_NAME}.log"
write_anchor_AI "${lAI_ANCHOR_}" "${lS20_SOURCE_DIR}/${lSH_NAME}.log"
fi
echo "${lVULNS}" >>"${TMP_DIR}"/S20_VULNS.tmp
modules/S21_python_check.sh
+25 -13
View File
@@ -92,7 +92,8 @@ s21_script_bandit() {
local lPY_LOG=""
local lVULNS=""
local lGPT_PRIO_=2
local lGPT_ANCHOR_=""
local lAI_ANCHOR=""
local lS21_SOURCE_DIR="${LOG_PATH_MODULE}/py_sources"
lSCRIPT_NAME=$(basename "${lPY_SCRIPT_}" 2>/dev/null | sed -e 's/:/_/g')
lPY_LOG="${LOG_PATH_MODULE}""/bandit_""${lSCRIPT_NAME}"".txt"
@@ -100,6 +101,13 @@ s21_script_bandit() {
lVULNS=$(grep -c ">> Issue: " "${lPY_LOG}" 2>/dev/null || true)
if [[ "${lVULNS}" -ne 0 ]]; then
[[ ! -d "${lS21_SOURCE_DIR}" ]] && mkdir -p "${lS21_SOURCE_DIR}"
# safe the sources and link it in the lSHELL_LOG
write_log "" "${lPY_LOG}"
write_log "[*] Source file ${ORANGE}${lSCRIPT_NAME}${NC}" "${lPY_LOG}"
copy_and_link_file "${lPY_SCRIPT_}" "${lS21_SOURCE_DIR}/${lSCRIPT_NAME}.log" "${lPY_LOG}"
# check if this is common linux file:
local lCOMMON_FILES_FOUND=""
local lCFF=""
@@ -115,26 +123,30 @@ s21_script_bandit() {
lCFF="NA"
fi
if [[ "${lVULNS}" -gt 20 ]]; then
print_output "[+] Found ""${RED}""${lVULNS}"" issues""${GREEN}"" in script ""${lCOMMON_FILES_FOUND}"":""${NC}"" ""$(print_path "${lPY_SCRIPT_}")" "" "${lPY_LOG}"
print_output "[+] Found ${RED}${lVULNS} issues${GREEN} in script ${lCOMMON_FILES_FOUND}:${NC} $(print_path "${lPY_SCRIPT_}")" "" "${lPY_LOG}"
lGPT_PRIO_=3
else
print_output "[+] Found ""${ORANGE}""${lVULNS}"" issues""${GREEN}"" in script ""${lCOMMON_FILES_FOUND}"":""${NC}"" ""$(print_path "${lPY_SCRIPT_}")" "" "${lPY_LOG}"
print_output "[+] Found ${ORANGE}${lVULNS} issues${GREEN} in script ${lCOMMON_FILES_FOUND}:${NC} $(print_path "${lPY_SCRIPT_}")" "" "${lPY_LOG}"
fi
write_csv_log "$(print_path "${lPY_SCRIPT_}")" "${lVULNS}" "${lCFF}" "NA"
if [[ "${GPT_OPTION}" -gt 0 ]]; then
lGPT_ANCHOR_="$(openssl rand -hex 8)"
if [[ -f "${BASE_LINUX_FILES}" ]]; then
# if we have the base linux config file we are checking it:
if ! grep -E -q "^$(basename "${lPY_SCRIPT_}")$" "${BASE_LINUX_FILES}" 2>/dev/null; then
lGPT_PRIO_=$((lGPT_PRIO_ + 1))
fi
if [[ "${AI_OPTION}" -gt 0 ]]; then
lAI_ANCHOR="$(openssl rand -hex 8)"
# if we have some default python packages we rate them lower
if [[ "${lPY_SCRIPT_}" == *"dist-packages"* || "${lPY_SCRIPT_}" == *"site-packages"* ]]; then
lGPT_PRIO_=$((lGPT_PRIO_ - 1))
elif ! grep -E -q "^$(basename "${lPY_SCRIPT_}")$" "${BASE_LINUX_FILES}" 2>/dev/null; then
# if no entry in our BASE_LINUX_FILES config we do not have a default
# linux file and we rate it higher for faster AI test
lGPT_PRIO_=$((lGPT_PRIO_ + 1))
fi
# "${GPT_INPUT_FILE_}" "${lGPT_ANCHOR_}" "${lGPT_PRIO_}" "${GPT_QUESTION_}" "${GPT_OUTPUT_FILE_}" "cost=$GPT_TOKENS_" "${GPT_RESPONSE_}"
write_csv_gpt_tmp "$(cut_path "${lPY_SCRIPT_}")" "${lGPT_ANCHOR_}" "${lGPT_PRIO_}" "${GPT_QUESTION}" "${lPY_LOG}" "" ""
# "${GPT_INPUT_FILE_}" "${lAI_ANCHOR}" "${lGPT_PRIO_}" "${GPT_QUESTION_}" "${GPT_OUTPUT_FILE_}" "cost=$GPT_TOKENS_" "${GPT_RESPONSE_}"
write_csv_AI_tmp "${lS21_SOURCE_DIR}/${lSCRIPT_NAME}.log" "${lAI_ANCHOR}" "${lGPT_PRIO_}" "${GPT_QUESTION}" "${lPY_LOG}" "" ""
# add ChatGPT link to output file
printf '%s\n\n' "" >>"${lPY_LOG}"
write_anchor_gpt "${lGPT_ANCHOR_}" "${lPY_LOG}"
write_anchor_AI "${lAI_ANCHOR}" "${lPY_LOG}"
printf '%s\n\n' "" >>"${lS21_SOURCE_DIR}/${lSCRIPT_NAME}.log"
write_anchor_AI "${lAI_ANCHOR}" "${lS21_SOURCE_DIR}/${lSCRIPT_NAME}.log"
fi
echo "${lVULNS}" >>"${TMP_DIR}"/S21_VULNS.tmp
fi
modules/S22_php_check.sh
+53 -26
View File
@@ -84,6 +84,8 @@ s22_vuln_check_semgrep() {
local lS22_SEMGREP_SCRIPTS=""
local lSEMG_SOURCE_NOTE=""
local lS22_SOURCE_DIR="${LOG_PATH_MODULE}/php_sources"
# multiple output options would be nice. Currently we have the xml output to parse it easily for getting the line number of the issue
# but this output is not very beautiful to show in the report.
semgrep --disable-version-check --metrics=off --junit-xml --config "${EXT_DIR}"/semgrep-rules/php "${LOG_DIR}"/firmware/ &>"${lPHP_SEMGREP_LOG}" || true
@@ -111,10 +113,12 @@ s22_vuln_check_semgrep() {
local lSEMG_SOURCE_FILE=""
local lSEMG_SOURCE_FILE_NAME=""
local lSEMG_LINE_NR=""
local lGPT_PRIO_=4
local lGPT_ANCHOR_=""
# we set the priority higher as the semgrep analysis on scripts is more reliable
local lGPT_PRIO_=5
local lAI_ANCHOR=""
! [[ -d "${LOG_PATH_MODULE}"/semgrep_sources/ ]] && mkdir "${LOG_PATH_MODULE}"/semgrep_sources/
[[ ! -d "${LOG_PATH_MODULE}"/semgrep_sources/ ]] && mkdir "${LOG_PATH_MODULE}"/semgrep_sources/
[[ ! -d "${lS22_SOURCE_DIR}" ]] && mkdir -p "${lS22_SOURCE_DIR}"
lSEMG_ISSUE_NAME=$(echo "${lSEMG_SOURCE_NOTE}" | tr ' ' '\n' | grep "^name=")
lSEMG_ISSUE_NAME="${lSEMG_ISSUE_NAME/name=\"/}"
@@ -122,29 +126,39 @@ s22_vuln_check_semgrep() {
lSEMG_SOURCE_FILE=$(echo "${lSEMG_SOURCE_NOTE}" | tr ' ' '\n' | grep "^file=")
lSEMG_SOURCE_FILE="${lSEMG_SOURCE_FILE/file=\"/}"
lSEMG_SOURCE_FILE_NAME=$(basename "${lSEMG_SOURCE_FILE}")
local lSEMG_SOURCE_FILE_log="${LOG_PATH_MODULE}/semgrep_sources/${lSEMG_SOURCE_FILE_NAME}.log"
[[ -f "${lSEMG_SOURCE_FILE}" && ! -f "${LOG_PATH_MODULE}"/semgrep_sources/"${lSEMG_SOURCE_FILE_NAME}".log ]] && cp "${lSEMG_SOURCE_FILE}" "${LOG_PATH_MODULE}"/semgrep_sources/"${lSEMG_SOURCE_FILE_NAME}".log
[[ -f "${lSEMG_SOURCE_FILE}" && ! -f "${LOG_PATH_MODULE}"/semgrep_sources/"${lSEMG_SOURCE_FILE_NAME}".log ]] && cp "${lSEMG_SOURCE_FILE}" "${lSEMG_SOURCE_FILE_log}"
lSEMG_LINE_NR=$(echo "${lSEMG_SOURCE_NOTE}" | tr ' ' '\n' | grep "^line=")
lSEMG_LINE_NR="${lSEMG_LINE_NR/line=\"/}"
sed -i -r "${lSEMG_LINE_NR}s/.*/\x1b[32m&\x1b[0m/" "${LOG_PATH_MODULE}"/semgrep_sources/"${lSEMG_SOURCE_FILE_NAME}".log || true
print_output "[+] Found possible PHP vulnerability ${ORANGE}${lSEMG_ISSUE_NAME}${GREEN} in ${ORANGE}${lSEMG_SOURCE_FILE_NAME}${GREEN}" "" "${LOG_PATH_MODULE}/semgrep_sources/${lSEMG_SOURCE_FILE_NAME}.log"
# make the identified line of code green and add a note
sed -i -r "${lSEMG_LINE_NR}s/.*/\x1b[32m&\x1b[0m \/\/possible issue identified - semgrep/" "${lSEMG_SOURCE_FILE_log}" || true
print_output "[+] Found possible PHP vulnerability ${ORANGE}${lSEMG_ISSUE_NAME}${GREEN} in ${ORANGE}${lSEMG_SOURCE_FILE_NAME}${GREEN}" "" "${lSEMG_SOURCE_FILE_log}"
write_csv_log "${lSEMG_SOURCE_FILE}" "${lSEMG_ISSUE_NAME}" "semgrep" "unknown"
if [[ "${GPT_OPTION}" -gt 0 ]]; then
lGPT_ANCHOR_="$(openssl rand -hex 8)"
if [[ -f "${BASE_LINUX_FILES}" ]]; then
# if we have the base linux config file we are checking it:
if ! grep -E -q "^${lSEMG_SOURCE_FILE_NAME}$" "${BASE_LINUX_FILES}" 2>/dev/null; then
lGPT_PRIO_=$((lGPT_PRIO_ + 1))
fi
if [[ "${AI_OPTION}" -gt 0 ]]; then
lAI_ANCHOR="$(openssl rand -hex 8)"
if ! grep -E -q "^${lSEMG_SOURCE_FILE_NAME}$" "${BASE_LINUX_FILES}" 2>/dev/null; then
lGPT_PRIO_=$((lGPT_PRIO_ + 1))
else
# on well known files we lower the priority
lGPT_PRIO_=$((lGPT_PRIO_ - 2))
fi
# "${GPT_INPUT_FILE_}" "${lGPT_ANCHOR_}" "${lGPT_PRIO_}" "${GPT_QUESTION_}" "${GPT_OUTPUT_FILE_}" "cost=$GPT_TOKENS_" "${GPT_RESPONSE_}"
write_csv_gpt_tmp "$(cut_path "${lSEMG_SOURCE_FILE}")" "${lGPT_ANCHOR_}" "${lGPT_PRIO_}" "${GPT_QUESTION} And I think there might be something in line ${lSEMG_LINE_NR}" "${LOG_PATH_MODULE}/semgrep_sources/${lSEMG_SOURCE_FILE_NAME}.log" "" ""
# if we have some semgrep results for this function we increase our priority
if grep -q "possible issue identified - semgrep" "${lSEMG_SOURCE_FILE_log}"; then
local lSEMGREP_CNT=0
lSEMGREP_CNT=$(grep -c "possible issue identified - semgrep" "${lSEMG_SOURCE_FILE_log}" || echo 0)
lGPT_PRIO_=$((lGPT_PRIO_ + lSEMGREP_CNT))
fi
# "${GPT_INPUT_FILE_}" "${lAI_ANCHOR}" "${lGPT_PRIO_}" "${GPT_QUESTION_}" "${GPT_OUTPUT_FILE_}" "cost=$GPT_TOKENS_" "${GPT_RESPONSE_}"
write_csv_AI_tmp "${lSEMG_SOURCE_FILE_log}" "${lAI_ANCHOR}" "${lGPT_PRIO_}" "${GPT_QUESTION} And I think there might be something in line ${lSEMG_LINE_NR}" "${lSEMG_SOURCE_FILE_log}" "" ""
# add ChatGPT link
printf '%s\n\n' "" >>"${LOG_PATH_MODULE}/semgrep_sources/${lSEMG_SOURCE_FILE_NAME}.log"
write_anchor_gpt "${lGPT_ANCHOR_}" "${LOG_PATH_MODULE}/semgrep_sources/${lSEMG_SOURCE_FILE_NAME}.log"
printf '%s\n\n' "" >>"${lSEMG_SOURCE_FILE_log}"
write_anchor_AI "${lAI_ANCHOR}" "${lSEMG_SOURCE_FILE_log}"
fi
done
fi
@@ -193,6 +207,8 @@ s22_vuln_check() {
return
fi
local lS22_SOURCE_DIR="${LOG_PATH_MODULE}/php_sources"
local lPHP_SCRIPT_NAME=""
local lVULNS=0
local lTOTAL_MEMORY=0
@@ -201,7 +217,7 @@ s22_vuln_check() {
local lMEM_LIMIT=$(("${lTOTAL_MEMORY}" / 2))
lPHP_SCRIPT_NAME=$(basename "${lPHP_SCRIPT_}" 2>/dev/null | sed -e 's/:/_/g')
local lPHP_LOG="${LOG_PATH_MODULE}""/php_vuln_""${lPHP_SCRIPT_NAME}""-${RANDOM}.txt"
local lPHP_LOG="${LOG_PATH_MODULE}/php_vuln_${lPHP_SCRIPT_NAME}-${RANDOM}.txt"
ulimit -Sv "${lMEM_LIMIT}"
"${EXT_DIR}"/progpilot "${lPHP_SCRIPT_}" >>"${lPHP_LOG}" 2>&1 || true
@@ -209,8 +225,9 @@ s22_vuln_check() {
lVULNS=$(grep -c "vuln_name" "${lPHP_LOG}" 2>/dev/null || true)
local lGPT_PRIO_=4
local lGPT_ANCHOR_=""
local lAI_ANCHOR=""
if [[ "${lVULNS}" -gt 0 ]]; then
[[ ! -d "${lS22_SOURCE_DIR}" ]] && mkdir -p "${lS22_SOURCE_DIR}"
# check if this is common linux file:
local lCOMMON_FILES_FOUND=""
local lCFF=""
@@ -227,15 +244,25 @@ s22_vuln_check() {
lCFF="NA"
fi
print_output "[+] Found ${ORANGE}${lVULNS} vulnerabilities${GREEN} in php file: ${ORANGE}$(print_path "${lPHP_SCRIPT_}")${GREEN}${lCOMMON_FILES_FOUND}${NC}" "" "${lPHP_LOG}"
# safe the sources and link it in the lSHELL_LOG
write_log "" "${lPHP_LOG}"
write_log "[*] Source file ${ORANGE}${lPHP_SCRIPT_NAME}${NC}" "${lPHP_LOG}"
copy_and_link_file "${lPHP_SCRIPT_}" "${lS22_SOURCE_DIR}/${lPHP_SCRIPT_NAME}.log" "${lPHP_LOG}"
write_csv_log "${lPHP_SCRIPT_}" "TODO" "progpilot" "${lCFF}"
if [[ "${GPT_OPTION}" -gt 0 ]]; then
lGPT_ANCHOR_="$(openssl rand -hex 8)"
# "${GPT_INPUT_FILE_}" "${lGPT_ANCHOR_}" "GPT-Prio-$lGPT_PRIO_" "${GPT_QUESTION_}" "${GPT_OUTPUT_FILE_}" "cost=$GPT_TOKENS_" "${GPT_RESPONSE_}"
write_csv_gpt_tmp "$(cut_path "${lPHP_SCRIPT_}")" "${lGPT_ANCHOR_}" "${lGPT_PRIO_}" "${GPT_QUESTION}" "${TMP_DIR}/S22_VULNS.tmp" "" ""
# add ChatGPT link
printf '%s\n\n' "" >>"${TMP_DIR}"/S22_VULNS.tmp
write_anchor_gpt "${lGPT_ANCHOR_}" "${TMP_DIR}"/S22_VULNS.tmp
if [[ "${AI_OPTION}" -gt 0 ]]; then
lAI_ANCHOR="$(openssl rand -hex 8)"
lGPT_PRIO_=$((lGPT_PRIO_ + lVULNS))
# "${GPT_INPUT_FILE_}" "${lAI_ANCHOR}" "GPT-Prio-$lGPT_PRIO_" "${GPT_QUESTION_}" "${GPT_OUTPUT_FILE_}" "cost=$GPT_TOKENS_" "${GPT_RESPONSE_}"
write_csv_AI_tmp "${lS22_SOURCE_DIR}/${lPHP_SCRIPT_NAME}.log" "${lAI_ANCHOR}" "${lGPT_PRIO_}" "${GPT_QUESTION}" "${lPHP_SCRIPT_}" "" ""
# add AI link
printf '%s\n\n' "" >>"${lPHP_SCRIPT_}"
write_anchor_AI "${lAI_ANCHOR}" "${lPHP_SCRIPT_}"
printf '%s\n\n' "" >>"${lS22_SOURCE_DIR}/${lPHP_SCRIPT_NAME}.log"
write_anchor_AI "${lAI_ANCHOR}" "${lS22_SOURCE_DIR}/${lPHP_SCRIPT_NAME}.log"
fi
echo "${lVULNS}" >>"${TMP_DIR}"/S22_VULNS.tmp
else
modules/S23_lua_check.sh
+26 -28
View File
@@ -67,12 +67,9 @@ S23_lua_check() {
# extract the not 0 results of the vulnerabilities
lS23_LUA_VULNS=$(cut -d ';' -f 3 "${S23_CSV_LOG}" 2>/dev/null | tail +2 | grep -v -c "^0$" || true)
if [[ "${lS23_LUA_VULNS}" -gt 0 ]]; then
print_ln
if [[ "${lS23_LUA_VULNS}" -gt 0 || "${lS23_LUA_ISSUES}" -gt 0 ]]; then
sub_module_title "LUA Security results overview"
print_output "[+] Found ${ORANGE}${lS23_LUA_VULNS} possible security issues${GREEN} in ${ORANGE}${#LUA_CGI_FILES_ARR[@]}${GREEN} lua files.${NC}"
fi
if [[ "${lS23_LUA_ISSUES}" -gt 0 ]]; then
print_ln
print_output "[+] Found ${ORANGE}${lS23_LUA_ISSUES} coding issues${GREEN} in ${ORANGE}${#LUA_CGI_FILES_ARR[@]}${GREEN} lua files.${NC}"
fi
@@ -83,8 +80,8 @@ S23_lua_check() {
# this is a very basic checker for LUA issues
s23_luaseccheck() {
local lGPT_ANCHOR_=""
local lGPT_PRIO_=3
local lAI_ANCHOR=""
local lAI_PRIO=3
local lENTRY=""
local lQUERY_ENTRIES_ARR=()
local lQUERY_FILE=""
@@ -95,6 +92,8 @@ s23_luaseccheck() {
for lQUERY_FILE in "${LUA_CGI_FILES_ARR[@]}"; do
[[ ! -f "${lQUERY_FILE}" ]] && continue
local lISSUES_FILE=0
local lLOG_QUERYFILE=""
lLOG_QUERYFILE="${LOG_PATH_MODULE}/$(basename "${lQUERY_FILE}").log"
mapfile -t lQUERY_ENTRIES_ARR < <(grep -E "=.*cgilua\.QUERY" "${lQUERY_FILE}" | tr ' ' '\n' | sed 's/.*cgilua.QUERY.//' |
sed 's/.*cgilua.QUERY.//' | grep -o -E "^[[:alnum:]]+" | grep -v "^local$" | sort -u || true)
@@ -109,18 +108,18 @@ s23_luaseccheck() {
lS23_LUA_VULNS=$((lS23_LUA_VULNS + 1))
lISSUES_FILE=$((lISSUES_FILE + 1))
print_output "[+] Found lua QUERY (GET/POST) entry: ${ORANGE}${lENTRY}${GREEN} in file ${ORANGE}${lQUERY_FILE}${GREEN} with file access capabilities."
copy_and_link_file "${lQUERY_FILE}" "${LOG_PATH_MODULE}/$(basename "${lQUERY_FILE}").log"
sed -i -r "s/.*io\.(p)?open.*/\x1b[32m&\x1b[0m/" "${LOG_PATH_MODULE}/$(basename "${lQUERY_FILE}").log"
lGPT_PRIO_=$((lGPT_PRIO_ + 1))
copy_and_link_file "${lQUERY_FILE}" "${lLOG_QUERYFILE}"
sed -i -r "s/.*io\.(p)?open.*/\x1b[32m&\x1b[0m/" "${lLOG_QUERYFILE}"
lAI_PRIO=$((lAI_PRIO + 1))
fi
if grep "${lENTRY}" "${lQUERY_FILE}" 2>/dev/null | grep -q "os.execute"; then
# command exec - critical
lS23_LUA_VULNS=$((lS23_LUA_VULNS + 1))
lISSUES_FILE=$((lISSUES_FILE + 1))
print_output "[+] Found lua QUERY (GET/POST) entry: ${ORANGE}${lENTRY}${GREEN} in file ${ORANGE}${lQUERY_FILE}${GREEN} with command execution capabilities."
copy_and_link_file "${lQUERY_FILE}" "${LOG_PATH_MODULE}/$(basename "${lQUERY_FILE}").log"
sed -i -r "s/.*os\.execute.*/\x1b[32m&\x1b[0m/" "${LOG_PATH_MODULE}/$(basename "${lQUERY_FILE}").log"
lGPT_PRIO_=$((lGPT_PRIO_ + 1))
copy_and_link_file "${lQUERY_FILE}" "${lLOG_QUERYFILE}"
sed -i -r "s/.*os\.execute.*/\x1b[32m&\x1b[0m/" "${lLOG_QUERYFILE}"
lAI_PRIO=$((lAI_PRIO + 1))
fi
done
@@ -129,8 +128,6 @@ s23_luaseccheck() {
# os.execute(string.format("sleep %d", tonumber(sec)))
# os.execute(asdf_cmd)
if [[ "${lISSUES_FILE}" -eq 0 ]] && grep -E -q "os\.execute\(.*\.\..*\)|os\.execute\(.*\%.*\)|os\.execute\([[:alnum:]_]+\)" "${lQUERY_FILE}"; then
local lLOG_QUERYFILE=""
lLOG_QUERYFILE="${LOG_PATH_MODULE}/$(basename "${lQUERY_FILE}").log"
print_output "[+] Found lua file ${ORANGE}${lQUERY_FILE}${GREEN} with possible command execution for review."
copy_and_link_file "${lQUERY_FILE}" "${lLOG_QUERYFILE}"
if [[ -f "${lLOG_QUERYFILE}" ]]; then
@@ -142,25 +139,26 @@ s23_luaseccheck() {
sed -i -r "s/.*os\.execute\([[:alnum:]_]+\)/\x1b[32m&\x1b[0m/" "${lLOG_QUERYFILE}"
fi
lISSUES_FILE=$((lISSUES_FILE + 1))
lAI_PRIO=$((lAI_PRIO + 1))
fi
if [[ "${lISSUES_FILE}" -eq 0 ]] && grep -E -q "io\.(p)?open" "${lQUERY_FILE}"; then
# command exec - not our parameter but we check it
print_output "[+] Found lua file ${ORANGE}${lQUERY_FILE}${GREEN} with possible file access for review."
copy_and_link_file "${lQUERY_FILE}" "${LOG_PATH_MODULE}/$(basename "${lQUERY_FILE}").log"
sed -i -r "s/.*io\.(p)?open.*/\x1b[32m&\x1b[0m/" "${LOG_PATH_MODULE}/$(basename "${lQUERY_FILE}").log"
copy_and_link_file "${lQUERY_FILE}" "${lLOG_QUERYFILE}"
sed -i -r "s/.*io\.(p)?open.*/\x1b[32m&\x1b[0m/" "${lLOG_QUERYFILE}"
lISSUES_FILE=$((lISSUES_FILE + 1))
lAI_PRIO=$((lAI_PRIO + 1))
fi
if [[ "${lISSUES_FILE}" -gt 0 ]]; then
write_csv_log "$(print_path "${lQUERY_FILE}")" "0" "${lISSUES_FILE}" "NA"
if [[ "${GPT_OPTION}" -gt 0 ]]; then
lGPT_ANCHOR_="$(openssl rand -hex 8)"
# "${GPT_INPUT_FILE_}" "${lGPT_ANCHOR_}" "${lGPT_PRIO_}" "${GPT_QUESTION_}" "${GPT_OUTPUT_FILE_}" "cost=$GPT_TOKENS_" "${GPT_RESPONSE_}"
write_csv_gpt_tmp "$(cut_path "${lQUERY_FILE}")" "${lGPT_ANCHOR_}" "${lGPT_PRIO_}" "${GPT_QUESTION}" "${S23_CSV_LOG}" "" ""
if [[ "${AI_OPTION}" -gt 0 ]]; then
lAI_ANCHOR="$(openssl rand -hex 8)"
# "${GPT_INPUT_FILE_}" "${lAI_ANCHOR}" "${lAI_PRIO}" "${GPT_QUESTION_}" "${GPT_OUTPUT_FILE_}" "cost=$GPT_TOKENS_" "${GPT_RESPONSE_}"
write_csv_AI_tmp "${lLOG_QUERYFILE}" "${lAI_ANCHOR}" "${lAI_PRIO}" "${GPT_QUESTION}" "${lLOG_QUERYFILE}" "" ""
# add ChatGPT link
print_ln
print_ln
write_anchor_gpt "${lGPT_ANCHOR_}"
print_output "" "${lLOG_QUERYFILE}"
write_anchor_AI "${lAI_ANCHOR}" "${lLOG_QUERYFILE}"
fi
fi
done
@@ -173,7 +171,7 @@ s23_luacheck() {
local lLUA_ISSUES=""
lSCRIPT_NAME=$(basename "${lLUA_SCRIPT_}" 2>/dev/null | sed -e 's/:/_/g')
lLUA_LOG="${LOG_PATH_MODULE}""/luacheck_""${lSCRIPT_NAME}"".txt"
lLUA_LOG="${LOG_PATH_MODULE}/luacheck_${lSCRIPT_NAME}.txt"
luacheck "${lLUA_SCRIPT_}" >"${lLUA_LOG}" 2>/dev/null || true
lLUA_ISSUES=$(strip_color_codes "$(grep Total "${lLUA_LOG}" | awk '{print $2}' 2>/dev/null || true)")
@@ -182,17 +180,17 @@ s23_luacheck() {
local lCOMMON_FILES_FOUND=""
local lCFF=""
if [[ -f "${BASE_LINUX_FILES}" ]]; then
lCOMMON_FILES_FOUND="(""${RED}""common linux file: no""${GREEN}"")"
lCOMMON_FILES_FOUND="(${RED}common linux file: no${GREEN})"
lCFF="no"
if grep -q "^${lSCRIPT_NAME}\$" "${BASE_LINUX_FILES}" 2>/dev/null; then
lCOMMON_FILES_FOUND="(""${CYAN}""common linux file: yes""${GREEN}"")"
lCOMMON_FILES_FOUND="(${CYAN}common linux file: yes${GREEN})"
lCFF="yes"
fi
else
lCOMMON_FILES_FOUND=""
lCFF="NA"
fi
print_output "[+] Found ""${ORANGE}""${lLUA_ISSUES}"" coding issues""${GREEN}"" in lua script ""${lCOMMON_FILES_FOUND}"":""${NC}"" ""$(print_path "${lLUA_SCRIPT_}")" "" "${lLUA_LOG}"
print_output "[+] Found ${ORANGE}${lLUA_ISSUES} coding issues${GREEN} in lua script ${lCOMMON_FILES_FOUND}:${NC} $(print_path "${lLUA_SCRIPT_}")" "" "${lLUA_LOG}"
write_csv_log "$(print_path "${lLUA_SCRIPT_}")" "${lLUA_ISSUES}" "0" "${lCFF}"
fi
}
modules/S27_perl_check.sh
+32 -38
View File
@@ -38,7 +38,6 @@ S27_perl_check() {
export PERL5LIB="${EXT_DIR}"/zarn/lib
write_csv_log "Script path" "Perl issues detected" "common linux file" "vuln title" "vuln line nr" "vuln note"
# mapfile -t lPERL_SCRIPTS_ARR < <(find "${FIRMWARE_PATH}" -xdev -type f \( -name "*.pl" -o -name "*.pm" -o -name "*.cgi" \) -print0|xargs -r -0 -P 16 -I % sh -c 'md5sum "%" 2>/dev/null' | sort -u -k1,1 | cut -d\ -f3 )
mapfile -t lPERL_SCRIPTS_ARR < <(grep "Perl script.*executable" "${P99_CSV_LOG}" | sort -u || true)
for lPL_SCRIPT in "${lPERL_SCRIPTS_ARR[@]}"; do
if [[ -f "${BASE_LINUX_FILES}" && "${FULL_TEST}" -eq 0 ]]; then
@@ -50,22 +49,16 @@ S27_perl_check() {
fi
fi
((lS27_PL_SCRIPTS += 1))
if [[ "${THREADED}" -eq 1 ]]; then
s27_zarn_perl_checks "$(echo "${lPL_SCRIPT}" | cut -d';' -f2)" &
local lTMP_PID="$!"
store_kill_pids "${lTMP_PID}"
lWAIT_PIDS_S27+=("${lTMP_PID}")
max_pids_protection "${MAX_MOD_THREADS}" lWAIT_PIDS_S27
continue
else
s27_zarn_perl_checks "$(echo "${lPL_SCRIPT}" | cut -d';' -f2)"
fi
s27_zarn_perl_checks "$(echo "${lPL_SCRIPT}" | cut -d';' -f2)" &
local lTMP_PID="$!"
lWAIT_PIDS_S27+=("${lTMP_PID}")
max_pids_protection "${MAX_MOD_THREADS}" lWAIT_PIDS_S27
done
[[ "${THREADED}" -eq 1 ]] && wait_for_pid "${lWAIT_PIDS_S27[@]}"
[[ -d "${TMP_DIR}"/s27 ]] && (rm -r "${TMP_DIR}"/s27 || true)
[[ -d "${TMP_DIR}/s27" ]] && (rm -r "${TMP_DIR}"/s27 || true)
if [[ -f "${TMP_DIR}"/S27_VULNS.tmp ]]; then
if [[ -f "${TMP_DIR}/S27_VULNS.tmp" ]]; then
lS27_PL_VULNS=$(awk '{sum += $1 } END { print sum }' "${TMP_DIR}"/S27_VULNS.tmp)
fi
@@ -80,7 +73,7 @@ s27_zarn_perl_checks() {
local lPL_LOG="" # text log with original output from zarn analysis
local lVULNS=""
local lGPT_PRIO=2
local lGPT_ANCHOR=""
local lAI_ANCHOR=""
local lZARN_SARIF_RESULTS_ARR=()
local lSARIF_RESULT=""
local lZARN_VULN_TITLE=""
@@ -89,54 +82,53 @@ s27_zarn_perl_checks() {
local lCODE_LINE=""
local lPL_LOG_LINKED="" # 2nd stage log which is linked from the main log
local lS27_SOURCE_DIR="${LOG_PATH_MODULE}/pl_sources"
lNAME=$(basename "${lPL_SCRIPT}" 2>/dev/null | sed -e 's/:/_/g')
print_output "[*] Testing perl script ${ORANGE}${lPL_SCRIPT}${NC}" "no_log"
lPL_LOG="${LOG_PATH_MODULE}""/zarn_""${lNAME}"".txt"
lPL_LOG="${LOG_PATH_MODULE}/zarn_${lNAME}.txt"
if [[ "${lPL_SCRIPT: -4}" == ".cgi" ]]; then
if ! [[ -d "${TMP_DIR}"/s27 ]]; then
mkdir "${TMP_DIR}"/s27
fi
# as zarn does not like cgi's, we need to temp rename these files now
cp "${lPL_SCRIPT}" "${TMP_DIR}"/s27/"${lNAME/.cgi/.pl}"
if [[ -f "${TMP_DIR}"/s27/"${lNAME/.cgi/.pl}" ]]; then
perl "${EXT_DIR}"/zarn/zarn.pl -r "${EXT_DIR}"/zarn/rules/default.yml --source "${TMP_DIR}"/s27/"${lNAME/.cgi/.pl}" --sarif "${LOG_PATH_MODULE}""/zarn_""${lNAME}"".sarif" | tee -a "${lPL_LOG}" || print_error "[-] Analysis of ${lNAME} failed"
rm "${TMP_DIR}"/s27/"${lNAME/.cgi/.pl}" || true
if [[ -f "${TMP_DIR}/s27/${lNAME/.cgi/.pl}" ]]; then
perl "${EXT_DIR}"/zarn/zarn.pl -r "${EXT_DIR}"/zarn/rules/default.yml --source "${TMP_DIR}/s27/${lNAME/.cgi/.pl}" --sarif "${LOG_PATH_MODULE}/zarn_${lNAME}.sarif" | tee -a "${lPL_LOG}" || print_error "[-] Analysis of ${lNAME} failed"
rm "${TMP_DIR}/s27/${lNAME/.cgi/.pl}" || true
fi
else
perl "${EXT_DIR}"/zarn/zarn.pl -r "${EXT_DIR}"/zarn/rules/default.yml --source "${lPL_SCRIPT}" --sarif "${LOG_PATH_MODULE}""/zarn_""${lNAME}"".sarif" | tee -a "${lPL_LOG}" || print_error "[-] Analysis of ${lNAME} failed"
perl "${EXT_DIR}"/zarn/zarn.pl -r "${EXT_DIR}"/zarn/rules/default.yml --source "${lPL_SCRIPT}" --sarif "${LOG_PATH_MODULE}/zarn_${lNAME}.sarif" | tee -a "${lPL_LOG}" || print_error "[-] Analysis of ${lNAME} failed"
fi
## reporting starts here
lVULNS=$(grep -c "\[vuln\]" "${lPL_LOG}" 2>/dev/null || true)
if [[ "${lVULNS}" -gt 0 ]]; then
[[ ! -d "${lS27_SOURCE_DIR}" ]] && mkdir -p "${lS27_SOURCE_DIR}"
# check if this is common linux file:
local lCOMMON_FILES_FOUND=""
local lCFF=""
if [[ -f "${BASE_LINUX_FILES}" ]]; then
lCOMMON_FILES_FOUND="(""${RED}""common linux file: no""${GREEN}"")"
lCOMMON_FILES_FOUND="(${RED}common linux file: no${GREEN})"
lCFF="no"
if grep -q "^${lNAME}\$" "${BASE_LINUX_FILES}" 2>/dev/null; then
lCOMMON_FILES_FOUND="(""${CYAN}""common linux file: yes""${GREEN}"")"
lCOMMON_FILES_FOUND="(${CYAN}common linux file: yes${GREEN})"
lCFF="yes"
fi
else
lCOMMON_FILES_FOUND=""
lCFF="NA"
fi
lPL_LOG_LINKED="${LOG_PATH_MODULE}""/zarn_""${lNAME}""_details.txt"
print_output "[+] Found ""${RED}""${lVULNS}"" possible issue(s)""${GREEN}"" in perl script ""${lCOMMON_FILES_FOUND}"":""${ORANGE}"" ""$(print_path "${lPL_SCRIPT}")" "" "${lPL_LOG_LINKED}"
lPL_LOG_LINKED="${LOG_PATH_MODULE}/zarn_${lNAME}_details.txt"
print_output "[+] Found ${RED}${lVULNS} possible issue(s)${GREEN} in perl script ${lCOMMON_FILES_FOUND}:${ORANGE} $(print_path "${lPL_SCRIPT}")" "" "${lPL_LOG_LINKED}"
if [[ "${lVULNS}" -gt 20 ]]; then
lGPT_PRIO=$((lGPT_PRIO + 1))
fi
mapfile -t lZARN_SARIF_RESULTS_ARR < <(jq -rc '.runs[].results[]' "${LOG_PATH_MODULE}""/zarn_""${lNAME}"".sarif")
mapfile -t lZARN_SARIF_RESULTS_ARR < <(jq -rc '.runs[].results[]' "${LOG_PATH_MODULE}/zarn_${lNAME}.sarif")
if ! [[ -d "${LOG_PATH_MODULE}"/pl_source_files/ ]]; then
mkdir "${LOG_PATH_MODULE}"/pl_source_files/ || true
fi
write_log "[+] Found ""${ORANGE}""${lVULNS}"" possible issue(s)""${GREEN}"" in perl script ""${lCOMMON_FILES_FOUND}"":""${ORANGE}"" ""$(print_path "${lPL_SCRIPT}")" "${lPL_LOG_LINKED}"
write_link "${LOG_PATH_MODULE}"/pl_source_files/"${lNAME}".txt "${lPL_LOG_LINKED}"
write_log "[+] Found ${ORANGE}${lVULNS} possible issue(s)${GREEN} in perl script ${lCOMMON_FILES_FOUND}:${ORANGE} $(print_path "${lPL_SCRIPT}")" "${lPL_LOG_LINKED}"
copy_and_link_file "${lPL_SCRIPT}" "${lS27_SOURCE_DIR}/${lNAME}.log" "${lPL_LOG_LINKED}"
write_log "$(indent "$(orange "$(file "${lPL_SCRIPT}")")")" "${lPL_LOG_LINKED}"
write_log "\n" "${lPL_LOG_LINKED}"
@@ -151,19 +143,19 @@ s27_zarn_perl_checks() {
write_log "" "${lPL_LOG_LINKED}"
# we need the original perl script to link to it and to highlight the affected lines of code
if ! [[ -f "${LOG_PATH_MODULE}"/pl_source_files/"${lNAME}".txt ]]; then
cp "${lPL_SCRIPT}" "${LOG_PATH_MODULE}"/pl_source_files/"${lNAME}".txt || print_error "[-] Copy of ${lPL_SCRIPT} to log directory failed"
if ! [[ -f "${lS27_SOURCE_DIR}/${lNAME}.log" ]]; then
cp "${lPL_SCRIPT}" "${lS27_SOURCE_DIR}/${lNAME}.log" || print_error "[-] Copy of ${lPL_SCRIPT} to log directory failed"
fi
lCODE_LINE="$(strip_color_codes "$(sed -n "${lZARN_VULN_LINE}"p "${LOG_PATH_MODULE}/pl_source_files/${lNAME}.txt" 2>/dev/null)")"
sed -i -r "${lZARN_VULN_LINE}s/.*/\x1b[32m&\x1b[0m/" "${LOG_PATH_MODULE}/pl_source_files/${lNAME}.txt" || true
lCODE_LINE="$(strip_color_codes "$(sed -n "${lZARN_VULN_LINE}"p "${lS27_SOURCE_DIR}/${lNAME}.log" 2>/dev/null)")"
sed -i -r "${lZARN_VULN_LINE}s/.*/\x1b[32m&\x1b[0m/" "${lS27_SOURCE_DIR}/${lNAME}.log" || true
write_log "$(indent "$(indent "${GREEN}${lZARN_VULN_LINE}${NC} - ${ORANGE}${lCODE_LINE}${NC}")")" "${lPL_LOG_LINKED}"
write_log "\\n-----------------------------------------------------------------\\n" "${lPL_LOG_LINKED}"
done
# writing gpt details:
if [[ "${GPT_OPTION}" -gt 0 ]]; then
lGPT_ANCHOR="$(openssl rand -hex 8)"
if [[ "${AI_OPTION}" -gt 0 ]]; then
lAI_ANCHOR="$(openssl rand -hex 8)"
if [[ -f "${BASE_LINUX_FILES}" ]]; then
# if we have the base linux config file we are checking it:
if ! grep -E -q "^$(basename "${lPL_SCRIPT}")$" "${BASE_LINUX_FILES}" 2>/dev/null; then
@@ -171,10 +163,12 @@ s27_zarn_perl_checks() {
fi
fi
# "${GPT_INPUT_FILE_}" "${GPT_ANCHOR_}" "${GPT_PRIO_}" "${GPT_QUESTION_}" "${GPT_OUTPUT_FILE_}" "cost=$GPT_TOKENS_" "${GPT_RESPONSE_}"
write_csv_gpt_tmp "$(cut_path "${lPL_SCRIPT}")" "${lGPT_ANCHOR}" "${lGPT_PRIO}" "${GPT_QUESTION}" "${lPL_LOG}" "" ""
write_csv_AI_tmp "${lS27_SOURCE_DIR}/${lNAME}.log" "${lAI_ANCHOR}" "${lGPT_PRIO}" "${GPT_QUESTION}" "${lPL_LOG}" "" ""
# add ChatGPT link to output file
printf '%s\n\n' "" >>"${lPL_LOG}"
write_anchor_gpt "${lGPT_ANCHOR}" "${lPL_LOG}"
write_anchor_AI "${lAI_ANCHOR}" "${lPL_LOG}"
printf '%s\n\n' "" >>"${lS27_SOURCE_DIR}/${lNAME}.log"
write_anchor_AI "${lAI_ANCHOR}" "${lS27_SOURCE_DIR}/${lNAME}.log"
fi
echo "${lVULNS}" >>"${TMP_DIR}"/S27_VULNS.tmp
fi
modules/S28_java_check.sh
+40 -14
View File
@@ -27,8 +27,9 @@ S28_java_check() {
local lJAVA_BINS_ARR=()
local lJAVA_BINARY=""
local lJNAME=""
local lJAVA_VULNS=0
local lJAVA_VULNS_CNT=0
local lWAIT_PIDS_S28=()
local MAX_MOD_THREADS=$((MAX_MOD_THREADS/2))
export JAVA_DECOMPILER="${EXT_DIR}/vineflower-1.11.2.jar"
@@ -70,21 +71,45 @@ S28_java_check() {
wait_for_pid "${lWAIT_PIDS_S28[@]}"
local lSEMGREP_RESULTS_ARR=()
local lSEMGREP_RESULT=""
mapfile -t lSEMGREP_RESULTS_ARR < <(find "${LOG_PATH_MODULE}/java_semgrep/" -name "semgrep_*.json")
for lSEMGREP_RESULT in "${lSEMGREP_RESULTS_ARR[@]}"; do
if grep -q '\"results\":\[{\"' "${lSEMGREP_RESULT}"; then
print_output "[+] Semgrep security scanning results for $(basename "${lSEMGREP_RESULT}")" "" "${lSEMGREP_RESULT}"
# todo: count vulns to lJAVA_VULNS
fi
done
if [[ -d "${LOG_PATH_MODULE}/java_semgrep/" ]]; then
local lSEMGREP_RESULTS_ARR=()
local lSEMGREP_RESULT=""
mapfile -t lSEMGREP_RESULTS_ARR < <(find "${LOG_PATH_MODULE}/java_semgrep/" -name "semgrep_*.json")
for lSEMGREP_RESULT in "${lSEMGREP_RESULTS_ARR[@]}"; do
if grep -q '\"results\":' "${lSEMGREP_RESULT}"; then
local lJAVA_SOURCE_FILE_ARR=()
mapfile -t lJAVA_SOURCE_FILE_ARR < <(jq -r '.results[].path' "${lSEMGREP_RESULT}" | sort -u)
lJAVA_VULNS_CNT=$((lJAVA_VULNS_CNT+$(jq -r '.results[].check_id' "${lSEMGREP_RESULT}" | wc -l)))
local lJAVA_SOURCE_FILE=""
for lJAVA_SOURCE_FILE in "${lJAVA_SOURCE_FILE_ARR[@]}"; do
print_output "[+] Semgrep security scanning results for ${ORANGE}$(basename "${lJAVA_SOURCE_FILE}")${NC}" "" "${lSEMGREP_RESULT}"
# we log ".log" for the web reporter
mv "${lJAVA_SOURCE_FILE}" "${lJAVA_SOURCE_FILE}.log" || true
print_output "$(indent "$(orange "Decompiled Java sources for ${ORANGE}$(basename "${lJAVA_SOURCE_FILE}")${NC}")")" "" "${lJAVA_SOURCE_FILE}.log"
# AI integration
local lAI_ANCHOR=""
lAI_ANCHOR="$(openssl rand -hex 8)"
lAI_PRIO=4
write_csv_AI_tmp "${lJAVA_SOURCE_FILE}.log" "${lAI_ANCHOR}" "${lAI_PRIO}" "NA" "${lSEMGREP_RESULT}" "" ""
printf '%s\n\n' "" >>"${lJAVA_SOURCE_FILE}.log"
write_anchor_AI "${lAI_ANCHOR}" "${lJAVA_SOURCE_FILE}.log"
done
# Todo: color the affected lines of code
fi
done
fi
local lJAVA_DECOMPILED=0
lJAVA_DECOMPILED=$(find "${LOG_PATH_MODULE}/java_decompile/" -type f | wc -l)
if [[ -d "${LOG_PATH_MODULE}/java_decompile/" ]]; then
lJAVA_DECOMPILED=$(find "${LOG_PATH_MODULE}/java_decompile/" -type f | wc -l)
fi
print_ln ""
print_output "[+] Found ${ORANGE}${lJAVA_VULNS_CNT} vulnerabilities${GREEN} in ${ORANGE}${lJAVA_DECOMPILED}${GREEN} decompiled Java binaries.${NC}"
write_log ""
write_log "[*] Statistics:${#lJAVA_BINS_ARR[@]}:${lJAVA_DECOMPILED}:${lJAVA_VULNS}"
write_log "[*] Statistics:${#lJAVA_BINS_ARR[@]}:${lJAVA_DECOMPILED}:${lJAVA_VULNS_CNT}"
module_end_log "${FUNCNAME[0]}" "${lJAVA_DECOMPILED}"
}
@@ -98,7 +123,7 @@ s28_java_semgrep() {
local lJ_ANALYSE_RESULTS="${lJ_ANALYSE_DIR}/semgrep_${lJNAME}.json"
semgrep --disable-version-check --metrics=off --severity ERROR --severity WARNING --json --config "${EXT_DIR}"/semgrep-rules/java "${lJAVA_BIN_DIR}" >"${lJ_ANALYSE_RESULTS}" || true
if [[ -f "${lJ_ANALYSE_RESULTS}" ]] && [[ "$(grep -c '\"results\":\[{\"' "${lJ_ANALYSE_RESULTS}")" -eq 0 ]]; then
if [[ -f "${lJ_ANALYSE_RESULTS}" ]] && [[ "$(grep -c '\"results\":' "${lJ_ANALYSE_RESULTS}")" -eq 0 ]]; then
rm "${lJ_ANALYSE_RESULTS}" || true
fi
}
@@ -108,8 +133,9 @@ s28_java_decompile() {
local lJNAME=""
lJNAME=$(basename "${lJAVA_BINARY}" 2>/dev/null | sed -e 's/:/_/g')
print_output "[*] Decompiling Java binary ${ORANGE}${lJNAME}${NC}" "no_log"
lJNAME=$(iconv -f UTF-8 -t ASCII//TRANSLIT <<<"${lJNAME}" || true)
local lJ_DECOMPILE_DIR="${LOG_PATH_MODULE}/java_decompile/${lJNAME}_${RANDOM}"
print_output "[*] Decompiling Java binary ${ORANGE}${lJNAME}${NC} to ${lJ_DECOMPILE_DIR}" "no_log"
[[ ! -d "${lJ_DECOMPILE_DIR}" ]] && mkdir -p "${lJ_DECOMPILE_DIR}"
java -jar "${JAVA_DECOMPILER}" --silent "${lJAVA_BINARY}" "${lJ_DECOMPILE_DIR}" || true
modules/template_module.sh
+9 -9
View File
@@ -35,21 +35,21 @@
=head1 MODULE_NAME
==head2 MODULE_NAME Short description
=head2 MODULE_NAME Short description
Please write a short description of your module. Usually ~2-3 sentences are fine to get an idea.
==head2 MODULE_NAME Detailed description
=head2 MODULE_NAME Detailed description
Please write a longer description of your module. This should also include further references and links
that were used.
==head2 MODULE_NAME 3rd party tools
=head2 MODULE_NAME 3rd party tools
Any 3rd party tool that is needed from your module. Also include the tested and known working version and
download link.
==head2 MODULE_NAME Testfirmware
=head2 MODULE_NAME Testfirmware
For verification of the module we need some testfirmware.
@@ -59,26 +59,26 @@ Testfirmware details:
- Checksum (MD5/SHA1/SHA256):
- Download Link:
==head2 MODULE_NAME Output
=head2 MODULE_NAME Output
Example output of module
==head2 MODULE_NAME License
=head2 MODULE_NAME License
EMBA module MODULE_NAME is licensed under GPLv3
SPDX-License-Identifier: GPL-3.0-only
Link to license document: https://github.com/e-m-b-a/emba/blob/master/LICENSE
Note: Only GPL-3.0 will be accepted in the master EMBA repository
==head2 MODULE_NAME Todo
=head2 MODULE_NAME Todo
Missing stuff that we need to consider.
==head2 MODULE_NAME Known issues
=head2 MODULE_NAME Known issues
Any known issues or known limitations.
==head2 MODULE_NAME Author(s)
=head2 MODULE_NAME Author(s)
Michael Messner, Pascal Eckmann
Note: List all authors including contributors to this module
scan-profiles/default-scan-gpt.emba → scan-profiles/default-scan-AI.emba
+7 -6
View File
@@ -1,6 +1,6 @@
# EMBA - EMBEDDED LINUX ANALYZER
#
# Copyright 2020-2025 Siemens Energy AG
# Copyright 2020-2026 Siemens Energy AG
#
# EMBA comes with ABSOLUTELY NO WARRANTY. This is free software, and you are
# welcome to redistribute it under the terms of the GNU General Public License.
@@ -11,7 +11,7 @@
# Author(s): Benedikt Kuehne
# Contributor(s): Michael Messner
#
# Description: Profile for ChatGPT
# Description: Profile for AI usage
export FORMAT_LOG=1
export THREADED=1
@@ -20,7 +20,8 @@ export HTML=1
# QEMULATION - for user-mode emulation to generate an optimised SBOM
export QEMULATION=1
export GPT_OPTION=1
# 0 -> off, 1-> unpaid GPT plan, 2 -> no GPT rate-limit, 3 -> LocalAI Server
export AI_OPTION=3
# extended binary tests are now only testing non-linux binaries. With this mechanism we can enable
# it in the default profile.
@@ -33,11 +34,11 @@ export VEX_METRICS=1
export MODULE_BLACKLIST+=( "S10_binaries_basic_check" "S15_radare_decompile_checks" "S26_kernel_vuln_verifier" "S99_grepit" "S110_yara_check" "F20_vul_aggregator" )
# enable silent mode and status bar
export DISABLE_STATUS_BAR=0
export SILENT=1
export DISABLE_STATUS_BAR=1
export SILENT=0
# enable EMBA binary map dependency generator
EMBA_MAP_GENERATOR=1
EMBA_MAP_GENERATOR=0
# we output the profile only at the beginning - outside the docker environment
if [[ $IN_DOCKER -ne 1 ]] ; then
scan-profiles/default-scan.emba
+2 -2
View File
@@ -41,8 +41,8 @@ export MODULE_BLACKLIST+=( "S10_binaries_basic_check" "S15_radare_decompile_chec
export DISABLE_STATUS_BAR=0
export SILENT=1
# enable GPT connection:
export GPT_OPTION=0
# enable AI connection:
export AI_OPTION=0
# enable EMBA binary map dependency generator
EMBA_MAP_GENERATOR=1