diff --git a/config/ai_config.env.template b/config/ai_config.env.template new file mode 100644 index 00000000..7476e667 --- /dev/null +++ b/config/ai_config.env.template @@ -0,0 +1,8 @@ +LOCAL_AI_IP="192.168.111.1" +# EMBA checks if this model is available for chatting +# if this model is not available EMBA can't proceed +LOCAL_AI_MODEL="Qwen2.5-Coder-7B-Instruct-GGUF" +# if EMBA will finish earlier with the testing phase she will wait for Q03 until AI requests are done or the timeperiod is over +AI_MIN_RUNTIME="12h" +# maximal code size for AI analysis +AI_MAX_CHARS_TO_ANALYSE=5000 diff --git a/config/gpt_config.env.template b/config/gpt_config.env.template deleted file mode 100644 index 62146bf9..00000000 --- a/config/gpt_config.env.template +++ /dev/null @@ -1,3 +0,0 @@ -# OPENAI_API_KEY="" -# GPT_QUESTION="For the following code I need you to tell me how an attacker could exploit it and point out all vulnerabilities." -# MAXMIUM_GPT_PRIO=3 diff --git a/helpers/helpers_emba_defaults.sh b/helpers/helpers_emba_defaults.sh index 8fe871cc..44a602ef 100755 --- a/helpers/helpers_emba_defaults.sh +++ b/helpers/helpers_emba_defaults.sh @@ -91,7 +91,7 @@ set_defaults() { export SHELLCHECK=1 - export GPT_OPTION=0 # 0 -> off 1-> unpaid plan 2 -> no rate-limit + export AI_OPTION=0 # 0 -> off, 1-> unpaid plan, 2 -> no rate-limit, 3 -> LocalAI Server export GPT_QUESTION="For the following code I need you to tell me how an attacker could exploit it and point out all vulnerabilities:" export MINIMUM_GPT_PRIO=1 # everything above this value gets checked @@ -175,6 +175,9 @@ set_defaults() { export SVG_BUILD_TIMEOUT="12h" export SVG_FILE="${LOG_PATH_MODULE}/EMBA-dependency-map.svg" + DEPENDENCY_TRACK_ENABLED=0 + OPENAI_API_KEY="" + # we can enable/disable the s08 submodules with the following array configuration # -> just comment the submodule that should not be used # usually this should be done via a scan-profile @@ -209,6 +212,7 @@ set_log_paths() { export P02_LOG="${LOG_DIR}/p02_firmware_bin_file_check.txt" export P02_LOG_DIR="${P02_LOG/\.txt/\/}" export P02_CSV_LOG="${CSV_DIR}/p02_firmware_bin_file_check.csv" + export P99_LOG="${LOG_DIR}/p99_prepare_analyzer.txt" export P99_CSV_LOG="${CSV_DIR}/p99_prepare_analyzer.csv" export P55_LOG="${LOG_DIR}/p55_unblob_extractor.txt" export P60_LOG="${LOG_DIR}/p60_deep_extractor.txt" @@ -270,6 +274,7 @@ set_log_paths() { export S118_LOG="${LOG_DIR}/s118_busybox_verifier.txt" export S118_LOG_DIR="${S118_LOG/\.txt/\/}" export Q02_LOG="${LOG_DIR}/q02_openai_question.txt" + export Q03_LOG="${LOG_DIR}/q03_localai_connector.txt" export L10_LOG="${LOG_DIR}/l10_system_emulator.txt" export L10_SYS_EMU_RESULTS="${LOG_DIR}/emulator_online_results.log" export L15_LOG="${LOG_DIR}/l15_emulated_checks_init.txt" diff --git a/helpers/helpers_emba_dependency_check.sh b/helpers/helpers_emba_dependency_check.sh index c634d919..be479a7c 100755 --- a/helpers/helpers_emba_dependency_check.sh +++ b/helpers/helpers_emba_dependency_check.sh @@ -297,7 +297,7 @@ dependency_check() { check_nvd_db "${lNVD_GITHUB_HASH}" fi else - echo -e "${RED}""not ok""${NC}" + echo -e "${RED}not ok${NC}" print_output "[!] Warning: EMBA has NO internet connection!" "no_log" print_output "[!] Warning: Update checks and multiple EMBA modules are disabled!" "no_log" print_output "[!] Warning: GPT (Q02), kernel verification (S26) and further online modules are disabled!" "no_log" @@ -306,14 +306,14 @@ dependency_check() { # running into this in Quest container and on host, but not on isolated EMBA container (as it is CONTAINER_NUMBER 1): if [[ "${CONTAINER_NUMBER}" -ne 1 ]]; then - if [[ -f "${CONFIG_DIR}/gpt_config.env" ]]; then - if grep -v -q "#" "${CONFIG_DIR}/gpt_config.env"; then - # readin gpt_config.env + if [[ -f "${CONFIG_DIR}/ai_config.env" ]]; then + if grep -v -q "#" "${CONFIG_DIR}/ai_config.env"; then + # readin ai_config.env while read -r LINE; do if [[ "${LINE}" == *'='* ]] && [[ "${LINE}" != '#'* ]]; then export "$(echo "${LINE}" | xargs)" fi - done <"${CONFIG_DIR}/gpt_config.env" + done <"${CONFIG_DIR}/ai_config.env" fi fi @@ -344,7 +344,7 @@ dependency_check() { print_output "$(indent "ChatGPT-API key not set - ${ORANGE}see https://github.com/e-m-b-a/emba/wiki/AI-supported-firmware-analysis for more information${NC}")" "no_log" # The following if clause is currently not working! We have not loaded the profile in this stage # TODO: Find a workaround! - if [[ "${GPT_OPTION}" -eq 1 ]]; then + if [[ "${AI_OPTION}" -eq 1 ]]; then DEP_ERROR=1 fi else diff --git a/helpers/helpers_emba_helpers.sh b/helpers/helpers_emba_helpers.sh index 5f9da37c..70d47bd6 100755 --- a/helpers/helpers_emba_helpers.sh +++ b/helpers/helpers_emba_helpers.sh @@ -42,6 +42,43 @@ run_web_reporter_mod_name() { fi } +# simple markdown to EMBA log parser +# Parameter 1: markdown file +# Parameter 2: EMBA log file to create and log to +# Parameter 3: How many # symbols are used for the first headline -> module_title +parse_markdown_to_emba_txt() { + local lMD_LOG="${1:-}" + local lEMBA_TXT_LOG="${2:-}" + # how many # signs are used for the first headline + local lHEADLINE_STARTER="${3:-1}" + [[ -z "${lEMBA_TXT_LOG}" ]] && lEMBA_TXT_LOG="${lMD_LOG//\.md/\.txt}" + [[ ! -f "${lEMBA_TXT_LOG}" ]] && touch "${lEMBA_TXT_LOG}" + + local lMD_LINE="" + local lSTORED_IFS="${IFS}" + + IFS='' + while read -r lMD_LINE; do + if [[ "${lMD_LINE}" =~ ^#{$((lHEADLINE_STARTER+1))}.* ]]; then + # other headlines -> sub-module-titles + # remove the # signs + lMD_LINE="${lMD_LINE//#}" + lMD_LINE="${lMD_LINE#\ }" + sub_module_title "${lMD_LINE}" "${lEMBA_TXT_LOG}" + elif [[ "${lMD_LINE}" =~ ^#{${lHEADLINE_STARTER}}\ .* ]]; then + # main headline + # remove the # signs + lMD_LINE="${lMD_LINE:${lHEADLINE_STARTER}}" + lMD_LINE="${lMD_LINE#\ }" + module_title "${lMD_LINE}" "${lEMBA_TXT_LOG}" + else + lMD_LINE=${lMD_LINE//\*/} + write_log "${lMD_LINE}" "${lEMBA_TXT_LOG}" + fi + done < "${lMD_LOG}" + IFS="${lSTORED_IFS}" +} + wait_for_pid() { local lWAIT_PIDS_ARR=("$@") local lPID="" @@ -54,7 +91,7 @@ wait_for_pid() { continue fi while [[ -e /proc/"${lPID}" ]]; do - print_debug "[*] wait pid protection - running pid: ${lPID}" "no_log" + # print_debug "[*] wait pid protection - running pid: ${lPID}" "no_log" print_dot # if S115 is running we have to kill old qemu processes if [[ -f "${LOG_DIR}/${MAIN_LOG_FILE}" ]] && [[ $(grep -i -c S115_ "${LOG_DIR}/${MAIN_LOG_FILE}") -gt 0 && -v QRUNTIME ]]; then diff --git a/helpers/helpers_emba_html_generator.sh b/helpers/helpers_emba_html_generator.sh index 37a92f6a..c4a23f86 100755 --- a/helpers/helpers_emba_html_generator.sh +++ b/helpers/helpers_emba_html_generator.sh @@ -113,18 +113,24 @@ add_link_tags() { lREF_ANCHOR="$(echo "${lREF_LINK}" | cut -d"#" -f2 || true)" lREF_LINK="$(echo "${lREF_LINK}" | cut -d"#" -f1 || true)" fi - print_debug "[*] Generate info_file - REF link ${lREF_LINK} / BACK_LINK ${lBACK_LINK}" "no_log" - # in some cases we link to files from different modules. Now we check if the html file is already + # in some cases we link to files from different modules. Now we check if the html report file is already # generated and we can directly use the already available html report file - lORIG_SRC_MODULE_DIR=$(dirname "${lREF_LINK}") - lORIG_SRC_MODULE_DIR=${lORIG_SRC_MODULE_DIR//*\//} + lORIG_SRC_MODULE_DIR=$(echo "${lREF_LINK}" | tr '/' '\n' | grep -E "^[psqfl][0-9]{2,3}_.*") + print_debug "[*] REF link ${lREF_LINK} / BACK_LINK ${lBACK_LINK} / lORIG_SRC_MODULE_DIR: ${lORIG_SRC_MODULE_DIR} / lLINK_FILE: ${lLINK_FILE}" "no_log" if [[ -f "${ABS_HTML_PATH%/}/${lORIG_SRC_MODULE_DIR}/${lMD5_REF_LINK}" ]]; then print_debug "[*] Found already generated log file in ${ABS_HTML_PATH%/}/${lORIG_SRC_MODULE_DIR}/${lMD5_REF_LINK}" "no_log" - # link to the already available report file without generating a new one: - lBACK_LINK_NEW="./${lORIG_SRC_MODULE_DIR}/${lMD5_REF_LINK}" + if [[ "$(basename ${lLINK_FILE})" =~ ^(d|p|l|s|q|f){1}[0-9]{2,3}_.*$ ]]; then + # link to the already available report file without generating a new one: + # we need to link to the original module directory of the file if we are in a main module log file + lBACK_LINK_NEW="${lORIG_SRC_MODULE_DIR}/${lMD5_REF_LINK}" + else + # if we are in our own sub-module log directory we do not need to include the module directory + lBACK_LINK_NEW="${lMD5_REF_LINK}" + fi else # generate reference file if it not already available + print_debug "[*] Generating info file lREF_LINK: ${lREF_LINK} / lBACK_LINK: ${lBACK_LINK}" generate_info_file "${lREF_LINK}" "${lBACK_LINK}" & lWAIT_PIDS_WR+=("$!") @@ -141,6 +147,7 @@ add_link_tags() { else lHTML_LINK="$(echo "${REFERENCE_LINK}" | sed -e "s@LINK@${DEPTH}/${lBACK_LINK_NEW}@g" || true)" fi + print_debug "[*] Generating ${lBACK_LINK_NEW} / lBACK_LINK_NEW: ${lBACK_LINK_NEW} / lREF_LINK: ${lREF_LINK} / DEPTH: ${DEPTH} / lHTML_LINK: ${lHTML_LINK}" lLINE_NUMBER_INFO_PREV="$((lREF_LINK_NUMBER - 1))" while [[ ("$(sed "${lLINE_NUMBER_INFO_PREV}""q;d" "${lLINK_FILE}")" == "${P_START}${SPAN_END}${P_END}") || ("$(sed "${lLINE_NUMBER_INFO_PREV}""q;d" "${lLINK_FILE}")" == "${BR}") ]]; do lLINE_NUMBER_INFO_PREV=$((lLINE_NUMBER_INFO_PREV - 1)) @@ -570,7 +577,8 @@ generate_info_file() { sed -i -e "s:^:${P_START}: ; s:$:${P_END}:" "${lTMP_INFO_FILE}" || true # add html tags for style add_color_tags "${lTMP_INFO_FILE}" - sed -i -e "s:[=]{65}:${HR_DOUBLE}:g ; s:^[-]{65}$:${HR_MONO}:g" "${lTMP_INFO_FILE}" || true + sed -i -E "s:[=]{65}:${HR_DOUBLE}:g" "${lTMP_INFO_FILE}" || true + sed -i -E "s:[-]{65}:${HR_MONO}:g" "${lTMP_INFO_FILE}" || true # add link tags to links/generate info files and link to them and write line to tmp file add_link_tags "${lTMP_INFO_FILE}" "${lINFO_HTML_FILE}" diff --git a/helpers/helpers_emba_print.sh b/helpers/helpers_emba_print.sh index 51d1ae5f..249b267b 100755 --- a/helpers/helpers_emba_print.sh +++ b/helpers/helpers_emba_print.sh @@ -447,9 +447,12 @@ write_link() { # The copy_and_link_file is used to copy files from the filesystem to the web-report and # automatically link it # This is usually used after a print_output "[*] asdf" +# 3rd parameter is optional log file to write this link into copy_and_link_file() { local lSRC_FILE="${1:-}" local lDST_FILE="${2:-}" + # optional + local lLOG_FILE="${3:-}" if ! [[ -d "$(dirname "${lDST_FILE}")" ]]; then mkdir -p "$(dirname "${lDST_FILE}")" || true @@ -460,30 +463,13 @@ copy_and_link_file() { cp "${lSRC_FILE}" "${lDST_FILE}" 2>/dev/null || true fi fi - if [[ -f "${lDST_FILE}" ]]; then + if [[ -f "${lDST_FILE}" && -n "${lLOG_FILE}" ]]; then + write_link "${lDST_FILE}" "${lLOG_FILE}" + elif [[ -f "${lDST_FILE}" ]]; then write_link "${lDST_FILE}" fi } -# we add an entry like -# [LOV] local_link -# this entry is later replaced from web reporter with the -# correct call to JS function -write_local_overlay_link() { - if [[ ${HTML} -eq 1 ]]; then - local lLINK="${1:-}" - lLINK="$(format_log "[LOV] ""${lLINK}" 1)" - local lLOG_FILE_ALT="${2:-}" - if [[ "${lLOG_FILE_ALT}" != "no_log" ]] && [[ "${lLOG_FILE_ALT}" != "main" ]]; then - if [[ -f "${lLOG_FILE_ALT}" ]]; then - echo -e "${lLINK}" | tee -a "${lLOG_FILE_ALT}" >/dev/null - else - echo -e "${lLINK}" | tee -a "${LOG_FILE}" >/dev/null - fi - fi - fi -} - write_anchor() { if [[ ${HTML} -eq 1 ]]; then local lANCHOR="${1:-}" @@ -993,32 +979,32 @@ print_notification() { # writes inputs into csv for chatgpt # Args: "${GPT_INPUT_FILE_}" "${GPT_ANCHOR_}" "${GPT_PRIO_}" "${GPT_QUESTION_}" "${GPT_OUTPUT_FILE_}" "cost=${GPT_TOKENS_}" "${GPT_RESPONSE_}" -write_csv_gpt() { +write_csv_AI() { local lCSV_ITEMS_ARR=("$@") if ! [[ -d "${CSV_DIR}" ]]; then print_output "[-] WARNING: CSV directory ${ORANGE}${CSV_DIR}${NC} not found" return fi - printf '%s;' "${lCSV_ITEMS_ARR[@]}" >>"${CSV_DIR}/q02_openai_question.csv" || true - printf '\n' >>"${CSV_DIR}/q02_openai_question.csv" || true + printf '%s;' "${lCSV_ITEMS_ARR[@]}" >>"${CSV_DIR}/ai_question.csv" || true + printf '\n' >>"${CSV_DIR}/ai_question.csv" || true } # writes inputs into tmp csv for chatgpt # Args: "${GPT_INPUT_FILE_}" "${GPT_ANCHOR_}" "${GPT_PRIO_}" "${GPT_QUESTION_}" "${GPT_OUTPUT_FILE_}" "cost=${GPT_TOKENS_}" "${GPT_RESPONSE_}" -write_csv_gpt_tmp() { +write_csv_AI_tmp() { local lCSV_ITEMS_ARR=("$@") if ! [[ -d "${CSV_DIR}" ]]; then print_output "[-] WARNING: CSV directory ${ORANGE}${CSV_DIR}${NC} not found" return fi - printf '%s;' "${lCSV_ITEMS_ARR[@]}" >>"${CSV_DIR}/q02_openai_question.csv.tmp" || true - printf '\n' >>"${CSV_DIR}/q02_openai_question.csv.tmp" || true + printf '%s;' "${lCSV_ITEMS_ARR[@]}" >>"${CSV_DIR}/ai_question.csv.tmp" || true + printf '\n' >>"${CSV_DIR}/ai_question.csv.tmp" || true } -write_anchor_gpt() { +write_anchor_AI() { if [[ ${HTML} -eq 1 ]]; then local lLINK="${1:-}" - lLINK="$(format_log "[ASK_GPT] ""${lLINK}" 1)" + lLINK="$(format_log "[ASK_AI] ""${lLINK}" 1)" local lLOG_FILE_ALT="${2:-}" if [[ "${lLOG_FILE_ALT}" != "no_log" ]] && [[ "${lLOG_FILE_ALT}" != "main" ]]; then if [[ -f "${lLOG_FILE_ALT}" ]]; then diff --git a/helpers/style.css b/helpers/style.css index 9a4a81aa..9ee7f3c7 100644 --- a/helpers/style.css +++ b/helpers/style.css @@ -81,6 +81,11 @@ a.f { padding-right: 18px !important; } +.double { + border: 0; + border-top: 5px double #9f9f9f; +} + hr.double { border: 0; border-top: 5px double #9f9f9f; diff --git a/modules/F05_qs_resolver.sh b/modules/F05_qs_resolver.sh index 858d4c4e..55cf78f3 100755 --- a/modules/F05_qs_resolver.sh +++ b/modules/F05_qs_resolver.sh @@ -18,90 +18,87 @@ F05_qs_resolver() { module_log_init "${FUNCNAME[0]}" - module_title "GPT Resolver" + module_title "AI Resolver" - if [[ "${GPT_OPTION}" -gt 0 ]]; then - # wait for completion or 1m - if grep -q "Q02_openai_question starting" "${LOG_DIR}"/"${MAIN_LOG_FILE}"; then - print_output "[*] Waiting for the GPT testing module to stop ... " "no_log" - grep -q "Q02_openai_question finished" "${LOG_DIR}"/"${MAIN_LOG_FILE}" || sleep 1m - print_output "[*] GPT testing module stopped ... " "no_log" + if [[ "${AI_OPTION}" -gt 0 ]]; then + if grep -q "Q02_openai_question starting" "${MAIN_LOG}"; then + print_output "[*] Waiting for the GPT analysis module to stop ... " "no_log" + while ! grep -q "Q02_openai_question finished\|Quest container finished" "${MAIN_LOG}"; do + print_output "[*] Waiting for Q02 module" + sleep 5 + done + print_output "[*] GPT testing module finished ... " "no_log" + fi + + if grep -q "Q03_localai_connector starting" "${MAIN_LOG}"; then + print_output "[*] Waiting for the LocalAI analysis module to stop ... " "no_log" + while ! grep -q "Q03_localai_connector finished\|Quest container finished" "${MAIN_LOG}"; do + print_output "[*] Waiting for Q03 module" + sleep 5 + done + print_output "[*] LocalAI testing module finished ... " "no_log" fi # local _GPT_INPUT_FILE_="" - local lGPT_ANCHOR_="" + local lAI_ANCHOR="" local l_GPT_PRIO_=3 local lGPT_QUESTION_="" local lGPT_RESPONSE_="" + local lAI_RESPONSE_FILE="" local lGPT_TOKENS_=0 local lGPT_OUTPUT_FILE_="" local lGPT_OUTPUT_FILE_HTML_ARR_=() local lGPT_REVERSE_LINK_="" local lWAIT_PIDS_F05_ARR=() - if [[ -f "${CSV_DIR}/q02_openai_question.csv" ]]; then + if [[ -f "${CSV_DIR}/ai_question.csv" ]]; then while IFS=";" read -r COL1_ COL2_ COL3_ COL4_ COL5_ COL6_ COL7_; do lGPT_INPUT_FILE_="${COL1_}" - lGPT_ANCHOR_="${COL2_}" + lAI_ANCHOR="${COL2_}" l_GPT_PRIO_="${COL3_}" lGPT_QUESTION_="${COL4_}" lGPT_OUTPUT_FILE_="${COL5_}" lGPT_TOKENS_="${COL6_//cost\=/}" - lGPT_RESPONSE_="${COL7_//\"/}" + # file with AI response: + lAI_RESPONSE_FILE="${COL7_}" - print_output "[*] GPT resolver - testing ${ORANGE}${CSV_DIR}/q02_openai_question.csv${NC}" + print_output "[*] AI resolver - testing ${ORANGE}${CSV_DIR}/ai_question.csv${NC}" - if [[ "${THREADED}" -eq 1 ]]; then - gpt_resolver_csv "${lGPT_INPUT_FILE_}" "${lGPT_ANCHOR_}" "${l_GPT_PRIO_}" "${lGPT_QUESTION_}" "${lGPT_OUTPUT_FILE_}" "${lGPT_TOKENS_}" "${lGPT_RESPONSE_}" & - local lTMP_PID="$!" - store_kill_pids "${lTMP_PID}" - lWAIT_PIDS_F05_ARR+=("${lTMP_PID}") - # max_pids_protection "${MAX_MOD_THREADS}" "${lWAIT_PIDS_F05_ARR[@]}" - else - gpt_resolver_csv "${lGPT_INPUT_FILE_}" "${lGPT_ANCHOR_}" "${l_GPT_PRIO_}" "${lGPT_QUESTION_}" "${lGPT_OUTPUT_FILE_}" "${lGPT_TOKENS_}" "${lGPT_RESPONSE_}" - fi - - done <"${CSV_DIR}/q02_openai_question.csv" - - [[ "${THREADED}" -eq 1 ]] && wait_for_pid "${lWAIT_PIDS_F05_ARR[@]}" + gpt_resolver_csv "${lGPT_INPUT_FILE_}" "${lAI_ANCHOR}" "${l_GPT_PRIO_}" "${lGPT_QUESTION_}" "${lGPT_OUTPUT_FILE_}" "${lGPT_TOKENS_}" "${lAI_RESPONSE_FILE}" & + done <"${CSV_DIR}/ai_question.csv" + wait_for_pid "${lWAIT_PIDS_F05_ARR[@]}" fi - if [[ -f "${CSV_DIR}/q02_openai_question.csv.tmp" ]]; then + if [[ -f "${CSV_DIR}/ai_question.csv.tmp" ]]; then while IFS=";" read -r COL1_ COL2_ COL3_ COL4_ COL5_ COL6_ COL7_; do local lGPT_INPUT_FILE_="${COL1_}" - local lGPT_ANCHOR_="${COL2_}" + local lAI_ANCHOR="${COL2_}" local l_GPT_PRIO_="${COL3_}" local lGPT_QUESTION_="${COL4_}" local lGPT_OUTPUT_FILE_="${COL5_}" local lGPT_TOKENS_="${COL6_//cost\=/}" local lGPT_RESPONSE_="${COL7_//\"/}" - print_output "[*] Trying to resolve ${ORANGE}Anchor ${lGPT_ANCHOR_}${NC} in ${ORANGE}Output_file ${lGPT_OUTPUT_FILE_}${NC}." + print_output "[*] Trying to resolve ${ORANGE}Anchor ${lAI_ANCHOR}${NC} in ${ORANGE}Output_file ${lGPT_OUTPUT_FILE_}${NC}." - if [[ "${THREADED}" -eq 1 ]]; then - gpt_resolver_csv_tmp "${lGPT_INPUT_FILE_}" "${lGPT_ANCHOR_}" "${l_GPT_PRIO_}" "${lGPT_QUESTION_}" "${lGPT_OUTPUT_FILE_}" "${lGPT_TOKENS_}" "${lGPT_RESPONSE_}" & - local lTMP_PID="$!" - store_kill_pids "${lTMP_PID}" - lWAIT_PIDS_F05_ARR+=("${lTMP_PID}") - # max_pids_protection "${MAX_MOD_THREADS}" "${lWAIT_PIDS_F05_ARR[@]}" - else - gpt_resolver_csv_tmp "${lGPT_INPUT_FILE_}" "${lGPT_ANCHOR_}" "${l_GPT_PRIO_}" "${lGPT_QUESTION_}" "${lGPT_OUTPUT_FILE_}" "${lGPT_TOKENS_}" "${lGPT_RESPONSE_}" - fi - done <"${CSV_DIR}/q02_openai_question.csv.tmp" - - [[ "${THREADED}" -eq 1 ]] && wait_for_pid "${lWAIT_PIDS_F05_ARR[@]}" + gpt_resolver_csv_tmp "${lGPT_INPUT_FILE_}" "${lAI_ANCHOR}" "${l_GPT_PRIO_}" "${lGPT_QUESTION_}" "${lGPT_OUTPUT_FILE_}" "${lGPT_TOKENS_}" "${lGPT_RESPONSE_}" & + local lTMP_PID="$!" + lWAIT_PIDS_F05_ARR+=("${lTMP_PID}") + # max_pids_protection "${MAX_MOD_THREADS}" "${lWAIT_PIDS_F05_ARR[@]}" + done <"${CSV_DIR}/ai_question.csv.tmp" + wait_for_pid "${lWAIT_PIDS_F05_ARR[@]}" fi fi - if [[ -d "${HTML_PATH}" ]]; then - # lets do a final cleanup to get rid of all the ASK_GPT entries: - find "${HTML_PATH}" -type f -name "*.html" -exec sed -i '/ASK_GPT/d' {} \; - fi + #if [[ -d "${HTML_PATH}" ]]; then + # lets do a final cleanup to get rid of all the ASK_AI entries: + # find "${HTML_PATH}" -type f -name "*.html" -exec sed -i '/ASK_AI/d' {} \; + #fi - # lets do a final cleanup to get rid of all the ASK_GPT entries: - find "${LOG_DIR}" -maxdepth 1 -type f -name "*.txt" -exec sed -i '/ASK_GPT/d' {} \; + # lets do a final cleanup to get rid of all the ASK_AI entries: + #find "${LOG_DIR}" -maxdepth 1 -type f -name "*.txt" -exec sed -i '/ASK_AI/d' {} \; # do not create a web reporter page module_end_log "${FUNCNAME[0]}" 0 @@ -109,69 +106,58 @@ F05_qs_resolver() { gpt_resolver_csv() { local lGPT_INPUT_FILE_="${1:-}" - local lGPT_ANCHOR_="${2:-}" + local lAI_ANCHOR="${2:-}" local l_GPT_PRIO_="${3:-}" local lGPT_QUESTION_="${4:-}" local lGPT_OUTPUT_FILE_="${5:-}" local lGPT_TOKENS_="${6:-}" - local lGPT_RESPONSE_="${7:-}" + local lAI_RESPONSE_FILE="${7:-}" local lGPT_OUTPUT_FILE_NAME="" - local lGPT_OUTPUT_FILE_NAME_bak="" local lHTML_FILE_="" local lHTML_FILE_X="" - print_output "[*] Trying to resolve ${ORANGE}Anchor ${lGPT_ANCHOR_}${NC} in ${ORANGE}Output_file ${lGPT_OUTPUT_FILE_}${NC}." + print_output "[*] Trying to resolve ${ORANGE}Anchor ${lAI_ANCHOR}${NC}." if [[ ${lGPT_TOKENS_} -ne 0 ]]; then - if ! [ -f "${lGPT_OUTPUT_FILE_}" ]; then - print_output "[-] Something went wrong with the Output file ${lGPT_OUTPUT_FILE_}" - if [[ -z ${lGPT_OUTPUT_FILE_} ]]; then - print_output " there is no file name for anchor: ${lGPT_ANCHOR_}" + # replace anchor in html-report with link to response + + local lMD5_OF_AI_RESPONSE_FILE="" + lMD5_OF_AI_RESPONSE_FILE=$(md5sum "${lAI_RESPONSE_FILE}" | awk '{print $1}') + lAI_RESPONSE_FILE=$(find "${HTML_PATH}" -iname "${lMD5_OF_AI_RESPONSE_FILE}.html" 2>/dev/null | sort -u | head -1) + lAI_RESPONSE_FILE_NAME=$(basename "${lAI_RESPONSE_FILE}") + print_output "[*] Testing ${lAI_RESPONSE_FILE} with md5sum of ${lMD5_OF_AI_RESPONSE_FILE}" "no_log" + + readarray -t lGPT_OUTPUT_FILE_HTML_ARR_ < <(grep -r -l "\[ASK_AI\]\ ${lAI_ANCHOR}" "${HTML_PATH}" 2>/dev/null || true) + + for lHTML_FILE_ in "${lGPT_OUTPUT_FILE_HTML_ARR_[@]}"; do + print_output "[*] Testing ${lHTML_FILE_} for anchor ${lAI_ANCHOR}" "no_log" + # should point back to q02-submodule with name "${lGPT_INPUT_FILE_}" + lGPT_REVERSE_LINK_="$(tr "[:upper:]" "[:lower:]" <<<"${lGPT_INPUT_FILE_}" | sed -e "s@[^a-zA-Z0-9]@@g")" + # shellcheck disable=SC2001 + lHTML_FILE_X=$(echo "${lHTML_FILE_}" | sed 's#'"${HTML_PATH}"'##') + print_output "[*] Linking AI results ${ORANGE}${lGPT_REVERSE_LINK_}${NC} into ${ORANGE}${lHTML_FILE_X}${NC}" "no_log" + local lDEPTH="\.\.\/" + + if [[ "${AI_OPTION}" -eq 3 ]]; then + if [[ -f "${lAI_RESPONSE_FILE}" ]]; then + print_output "[*] Replacing Anchor ${lAI_ANCHOR} with link to ${lAI_RESPONSE_FILE_NAME} - ${lAI_RESPONSE_FILE} in ${lHTML_FILE_}" + sed -i "s/\[ASK_AI\]\ ${lAI_ANCHOR}/\ \ \ \ \\[+] LocalAI results are available\<\/span\>\<\/a\>\n/1" "${lHTML_FILE_}" + else + # link to localAI module results - q03_localai_connector.html + sed -i "s/\[ASK_AI\]\ ${lAI_ANCHOR}/\ \ \ \ \\[+] LocalAI results are available\<\/span\>\<\/a\>\n/1" "${lHTML_FILE_}" + fi + else + # openai mode - deprected and probably not working anymore + # Todo: remove OpenAI module environment from EMBA + sed -i "s/\[ASK_AI\]\ ${lAI_ANCHOR}/\ \ \ \ \\[+] OpenAI results are available\<\/span\>\<\/a\>\n/1" "${lHTML_FILE_}" fi - else - sed -i "s/${lGPT_ANCHOR_}/Q\: ${lGPT_QUESTION_}\nA\: /1" "${lGPT_OUTPUT_FILE_}" - # grep "${lGPT_ANCHOR_}" "${CSV_DIR}/q02_openai_question.csv" | cut -d";" -f7 >> "${lGPT_OUTPUT_FILE_}" - printf '%q\n' "${lGPT_RESPONSE_//\\/}" >>"${lGPT_OUTPUT_FILE_}" - # replace anchor in html-report with link to response - - if [[ "${lGPT_OUTPUT_FILE_}" == *".log" ]]; then - lGPT_OUTPUT_FILE_NAME="$(basename "${lGPT_OUTPUT_FILE_//\.log/}.html")" - elif [[ "${lGPT_OUTPUT_FILE_}" == *".txt" ]]; then - lGPT_OUTPUT_FILE_NAME="$(basename "${lGPT_OUTPUT_FILE_//\.txt/}.html")" - elif [[ "${lGPT_OUTPUT_FILE_}" == *".c" ]]; then - lGPT_OUTPUT_FILE_NAME="$(basename "${lGPT_OUTPUT_FILE_//\.c/}.html")" - fi - - readarray -t lGPT_OUTPUT_FILE_HTML_ARR_ < <(find "${HTML_PATH}" -iname "${lGPT_OUTPUT_FILE_NAME}" 2>/dev/null) - # the following search is because of inconsistency in file names. - # Todo: check this and fix it to only use the rules above - lGPT_OUTPUT_FILE_NAME_bak="$(basename "${lGPT_OUTPUT_FILE_//\./}.html")" - readarray -t GPT_OUTPUT_FILE_HTML_ARR_bak < <(find "${HTML_PATH}" -iname "${lGPT_OUTPUT_FILE_NAME_bak}" 2>/dev/null) - lGPT_OUTPUT_FILE_HTML_ARR_+=("${GPT_OUTPUT_FILE_HTML_ARR_bak[@]}") - - for lHTML_FILE_ in "${lGPT_OUTPUT_FILE_HTML_ARR_[@]}"; do - # should point back to q02-submodule with name "${lGPT_INPUT_FILE_}" - lGPT_REVERSE_LINK_="$(tr "[:upper:]" "[:lower:]" <<<"${lGPT_INPUT_FILE_}" | sed -e "s@[^a-zA-Z0-9]@@g")" - # we need to find the depth which we need to link to the file - # shellcheck disable=SC2001 - lHTML_FILE_X=$(echo "${lHTML_FILE_}" | sed 's#'"${HTML_PATH}"'##') - print_output "[*] Linking GPT results ${ORANGE}${lGPT_REVERSE_LINK_}${NC} into ${ORANGE}${lHTML_FILE_X}${NC}" "no_log" - depth_cnt="${lHTML_FILE_X//[^\/]/}" - depth_cnt="$(("${#depth_cnt}" - 1))" - local lDEPTH="\.\.\/" - local lmyDEPTH="" - lmyDEPTH=$(printf "%${depth_cnt}s") - lDEPTH="${lmyDEPTH// /${lDEPTH}}" - - sed -i "s/\[ASK_GPT\]\ ${lGPT_ANCHOR_}/\ \ \ \ \\[+] OpenAI results are available\<\/span\>\<\/a\>\n/1" "${lHTML_FILE_}" - done - fi + done fi } gpt_resolver_csv_tmp() { local lGPT_INPUT_FILE_="${1:-}" - local lGPT_ANCHOR_="${2:-}" + local lAI_ANCHOR="${2:-}" local l_GPT_PRIO_="${3:-}" local lGPT_QUESTION_="${4:-}" local lGPT_OUTPUT_FILE_="${5:-}" @@ -181,44 +167,31 @@ gpt_resolver_csv_tmp() { local lHTML_FILE_="" local lHTML_FILE_X="" - print_output "[*] Trying to resolve ${ORANGE}Anchor ${lGPT_ANCHOR_}${NC} in ${ORANGE}output_file ${lGPT_OUTPUT_FILE_}${NC}." + print_output "[*] Trying to resolve ${ORANGE}Anchor ${lAI_ANCHOR}${NC}." - if ! [ -f "${lGPT_OUTPUT_FILE_}" ]; then - print_output "[-] Something went wrong with the Output file ${lGPT_OUTPUT_FILE_}" - if [[ -z ${lGPT_OUTPUT_FILE_} ]]; then - print_output " there is no file name for anchor: ${lGPT_ANCHOR_}" + print_output "[*] AI module didn't check ${lGPT_INPUT_FILE_}, linking to the GPT module page instead" + + local lMD5_OF_ORIG_SRC_FILE="" + lMD5_OF_ORIG_SRC_FILE=$(md5sum "${lGPT_OUTPUT_FILE_}" | awk '{print $1}') + + readarray -t lGPT_OUTPUT_FILE_HTML_ARR_ < <(grep -r -l "\[ASK_AI\]\ ${lAI_ANCHOR}" "${HTML_PATH}" 2>/dev/null || true) + + for lHTML_FILE_ in "${lGPT_OUTPUT_FILE_HTML_ARR_[@]}"; do + # should point back to q02-submodule with name "${lGPT_INPUT_FILE_}" + lGPT_REVERSE_LINK_="$(tr "[:upper:]" "[:lower:]" <<<"${lGPT_INPUT_FILE_}" | sed -e "s@[^a-zA-Z0-9]@@g")" + + # we need to find the depth which we need to link to the file + # shellcheck disable=SC2001 + lHTML_FILE_X=$(echo "${lHTML_FILE_}" | sed 's#'"${HTML_PATH}"'##') + print_output "[*] Linking AI results ${ORANGE}${lGPT_REVERSE_LINK_}${NC} into ${ORANGE}${lHTML_FILE_X}${NC}" "no_log" + depth_cnt="${lHTML_FILE_X//[^\/]/}" + depth_cnt="$(("${#depth_cnt}" - 1))" + local lDEPTH="\.\.\/" + + if [[ "${AI_OPTION}" -eq 3 ]]; then + sed -i "s/\[ASK_AI\]\ ${lAI_ANCHOR}/\ \ \ \ \\[*] LocalAI module did not finish\<\/span\>\<\/a\>\n/1" "${lHTML_FILE_}" + else + sed -i "s/\[ASK_AI\]\ ${lAI_ANCHOR}/\ \ \ \ \\[*] OpenAI module did not finish\<\/span\>\<\/a\>\n/1" "${lHTML_FILE_}" fi - else - print_output "[*] Q02 didn't check ${lGPT_INPUT_FILE_}, linking to the GPT module page instead" - - # sed -i "s/${lGPT_ANCHOR_}/Check did not finish!/1" "${lGPT_OUTPUT_FILE_}" - - if [[ "${lGPT_OUTPUT_FILE_}" == *".log" ]]; then - lGPT_OUTPUT_FILE_NAME="$(basename "${lGPT_OUTPUT_FILE_//\.log/}.html")" - elif [[ "${lGPT_OUTPUT_FILE_}" == *".txt" ]]; then - lGPT_OUTPUT_FILE_NAME="$(basename "${lGPT_OUTPUT_FILE_//\.txt/}.html")" - elif [[ "${lGPT_OUTPUT_FILE_}" == *".c" ]]; then - lGPT_OUTPUT_FILE_NAME="$(basename "${lGPT_OUTPUT_FILE_//\.c/}.html")" - fi - - readarray -t lGPT_OUTPUT_FILE_HTML_ARR_ < <(find "${HTML_PATH}" -iname "${lGPT_OUTPUT_FILE_NAME}" 2>/dev/null) - - for lHTML_FILE_ in "${lGPT_OUTPUT_FILE_HTML_ARR_[@]}"; do - # should point back to q02-submodule with name "${lGPT_INPUT_FILE_}" - lGPT_REVERSE_LINK_="$(tr "[:upper:]" "[:lower:]" <<<"${lGPT_INPUT_FILE_}" | sed -e "s@[^a-zA-Z0-9]@@g")" - - # we need to find the depth which we need to link to the file - # shellcheck disable=SC2001 - lHTML_FILE_X=$(echo "${lHTML_FILE_}" | sed 's#'"${HTML_PATH}"'##') - print_output "[*] Linking GPT results ${ORANGE}${lGPT_REVERSE_LINK_}${NC} into ${ORANGE}${lHTML_FILE_X}${NC}" "no_log" - depth_cnt="${lHTML_FILE_X//[^\/]/}" - depth_cnt="$(("${#depth_cnt}" - 1))" - local lDEPTH="\.\.\/" - local lmyDEPTH="" - lmyDEPTH=$(printf "%${depth_cnt}s") - lDEPTH="${lmyDEPTH// /${lDEPTH}}" - - sed -i "s/\[ASK_GPT\]\ ${lGPT_ANCHOR_}/\ \ \ \ \\[*] OpenAI module did not finish\<\/span\>\<\/a\>\n/1" "${lHTML_FILE_}" - done - fi + done } diff --git a/modules/F50_base_aggregator.sh b/modules/F50_base_aggregator.sh index 9a8411d7..0ef1ae56 100755 --- a/modules/F50_base_aggregator.sh +++ b/modules/F50_base_aggregator.sh @@ -96,14 +96,6 @@ output_overview() { print_output "[+] EMBA version: ""${ORANGE}""${lSBOM_TOOL_VERS}""${NC}" write_csv_log "EMBA_version" "${lSBOM_TOOL_VERS}" "NA" "NA" "NA" "NA" "NA" "NA" "NA" - if [[ -f "${Q02_LOG}" ]] && [[ "${GPT_OPTION}" -gt 0 ]]; then - lGPT_RESULTS_CNT=$(grep -c "AI analysis for" "${Q02_LOG}" || true) - if [[ "${lGPT_RESULTS_CNT}" -gt 0 ]]; then - print_output "[+] EMBA AI analysis discovered ${ORANGE}${lGPT_RESULTS_CNT}${GREEN} results." - write_link "q02" - fi - fi - if [[ -n "${ARCH}" ]] && [[ "${ARCH}" != "NA" ]]; then if [[ -n "${D_END:-"NA"}" ]]; then write_csv_log "architecture_verified" "${ARCH}" "${D_END}" "NA" "NA" "NA" "NA" "NA" "NA" @@ -263,10 +255,27 @@ output_details() { fi fi + if [[ -f "${Q02_LOG}" ]] && [[ "${AI_OPTION}" -gt 0 ]]; then + lGPT_RESULTS_CNT=$(grep -c "AI analysis for" "${Q02_LOG}" || true) + if [[ "${lGPT_RESULTS_CNT}" -gt 0 ]]; then + print_output "[+] AI analysis identified ${ORANGE}${lGPT_RESULTS_CNT}${GREEN} results via ChatGPT." + write_link "q02" + write_csv_log "AI results" "${lGPT_RESULTS_CNT}" "ChatGPT" "NA" "NA" "NA" "NA" "NA" "NA" + fi + fi + + if [[ -f "${Q03_LOG}" ]] && [[ "${AI_OPTION}" -gt 0 ]]; then + lAI_RESULTS_CNT=$(grep -c "AI analysis results for" "${Q03_LOG}" || true) + if [[ "${lAI_RESULTS_CNT}" -gt 0 ]]; then + print_output "[+] AI analysis discovered ${ORANGE}${lAI_RESULTS_CNT}${GREEN} results via LocalAI." + write_link "q03" + write_csv_log "AI results" "${lAI_RESULTS_CNT}" "LocalAI" "NA" "NA" "NA" "NA" "NA" "NA" + fi + fi + + if [[ "${lGPT_RESULTS_CNT:-0}" -gt 0 ]]; then - print_output "[+] EMBA AI tests identified ${ORANGE}${lGPT_RESULTS_CNT}${GREEN} results via ChatGPT." write_link "q02" - write_csv_log "AI results" "${lGPT_RESULTS_CNT}" "NA" "NA" "NA" "NA" "NA" "NA" "NA" fi if [[ "${BOOTED:-0}" -gt 0 ]] || [[ "${IP_ADDR_CNT:-0}" -gt 0 ]]; then diff --git a/modules/Q02_openai_question.sh b/modules/Q02_openai_question.sh index 270b5530..b032aa7c 100755 --- a/modules/Q02_openai_question.sh +++ b/modules/Q02_openai_question.sh @@ -19,7 +19,7 @@ Q02_openai_question() { module_log_init "${FUNCNAME[0]}" export CHATGPT_RESULT_CNT=0 - if [[ "${GPT_OPTION}" -gt 0 ]] && [[ -n "${OPENAI_API_KEY}" ]]; then + if [[ "${AI_OPTION}" -gt 0 ]] && [[ -n "${OPENAI_API_KEY}" ]]; then # Prints title to CLI and into log module_title "AI analysis via OpenAI" pre_module_reporter "${FUNCNAME[0]}" @@ -34,13 +34,13 @@ Q02_openai_question() { done if [[ -f "${LOG_DIR}"/"${MAIN_LOG_FILE}" ]]; then - while ! [[ -f "${CSV_DIR}/q02_openai_question.csv.tmp" ]]; do + while ! [[ -f "${CSV_DIR}/ai_question.csv.tmp" ]]; do sleep 3 done fi export GTP_CHECKED_ARR=() - while ! grep -q "Testing phase ended" "${LOG_DIR}"/"${MAIN_LOG_FILE}"; do + while ! grep -q "Reporting phase started" "${LOG_DIR}"/"${MAIN_LOG_FILE}" && [[ "${lMODULE_RUNTIME}" -le "${LOCAL_AI_MIN_RUNTIME}" ]]; do if [[ "${CHATGPT_RESULT_CNT}" -ge 0 ]]; then ask_chatgpt fi @@ -79,7 +79,7 @@ ask_chatgpt() { # generating Array for GPT requests - sorting according the prio in field 3 # this array gets regenerated on every round - readarray -t Q02_OPENAI_QUESTIONS < <(sort -k 3 -t ';' -r "${CSV_DIR}/q02_openai_question.csv.tmp") + readarray -t Q02_OPENAI_QUESTIONS < <(sort -k 3 -t ';' -r "${CSV_DIR}/ai_question.csv.tmp") for ((lELE_INDEX = 0; lELE_INDEX < "${#Q02_OPENAI_QUESTIONS[@]}"; lELE_INDEX++)); do local lELEM="${Q02_OPENAI_QUESTIONS["${lELE_INDEX}"]}" @@ -197,7 +197,7 @@ ask_chatgpt() { lCNT=$((lCNT + 1)) local lTEMP_VAR="$(("${lCNT}" % 100))" (("${lTEMP_VAR}" == 0)) && print_output "[*] Rate limit handling ... sleep mode - ${lCNT}" "no_log" - if grep -q "Testing phase ended" "${LOG_DIR}"/"${MAIN_LOG_FILE}"; then + if grep -q "Reporting phase started" "${LOG_DIR}"/"${MAIN_LOG_FILE}" && [[ "${lMODULE_RUNTIME}" -ge "${LOCAL_AI_MIN_RUNTIME}" ]]; then break 2 fi sleep 1 @@ -211,10 +211,10 @@ ask_chatgpt() { fi cat "${TMP_DIR}/${lGPT_INPUT_FILE_mod}_response.json" >>"${lGPT_FILE_DIR}/openai_server_errors.log" - readarray -t Q02_OPENAI_QUESTIONS < <(sort -k 3 -t ';' -r "${CSV_DIR}/q02_openai_question.csv.tmp") + readarray -t Q02_OPENAI_QUESTIONS < <(sort -k 3 -t ';' -r "${CSV_DIR}/ai_question.csv.tmp") # reset the array index to start again with the highest rated entry lELE_INDEX=0 - if grep -q "Testing phase ended" "${LOG_DIR}"/"${MAIN_LOG_FILE}"; then + if grep -q "Reporting phase started" "${LOG_DIR}"/"${MAIN_LOG_FILE}" && [[ "${lMODULE_RUNTIME}" -ge "${LOCAL_AI_MIN_RUNTIME}" ]]; then return fi sleep 30s @@ -235,7 +235,7 @@ ask_chatgpt() { if [[ ${lGPT_TOKENS} -ne 0 ]]; then GTP_CHECKED_ARR+=("${lSCRIPT_PATH_TMP}") # write new into done csv - write_csv_gpt "${lGPT_INPUT_FILE}" "${lGPT_ANCHOR}" "${lGPT_PRIO}" "${lGPT_QUESTION}" "${lGPT_OUTPUT_FILE}" "cost=${lGPT_TOKENS}" "'${lGPT_RESPONSE_CLEANED//\'/}'" + write_csv_AI "${lGPT_INPUT_FILE}" "${lGPT_ANCHOR}" "${lGPT_PRIO}" "${lGPT_QUESTION}" "${lGPT_OUTPUT_FILE}" "cost=${lGPT_TOKENS}" "'${lGPT_RESPONSE_CLEANED//\'/}'" # we store the answers in dedicated files for further interlinking within the report if ! [[ -d "${LOG_PATH_MODULE}"/gpt_answers ]]; then @@ -275,26 +275,26 @@ ask_chatgpt() { fi fi - if grep -q "Testing phase ended" "${LOG_DIR}"/"${MAIN_LOG_FILE}"; then + if grep -q "Reporting phase started" "${LOG_DIR}"/"${MAIN_LOG_FILE}" && [[ "${lMODULE_RUNTIME}" -ge "${LOCAL_AI_MIN_RUNTIME}" ]]; then break fi - if [[ "${GPT_OPTION}" -ne 2 ]]; then + if [[ "${AI_OPTION}" -ne 2 ]]; then sleep 20s fi # reload q02 results: print_output "[*] Regenerate analysis array ..." "no_log" - readarray -t Q02_OPENAI_QUESTIONS < <(sort -k 3 -t ';' -r "${CSV_DIR}/q02_openai_question.csv.tmp") + readarray -t Q02_OPENAI_QUESTIONS < <(sort -k 3 -t ';' -r "${CSV_DIR}/ai_question.csv.tmp") # reset the array index to start again with the highest rated entry lELE_INDEX=0 done - if [[ -f "${CSV_DIR}/q02_openai_question.csv" ]]; then + if [[ -f "${CSV_DIR}/ai_question.csv" ]]; then local lGPT_ENTRY_LINE="" while read -r lGPT_ENTRY_LINE; do lGPT_ANCHOR="$(echo "${lGPT_ENTRY_LINE}" | cut -d ';' -f2)" - sed -i "/${lGPT_ANCHOR}/d" "${CSV_DIR}/q02_openai_question.csv.tmp" - done <"${CSV_DIR}/q02_openai_question.csv" + sed -i "/${lGPT_ANCHOR}/d" "${CSV_DIR}/ai_question.csv.tmp" + done <"${CSV_DIR}/ai_question.csv" fi } diff --git a/modules/Q03_localai_connector.sh b/modules/Q03_localai_connector.sh new file mode 100755 index 00000000..fafa6475 --- /dev/null +++ b/modules/Q03_localai_connector.sh @@ -0,0 +1,592 @@ +#!/bin/bash -p + +# EMBA - EMBEDDED LINUX ANALYZER +# +# Copyright 2020-2026 Siemens Energy AG +# +# EMBA comes with ABSOLUTELY NO WARRANTY. This is free software, and you are +# welcome to redistribute it under the terms of the GNU General Public License. +# See LICENSE file for usage of this software. +# +# EMBA is licensed under GPLv3 +# SPDX-License-Identifier: GPL-3.0-only +# +# Author(s): Benedikt Kuehne, Michael Messner + +# Description: LocalAI questioning module for container #2 +# Note: Important requirement for Q-modules is the self termination when a certain phase ends +# This module has an additional minimum runtime mechanism. Ensure the F05 module is respecting +# the minimum runtime and waits for the module to finish. + +# set DEBUG to 1 to enable further debug messages +: "${DEBUG:=0}" +export DEBUG + +Q03_localai_connector() { + module_log_init "${FUNCNAME[0]}" + export AI_RESULT_CNT=0 + + if [[ "${AI_OPTION}" -eq 3 ]]; then + module_title "AI analysis via LocalAI" + pre_module_reporter "${FUNCNAME[0]}" + + export AI_RESULT_CNT=0 + export AI_ERROR_CNT=0 + local lMAX_AI_ERROR_CNT=10 + + local lMODEL_LOCALAI="" + lMODEL_LOCALAI=$(identify_ai_model) + if [[ -z "${lMODEL_LOCALAI}" ]]; then + print_output "[-] No model identified - is the AI responsive? Exit now." + module_end_log "${FUNCNAME[0]}" 0 + return + fi + + print_output "[*] Identified AI model: ${lMODEL_LOCALAI}" + + if [[ "${lMODEL_LOCALAI}" != *"${LOCAL_AI_MODEL}"* ]]; then + print_output "[-] Configured model ${LOCAL_AI_MODEL} does not match the identified model ${lMODEL_LOCALAI} - exit now" + module_end_log "${FUNCNAME[0]}" 0 + return + fi + + # we wait until there arer entries in the question csv + while ! [[ -f "${MAIN_LOG}" ]]; do + if ! [[ -d "${LOG_DIR}" ]]; then + # this usually happens if we automate analysis and remove the logging directory while this module was not finished at all + return + fi + sleep 10 + done + + if [[ -f "${MAIN_LOG}" ]]; then + while ! [[ -f "${CSV_DIR}/ai_question.csv.tmp" ]]; do + sleep 3 + done + fi + + export SECONDS=0 + if [[ "${AI_MIN_RUNTIME}" == *"h" ]]; then + AI_MIN_RUNTIME=${AI_MIN_RUNTIME//h/} + AI_MIN_RUNTIME=$((AI_MIN_RUNTIME*3600)) + fi + if [[ "${AI_MIN_RUNTIME}" == *"m" ]]; then + AI_MIN_RUNTIME=${AI_MIN_RUNTIME//m/} + AI_MIN_RUNTIME=$((AI_MIN_RUNTIME*60)) + fi + if [[ "${AI_MIN_RUNTIME}" == *"d" ]]; then + AI_MIN_RUNTIME=${AI_MIN_RUNTIME//d/} + AI_MIN_RUNTIME=$((AI_MIN_RUNTIME*24*3600)) + fi + + export GTP_CHECKED_ARR=() + local lMODULE_RUNTIME=${SECONDS} + adjust_minimum_runtime "${lMODULE_RUNTIME}" + # check for runtime + while [[ "${lMODULE_RUNTIME}" -lt "${AI_MIN_RUNTIME}" ]]; do + if [[ "${AI_ERROR_CNT}" -lt "${lMAX_AI_ERROR_CNT}" ]]; then + ask_localai "${lMODEL_LOCALAI}" + else + # exit if too many errors happened + break + fi + local lMODULE_RUNTIME=${SECONDS} + adjust_minimum_runtime "${lMODULE_RUNTIME}" + done + fi + module_end_log "${FUNCNAME[0]}" "${AI_RESULT_CNT}" +} + +adjust_minimum_runtime() { + local lMODULE_RUNTIME="${1:-}" + + # if the minimum runtime is not set we adjust it to something bigger then the module runtime + # with this we can start and the later rules to end this module will jump in + if [[ -z "${AI_MIN_RUNTIME}" ]]; then + AI_MIN_RUNTIME=$((lMODULE_RUNTIME+10)) + return + fi + # to enusure the AI module is running at least until the reporting phase started we tweak the minimum + # runtime to a higher value as the module runtime as long as the reporting phase is not started + if ! grep -q "Reporting phase started" "${MAIN_LOG}" && [[ "${lMODULE_RUNTIME}" -ge "${AI_MIN_RUNTIME}" ]]; then + AI_MIN_RUNTIME=$((lMODULE_RUNTIME+10)) + fi +} + +# looks through the modules and finds localAI questions inside the csv +ask_localai() { + local lMODEL_LOCALAI="${1:-}" + + local lAI_FILE_DIR="${LOG_PATH_MODULE}/ai_files" + export AI_PROMPT_DIR="${LOG_PATH_MODULE}/ai_prompt" + local lAI_PRIO=3 + # default vars + local lHTTP_CODE=200 + local lORIGIN_MODULE="" + local lELE_INDEX=0 + local lAI_ANCHOR="" + local lAI_INPUT_FILE="" + local lAI_INPUT_FILE_mod="" + local lAI_SOURCE_FILE="" + local lSCRIPT_PATH_TMP="" + + print_output "[*] Checking scripts with LocalAI that have a priority of ${ORANGE}${MINIMUM_GPT_PRIO}${NC} or lower" "no_log" + if ! [[ -d "${lAI_FILE_DIR}" ]]; then + mkdir "${lAI_FILE_DIR}" + fi + if ! [[ -d "${AI_PROMPT_DIR}" ]]; then + mkdir "${AI_PROMPT_DIR}" + fi + + # generating Array for AI requests - sorting according the prio in field 3 + # this array gets regenerated on every round + regenerate_ai_todo_list + + for ((lELE_INDEX = 0; lELE_INDEX < "${#Q03_LOCALAI_QUESTIONS[@]}"; lELE_INDEX++)); do + local lELEM="${Q03_LOCALAI_QUESTIONS["${lELE_INDEX}"]}" + lSCRIPT_PATH_TMP="$(echo "${lELEM}" | cut -d\; -f1)" + + # as we always start with the highest rated entry, we need to check if this entry was already tested: + if [[ " ${GTP_CHECKED_ARR[*]} " =~ ${lSCRIPT_PATH_TMP} ]]; then + print_output "[*] AI - Already tested ${lSCRIPT_PATH_TMP}" "no_log" + # lets test the next entry + continue + fi + + lAI_ANCHOR="$(echo "${lELEM}" | cut -d\; -f2)" + lAI_PRIO="$(echo "${lELEM}" | cut -d\; -f3)" + lAI_SOURCE_FILE="$(echo "${lELEM}" | cut -d\; -f5)" + lAI_INPUT_FILE="$(basename "${lSCRIPT_PATH_TMP}")" + lAI_INPUT_FILE_mod="${lAI_INPUT_FILE//\./}" + + # in case we have nothing we are going to move on + [[ -z "${lSCRIPT_PATH_TMP}" ]] && continue + + if [[ "${lSCRIPT_PATH_TMP}" == *"s16_ghidra_decompile_checks"* ]]; then + print_output "[*] Ghidra decompiled code found ${lSCRIPT_PATH_TMP}" "no_log" + print_output "[*] AI-Assisted analysis of binary function ${ORANGE}${lSCRIPT_PATH_TMP}${NC}" "no_log" + elif [[ "${lSCRIPT_PATH_TMP}" == *"s28_java_check"* ]]; then + print_output "[*] Decompiled Java code found ${lSCRIPT_PATH_TMP}" "no_log" + print_output "[*] AI-Assisted analysis of Java binary ${ORANGE}${lSCRIPT_PATH_TMP}${NC}" "no_log" + elif [[ "${lSCRIPT_PATH_TMP}" == *"s20_shell_check"* ]]; then + print_output "[*] Shell code found ${lSCRIPT_PATH_TMP}" "no_log" + print_output "[*] AI-Assisted analysis of shell script ${ORANGE}${lSCRIPT_PATH_TMP}${NC}" "no_log" + elif [[ "${lSCRIPT_PATH_TMP}" == *"s21_python_check"* ]]; then + print_output "[*] Python code found ${lSCRIPT_PATH_TMP}" "no_log" + print_output "[*] AI-Assisted analysis of python script ${ORANGE}${lSCRIPT_PATH_TMP}${NC}" "no_log" + elif [[ "${lSCRIPT_PATH_TMP}" == *"s22_php_check"* ]]; then + print_output "[*] PHP code found ${lSCRIPT_PATH_TMP}" "no_log" + print_output "[*] AI-Assisted analysis of PHP script ${ORANGE}${lSCRIPT_PATH_TMP}${NC}" "no_log" + elif [[ "${lSCRIPT_PATH_TMP}" == *"s23_lua_check"* ]]; then + print_output "[*] Lua code found ${lSCRIPT_PATH_TMP}" "no_log" + print_output "[*] AI-Assisted analysis of lua script ${ORANGE}${lSCRIPT_PATH_TMP}${NC}" "no_log" + elif [[ "${lSCRIPT_PATH_TMP}" == *"s27_perl_check"* ]]; then + print_output "[*] Perl code found ${lSCRIPT_PATH_TMP}" "no_log" + print_output "[*] AI-Assisted analysis of perl script ${ORANGE}${lSCRIPT_PATH_TMP}${NC}" "no_log" + else + # this should not happen. If we land here we have changed something in the module area + # or we have a new module that is not handled correctly + print_output "[*] Identification of ${ORANGE}${lSCRIPT_PATH_TMP} / ${lAI_INPUT_FILE}${NC} inside ${ORANGE}${LOG_DIR}/firmware${NC}" "no_log" + if [[ "${lSCRIPT_PATH_TMP}" == ".""${LOG_DIR}"* ]]; then + print_output "[*] Warning: System path is not stripped with the root directory - we try to fix it now" "no_log" + # remove the '.' + lSCRIPT_PATH_TMP="${lSCRIPT_PATH_TMP:1}" + # remove the LOG_DIR + # shellcheck disable=SC2001 + lSCRIPT_PATH_TMP="$(echo "${lSCRIPT_PATH_TMP}" | sed 's#'"${LOG_DIR}"'##')" + print_output "[*] Stripped path ${lSCRIPT_PATH_TMP}" "no_log" + fi + # dirty fix - Todo: use array in future + lSCRIPT_PATH_TMP="$(find "${LOG_DIR}/firmware" -wholename "*${lSCRIPT_PATH_TMP}" | head -1)" + + # in case we have nothing we are going to move on + ! [[ -f "${lSCRIPT_PATH_TMP}" ]] && continue + print_output "[*] AI-Assisted analysis of script ${ORANGE}${lSCRIPT_PATH_TMP}${NC}" "no_log" + fi + + print_output "[*] Current priority for testing is ${lAI_PRIO}" "no_log" + + if [[ ${lAI_PRIO} -ge ${MINIMUM_GPT_PRIO} ]] && [[ "${lSCRIPT_PATH_TMP}" != '' ]]; then + if [[ -f "${lSCRIPT_PATH_TMP}" ]]; then + sub_module_title "AI analysis for ${lAI_INPUT_FILE//\.log}" + + local lBINARY_NAME="NA" + local lFUNCTION_NAME="NA" + + # Starting with special attention modules + if [[ "${lSCRIPT_PATH_TMP}" == *"s16_ghidra_decompile_checks"* ]]; then + lBINARY_NAME="$(echo "${lSCRIPT_PATH_TMP}" | rev | cut -d '/' -f2 | rev | sed 's/haruspex_//')" + lFUNCTION_NAME=$(extract_s16_fct_name "${lSCRIPT_PATH_TMP}") + print_output "[*] AI-Assisted analysis for function ${ORANGE}${lFUNCTION_NAME}${NC} of decompiled binary ${ORANGE}${lBINARY_NAME}${NC}" "" "${lSCRIPT_PATH_TMP}" + print_output "$(indent "$(orange "$(print_path "${lSCRIPT_PATH_TMP}")")")" + generate_prompt_binary "${lSCRIPT_PATH_TMP}" + elif [[ "${lSCRIPT_PATH_TMP}" == *"s28_java_check"* ]]; then + # we start with a path like /logs/s28_java_check/java_decompile/i.class_16708/i.java + # -> the last are we use as function name + lFUNCTION_NAME="${lSCRIPT_PATH_TMP//*\/}" + # -> the middle area is the binary name + lBINARY_NAME="${lSCRIPT_PATH_TMP%\/*}" + lBINARY_NAME="${lBINARY_NAME//*\/}" + lBINARY_NAME="${lBINARY_NAME//_*}" + print_output "[*] AI-Assisted analysis for decompiled Java binary function ${ORANGE}${lFUNCTION_NAME}${NC} of ${ORANGE}${lBINARY_NAME}${NC}" "" "${lSCRIPT_PATH_TMP}" + # as we are using lSCRIPT_PATH_TMP in the prompt generator we get a prompt file like prompt_localai_NeoterisStatic.java_Java.txt which is the current lFUNCTION_NAME + # later on we can't find the prompt file as we look for the lBINARY_NAME. This means we rewrite the lBINARY_NAME with the lFUNCTION_NAME + lBINARY_NAME="${lFUNCTION_NAME//\.log}" + + # we need to redefine the lFUNCTION_NAME now as the type of analysed binary + # Todo: rename and cleanup variable names + lFUNCTION_NAME="Java" + print_output "$(indent "$(orange "$(print_path "${lSCRIPT_PATH_TMP}")")")" + generate_prompt_script "${lSCRIPT_PATH_TMP}" "${lFUNCTION_NAME}" + else + # for all the default modules + lBINARY_NAME=$(basename "${lSCRIPT_PATH_TMP}") + lBINARY_NAME="${lBINARY_NAME//*\/}" + lBINARY_NAME="${lBINARY_NAME//\.log}" + + if [[ "${lAI_SOURCE_FILE}" == *"s22_php_check"* ]]; then + lFUNCTION_NAME="PHP" + elif [[ "${lAI_SOURCE_FILE}" == *"s20_shell_check"* ]]; then + lFUNCTION_NAME="shell" + elif [[ "${lAI_SOURCE_FILE}" == *"s21_python_check"* ]]; then + lFUNCTION_NAME="python" + elif [[ "${lAI_SOURCE_FILE}" == *"s23_lua_check"* ]]; then + lFUNCTION_NAME="lua" + elif [[ "${lAI_SOURCE_FILE}" == *"s27_perl_check"* ]]; then + lFUNCTION_NAME="perl" + else + # fallback + lBINARY_NAME=$(basename "${lSCRIPT_PATH_TMP}") + lFUNCTION_NAME="unknown" + fi + + print_output "[*] AI-Assisted analysis for ${ORANGE}${lFUNCTION_NAME}${NC} script ${ORANGE}${lBINARY_NAME}${NC}" "" "${lSCRIPT_PATH_TMP}" + print_output "$(indent "$(orange "$(print_path "${lSCRIPT_PATH_TMP}")")")" + generate_prompt_script "${lSCRIPT_PATH_TMP}" "${lFUNCTION_NAME}" + fi + + if [[ ! -f "${AI_PROMPT_DIR}/prompt_localai_${lBINARY_NAME}_${lFUNCTION_NAME}.txt" ]]; then + print_output "[-] No prompt generated for ${AI_PROMPT_DIR}/prompt_localai_${lBINARY_NAME}_${lFUNCTION_NAME}.txt ... next one" + continue + fi + + lPROMPT=$(cat "${AI_PROMPT_DIR}/prompt_localai_${lBINARY_NAME}_${lFUNCTION_NAME}.txt") + local lAI_LOG_FILE="${lAI_FILE_DIR}/ai_response_${lBINARY_NAME}_${lFUNCTION_NAME}.md" + + local lWC_M_FILE="" + local lWC_L_FILE="" + local lTOKENS_PRE="" + # lWC_M_FILE=$(echo "${lPROMPT}" | wc -m) + lWC_M_FILE="${#lPROMPT}" + lWC_L_FILE=$(echo "${lPROMPT}" | wc -l) + # just a very rough estimation + lTOKENS_PRE=$((lWC_M_FILE/2)) + + print_output "[*] Testing prompt" "" "${AI_PROMPT_DIR}/prompt_localai_${lBINARY_NAME}_${lFUNCTION_NAME}.txt" + print_output "${ORANGE}${lPROMPT}${NC}" "no_log" + print_output "[*] Testing code with ${ORANGE}${lWC_L_FILE}${NC} lines and ${ORANGE}${lWC_M_FILE}${NC} characters" + print_output "[*] Testing will need at least ${ORANGE}${lTOKENS_PRE}${NC} tokens\n" + + local lJSON_PAYLOAD="" + local lstart_time="" + local lTOKENS_POST="" + local lfinished_time="" + local lruntime="" + + lJSON_PAYLOAD=$(jq -n \ + --arg model "${lMODEL_LOCALAI}" \ + --arg prompt "${lPROMPT}" \ + '{model: $model, messages: [{role: "user", content: $prompt}], temperature: 0.2}') + + lstart_time=$(date +%s) + lHTTP_CODE=$(curl --connect-timeout 10 --max-time 600 -s http://"${LOCAL_AI_IP}":8080/v1/chat/completions \ + -H "Content-Type: application/json" \ + -d "${lJSON_PAYLOAD}" -o "${lAI_LOG_FILE}.json" --write-out "%{http_code}" || true) + + if [[ -f "${lAI_LOG_FILE}.json" ]]; then + # reformat the output to also have the token usage avaialble + jq -r '. | "\(.choices[].message.content)\n\nTokens used: \(.usage.total_tokens)"' "${lAI_LOG_FILE}.json" > "${lAI_LOG_FILE}" || print_error "[-] Q03 - AI parsing error for ${lAI_LOG_FILE}.json" + fi + + if [[ ! -f "${lAI_LOG_FILE}" ]] || [[ ! -s "${lAI_LOG_FILE}" ]]; then + print_output "[-] WARNING: No AI respoonse for function ${lFUNCTION_NAME}" + continue + fi + # our currenty AI output starts the headlines with level 3 headlines ### + parse_markdown_to_emba_txt "${lAI_LOG_FILE}" "${lAI_LOG_FILE//\.md/\.txt}" 3 + + # just for the user output on cli: + cat "${lAI_LOG_FILE//\.md/\.txt}" + + lTOKENS_POST=$(grep "Tokens used:" "${lAI_LOG_FILE}" | cut -d ':' -f2) + lTOKENS_POST=${lTOKENS_POST//\ /} + lfinished_time=$(date +%s) + lruntime=$((lfinished_time-lstart_time)) + + if [[ "${lTOKENS_POST}" -gt 0 && "${lruntime}" -gt 0 ]]; then + print_output "[*] AI took ${ORANGE}${lruntime}${NC} seconds and ${ORANGE}${lTOKENS_POST}${NC} tokens to complete this request." + print_output "[*] Tokens/sec: ${ORANGE}~$((lTOKENS_POST/lruntime))${NC}" || true + fi + + if [[ "${lHTTP_CODE}" -ne 200 ]]; then + print_output "[-] Something went wrong with the LocalAI requests" + # track the errors. If we get too many we are going to stop AI requests + AI_ERROR_CNT=$((AI_ERROR_CNT+1)) + + if [[ -f "${lAI_LOG_FILE}" ]]; then + print_output "[-] ERROR response: $(cat "${lAI_LOG_FILE}")" + fi + + # if the testing is over we can just return now + local lMODULE_RUNTIME=${SECONDS} + adjust_minimum_runtime "${lMODULE_RUNTIME}" + if [[ "${lMODULE_RUNTIME}" -gt "${AI_MIN_RUNTIME}" ]]; then + return + fi + + continue + fi + + if ! [[ -f "${lAI_LOG_FILE}" ]]; then + # catches: (56) Recv failure: Connection reset by peer + print_output "[-] Something went wrong with the LocalAI request for ${lAI_INPUT_FILE}" + AI_ERROR_CNT=$((AI_ERROR_CNT+1)) + + continue + fi + + if [[ ${lTOKENS_POST} -ne 0 ]]; then + GTP_CHECKED_ARR+=("${lSCRIPT_PATH_TMP}") + # write results into done csv + # lAI_INPUT_FILE - lAI_ANCHOR - Priority - used Prompt file - source file to test - cost - ai final log file with results + write_csv_AI "${lAI_INPUT_FILE}" "${lAI_ANCHOR}" "${lAI_PRIO}" "${AI_PROMPT_DIR}/prompt_localai_${lBINARY_NAME}_${lFUNCTION_NAME}.txt" "${lAI_SOURCE_FILE}" "cost=${lTOKENS_POST}" "${lAI_LOG_FILE//\.md/\.txt}" + + # add proper module link + if [[ "${lAI_SOURCE_FILE}" == *'/csv_logs/'* ]]; then + # if we have a csv_logs path we need to adjust the cut + lORIGIN_MODULE="$(echo "${lAI_SOURCE_FILE}" | cut -d / -f4 | cut -d_ -f1)" + elif [[ "${lAI_SOURCE_FILE}" == '/logs/'* ]]; then + lORIGIN_MODULE="$(echo "${lAI_SOURCE_FILE}" | grep -E "[splfq][0-9]{2,3}_.*" -o | cut -d '_' -f1)" + else + lORIGIN_MODULE="$(basename "$(dirname "${lAI_SOURCE_FILE}")" | cut -d_ -f1)" + fi + + print_ln + print_output "[+] Further analysis results for ${ORANGE}${lAI_INPUT_FILE_mod}${GREEN} available in module ${ORANGE}${lORIGIN_MODULE}${NC}" "" "${lAI_SOURCE_FILE}" + if [[ -f "${lAI_LOG_FILE}" ]]; then + print_output "[+] AI analysis results for ${ORANGE}${lAI_INPUT_FILE_mod}${GREEN} available${NC}" "" "${lAI_LOG_FILE//\.md/\.txt}" + local lSECURITY_RATING="" + lSECURITY_RATING=$(grep -h "Security rating: " "${lAI_LOG_FILE}" | tr '.' '\n' | grep -o -E "[0-9]+/10" || true) + if [[ -n "${lSECURITY_RATING}" ]]; then + print_output "[+] AI security rating for ${ORANGE}${lAI_INPUT_FILE_mod}${GREEN}: ${ORANGE}${lSECURITY_RATING}${NC}" "" "${lAI_LOG_FILE//\.md/\.txt}" + fi + fi + + print_ln + ((AI_RESULT_CNT += 1)) + fi + else + print_output "[-] Couldn't find ${ORANGE}$(print_path "${lSCRIPT_PATH_TMP}")${NC}" + fi + fi + + # exit conditions + local lMODULE_RUNTIME=${SECONDS} + adjust_minimum_runtime "${lMODULE_RUNTIME}" + print_output "[*] Current AI module runtime: ${lMODULE_RUNTIME} / min module runtime ${AI_MIN_RUNTIME}" "no_log" + + # if our module runtime is longer as our configured min runtime: + if [[ "${lMODULE_RUNTIME}" -gt "${AI_MIN_RUNTIME}" ]]; then + break + fi + + regenerate_ai_todo_list + # reset the array index to start again with the highest rated entry + lELE_INDEX=-1 + done +} + + +regenerate_ai_todo_list() { + print_output "[*] Regenerate analysis array ..." "no_log" + + if [[ -f "${CSV_DIR}/ai_question.csv" ]]; then + local lGPT_ENTRY_LINE="" + while read -r lGPT_ENTRY_LINE; do + lAI_ANCHOR="$(echo "${lGPT_ENTRY_LINE}" | cut -d ';' -f2)" + sed -i "/${lAI_ANCHOR}/d" "${CSV_DIR}/ai_question.csv.tmp" + done <"${CSV_DIR}/ai_question.csv" + fi + + # reload and resort current state of questions: + readarray -t Q03_LOCALAI_QUESTIONS < <(sort -k 3 -t ';' -r "${CSV_DIR}/ai_question.csv.tmp") +} + +identify_ai_model() { + local lMODEL_LOCALAI="" + local lCNT=1 + while [[ -z "${lMODEL_LOCALAI}" ]]; do + lMODEL_LOCALAI=$(curl --connect-timeout 10 --max-time 30 http://"${LOCAL_AI_IP}":8080/v1/models 2>/dev/null | jq -r .data[].id || true) + [[ -z "${lMODEL_LOCALAI}" ]] && sleep 5 + lCNT=$((lCNT+1)) + [[ "${lCNT}" -gt 5 ]] && break + done + echo "${lMODEL_LOCALAI}" +} + +extract_s16_fct_name() { + local lFUNCTION_NAME="${1:-}" + # extract the function name from the beginning of the first part of the file + # e.g.: bool fct_name(asdf) + local lBINARY_NAME="" + lBINARY_NAME="$(echo "${lSCRIPT_PATH_TMP}" | rev | cut -d '/' -f2 | rev | sed 's/haruspex_//')" + lFUNCTION_NAME="$(strip_color_codes "$(head "${lSCRIPT_PATH_TMP}")")" + lFUNCTION_NAME="$(echo "${lFUNCTION_NAME}" | grep "^bool\|^uint\|^ulong\|^long\|^int\|^undefined\|^void\|^char\|^ssize\|^size" | grep "(.*" || true)" + if [[ -z "${lFUNCTION_NAME}" ]]; then + # e.g.: fct_name(asdf) + lFUNCTION_NAME="$(strip_color_codes "$(head "${lSCRIPT_PATH_TMP}")")" + lFUNCTION_NAME="$(echo "${lFUNCTION_NAME}" | grep "^[a-zA-Z0-9_-]*(.*" || true)" + fi + if [[ -z "${lFUNCTION_NAME}" ]]; then + lFUNCTION_NAME="unknown_${lBINARY_NAME:-NA}" + fi + lFUNCTION_NAME="${lFUNCTION_NAME//\(*}" + lFUNCTION_NAME="${lFUNCTION_NAME//* }" + echo "${lFUNCTION_NAME}" +} + +generate_prompt_script() { + local lSCRIPT_PATH_TMP="${1:-}" + local lSCRIPT_TYPE="${2:-unknown}" + + local lCODE_CONTENT_tmp="" + local lCODE_CONTENT="" + local lPROMPT="" + local lBINARY_NAME="" + lBINARY_NAME=$(basename "${lSCRIPT_PATH_TMP}") + lBINARY_NAME="${lBINARY_NAME//*\/}" + lBINARY_NAME="${lBINARY_NAME//\.log}" + + print_output "[*] Build prompt for ${lSCRIPT_PATH_TMP} - lSCRIPT_NAME: ${lBINARY_NAME} - script type: ${lSCRIPT_TYPE}" "no_log" + + # Read the file content and remove the ASK_AI marker + lCODE_CONTENT_tmp=$(grep -v "ASK_AI" "${lSCRIPT_PATH_TMP}") + # remove REF entries for HTML links + lCODE_CONTENT_tmp=$(sed '/^\[REF\] .*/d' <<< "${lCODE_CONTENT_tmp}") + # remove semgrep identifiers + lCODE_CONTENT_tmp=$(sed 's/\/\/possible issue identified -.*//' <<< "${lCODE_CONTENT_tmp}") + # remove color codes + lCODE_CONTENT_tmp=$(strip_color_codes "${lCODE_CONTENT_tmp}") + # remove shell comments and empty lines: + lCODE_CONTENT=$(sed '/^[[:blank:]]*#/d;s/[[:blank:]]*#.*//' <<< "${lCODE_CONTENT_tmp}") + + lPROMPT="You are an expert software architect. Analyze the provided source code." + lPROMPT+=" The following source code is from a Linux based firmware ${lSCRIPT_TYPE} script called ${lBINARY_NAME}." + lPROMPT+=" Give a short summary of the functionality of the provided code" + lPROMPT+=" Improve the readability of the original source code inline with comments on the functionality. Perform an indepth security and vulnerability analysis of the provided source code. Use all available information for your security and vulnerability analysis. Include security notes inline in the original source code for security relevant areas and for identified vulnerabilities inline to the improved source code. Finally, give a security rating between 1 and 10 points, while 1 point is very insecure and 10 is a good security rating." + lPROMPT+=" Structure your report the following way:" + lPROMPT+=" 1.) Summary of ${lSCRIPT_TYPE} script ${lBINARY_NAME}" + lPROMPT+=" 2.) Optimized source code of ${lSCRIPT_TYPE} script ${lBINARY_NAME} with comments" + lPROMPT+=" 3.) Security overview with identified vulnerabilities" + lPROMPT+=" 4.) Final security rating in the form \"Security rating: /10.\"" + + if [[ "${#lCODE_CONTENT}" -gt "${AI_MAX_CHARS_TO_ANALYSE}" ]]; then + print_output "[*] Limit code of ${lBINARY_NAME}/${lSCRIPT_TYPE} for analysis to ${AI_MAX_CHARS_TO_ANALYSE} characters" "no_log" + fi + lPROMPT="${lPROMPT}\nHere is the code:\n\n${lCODE_CONTENT:0:${AI_MAX_CHARS_TO_ANALYSE}}\n" + + echo -n "${lPROMPT}" | iconv -f ISO-8859-1 -t UTF-8 > "${AI_PROMPT_DIR}/prompt_localai_${lBINARY_NAME}_${lSCRIPT_TYPE}.txt" +} + +generate_prompt_binary() { + local lSCRIPT_PATH_TMP="${1:-}" + + local lARCH="" + local lEND="" + local lKERNELV="" + local lRELRO="" + local lCANARIES="" + local lNX="" + local lSYMBOLS="" + local lCODE_CONTENT_tmp="" + local lCODE_CONTENT="" + local lPROMPT="" + local lBINARY_NAME="" + local lFUNCTION_NAME="" + + lBINARY_NAME="$(echo "${lSCRIPT_PATH_TMP}" | rev | cut -d '/' -f2 | rev | sed 's/haruspex_//')" + lFUNCTION_NAME=$(extract_s16_fct_name "${lSCRIPT_PATH_TMP}") + + print_output "[*] Build prompt for ${lSCRIPT_PATH_TMP} - lBINARY_NAME: ${lBINARY_NAME} - lFUNCTION_NAME: ${lFUNCTION_NAME}" "no_log" + + # extract further binary details from the current EMBA report + if [[ -f "${P99_LOG}" ]]; then + lARCH="$(grep Statistics "${P99_LOG}" | cut -d ':' -f2)" + lEND="$(grep Statistics "${P99_LOG}" | cut -d ':' -f3)" + fi + if [[ -f "${S24_CSV_LOG}" ]]; then + lKERNELV=$(tail -n +2 "${S24_CSV_LOG}" | cut -d ';' -f2 | sort -u | head -1) + fi + if [[ -f "${S12_CSV_LOG}" ]]; then + lRELRO=$(grep "/${lBINARY_NAME}$" "${S12_CSV_LOG}" | cut -d ';' -f1 | sort -u | head -1) + lCANARIES=$(grep "/${lBINARY_NAME}$" "${S12_CSV_LOG}" | cut -d ';' -f2 | sort -u | head -1) + lNX=$(grep "/${lBINARY_NAME}$" "${S12_CSV_LOG}" | cut -d ';' -f3 | sort -u | head -1) + lSYMBOLS=$(grep "/${lBINARY_NAME}$" "${S12_CSV_LOG}" | cut -d ';' -f7 | sort -u | head -1) + fi + + # Read the file content and remove the ASK_AI marker + lCODE_CONTENT_tmp=$(grep -v "ASK_AI" "${lSCRIPT_PATH_TMP}") + # remove REF entries for HTML links + lCODE_CONTENT_tmp=$(sed '/^\[REF\] .*/d' <<< "${lCODE_CONTENT_tmp}") + # remove semgrep identifiers + lCODE_CONTENT_tmp=$(sed 's/\/\/possible issue identified -.*//' <<< "${lCODE_CONTENT_tmp}") + # remove color codes + lCODE_CONTENT=$(strip_color_codes "${lCODE_CONTENT_tmp}") + + lPROMPT="You are an expert software architect. Analyze the provided source code." + lPROMPT+=" The following source code is from a Linux based firmware binary called ${lBINARY_NAME}." + lPROMPT+=" It was automatically generated with the reverse engineering framework Ghidra." + if [[ -n "${lARCH}" ]]; then + lPROMPT+=" The architecture of the firmware is ${lARCH}." + if [[ -n "${lEND}" ]]; then + lPROMPT+=" The endianess of the firmware is ${lEND}." + fi + fi + if [[ -n "${lKERNELV}" ]]; then + lPROMPT+=" The original device was running a Linux kernel in version ${lKERNELV}." + fi + if [[ -n "${lRELRO}" ]]; then + lPROMPT+=" The original binary has the following RELRO security setting: ${lRELRO}." + fi + if [[ -n "${lCANARIES}" ]]; then + lPROMPT+=" The original binary has the following Stack Canaries security setting: ${lCANARIES}." + fi + if [[ -n "${lNX}" ]]; then + lPROMPT+=" The original binary has the following NX security setting: ${lNX}." + fi + if [[ -n "${lSYMBOLS}" ]]; then + if [[ "${lSYMBOLS}" == "No Symbols" ]]; then + lPROMPT+=" The original binary has no symbols included." + else + lPROMPT+=" The original binary has symbols included." + fi + fi + + lPROMPT+=" Give a short summary of the functionality of the provided code with the functionname ${lFUNCTION_NAME}." + lPROMPT+=" Improve the readability of the original source code inline with comments on the functionality and useful variable names. Perform an indepth security and vulnerability analysis of the provided source code. Use all available information for your security and vulnerability analysis. Include security notes inline in the original source code for security relevant areas and for identified vulnerabilities inline to the improved source code. The variable names of the provided source code should be always renamed for better readability to support manual verification and further manual analysis. Finally, give a security rating between 1 and 10 points, while 1 point is very insecure and 10 is a good security rating." + lPROMPT+=" Structure your report the following way:" + lPROMPT+=" 1.) Summary of function ${lFUNCTION_NAME}" + lPROMPT+=" 2.) Optimized source code of function ${lFUNCTION_NAME} with renamed variables and comments" + lPROMPT+=" 3.) Security overview with identified vulnerabilities" + lPROMPT+=" 4.) Final security rating in the form \"Security rating: /10.\"" + + if [[ "${#lCODE_CONTENT}" -gt "${AI_MAX_CHARS_TO_ANALYSE}" ]]; then + print_output "[*] Limit code of ${lBINARY_NAME}/${lFUNCTION_NAME} for analysis to ${AI_MAX_CHARS_TO_ANALYSE} characters" + fi + # limit the code to analyse to 5000 characters + lPROMPT="${lPROMPT}\nHere is the code:\n\n${lCODE_CONTENT:0:${AI_MAX_CHARS_TO_ANALYSE}}\n" + + # print_output "[*] Wrting prompt for ${lSCRIPT_PATH_TMP} - ${lPROMPT} to ${AI_PROMPT_DIR}/prompt_localai_${lBINARY_NAME}_${lFUNCTION_NAME}.txt" + + echo -n "${lPROMPT}" | iconv -f ISO-8859-1 -t UTF-8 > "${AI_PROMPT_DIR}/prompt_localai_${lBINARY_NAME}_${lFUNCTION_NAME}.txt" +} diff --git a/modules/S15_radare_decompile_checks.sh b/modules/S15_radare_decompile_checks.sh index 34f91bc6..a6338a5e 100755 --- a/modules/S15_radare_decompile_checks.sh +++ b/modules/S15_radare_decompile_checks.sh @@ -217,14 +217,14 @@ radare_decomp_print_top10_statistics() { if [[ -f "${lR2_BIN_LOG_PATH}" ]]; then write_link "${lR2_BIN_LOG_PATH}" # FIXME - if [[ "${GPT_OPTION}" -gt 0 ]]; then + if [[ "${AI_OPTION}" -gt 0 ]]; then print_output "[*] Asking OpenAI chatbot about ${lR2_BIN_LOG_PATH}" lGPT_ANCHOR_="$(openssl rand -hex 8)" # "${GPT_INPUT_FILE_}" "${lGPT_ANCHOR_}" "${GPT_PRIO_}" "${GPT_QUESTION_}" "${GPT_OUTPUT_FILE_}" "cost=$GPT_TOKENS_" "${GPT_RESPONSE_}" - write_csv_gpt_tmp "${lR2_BIN_LOG_PATH}" "${lGPT_ANCHOR_}" "${lGPT_PRIO}" "Can you give me a side by side desciption of the following code in a table, where on the left is the code and on the right the desciption. And please use proper spacing and | to make it terminal friendly:" "${lR2_BIN_LOG_PATH}" "" "" + write_csv_AI_tmp "${lR2_BIN_LOG_PATH}" "${lGPT_ANCHOR_}" "${lGPT_PRIO}" "Can you give me a side by side desciption of the following code in a table, where on the left is the code and on the right the desciption. And please use proper spacing and | to make it terminal friendly:" "${lR2_BIN_LOG_PATH}" "" "" # add ChatGPT link printf '%s\n\n' "" >>"${lR2_BIN_LOG_PATH}" - write_anchor_gpt "${lGPT_ANCHOR_}" "${lR2_BIN_LOG_PATH}" + write_anchor_AI "${lGPT_ANCHOR_}" "${lR2_BIN_LOG_PATH}" fi fi done diff --git a/modules/S16_ghidra_decompile_checks.sh b/modules/S16_ghidra_decompile_checks.sh index 74a44779..53a6942a 100755 --- a/modules/S16_ghidra_decompile_checks.sh +++ b/modules/S16_ghidra_decompile_checks.sh @@ -18,10 +18,9 @@ # The generated source code is further analysed with semgrep and the rules provided by 0xdea # (https://github.com/0xdea/semgrep-rules) -# set DEBUG to 1 to enable further print statements -export DEBUG=0 S16_ghidra_decompile_checks() { + # export MAX_EXT_CHECK_BINS=3 module_log_init "${FUNCNAME[0]}" module_title "Check decompiled binary source code for vulnerabilities" pre_module_reporter "${FUNCNAME[0]}" @@ -47,6 +46,9 @@ S16_ghidra_decompile_checks() { local lNAME="" local lBINS_CHECKED_ARR=() + # the blacklist elements are handled as regex in the form *pattern* + local lBIN_BLACKLIST_ARR=("libc") + if [[ "${FULL_TEST}" -ne 1 ]]; then # we need to wait in default mode for the results of S13 and S14 module_wait "S13_weak_func_check" @@ -64,6 +66,16 @@ S16_ghidra_decompile_checks() { for lBIN_TO_CHECK in "${lBINARIES_ARR[@]}"; do lNAME=$(basename "${lBIN_TO_CHECK}" 2>/dev/null) + if [[ "${FULL_TEST}" -ne 1 ]]; then + # first of all we check our binary name against our blacklist + # the blacklist are handled as regex in the form *pattern* + for lBIN_BLACKLIST in "${lBIN_BLACKLIST_ARR[@]}"; do + if [[ "${lNAME}" == *"${lBIN_BLACKLIST}"* ]]; then + continue 2 + fi + done + fi + if [[ -f "${BASE_LINUX_FILES}" ]]; then # if we have the base linux config file we only test non known Linux binaries # with this we do not waste too much time on open source Linux stuff @@ -379,14 +391,18 @@ s16_semgrep_logger() { # GPT integration lGPT_ANCHOR="$(openssl rand -hex 8)" - if [[ -f "${BASE_LINUX_FILES}" ]]; then - # if we have the base linux config file we are checking it: - if ! grep -E -q "^${lNAME}$" "${BASE_LINUX_FILES}" 2>/dev/null; then - lGPT_PRIO=$((lGPT_PRIO + 1)) - fi + # if we have some semgrep results for this function we increase our priority + if grep -q "possible issue identified - semgrep" "${lC_SRC_FCT}"; then + local lSEMGREP_CNT=0 + lSEMGREP_CNT=$(grep -c "possible issue identified - semgrep" "${lC_SRC_FCT}" || echo 0) + lGPT_PRIO=$((lGPT_PRIO + lSEMGREP_CNT)) fi - write_csv_gpt_tmp "${LOG_PATH_MODULE}/haruspex_${lNAME}/${lHARUSPEX_FILE_NAME}" "${lGPT_ANCHOR}" "${lGPT_PRIO}" "${GPT_QUESTION}" "${LOG_PATH_MODULE}/haruspex_${lNAME}/${lHARUSPEX_FILE_NAME}" "" "" - write_anchor_gpt "${lGPT_ANCHOR}" "${LOG_PATH_MODULE}"/haruspex_"${lNAME}"/"${lHARUSPEX_FILE_NAME}" + # if it is not a default (known) linux file we increase the priority + if ! grep -E -q "^${lNAME}$" "${BASE_LINUX_FILES}" 2>/dev/null; then + lGPT_PRIO=$((lGPT_PRIO + 1)) + fi + write_csv_AI_tmp "${LOG_PATH_MODULE}/haruspex_${lNAME}/${lHARUSPEX_FILE_NAME}" "${lGPT_ANCHOR}" "${lGPT_PRIO}" "${GPT_QUESTION}" "${LOG_PATH_MODULE}/haruspex_${lNAME}/${lHARUSPEX_FILE_NAME}" "" "" + write_anchor_AI "${lGPT_ANCHOR}" "${LOG_PATH_MODULE}"/haruspex_"${lNAME}"/"${lHARUSPEX_FILE_NAME}" if [[ -f "${lSEMGREPLOG_TMP}" ]]; then cat "${lSEMGREPLOG_TMP}" >>"${lSEMGREPLOG_TXT}" diff --git a/modules/S19_apk_check.sh b/modules/S19_apk_check.sh index 92b30036..eef633ff 100755 --- a/modules/S19_apk_check.sh +++ b/modules/S19_apk_check.sh @@ -80,7 +80,7 @@ apk_checker_helper() { lAPK_NAME=$(basename -s .apk "${lAPK_FILE}") print_output "[*] Testing Android apk with APKHunt - ${ORANGE}$(print_path "${lAPK_FILE}")${NC}" - go run "${EXT_DIR}"/APKHunt/apkhunt.go -p "${lAPK_FILE}" -l 2>&1 | tee -a "${LOG_PATH_MODULE}/APKHunt-${lAPK_NAME}.txt" + go run "${EXT_DIR}"/APKHunt/apkhunt.go -p "${lAPK_FILE}" -l 2>&1 | tee -a "${LOG_PATH_MODULE}/APKHunt-${lAPK_NAME}.txt" || print_error "[-] Failed to analyze apk file ${lAPK_FILE} with apkhunt" if [[ -f "${LOG_PATH_MODULE}/APKHunt-${lAPK_NAME}.txt" ]]; then lAPK_ISSUES=$(grep -c -E "^[0-9]+:" "${LOG_PATH_MODULE}/APKHunt-${lAPK_NAME}.txt" || true) diff --git a/modules/S20_shell_check.sh b/modules/S20_shell_check.sh index 34b0ad87..f977cde9 100755 --- a/modules/S20_shell_check.sh +++ b/modules/S20_shell_check.sh @@ -115,8 +115,9 @@ S20_shell_check() { s20_eval_script_check() { local lSH_SCRIPTS_ARR=("${@}") local lSH_SCRIPT="" - local lGPT_PRIO_=3 - local lGPT_ANCHOR_="" + # we rate eval usages higher compared to other shellcheck issues + local lGPT_PRIO_=4 + local lAI_ANCHOR_="" local lEVAL_RESULTS=0 local lSH_SCRIPT_NAME="" @@ -133,19 +134,19 @@ s20_eval_script_check() { sed -i -r "s/.*eval\ .*/\x1b[32m&\x1b[0m/" "${lSHELL_LOG}" print_output "[+] Found ${ORANGE}eval${GREEN} usage in ${ORANGE}${lSH_SCRIPT_NAME}${NC}" "" "${lSHELL_LOG}" - if [[ "${GPT_OPTION}" -gt 0 ]]; then - lGPT_ANCHOR_="$(openssl rand -hex 8)" + if [[ "${AI_OPTION}" -gt 0 ]]; then + lAI_ANCHOR_="$(openssl rand -hex 8)" if [[ -f "${BASE_LINUX_FILES}" ]]; then # if we have the base linux config file we are checking it: if ! grep -E -q "^${lSH_SCRIPT_NAME}$" "${BASE_LINUX_FILES}" 2>/dev/null; then lGPT_PRIO_=$((lGPT_PRIO_ + 1)) fi fi - # "${GPT_INPUT_FILE_}" "${lGPT_ANCHOR_}" "GPT-Prio-$lGPT_PRIO_" "${GPT_QUESTION_}" "${GPT_OUTPUT_FILE_}" "cost=$GPT_TOKENS_" "${GPT_RESPONSE_}" - write_csv_gpt_tmp "$(cut_path "${lSH_SCRIPT/;*/}")" "${lGPT_ANCHOR_}" "${lGPT_PRIO_}" "${GPT_QUESTION}" "${lSHELL_LOG}" "" "" + # "${GPT_INPUT_FILE_}" "${lAI_ANCHOR_}" "GPT-Prio-$lGPT_PRIO_" "${GPT_QUESTION_}" "${GPT_OUTPUT_FILE_}" "cost=$GPT_TOKENS_" "${GPT_RESPONSE_}" + write_csv_AI_tmp "$(cut_path "${lSH_SCRIPT/;*/}")" "${lAI_ANCHOR_}" "${lGPT_PRIO_}" "${GPT_QUESTION}" "${lSHELL_LOG}" "" "" # add ChatGPT link printf '%s\n\n' "" >>"${lSHELL_LOG}" - write_anchor_gpt "${lGPT_ANCHOR_}" "${lSHELL_LOG}" + write_anchor_AI "${lAI_ANCHOR_}" "${lSHELL_LOG}" fi fi done @@ -173,11 +174,19 @@ s20_reporter() { local lSH_SCRIPT_="${2:-}" local lSHELL_LOG="${3:-}" local lGPT_PRIO_=2 - local lGPT_ANCHOR_="" + local lAI_ANCHOR_="" local lSH_NAME="" lSH_NAME=$(basename "${lSH_SCRIPT_}" 2>/dev/null | sed -e 's/:/_/g') + local lS20_SOURCE_DIR="${LOG_PATH_MODULE}/sh_sources" if [[ "${lVULNS}" -ne 0 ]]; then + [[ ! -d "${lS20_SOURCE_DIR}" ]] && mkdir -p "${lS20_SOURCE_DIR}" + + # safe the sources and link it in the lSHELL_LOG + write_log "" "${lSHELL_LOG}" + write_log "[*] Source file ${ORANGE}${lSH_NAME}${NC}" "${lSHELL_LOG}" + copy_and_link_file "${lSH_SCRIPT_}" "${lS20_SOURCE_DIR}/${lSH_NAME}.log" "${lSHELL_LOG}" + # check if this is common linux file: local lCOMMON_FILES_FOUND="" local lCFF="" @@ -198,15 +207,21 @@ s20_reporter() { else print_output "[+] Found ${ORANGE}${lVULNS} issues${GREEN} in script ${lCOMMON_FILES_FOUND}:${NC} $(print_path "${lSH_SCRIPT_}")" "" "${lSHELL_LOG}" fi + # scripts with eval usage are higher rated + if grep -q "eval" "${lS20_SOURCE_DIR}/${lSH_NAME}.log"; then + lGPT_PRIO_=$((lGPT_PRIO_ + 2)) + fi write_csv_log "$(print_path "${lSH_SCRIPT_}")" "${lVULNS}" "${lCFF}" "NA" - if [[ "${GPT_OPTION}" -gt 0 ]]; then - lGPT_ANCHOR_="$(openssl rand -hex 8)" - # "${GPT_INPUT_FILE_}" "${lGPT_ANCHOR_}" "GPT-Prio-$lGPT_PRIO_" "${GPT_QUESTION_}" "${GPT_OUTPUT_FILE_}" "cost=$GPT_TOKENS_" "${GPT_RESPONSE_}" - write_csv_gpt_tmp "$(cut_path "${lSH_SCRIPT_}")" "${lGPT_ANCHOR_}" "${lGPT_PRIO_}" "${GPT_QUESTION}" "${lSHELL_LOG}" "" "" - # add ChatGPT link + if [[ "${AI_OPTION}" -gt 0 ]]; then + lAI_ANCHOR_="$(openssl rand -hex 8)" + # "${GPT_INPUT_FILE_}" "${lAI_ANCHOR_}" "GPT-Prio-$lGPT_PRIO_" "${GPT_QUESTION_}" "${GPT_OUTPUT_FILE_}" "cost=$GPT_TOKENS_" "${GPT_RESPONSE_}" + write_csv_AI_tmp "${lS20_SOURCE_DIR}/${lSH_NAME}.log" "${lAI_ANCHOR_}" "${lGPT_PRIO_}" "${GPT_QUESTION}" "${lSHELL_LOG}" "" "" + # add AI link printf '%s\n\n' "" >>"${lSHELL_LOG}" - write_anchor_gpt "${lGPT_ANCHOR_}" "${lSHELL_LOG}" + write_anchor_AI "${lAI_ANCHOR_}" "${lSHELL_LOG}" + printf '%s\n\n' "" >>"${lS20_SOURCE_DIR}/${lSH_NAME}.log" + write_anchor_AI "${lAI_ANCHOR_}" "${lS20_SOURCE_DIR}/${lSH_NAME}.log" fi echo "${lVULNS}" >>"${TMP_DIR}"/S20_VULNS.tmp diff --git a/modules/S21_python_check.sh b/modules/S21_python_check.sh index a1199dcc..5ab3ddfb 100755 --- a/modules/S21_python_check.sh +++ b/modules/S21_python_check.sh @@ -92,7 +92,8 @@ s21_script_bandit() { local lPY_LOG="" local lVULNS="" local lGPT_PRIO_=2 - local lGPT_ANCHOR_="" + local lAI_ANCHOR="" + local lS21_SOURCE_DIR="${LOG_PATH_MODULE}/py_sources" lSCRIPT_NAME=$(basename "${lPY_SCRIPT_}" 2>/dev/null | sed -e 's/:/_/g') lPY_LOG="${LOG_PATH_MODULE}""/bandit_""${lSCRIPT_NAME}"".txt" @@ -100,6 +101,13 @@ s21_script_bandit() { lVULNS=$(grep -c ">> Issue: " "${lPY_LOG}" 2>/dev/null || true) if [[ "${lVULNS}" -ne 0 ]]; then + [[ ! -d "${lS21_SOURCE_DIR}" ]] && mkdir -p "${lS21_SOURCE_DIR}" + + # safe the sources and link it in the lSHELL_LOG + write_log "" "${lPY_LOG}" + write_log "[*] Source file ${ORANGE}${lSCRIPT_NAME}${NC}" "${lPY_LOG}" + copy_and_link_file "${lPY_SCRIPT_}" "${lS21_SOURCE_DIR}/${lSCRIPT_NAME}.log" "${lPY_LOG}" + # check if this is common linux file: local lCOMMON_FILES_FOUND="" local lCFF="" @@ -115,26 +123,30 @@ s21_script_bandit() { lCFF="NA" fi if [[ "${lVULNS}" -gt 20 ]]; then - print_output "[+] Found ""${RED}""${lVULNS}"" issues""${GREEN}"" in script ""${lCOMMON_FILES_FOUND}"":""${NC}"" ""$(print_path "${lPY_SCRIPT_}")" "" "${lPY_LOG}" + print_output "[+] Found ${RED}${lVULNS} issues${GREEN} in script ${lCOMMON_FILES_FOUND}:${NC} $(print_path "${lPY_SCRIPT_}")" "" "${lPY_LOG}" lGPT_PRIO_=3 else - print_output "[+] Found ""${ORANGE}""${lVULNS}"" issues""${GREEN}"" in script ""${lCOMMON_FILES_FOUND}"":""${NC}"" ""$(print_path "${lPY_SCRIPT_}")" "" "${lPY_LOG}" + print_output "[+] Found ${ORANGE}${lVULNS} issues${GREEN} in script ${lCOMMON_FILES_FOUND}:${NC} $(print_path "${lPY_SCRIPT_}")" "" "${lPY_LOG}" fi write_csv_log "$(print_path "${lPY_SCRIPT_}")" "${lVULNS}" "${lCFF}" "NA" - if [[ "${GPT_OPTION}" -gt 0 ]]; then - lGPT_ANCHOR_="$(openssl rand -hex 8)" - if [[ -f "${BASE_LINUX_FILES}" ]]; then - # if we have the base linux config file we are checking it: - if ! grep -E -q "^$(basename "${lPY_SCRIPT_}")$" "${BASE_LINUX_FILES}" 2>/dev/null; then - lGPT_PRIO_=$((lGPT_PRIO_ + 1)) - fi + if [[ "${AI_OPTION}" -gt 0 ]]; then + lAI_ANCHOR="$(openssl rand -hex 8)" + # if we have some default python packages we rate them lower + if [[ "${lPY_SCRIPT_}" == *"dist-packages"* || "${lPY_SCRIPT_}" == *"site-packages"* ]]; then + lGPT_PRIO_=$((lGPT_PRIO_ - 1)) + elif ! grep -E -q "^$(basename "${lPY_SCRIPT_}")$" "${BASE_LINUX_FILES}" 2>/dev/null; then + # if no entry in our BASE_LINUX_FILES config we do not have a default + # linux file and we rate it higher for faster AI test + lGPT_PRIO_=$((lGPT_PRIO_ + 1)) fi - # "${GPT_INPUT_FILE_}" "${lGPT_ANCHOR_}" "${lGPT_PRIO_}" "${GPT_QUESTION_}" "${GPT_OUTPUT_FILE_}" "cost=$GPT_TOKENS_" "${GPT_RESPONSE_}" - write_csv_gpt_tmp "$(cut_path "${lPY_SCRIPT_}")" "${lGPT_ANCHOR_}" "${lGPT_PRIO_}" "${GPT_QUESTION}" "${lPY_LOG}" "" "" + # "${GPT_INPUT_FILE_}" "${lAI_ANCHOR}" "${lGPT_PRIO_}" "${GPT_QUESTION_}" "${GPT_OUTPUT_FILE_}" "cost=$GPT_TOKENS_" "${GPT_RESPONSE_}" + write_csv_AI_tmp "${lS21_SOURCE_DIR}/${lSCRIPT_NAME}.log" "${lAI_ANCHOR}" "${lGPT_PRIO_}" "${GPT_QUESTION}" "${lPY_LOG}" "" "" # add ChatGPT link to output file printf '%s\n\n' "" >>"${lPY_LOG}" - write_anchor_gpt "${lGPT_ANCHOR_}" "${lPY_LOG}" + write_anchor_AI "${lAI_ANCHOR}" "${lPY_LOG}" + printf '%s\n\n' "" >>"${lS21_SOURCE_DIR}/${lSCRIPT_NAME}.log" + write_anchor_AI "${lAI_ANCHOR}" "${lS21_SOURCE_DIR}/${lSCRIPT_NAME}.log" fi echo "${lVULNS}" >>"${TMP_DIR}"/S21_VULNS.tmp fi diff --git a/modules/S22_php_check.sh b/modules/S22_php_check.sh index 92cd87ca..ab5ef21e 100755 --- a/modules/S22_php_check.sh +++ b/modules/S22_php_check.sh @@ -84,6 +84,8 @@ s22_vuln_check_semgrep() { local lS22_SEMGREP_SCRIPTS="" local lSEMG_SOURCE_NOTE="" + local lS22_SOURCE_DIR="${LOG_PATH_MODULE}/php_sources" + # multiple output options would be nice. Currently we have the xml output to parse it easily for getting the line number of the issue # but this output is not very beautiful to show in the report. semgrep --disable-version-check --metrics=off --junit-xml --config "${EXT_DIR}"/semgrep-rules/php "${LOG_DIR}"/firmware/ &>"${lPHP_SEMGREP_LOG}" || true @@ -111,10 +113,12 @@ s22_vuln_check_semgrep() { local lSEMG_SOURCE_FILE="" local lSEMG_SOURCE_FILE_NAME="" local lSEMG_LINE_NR="" - local lGPT_PRIO_=4 - local lGPT_ANCHOR_="" + # we set the priority higher as the semgrep analysis on scripts is more reliable + local lGPT_PRIO_=5 + local lAI_ANCHOR="" - ! [[ -d "${LOG_PATH_MODULE}"/semgrep_sources/ ]] && mkdir "${LOG_PATH_MODULE}"/semgrep_sources/ + [[ ! -d "${LOG_PATH_MODULE}"/semgrep_sources/ ]] && mkdir "${LOG_PATH_MODULE}"/semgrep_sources/ + [[ ! -d "${lS22_SOURCE_DIR}" ]] && mkdir -p "${lS22_SOURCE_DIR}" lSEMG_ISSUE_NAME=$(echo "${lSEMG_SOURCE_NOTE}" | tr ' ' '\n' | grep "^name=") lSEMG_ISSUE_NAME="${lSEMG_ISSUE_NAME/name=\"/}" @@ -122,29 +126,39 @@ s22_vuln_check_semgrep() { lSEMG_SOURCE_FILE=$(echo "${lSEMG_SOURCE_NOTE}" | tr ' ' '\n' | grep "^file=") lSEMG_SOURCE_FILE="${lSEMG_SOURCE_FILE/file=\"/}" lSEMG_SOURCE_FILE_NAME=$(basename "${lSEMG_SOURCE_FILE}") + local lSEMG_SOURCE_FILE_log="${LOG_PATH_MODULE}/semgrep_sources/${lSEMG_SOURCE_FILE_NAME}.log" - [[ -f "${lSEMG_SOURCE_FILE}" && ! -f "${LOG_PATH_MODULE}"/semgrep_sources/"${lSEMG_SOURCE_FILE_NAME}".log ]] && cp "${lSEMG_SOURCE_FILE}" "${LOG_PATH_MODULE}"/semgrep_sources/"${lSEMG_SOURCE_FILE_NAME}".log + [[ -f "${lSEMG_SOURCE_FILE}" && ! -f "${LOG_PATH_MODULE}"/semgrep_sources/"${lSEMG_SOURCE_FILE_NAME}".log ]] && cp "${lSEMG_SOURCE_FILE}" "${lSEMG_SOURCE_FILE_log}" lSEMG_LINE_NR=$(echo "${lSEMG_SOURCE_NOTE}" | tr ' ' '\n' | grep "^line=") lSEMG_LINE_NR="${lSEMG_LINE_NR/line=\"/}" - sed -i -r "${lSEMG_LINE_NR}s/.*/\x1b[32m&\x1b[0m/" "${LOG_PATH_MODULE}"/semgrep_sources/"${lSEMG_SOURCE_FILE_NAME}".log || true - print_output "[+] Found possible PHP vulnerability ${ORANGE}${lSEMG_ISSUE_NAME}${GREEN} in ${ORANGE}${lSEMG_SOURCE_FILE_NAME}${GREEN}" "" "${LOG_PATH_MODULE}/semgrep_sources/${lSEMG_SOURCE_FILE_NAME}.log" + # make the identified line of code green and add a note + sed -i -r "${lSEMG_LINE_NR}s/.*/\x1b[32m&\x1b[0m \/\/possible issue identified - semgrep/" "${lSEMG_SOURCE_FILE_log}" || true + + print_output "[+] Found possible PHP vulnerability ${ORANGE}${lSEMG_ISSUE_NAME}${GREEN} in ${ORANGE}${lSEMG_SOURCE_FILE_NAME}${GREEN}" "" "${lSEMG_SOURCE_FILE_log}" write_csv_log "${lSEMG_SOURCE_FILE}" "${lSEMG_ISSUE_NAME}" "semgrep" "unknown" - if [[ "${GPT_OPTION}" -gt 0 ]]; then - lGPT_ANCHOR_="$(openssl rand -hex 8)" - if [[ -f "${BASE_LINUX_FILES}" ]]; then - # if we have the base linux config file we are checking it: - if ! grep -E -q "^${lSEMG_SOURCE_FILE_NAME}$" "${BASE_LINUX_FILES}" 2>/dev/null; then - lGPT_PRIO_=$((lGPT_PRIO_ + 1)) - fi + if [[ "${AI_OPTION}" -gt 0 ]]; then + lAI_ANCHOR="$(openssl rand -hex 8)" + if ! grep -E -q "^${lSEMG_SOURCE_FILE_NAME}$" "${BASE_LINUX_FILES}" 2>/dev/null; then + lGPT_PRIO_=$((lGPT_PRIO_ + 1)) + else + # on well known files we lower the priority + lGPT_PRIO_=$((lGPT_PRIO_ - 2)) fi - # "${GPT_INPUT_FILE_}" "${lGPT_ANCHOR_}" "${lGPT_PRIO_}" "${GPT_QUESTION_}" "${GPT_OUTPUT_FILE_}" "cost=$GPT_TOKENS_" "${GPT_RESPONSE_}" - write_csv_gpt_tmp "$(cut_path "${lSEMG_SOURCE_FILE}")" "${lGPT_ANCHOR_}" "${lGPT_PRIO_}" "${GPT_QUESTION} And I think there might be something in line ${lSEMG_LINE_NR}" "${LOG_PATH_MODULE}/semgrep_sources/${lSEMG_SOURCE_FILE_NAME}.log" "" "" + # if we have some semgrep results for this function we increase our priority + if grep -q "possible issue identified - semgrep" "${lSEMG_SOURCE_FILE_log}"; then + local lSEMGREP_CNT=0 + lSEMGREP_CNT=$(grep -c "possible issue identified - semgrep" "${lSEMG_SOURCE_FILE_log}" || echo 0) + lGPT_PRIO_=$((lGPT_PRIO_ + lSEMGREP_CNT)) + fi + + # "${GPT_INPUT_FILE_}" "${lAI_ANCHOR}" "${lGPT_PRIO_}" "${GPT_QUESTION_}" "${GPT_OUTPUT_FILE_}" "cost=$GPT_TOKENS_" "${GPT_RESPONSE_}" + write_csv_AI_tmp "${lSEMG_SOURCE_FILE_log}" "${lAI_ANCHOR}" "${lGPT_PRIO_}" "${GPT_QUESTION} And I think there might be something in line ${lSEMG_LINE_NR}" "${lSEMG_SOURCE_FILE_log}" "" "" # add ChatGPT link - printf '%s\n\n' "" >>"${LOG_PATH_MODULE}/semgrep_sources/${lSEMG_SOURCE_FILE_NAME}.log" - write_anchor_gpt "${lGPT_ANCHOR_}" "${LOG_PATH_MODULE}/semgrep_sources/${lSEMG_SOURCE_FILE_NAME}.log" + printf '%s\n\n' "" >>"${lSEMG_SOURCE_FILE_log}" + write_anchor_AI "${lAI_ANCHOR}" "${lSEMG_SOURCE_FILE_log}" fi done fi @@ -193,6 +207,8 @@ s22_vuln_check() { return fi + local lS22_SOURCE_DIR="${LOG_PATH_MODULE}/php_sources" + local lPHP_SCRIPT_NAME="" local lVULNS=0 local lTOTAL_MEMORY=0 @@ -201,7 +217,7 @@ s22_vuln_check() { local lMEM_LIMIT=$(("${lTOTAL_MEMORY}" / 2)) lPHP_SCRIPT_NAME=$(basename "${lPHP_SCRIPT_}" 2>/dev/null | sed -e 's/:/_/g') - local lPHP_LOG="${LOG_PATH_MODULE}""/php_vuln_""${lPHP_SCRIPT_NAME}""-${RANDOM}.txt" + local lPHP_LOG="${LOG_PATH_MODULE}/php_vuln_${lPHP_SCRIPT_NAME}-${RANDOM}.txt" ulimit -Sv "${lMEM_LIMIT}" "${EXT_DIR}"/progpilot "${lPHP_SCRIPT_}" >>"${lPHP_LOG}" 2>&1 || true @@ -209,8 +225,9 @@ s22_vuln_check() { lVULNS=$(grep -c "vuln_name" "${lPHP_LOG}" 2>/dev/null || true) local lGPT_PRIO_=4 - local lGPT_ANCHOR_="" + local lAI_ANCHOR="" if [[ "${lVULNS}" -gt 0 ]]; then + [[ ! -d "${lS22_SOURCE_DIR}" ]] && mkdir -p "${lS22_SOURCE_DIR}" # check if this is common linux file: local lCOMMON_FILES_FOUND="" local lCFF="" @@ -227,15 +244,25 @@ s22_vuln_check() { lCFF="NA" fi print_output "[+] Found ${ORANGE}${lVULNS} vulnerabilities${GREEN} in php file: ${ORANGE}$(print_path "${lPHP_SCRIPT_}")${GREEN}${lCOMMON_FILES_FOUND}${NC}" "" "${lPHP_LOG}" + + # safe the sources and link it in the lSHELL_LOG + write_log "" "${lPHP_LOG}" + write_log "[*] Source file ${ORANGE}${lPHP_SCRIPT_NAME}${NC}" "${lPHP_LOG}" + copy_and_link_file "${lPHP_SCRIPT_}" "${lS22_SOURCE_DIR}/${lPHP_SCRIPT_NAME}.log" "${lPHP_LOG}" + write_csv_log "${lPHP_SCRIPT_}" "TODO" "progpilot" "${lCFF}" - if [[ "${GPT_OPTION}" -gt 0 ]]; then - lGPT_ANCHOR_="$(openssl rand -hex 8)" - # "${GPT_INPUT_FILE_}" "${lGPT_ANCHOR_}" "GPT-Prio-$lGPT_PRIO_" "${GPT_QUESTION_}" "${GPT_OUTPUT_FILE_}" "cost=$GPT_TOKENS_" "${GPT_RESPONSE_}" - write_csv_gpt_tmp "$(cut_path "${lPHP_SCRIPT_}")" "${lGPT_ANCHOR_}" "${lGPT_PRIO_}" "${GPT_QUESTION}" "${TMP_DIR}/S22_VULNS.tmp" "" "" - # add ChatGPT link - printf '%s\n\n' "" >>"${TMP_DIR}"/S22_VULNS.tmp - write_anchor_gpt "${lGPT_ANCHOR_}" "${TMP_DIR}"/S22_VULNS.tmp + if [[ "${AI_OPTION}" -gt 0 ]]; then + lAI_ANCHOR="$(openssl rand -hex 8)" + lGPT_PRIO_=$((lGPT_PRIO_ + lVULNS)) + # "${GPT_INPUT_FILE_}" "${lAI_ANCHOR}" "GPT-Prio-$lGPT_PRIO_" "${GPT_QUESTION_}" "${GPT_OUTPUT_FILE_}" "cost=$GPT_TOKENS_" "${GPT_RESPONSE_}" + write_csv_AI_tmp "${lS22_SOURCE_DIR}/${lPHP_SCRIPT_NAME}.log" "${lAI_ANCHOR}" "${lGPT_PRIO_}" "${GPT_QUESTION}" "${lPHP_SCRIPT_}" "" "" + + # add AI link + printf '%s\n\n' "" >>"${lPHP_SCRIPT_}" + write_anchor_AI "${lAI_ANCHOR}" "${lPHP_SCRIPT_}" + printf '%s\n\n' "" >>"${lS22_SOURCE_DIR}/${lPHP_SCRIPT_NAME}.log" + write_anchor_AI "${lAI_ANCHOR}" "${lS22_SOURCE_DIR}/${lPHP_SCRIPT_NAME}.log" fi echo "${lVULNS}" >>"${TMP_DIR}"/S22_VULNS.tmp else diff --git a/modules/S23_lua_check.sh b/modules/S23_lua_check.sh index bcdffc0b..f4fb5380 100755 --- a/modules/S23_lua_check.sh +++ b/modules/S23_lua_check.sh @@ -67,12 +67,9 @@ S23_lua_check() { # extract the not 0 results of the vulnerabilities lS23_LUA_VULNS=$(cut -d ';' -f 3 "${S23_CSV_LOG}" 2>/dev/null | tail +2 | grep -v -c "^0$" || true) - if [[ "${lS23_LUA_VULNS}" -gt 0 ]]; then - print_ln + if [[ "${lS23_LUA_VULNS}" -gt 0 || "${lS23_LUA_ISSUES}" -gt 0 ]]; then + sub_module_title "LUA Security results overview" print_output "[+] Found ${ORANGE}${lS23_LUA_VULNS} possible security issues${GREEN} in ${ORANGE}${#LUA_CGI_FILES_ARR[@]}${GREEN} lua files.${NC}" - fi - if [[ "${lS23_LUA_ISSUES}" -gt 0 ]]; then - print_ln print_output "[+] Found ${ORANGE}${lS23_LUA_ISSUES} coding issues${GREEN} in ${ORANGE}${#LUA_CGI_FILES_ARR[@]}${GREEN} lua files.${NC}" fi @@ -83,8 +80,8 @@ S23_lua_check() { # this is a very basic checker for LUA issues s23_luaseccheck() { - local lGPT_ANCHOR_="" - local lGPT_PRIO_=3 + local lAI_ANCHOR="" + local lAI_PRIO=3 local lENTRY="" local lQUERY_ENTRIES_ARR=() local lQUERY_FILE="" @@ -95,6 +92,8 @@ s23_luaseccheck() { for lQUERY_FILE in "${LUA_CGI_FILES_ARR[@]}"; do [[ ! -f "${lQUERY_FILE}" ]] && continue local lISSUES_FILE=0 + local lLOG_QUERYFILE="" + lLOG_QUERYFILE="${LOG_PATH_MODULE}/$(basename "${lQUERY_FILE}").log" mapfile -t lQUERY_ENTRIES_ARR < <(grep -E "=.*cgilua\.QUERY" "${lQUERY_FILE}" | tr ' ' '\n' | sed 's/.*cgilua.QUERY.//' | sed 's/.*cgilua.QUERY.//' | grep -o -E "^[[:alnum:]]+" | grep -v "^local$" | sort -u || true) @@ -109,18 +108,18 @@ s23_luaseccheck() { lS23_LUA_VULNS=$((lS23_LUA_VULNS + 1)) lISSUES_FILE=$((lISSUES_FILE + 1)) print_output "[+] Found lua QUERY (GET/POST) entry: ${ORANGE}${lENTRY}${GREEN} in file ${ORANGE}${lQUERY_FILE}${GREEN} with file access capabilities." - copy_and_link_file "${lQUERY_FILE}" "${LOG_PATH_MODULE}/$(basename "${lQUERY_FILE}").log" - sed -i -r "s/.*io\.(p)?open.*/\x1b[32m&\x1b[0m/" "${LOG_PATH_MODULE}/$(basename "${lQUERY_FILE}").log" - lGPT_PRIO_=$((lGPT_PRIO_ + 1)) + copy_and_link_file "${lQUERY_FILE}" "${lLOG_QUERYFILE}" + sed -i -r "s/.*io\.(p)?open.*/\x1b[32m&\x1b[0m/" "${lLOG_QUERYFILE}" + lAI_PRIO=$((lAI_PRIO + 1)) fi if grep "${lENTRY}" "${lQUERY_FILE}" 2>/dev/null | grep -q "os.execute"; then # command exec - critical lS23_LUA_VULNS=$((lS23_LUA_VULNS + 1)) lISSUES_FILE=$((lISSUES_FILE + 1)) print_output "[+] Found lua QUERY (GET/POST) entry: ${ORANGE}${lENTRY}${GREEN} in file ${ORANGE}${lQUERY_FILE}${GREEN} with command execution capabilities." - copy_and_link_file "${lQUERY_FILE}" "${LOG_PATH_MODULE}/$(basename "${lQUERY_FILE}").log" - sed -i -r "s/.*os\.execute.*/\x1b[32m&\x1b[0m/" "${LOG_PATH_MODULE}/$(basename "${lQUERY_FILE}").log" - lGPT_PRIO_=$((lGPT_PRIO_ + 1)) + copy_and_link_file "${lQUERY_FILE}" "${lLOG_QUERYFILE}" + sed -i -r "s/.*os\.execute.*/\x1b[32m&\x1b[0m/" "${lLOG_QUERYFILE}" + lAI_PRIO=$((lAI_PRIO + 1)) fi done @@ -129,8 +128,6 @@ s23_luaseccheck() { # os.execute(string.format("sleep %d", tonumber(sec))) # os.execute(asdf_cmd) if [[ "${lISSUES_FILE}" -eq 0 ]] && grep -E -q "os\.execute\(.*\.\..*\)|os\.execute\(.*\%.*\)|os\.execute\([[:alnum:]_]+\)" "${lQUERY_FILE}"; then - local lLOG_QUERYFILE="" - lLOG_QUERYFILE="${LOG_PATH_MODULE}/$(basename "${lQUERY_FILE}").log" print_output "[+] Found lua file ${ORANGE}${lQUERY_FILE}${GREEN} with possible command execution for review." copy_and_link_file "${lQUERY_FILE}" "${lLOG_QUERYFILE}" if [[ -f "${lLOG_QUERYFILE}" ]]; then @@ -142,25 +139,26 @@ s23_luaseccheck() { sed -i -r "s/.*os\.execute\([[:alnum:]_]+\)/\x1b[32m&\x1b[0m/" "${lLOG_QUERYFILE}" fi lISSUES_FILE=$((lISSUES_FILE + 1)) + lAI_PRIO=$((lAI_PRIO + 1)) fi if [[ "${lISSUES_FILE}" -eq 0 ]] && grep -E -q "io\.(p)?open" "${lQUERY_FILE}"; then # command exec - not our parameter but we check it print_output "[+] Found lua file ${ORANGE}${lQUERY_FILE}${GREEN} with possible file access for review." - copy_and_link_file "${lQUERY_FILE}" "${LOG_PATH_MODULE}/$(basename "${lQUERY_FILE}").log" - sed -i -r "s/.*io\.(p)?open.*/\x1b[32m&\x1b[0m/" "${LOG_PATH_MODULE}/$(basename "${lQUERY_FILE}").log" + copy_and_link_file "${lQUERY_FILE}" "${lLOG_QUERYFILE}" + sed -i -r "s/.*io\.(p)?open.*/\x1b[32m&\x1b[0m/" "${lLOG_QUERYFILE}" lISSUES_FILE=$((lISSUES_FILE + 1)) + lAI_PRIO=$((lAI_PRIO + 1)) fi if [[ "${lISSUES_FILE}" -gt 0 ]]; then write_csv_log "$(print_path "${lQUERY_FILE}")" "0" "${lISSUES_FILE}" "NA" - if [[ "${GPT_OPTION}" -gt 0 ]]; then - lGPT_ANCHOR_="$(openssl rand -hex 8)" - # "${GPT_INPUT_FILE_}" "${lGPT_ANCHOR_}" "${lGPT_PRIO_}" "${GPT_QUESTION_}" "${GPT_OUTPUT_FILE_}" "cost=$GPT_TOKENS_" "${GPT_RESPONSE_}" - write_csv_gpt_tmp "$(cut_path "${lQUERY_FILE}")" "${lGPT_ANCHOR_}" "${lGPT_PRIO_}" "${GPT_QUESTION}" "${S23_CSV_LOG}" "" "" + if [[ "${AI_OPTION}" -gt 0 ]]; then + lAI_ANCHOR="$(openssl rand -hex 8)" + # "${GPT_INPUT_FILE_}" "${lAI_ANCHOR}" "${lAI_PRIO}" "${GPT_QUESTION_}" "${GPT_OUTPUT_FILE_}" "cost=$GPT_TOKENS_" "${GPT_RESPONSE_}" + write_csv_AI_tmp "${lLOG_QUERYFILE}" "${lAI_ANCHOR}" "${lAI_PRIO}" "${GPT_QUESTION}" "${lLOG_QUERYFILE}" "" "" # add ChatGPT link - print_ln - print_ln - write_anchor_gpt "${lGPT_ANCHOR_}" + print_output "" "${lLOG_QUERYFILE}" + write_anchor_AI "${lAI_ANCHOR}" "${lLOG_QUERYFILE}" fi fi done @@ -173,7 +171,7 @@ s23_luacheck() { local lLUA_ISSUES="" lSCRIPT_NAME=$(basename "${lLUA_SCRIPT_}" 2>/dev/null | sed -e 's/:/_/g') - lLUA_LOG="${LOG_PATH_MODULE}""/luacheck_""${lSCRIPT_NAME}"".txt" + lLUA_LOG="${LOG_PATH_MODULE}/luacheck_${lSCRIPT_NAME}.txt" luacheck "${lLUA_SCRIPT_}" >"${lLUA_LOG}" 2>/dev/null || true lLUA_ISSUES=$(strip_color_codes "$(grep Total "${lLUA_LOG}" | awk '{print $2}' 2>/dev/null || true)") @@ -182,17 +180,17 @@ s23_luacheck() { local lCOMMON_FILES_FOUND="" local lCFF="" if [[ -f "${BASE_LINUX_FILES}" ]]; then - lCOMMON_FILES_FOUND="(""${RED}""common linux file: no""${GREEN}"")" + lCOMMON_FILES_FOUND="(${RED}common linux file: no${GREEN})" lCFF="no" if grep -q "^${lSCRIPT_NAME}\$" "${BASE_LINUX_FILES}" 2>/dev/null; then - lCOMMON_FILES_FOUND="(""${CYAN}""common linux file: yes""${GREEN}"")" + lCOMMON_FILES_FOUND="(${CYAN}common linux file: yes${GREEN})" lCFF="yes" fi else lCOMMON_FILES_FOUND="" lCFF="NA" fi - print_output "[+] Found ""${ORANGE}""${lLUA_ISSUES}"" coding issues""${GREEN}"" in lua script ""${lCOMMON_FILES_FOUND}"":""${NC}"" ""$(print_path "${lLUA_SCRIPT_}")" "" "${lLUA_LOG}" + print_output "[+] Found ${ORANGE}${lLUA_ISSUES} coding issues${GREEN} in lua script ${lCOMMON_FILES_FOUND}:${NC} $(print_path "${lLUA_SCRIPT_}")" "" "${lLUA_LOG}" write_csv_log "$(print_path "${lLUA_SCRIPT_}")" "${lLUA_ISSUES}" "0" "${lCFF}" fi } diff --git a/modules/S27_perl_check.sh b/modules/S27_perl_check.sh index 1fb54d75..454231c6 100755 --- a/modules/S27_perl_check.sh +++ b/modules/S27_perl_check.sh @@ -38,7 +38,6 @@ S27_perl_check() { export PERL5LIB="${EXT_DIR}"/zarn/lib write_csv_log "Script path" "Perl issues detected" "common linux file" "vuln title" "vuln line nr" "vuln note" - # mapfile -t lPERL_SCRIPTS_ARR < <(find "${FIRMWARE_PATH}" -xdev -type f \( -name "*.pl" -o -name "*.pm" -o -name "*.cgi" \) -print0|xargs -r -0 -P 16 -I % sh -c 'md5sum "%" 2>/dev/null' | sort -u -k1,1 | cut -d\ -f3 ) mapfile -t lPERL_SCRIPTS_ARR < <(grep "Perl script.*executable" "${P99_CSV_LOG}" | sort -u || true) for lPL_SCRIPT in "${lPERL_SCRIPTS_ARR[@]}"; do if [[ -f "${BASE_LINUX_FILES}" && "${FULL_TEST}" -eq 0 ]]; then @@ -50,22 +49,16 @@ S27_perl_check() { fi fi ((lS27_PL_SCRIPTS += 1)) - if [[ "${THREADED}" -eq 1 ]]; then - s27_zarn_perl_checks "$(echo "${lPL_SCRIPT}" | cut -d';' -f2)" & - local lTMP_PID="$!" - store_kill_pids "${lTMP_PID}" - lWAIT_PIDS_S27+=("${lTMP_PID}") - max_pids_protection "${MAX_MOD_THREADS}" lWAIT_PIDS_S27 - continue - else - s27_zarn_perl_checks "$(echo "${lPL_SCRIPT}" | cut -d';' -f2)" - fi + s27_zarn_perl_checks "$(echo "${lPL_SCRIPT}" | cut -d';' -f2)" & + local lTMP_PID="$!" + lWAIT_PIDS_S27+=("${lTMP_PID}") + max_pids_protection "${MAX_MOD_THREADS}" lWAIT_PIDS_S27 done [[ "${THREADED}" -eq 1 ]] && wait_for_pid "${lWAIT_PIDS_S27[@]}" - [[ -d "${TMP_DIR}"/s27 ]] && (rm -r "${TMP_DIR}"/s27 || true) + [[ -d "${TMP_DIR}/s27" ]] && (rm -r "${TMP_DIR}"/s27 || true) - if [[ -f "${TMP_DIR}"/S27_VULNS.tmp ]]; then + if [[ -f "${TMP_DIR}/S27_VULNS.tmp" ]]; then lS27_PL_VULNS=$(awk '{sum += $1 } END { print sum }' "${TMP_DIR}"/S27_VULNS.tmp) fi @@ -80,7 +73,7 @@ s27_zarn_perl_checks() { local lPL_LOG="" # text log with original output from zarn analysis local lVULNS="" local lGPT_PRIO=2 - local lGPT_ANCHOR="" + local lAI_ANCHOR="" local lZARN_SARIF_RESULTS_ARR=() local lSARIF_RESULT="" local lZARN_VULN_TITLE="" @@ -89,54 +82,53 @@ s27_zarn_perl_checks() { local lCODE_LINE="" local lPL_LOG_LINKED="" # 2nd stage log which is linked from the main log + local lS27_SOURCE_DIR="${LOG_PATH_MODULE}/pl_sources" + lNAME=$(basename "${lPL_SCRIPT}" 2>/dev/null | sed -e 's/:/_/g') print_output "[*] Testing perl script ${ORANGE}${lPL_SCRIPT}${NC}" "no_log" - lPL_LOG="${LOG_PATH_MODULE}""/zarn_""${lNAME}"".txt" + lPL_LOG="${LOG_PATH_MODULE}/zarn_${lNAME}.txt" if [[ "${lPL_SCRIPT: -4}" == ".cgi" ]]; then if ! [[ -d "${TMP_DIR}"/s27 ]]; then mkdir "${TMP_DIR}"/s27 fi # as zarn does not like cgi's, we need to temp rename these files now cp "${lPL_SCRIPT}" "${TMP_DIR}"/s27/"${lNAME/.cgi/.pl}" - if [[ -f "${TMP_DIR}"/s27/"${lNAME/.cgi/.pl}" ]]; then - perl "${EXT_DIR}"/zarn/zarn.pl -r "${EXT_DIR}"/zarn/rules/default.yml --source "${TMP_DIR}"/s27/"${lNAME/.cgi/.pl}" --sarif "${LOG_PATH_MODULE}""/zarn_""${lNAME}"".sarif" | tee -a "${lPL_LOG}" || print_error "[-] Analysis of ${lNAME} failed" - rm "${TMP_DIR}"/s27/"${lNAME/.cgi/.pl}" || true + if [[ -f "${TMP_DIR}/s27/${lNAME/.cgi/.pl}" ]]; then + perl "${EXT_DIR}"/zarn/zarn.pl -r "${EXT_DIR}"/zarn/rules/default.yml --source "${TMP_DIR}/s27/${lNAME/.cgi/.pl}" --sarif "${LOG_PATH_MODULE}/zarn_${lNAME}.sarif" | tee -a "${lPL_LOG}" || print_error "[-] Analysis of ${lNAME} failed" + rm "${TMP_DIR}/s27/${lNAME/.cgi/.pl}" || true fi else - perl "${EXT_DIR}"/zarn/zarn.pl -r "${EXT_DIR}"/zarn/rules/default.yml --source "${lPL_SCRIPT}" --sarif "${LOG_PATH_MODULE}""/zarn_""${lNAME}"".sarif" | tee -a "${lPL_LOG}" || print_error "[-] Analysis of ${lNAME} failed" + perl "${EXT_DIR}"/zarn/zarn.pl -r "${EXT_DIR}"/zarn/rules/default.yml --source "${lPL_SCRIPT}" --sarif "${LOG_PATH_MODULE}/zarn_${lNAME}.sarif" | tee -a "${lPL_LOG}" || print_error "[-] Analysis of ${lNAME} failed" fi ## reporting starts here lVULNS=$(grep -c "\[vuln\]" "${lPL_LOG}" 2>/dev/null || true) if [[ "${lVULNS}" -gt 0 ]]; then + [[ ! -d "${lS27_SOURCE_DIR}" ]] && mkdir -p "${lS27_SOURCE_DIR}" # check if this is common linux file: local lCOMMON_FILES_FOUND="" local lCFF="" if [[ -f "${BASE_LINUX_FILES}" ]]; then - lCOMMON_FILES_FOUND="(""${RED}""common linux file: no""${GREEN}"")" + lCOMMON_FILES_FOUND="(${RED}common linux file: no${GREEN})" lCFF="no" if grep -q "^${lNAME}\$" "${BASE_LINUX_FILES}" 2>/dev/null; then - lCOMMON_FILES_FOUND="(""${CYAN}""common linux file: yes""${GREEN}"")" + lCOMMON_FILES_FOUND="(${CYAN}common linux file: yes${GREEN})" lCFF="yes" fi else lCOMMON_FILES_FOUND="" lCFF="NA" fi - lPL_LOG_LINKED="${LOG_PATH_MODULE}""/zarn_""${lNAME}""_details.txt" - print_output "[+] Found ""${RED}""${lVULNS}"" possible issue(s)""${GREEN}"" in perl script ""${lCOMMON_FILES_FOUND}"":""${ORANGE}"" ""$(print_path "${lPL_SCRIPT}")" "" "${lPL_LOG_LINKED}" + lPL_LOG_LINKED="${LOG_PATH_MODULE}/zarn_${lNAME}_details.txt" + print_output "[+] Found ${RED}${lVULNS} possible issue(s)${GREEN} in perl script ${lCOMMON_FILES_FOUND}:${ORANGE} $(print_path "${lPL_SCRIPT}")" "" "${lPL_LOG_LINKED}" if [[ "${lVULNS}" -gt 20 ]]; then lGPT_PRIO=$((lGPT_PRIO + 1)) fi - mapfile -t lZARN_SARIF_RESULTS_ARR < <(jq -rc '.runs[].results[]' "${LOG_PATH_MODULE}""/zarn_""${lNAME}"".sarif") + mapfile -t lZARN_SARIF_RESULTS_ARR < <(jq -rc '.runs[].results[]' "${LOG_PATH_MODULE}/zarn_${lNAME}.sarif") - if ! [[ -d "${LOG_PATH_MODULE}"/pl_source_files/ ]]; then - mkdir "${LOG_PATH_MODULE}"/pl_source_files/ || true - fi - - write_log "[+] Found ""${ORANGE}""${lVULNS}"" possible issue(s)""${GREEN}"" in perl script ""${lCOMMON_FILES_FOUND}"":""${ORANGE}"" ""$(print_path "${lPL_SCRIPT}")" "${lPL_LOG_LINKED}" - write_link "${LOG_PATH_MODULE}"/pl_source_files/"${lNAME}".txt "${lPL_LOG_LINKED}" + write_log "[+] Found ${ORANGE}${lVULNS} possible issue(s)${GREEN} in perl script ${lCOMMON_FILES_FOUND}:${ORANGE} $(print_path "${lPL_SCRIPT}")" "${lPL_LOG_LINKED}" + copy_and_link_file "${lPL_SCRIPT}" "${lS27_SOURCE_DIR}/${lNAME}.log" "${lPL_LOG_LINKED}" write_log "$(indent "$(orange "$(file "${lPL_SCRIPT}")")")" "${lPL_LOG_LINKED}" write_log "\n" "${lPL_LOG_LINKED}" @@ -151,19 +143,19 @@ s27_zarn_perl_checks() { write_log "" "${lPL_LOG_LINKED}" # we need the original perl script to link to it and to highlight the affected lines of code - if ! [[ -f "${LOG_PATH_MODULE}"/pl_source_files/"${lNAME}".txt ]]; then - cp "${lPL_SCRIPT}" "${LOG_PATH_MODULE}"/pl_source_files/"${lNAME}".txt || print_error "[-] Copy of ${lPL_SCRIPT} to log directory failed" + if ! [[ -f "${lS27_SOURCE_DIR}/${lNAME}.log" ]]; then + cp "${lPL_SCRIPT}" "${lS27_SOURCE_DIR}/${lNAME}.log" || print_error "[-] Copy of ${lPL_SCRIPT} to log directory failed" fi - lCODE_LINE="$(strip_color_codes "$(sed -n "${lZARN_VULN_LINE}"p "${LOG_PATH_MODULE}/pl_source_files/${lNAME}.txt" 2>/dev/null)")" - sed -i -r "${lZARN_VULN_LINE}s/.*/\x1b[32m&\x1b[0m/" "${LOG_PATH_MODULE}/pl_source_files/${lNAME}.txt" || true + lCODE_LINE="$(strip_color_codes "$(sed -n "${lZARN_VULN_LINE}"p "${lS27_SOURCE_DIR}/${lNAME}.log" 2>/dev/null)")" + sed -i -r "${lZARN_VULN_LINE}s/.*/\x1b[32m&\x1b[0m/" "${lS27_SOURCE_DIR}/${lNAME}.log" || true write_log "$(indent "$(indent "${GREEN}${lZARN_VULN_LINE}${NC} - ${ORANGE}${lCODE_LINE}${NC}")")" "${lPL_LOG_LINKED}" write_log "\\n-----------------------------------------------------------------\\n" "${lPL_LOG_LINKED}" done # writing gpt details: - if [[ "${GPT_OPTION}" -gt 0 ]]; then - lGPT_ANCHOR="$(openssl rand -hex 8)" + if [[ "${AI_OPTION}" -gt 0 ]]; then + lAI_ANCHOR="$(openssl rand -hex 8)" if [[ -f "${BASE_LINUX_FILES}" ]]; then # if we have the base linux config file we are checking it: if ! grep -E -q "^$(basename "${lPL_SCRIPT}")$" "${BASE_LINUX_FILES}" 2>/dev/null; then @@ -171,10 +163,12 @@ s27_zarn_perl_checks() { fi fi # "${GPT_INPUT_FILE_}" "${GPT_ANCHOR_}" "${GPT_PRIO_}" "${GPT_QUESTION_}" "${GPT_OUTPUT_FILE_}" "cost=$GPT_TOKENS_" "${GPT_RESPONSE_}" - write_csv_gpt_tmp "$(cut_path "${lPL_SCRIPT}")" "${lGPT_ANCHOR}" "${lGPT_PRIO}" "${GPT_QUESTION}" "${lPL_LOG}" "" "" + write_csv_AI_tmp "${lS27_SOURCE_DIR}/${lNAME}.log" "${lAI_ANCHOR}" "${lGPT_PRIO}" "${GPT_QUESTION}" "${lPL_LOG}" "" "" # add ChatGPT link to output file printf '%s\n\n' "" >>"${lPL_LOG}" - write_anchor_gpt "${lGPT_ANCHOR}" "${lPL_LOG}" + write_anchor_AI "${lAI_ANCHOR}" "${lPL_LOG}" + printf '%s\n\n' "" >>"${lS27_SOURCE_DIR}/${lNAME}.log" + write_anchor_AI "${lAI_ANCHOR}" "${lS27_SOURCE_DIR}/${lNAME}.log" fi echo "${lVULNS}" >>"${TMP_DIR}"/S27_VULNS.tmp fi diff --git a/modules/S28_java_check.sh b/modules/S28_java_check.sh index 1f66479b..9d2687fe 100755 --- a/modules/S28_java_check.sh +++ b/modules/S28_java_check.sh @@ -27,8 +27,9 @@ S28_java_check() { local lJAVA_BINS_ARR=() local lJAVA_BINARY="" local lJNAME="" - local lJAVA_VULNS=0 + local lJAVA_VULNS_CNT=0 local lWAIT_PIDS_S28=() + local MAX_MOD_THREADS=$((MAX_MOD_THREADS/2)) export JAVA_DECOMPILER="${EXT_DIR}/vineflower-1.11.2.jar" @@ -70,21 +71,45 @@ S28_java_check() { wait_for_pid "${lWAIT_PIDS_S28[@]}" - local lSEMGREP_RESULTS_ARR=() - local lSEMGREP_RESULT="" - mapfile -t lSEMGREP_RESULTS_ARR < <(find "${LOG_PATH_MODULE}/java_semgrep/" -name "semgrep_*.json") - for lSEMGREP_RESULT in "${lSEMGREP_RESULTS_ARR[@]}"; do - if grep -q '\"results\":\[{\"' "${lSEMGREP_RESULT}"; then - print_output "[+] Semgrep security scanning results for $(basename "${lSEMGREP_RESULT}")" "" "${lSEMGREP_RESULT}" - # todo: count vulns to lJAVA_VULNS - fi - done + + if [[ -d "${LOG_PATH_MODULE}/java_semgrep/" ]]; then + local lSEMGREP_RESULTS_ARR=() + local lSEMGREP_RESULT="" + mapfile -t lSEMGREP_RESULTS_ARR < <(find "${LOG_PATH_MODULE}/java_semgrep/" -name "semgrep_*.json") + for lSEMGREP_RESULT in "${lSEMGREP_RESULTS_ARR[@]}"; do + if grep -q '\"results\":' "${lSEMGREP_RESULT}"; then + local lJAVA_SOURCE_FILE_ARR=() + mapfile -t lJAVA_SOURCE_FILE_ARR < <(jq -r '.results[].path' "${lSEMGREP_RESULT}" | sort -u) + lJAVA_VULNS_CNT=$((lJAVA_VULNS_CNT+$(jq -r '.results[].check_id' "${lSEMGREP_RESULT}" | wc -l))) + local lJAVA_SOURCE_FILE="" + for lJAVA_SOURCE_FILE in "${lJAVA_SOURCE_FILE_ARR[@]}"; do + print_output "[+] Semgrep security scanning results for ${ORANGE}$(basename "${lJAVA_SOURCE_FILE}")${NC}" "" "${lSEMGREP_RESULT}" + # we log ".log" for the web reporter + mv "${lJAVA_SOURCE_FILE}" "${lJAVA_SOURCE_FILE}.log" || true + print_output "$(indent "$(orange "Decompiled Java sources for ${ORANGE}$(basename "${lJAVA_SOURCE_FILE}")${NC}")")" "" "${lJAVA_SOURCE_FILE}.log" + # AI integration + local lAI_ANCHOR="" + lAI_ANCHOR="$(openssl rand -hex 8)" + lAI_PRIO=4 + write_csv_AI_tmp "${lJAVA_SOURCE_FILE}.log" "${lAI_ANCHOR}" "${lAI_PRIO}" "NA" "${lSEMGREP_RESULT}" "" "" + printf '%s\n\n' "" >>"${lJAVA_SOURCE_FILE}.log" + write_anchor_AI "${lAI_ANCHOR}" "${lJAVA_SOURCE_FILE}.log" + done + # Todo: color the affected lines of code + fi + done + fi local lJAVA_DECOMPILED=0 - lJAVA_DECOMPILED=$(find "${LOG_PATH_MODULE}/java_decompile/" -type f | wc -l) + if [[ -d "${LOG_PATH_MODULE}/java_decompile/" ]]; then + lJAVA_DECOMPILED=$(find "${LOG_PATH_MODULE}/java_decompile/" -type f | wc -l) + fi + + print_ln "" + print_output "[+] Found ${ORANGE}${lJAVA_VULNS_CNT} vulnerabilities${GREEN} in ${ORANGE}${lJAVA_DECOMPILED}${GREEN} decompiled Java binaries.${NC}" write_log "" - write_log "[*] Statistics:${#lJAVA_BINS_ARR[@]}:${lJAVA_DECOMPILED}:${lJAVA_VULNS}" + write_log "[*] Statistics:${#lJAVA_BINS_ARR[@]}:${lJAVA_DECOMPILED}:${lJAVA_VULNS_CNT}" module_end_log "${FUNCNAME[0]}" "${lJAVA_DECOMPILED}" } @@ -98,7 +123,7 @@ s28_java_semgrep() { local lJ_ANALYSE_RESULTS="${lJ_ANALYSE_DIR}/semgrep_${lJNAME}.json" semgrep --disable-version-check --metrics=off --severity ERROR --severity WARNING --json --config "${EXT_DIR}"/semgrep-rules/java "${lJAVA_BIN_DIR}" >"${lJ_ANALYSE_RESULTS}" || true - if [[ -f "${lJ_ANALYSE_RESULTS}" ]] && [[ "$(grep -c '\"results\":\[{\"' "${lJ_ANALYSE_RESULTS}")" -eq 0 ]]; then + if [[ -f "${lJ_ANALYSE_RESULTS}" ]] && [[ "$(grep -c '\"results\":' "${lJ_ANALYSE_RESULTS}")" -eq 0 ]]; then rm "${lJ_ANALYSE_RESULTS}" || true fi } @@ -108,8 +133,9 @@ s28_java_decompile() { local lJNAME="" lJNAME=$(basename "${lJAVA_BINARY}" 2>/dev/null | sed -e 's/:/_/g') - print_output "[*] Decompiling Java binary ${ORANGE}${lJNAME}${NC}" "no_log" + lJNAME=$(iconv -f UTF-8 -t ASCII//TRANSLIT <<<"${lJNAME}" || true) local lJ_DECOMPILE_DIR="${LOG_PATH_MODULE}/java_decompile/${lJNAME}_${RANDOM}" + print_output "[*] Decompiling Java binary ${ORANGE}${lJNAME}${NC} to ${lJ_DECOMPILE_DIR}" "no_log" [[ ! -d "${lJ_DECOMPILE_DIR}" ]] && mkdir -p "${lJ_DECOMPILE_DIR}" java -jar "${JAVA_DECOMPILER}" --silent "${lJAVA_BINARY}" "${lJ_DECOMPILE_DIR}" || true diff --git a/modules/template_module.sh b/modules/template_module.sh index 394d4055..060087a5 100755 --- a/modules/template_module.sh +++ b/modules/template_module.sh @@ -35,21 +35,21 @@ =head1 MODULE_NAME -==head2 MODULE_NAME Short description +=head2 MODULE_NAME Short description Please write a short description of your module. Usually ~2-3 sentences are fine to get an idea. -==head2 MODULE_NAME Detailed description +=head2 MODULE_NAME Detailed description Please write a longer description of your module. This should also include further references and links that were used. -==head2 MODULE_NAME 3rd party tools +=head2 MODULE_NAME 3rd party tools Any 3rd party tool that is needed from your module. Also include the tested and known working version and download link. -==head2 MODULE_NAME Testfirmware +=head2 MODULE_NAME Testfirmware For verification of the module we need some testfirmware. @@ -59,26 +59,26 @@ Testfirmware details: - Checksum (MD5/SHA1/SHA256): - Download Link: -==head2 MODULE_NAME Output +=head2 MODULE_NAME Output Example output of module -==head2 MODULE_NAME License +=head2 MODULE_NAME License EMBA module MODULE_NAME is licensed under GPLv3 SPDX-License-Identifier: GPL-3.0-only Link to license document: https://github.com/e-m-b-a/emba/blob/master/LICENSE Note: Only GPL-3.0 will be accepted in the master EMBA repository -==head2 MODULE_NAME Todo +=head2 MODULE_NAME Todo Missing stuff that we need to consider. -==head2 MODULE_NAME Known issues +=head2 MODULE_NAME Known issues Any known issues or known limitations. -==head2 MODULE_NAME Author(s) +=head2 MODULE_NAME Author(s) Michael Messner, Pascal Eckmann Note: List all authors including contributors to this module diff --git a/scan-profiles/default-scan-gpt.emba b/scan-profiles/default-scan-AI.emba similarity index 90% rename from scan-profiles/default-scan-gpt.emba rename to scan-profiles/default-scan-AI.emba index ef28dfd8..32d9a5ad 100644 --- a/scan-profiles/default-scan-gpt.emba +++ b/scan-profiles/default-scan-AI.emba @@ -1,6 +1,6 @@ # EMBA - EMBEDDED LINUX ANALYZER # -# Copyright 2020-2025 Siemens Energy AG +# Copyright 2020-2026 Siemens Energy AG # # EMBA comes with ABSOLUTELY NO WARRANTY. This is free software, and you are # welcome to redistribute it under the terms of the GNU General Public License. @@ -11,7 +11,7 @@ # Author(s): Benedikt Kuehne # Contributor(s): Michael Messner # -# Description: Profile for ChatGPT +# Description: Profile for AI usage export FORMAT_LOG=1 export THREADED=1 @@ -20,7 +20,8 @@ export HTML=1 # QEMULATION - for user-mode emulation to generate an optimised SBOM export QEMULATION=1 -export GPT_OPTION=1 +# 0 -> off, 1-> unpaid GPT plan, 2 -> no GPT rate-limit, 3 -> LocalAI Server +export AI_OPTION=3 # extended binary tests are now only testing non-linux binaries. With this mechanism we can enable # it in the default profile. @@ -33,11 +34,11 @@ export VEX_METRICS=1 export MODULE_BLACKLIST+=( "S10_binaries_basic_check" "S15_radare_decompile_checks" "S26_kernel_vuln_verifier" "S99_grepit" "S110_yara_check" "F20_vul_aggregator" ) # enable silent mode and status bar -export DISABLE_STATUS_BAR=0 -export SILENT=1 +export DISABLE_STATUS_BAR=1 +export SILENT=0 # enable EMBA binary map dependency generator -EMBA_MAP_GENERATOR=1 +EMBA_MAP_GENERATOR=0 # we output the profile only at the beginning - outside the docker environment if [[ $IN_DOCKER -ne 1 ]] ; then diff --git a/scan-profiles/default-scan.emba b/scan-profiles/default-scan.emba index cdade439..90a329c9 100644 --- a/scan-profiles/default-scan.emba +++ b/scan-profiles/default-scan.emba @@ -41,8 +41,8 @@ export MODULE_BLACKLIST+=( "S10_binaries_basic_check" "S15_radare_decompile_chec export DISABLE_STATUS_BAR=0 export SILENT=1 -# enable GPT connection: -export GPT_OPTION=0 +# enable AI connection: +export AI_OPTION=0 # enable EMBA binary map dependency generator EMBA_MAP_GENERATOR=1