Compare commits

..
5 Commits
Author SHA1 Message Date
pancake 83b96f7a3e fiif 2025-10-03 18:21:20 +02:00
pancake e2ee8cdbb6 rjo 2025-10-03 17:36:29 +02:00
pancake 8630e95d83 morebecnh 2025-10-03 17:06:20 +02:00
pancake e07293e40f Initial benchmark working 2025-10-03 16:48:56 +02:00
pancake 415c546f92 Bench 2025-10-03 16:01:31 +02:00
18 changed files with 299 additions and 67 deletions

No files matched your search

+2 -5
View File
@@ -93,11 +93,8 @@ jobs:
shell: bash
run: |
mkdir -p release
# Collect common binary artefact names produced by Make/Meson
# Include both "r2ai.*" and "libr2ai.*" variants for DLL/dylib/so on all platforms.
find src -type f \( -iname "r2ai.dll" -o -iname "libr2ai.dll" -o -iname "r2ai.dylib" -o -iname "libr2ai.dylib" -o -iname "r2ai.so" -o -iname "libr2ai.so" -o -iname "r2ai.exe" \) -exec cp -f {} release/ \; || true
# Also check top-level build output directories
find . -maxdepth 3 -type f \( -iname "r2ai.dll" -o -iname "libr2ai.dll" -o -iname "r2ai.dylib" -o -iname "libr2ai.dylib" -o -iname "r2ai.so" -o -iname "libr2ai.so" -o -iname "r2ai.exe" \) -exec cp -f {} release/ \; || true
cp -rf src/r2ai.dll src/r2ai.exe src/r2ai src/r2ai.dylib src/r2ai.so release/ || true
cp -rf src/build/r2ai src/build/r2ai.dylib src/build/r2ai.so release/ || true
cp -f decai/decai.r2.js release/ || true
if command -v zip >/dev/null 2>&1; then
zip -j "release/r2ai-${{ github.ref_name }}-${{ matrix.os }}-${{ matrix.build_system }}.zip" release/*
+1 -4
View File
@@ -89,11 +89,8 @@ r2ai -e max_tokens=64000
* There's [a chapter](https://book.rada.re/plugins/r2ai.html) in the official r2book
* Cryptax on [lmstudio+gptoss](https://cryptax.medium.com/r2ai-with-lmstudio-and-gpt-oss-08efa5ea2476) blog post
* Malware analysis [with r2ai](https://arxiv.org/pdf/2504.07574) by Cryptax and Daniel Nakov
* Analysis of [Linux/Trigona ransomware](https://cryptax.medium.com/linux-trigona-analysis-with-r2ai-3e2bd1815e52), [Linux/Prometei botnet](https://cryptax.medium.com/reversing-a-prometei-botnet-binary-with-r2-and-ai-part-one-3cdb3dc6ffab) and [W32/SkyAI](https://cryptax.medium.com/w32-skyai-uses-ai-so-do-i-d33f04d63534with) with r2ai
* Malware analysis [with r2ai](https://github.com/cryptax/talks/blob/master/BSidesKristiansand-2025/r2ai.pdf) by Cryptax
## Videos
- https://infosec.exchange/@radareorg/111946255058894583
- [De-obfuscation of malware Linux/Ladvix](https://asciinema.org/a/724126)
- [Analysis of the /fast option inside Linux/Trigona ransomware](https://asciinema.org/a/pBPEaJhp6cunWSKFpBUDTgPt4)
+1 -1
View File
@@ -1 +1 @@
1.1.2
1.1.1
+3 -10
View File
@@ -765,20 +765,13 @@ Use radare2 to resolve user requests.
try {
const res = http.post(url, [], JSON.stringify(payload));
if (res.error) {
throw new Error(res.error);
}
if (res.message && res.message.content) {
return utils.filterResponse(res.message.content);
}
throw new Error(JSON.stringify(res));
return utils.filterResponse(res.message.content);
} catch (e) {
if (res.error && res.error.indexOf("try pulling")) {
const modelName = res.error.split(/"/g)[1];
res.error += "\n!ollama run " + modelName;
}
} catch (e) {
console.error(e.stack);
return "ERROR: " + e.message;
return "ERROR: " + (res.error || e.message);
}
},
+34 -7
View File
@@ -1,14 +1,41 @@
SHELL=/bin/bash
MODELS=gpt-oss:20b gemma3:4b gemma3:12b
MODELS+=granite3-dense:8b mannix/jan-nano:latest yi-coder:latest qwen3:latest deepseek-coder-v2:latest hhao/qwen2.5-coder-tools:latest qwen2.5-coder:latest
PROVIDER=ollama
MODELS=granite3-dense:8b mannix/jan-nano:latest yi-coder:latest qwen3:latest deepseek-coder-v2:latest hhao/qwen2.5-coder-tools:latest qwen2.5-coder:latest
MODELS=gemma3:1b
LANGS=Dart Perl Ruby Javascript PHP C
BINARIES=files/complex1 files/complex2 files/complex3
MODELS=gpt-4o
PROVIDER=openai
all:
for a in ${MODELS} ; do \
declare -a ARGS; \
for l in ${LANGS} ; do \
ARGS+=("echo \"decai -e model=$$a;decai -e lang=$$l;s 0x100000924;af;decai -d\" | r2 -q /bin/ls"); \
mkdir -p tmp
for b in $(BINARIES) ; do \
for a in $(MODELS) ; do \
echo "$$b $$a"; \
./run.sh $$b $$a ${PROVIDER} ${LANGS} > tmp/$${b##*/}_$${a//\//_}.html ; \
done ; \
./run.sh "$${ARGS[@]}" > ${a//\//_}.html ; done
done
.PHONY: run index evaluate
compile: $(BINARIES)
files/complex1: files/complex1.c
gcc -o $@ $<
files/complex2: files/complex2.c
gcc -o $@ $<
files/complex3: files/complex3.c
gcc -o $@ $<
lmstudio:
for b in $(BINARIES) ; do \
for a in $(MODELS) ; do \
./run.sh $$b $$a lmstudio ${LANGS} > $${b##*/}_$${a//\//_}.html ; \
done ; \
done
clean:
rm -rf tmp
+59
View File
@@ -0,0 +1,59 @@
#!/bin/bash
# Evaluate decompiled outputs using mai and determine best model+lang
echo "Evaluating decompiled outputs..."
tmpfile=$(mktemp)
MAI="mai -q -p openai -m gpt-4o"
# MAI="mai -q"
for orig in files/complex1.c files/complex2.c files/complex3.c; do
base=$(basename $orig .c)
original_code=$(cat $orig)
for f in tmp/${base}_*.txt; do
if [ -f "$f" ]; then
decompiled_code=$(cat $f)
#echo "Comparing $orig $f"
result=`echo n | ${MAI} -q -r "/template files/CMPROMPT.txt original=@${orig} decompiled=@${f}"`
score=`echo "$result" | head -n 1 | grep -oE '[0-9]+'` # |head -n 1`
perfect=`echo "$result" | grep 'No sign' && echo yes`
[ -z "$score" ] && score=0
if [ "$perfect" = yes ]; then
score=100
else
score="$((0+${score}))"
fi
echo "$score\t$f"
if [ -n "$score" ]; then
key="${f%.*}" # remove .txt
# Check if key exists in tmpfile
existing=$(grep "^$key " $tmpfile | awk '{print $2}')
if [ -n "$existing" ]; then
new_score=$(( existing + score ))
sed -i "s/^$key .*/$key $new_score/" $tmpfile
else
echo "$key $score" >> $tmpfile
fi
fi
fi
done
done
# Find the best (highest total score)
best_key=""
best_score=0
while read key score; do
if [ "$score" -gt "$best_score" ]; then
best_score=$score
best_key=$key
fi
done < $tmpfile
rm $tmpfile
if [ -n "$best_key" ]; then
echo "Best model+language combination: $best_key with total score $best_score"
else
echo "No scores found"
fi
+17
View File
@@ -0,0 +1,17 @@
Compare the two provided code in C with the decompiled counterpart. Ignore function or variable names, syntax differences, language-specific constructs, and formatting. Focus solely on what the code does: the operations performed, control flow, logic, and functional behavior. Determine if they are functionally equivalent in terms of their effects and computations.
Output only:
- A numeric score from 1 to 100 (100 being identical logic and functionality)
- Followed by a single sentence describing the most relevant semantic differences (or "No significant semantic differences" if score is 95+)
**Original Source Code:**
```C
{original}
```
**Decompiled Code:**
```
{decompiled}
```
+21
View File
@@ -0,0 +1,21 @@
#include <stdio.h>
#include <stdlib.h>
int main() {
int n = 5;
int *arr = (int *)malloc(n * sizeof(int));
if (arr == NULL) return 1;
for (int i = 0; i < n; i++) {
arr[i] = i * i;
}
int sum = 0;
for (int i = 0; i < n; i++) {
sum += arr[i];
for (int j = 0; j < i; j++) {
sum -= j;
}
}
printf("Sum: %d\n", sum);
free(arr);
return 0;
}
+39
View File
@@ -0,0 +1,39 @@
#include <stdio.h>
#include <stdlib.h>
struct Node {
int data;
struct Node *next;
};
int main() {
struct Node *head = NULL;
struct Node *tail = NULL;
for (int i = 0; i < 10; i++) {
struct Node *newNode = (struct Node *)malloc(sizeof(struct Node));
if (newNode == NULL) return 1;
newNode->data = i * 2;
newNode->next = NULL;
if (head == NULL) {
head = newNode;
tail = newNode;
} else {
tail->next = newNode;
tail = newNode;
}
}
struct Node *current = head;
int sum = 0;
while (current != NULL) {
sum += current->data;
current = current->next;
}
printf("Sum: %d\n", sum);
current = head;
while (current != NULL) {
struct Node *temp = current;
current = current->next;
free(temp);
}
return 0;
}
+33
View File
@@ -0,0 +1,33 @@
#include <stdio.h>
int main() {
int x = 42;
int y = 0;
switch (x & 15) {
case 0:
y = x << 1;
break;
case 1:
y = x >> 1;
break;
case 2:
y = x ^ 255;
break;
case 3:
y = ~x;
break;
default:
y = x | 128;
break;
}
if ((y & 1) == 0) {
y += 10;
} else {
y -= 5;
}
for (int i = 0; i < 5; i++) {
y = (y << 2) | (y >> 30);
}
printf("Result: %d\n", y);
return 0;
}
+2 -1
View File
@@ -1,5 +1,6 @@
#!/bin/bash
cp decai/BENCH.txt tmp/BENCH.txt
# Find all directories containing BENCH.txt
dirs=$(find . -name "BENCH.txt" -type f | xargs dirname | sort | uniq)
@@ -46,4 +47,4 @@ cat <<EOF >> benchmark_index.html
<br />
<br />
</html>
EOF
EOF
+62 -8
View File
@@ -1,12 +1,55 @@
#!/bin/bash
# Generate HTML output for command executions with collapsible sections and width control
# If PLAIN=1, output plain text instead
if [ $# -eq 0 ]; then
echo "Usage: $0 command1 [command2 ...]"
if [ $# -lt 4 ]; then
echo "Usage: $0 <binary> <model> <provider> <lang1> [lang2 ...]"
echo "Or: $0 command1 [command2 ...] (legacy)"
exit 1
fi
# Check if first arg is a file (binary), assume new format
if [ -f "$1" ]; then
binary="$1"
model="$2"
provider="$3"
shift 3
langs=("$@")
commands=()
for lang in "${langs[@]}"; do
# commands+=("# $lang echo \"decai -e model=$model;decai -e lang=$lang;decai -e api=$provider;af;decai -d\" | r2 -q $binary 2> /dev/null")
commands+=("# $lang r2 -e scr.color=0 -qcq -qc \"decai -e model=$model;decai -e lang=$lang;decai -e api=$provider;af;decai -d\" $binary")
done
set -- "${commands[@]}"
else
# Legacy mode
commands=("$@")
fi
if [ "$PLAIN" = "1" ]; then
# In plain mode, execute each command and print raw output.
# Also, if TXTFILE is set, save combined raw output there.
tmp_plain=$(mktemp)
for cmd in "$@"; do
if [[ "$cmd" =~ ^#[[:space:]]*(.*) ]]; then
exec_cmd="${BASH_REMATCH[1]}"
else
exec_cmd="$cmd"
fi
tmpout=$(mktemp)
eval "$exec_cmd" > "$tmpout" 2>&1 || true
mv "$tmpout" "$tmp_plain"
rm -f "$tmpout"
done
if [ -n "$TXTFILE" ]; then
mv "$tmp_plain" "$TXTFILE"
else
rm -f "$tmp_plain"
fi
exit 0
fi
# Timeout in seconds for each command (configurable via TIMEOUT environment variable)
TIMEOUT=${TIMEOUT:-30}
@@ -109,24 +152,27 @@ EOF
# Process each command
for cmd in "$@"; do
# Extract title: if command starts with #, use the rest as title, else use the command
if [[ "$cmd" =~ ^#[[:space:]]*(.*) ]]; then
# Extract title: if command starts with #, parse title and command, else use the command
if [[ "$cmd" =~ ^#[[:space:]]*([^[:space:]]+)[[:space:]]*(.*) ]]; then
title="${BASH_REMATCH[1]}"
exec_cmd="${cmd#*#}"
exec_cmd="${exec_cmd#"${exec_cmd%%[![:space:]]*}"}" # trim leading spaces
exec_cmd="${BASH_REMATCH[2]}"
else
title="$cmd"
exec_cmd="$cmd"
fi
echo "RUN $exec_cmd" > /dev/stderr
# Execute command and time it
start_time=$(date +%s.%N)
tmpfile=$(mktemp)
eval "$exec_cmd" > "$tmpfile" 2>&1 &
# echo "eval $exec_cmd > $tmpfile" > /dev/stder
eval "$exec_cmd" > "$tmpfile" 2> /dev/null &
pid=$!
(sleep $TIMEOUT && kill $pid 2>/dev/null) &
wait $pid 2>/dev/null
output=$(cat "$tmpfile")
#echo "$output" > /dev/stderr
rm "$tmpfile"
exit_code=$?
end_time=$(date +%s.%N)
@@ -139,8 +185,16 @@ for cmd in "$@"; do
execution_time=$(printf "%.3f" "$execution_time")
fi
# Save plain output to .txt file if in new format
if [ -n "$binary" ]; then
txtfile="tmp/${binary##*/}_${model//\//_}_${title}.txt"
mkdir -p "$(dirname "$txtfile")" 2>/dev/null || true
echo "$output" > "$txtfile"
fi
# Escape HTML in output
escaped_output=$(echo "$output" | sed 's/&/\&amp;/g; s/</\&lt;/g; s/>/\&gt;/g')
#escaped_output=$(echo "$output" | sed 's/&/\&amp;/g; s/</\&lt;/g; s/>/\&gt;/g')
escaped_output="$output"
cat << EOF
<div class="command-section expanded">
+3 -11
View File
@@ -59,25 +59,17 @@ indent:
user-install: user-uninstall
mkdir -p $(R2USRPLUGDIR)
mkdir -p $(R2BINDIR)
$(MAKE) user-uninstall
cp -f r2ai$(DOTLIB) $(R2USRPLUGDIR)/r2ai$(DOTLIB)
cp -f r2ai$(DOTEXE) $(R2BINDIR)/r2ai$(DOTEXE)
cp -f r2ai$(DOTLIB) $(R2USRPLUGDIR)
cp -f r2ai$(DOTEXE) $(R2BINDIR)
user-uninstall:
rm -f $(R2USRPLUGDIR)/r2ai$(DOTLIB)
rm -f $(R2BINDIR)/r2ai$(DOTEXE)
install:
$(MAKE) uninstall
mkdir -p $(R2USRPLUGDIR)
mkdir -p $(R2BINDIR)
cp -f r2ai$(DOTLIB) $(R2SYSPLUGDIR)/r2ai$(DOTLIB)
cp -f r2ai$(DOTEXE) $(R2BINDIR)/r2ai$(DOTEXE)
cp -f r2ai$(DOTLIB) $(R2SYSPLUGDIR)
uninstall:
rm -f $(R2SYSPLUGDIR)/r2ai$(DOTLIB)
rm -f $(R2BINDIR)/r2ai$(DOTEXE)
r2check:
@r2 -qcq --
+1 -1
View File
@@ -233,7 +233,7 @@ R_API void process_messages(RCore *core, R2AI_Messages *messages, const char *sy
// Create a tool call response message
R2AI_Message tool_response = {
.role = "tool",
.tool_call_id = strdup (tool_call->id),
.tool_call_id = tool_call->id,
.content = cmd_output
};
+1 -1
View File
@@ -15,7 +15,7 @@ R_API void r2ai_message_free(R2AI_Message *msg) {
}
free (msg->role);
free ((void *)msg->content); // maybe double free
free (msg->content);
free (msg->reasoning_content);
free (msg->tool_call_id);
+6 -4
View File
@@ -161,9 +161,12 @@ R_IPI R2AI_ChatResponse *r2ai_openai(RCore *core, R2AIArgs args) {
headers[1] = auth_header;
}
const char *urlfmt = strcmp (args.provider, "ollama")
? "%s/chat/completions" : "%s/chat";
char *openai_url = r_str_newf (urlfmt, base_url);
const char *openai_url;
if (strcmp (args.provider, "ollama") == 0) {
openai_url = r_str_newf ("%s/chat", base_url);
} else {
openai_url = r_str_newf ("%s/chat/completions", base_url);
}
R_LOG_DEBUG ("OpenAI URL: %s", openai_url);
// Create a messages JSON object, either from input messages or from content
@@ -271,7 +274,6 @@ R_IPI R2AI_ChatResponse *r2ai_openai(RCore *core, R2AIArgs args) {
int code = 0;
res = r2ai_http_post (core, openai_url, headers, complete_json, &code, NULL);
free (complete_json);
free (openai_url);
if (code != 200) {
R_LOG_ERROR ("OpenAI API error %d", code);
+1 -1
View File
@@ -1032,7 +1032,7 @@ RCorePlugin r_core_plugin_r2ai_client = {
.name = "r2ai",
.desc = "r2ai plugin in plain C",
.author = "pancake",
.version = "1.1.2",
.version = "1.1.1",
.license = "MIT",
},
.init = r2ai_init,
+13 -13
View File
@@ -40,14 +40,14 @@ typedef struct {
} R2AI_ToolCall;
typedef struct {
char *type;
char *id;
char *name;
char *input;
char *data;
char *thinking;
char *signature;
char *text;
const char *type;
const char *id;
const char *name;
const char *input;
const char *data;
const char *thinking;
const char *signature;
const char *text;
} R2AI_ContentBlock;
typedef struct {
@@ -56,12 +56,12 @@ typedef struct {
} R2AI_ContentBlocks;
typedef struct {
char *role;
const char *role;
const char *content;
char *reasoning_content;
R2AI_ContentBlocks *content_blocks;
char *tool_call_id;
R2AI_ToolCall *tool_calls;
const char *reasoning_content;
const R2AI_ContentBlocks *content_blocks;
const char *tool_call_id;
const R2AI_ToolCall *tool_calls;
int n_tool_calls;
} R2AI_Message;