Compare commits

..
22 Commits
Author SHA1 Message Date
pancake 83b96f7a3e fiif 2025-10-03 18:21:20 +02:00
pancake e2ee8cdbb6 rjo 2025-10-03 17:36:29 +02:00
pancake 8630e95d83 morebecnh 2025-10-03 17:06:20 +02:00
pancake e07293e40f Initial benchmark working 2025-10-03 16:48:56 +02:00
pancake 415c546f92 Bench 2025-10-03 16:01:31 +02:00
pancake 8446af7385 Add description to all the config vars 2025-10-03 15:35:48 +02:00
pancake 5a156a0126 Do not use temperature for the openai provider in decai 2025-10-03 15:35:43 +02:00
pancake a28efa7a33 Add config var descriptions (fixes e??) 2025-10-03 12:46:25 +02:00
pancake bc5adb3fcb Refactor the model listing logic in decai 2025-10-02 14:00:30 +02:00
pancake 4bd1d911e0 Bring back the XAI Provider 2025-10-02 13:28:17 +02:00
pancake 41c9734c69 Bring back the mistral provider 2025-10-02 13:25:31 +02:00
pancake 1a41298dd6 Bring back the gemini provider 2025-10-02 13:22:47 +02:00
pancake c618f8ed09 Bring back the readable prompts 2025-10-02 13:20:42 +02:00
pancake 48932c5c64 Add support for listing models from openai endpoints in the decai 2025-10-02 13:17:08 +02:00
pancake 4da20a5c33 Bring back the OpenAI backend for Decai 2025-10-02 11:23:21 +02:00
pancake b7f94ad419 Release 1.1.1 2025-10-01 15:48:01 +02:00
pancake 2e45fb9e9c Fix spaces 2025-10-01 15:44:20 +02:00
pancake 9775e94455 Add support for ollamacloud to decai 2025-10-01 12:16:12 +02:00
pancake 67416766e2 Move the benchmark scripts into its own directory 2025-10-01 12:07:42 +02:00
pancake 5c1be40048 Improve benchmark2html.sh and makefile 2025-10-01 11:00:13 +02:00
pancake 49e87c7b7a Add script to generate benchmark results in html 2025-10-01 10:22:37 +02:00
pancake 11e7c288e0 Include decai.r2.js in the release page 2025-10-01 02:37:59 +02:00
13 changed files with 1164 additions and 260 deletions

No files matched your search

+4
View File
@@ -139,9 +139,13 @@ jobs:
with:
path: release-artifacts
- name: Add decai.r2.js to release artifacts
run: cp decai/decai.r2.js release-artifacts/
- name: Create GitHub Release
uses: softprops/action-gh-release@v1
with:
files: |
release-artifacts/**/*.zip
release-artifacts/decai.r2.js
generate_release_notes: true
+1 -1
View File
@@ -1 +1 @@
1.1.0
1.1.1
+638 -255
View File
File diff suppressed because it is too large. Load diff
+41
View File
@@ -0,0 +1,41 @@
SHELL=/bin/bash
PROVIDER=ollama
MODELS=granite3-dense:8b mannix/jan-nano:latest yi-coder:latest qwen3:latest deepseek-coder-v2:latest hhao/qwen2.5-coder-tools:latest qwen2.5-coder:latest
MODELS=gemma3:1b
LANGS=Dart Perl Ruby Javascript PHP C
BINARIES=files/complex1 files/complex2 files/complex3
MODELS=gpt-4o
PROVIDER=openai
all:
mkdir -p tmp
for b in $(BINARIES) ; do \
for a in $(MODELS) ; do \
echo "$$b $$a"; \
./run.sh $$b $$a ${PROVIDER} ${LANGS} > tmp/$${b##*/}_$${a//\//_}.html ; \
done ; \
done
.PHONY: run index evaluate
compile: $(BINARIES)
files/complex1: files/complex1.c
gcc -o $@ $<
files/complex2: files/complex2.c
gcc -o $@ $<
files/complex3: files/complex3.c
gcc -o $@ $<
lmstudio:
for b in $(BINARIES) ; do \
for a in $(MODELS) ; do \
./run.sh $$b $$a lmstudio ${LANGS} > $${b##*/}_$${a//\//_}.html ; \
done ; \
done
clean:
rm -rf tmp
+4
View File
@@ -0,0 +1,4 @@
<b>
decai -e model=$a;decai -e lang=$l;s 0x100000924;af;decai -d
</b><br/>
Transform this pseudocode and respond ONLY with plain code (NO explanations, comments or markdown), Change 'goto' into if/else/for/while, Simplify as much as possible, use better variable names, take function arguments and strings from comments like 'string:', Reduce lines of code and fit everything in a single function, Remove all dead code
+59
View File
@@ -0,0 +1,59 @@
#!/bin/bash
# Evaluate decompiled outputs using mai and determine best model+lang
echo "Evaluating decompiled outputs..."
tmpfile=$(mktemp)
MAI="mai -q -p openai -m gpt-4o"
# MAI="mai -q"
for orig in files/complex1.c files/complex2.c files/complex3.c; do
base=$(basename $orig .c)
original_code=$(cat $orig)
for f in tmp/${base}_*.txt; do
if [ -f "$f" ]; then
decompiled_code=$(cat $f)
#echo "Comparing $orig $f"
result=`echo n | ${MAI} -q -r "/template files/CMPROMPT.txt original=@${orig} decompiled=@${f}"`
score=`echo "$result" | head -n 1 | grep -oE '[0-9]+'` # |head -n 1`
perfect=`echo "$result" | grep 'No sign' && echo yes`
[ -z "$score" ] && score=0
if [ "$perfect" = yes ]; then
score=100
else
score="$((0+${score}))"
fi
echo "$score\t$f"
if [ -n "$score" ]; then
key="${f%.*}" # remove .txt
# Check if key exists in tmpfile
existing=$(grep "^$key " $tmpfile | awk '{print $2}')
if [ -n "$existing" ]; then
new_score=$(( existing + score ))
sed -i "s/^$key .*/$key $new_score/" $tmpfile
else
echo "$key $score" >> $tmpfile
fi
fi
fi
done
done
# Find the best (highest total score)
best_key=""
best_score=0
while read key score; do
if [ "$score" -gt "$best_score" ]; then
best_score=$score
best_key=$key
fi
done < $tmpfile
rm $tmpfile
if [ -n "$best_key" ]; then
echo "Best model+language combination: $best_key with total score $best_score"
else
echo "No scores found"
fi
+17
View File
@@ -0,0 +1,17 @@
Compare the two provided code in C with the decompiled counterpart. Ignore function or variable names, syntax differences, language-specific constructs, and formatting. Focus solely on what the code does: the operations performed, control flow, logic, and functional behavior. Determine if they are functionally equivalent in terms of their effects and computations.
Output only:
- A numeric score from 1 to 100 (100 being identical logic and functionality)
- Followed by a single sentence describing the most relevant semantic differences (or "No significant semantic differences" if score is 95+)
**Original Source Code:**
```C
{original}
```
**Decompiled Code:**
```
{decompiled}
```
+21
View File
@@ -0,0 +1,21 @@
#include <stdio.h>
#include <stdlib.h>
int main() {
int n = 5;
int *arr = (int *)malloc(n * sizeof(int));
if (arr == NULL) return 1;
for (int i = 0; i < n; i++) {
arr[i] = i * i;
}
int sum = 0;
for (int i = 0; i < n; i++) {
sum += arr[i];
for (int j = 0; j < i; j++) {
sum -= j;
}
}
printf("Sum: %d\n", sum);
free(arr);
return 0;
}
+39
View File
@@ -0,0 +1,39 @@
#include <stdio.h>
#include <stdlib.h>
struct Node {
int data;
struct Node *next;
};
int main() {
struct Node *head = NULL;
struct Node *tail = NULL;
for (int i = 0; i < 10; i++) {
struct Node *newNode = (struct Node *)malloc(sizeof(struct Node));
if (newNode == NULL) return 1;
newNode->data = i * 2;
newNode->next = NULL;
if (head == NULL) {
head = newNode;
tail = newNode;
} else {
tail->next = newNode;
tail = newNode;
}
}
struct Node *current = head;
int sum = 0;
while (current != NULL) {
sum += current->data;
current = current->next;
}
printf("Sum: %d\n", sum);
current = head;
while (current != NULL) {
struct Node *temp = current;
current = current->next;
free(temp);
}
return 0;
}
+33
View File
@@ -0,0 +1,33 @@
#include <stdio.h>
int main() {
int x = 42;
int y = 0;
switch (x & 15) {
case 0:
y = x << 1;
break;
case 1:
y = x >> 1;
break;
case 2:
y = x ^ 255;
break;
case 3:
y = ~x;
break;
default:
y = x | 128;
break;
}
if ((y & 1) == 0) {
y += 10;
} else {
y -= 5;
}
for (int i = 0; i < 5; i++) {
y = (y << 2) | (y >> 30);
}
printf("Result: %d\n", y);
return 0;
}
+50
View File
@@ -0,0 +1,50 @@
#!/bin/bash
cp decai/BENCH.txt tmp/BENCH.txt
# Find all directories containing BENCH.txt
dirs=$(find . -name "BENCH.txt" -type f | xargs dirname | sort | uniq)
# Start HTML output
cat <<EOF > benchmark_index.html
<html>
<table>
EOF
# Process each directory
for dir in $dirs; do
cat <<EOF >> benchmark_index.html
<tr> <td>
<div>
EOF
# Add contents of BENCH.txt
cat "$dir/BENCH.txt" >> benchmark_index.html
cat <<EOF >> benchmark_index.html
</div>
</td>
<td>
EOF
# Find and add links to HTML files
for html_file in $(ls "$dir"/*.html 2>/dev/null | sort); do
base=$(basename "$html_file" .html)
cat <<EOF >> benchmark_index.html
<a href="$dir/$base.html">$base</a>
<br />
EOF
done
cat <<EOF >> benchmark_index.html
</td>
</tr>
EOF
done
# End HTML output
cat <<EOF >> benchmark_index.html
</table>
<br />
<br />
</html>
EOF
+231
View File
@@ -0,0 +1,231 @@
#!/bin/bash
# Generate HTML output for command executions with collapsible sections and width control
# If PLAIN=1, output plain text instead
if [ $# -lt 4 ]; then
echo "Usage: $0 <binary> <model> <provider> <lang1> [lang2 ...]"
echo "Or: $0 command1 [command2 ...] (legacy)"
exit 1
fi
# Check if first arg is a file (binary), assume new format
if [ -f "$1" ]; then
binary="$1"
model="$2"
provider="$3"
shift 3
langs=("$@")
commands=()
for lang in "${langs[@]}"; do
# commands+=("# $lang echo \"decai -e model=$model;decai -e lang=$lang;decai -e api=$provider;af;decai -d\" | r2 -q $binary 2> /dev/null")
commands+=("# $lang r2 -e scr.color=0 -qcq -qc \"decai -e model=$model;decai -e lang=$lang;decai -e api=$provider;af;decai -d\" $binary")
done
set -- "${commands[@]}"
else
# Legacy mode
commands=("$@")
fi
if [ "$PLAIN" = "1" ]; then
# In plain mode, execute each command and print raw output.
# Also, if TXTFILE is set, save combined raw output there.
tmp_plain=$(mktemp)
for cmd in "$@"; do
if [[ "$cmd" =~ ^#[[:space:]]*(.*) ]]; then
exec_cmd="${BASH_REMATCH[1]}"
else
exec_cmd="$cmd"
fi
tmpout=$(mktemp)
eval "$exec_cmd" > "$tmpout" 2>&1 || true
mv "$tmpout" "$tmp_plain"
rm -f "$tmpout"
done
if [ -n "$TXTFILE" ]; then
mv "$tmp_plain" "$TXTFILE"
else
rm -f "$tmp_plain"
fi
exit 0
fi
# Timeout in seconds for each command (configurable via TIMEOUT environment variable)
TIMEOUT=${TIMEOUT:-30}
cat << 'EOF'
<!DOCTYPE html>
<html lang="en">
<head>
<meta charset="UTF-8">
<meta name="viewport" content="width=device-width, initial-scale=1.0">
<title>Command Execution Results</title>
<style>
* {
box-sizing: border-box;
}
:root {
--max-width: 400px;
}
body {
font-family: monospace;
margin: 0;
padding: 20px;
background-color: #f5f5f5;
overflow-x: hidden;
}
.slider-container {
background: white;
padding: 15px;
margin-bottom: 20px;
box-shadow: 0 2px 4px rgba(0,0,0,0.1);
width: 100%;
}
.slider-container label {
display: block;
margin-bottom: 10px;
font-weight: bold;
}
.slider-container input {
width: 100%;
}
.commands-container {
display: flex;
flex-wrap: wrap;
gap: 15px;
align-items: flex-start;
}
.command-section {
background: white;
border-radius: 8px;
box-shadow: 0 2px 4px rgba(0,0,0,0.1);
overflow: hidden;
width: var(--max-width);
}
.command-title {
background: #e9ecef;
padding: 12px 15px;
cursor: pointer;
border: none;
width: 100%;
text-align: left;
font-family: inherit;
font-size: inherit;
border-radius: 8px 8px 0 0;
position: relative;
}
.command-title:hover {
background: #dee2e6;
}
.command-time {
position: absolute;
top: 12px;
right: 15px;
color: #6c757d;
font-size: 0.9em;
}
.command-body {
padding: 15px;
border-top: 1px solid #dee2e6;
display: none;
}
.command-output {
background: #f8f9fa;
padding: 10px;
border-radius: 4px;
white-space: pre-wrap;
word-wrap: break-word;
overflow-x: auto;
}
.command-section.expanded .command-body {
display: block;
}
</style>
</head>
<body>
<div class="slider-container">
<label for="width-slider">Maximum Width: <span id="width-value">400</span>px</label>
<input type="range" id="width-slider" min="200" max="2000" value="400" step="50">
</div>
<div class="commands-container">
EOF
# Process each command
for cmd in "$@"; do
# Extract title: if command starts with #, parse title and command, else use the command
if [[ "$cmd" =~ ^#[[:space:]]*([^[:space:]]+)[[:space:]]*(.*) ]]; then
title="${BASH_REMATCH[1]}"
exec_cmd="${BASH_REMATCH[2]}"
else
title="$cmd"
exec_cmd="$cmd"
fi
echo "RUN $exec_cmd" > /dev/stderr
# Execute command and time it
start_time=$(date +%s.%N)
tmpfile=$(mktemp)
# echo "eval $exec_cmd > $tmpfile" > /dev/stder
eval "$exec_cmd" > "$tmpfile" 2> /dev/null &
pid=$!
(sleep $TIMEOUT && kill $pid 2>/dev/null) &
wait $pid 2>/dev/null
output=$(cat "$tmpfile")
#echo "$output" > /dev/stderr
rm "$tmpfile"
exit_code=$?
end_time=$(date +%s.%N)
# Calculate execution time
execution_time=$(echo "$end_time - $start_time" | bc -l 2>/dev/null || echo "0")
if [ $? -ne 0 ]; then
execution_time="N/A"
else
execution_time=$(printf "%.3f" "$execution_time")
fi
# Save plain output to .txt file if in new format
if [ -n "$binary" ]; then
txtfile="tmp/${binary##*/}_${model//\//_}_${title}.txt"
mkdir -p "$(dirname "$txtfile")" 2>/dev/null || true
echo "$output" > "$txtfile"
fi
# Escape HTML in output
#escaped_output=$(echo "$output" | sed 's/&/\&amp;/g; s/</\&lt;/g; s/>/\&gt;/g')
escaped_output="$output"
cat << EOF
<div class="command-section expanded">
<button class="command-title" onclick="toggleSection(this)">$title<span class="command-time">${execution_time}s</span></button>
<div class="command-body">
<div class="command-output">$escaped_output</div>
</div>
</div>
EOF
done
cat << 'EOF'
</div>
<script>
// Slider functionality
const slider = document.getElementById('width-slider');
const widthValue = document.getElementById('width-value');
slider.addEventListener('input', function() {
const value = this.value;
document.documentElement.style.setProperty('--max-width', value + 'px');
widthValue.textContent = value;
});
// Toggle functionality
function toggleSection(button) {
const section = button.parentElement;
section.classList.toggle('expanded');
}
</script>
</body>
</html>
EOF
+26 -4
View File
@@ -907,18 +907,31 @@ static int r2ai_init(void *user, const char *input) {
r_config_lock (core->config, false);
r_config_set_cb (core->config, "r2ai.api", "openai", &cb_r2ai_api);
r_config_desc (core->config, "r2ai.api", "LLM provider to use (openai, gemini, anthropic, ollama, ...)");
r_config_set_cb (core->config, "r2ai.model", "gpt-5-mini", &cb_r2ai_model);
r_config_desc (core->config, "r2ai.model", "Model identifier for the selected provider (e.g. gpt-5-mini)");
r_config_set (core->config, "r2ai.baseurl", "");
r_config_desc (core->config, "r2ai.baseurl", "Base URL for provider API (overrides default endpoints)");
r_config_set_i (core->config, "r2ai.max_tokens", 4096); // max output tokens, or max total tokens
r_config_desc (core->config, "r2ai.max_tokens", "Maximum tokens for LLM responses (output/total depending on provider)");
r_config_set_i (core->config, "r2ai.thinking_tokens", 0);
r_config_desc (core->config, "r2ai.thinking_tokens", "Number of tokens reserved for internal thinking/context messages");
r_config_set (core->config, "r2ai.temperature", "0.01");
r_config_desc (core->config, "r2ai.temperature", "Sampling temperature for LLM output (0 = deterministic)");
r_config_set (core->config, "r2ai.cmds", "pdc");
r_config_desc (core->config, "r2ai.cmds", "Default command sequence used by automation (e.g. 'pdc')");
r_config_set (core->config, "r2ai.lang", "C");
r_config_desc (core->config, "r2ai.lang", "Programming language hint used in prompts (e.g. C, C++, Rust)");
r_config_set_b (core->config, "r2ai.data", false);
r_config_desc (core->config, "r2ai.data", "Enable local data/embeddings for query context retrieval");
r_config_set_b (core->config, "r2ai.data.reason", false);
r_config_desc (core->config, "r2ai.data.reason", "Include reasoning/explanations from local data when building context");
r_config_set (core->config, "r2ai.data.path", "/tmp/embeds");
r_config_desc (core->config, "r2ai.data.path", "Path to local embeddings/text files used for context (one .txt per document)");
r_config_set_i (core->config, "r2ai.data.nth", 10);
r_config_desc (core->config, "r2ai.data.nth", "Number of top-matching documents to include from local data for context");
r_config_set (core->config, "r2ai.hlang", "english");
r_config_desc (core->config, "r2ai.hlang", "Human language for prompts/messages (e.g. english)");
r_config_set (
core->config, "r2ai.system",
"You are a reverse engineer. The user is reversing a binary, using "
@@ -931,21 +944,30 @@ static int r2ai_init(void *user, const char *input) {
"possible, use better variable names, take function arguments and "
"strings from comments like 'string:'");
r_config_set_b (core->config, "r2ai.stream", false);
r_config_desc (core->config, "r2ai.stream", "Enable streaming responses from the LLM (true/false)");
r_config_set_i (core->config, "r2ai.auto.max_runs", 50);
r_config_desc (core->config, "r2ai.auto.max_runs", "Maximum number of automated steps/runs in auto mode");
r_config_set_b (core->config, "r2ai.auto.hide_tool_output", false);
r_config_desc (core->config, "r2ai.auto.hide_tool_output", "Hide tool output when running automated actions");
r_config_set (core->config, "r2ai.auto.init_commands", "aaa;iI;afl");
r_config_desc (core->config, "r2ai.auto.init_commands", "Initial commands executed when auto mode starts (semicolon separated)");
r_config_set_b (core->config, "r2ai.auto.yolo", false);
r_config_desc (core->config, "r2ai.auto.yolo", "Execute potentially dangerous commands in auto mode without asking");
r_config_set_b (core->config, "r2ai.auto.reset_on_query", false);
r_config_desc (core->config, "r2ai.auto.reset_on_query", "Reset auto-mode conversation state on new user queries");
r_config_set_b (core->config, "r2ai.chat.show_cost", true);
r_config_desc (core->config, "r2ai.chat.show_cost", "Display estimated API cost for chat interactions");
// Configure HTTP timeout in seconds
r_config_set_i (core->config, "r2ai.http.timeout", 120);
// Configure HTTP rate limiting and retry parameters
r_config_desc (core->config, "r2ai.http.timeout", "HTTP client timeout (seconds) for provider API calls");
r_config_set_i (core->config, "r2ai.http.max_retries", 10);
r_config_desc (core->config, "r2ai.http.max_retries", "Maximum number of HTTP retries for failed requests");
r_config_set_i (core->config, "r2ai.http.max_backoff", 30);
// Configure HTTP backend and options
r_config_desc (core->config, "r2ai.http.max_backoff", "Maximum backoff time (seconds) between HTTP retries");
r_config_set (core->config, "r2ai.http.backend", "auto"); // Options: auto, libcurl, socket, system
r_config_desc (core->config, "r2ai.http.backend", "HTTP backend to use (auto, libcurl, socket, system)");
r_config_set_b (core->config, "r2ai.http.use_files", false);
r_config_desc (core->config, "r2ai.http.use_files", "Use temporary files to pass HTTP request/response payloads (true/false)");
r_config_lock (core->config, true);
return true;
}
@@ -1010,7 +1032,7 @@ RCorePlugin r_core_plugin_r2ai_client = {
.name = "r2ai",
.desc = "r2ai plugin in plain C",
.author = "pancake",
.version = "1.1.0",
.version = "1.1.1",
.license = "MIT",
},
.init = r2ai_init,