fiif

examples/benchmark/Makefile

@@ -1,14 +1,41 @@
 SHELL=/bin/bash
 
-MODELS=gpt-oss:20b gemma3:4b gemma3:12b
-MODELS+=granite3-dense:8b mannix/jan-nano:latest yi-coder:latest qwen3:latest deepseek-coder-v2:latest hhao/qwen2.5-coder-tools:latest qwen2.5-coder:latest
+PROVIDER=ollama
 MODELS=granite3-dense:8b mannix/jan-nano:latest yi-coder:latest qwen3:latest deepseek-coder-v2:latest hhao/qwen2.5-coder-tools:latest qwen2.5-coder:latest
+MODELS=gemma3:1b
 LANGS=Dart Perl Ruby Javascript PHP C
+BINARIES=files/complex1 files/complex2 files/complex3
+MODELS=gpt-4o
+PROVIDER=openai
 
 all:
-	for a in ${MODELS} ; do \
-		declare -a ARGS; \
-		for l in ${LANGS} ; do \
-			ARGS+=("echo \"decai -e model=$$a;decai -e lang=$$l;s 0x100000924;af;decai -d\" | r2 -q /bin/ls"); \
+	mkdir -p tmp
+	for b in $(BINARIES) ; do \
+		for a in $(MODELS) ; do \
+			echo "$$b $$a"; \
+			./run.sh $$b $$a ${PROVIDER} ${LANGS} > tmp/$${b##*/}_$${a//\//_}.html ; \
 		done ; \
-		./run.sh "$${ARGS[@]}" > ${a//\//_}.html ; done
+	done
+
+.PHONY: run index evaluate
+
+compile: $(BINARIES)
+
+files/complex1: files/complex1.c
+	gcc -o $@ $<
+
+files/complex2: files/complex2.c
+	gcc -o $@ $<
+
+files/complex3: files/complex3.c
+	gcc -o $@ $<
+
+lmstudio:
+	for b in $(BINARIES) ; do \
+		for a in $(MODELS) ; do \
+			./run.sh $$b $$a lmstudio ${LANGS} > $${b##*/}_$${a//\//_}.html ; \
+		done ; \
+	done
+
+clean:
+	rm -rf tmp

examples/benchmark/evaluate.sh

@@ -0,0 +1,59 @@
+#!/bin/bash
+
+# Evaluate decompiled outputs using mai and determine best model+lang
+
+echo "Evaluating decompiled outputs..."
+
+tmpfile=$(mktemp)
+MAI="mai -q -p openai -m gpt-4o"
+# MAI="mai -q"
+
+for orig in files/complex1.c files/complex2.c files/complex3.c; do
+    base=$(basename $orig .c)
+    original_code=$(cat $orig)
+    for f in tmp/${base}_*.txt; do
+        if [ -f "$f" ]; then
+            decompiled_code=$(cat $f)
+	    #echo "Comparing $orig $f"
+	    result=`echo n | ${MAI} -q -r "/template files/CMPROMPT.txt original=@${orig} decompiled=@${f}"`
+	    score=`echo "$result" | head -n 1 | grep -oE '[0-9]+'` # |head -n 1`
+	    perfect=`echo "$result" | grep 'No sign' && echo yes`
+		    [ -z "$score" ] && score=0
+	    if [ "$perfect" = yes ]; then
+		    score=100
+	    else
+		    score="$((0+${score}))"
+	    fi
+	    echo "$score\t$f"
+            if [ -n "$score" ]; then
+                key="${f%.*}"  # remove .txt
+                # Check if key exists in tmpfile
+                existing=$(grep "^$key " $tmpfile | awk '{print $2}')
+                if [ -n "$existing" ]; then
+                    new_score=$(( existing + score ))
+                    sed -i "s/^$key .*/$key $new_score/" $tmpfile
+                else
+                    echo "$key $score" >> $tmpfile
+                fi
+            fi
+        fi
+    done
+done
+
+# Find the best (highest total score)
+best_key=""
+best_score=0
+while read key score; do
+    if [ "$score" -gt "$best_score" ]; then
+        best_score=$score
+        best_key=$key
+    fi
+done < $tmpfile
+
+rm $tmpfile
+
+if [ -n "$best_key" ]; then
+    echo "Best model+language combination: $best_key with total score $best_score"
+else
+    echo "No scores found"
+fi

examples/benchmark/files/CMPROMPT.txt

@@ -0,0 +1,17 @@
+Compare the two provided code in C with the decompiled counterpart. Ignore function or variable names, syntax differences, language-specific constructs, and formatting. Focus solely on what the code does: the operations performed, control flow, logic, and functional behavior. Determine if they are functionally equivalent in terms of their effects and computations.
+
+Output only:
+- A numeric score from 1 to 100 (100 being identical logic and functionality)
+- Followed by a single sentence describing the most relevant semantic differences (or "No significant semantic differences" if score is 95+)
+
+**Original Source Code:**
+
+```C
+{original}
+```
+
+**Decompiled Code:**
+
+```
+{decompiled}
+```

examples/benchmark/files/complex1.c

@@ -0,0 +1,21 @@
+#include <stdio.h>
+#include <stdlib.h>
+
+int main() {
+    int n = 5;
+    int *arr = (int *)malloc(n * sizeof(int));
+    if (arr == NULL) return 1;
+    for (int i = 0; i < n; i++) {
+        arr[i] = i * i;
+    }
+    int sum = 0;
+    for (int i = 0; i < n; i++) {
+        sum += arr[i];
+        for (int j = 0; j < i; j++) {
+            sum -= j;
+        }
+    }
+    printf("Sum: %d\n", sum);
+    free(arr);
+    return 0;
+}

examples/benchmark/files/complex2.c

@@ -0,0 +1,39 @@
+#include <stdio.h>
+#include <stdlib.h>
+
+struct Node {
+    int data;
+    struct Node *next;
+};
+
+int main() {
+    struct Node *head = NULL;
+    struct Node *tail = NULL;
+    for (int i = 0; i < 10; i++) {
+        struct Node *newNode = (struct Node *)malloc(sizeof(struct Node));
+        if (newNode == NULL) return 1;
+        newNode->data = i * 2;
+        newNode->next = NULL;
+        if (head == NULL) {
+            head = newNode;
+            tail = newNode;
+        } else {
+            tail->next = newNode;
+            tail = newNode;
+        }
+    }
+    struct Node *current = head;
+    int sum = 0;
+    while (current != NULL) {
+        sum += current->data;
+        current = current->next;
+    }
+    printf("Sum: %d\n", sum);
+    current = head;
+    while (current != NULL) {
+        struct Node *temp = current;
+        current = current->next;
+        free(temp);
+    }
+    return 0;
+}

examples/benchmark/files/complex3.c

@@ -0,0 +1,33 @@
+#include <stdio.h>
+
+int main() {
+    int x = 42;
+    int y = 0;
+    switch (x & 15) {
+        case 0:
+            y = x << 1;
+            break;
+        case 1:
+            y = x >> 1;
+            break;
+        case 2:
+            y = x ^ 255;
+            break;
+        case 3:
+            y = ~x;
+            break;
+        default:
+            y = x | 128;
+            break;
+    }
+    if ((y & 1) == 0) {
+        y += 10;
+    } else {
+        y -= 5;
+    }
+    for (int i = 0; i < 5; i++) {
+        y = (y << 2) | (y >> 30);
+    }
+    printf("Result: %d\n", y);
+    return 0;
+}

examples/benchmark/index.sh

@@ -1,5 +1,6 @@
 #!/bin/bash
 
+cp decai/BENCH.txt tmp/BENCH.txt
 # Find all directories containing BENCH.txt
 dirs=$(find . -name "BENCH.txt" -type f | xargs dirname | sort | uniq)
 
@@ -46,4 +47,4 @@ cat <<EOF >> benchmark_index.html
 <br />
 <br />
 </html>
-EOF
+EOF

examples/benchmark/run.sh

@@ -1,12 +1,55 @@
 #!/bin/bash
 
 # Generate HTML output for command executions with collapsible sections and width control
+# If PLAIN=1, output plain text instead
 
-if [ $# -eq 0 ]; then
-    echo "Usage: $0 command1 [command2 ...]"
+if [ $# -lt 4 ]; then
+    echo "Usage: $0 <binary> <model> <provider> <lang1> [lang2 ...]"
+    echo "Or: $0 command1 [command2 ...] (legacy)"
     exit 1
 fi
 
+# Check if first arg is a file (binary), assume new format
+if [ -f "$1" ]; then
+    binary="$1"
+    model="$2"
+    provider="$3"
+    shift 3
+    langs=("$@")
+    commands=()
+    for lang in "${langs[@]}"; do
+       # commands+=("# $lang echo \"decai -e model=$model;decai -e lang=$lang;decai -e api=$provider;af;decai -d\" | r2 -q $binary 2> /dev/null")
+       commands+=("# $lang r2 -e scr.color=0 -qcq -qc \"decai -e model=$model;decai -e lang=$lang;decai -e api=$provider;af;decai -d\" $binary")
+    done
+    set -- "${commands[@]}"
+else
+    # Legacy mode
+    commands=("$@")
+fi
+
+if [ "$PLAIN" = "1" ]; then
+    # In plain mode, execute each command and print raw output.
+    # Also, if TXTFILE is set, save combined raw output there.
+    tmp_plain=$(mktemp)
+    for cmd in "$@"; do
+        if [[ "$cmd" =~ ^#[[:space:]]*(.*) ]]; then
+            exec_cmd="${BASH_REMATCH[1]}"
+        else
+            exec_cmd="$cmd"
+        fi
+        tmpout=$(mktemp)
+        eval "$exec_cmd" > "$tmpout" 2>&1 || true
+        mv "$tmpout" "$tmp_plain"
+        rm -f "$tmpout"
+    done
+    if [ -n "$TXTFILE" ]; then
+        mv "$tmp_plain" "$TXTFILE"
+    else
+        rm -f "$tmp_plain"
+    fi
+    exit 0
+fi
+
 # Timeout in seconds for each command (configurable via TIMEOUT environment variable)
 TIMEOUT=${TIMEOUT:-30}
 
@@ -109,24 +152,27 @@ EOF
 
 # Process each command
 for cmd in "$@"; do
-    # Extract title: if command starts with #, use the rest as title, else use the command
-    if [[ "$cmd" =~ ^#[[:space:]]*(.*) ]]; then
+    # Extract title: if command starts with #, parse title and command, else use the command
+    if [[ "$cmd" =~ ^#[[:space:]]*([^[:space:]]+)[[:space:]]*(.*) ]]; then
         title="${BASH_REMATCH[1]}"
-        exec_cmd="${cmd#*#}"
-        exec_cmd="${exec_cmd#"${exec_cmd%%[![:space:]]*}"}"  # trim leading spaces
+        exec_cmd="${BASH_REMATCH[2]}"
     else
         title="$cmd"
         exec_cmd="$cmd"
     fi
 
+    echo "RUN $exec_cmd" > /dev/stderr
     # Execute command and time it
     start_time=$(date +%s.%N)
     tmpfile=$(mktemp)
-    eval "$exec_cmd" > "$tmpfile" 2>&1 &
+    # echo "eval $exec_cmd > $tmpfile" > /dev/stder
+    eval "$exec_cmd" > "$tmpfile" 2> /dev/null &
     pid=$!
     (sleep $TIMEOUT && kill $pid 2>/dev/null) &
     wait $pid 2>/dev/null
     output=$(cat "$tmpfile")
+    #echo "$output" > /dev/stderr
+
     rm "$tmpfile"
     exit_code=$?
     end_time=$(date +%s.%N)
@@ -139,8 +185,16 @@ for cmd in "$@"; do
         execution_time=$(printf "%.3f" "$execution_time")
     fi
 
+    # Save plain output to .txt file if in new format
+    if [ -n "$binary" ]; then
+        txtfile="tmp/${binary##*/}_${model//\//_}_${title}.txt"
+        mkdir -p "$(dirname "$txtfile")" 2>/dev/null || true
+        echo "$output" > "$txtfile"
+    fi
+
     # Escape HTML in output
-    escaped_output=$(echo "$output" | sed 's/&/\&amp;/g; s/</\&lt;/g; s/>/\&gt;/g')
+    #escaped_output=$(echo "$output" | sed 's/&/\&amp;/g; s/</\&lt;/g; s/>/\&gt;/g')
+    escaped_output="$output"
 
     cat << EOF
         <div class="command-section expanded">