mirror of
https://github.com/KeygraphHQ/validation-benchmarks.git
synced 2026-11-07 10:56:03 -10:00
cleanup
This commit is contained in:
ajmallesh committed
1 parent
243198d8e5
commit
6b38ee518e
6 files changed
-462
No files matched your search
@@ -1,67 +0,0 @@
|
||||
# Validation Benchmarks - Pentest Orchestration Guide
|
||||
|
||||
## One-Time Setup
|
||||
|
||||
### 1. Create workspace and results directories
|
||||
```bash
|
||||
mkdir -p /Users/arjunmalleswaran/Code/pentest-workspaces
|
||||
mkdir -p /Users/arjunmalleswaran/Code/pentest-results
|
||||
```
|
||||
|
||||
### 2. Get list of all benchmarks
|
||||
```bash
|
||||
cd /Users/arjunmalleswaran/Code/validation-benchmarks
|
||||
ls benchmarks/ > benchmark-list.txt
|
||||
```
|
||||
|
||||
## Per-Benchmark Workflow
|
||||
|
||||
### Step 1: Prepare Isolated Workspace
|
||||
```bash
|
||||
# Set benchmark name
|
||||
export BENCHMARK="XBEN-001-24"
|
||||
|
||||
# Copy benchmark to isolated workspace
|
||||
cp -r /Users/arjunmalleswaran/Code/validation-benchmarks/benchmarks/$BENCHMARK \
|
||||
/Users/arjunmalleswaran/Code/pentest-workspaces/$BENCHMARK
|
||||
|
||||
# Navigate to workspace
|
||||
cd /Users/arjunmalleswaran/Code/pentest-workspaces/$BENCHMARK
|
||||
```
|
||||
|
||||
### Step 2: Build and Run Benchmark
|
||||
update the XBEN Makefile from `include ../../common.mk` to `include ../common.mk`
|
||||
|
||||
```bash
|
||||
# Build the benchmark
|
||||
make build
|
||||
|
||||
# Run the benchmark
|
||||
make run
|
||||
|
||||
# Get the URL/port from docker-compose or logs
|
||||
# Common pattern: http://localhost:XXXX
|
||||
```
|
||||
|
||||
### Step 4: Run Shannon Pentest
|
||||
```bash
|
||||
# Example command structure:
|
||||
cd /Users/arjunmalleswaran/Code/shannon
|
||||
export CLAUDE_CODE_MAX_OUTPUT_TOKENS=64000
|
||||
./shannon.mjs http://localhost:X /Users/arjunmalleswaran/Code/pentest-workspaces/$BENCHMARK
|
||||
```
|
||||
|
||||
### Step 5: Collect Results
|
||||
```bash
|
||||
# Create results directory for this benchmark
|
||||
mkdir -p /Users/arjunmalleswaran/Code/pentest-results/$BENCHMARK
|
||||
|
||||
# Copy deliverables from workspace
|
||||
cp -r /Users/arjunmalleswaran/Code/pentest-workspaces/$BENCHMARK/deliverables \
|
||||
/Users/arjunmalleswaran/Code/pentest-results/$BENCHMARK/
|
||||
|
||||
# Find and copy the latest audit-logs folder
|
||||
export LATEST_AUDIT_LOG=folder-name-of-logs-in-shannon
|
||||
cp -r /Users/arjunmalleswaran/Code/shannon/audit-logs/$LATEST_AUDIT_LOG \
|
||||
/Users/arjunmalleswaran/Code/pentest-results/$BENCHMARK/audit-logs
|
||||
```
|
||||
@@ -1,104 +0,0 @@
|
||||
XBEN-001-24
|
||||
XBEN-002-24
|
||||
XBEN-003-24
|
||||
XBEN-004-24
|
||||
XBEN-005-24
|
||||
XBEN-006-24
|
||||
XBEN-007-24
|
||||
XBEN-008-24
|
||||
XBEN-009-24
|
||||
XBEN-010-24
|
||||
XBEN-011-24
|
||||
XBEN-012-24
|
||||
XBEN-013-24
|
||||
XBEN-014-24
|
||||
XBEN-015-24
|
||||
XBEN-016-24
|
||||
XBEN-017-24
|
||||
XBEN-018-24
|
||||
XBEN-019-24
|
||||
XBEN-020-24
|
||||
XBEN-021-24
|
||||
XBEN-022-24
|
||||
XBEN-023-24
|
||||
XBEN-024-24
|
||||
XBEN-025-24
|
||||
XBEN-026-24
|
||||
XBEN-027-24
|
||||
XBEN-028-24
|
||||
XBEN-029-24
|
||||
XBEN-030-24
|
||||
XBEN-031-24
|
||||
XBEN-032-24
|
||||
XBEN-033-24
|
||||
XBEN-034-24
|
||||
XBEN-035-24
|
||||
XBEN-036-24
|
||||
XBEN-037-24
|
||||
XBEN-038-24
|
||||
XBEN-039-24
|
||||
XBEN-040-24
|
||||
XBEN-041-24
|
||||
XBEN-042-24
|
||||
XBEN-043-24
|
||||
XBEN-044-24
|
||||
XBEN-045-24
|
||||
XBEN-046-24
|
||||
XBEN-047-24
|
||||
XBEN-048-24
|
||||
XBEN-049-24
|
||||
XBEN-050-24
|
||||
XBEN-051-24
|
||||
XBEN-052-24
|
||||
XBEN-053-24
|
||||
XBEN-054-24
|
||||
XBEN-055-24
|
||||
XBEN-056-24
|
||||
XBEN-057-24
|
||||
XBEN-058-24
|
||||
XBEN-059-24
|
||||
XBEN-060-24
|
||||
XBEN-061-24
|
||||
XBEN-062-24
|
||||
XBEN-063-24
|
||||
XBEN-064-24
|
||||
XBEN-065-24
|
||||
XBEN-066-24
|
||||
XBEN-067-24
|
||||
XBEN-068-24
|
||||
XBEN-069-24
|
||||
XBEN-070-24
|
||||
XBEN-071-24
|
||||
XBEN-072-24
|
||||
XBEN-073-24
|
||||
XBEN-074-24
|
||||
XBEN-075-24
|
||||
XBEN-076-24
|
||||
XBEN-077-24
|
||||
XBEN-078-24
|
||||
XBEN-079-24
|
||||
XBEN-080-24
|
||||
XBEN-081-24
|
||||
XBEN-082-24
|
||||
XBEN-083-24
|
||||
XBEN-084-24
|
||||
XBEN-085-24
|
||||
XBEN-086-24
|
||||
XBEN-087-24
|
||||
XBEN-088-24
|
||||
XBEN-089-24
|
||||
XBEN-090-24
|
||||
XBEN-091-24
|
||||
XBEN-092-24
|
||||
XBEN-093-24
|
||||
XBEN-094-24
|
||||
XBEN-095-24
|
||||
XBEN-096-24
|
||||
XBEN-097-24
|
||||
XBEN-098-24
|
||||
XBEN-099-24
|
||||
XBEN-100-24
|
||||
XBEN-101-24
|
||||
XBEN-102-24
|
||||
XBEN-103-24
|
||||
XBEN-104-24
|
||||
@@ -1,34 +0,0 @@
|
||||
#!/bin/bash
|
||||
|
||||
# Absolute path to where benchmarks/ lives relative to this script
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
BENCHMARKS_DIR="$SCRIPT_DIR/benchmarks"
|
||||
|
||||
echo "Enter benchmark range (e.g., 005-010):"
|
||||
read range
|
||||
|
||||
IFS='-' read -r start end <<< "$range"
|
||||
|
||||
if ! [[ $start =~ ^[0-9]{3}$ && $end =~ ^[0-9]{3}$ ]]; then
|
||||
echo "❌ Invalid format. Use 3-digit range like 005-010"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Convert to integers for iteration
|
||||
start_num=$((10#$start)) # The 10# prefix avoids octal interpretation
|
||||
end_num=$((10#$end))
|
||||
|
||||
for ((i=start_num; i<=end_num; i++)); do
|
||||
padded=$(printf "%03d" "$i")
|
||||
folder="XBEN-${padded}-24"
|
||||
full_path="$BENCHMARKS_DIR/$folder"
|
||||
|
||||
if [ -f "$full_path/docker-compose.yml" ]; then
|
||||
echo "🚀 Launching $folder ..."
|
||||
(cd "$full_path" && docker-compose up -d --build)
|
||||
else
|
||||
echo "⚠️ Skipping $folder (docker-compose.yml not found)"
|
||||
fi
|
||||
done
|
||||
|
||||
echo "✅ Done."
|
||||
@@ -1,89 +0,0 @@
|
||||
#!/bin/bash
|
||||
|
||||
# Test all 104 XBEN benchmarks in parallel
|
||||
# Outputs failed benchmark names to failed_builds.txt
|
||||
|
||||
set -e
|
||||
|
||||
BENCHMARKS_DIR="benchmarks"
|
||||
FAILED_FILE="failed_builds.txt"
|
||||
SUCCESS_FILE="successful_builds.txt"
|
||||
LOG_DIR="build_logs"
|
||||
PARALLEL_JOBS=8 # Adjust based on your CPU cores
|
||||
|
||||
# Clean up old results
|
||||
rm -f "$FAILED_FILE" "$SUCCESS_FILE"
|
||||
rm -rf "$LOG_DIR"
|
||||
mkdir -p "$LOG_DIR"
|
||||
|
||||
echo "========================================="
|
||||
echo "Testing builds for all 104 XBEN benchmarks"
|
||||
echo "Parallel jobs: $PARALLEL_JOBS"
|
||||
echo "========================================="
|
||||
echo ""
|
||||
|
||||
# Function to test a single benchmark
|
||||
test_benchmark() {
|
||||
local benchmark=$1
|
||||
local benchmark_name=$(basename "$benchmark")
|
||||
local base_dir=$(pwd)
|
||||
local log_file="$base_dir/$LOG_DIR/${benchmark_name}.log"
|
||||
|
||||
echo "[TESTING] $benchmark_name"
|
||||
|
||||
if (cd "$benchmark" && make build > "$log_file" 2>&1); then
|
||||
echo "[SUCCESS] $benchmark_name"
|
||||
echo "$benchmark_name" >> "$base_dir/$SUCCESS_FILE"
|
||||
return 0
|
||||
else
|
||||
echo "[FAILED] $benchmark_name"
|
||||
echo "$benchmark_name" >> "$base_dir/$FAILED_FILE"
|
||||
return 1
|
||||
fi
|
||||
}
|
||||
|
||||
# Export function and variables for parallel execution
|
||||
export -f test_benchmark
|
||||
export LOG_DIR
|
||||
export FAILED_FILE
|
||||
export SUCCESS_FILE
|
||||
|
||||
# Find all benchmark directories and run tests in parallel
|
||||
find "$BENCHMARKS_DIR" -maxdepth 1 -type d -name "XBEN-*-24" | sort | \
|
||||
xargs -I {} -P "$PARALLEL_JOBS" bash -c 'test_benchmark "$@"' _ {}
|
||||
|
||||
echo ""
|
||||
echo "========================================="
|
||||
echo "BUILD TEST COMPLETE"
|
||||
echo "========================================="
|
||||
|
||||
# Count results
|
||||
TOTAL=104
|
||||
SUCCESS_COUNT=0
|
||||
FAIL_COUNT=0
|
||||
|
||||
if [ -f "$SUCCESS_FILE" ]; then
|
||||
SUCCESS_COUNT=$(wc -l < "$SUCCESS_FILE")
|
||||
fi
|
||||
|
||||
if [ -f "$FAILED_FILE" ]; then
|
||||
FAIL_COUNT=$(wc -l < "$FAILED_FILE")
|
||||
fi
|
||||
|
||||
echo "Total benchmarks: $TOTAL"
|
||||
echo "Successful builds: $SUCCESS_COUNT"
|
||||
echo "Failed builds: $FAIL_COUNT"
|
||||
echo ""
|
||||
|
||||
if [ -f "$FAILED_FILE" ] && [ -s "$FAILED_FILE" ]; then
|
||||
echo "Failed benchmarks (saved to $FAILED_FILE):"
|
||||
echo "========================================="
|
||||
cat "$FAILED_FILE"
|
||||
echo ""
|
||||
echo "Build logs available in: $LOG_DIR/"
|
||||
exit 1
|
||||
else
|
||||
echo "All builds passed! ✓"
|
||||
rm -f "$FAILED_FILE"
|
||||
exit 0
|
||||
fi
|
||||
@@ -1,51 +0,0 @@
|
||||
#!/bin/bash
|
||||
|
||||
echo "Testing make build for XBEN-001 through XBEN-024..."
|
||||
echo "=================================================="
|
||||
echo ""
|
||||
|
||||
SUCCESS_COUNT=0
|
||||
FAIL_COUNT=0
|
||||
FAILED_BENCHMARKS=()
|
||||
|
||||
for i in $(seq -f "%03g" 1 24); do
|
||||
BENCHMARK="XBEN-${i}-24"
|
||||
BENCHMARK_PATH="benchmarks/${BENCHMARK}"
|
||||
|
||||
if [ ! -d "$BENCHMARK_PATH" ]; then
|
||||
echo "⚠️ ${BENCHMARK}: Directory not found, skipping"
|
||||
continue
|
||||
fi
|
||||
|
||||
echo "Building ${BENCHMARK}..."
|
||||
|
||||
# Run make build with timeout of 5 minutes
|
||||
cd "$BENCHMARK_PATH"
|
||||
timeout 300 make build > /tmp/${BENCHMARK}_build.log 2>&1
|
||||
BUILD_EXIT_CODE=$?
|
||||
cd - > /dev/null
|
||||
|
||||
if [ $BUILD_EXIT_CODE -eq 0 ]; then
|
||||
echo "✅ ${BENCHMARK}: SUCCESS"
|
||||
((SUCCESS_COUNT++))
|
||||
else
|
||||
echo "❌ ${BENCHMARK}: FAILED (exit code: $BUILD_EXIT_CODE)"
|
||||
FAILED_BENCHMARKS+=("$BENCHMARK")
|
||||
((FAIL_COUNT++))
|
||||
echo " Log saved to: /tmp/${BENCHMARK}_build.log"
|
||||
fi
|
||||
echo ""
|
||||
done
|
||||
|
||||
echo "=================================================="
|
||||
echo "Build Test Summary:"
|
||||
echo " Successful: ${SUCCESS_COUNT}/24"
|
||||
echo " Failed: ${FAIL_COUNT}/24"
|
||||
|
||||
if [ ${FAIL_COUNT} -gt 0 ]; then
|
||||
echo ""
|
||||
echo "Failed benchmarks:"
|
||||
for benchmark in "${FAILED_BENCHMARKS[@]}"; do
|
||||
echo " - $benchmark"
|
||||
done
|
||||
fi
|
||||
@@ -1,117 +0,0 @@
|
||||
#!/bin/bash
|
||||
|
||||
# Test only the failed benchmarks from failed_builds.txt
|
||||
# Useful for iteratively fixing and retesting broken builds
|
||||
|
||||
set -e
|
||||
|
||||
BENCHMARKS_DIR="benchmarks"
|
||||
FAILED_INPUT="failed_builds.txt"
|
||||
STILL_FAILING="still_failing.txt"
|
||||
NOW_PASSING="now_passing.txt"
|
||||
LOG_DIR="build_logs_failed"
|
||||
PARALLEL_JOBS=8 # Adjust based on your CPU cores
|
||||
|
||||
# Check if failed_builds.txt exists
|
||||
if [ ! -f "$FAILED_INPUT" ]; then
|
||||
echo "Error: $FAILED_INPUT not found!"
|
||||
echo "Run ./test_all_builds.sh first to generate the list of failed builds."
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Count how many to test
|
||||
TOTAL_FAILED=$(wc -l < "$FAILED_INPUT")
|
||||
|
||||
if [ "$TOTAL_FAILED" -eq 0 ]; then
|
||||
echo "No failed builds to test!"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
# Clean up old results
|
||||
rm -f "$STILL_FAILING" "$NOW_PASSING"
|
||||
rm -rf "$LOG_DIR"
|
||||
mkdir -p "$LOG_DIR"
|
||||
|
||||
echo "========================================="
|
||||
echo "Testing $TOTAL_FAILED previously failed benchmarks"
|
||||
echo "Parallel jobs: $PARALLEL_JOBS"
|
||||
echo "========================================="
|
||||
echo ""
|
||||
|
||||
# Function to test a single benchmark
|
||||
test_benchmark() {
|
||||
local benchmark_name=$1
|
||||
local base_dir=$(pwd)
|
||||
local benchmark="$base_dir/$BENCHMARKS_DIR/$benchmark_name"
|
||||
local log_file="$base_dir/$LOG_DIR/${benchmark_name}.log"
|
||||
|
||||
echo "[TESTING] $benchmark_name"
|
||||
|
||||
if [ ! -d "$benchmark" ]; then
|
||||
echo "[ERROR] $benchmark_name - directory not found"
|
||||
echo "$benchmark_name" >> "$base_dir/$STILL_FAILING"
|
||||
return 1
|
||||
fi
|
||||
|
||||
if (cd "$benchmark" && make build > "$log_file" 2>&1); then
|
||||
echo "[FIXED] $benchmark_name ✓"
|
||||
echo "$benchmark_name" >> "$base_dir/$NOW_PASSING"
|
||||
return 0
|
||||
else
|
||||
echo "[FAILED] $benchmark_name"
|
||||
echo "$benchmark_name" >> "$base_dir/$STILL_FAILING"
|
||||
return 1
|
||||
fi
|
||||
}
|
||||
|
||||
# Export function and variables for parallel execution
|
||||
export -f test_benchmark
|
||||
export BENCHMARKS_DIR
|
||||
export LOG_DIR
|
||||
export STILL_FAILING
|
||||
export NOW_PASSING
|
||||
|
||||
# Read failed benchmarks and test them in parallel
|
||||
cat "$FAILED_INPUT" | xargs -I {} -P "$PARALLEL_JOBS" bash -c 'test_benchmark "$@"' _ {}
|
||||
|
||||
echo ""
|
||||
echo "========================================="
|
||||
echo "RETEST COMPLETE"
|
||||
echo "========================================="
|
||||
|
||||
# Count results
|
||||
STILL_FAILING_COUNT=0
|
||||
NOW_PASSING_COUNT=0
|
||||
|
||||
if [ -f "$NOW_PASSING" ]; then
|
||||
NOW_PASSING_COUNT=$(wc -l < "$NOW_PASSING")
|
||||
fi
|
||||
|
||||
if [ -f "$STILL_FAILING" ]; then
|
||||
STILL_FAILING_COUNT=$(wc -l < "$STILL_FAILING")
|
||||
fi
|
||||
|
||||
echo "Previously failed: $TOTAL_FAILED"
|
||||
echo "Now passing: $NOW_PASSING_COUNT ✓"
|
||||
echo "Still failing: $STILL_FAILING_COUNT"
|
||||
echo ""
|
||||
|
||||
if [ "$NOW_PASSING_COUNT" -gt 0 ]; then
|
||||
echo "Fixed benchmarks (saved to $NOW_PASSING):"
|
||||
echo "-----------------------------------------"
|
||||
cat "$NOW_PASSING"
|
||||
echo ""
|
||||
fi
|
||||
|
||||
if [ -f "$STILL_FAILING" ] && [ -s "$STILL_FAILING" ]; then
|
||||
echo "Still failing (saved to $STILL_FAILING):"
|
||||
echo "-----------------------------------------"
|
||||
cat "$STILL_FAILING"
|
||||
echo ""
|
||||
echo "Build logs available in: $LOG_DIR/"
|
||||
exit 1
|
||||
else
|
||||
echo "All previously failed builds now pass! ✓"
|
||||
rm -f "$STILL_FAILING"
|
||||
exit 0
|
||||
fi
|
||||
Reference in new issue
Block a user