diff --git a/.gitignore b/.gitignore index e69de29..ababa4f 100644 --- a/.gitignore +++ b/.gitignore @@ -0,0 +1,7 @@ +.xben_build_done +build_logs/ +build_logs_failed/ +failed_builds.txt +successful_builds.txt +still_failing.txt +now_passing.txt diff --git a/benchmarks/XBEN-012-24/src/Dockerfile b/benchmarks/XBEN-012-24/src/Dockerfile index 8513f6c..4dad1b3 100644 --- a/benchmarks/XBEN-012-24/src/Dockerfile +++ b/benchmarks/XBEN-012-24/src/Dockerfile @@ -1,7 +1,14 @@ FROM python:2.7.18-slim -RUN apt-get update && apt-get install -y phantomjs +RUN echo "deb http://archive.debian.org/debian/ buster main" > /etc/apt/sources.list \ + && echo "deb http://archive.debian.org/debian/ buster-updates main" >> /etc/apt/sources.list \ + && echo "deb http://archive.debian.org/debian-security buster/updates main" >> /etc/apt/sources.list + +# Now apt-get update will work +RUN apt-get update && apt-get install -y --no-install-recommends \ + phantomjs \ + && rm -rf /var/lib/apt/lists/* ENV QT_QPA_PLATFORM=offscreen ENV OPENSSL_CONF=/dev/null diff --git a/benchmarks/XBEN-013-24/src/Dockerfile b/benchmarks/XBEN-013-24/src/Dockerfile index 8513f6c..4dad1b3 100644 --- a/benchmarks/XBEN-013-24/src/Dockerfile +++ b/benchmarks/XBEN-013-24/src/Dockerfile @@ -1,7 +1,14 @@ FROM python:2.7.18-slim -RUN apt-get update && apt-get install -y phantomjs +RUN echo "deb http://archive.debian.org/debian/ buster main" > /etc/apt/sources.list \ + && echo "deb http://archive.debian.org/debian/ buster-updates main" >> /etc/apt/sources.list \ + && echo "deb http://archive.debian.org/debian-security buster/updates main" >> /etc/apt/sources.list + +# Now apt-get update will work +RUN apt-get update && apt-get install -y --no-install-recommends \ + phantomjs \ + && rm -rf /var/lib/apt/lists/* ENV QT_QPA_PLATFORM=offscreen ENV OPENSSL_CONF=/dev/null diff --git a/benchmarks/XBEN-018-24/src/Dockerfile b/benchmarks/XBEN-018-24/src/Dockerfile index 8513f6c..4dad1b3 100644 --- a/benchmarks/XBEN-018-24/src/Dockerfile +++ b/benchmarks/XBEN-018-24/src/Dockerfile @@ -1,7 +1,14 @@ FROM python:2.7.18-slim -RUN apt-get update && apt-get install -y phantomjs +RUN echo "deb http://archive.debian.org/debian/ buster main" > /etc/apt/sources.list \ + && echo "deb http://archive.debian.org/debian/ buster-updates main" >> /etc/apt/sources.list \ + && echo "deb http://archive.debian.org/debian-security buster/updates main" >> /etc/apt/sources.list + +# Now apt-get update will work +RUN apt-get update && apt-get install -y --no-install-recommends \ + phantomjs \ + && rm -rf /var/lib/apt/lists/* ENV QT_QPA_PLATFORM=offscreen ENV OPENSSL_CONF=/dev/null diff --git a/benchmarks/XBEN-051-24/src/Dockerfile b/benchmarks/XBEN-051-24/src/Dockerfile index 8513f6c..4dad1b3 100644 --- a/benchmarks/XBEN-051-24/src/Dockerfile +++ b/benchmarks/XBEN-051-24/src/Dockerfile @@ -1,7 +1,14 @@ FROM python:2.7.18-slim -RUN apt-get update && apt-get install -y phantomjs +RUN echo "deb http://archive.debian.org/debian/ buster main" > /etc/apt/sources.list \ + && echo "deb http://archive.debian.org/debian/ buster-updates main" >> /etc/apt/sources.list \ + && echo "deb http://archive.debian.org/debian-security buster/updates main" >> /etc/apt/sources.list + +# Now apt-get update will work +RUN apt-get update && apt-get install -y --no-install-recommends \ + phantomjs \ + && rm -rf /var/lib/apt/lists/* ENV QT_QPA_PLATFORM=offscreen ENV OPENSSL_CONF=/dev/null diff --git a/benchmarks/XBEN-059-24/src/Dockerfile b/benchmarks/XBEN-059-24/src/Dockerfile index 8513f6c..4dad1b3 100644 --- a/benchmarks/XBEN-059-24/src/Dockerfile +++ b/benchmarks/XBEN-059-24/src/Dockerfile @@ -1,7 +1,14 @@ FROM python:2.7.18-slim -RUN apt-get update && apt-get install -y phantomjs +RUN echo "deb http://archive.debian.org/debian/ buster main" > /etc/apt/sources.list \ + && echo "deb http://archive.debian.org/debian/ buster-updates main" >> /etc/apt/sources.list \ + && echo "deb http://archive.debian.org/debian-security buster/updates main" >> /etc/apt/sources.list + +# Now apt-get update will work +RUN apt-get update && apt-get install -y --no-install-recommends \ + phantomjs \ + && rm -rf /var/lib/apt/lists/* ENV QT_QPA_PLATFORM=offscreen ENV OPENSSL_CONF=/dev/null diff --git a/benchmarks/XBEN-074-24/src/Dockerfile b/benchmarks/XBEN-074-24/src/Dockerfile index 8513f6c..4dad1b3 100644 --- a/benchmarks/XBEN-074-24/src/Dockerfile +++ b/benchmarks/XBEN-074-24/src/Dockerfile @@ -1,7 +1,14 @@ FROM python:2.7.18-slim -RUN apt-get update && apt-get install -y phantomjs +RUN echo "deb http://archive.debian.org/debian/ buster main" > /etc/apt/sources.list \ + && echo "deb http://archive.debian.org/debian/ buster-updates main" >> /etc/apt/sources.list \ + && echo "deb http://archive.debian.org/debian-security buster/updates main" >> /etc/apt/sources.list + +# Now apt-get update will work +RUN apt-get update && apt-get install -y --no-install-recommends \ + phantomjs \ + && rm -rf /var/lib/apt/lists/* ENV QT_QPA_PLATFORM=offscreen ENV OPENSSL_CONF=/dev/null diff --git a/benchmarks/XBEN-084-24/docker-compose.yml b/benchmarks/XBEN-084-24/docker-compose.yml index c700186..84d7242 100644 --- a/benchmarks/XBEN-084-24/docker-compose.yml +++ b/benchmarks/XBEN-084-24/docker-compose.yml @@ -51,5 +51,7 @@ services: retries: 3 networks: + default: + driver: bridge internal: driver: bridge diff --git a/benchmarks/XBEN-087-24/src/Dockerfile b/benchmarks/XBEN-087-24/src/Dockerfile index 8513f6c..4dad1b3 100644 --- a/benchmarks/XBEN-087-24/src/Dockerfile +++ b/benchmarks/XBEN-087-24/src/Dockerfile @@ -1,7 +1,14 @@ FROM python:2.7.18-slim -RUN apt-get update && apt-get install -y phantomjs +RUN echo "deb http://archive.debian.org/debian/ buster main" > /etc/apt/sources.list \ + && echo "deb http://archive.debian.org/debian/ buster-updates main" >> /etc/apt/sources.list \ + && echo "deb http://archive.debian.org/debian-security buster/updates main" >> /etc/apt/sources.list + +# Now apt-get update will work +RUN apt-get update && apt-get install -y --no-install-recommends \ + phantomjs \ + && rm -rf /var/lib/apt/lists/* ENV QT_QPA_PLATFORM=offscreen ENV OPENSSL_CONF=/dev/null diff --git a/benchmarks/XBEN-089-24/app/Dockerfile b/benchmarks/XBEN-089-24/app/Dockerfile index 1c8e874..1e7cd87 100644 --- a/benchmarks/XBEN-089-24/app/Dockerfile +++ b/benchmarks/XBEN-089-24/app/Dockerfile @@ -3,7 +3,14 @@ FROM python:3.8-slim-buster WORKDIR /app -RUN apt-get update && apt-get install -y curl +RUN echo "deb http://archive.debian.org/debian/ buster main" > /etc/apt/sources.list \ + && echo "deb http://archive.debian.org/debian/ buster-updates main" >> /etc/apt/sources.list \ + && echo "deb http://archive.debian.org/debian-security buster/updates main" >> /etc/apt/sources.list + +# Now apt-get update will work +RUN apt-get update && apt-get install -y --no-install-recommends \ + phantomjs \ + && rm -rf /var/lib/apt/lists/* COPY requirements.txt requirements.txt RUN pip install -r requirements.txt diff --git a/benchmarks/XBEN-090-24/app/Dockerfile b/benchmarks/XBEN-090-24/app/Dockerfile index 1c8e874..1e7cd87 100644 --- a/benchmarks/XBEN-090-24/app/Dockerfile +++ b/benchmarks/XBEN-090-24/app/Dockerfile @@ -3,7 +3,14 @@ FROM python:3.8-slim-buster WORKDIR /app -RUN apt-get update && apt-get install -y curl +RUN echo "deb http://archive.debian.org/debian/ buster main" > /etc/apt/sources.list \ + && echo "deb http://archive.debian.org/debian/ buster-updates main" >> /etc/apt/sources.list \ + && echo "deb http://archive.debian.org/debian-security buster/updates main" >> /etc/apt/sources.list + +# Now apt-get update will work +RUN apt-get update && apt-get install -y --no-install-recommends \ + phantomjs \ + && rm -rf /var/lib/apt/lists/* COPY requirements.txt requirements.txt RUN pip install -r requirements.txt diff --git a/benchmarks/XBEN-091-24/app/Dockerfile b/benchmarks/XBEN-091-24/app/Dockerfile index 7dd356c..1216a4e 100644 --- a/benchmarks/XBEN-091-24/app/Dockerfile +++ b/benchmarks/XBEN-091-24/app/Dockerfile @@ -3,7 +3,14 @@ FROM python:3.8-slim-buster WORKDIR /app -RUN apt-get update && apt-get install -y curl +RUN echo "deb http://archive.debian.org/debian/ buster main" > /etc/apt/sources.list \ + && echo "deb http://archive.debian.org/debian/ buster-updates main" >> /etc/apt/sources.list \ + && echo "deb http://archive.debian.org/debian-security buster/updates main" >> /etc/apt/sources.list + +# Now apt-get update will work +RUN apt-get update && apt-get install -y --no-install-recommends \ + phantomjs \ + && rm -rf /var/lib/apt/lists/* COPY requirements.txt requirements.txt RUN pip install -r requirements.txt diff --git a/benchmarks/XBEN-093-24/app/Dockerfile b/benchmarks/XBEN-093-24/app/Dockerfile index 1c8e874..1e7cd87 100644 --- a/benchmarks/XBEN-093-24/app/Dockerfile +++ b/benchmarks/XBEN-093-24/app/Dockerfile @@ -3,7 +3,14 @@ FROM python:3.8-slim-buster WORKDIR /app -RUN apt-get update && apt-get install -y curl +RUN echo "deb http://archive.debian.org/debian/ buster main" > /etc/apt/sources.list \ + && echo "deb http://archive.debian.org/debian/ buster-updates main" >> /etc/apt/sources.list \ + && echo "deb http://archive.debian.org/debian-security buster/updates main" >> /etc/apt/sources.list + +# Now apt-get update will work +RUN apt-get update && apt-get install -y --no-install-recommends \ + phantomjs \ + && rm -rf /var/lib/apt/lists/* COPY requirements.txt requirements.txt RUN pip install -r requirements.txt diff --git a/benchmarks/XBEN-094-24/app/Dockerfile b/benchmarks/XBEN-094-24/app/Dockerfile index 1c8e874..1e7cd87 100644 --- a/benchmarks/XBEN-094-24/app/Dockerfile +++ b/benchmarks/XBEN-094-24/app/Dockerfile @@ -3,7 +3,14 @@ FROM python:3.8-slim-buster WORKDIR /app -RUN apt-get update && apt-get install -y curl +RUN echo "deb http://archive.debian.org/debian/ buster main" > /etc/apt/sources.list \ + && echo "deb http://archive.debian.org/debian/ buster-updates main" >> /etc/apt/sources.list \ + && echo "deb http://archive.debian.org/debian-security buster/updates main" >> /etc/apt/sources.list + +# Now apt-get update will work +RUN apt-get update && apt-get install -y --no-install-recommends \ + phantomjs \ + && rm -rf /var/lib/apt/lists/* COPY requirements.txt requirements.txt RUN pip install -r requirements.txt diff --git a/benchmarks/XBEN-100-24/app/Dockerfile b/benchmarks/XBEN-100-24/app/Dockerfile index b4283c3..16e1da7 100644 --- a/benchmarks/XBEN-100-24/app/Dockerfile +++ b/benchmarks/XBEN-100-24/app/Dockerfile @@ -3,7 +3,14 @@ FROM python:3.8-slim-buster WORKDIR /app -RUN apt-get update && apt-get install -y curl +RUN echo "deb http://archive.debian.org/debian/ buster main" > /etc/apt/sources.list \ + && echo "deb http://archive.debian.org/debian/ buster-updates main" >> /etc/apt/sources.list \ + && echo "deb http://archive.debian.org/debian-security buster/updates main" >> /etc/apt/sources.list + +# Now apt-get update will work +RUN apt-get update && apt-get install -y --no-install-recommends \ + phantomjs \ + && rm -rf /var/lib/apt/lists/* COPY requirements.txt requirements.txt RUN pip install -r requirements.txt diff --git a/test_all_builds.sh b/test_all_builds.sh new file mode 100755 index 0000000..e8c2bbb --- /dev/null +++ b/test_all_builds.sh @@ -0,0 +1,89 @@ +#!/bin/bash + +# Test all 104 XBEN benchmarks in parallel +# Outputs failed benchmark names to failed_builds.txt + +set -e + +BENCHMARKS_DIR="benchmarks" +FAILED_FILE="failed_builds.txt" +SUCCESS_FILE="successful_builds.txt" +LOG_DIR="build_logs" +PARALLEL_JOBS=8 # Adjust based on your CPU cores + +# Clean up old results +rm -f "$FAILED_FILE" "$SUCCESS_FILE" +rm -rf "$LOG_DIR" +mkdir -p "$LOG_DIR" + +echo "=========================================" +echo "Testing builds for all 104 XBEN benchmarks" +echo "Parallel jobs: $PARALLEL_JOBS" +echo "=========================================" +echo "" + +# Function to test a single benchmark +test_benchmark() { + local benchmark=$1 + local benchmark_name=$(basename "$benchmark") + local base_dir=$(pwd) + local log_file="$base_dir/$LOG_DIR/${benchmark_name}.log" + + echo "[TESTING] $benchmark_name" + + if (cd "$benchmark" && make build > "$log_file" 2>&1); then + echo "[SUCCESS] $benchmark_name" + echo "$benchmark_name" >> "$base_dir/$SUCCESS_FILE" + return 0 + else + echo "[FAILED] $benchmark_name" + echo "$benchmark_name" >> "$base_dir/$FAILED_FILE" + return 1 + fi +} + +# Export function and variables for parallel execution +export -f test_benchmark +export LOG_DIR +export FAILED_FILE +export SUCCESS_FILE + +# Find all benchmark directories and run tests in parallel +find "$BENCHMARKS_DIR" -maxdepth 1 -type d -name "XBEN-*-24" | sort | \ + xargs -I {} -P "$PARALLEL_JOBS" bash -c 'test_benchmark "$@"' _ {} + +echo "" +echo "=========================================" +echo "BUILD TEST COMPLETE" +echo "=========================================" + +# Count results +TOTAL=104 +SUCCESS_COUNT=0 +FAIL_COUNT=0 + +if [ -f "$SUCCESS_FILE" ]; then + SUCCESS_COUNT=$(wc -l < "$SUCCESS_FILE") +fi + +if [ -f "$FAILED_FILE" ]; then + FAIL_COUNT=$(wc -l < "$FAILED_FILE") +fi + +echo "Total benchmarks: $TOTAL" +echo "Successful builds: $SUCCESS_COUNT" +echo "Failed builds: $FAIL_COUNT" +echo "" + +if [ -f "$FAILED_FILE" ] && [ -s "$FAILED_FILE" ]; then + echo "Failed benchmarks (saved to $FAILED_FILE):" + echo "=========================================" + cat "$FAILED_FILE" + echo "" + echo "Build logs available in: $LOG_DIR/" + exit 1 +else + echo "All builds passed! ✓" + rm -f "$FAILED_FILE" + exit 0 +fi diff --git a/test_failed_builds.sh b/test_failed_builds.sh new file mode 100755 index 0000000..11f5d66 --- /dev/null +++ b/test_failed_builds.sh @@ -0,0 +1,117 @@ +#!/bin/bash + +# Test only the failed benchmarks from failed_builds.txt +# Useful for iteratively fixing and retesting broken builds + +set -e + +BENCHMARKS_DIR="benchmarks" +FAILED_INPUT="failed_builds.txt" +STILL_FAILING="still_failing.txt" +NOW_PASSING="now_passing.txt" +LOG_DIR="build_logs_failed" +PARALLEL_JOBS=8 # Adjust based on your CPU cores + +# Check if failed_builds.txt exists +if [ ! -f "$FAILED_INPUT" ]; then + echo "Error: $FAILED_INPUT not found!" + echo "Run ./test_all_builds.sh first to generate the list of failed builds." + exit 1 +fi + +# Count how many to test +TOTAL_FAILED=$(wc -l < "$FAILED_INPUT") + +if [ "$TOTAL_FAILED" -eq 0 ]; then + echo "No failed builds to test!" + exit 0 +fi + +# Clean up old results +rm -f "$STILL_FAILING" "$NOW_PASSING" +rm -rf "$LOG_DIR" +mkdir -p "$LOG_DIR" + +echo "=========================================" +echo "Testing $TOTAL_FAILED previously failed benchmarks" +echo "Parallel jobs: $PARALLEL_JOBS" +echo "=========================================" +echo "" + +# Function to test a single benchmark +test_benchmark() { + local benchmark_name=$1 + local base_dir=$(pwd) + local benchmark="$base_dir/$BENCHMARKS_DIR/$benchmark_name" + local log_file="$base_dir/$LOG_DIR/${benchmark_name}.log" + + echo "[TESTING] $benchmark_name" + + if [ ! -d "$benchmark" ]; then + echo "[ERROR] $benchmark_name - directory not found" + echo "$benchmark_name" >> "$base_dir/$STILL_FAILING" + return 1 + fi + + if (cd "$benchmark" && make build > "$log_file" 2>&1); then + echo "[FIXED] $benchmark_name ✓" + echo "$benchmark_name" >> "$base_dir/$NOW_PASSING" + return 0 + else + echo "[FAILED] $benchmark_name" + echo "$benchmark_name" >> "$base_dir/$STILL_FAILING" + return 1 + fi +} + +# Export function and variables for parallel execution +export -f test_benchmark +export BENCHMARKS_DIR +export LOG_DIR +export STILL_FAILING +export NOW_PASSING + +# Read failed benchmarks and test them in parallel +cat "$FAILED_INPUT" | xargs -I {} -P "$PARALLEL_JOBS" bash -c 'test_benchmark "$@"' _ {} + +echo "" +echo "=========================================" +echo "RETEST COMPLETE" +echo "=========================================" + +# Count results +STILL_FAILING_COUNT=0 +NOW_PASSING_COUNT=0 + +if [ -f "$NOW_PASSING" ]; then + NOW_PASSING_COUNT=$(wc -l < "$NOW_PASSING") +fi + +if [ -f "$STILL_FAILING" ]; then + STILL_FAILING_COUNT=$(wc -l < "$STILL_FAILING") +fi + +echo "Previously failed: $TOTAL_FAILED" +echo "Now passing: $NOW_PASSING_COUNT ✓" +echo "Still failing: $STILL_FAILING_COUNT" +echo "" + +if [ "$NOW_PASSING_COUNT" -gt 0 ]; then + echo "Fixed benchmarks (saved to $NOW_PASSING):" + echo "-----------------------------------------" + cat "$NOW_PASSING" + echo "" +fi + +if [ -f "$STILL_FAILING" ] && [ -s "$STILL_FAILING" ]; then + echo "Still failing (saved to $STILL_FAILING):" + echo "-----------------------------------------" + cat "$STILL_FAILING" + echo "" + echo "Build logs available in: $LOG_DIR/" + exit 1 +else + echo "All previously failed builds now pass! ✓" + rm -f "$STILL_FAILING" + exit 0 +fi