mirror of
https://github.com/cloudstack-llc/mlx-knife.git
synced 2026-07-19 14:43:36 -04:00
The BROKE Cluster Team
bf7480d042
Major Features: - Audio transcription via mlx-audio backend (Whisper, >10min duration) - OpenAI /v1/audio/transcriptions endpoint - Memory Gate System (Vision: 8GB, Audio: 4GB) - Config-based backend routing (ADR-020) - Benchmark toolchain (memmon/memplot, Schema v0.2.2) Key Fixes: - EuroLLM tokenizer decoding - Vision-model text-only routing regression - Multimodal model context length detection - Memory cleanup bug (mx.metal.clear_cache) - Orphan process bug Test Results: - Unit tests: 647 passed, 11 skipped (Python 3.10-3.12) - wet-umbrella: 171 passed total See CHANGELOG.md for complete details and known issues.
256 lines
10 KiB
Bash
Executable File
256 lines
10 KiB
Bash
Executable File
#!/bin/bash
|
||
# Note: removed set -e to allow script to continue through all Python versions
|
||
# Individual error handling is done explicitly in each test section
|
||
|
||
echo "🧪 MLX Knife 2.0 (mlxk2) Multi-Python Version Testing"
|
||
echo "=========================================="
|
||
echo "Prerequisites: Python versions should be available as:"
|
||
echo " - python3.10, python3.11, python3.12 (full support: text + vision + audio)"
|
||
echo "Note: Python 3.9 not supported (MLX 0.30+ requires 3.10+)"
|
||
echo "Note: Python 3.13+ not supported (miniaudio lacks pre-built wheels)"
|
||
echo ""
|
||
|
||
# Colors for output
|
||
GREEN='\033[0;32m'
|
||
RED='\033[0;31m'
|
||
YELLOW='\033[1;33m'
|
||
NC='\033[0m' # No Color
|
||
|
||
# Python versions to test (bash 3.2 compatible)
|
||
# Note: Python 3.9 dropped (MLX 0.30+ requires 3.10+)
|
||
# Note: Python 3.13+ dropped (miniaudio wheel limitation)
|
||
PYTHON_COMMANDS=("python3.10" "python3.11" "python3.12")
|
||
VERSION_NAMES=("3.10" "3.11" "3.12")
|
||
RESULTS=()
|
||
|
||
# Test function
|
||
test_python_version() {
|
||
local index=$1
|
||
local version_name="${VERSION_NAMES[$index]}"
|
||
local python_cmd="${PYTHON_COMMANDS[$index]}"
|
||
|
||
echo -e "\n${YELLOW}🐍 Testing Python ${version_name}${NC}"
|
||
echo "----------------------------------------"
|
||
|
||
# Check if Python version is available
|
||
if ! command -v $python_cmd &> /dev/null; then
|
||
echo -e "${RED}❌ Python ${version_name} not found (tried: $python_cmd)${NC}"
|
||
RESULTS+=("${version_name}:NOT_FOUND")
|
||
return 1
|
||
fi
|
||
|
||
# Show actual version
|
||
local actual_version=$($python_cmd --version 2>&1)
|
||
echo "📍 Found: $actual_version"
|
||
|
||
# Create virtual environment
|
||
local venv_name="test_env_${version_name//./_}"
|
||
echo "🔧 Creating virtual environment: $venv_name"
|
||
|
||
if [ -d "$venv_name" ]; then
|
||
rm -rf "$venv_name"
|
||
fi
|
||
|
||
$python_cmd -m venv "$venv_name"
|
||
source "$venv_name/bin/activate"
|
||
|
||
# Upgrade pip and install MLX Knife
|
||
echo "📦 Installing MLX Knife (2.0) ..."
|
||
local install_log="install_${version_name//./_}.log"
|
||
pip install --upgrade pip setuptools wheel > "$install_log" 2>&1
|
||
|
||
# Install with vision + audio support (all supported versions are 3.10+)
|
||
local install_extras=".[test,vision,audio]"
|
||
echo " Including vision + audio support (Python $version_name)"
|
||
|
||
if pip install -e "$install_extras" >> "$install_log" 2>&1; then
|
||
echo -e "${GREEN}✅ Installation successful${NC}"
|
||
echo "🧰 Ensuring tooling (ruff, mypy)..."
|
||
pip install -q "ruff>=0.1.0" "mypy>=1.5.0" >> "$install_log" 2>&1 || true
|
||
|
||
# Run smoke test
|
||
echo "🧪 Running import test (mlxk2)..."
|
||
if python -c "import mlxk2, mlxk2.cli; print('Import successful')"; then
|
||
echo -e "${GREEN}✅ Import test passed${NC}"
|
||
|
||
# Try basic CLI command
|
||
echo "🧪 Testing CLI version (JSON)..."
|
||
if python -m mlxk2.cli --version --json > /dev/null 2>&1; then
|
||
echo -e "${GREEN}✅ CLI test (version) passed${NC}"
|
||
|
||
# Run complete test suite (exclude ALL live tests requiring models/network)
|
||
echo "🧪 Running 2.0 test suite..."
|
||
local test_log="test_results_${version_name//./_}.log"
|
||
if python -m pytest tests_2.0/ -m "not live" -v --tb=short > "$test_log" 2>&1; then
|
||
local passed_count=$(grep -c "PASSED" "$test_log" 2>/dev/null)
|
||
local failed_count=$(grep -c "FAILED" "$test_log" 2>/dev/null)
|
||
passed_count=${passed_count:-0}
|
||
failed_count=${failed_count:-0}
|
||
local test_count=$((passed_count + failed_count))
|
||
|
||
if [ "$failed_count" -eq 0 ] && [ "$passed_count" -gt 0 ]; then
|
||
echo -e "${GREEN}✅ Full test suite passed ($passed_count/$test_count tests)${NC}"
|
||
|
||
# Code quality checks
|
||
echo "🧪 Running code quality checks (mlxk2)..."
|
||
|
||
local ruff_log="ruff_${version_name//./_}.log"
|
||
echo "🧪 Running ruff check on mlxk2 (logging to $ruff_log)..."
|
||
if python -m ruff check mlxk2/ > "$ruff_log" 2>&1; then
|
||
echo -e "${GREEN}✅ ruff linting passed${NC}"
|
||
|
||
# Note: mypy might have many warnings, so we allow it to "fail" but still continue
|
||
python -m mypy mlxk2/ --ignore-missing-imports > mypy_${version_name//./_}.log 2>&1
|
||
local mypy_errors=$(grep -c "error:" mypy_${version_name//./_}.log 2>/dev/null || echo "0")
|
||
echo -e "${YELLOW}ℹ️ mypy check complete ($mypy_errors errors found)${NC}"
|
||
|
||
RESULTS+=("${version_name}:FULL_SUCCESS:${passed_count}tests")
|
||
else
|
||
local ruff_error_count=$(grep -c "Found .* error" "$ruff_log" 2>/dev/null || echo "unknown")
|
||
echo -e "${RED}❌ ruff linting failed ($ruff_error_count errors)${NC}"
|
||
echo " See $ruff_log for details"
|
||
RESULTS+=("${version_name}:RUFF_FAILED")
|
||
fi
|
||
else
|
||
echo -e "${RED}❌ Test suite failed ($passed_count passed, $failed_count failed)${NC}"
|
||
echo " See $test_log for details"
|
||
RESULTS+=("${version_name}:TESTS_FAILED:${failed_count}failures")
|
||
fi
|
||
else
|
||
echo -e "${RED}❌ Test suite timed out or crashed${NC}"
|
||
RESULTS+=("${version_name}:TESTS_TIMEOUT")
|
||
fi
|
||
else
|
||
echo -e "${RED}❌ CLI test failed${NC}"
|
||
RESULTS+=("${version_name}:CLI_FAILED")
|
||
fi
|
||
else
|
||
echo -e "${RED}❌ Import test failed${NC}"
|
||
RESULTS+=("${version_name}:IMPORT_FAILED")
|
||
fi
|
||
else
|
||
echo -e "${RED}❌ Installation failed${NC}"
|
||
echo " See $install_log for details"
|
||
RESULTS+=("${version_name}:INSTALL_FAILED")
|
||
fi
|
||
|
||
# Cleanup
|
||
deactivate 2>/dev/null || true
|
||
rm -rf "$venv_name"
|
||
}
|
||
|
||
# Run tests for all Python versions
|
||
for i in "${!PYTHON_COMMANDS[@]}"; do
|
||
test_python_version "$i"
|
||
done
|
||
|
||
# Summary
|
||
echo -e "\n${YELLOW}📊 SUMMARY${NC}"
|
||
echo "==========="
|
||
|
||
for result in "${RESULTS[@]}"; do
|
||
IFS=':' read -r version status details <<< "$result"
|
||
case $status in
|
||
"FULL_SUCCESS")
|
||
echo -e "${GREEN}✅ Python $version: FULLY VERIFIED ($details)${NC}"
|
||
;;
|
||
"NOT_FOUND")
|
||
echo -e "${YELLOW}⚠️ Python $version: NOT INSTALLED${NC}"
|
||
;;
|
||
"TESTS_FAILED")
|
||
echo -e "${RED}❌ Python $version: TESTS FAILED ($details)${NC}"
|
||
;;
|
||
"RUFF_FAILED")
|
||
echo -e "${RED}❌ Python $version: CODE QUALITY FAILED${NC}"
|
||
;;
|
||
"RUFF_INSTALL_FAILED")
|
||
echo -e "${RED}❌ Python $version: RUFF INSTALLATION FAILED${NC}"
|
||
;;
|
||
"TESTS_TIMEOUT")
|
||
echo -e "${RED}❌ Python $version: TESTS TIMED OUT${NC}"
|
||
;;
|
||
*)
|
||
echo -e "${RED}❌ Python $version: $status${NC}"
|
||
;;
|
||
esac
|
||
done
|
||
|
||
# Recommendations
|
||
echo -e "\n${YELLOW}💡 RECOMMENDATIONS${NC}"
|
||
echo "=================="
|
||
|
||
fully_verified_count=0
|
||
partial_count=0
|
||
failed_count=0
|
||
not_found_count=0
|
||
fully_verified_versions=()
|
||
|
||
for result in "${RESULTS[@]}"; do
|
||
IFS=':' read -r version status details <<< "$result"
|
||
case $status in
|
||
"FULL_SUCCESS")
|
||
((fully_verified_count++))
|
||
fully_verified_versions+=("$version")
|
||
;;
|
||
"NOT_FOUND")
|
||
((not_found_count++))
|
||
;;
|
||
*)
|
||
((failed_count++))
|
||
;;
|
||
esac
|
||
done
|
||
|
||
echo -e "${YELLOW}📊 VERIFICATION RESULTS:${NC}"
|
||
echo " Fully Verified: $fully_verified_count"
|
||
echo " Failed/Issues: $failed_count"
|
||
echo " Not Available: $not_found_count"
|
||
|
||
if [ $fully_verified_count -eq 0 ]; then
|
||
echo -e "\n${RED}🚨 CRITICAL: No Python versions fully verified!${NC}"
|
||
echo " → Cannot release without verified compatibility"
|
||
echo " → Fix blocking issues before any release"
|
||
elif [ $failed_count -eq 0 ] && [ $fully_verified_count -ge 2 ]; then
|
||
echo -e "\n${GREEN}🎉 PRODUCTION READY: All tested versions fully verified!${NC}"
|
||
echo " → Safe to release with confidence"
|
||
echo " → All versions pass: installation, tests, code quality"
|
||
echo " → Verified versions: ${fully_verified_versions[*]}"
|
||
elif [ $fully_verified_count -ge 2 ]; then
|
||
echo -e "\n${YELLOW}⚖️ PARTIAL SUCCESS: $fully_verified_count verified, $failed_count with issues${NC}"
|
||
echo " → Can release with verified versions: ${fully_verified_versions[*]}"
|
||
echo " → Document known issues with other versions"
|
||
echo " → Consider fixing compatibility or updating requirements"
|
||
else
|
||
echo -e "\n${RED}⚠️ INSUFFICIENT VERIFICATION: Only $fully_verified_count version(s) verified${NC}"
|
||
echo " → Need at least 2 fully verified versions for release"
|
||
echo " → Fix compatibility issues or verify more versions"
|
||
fi
|
||
|
||
echo -e "\n${YELLOW}📝 NEXT STEPS${NC}"
|
||
echo "============="
|
||
|
||
if [ $fully_verified_count -ge 2 ] && [ $failed_count -eq 0 ]; then
|
||
echo "✅ READY TO RELEASE:"
|
||
echo " 1. Update README.md with verified Python versions"
|
||
echo " 2. Update pyproject.toml requires-python based on results"
|
||
echo " 3. Document verified versions: ${fully_verified_versions[*]}"
|
||
echo " 4. Safe to tag and release (alpha.1)"
|
||
exit_code=0
|
||
else
|
||
echo "🔧 WORK NEEDED:"
|
||
echo " 1. Review detailed logs: test_results_*.log, mypy_*.log"
|
||
echo " 2. Fix compatibility issues for failed versions"
|
||
echo " 3. Re-run this script until all targeted versions pass"
|
||
echo " 4. Update documentation to reflect actual compatibility"
|
||
echo " 5. Consider reducing version scope if fixes are complex"
|
||
exit_code=1
|
||
fi
|
||
|
||
echo ""
|
||
echo -e "${YELLOW}📁 Generated Files:${NC}"
|
||
echo " - test_results_<version>.log: Detailed pytest results"
|
||
echo " - mypy_<version>.log: Type checking results"
|
||
echo " - Use these logs to debug specific compatibility issues"
|
||
|
||
exit $exit_code
|