diff --git a/scripts/ELECTRON_EXTERNAL_PYTHON.md b/scripts/ELECTRON_EXTERNAL_PYTHON.md new file mode 100644 index 0000000..355698e --- /dev/null +++ b/scripts/ELECTRON_EXTERNAL_PYTHON.md @@ -0,0 +1,101 @@ +# External Python Mode (Electron-embedded Python) + +This variant does not use PyInstaller. Instead, it vendors all Python deps (including `mlxk2`) into a folder and runs against the Python interpreter you ship inside your Electron app (`MyApp.app/Contents/Resources/python/bin/python3`). + +## Build + +```bash +# Recommended: point to your embedded Python +EMBEDDED_PYTHON="/path/to/MyApp.app/Contents/Resources/python/bin/python3" \ + scripts/build_external_python.sh + +# Dev fallback (uses ./python/bin/python3 if present, else system python3) +scripts/build_external_python.sh +``` + +Output: `dist-ext-python//` (default `msty-mlx-studio/`), containing: +- `_vendor/` – vendored site-packages (deps + project) +- `msty-mlx-studio` – launcher that runs `-m mlxk2.cli` with `PYTHONPATH=_vendor` + +Verify the Python in use: + +```bash +./dist-ext-python/msty-mlx-studio/msty-mlx-studio --python-info + +### Building without executing the embedded Python (avoids Gatekeeper during dev) + +If macOS blocks running your embedded Python during the build, you can vendor wheels for a target version without executing that interpreter: + +```bash +TARGET_PY_VERSION=3.10 \ +VENDOR_VIA_WHEELS=1 \ +scripts/build_external_python.sh +``` + +Notes: +- This uses your system Python to `pip download` wheels for macOS arm64 and the specified Python version, then unpacks them into `_vendor`. +- At runtime, the Electron-embedded Python imports from `_vendor` via `PYTHONPATH`. +- For compiled packages, you must choose a `TARGET_PY_VERSION` that matches your embedded Python minor version (e.g., 3.10 → cp310 wheels). +``` + +## Packaging into Electron + +Place the folder under your app resources, for example: + +``` +MyApp.app/Contents/Resources/ + python/bin/python3 # your embedded Python + mlxk2/ + _vendor/ + msty-mlx-studio # launcher +``` + +### Option A: Use the launcher + +- Command: `join(process.resourcesPath, 'mlxk2', 'msty-mlx-studio')` +- Env: set `RESOURCES_PATH=process.resourcesPath` (so the launcher finds `python/bin/python3`). + +```js +import { spawn } from 'child_process'; +import { join } from 'path'; +import { app } from 'electron'; + +const launcher = join(process.resourcesPath, 'mlxk2', 'msty-mlx-studio'); + +export function startServer({ host = '127.0.0.1', port = 8000, maxTokens } = {}) { + const args = ['serve', '--host', host, '--port', String(port)]; + if (maxTokens != null) args.push('--max-tokens', String(maxTokens)); + const env = { ...process.env, RESOURCES_PATH: process.resourcesPath }; + const child = spawn(launcher, args, { env, stdio: ['ignore', 'inherit', 'inherit'] }); + return child; +} +``` + +### Option B: Call Python directly + +- Python: `join(process.resourcesPath, 'python', 'bin', 'python3')` +- Vendor: `join(process.resourcesPath, 'mlxk2', '_vendor')` +- Env: `PYTHONPATH=[:$PYTHONPATH]`, `PYTHONNOUSERSITE=1` + +```js +import { spawn } from 'child_process'; +import { join } from 'path'; +import { app } from 'electron'; + +export function startServer({ host = '127.0.0.1', port = 8000, maxTokens } = {}) { + const pythonPath = join(process.resourcesPath, 'python', 'bin', 'python3'); + const vendor = join(process.resourcesPath, 'mlxk2', '_vendor'); + const args = ['-m', 'mlxk2.cli', 'serve', '--host', host, '--port', String(port)]; + if (maxTokens != null) args.push('--max-tokens', String(maxTokens)); + const env = { ...process.env, PYTHONNOUSERSITE: '1', PYTHONPATH: vendor }; + const child = spawn(pythonPath, args, { env, stdio: ['ignore', 'inherit', 'inherit'] }); + return child; +} +``` + +## Notes + +- You are responsible for shipping a compatible macOS arm64 Python with all required system libs. +- If using private or offline wheels, pass extra pip args: `PIP_ARGS="--no-index --find-links /path/to/wheels" scripts/build_external_python.sh`. +- To sanity check deps at runtime: + `PYTHONPATH= -c "import mlxk2, fastapi, uvicorn, mlx, mlx_lm; print('ok')"` diff --git a/scripts/build_external_python.sh b/scripts/build_external_python.sh new file mode 100755 index 0000000..267a6ce --- /dev/null +++ b/scripts/build_external_python.sh @@ -0,0 +1,180 @@ +#!/usr/bin/env bash +set -euo pipefail + +# Build a "no-PyInstaller" distribution that runs against an external Python +# (e.g., the interpreter embedded in your Electron app at +# MyApp.app/Contents/Resources/python/bin/python3). +# +# This script vendors all Python deps and this project into a folder and +# generates a small launcher that executes: -m mlxk2.cli ... +# with PYTHONPATH pointing at the vendored deps. +# +# Usage: +# # Preferred: point to your embedded Python +# EMBEDDED_PYTHON="/path/to/MyApp.app/Contents/Resources/python/bin/python3" \ +# scripts/build_external_python.sh +# +# # Dev fallback (uses repo-local python/bin if present, else system python3) +# scripts/build_external_python.sh +# +# Optional env: +# BIN_NAME=my-cli -> output folder/launcher name (default msty-mlx-studio) +# DIST_DIR=dist-ext-python -> output root (default dist-ext-python) +# PIP_ARGS="..." -> extra pip args (e.g., --no-index --find-links ...) +# CLEAN=1 -> delete output dir before build +# + +ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +cd "$ROOT_DIR" + +APP_NAME="${BIN_NAME:-${APP_NAME:-msty-mlx-studio}}" +OUT_ROOT="${DIST_DIR:-dist-ext-python}" +OUT_DIR="$OUT_ROOT/$APP_NAME" +VENDOR_DIR="$OUT_DIR/_vendor" + +# Resolve external Python +if [[ -n "${EMBEDDED_PYTHON:-}" ]]; then + PY="$EMBEDDED_PYTHON" + echo "Using EMBEDDED_PYTHON: $PY" +elif [[ -x "$ROOT_DIR/python/bin/python3" ]]; then + PY="$ROOT_DIR/python/bin/python3" + echo "Using repo-local Python: $PY" +else + PY="$(command -v python3)" + echo "WARNING: Falling back to system python3: $PY" >&2 +fi + +if [[ ! -x "$PY" ]]; then + echo "ERROR: Python interpreter not executable: $PY" >&2 + exit 1 +fi + +# Clean output if requested +if [[ "${CLEAN:-0}" == "1" ]]; then + rm -rf "$OUT_DIR" +fi +mkdir -p "$VENDOR_DIR" + +if [[ "${VENDOR_VIA_WHEELS:-0}" == "1" ]]; then + echo "[1/3] Vendoring via wheels (does not execute embedded Python)" + SYS_PY="$(command -v python3)" + if [[ -z "$SYS_PY" ]]; then + echo "ERROR: python3 not found on PATH for wheel download" >&2 + exit 1 + fi + : "${TARGET_PY_VERSION:?Set TARGET_PY_VERSION (e.g., 3.10)}" + TARGET_PLATFORM="${TARGET_PLATFORM:-macosx_11_0_arm64}" + # Derive ABI from version (e.g., 3.10 -> cp310) + ver_nodot="${TARGET_PY_VERSION/.}" # 3.10 -> 310 + TARGET_ABI="${TARGET_ABI:-cp${ver_nodot}}" + WHEELS_DIR="$OUT_DIR/_wheels" + rm -rf "$WHEELS_DIR" && mkdir -p "$WHEELS_DIR" + + echo "[1a] Downloading wheels for target Python ${TARGET_PY_VERSION} (${TARGET_ABI}) platform ${TARGET_PLATFORM}..." + if [[ -f requirements.txt ]]; then + "$SYS_PY" -m pip download -r requirements.txt -d "$WHEELS_DIR" \ + --only-binary=:all: --implementation cp --platform "$TARGET_PLATFORM" \ + --python-version "${TARGET_PY_VERSION}" --abi "$TARGET_ABI" ${PIP_ARGS:-} + fi + echo "[1b] Building wheel for local package..." + "$SYS_PY" -m pip wheel -w "$WHEELS_DIR" . ${PIP_ARGS:-} + + echo "[2/3] Unpacking wheels into vendor..." + for whl in "$WHEELS_DIR"/*.whl; do + [ -e "$whl" ] || { echo "No wheels found in $WHEELS_DIR" >&2; exit 1; } + unzip -oq "$whl" -d "$VENDOR_DIR" + done + + # Sanity: detect missing MLX wheel for the chosen Python version + if grep -Eqi '^\s*mlx([>=<]|\s|$)' "$ROOT_DIR/requirements.txt" 2>/dev/null; then + if ! ls "$WHEELS_DIR"/mlx-*.whl >/dev/null 2>&1; then + echo "" >&2 + echo "ERROR: No mlx wheel was downloaded for TARGET_PY_VERSION=${TARGET_PY_VERSION}." >&2 + echo "MLX publishes wheels only for certain Python versions (typically 3.11/3.12)." >&2 + echo "Try one of these options:" >&2 + echo " - Use TARGET_PY_VERSION=3.12 (or 3.11) with VENDOR_VIA_WHEELS=1" >&2 + echo " - Or omit VENDOR_VIA_WHEELS to let the embedded Python build from source (requires unquarantined/signed interpreter)" >&2 + exit 1 + fi + fi +else + echo "[1/3] Installing/Upgrading pip in external Python..." + "$PY" - <<'PY' +import ensurepip, sys +try: + ensurepip.bootstrap() +except Exception: + pass +PY + "$PY" -m pip install --upgrade pip setuptools wheel ${PIP_ARGS:-} + + echo "[2/3] Installing project dependencies into: $VENDOR_DIR" + if [[ -f requirements.txt ]]; then + "$PY" -m pip install --target "$VENDOR_DIR" -r requirements.txt ${PIP_ARGS:-} + fi + + echo "[2/3b] Installing mlxk2 package into vendor..." + "$PY" -m pip install --target "$VENDOR_DIR" . ${PIP_ARGS:-} +fi + +echo "[3/3] Writing launcher: $OUT_DIR/$APP_NAME" +cat > "$OUT_DIR/$APP_NAME" <<'SH' +#!/usr/bin/env bash +set -euo pipefail + +# Launcher for running mlxk2 against an external Python interpreter. +# +# Picks the interpreter from: +# 1) MLXK_PYTHON override +# 2) MyApp Electron resources path (macOS): +# "$RESOURCES_PATH/python/bin/python3" if RESOURCES_PATH is set +# 3) repo-local ./python/bin/python3 relative to this launcher +# 4) system python3 + +SELF_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +VENDOR_DIR="$SELF_DIR/_vendor" + +PY="${MLXK_PYTHON:-}" +if [[ -z "$PY" && -n "${RESOURCES_PATH:-}" && -x "$RESOURCES_PATH/python/bin/python3" ]]; then + PY="$RESOURCES_PATH/python/bin/python3" +fi +if [[ -z "$PY" && -x "$SELF_DIR/../python/bin/python3" ]]; then + PY="$SELF_DIR/../python/bin/python3" +fi +if [[ -z "$PY" ]]; then + PY="$(command -v python3)" +fi + +if [[ "${1:-}" == "--python-info" ]]; then + shift || true + "$PY" - <<'PY' +import sys, json +print(json.dumps({ + "python_version": sys.version.split(" (", 1)[0], + "executable": sys.executable, + "prefix": sys.prefix, + "base_prefix": getattr(sys, "base_prefix", None), + "platform": sys.platform +}, indent=2)) +PY + exit 0 +fi + +export PYTHONNOUSERSITE=1 +if [[ -n "${PYTHONPATH:-}" ]]; then + export PYTHONPATH="$VENDOR_DIR:$PYTHONPATH" +else + export PYTHONPATH="$VENDOR_DIR" +fi + +exec "$PY" -m mlxk2.cli "$@" +SH +chmod +x "$OUT_DIR/$APP_NAME" + +cp -f LICENSE "$OUT_DIR/" 2>/dev/null || true + +echo "Built external-Python dist at: $OUT_DIR" +echo "Quick test:" +echo " $OUT_DIR/$APP_NAME --python-info" +echo " MLXK_PYTHON=\"/path/to/MyApp.app/Contents/Resources/python/bin/python3\" \\ + $OUT_DIR/$APP_NAME --version" diff --git a/scripts/clear_quarantine_python.sh b/scripts/clear_quarantine_python.sh new file mode 100755 index 0000000..6bfc710 --- /dev/null +++ b/scripts/clear_quarantine_python.sh @@ -0,0 +1,68 @@ +#!/usr/bin/env bash +set -euo pipefail + +# Recursively remove macOS Gatekeeper quarantine from an embedded Python folder +# and verify any leftovers (executables, .dylib, .so, etc.). +# +# Usage: +# scripts/clear_quarantine_python.sh /path/to/MyApp.app/Contents/Resources/python +# # or, if your repo has ./python +# scripts/clear_quarantine_python.sh +# + +if [[ "${1:-}" == "-h" || "${1:-}" == "--help" ]]; then + echo "Usage: $0 [python_folder]" + echo "Example: $0 /Applications/MyApp.app/Contents/Resources/python" + exit 0 +fi + +PY_DIR="${1:-}" +if [[ -z "$PY_DIR" ]]; then + if [[ -d "python" ]]; then + PY_DIR="python" + else + echo "ERROR: No python folder provided and ./python not found." >&2 + echo "Provide the path to your embedded Python folder." >&2 + exit 1 + fi +fi + +if [[ ! -d "$PY_DIR" ]]; then + echo "ERROR: Not a directory: $PY_DIR" >&2 + exit 1 +fi + +if [[ "$(uname -s)" != "Darwin" ]]; then + echo "WARNING: This script is intended for macOS (Darwin). Proceeding anyway..." >&2 +fi + +echo "Target python folder: $PY_DIR" + +echo "Scanning for quarantined items before..." +before_list=$(mktemp) +find "$PY_DIR" -type f -print0 \ + | xargs -0 -I{} sh -c 'xattr -p com.apple.quarantine "$1" >/dev/null 2>&1 && printf "%s\n" "$1"' -- {} \ + | tee "$before_list" >/dev/null || true +before_count=$(wc -l < "$before_list" | tr -d ' ') +echo "Quarantined files found: ${before_count}" + +echo "Clearing quarantine recursively..." +xattr -dr com.apple.quarantine "$PY_DIR" || true + +echo "Re-scanning for leftovers..." +after_list=$(mktemp) +find "$PY_DIR" -type f -print0 \ + | xargs -0 -I{} sh -c 'xattr -p com.apple.quarantine "$1" >/dev/null 2>&1 && printf "%s\n" "$1"' -- {} \ + | tee "$after_list" >/dev/null || true +after_count=$(wc -l < "$after_list" | tr -d ' ') + +if [[ "$after_count" -eq 0 ]]; then + echo "Success: no quarantined files remain under $PY_DIR." +else + echo "WARNING: ${after_count} quarantined files remain (showing up to 20):" + head -n 20 "$after_list" + echo "You may need elevated permissions or to remove quarantine on parent folders." +fi + +rm -f "$before_list" "$after_list" +