chore: include self bundled python script

2026-07-19 14:43:36 -04:00 · 2025-09-25 11:15:28 -04:00
parent 0619854bb1
commit 42fd195095
3 changed files with 349 additions and 0 deletions
@@ -0,0 +1,101 @@
+# External Python Mode (Electron-embedded Python)
+
+This variant does not use PyInstaller. Instead, it vendors all Python deps (including `mlxk2`) into a folder and runs against the Python interpreter you ship inside your Electron app (`MyApp.app/Contents/Resources/python/bin/python3`).
+
+## Build
+
+```bash
+# Recommended: point to your embedded Python
+EMBEDDED_PYTHON="/path/to/MyApp.app/Contents/Resources/python/bin/python3" \
+  scripts/build_external_python.sh
+
+# Dev fallback (uses ./python/bin/python3 if present, else system python3)
+scripts/build_external_python.sh
+```
+
+Output: `dist-ext-python/<BIN_NAME>/` (default `msty-mlx-studio/`), containing:
+- `_vendor/` – vendored site-packages (deps + project)
+- `msty-mlx-studio` – launcher that runs `-m mlxk2.cli` with `PYTHONPATH=_vendor`
+
+Verify the Python in use:
+
+```bash
+./dist-ext-python/msty-mlx-studio/msty-mlx-studio --python-info
+
+### Building without executing the embedded Python (avoids Gatekeeper during dev)
+
+If macOS blocks running your embedded Python during the build, you can vendor wheels for a target version without executing that interpreter:
+
+```bash
+TARGET_PY_VERSION=3.10 \
+VENDOR_VIA_WHEELS=1 \
+scripts/build_external_python.sh
+```
+
+Notes:
+- This uses your system Python to `pip download` wheels for macOS arm64 and the specified Python version, then unpacks them into `_vendor`.
+- At runtime, the Electron-embedded Python imports from `_vendor` via `PYTHONPATH`.
+- For compiled packages, you must choose a `TARGET_PY_VERSION` that matches your embedded Python minor version (e.g., 3.10 → cp310 wheels).
+```
+
+## Packaging into Electron
+
+Place the folder under your app resources, for example:
+
+```
+MyApp.app/Contents/Resources/
+  python/bin/python3              # your embedded Python
+  mlxk2/
+    _vendor/
+    msty-mlx-studio               # launcher
+```
+
+### Option A: Use the launcher
+
+- Command: `join(process.resourcesPath, 'mlxk2', 'msty-mlx-studio')`
+- Env: set `RESOURCES_PATH=process.resourcesPath` (so the launcher finds `python/bin/python3`).
+
+```js
+import { spawn } from 'child_process';
+import { join } from 'path';
+import { app } from 'electron';
+
+const launcher = join(process.resourcesPath, 'mlxk2', 'msty-mlx-studio');
+
+export function startServer({ host = '127.0.0.1', port = 8000, maxTokens } = {}) {
+  const args = ['serve', '--host', host, '--port', String(port)];
+  if (maxTokens != null) args.push('--max-tokens', String(maxTokens));
+  const env = { ...process.env, RESOURCES_PATH: process.resourcesPath };
+  const child = spawn(launcher, args, { env, stdio: ['ignore', 'inherit', 'inherit'] });
+  return child;
+}
+```
+
+### Option B: Call Python directly
+
+- Python: `join(process.resourcesPath, 'python', 'bin', 'python3')`
+- Vendor: `join(process.resourcesPath, 'mlxk2', '_vendor')`
+- Env: `PYTHONPATH=<vendor>[:$PYTHONPATH]`, `PYTHONNOUSERSITE=1`
+
+```js
+import { spawn } from 'child_process';
+import { join } from 'path';
+import { app } from 'electron';
+
+export function startServer({ host = '127.0.0.1', port = 8000, maxTokens } = {}) {
+  const pythonPath = join(process.resourcesPath, 'python', 'bin', 'python3');
+  const vendor = join(process.resourcesPath, 'mlxk2', '_vendor');
+  const args = ['-m', 'mlxk2.cli', 'serve', '--host', host, '--port', String(port)];
+  if (maxTokens != null) args.push('--max-tokens', String(maxTokens));
+  const env = { ...process.env, PYTHONNOUSERSITE: '1', PYTHONPATH: vendor };
+  const child = spawn(pythonPath, args, { env, stdio: ['ignore', 'inherit', 'inherit'] });
+  return child;
+}
+```
+
+## Notes
+
+- You are responsible for shipping a compatible macOS arm64 Python with all required system libs.
+- If using private or offline wheels, pass extra pip args: `PIP_ARGS="--no-index --find-links /path/to/wheels" scripts/build_external_python.sh`.
+- To sanity check deps at runtime:
+  `PYTHONPATH=<vendor> <embedded-python> -c "import mlxk2, fastapi, uvicorn, mlx, mlx_lm; print('ok')"`
@@ -0,0 +1,180 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+# Build a "no-PyInstaller" distribution that runs against an external Python
+# (e.g., the interpreter embedded in your Electron app at
+#   MyApp.app/Contents/Resources/python/bin/python3).
+#
+# This script vendors all Python deps and this project into a folder and
+# generates a small launcher that executes: <external-python> -m mlxk2.cli ...
+# with PYTHONPATH pointing at the vendored deps.
+#
+# Usage:
+#   # Preferred: point to your embedded Python
+#   EMBEDDED_PYTHON="/path/to/MyApp.app/Contents/Resources/python/bin/python3" \
+#     scripts/build_external_python.sh
+#
+#   # Dev fallback (uses repo-local python/bin if present, else system python3)
+#   scripts/build_external_python.sh
+#
+# Optional env:
+#   BIN_NAME=my-cli             -> output folder/launcher name (default msty-mlx-studio)
+#   DIST_DIR=dist-ext-python    -> output root (default dist-ext-python)
+#   PIP_ARGS="..."              -> extra pip args (e.g., --no-index --find-links ...)
+#   CLEAN=1                     -> delete output dir before build
+#
+
+ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
+cd "$ROOT_DIR"
+
+APP_NAME="${BIN_NAME:-${APP_NAME:-msty-mlx-studio}}"
+OUT_ROOT="${DIST_DIR:-dist-ext-python}"
+OUT_DIR="$OUT_ROOT/$APP_NAME"
+VENDOR_DIR="$OUT_DIR/_vendor"
+
+# Resolve external Python
+if [[ -n "${EMBEDDED_PYTHON:-}" ]]; then
+  PY="$EMBEDDED_PYTHON"
+  echo "Using EMBEDDED_PYTHON: $PY"
+elif [[ -x "$ROOT_DIR/python/bin/python3" ]]; then
+  PY="$ROOT_DIR/python/bin/python3"
+  echo "Using repo-local Python: $PY"
+else
+  PY="$(command -v python3)"
+  echo "WARNING: Falling back to system python3: $PY" >&2
+fi
+
+if [[ ! -x "$PY" ]]; then
+  echo "ERROR: Python interpreter not executable: $PY" >&2
+  exit 1
+fi
+
+# Clean output if requested
+if [[ "${CLEAN:-0}" == "1" ]]; then
+  rm -rf "$OUT_DIR"
+fi
+mkdir -p "$VENDOR_DIR"
+
+if [[ "${VENDOR_VIA_WHEELS:-0}" == "1" ]]; then
+  echo "[1/3] Vendoring via wheels (does not execute embedded Python)"
+  SYS_PY="$(command -v python3)"
+  if [[ -z "$SYS_PY" ]]; then
+    echo "ERROR: python3 not found on PATH for wheel download" >&2
+    exit 1
+  fi
+  : "${TARGET_PY_VERSION:?Set TARGET_PY_VERSION (e.g., 3.10)}"
+  TARGET_PLATFORM="${TARGET_PLATFORM:-macosx_11_0_arm64}"
+  # Derive ABI from version (e.g., 3.10 -> cp310)
+  ver_nodot="${TARGET_PY_VERSION/.}"   # 3.10 -> 310
+  TARGET_ABI="${TARGET_ABI:-cp${ver_nodot}}"
+  WHEELS_DIR="$OUT_DIR/_wheels"
+  rm -rf "$WHEELS_DIR" && mkdir -p "$WHEELS_DIR"
+
+  echo "[1a] Downloading wheels for target Python ${TARGET_PY_VERSION} (${TARGET_ABI}) platform ${TARGET_PLATFORM}..."
+  if [[ -f requirements.txt ]]; then
+    "$SYS_PY" -m pip download -r requirements.txt -d "$WHEELS_DIR" \
+      --only-binary=:all: --implementation cp --platform "$TARGET_PLATFORM" \
+      --python-version "${TARGET_PY_VERSION}" --abi "$TARGET_ABI" ${PIP_ARGS:-}
+  fi
+  echo "[1b] Building wheel for local package..."
+  "$SYS_PY" -m pip wheel -w "$WHEELS_DIR" . ${PIP_ARGS:-}
+
+  echo "[2/3] Unpacking wheels into vendor..."
+  for whl in "$WHEELS_DIR"/*.whl; do
+    [ -e "$whl" ] || { echo "No wheels found in $WHEELS_DIR" >&2; exit 1; }
+    unzip -oq "$whl" -d "$VENDOR_DIR"
+  done
+
+  # Sanity: detect missing MLX wheel for the chosen Python version
+  if grep -Eqi '^\s*mlx([>=<]|\s|$)' "$ROOT_DIR/requirements.txt" 2>/dev/null; then
+    if ! ls "$WHEELS_DIR"/mlx-*.whl >/dev/null 2>&1; then
+      echo "" >&2
+      echo "ERROR: No mlx wheel was downloaded for TARGET_PY_VERSION=${TARGET_PY_VERSION}." >&2
+      echo "MLX publishes wheels only for certain Python versions (typically 3.11/3.12)." >&2
+      echo "Try one of these options:" >&2
+      echo "  - Use TARGET_PY_VERSION=3.12 (or 3.11) with VENDOR_VIA_WHEELS=1" >&2
+      echo "  - Or omit VENDOR_VIA_WHEELS to let the embedded Python build from source (requires unquarantined/signed interpreter)" >&2
+      exit 1
+    fi
+  fi
+else
+  echo "[1/3] Installing/Upgrading pip in external Python..."
+  "$PY" - <<'PY'
+import ensurepip, sys
+try:
+    ensurepip.bootstrap()
+except Exception:
+    pass
+PY
+  "$PY" -m pip install --upgrade pip setuptools wheel ${PIP_ARGS:-}
+
+  echo "[2/3] Installing project dependencies into: $VENDOR_DIR"
+  if [[ -f requirements.txt ]]; then
+    "$PY" -m pip install --target "$VENDOR_DIR" -r requirements.txt ${PIP_ARGS:-}
+  fi
+
+  echo "[2/3b] Installing mlxk2 package into vendor..."
+  "$PY" -m pip install --target "$VENDOR_DIR" . ${PIP_ARGS:-}
+fi
+
+echo "[3/3] Writing launcher: $OUT_DIR/$APP_NAME"
+cat > "$OUT_DIR/$APP_NAME" <<'SH'
+#!/usr/bin/env bash
+set -euo pipefail
+
+# Launcher for running mlxk2 against an external Python interpreter.
+#
+# Picks the interpreter from:
+#   1) MLXK_PYTHON override
+#   2) MyApp Electron resources path (macOS):
+#        "$RESOURCES_PATH/python/bin/python3" if RESOURCES_PATH is set
+#   3) repo-local ./python/bin/python3 relative to this launcher
+#   4) system python3
+
+SELF_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+VENDOR_DIR="$SELF_DIR/_vendor"
+
+PY="${MLXK_PYTHON:-}"
+if [[ -z "$PY" && -n "${RESOURCES_PATH:-}" && -x "$RESOURCES_PATH/python/bin/python3" ]]; then
+  PY="$RESOURCES_PATH/python/bin/python3"
+fi
+if [[ -z "$PY" && -x "$SELF_DIR/../python/bin/python3" ]]; then
+  PY="$SELF_DIR/../python/bin/python3"
+fi
+if [[ -z "$PY" ]]; then
+  PY="$(command -v python3)"
+fi
+
+if [[ "${1:-}" == "--python-info" ]]; then
+  shift || true
+  "$PY" - <<'PY'
+import sys, json
+print(json.dumps({
+  "python_version": sys.version.split(" (", 1)[0],
+  "executable": sys.executable,
+  "prefix": sys.prefix,
+  "base_prefix": getattr(sys, "base_prefix", None),
+  "platform": sys.platform
+}, indent=2))
+PY
+  exit 0
+fi
+
+export PYTHONNOUSERSITE=1
+if [[ -n "${PYTHONPATH:-}" ]]; then
+  export PYTHONPATH="$VENDOR_DIR:$PYTHONPATH"
+else
+  export PYTHONPATH="$VENDOR_DIR"
+fi
+
+exec "$PY" -m mlxk2.cli "$@"
+SH
+chmod +x "$OUT_DIR/$APP_NAME"
+
+cp -f LICENSE "$OUT_DIR/" 2>/dev/null || true
+
+echo "Built external-Python dist at: $OUT_DIR"
+echo "Quick test:"
+echo "  $OUT_DIR/$APP_NAME --python-info"
+echo "  MLXK_PYTHON=\"/path/to/MyApp.app/Contents/Resources/python/bin/python3\" \\
+    $OUT_DIR/$APP_NAME --version"
@@ -0,0 +1,68 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+# Recursively remove macOS Gatekeeper quarantine from an embedded Python folder
+# and verify any leftovers (executables, .dylib, .so, etc.).
+#
+# Usage:
+#   scripts/clear_quarantine_python.sh /path/to/MyApp.app/Contents/Resources/python
+#   # or, if your repo has ./python
+#   scripts/clear_quarantine_python.sh
+#
+
+if [[ "${1:-}" == "-h" || "${1:-}" == "--help" ]]; then
+  echo "Usage: $0 [python_folder]"
+  echo "Example: $0 /Applications/MyApp.app/Contents/Resources/python"
+  exit 0
+fi
+
+PY_DIR="${1:-}"
+if [[ -z "$PY_DIR" ]]; then
+  if [[ -d "python" ]]; then
+    PY_DIR="python"
+  else
+    echo "ERROR: No python folder provided and ./python not found." >&2
+    echo "Provide the path to your embedded Python folder." >&2
+    exit 1
+  fi
+fi
+
+if [[ ! -d "$PY_DIR" ]]; then
+  echo "ERROR: Not a directory: $PY_DIR" >&2
+  exit 1
+fi
+
+if [[ "$(uname -s)" != "Darwin" ]]; then
+  echo "WARNING: This script is intended for macOS (Darwin). Proceeding anyway..." >&2
+fi
+
+echo "Target python folder: $PY_DIR"
+
+echo "Scanning for quarantined items before..."
+before_list=$(mktemp)
+find "$PY_DIR" -type f -print0 \
+  | xargs -0 -I{} sh -c 'xattr -p com.apple.quarantine "$1" >/dev/null 2>&1 && printf "%s\n" "$1"' -- {} \
+  | tee "$before_list" >/dev/null || true
+before_count=$(wc -l < "$before_list" | tr -d ' ')
+echo "Quarantined files found: ${before_count}"
+
+echo "Clearing quarantine recursively..."
+xattr -dr com.apple.quarantine "$PY_DIR" || true
+
+echo "Re-scanning for leftovers..."
+after_list=$(mktemp)
+find "$PY_DIR" -type f -print0 \
+  | xargs -0 -I{} sh -c 'xattr -p com.apple.quarantine "$1" >/dev/null 2>&1 && printf "%s\n" "$1"' -- {} \
+  | tee "$after_list" >/dev/null || true
+after_count=$(wc -l < "$after_list" | tr -d ' ')
+
+if [[ "$after_count" -eq 0 ]]; then
+  echo "Success: no quarantined files remain under $PY_DIR."
+else
+  echo "WARNING: ${after_count} quarantined files remain (showing up to 20):"
+  head -n 20 "$after_list"
+  echo "You may need elevated permissions or to remove quarantine on parent folders."
+fi
+
+rm -f "$before_list" "$after_list"
+