mirror of
https://github.com/cloudstack-llc/mlx-knife.git
synced 2026-07-21 10:05:26 -04:00
The BROKE Cluster Team
bf7480d042
Major Features: - Audio transcription via mlx-audio backend (Whisper, >10min duration) - OpenAI /v1/audio/transcriptions endpoint - Memory Gate System (Vision: 8GB, Audio: 4GB) - Config-based backend routing (ADR-020) - Benchmark toolchain (memmon/memplot, Schema v0.2.2) Key Fixes: - EuroLLM tokenizer decoding - Vision-model text-only routing regression - Multimodal model context length detection - Memory cleanup bug (mx.metal.clear_cache) - Orphan process bug Test Results: - Unit tests: 647 passed, 11 skipped (Python 3.10-3.12) - wet-umbrella: 171 passed total See CHANGELOG.md for complete details and known issues.
65 lines
2.3 KiB
Python
65 lines
2.3 KiB
Python
from __future__ import annotations
|
|
|
|
import json
|
|
import os
|
|
from typing import Optional
|
|
|
|
|
|
def get_model_context_length(model_path: str) -> int:
|
|
"""Extract max_position_embeddings from model config with safe fallbacks.
|
|
|
|
Supports both flat configs (text-only models) and nested configs (multimodal models
|
|
like Mistral3, Pixtral with text_config/vision_config).
|
|
|
|
Returns a sensible default (4096) if the config is missing or malformed.
|
|
"""
|
|
config_path = os.path.join(model_path, "config.json")
|
|
try:
|
|
with open(config_path) as f:
|
|
config = json.load(f)
|
|
|
|
context_keys = [
|
|
"max_position_embeddings",
|
|
"n_positions",
|
|
"context_length",
|
|
"max_sequence_length",
|
|
"seq_len",
|
|
]
|
|
|
|
# Priority 1: Try top-level keys (text-only models)
|
|
for key in context_keys:
|
|
if key in config:
|
|
value = config[key]
|
|
if isinstance(value, int) and value > 0:
|
|
return value
|
|
if isinstance(value, str) and value.isdigit():
|
|
parsed = int(value)
|
|
if parsed > 0:
|
|
return parsed
|
|
|
|
# Priority 2: Try text_config for multimodal models (Mistral3, Pixtral)
|
|
# These models have separate text_config and vision_config
|
|
if "text_config" in config and isinstance(config["text_config"], dict):
|
|
text_config = config["text_config"]
|
|
for key in context_keys:
|
|
if key in text_config:
|
|
value = text_config[key]
|
|
if isinstance(value, int) and value > 0:
|
|
return value
|
|
if isinstance(value, str) and value.isdigit():
|
|
parsed = int(value)
|
|
if parsed > 0:
|
|
return parsed
|
|
|
|
return 4096
|
|
except (FileNotFoundError, json.JSONDecodeError, KeyError):
|
|
return 4096
|
|
|
|
|
|
def calculate_dynamic_max_tokens(context_length: Optional[int], server_mode: bool = True) -> int:
|
|
"""Compute an effective generation limit based on context and mode."""
|
|
if not context_length or context_length <= 0:
|
|
return 2048
|
|
return context_length // 2 if server_mode else context_length
|
|
|