mirror of
https://github.com/cloudstack-llc/mlx-knife.git
synced 2026-07-09 15:09:34 -04:00
The BROKE Team
1aad374d08
Major fixes: - Issue #19: Server response truncation resolved - large context models work at full capacity - Issue #20: End-Token filtering in non-streaming mode - clean professional output - Test stability: Fixed flaky server tests, improved lifecycle management Technical changes: - Server: Dynamic token limits by default (--max-tokens None) - MLXRunner: Added _filter_end_tokens_from_response() for batch consistency - Tests: 132/132 passing + 48 comprehensive server tests - Documentation: Updated CHANGELOG.md, README.md, TESTING.md
584 lines
20 KiB
HTML
584 lines
20 KiB
HTML
<!DOCTYPE html>
|
|
<html lang="en">
|
|
<head>
|
|
<meta charset="UTF-8">
|
|
<meta name="viewport" content="width=device-width, initial-scale=1.0">
|
|
<title>MLX Knife Chat</title>
|
|
<script src="https://cdn.jsdelivr.net/npm/marked@5.1.1/marked.min.js"></script>
|
|
<style>
|
|
body {
|
|
font-family: -apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, sans-serif;
|
|
max-width: 800px;
|
|
margin: 0 auto;
|
|
padding: 20px;
|
|
background: #f5f5f5;
|
|
height: 100vh;
|
|
box-sizing: border-box;
|
|
}
|
|
|
|
.chat-container {
|
|
background: white;
|
|
border-radius: 10px;
|
|
padding: 20px;
|
|
box-shadow: 0 2px 10px rgba(0,0,0,0.1);
|
|
height: 100%;
|
|
display: flex;
|
|
flex-direction: column;
|
|
}
|
|
|
|
.header {
|
|
text-align: center;
|
|
color: #333;
|
|
border-bottom: 1px solid #eee;
|
|
padding-bottom: 15px;
|
|
margin-bottom: 20px;
|
|
}
|
|
|
|
.model-selector {
|
|
margin-bottom: 20px;
|
|
}
|
|
|
|
select {
|
|
width: 100%;
|
|
padding: 8px;
|
|
border: 1px solid #ddd;
|
|
border-radius: 5px;
|
|
font-size: 14px;
|
|
}
|
|
|
|
.chat-messages {
|
|
flex: 1;
|
|
overflow-y: auto;
|
|
border: 1px solid #eee;
|
|
padding: 15px;
|
|
margin-bottom: 15px;
|
|
border-radius: 5px;
|
|
background: #fafafa;
|
|
min-height: 300px;
|
|
}
|
|
|
|
.message {
|
|
margin-bottom: 15px;
|
|
padding: 10px;
|
|
border-radius: 8px;
|
|
max-width: 80%;
|
|
}
|
|
|
|
.user-message {
|
|
background: #007AFF;
|
|
color: white;
|
|
margin-left: auto;
|
|
}
|
|
|
|
.assistant-message {
|
|
background: #e5e5e7;
|
|
color: #333;
|
|
}
|
|
|
|
.typing {
|
|
opacity: 0.7;
|
|
font-style: italic;
|
|
}
|
|
|
|
.input-container {
|
|
display: flex;
|
|
gap: 10px;
|
|
}
|
|
|
|
#messageInput {
|
|
flex: 1;
|
|
padding: 12px;
|
|
border: 1px solid #ddd;
|
|
border-radius: 25px;
|
|
font-size: 16px;
|
|
outline: none;
|
|
}
|
|
|
|
#clearButton, #sendButton {
|
|
padding: 12px 20px;
|
|
border: none;
|
|
border-radius: 25px;
|
|
cursor: pointer;
|
|
font-weight: bold;
|
|
}
|
|
|
|
#clearButton {
|
|
background: #6c757d;
|
|
color: white;
|
|
}
|
|
|
|
#clearButton:hover:not(:disabled) {
|
|
background: #5a6268;
|
|
}
|
|
|
|
#clearButton:disabled {
|
|
background: #ccc;
|
|
cursor: not-allowed;
|
|
}
|
|
|
|
#sendButton {
|
|
background: #007AFF;
|
|
color: white;
|
|
}
|
|
|
|
#sendButton:hover:not(:disabled) {
|
|
background: #0056b3;
|
|
}
|
|
|
|
#sendButton:disabled {
|
|
background: #ccc;
|
|
cursor: not-allowed;
|
|
}
|
|
|
|
.status {
|
|
text-align: center;
|
|
color: #666;
|
|
font-size: 12px;
|
|
margin-top: 10px;
|
|
}
|
|
|
|
.connected { color: #28a745; }
|
|
.disconnected { color: #dc3545; }
|
|
|
|
/* Markdown formatting */
|
|
.message pre {
|
|
background: #f8f9fa;
|
|
border: 1px solid #e9ecef;
|
|
border-radius: 4px;
|
|
padding: 12px;
|
|
overflow-x: auto;
|
|
margin: 8px 0;
|
|
}
|
|
|
|
.message code {
|
|
background: #f8f9fa;
|
|
border: 1px solid #e9ecef;
|
|
border-radius: 3px;
|
|
padding: 2px 4px;
|
|
font-family: 'Monaco', 'Menlo', 'Ubuntu Mono', monospace;
|
|
font-size: 14px;
|
|
}
|
|
|
|
.message pre code {
|
|
background: transparent;
|
|
border: none;
|
|
padding: 0;
|
|
}
|
|
|
|
.message h1, .message h2, .message h3, .message h4, .message h5, .message h6 {
|
|
margin: 16px 0 8px 0;
|
|
color: #333;
|
|
}
|
|
|
|
.message ul, .message ol {
|
|
margin: 8px 0;
|
|
padding-left: 20px;
|
|
}
|
|
|
|
.message blockquote {
|
|
border-left: 4px solid #ddd;
|
|
margin: 8px 0;
|
|
padding-left: 16px;
|
|
color: #666;
|
|
font-style: italic;
|
|
}
|
|
|
|
.message table {
|
|
border-collapse: collapse;
|
|
width: 100%;
|
|
margin: 8px 0;
|
|
}
|
|
|
|
.message th, .message td {
|
|
border: 1px solid #ddd;
|
|
padding: 8px;
|
|
text-align: left;
|
|
}
|
|
|
|
.message th {
|
|
background-color: #f8f9fa;
|
|
font-weight: bold;
|
|
}
|
|
|
|
/* Custom modal styles */
|
|
.modal-overlay {
|
|
position: fixed;
|
|
top: 0;
|
|
left: 0;
|
|
width: 100%;
|
|
height: 100%;
|
|
background: rgba(0,0,0,0.5);
|
|
display: flex;
|
|
align-items: center;
|
|
justify-content: center;
|
|
z-index: 1000;
|
|
}
|
|
|
|
.modal {
|
|
background: white;
|
|
padding: 30px;
|
|
border-radius: 10px;
|
|
box-shadow: 0 4px 20px rgba(0,0,0,0.3);
|
|
max-width: 400px;
|
|
text-align: center;
|
|
}
|
|
|
|
.modal-buttons {
|
|
margin-top: 20px;
|
|
display: flex;
|
|
gap: 15px;
|
|
justify-content: center;
|
|
}
|
|
|
|
.modal-button {
|
|
padding: 10px 20px;
|
|
border: none;
|
|
border-radius: 5px;
|
|
cursor: pointer;
|
|
font-weight: bold;
|
|
min-width: 100px;
|
|
}
|
|
|
|
.modal-button.primary {
|
|
background: #007AFF;
|
|
color: white;
|
|
}
|
|
|
|
.modal-button.secondary {
|
|
background: #6c757d;
|
|
color: white;
|
|
}
|
|
|
|
.modal-button:hover {
|
|
opacity: 0.8;
|
|
}
|
|
</style>
|
|
</head>
|
|
<body>
|
|
<div class="chat-container">
|
|
<div class="header">
|
|
<h1>🦫 MLX Knife Chat</h1>
|
|
<div class="status" id="status">Connecting...</div>
|
|
</div>
|
|
|
|
<div class="model-selector">
|
|
<select id="modelSelect">
|
|
<option value="">Loading models...</option>
|
|
</select>
|
|
</div>
|
|
|
|
<div class="chat-messages" id="chatMessages">
|
|
<div class="message assistant-message">
|
|
<strong>MLX Assistant:</strong> Hi! I'm ready to chat. Select a model and send me a message!
|
|
</div>
|
|
</div>
|
|
|
|
<div class="input-container">
|
|
<input type="text" id="messageInput" placeholder="Type your message..." disabled>
|
|
<button id="clearButton" onclick="clearChat()">Clear Chat</button>
|
|
<button id="sendButton" disabled>Send</button>
|
|
</div>
|
|
</div>
|
|
|
|
<script>
|
|
const API_BASE = 'http://localhost:8000';
|
|
let currentModel = localStorage.getItem('mlxk_selected_model') || '';
|
|
let isConnected = false;
|
|
let conversationHistory = JSON.parse(localStorage.getItem('mlxk_chat_history') || '[]');
|
|
let isStreaming = false;
|
|
|
|
// DOM elements
|
|
const statusEl = document.getElementById('status');
|
|
const modelSelect = document.getElementById('modelSelect');
|
|
const chatMessages = document.getElementById('chatMessages');
|
|
const messageInput = document.getElementById('messageInput');
|
|
const sendButton = document.getElementById('sendButton');
|
|
const clearButton = document.getElementById('clearButton');
|
|
|
|
// Update status
|
|
function updateStatus(connected, message = '') {
|
|
isConnected = connected;
|
|
statusEl.textContent = message || (connected ? 'Connected' : 'Disconnected');
|
|
statusEl.className = `status ${connected ? 'connected' : 'disconnected'}`;
|
|
|
|
messageInput.disabled = !connected || !currentModel || isStreaming;
|
|
sendButton.disabled = !connected || !currentModel || isStreaming;
|
|
clearButton.disabled = isStreaming || conversationHistory.length === 0;
|
|
}
|
|
|
|
// Check API health
|
|
async function checkHealth() {
|
|
try {
|
|
const response = await fetch(`${API_BASE}/health`);
|
|
if (response.ok) {
|
|
updateStatus(true, 'Connected to MLX Knife');
|
|
loadModels();
|
|
} else {
|
|
updateStatus(false, 'API Error');
|
|
}
|
|
} catch (error) {
|
|
updateStatus(false, 'Start server: mlxk server --port 8000');
|
|
}
|
|
}
|
|
|
|
// Load available models
|
|
async function loadModels() {
|
|
try {
|
|
const response = await fetch(`${API_BASE}/v1/models`);
|
|
if (response.ok) {
|
|
const data = await response.json();
|
|
modelSelect.innerHTML = '<option value="">Select a model...</option>';
|
|
|
|
// Store models data for context length lookup
|
|
window.modelsData = data.data;
|
|
|
|
data.data.forEach(model => {
|
|
const option = document.createElement('option');
|
|
option.value = model.id;
|
|
option.textContent = model.id.replace('mlx-community/', '');
|
|
modelSelect.appendChild(option);
|
|
});
|
|
|
|
// Restore or auto-select model
|
|
if (currentModel && data.data.some(m => m.id === currentModel)) {
|
|
// Restore previously selected model
|
|
modelSelect.value = currentModel;
|
|
updateStatus(true, getModelDisplayText(currentModel));
|
|
} else if (data.data.length > 0) {
|
|
// Auto-select first model if no valid stored model
|
|
currentModel = data.data[0].id;
|
|
modelSelect.value = currentModel;
|
|
localStorage.setItem('mlxk_selected_model', currentModel);
|
|
updateStatus(true, getModelDisplayText(currentModel));
|
|
}
|
|
}
|
|
} catch (error) {
|
|
console.error('Failed to load models:', error);
|
|
}
|
|
}
|
|
|
|
// Update status with model info including token capacity
|
|
function getModelDisplayText(modelId) {
|
|
if (!modelId) return 'Select a model';
|
|
|
|
const modelName = modelId.replace('mlx-community/', '');
|
|
|
|
if (!window.modelsData) {
|
|
// Fallback before models data is loaded
|
|
return `Ready with ${modelName}`;
|
|
}
|
|
|
|
const model = window.modelsData.find(m => m.id === modelId);
|
|
const contextLength = model?.context_length || 4096;
|
|
|
|
return `Ready with ${modelName} (${contextLength.toLocaleString()} tokens)`;
|
|
}
|
|
|
|
// Add message to chat
|
|
function addMessage(content, isUser = false) {
|
|
const messageDiv = document.createElement('div');
|
|
messageDiv.className = `message ${isUser ? 'user-message' : 'assistant-message'}`;
|
|
|
|
// Render markdown for assistant messages, plain text for user messages
|
|
const renderedContent = isUser ? content : marked.parse(content);
|
|
messageDiv.innerHTML = `<strong>${isUser ? 'You' : 'Assistant'}:</strong> ${renderedContent}`;
|
|
|
|
chatMessages.appendChild(messageDiv);
|
|
chatMessages.scrollTop = chatMessages.scrollHeight;
|
|
return messageDiv;
|
|
}
|
|
|
|
// Send chat message
|
|
async function sendMessage() {
|
|
const message = messageInput.value.trim();
|
|
if (!message || !currentModel) return;
|
|
|
|
// Add user message to history and display
|
|
conversationHistory.push({ role: 'user', content: message });
|
|
localStorage.setItem('mlxk_chat_history', JSON.stringify(conversationHistory));
|
|
addMessage(message, true);
|
|
messageInput.value = '';
|
|
isStreaming = true;
|
|
updateStatus(isConnected, `Generating response with ${currentModel.replace('mlx-community/', '')}...`);
|
|
|
|
// Add typing indicator
|
|
const typingDiv = addMessage('Typing...', false);
|
|
typingDiv.classList.add('typing');
|
|
|
|
try {
|
|
const response = await fetch(`${API_BASE}/v1/chat/completions`, {
|
|
method: 'POST',
|
|
headers: {
|
|
'Content-Type': 'application/json'
|
|
},
|
|
body: JSON.stringify({
|
|
model: currentModel,
|
|
messages: conversationHistory, // Send full conversation history
|
|
max_tokens: null, // Use 1.1.0-beta1 dynamic limits
|
|
temperature: 0.7,
|
|
stream: true
|
|
})
|
|
});
|
|
|
|
// Remove typing indicator
|
|
typingDiv.remove();
|
|
|
|
if (!response.ok) {
|
|
throw new Error(`HTTP ${response.status}`);
|
|
}
|
|
|
|
// Handle streaming response
|
|
const reader = response.body.getReader();
|
|
const decoder = new TextDecoder();
|
|
let assistantMessage = addMessage('', false);
|
|
let content = '';
|
|
|
|
while (true) {
|
|
const { done, value } = await reader.read();
|
|
if (done) break;
|
|
|
|
const chunk = decoder.decode(value);
|
|
const lines = chunk.split('\n');
|
|
|
|
for (const line of lines) {
|
|
if (line.startsWith('data: ')) {
|
|
const data = line.slice(6);
|
|
if (data === '[DONE]') continue;
|
|
|
|
try {
|
|
const parsed = JSON.parse(data);
|
|
const delta = parsed.choices?.[0]?.delta?.content;
|
|
if (delta) {
|
|
content += delta;
|
|
// Render markdown for streaming content
|
|
const renderedContent = marked.parse(content);
|
|
assistantMessage.innerHTML = `<strong>Assistant:</strong> ${renderedContent}`;
|
|
chatMessages.scrollTop = chatMessages.scrollHeight;
|
|
}
|
|
} catch (e) {
|
|
// Ignore parsing errors
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
// Add assistant response to conversation history
|
|
if (content.trim()) {
|
|
conversationHistory.push({ role: 'assistant', content: content });
|
|
localStorage.setItem('mlxk_chat_history', JSON.stringify(conversationHistory));
|
|
}
|
|
|
|
} catch (error) {
|
|
typingDiv.remove();
|
|
addMessage(`Error: ${error.message}`, false);
|
|
} finally {
|
|
isStreaming = false;
|
|
updateStatus(isConnected, currentModel ? getModelDisplayText(currentModel) : 'Select a model');
|
|
}
|
|
}
|
|
|
|
// Clear conversation
|
|
function clearChat() {
|
|
if (conversationHistory.length === 0) {
|
|
return; // Nothing to clear
|
|
}
|
|
|
|
if (isStreaming) {
|
|
return; // Don't allow clearing during streaming
|
|
}
|
|
|
|
showModal(
|
|
'Clear Chat',
|
|
'Are you sure you want to clear the entire chat history?',
|
|
'Clear All',
|
|
'Cancel',
|
|
() => {
|
|
conversationHistory = [];
|
|
localStorage.setItem('mlxk_chat_history', JSON.stringify(conversationHistory));
|
|
chatMessages.innerHTML = '<div class="message assistant-message"><strong>MLX Assistant:</strong> Hi! I\'m ready to chat. Select a model and send me a message!</div>';
|
|
updateStatus(isConnected, currentModel ? getModelDisplayText(currentModel) : 'Select a model');
|
|
},
|
|
() => {
|
|
// Cancel - do nothing
|
|
}
|
|
);
|
|
}
|
|
|
|
// Custom modal functions
|
|
function showModal(title, message, primaryText, secondaryText, onPrimary, onSecondary) {
|
|
const overlay = document.createElement('div');
|
|
overlay.className = 'modal-overlay';
|
|
|
|
overlay.innerHTML = `
|
|
<div class="modal">
|
|
<h3>${title}</h3>
|
|
<p>${message}</p>
|
|
<div class="modal-buttons">
|
|
<button class="modal-button primary">${primaryText}</button>
|
|
<button class="modal-button secondary">${secondaryText}</button>
|
|
</div>
|
|
</div>
|
|
`;
|
|
|
|
const buttons = overlay.querySelectorAll('.modal-button');
|
|
buttons[0].onclick = () => { document.body.removeChild(overlay); onPrimary(); };
|
|
buttons[1].onclick = () => { document.body.removeChild(overlay); onSecondary(); };
|
|
|
|
document.body.appendChild(overlay);
|
|
}
|
|
|
|
// Event listeners
|
|
modelSelect.addEventListener('change', (e) => {
|
|
currentModel = e.target.value;
|
|
localStorage.setItem('mlxk_selected_model', currentModel);
|
|
|
|
// Ask user if they want to keep chat history when switching models
|
|
if (conversationHistory.length > 0 && currentModel) {
|
|
showModal(
|
|
'Model Switch',
|
|
'Keep your chat history with the new model?',
|
|
'Keep History',
|
|
'Start Fresh',
|
|
() => {
|
|
// Keep history - do nothing
|
|
},
|
|
() => {
|
|
// Clear history
|
|
conversationHistory = [];
|
|
localStorage.setItem('mlxk_chat_history', JSON.stringify(conversationHistory));
|
|
chatMessages.innerHTML = '<div class="message assistant-message"><strong>MLX Assistant:</strong> Hi! I\'m ready to chat with the new model!</div>';
|
|
updateStatus(isConnected, currentModel ? getModelDisplayText(currentModel) : 'Select a model');
|
|
}
|
|
);
|
|
}
|
|
|
|
updateStatus(isConnected, currentModel ? getModelDisplayText(currentModel) : 'Select a model');
|
|
});
|
|
|
|
messageInput.addEventListener('keypress', (e) => {
|
|
if (e.key === 'Enter') {
|
|
sendMessage();
|
|
}
|
|
});
|
|
|
|
sendButton.addEventListener('click', sendMessage);
|
|
|
|
// Restore chat history on page load
|
|
function restoreChatHistory() {
|
|
if (conversationHistory.length > 0) {
|
|
chatMessages.innerHTML = '<div class="message assistant-message"><strong>MLX Assistant:</strong> Restored previous conversation.</div>';
|
|
conversationHistory.forEach(msg => {
|
|
addMessage(msg.content, msg.role === 'user');
|
|
});
|
|
}
|
|
// Update button states after restoring
|
|
updateStatus(isConnected, currentModel ? getModelDisplayText(currentModel) : 'Select a model');
|
|
}
|
|
|
|
// Initialize
|
|
checkHealth();
|
|
restoreChatHistory();
|
|
setInterval(checkHealth, 30000); // Health check every 30s
|
|
</script>
|
|
</body>
|
|
</html> |