mirror of
https://github.com/run-llama/LlamaIndexTS.git
synced 2026-07-16 07:14:29 -04:00
feat: add openai realtime api (#2006)
Co-authored-by: Marcus Schiesser <mail@marcusschiesser.de>
This commit is contained in:
ANKIT VARSHNEY
@@ -0,0 +1,7 @@
|
||||
---
|
||||
"@llamaindex/core": patch
|
||||
"@llamaindex/google": patch
|
||||
"@llamaindex/openai": patch
|
||||
---
|
||||
|
||||
Feat: add support for openai realtime API
|
||||
@@ -378,3 +378,186 @@ async function main() {
|
||||
## API Reference
|
||||
|
||||
- [OpenAI](/docs/api/classes/OpenAI)
|
||||
|
||||
|
||||
# OpenAI Live LLM
|
||||
|
||||
The OpenAI Live LLM integration in LlamaIndex provides real-time chat capabilities with support for audio streaming and tool calling.
|
||||
|
||||
## Basic Usage
|
||||
|
||||
```typescript
|
||||
import { openai } from "@llamaindex/openai";
|
||||
import { tool, ModalityType } from "llamaindex";
|
||||
|
||||
// Get the ephimeral key on the server
|
||||
const serverllm = openai({
|
||||
apiKey: "your-api-key",
|
||||
model: "gpt-4o-realtime-preview-2025-06-03",
|
||||
});
|
||||
|
||||
// Get an ephemeral key
|
||||
// Usually this code is run on the server and the ephemeral key is passed to the
|
||||
// client - the ephemeral key can be securely used on the client side
|
||||
const ephemeralKey = await serverllm.live.getEphemeralKey();
|
||||
|
||||
// Create a client-side LLM instance with the ephemeral key
|
||||
const llm = openai({
|
||||
apiKey: ephemeralKey,
|
||||
model: "gpt-4o-realtime-preview-2025-06-03"
|
||||
});
|
||||
|
||||
// Create a live sessionimport { tool } from "llamaindex";
|
||||
const session = await llm.live.connect({
|
||||
systemInstruction: "You are a helpful assistant.",
|
||||
});
|
||||
|
||||
// Send a message
|
||||
session.sendMessage({
|
||||
content: "Hello!",
|
||||
role: "user",
|
||||
});
|
||||
```
|
||||
|
||||
## Tool Integration
|
||||
|
||||
Tools are handled server-side, making it simple to pass them to the live session:
|
||||
|
||||
```typescript
|
||||
// Define your tools
|
||||
const weatherTool = tool({
|
||||
name: "weather",
|
||||
description: "Get the weather for a location",
|
||||
parameters: z.object({
|
||||
location: z.string().describe("The location to get weather for"),
|
||||
}),
|
||||
execute: async ({ location }) => {
|
||||
return `The weather in ${location} is sunny`;
|
||||
},
|
||||
});
|
||||
|
||||
// Create session with tools
|
||||
const session = await llm.live.connect({
|
||||
systemInstruction: "You are a helpful assistant.",
|
||||
tools: [weatherTool],
|
||||
});
|
||||
```
|
||||
|
||||
## Audio Support
|
||||
|
||||
For audio capabilities:
|
||||
|
||||
```typescript
|
||||
// Get microphone access
|
||||
const userStream = await navigator.mediaDevices.getUserMedia({
|
||||
audio: true,
|
||||
});
|
||||
|
||||
// Create session with audio
|
||||
const session = await llm.live.connect({
|
||||
audioConfig: {
|
||||
stream: userStream,
|
||||
onTrack: (remoteStream) => {
|
||||
// Handle incoming audio
|
||||
audioElement.srcObject = remoteStream;
|
||||
},
|
||||
},
|
||||
});
|
||||
```
|
||||
|
||||
## Event Handling
|
||||
|
||||
Listen to events from the session:
|
||||
|
||||
```typescript
|
||||
for await (const event of session.streamEvents()) {
|
||||
if (liveEvents.open.include(event)) {
|
||||
// Connection established
|
||||
console.log("Connected!");
|
||||
} else if (liveEvents.text.include(event)) {
|
||||
// Received text response
|
||||
console.log("Assistant:", event.text);
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
## Capabilities
|
||||
|
||||
The OpenAI Live LLM supports:
|
||||
|
||||
- Real-time text chat
|
||||
- Audio streaming (if configured)
|
||||
- Tool calling (server-side execution)
|
||||
- Ephemeral key generation for secure sessions
|
||||
|
||||
## API Reference
|
||||
|
||||
### LiveLLM Methods
|
||||
// Get an ephemeral key
|
||||
// Usually this code is run on the server and the ephemeral key is passed to the
|
||||
// client - the ephemeral key can be securely used on the client side
|
||||
|
||||
#### `connect(config?: LiveConnectConfig)`
|
||||
|
||||
Creates a new live session.
|
||||
|
||||
```typescript
|
||||
interface LiveConnectConfig {
|
||||
systemInstruction?: string;
|
||||
tools?: BaseTool[];
|
||||
audioConfig?: AudioConfig;
|
||||
responseModality?: ModalityType[];
|
||||
}
|
||||
```
|
||||
|
||||
#### `getEphemeralKey()`
|
||||
|
||||
Gets a temporary key for the session.
|
||||
|
||||
### LiveLLMSession Methods
|
||||
|
||||
#### `sendMessage(message: ChatMessage)`
|
||||
|
||||
Sends a message to the assistant.
|
||||
|
||||
```typescript
|
||||
interface ChatMessage {
|
||||
content: string | MessageContentDetail[];
|
||||
role: "user" | "assistant";
|
||||
}
|
||||
```
|
||||
|
||||
#### `disconnect()`
|
||||
|
||||
Closes the session and cleans up resources.
|
||||
|
||||
## Error Handling
|
||||
|
||||
```typescript
|
||||
try {
|
||||
const session = await llm.live.connect();
|
||||
} catch (error) {
|
||||
if (error instanceof Error) {
|
||||
console.error("Connection failed:", error.message);
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
## Best Practices
|
||||
|
||||
1. **Tool Definition**
|
||||
|
||||
- Keep tool implementations server-side
|
||||
- Use clear descriptions for tools
|
||||
- Handle tool errors gracefully
|
||||
|
||||
2. **Session Management**
|
||||
|
||||
- Always disconnect sessions when done
|
||||
- Clean up audio resources
|
||||
- Handle reconnection scenarios
|
||||
|
||||
3. **Security**
|
||||
- Use ephemeral keys for sessions
|
||||
- Validate tool inputs
|
||||
- Secure API key handling
|
||||
|
||||
@@ -1 +1,9 @@
|
||||
{"root":["./src/main.ts","./vite.config.ts"],"version":"5.7.3"}
|
||||
{
|
||||
"root": [
|
||||
"./src/main.ts",
|
||||
"./vite.config.ts",
|
||||
"./tsconfig.json"
|
||||
],
|
||||
"errors": true,
|
||||
"version": "5.7.3"
|
||||
}
|
||||
|
||||
@@ -23,10 +23,7 @@ async function main() {
|
||||
content: "Say something about you for 10 seconds",
|
||||
role: "user",
|
||||
});
|
||||
} else if (
|
||||
liveEvents.audio.include(event) &&
|
||||
typeof event.data === "string"
|
||||
) {
|
||||
} else if (liveEvents.audio.include(event)) {
|
||||
const chunk = Buffer.from(event.data, "base64");
|
||||
audioChunks.push(chunk);
|
||||
console.log(`Received audio chunk: ${chunk.length} bytes`);
|
||||
|
||||
@@ -0,0 +1,24 @@
|
||||
# Logs
|
||||
logs
|
||||
*.log
|
||||
npm-debug.log*
|
||||
yarn-debug.log*
|
||||
yarn-error.log*
|
||||
pnpm-debug.log*
|
||||
lerna-debug.log*
|
||||
|
||||
node_modules
|
||||
dist
|
||||
dist-ssr
|
||||
*.local
|
||||
|
||||
# Editor directories and files
|
||||
.vscode/*
|
||||
!.vscode/extensions.json
|
||||
.idea
|
||||
.DS_Store
|
||||
*.suo
|
||||
*.ntvs*
|
||||
*.njsproj
|
||||
*.sln
|
||||
*.sw?
|
||||
@@ -0,0 +1,54 @@
|
||||
# OpenAI Realtime Chat with LlamaIndex
|
||||
|
||||
This is a demo application showcasing real-time audio and text chat capabilities using OpenAI's GPT-4 with voice through LlamaIndex. The application demonstrates bidirectional audio communication and text chat with an AI assistant.
|
||||
|
||||
## Features
|
||||
|
||||
- Real-time voice communication with GPT-4
|
||||
- Text-based chat interface
|
||||
- WebRTC-based audio streaming
|
||||
- Bidirectional communication (both text and voice)
|
||||
- React + TypeScript implementation
|
||||
|
||||
## Prerequisites
|
||||
|
||||
- Node.js (v18 or higher)
|
||||
- OpenAI API key with access to GPT-4 voice models
|
||||
- Modern browser with WebRTC support
|
||||
|
||||
## Getting Started
|
||||
|
||||
1. Install dependencies:
|
||||
|
||||
```bash
|
||||
pnpm install
|
||||
```
|
||||
|
||||
2. Start the development server:
|
||||
|
||||
```bash
|
||||
pnpm run dev
|
||||
```
|
||||
|
||||
## Usage
|
||||
|
||||
The application provides a simple interface where you can:
|
||||
|
||||
- Start/Stop a chat session
|
||||
- Speak to the AI assistant through your microphone
|
||||
- Receive audio responses from the assistant
|
||||
- See text transcripts of the conversation
|
||||
|
||||
## Technical Details
|
||||
|
||||
This project uses:
|
||||
|
||||
- LlamaIndex for AI interaction management
|
||||
- WebRTC for real-time audio streaming
|
||||
- React for the UI
|
||||
- Vite for development and building
|
||||
- TypeScript for type safety
|
||||
|
||||
```
|
||||
|
||||
```
|
||||
@@ -0,0 +1,28 @@
|
||||
import js from "@eslint/js";
|
||||
import reactHooks from "eslint-plugin-react-hooks";
|
||||
import reactRefresh from "eslint-plugin-react-refresh";
|
||||
import globals from "globals";
|
||||
import tseslint from "typescript-eslint";
|
||||
|
||||
export default tseslint.config(
|
||||
{ ignores: ["dist"] },
|
||||
{
|
||||
extends: [js.configs.recommended, ...tseslint.configs.recommended],
|
||||
files: ["**/*.{ts,tsx}"],
|
||||
languageOptions: {
|
||||
ecmaVersion: 2020,
|
||||
globals: globals.browser,
|
||||
},
|
||||
plugins: {
|
||||
"react-hooks": reactHooks,
|
||||
"react-refresh": reactRefresh,
|
||||
},
|
||||
rules: {
|
||||
...reactHooks.configs.recommended.rules,
|
||||
"react-refresh/only-export-components": [
|
||||
"warn",
|
||||
{ allowConstantExport: true },
|
||||
],
|
||||
},
|
||||
},
|
||||
);
|
||||
@@ -0,0 +1,13 @@
|
||||
<!doctype html>
|
||||
<html lang="en">
|
||||
<head>
|
||||
<meta charset="UTF-8" />
|
||||
<link rel="icon" type="image/svg+xml" href="/vite.svg" />
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
|
||||
<title>Vite + React + TS</title>
|
||||
</head>
|
||||
<body>
|
||||
<div id="root"></div>
|
||||
<script type="module" src="/src/main.tsx"></script>
|
||||
</body>
|
||||
</html>
|
||||
@@ -0,0 +1,29 @@
|
||||
{
|
||||
"name": "open-ai-realtime",
|
||||
"private": true,
|
||||
"version": "0.0.0",
|
||||
"type": "module",
|
||||
"scripts": {
|
||||
"dev": "vite",
|
||||
"build": "tsc -b && vite build",
|
||||
"lint": "eslint .",
|
||||
"preview": "vite preview"
|
||||
},
|
||||
"dependencies": {
|
||||
"react": "^19.1.0",
|
||||
"react-dom": "^19.1.0"
|
||||
},
|
||||
"devDependencies": {
|
||||
"@eslint/js": "^9.25.0",
|
||||
"@types/react": "^19.1.2",
|
||||
"@types/react-dom": "^19.1.2",
|
||||
"@vitejs/plugin-react": "^4.5.2",
|
||||
"eslint": "^9.25.0",
|
||||
"eslint-plugin-react-hooks": "^5.2.0",
|
||||
"eslint-plugin-react-refresh": "^0.4.19",
|
||||
"globals": "^16.0.0",
|
||||
"typescript": "~5.8.3",
|
||||
"typescript-eslint": "^8.30.1",
|
||||
"vite": "^6.3.5"
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1 @@
|
||||
<svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" class="iconify iconify--logos" width="31.88" height="32" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 257"><defs><linearGradient id="IconifyId1813088fe1fbc01fb466" x1="-.828%" x2="57.636%" y1="7.652%" y2="78.411%"><stop offset="0%" stop-color="#41D1FF"></stop><stop offset="100%" stop-color="#BD34FE"></stop></linearGradient><linearGradient id="IconifyId1813088fe1fbc01fb467" x1="43.376%" x2="50.316%" y1="2.242%" y2="89.03%"><stop offset="0%" stop-color="#FFEA83"></stop><stop offset="8.333%" stop-color="#FFDD35"></stop><stop offset="100%" stop-color="#FFA800"></stop></linearGradient></defs><path fill="url(#IconifyId1813088fe1fbc01fb466)" d="M255.153 37.938L134.897 252.976c-2.483 4.44-8.862 4.466-11.382.048L.875 37.958c-2.746-4.814 1.371-10.646 6.827-9.67l120.385 21.517a6.537 6.537 0 0 0 2.322-.004l117.867-21.483c5.438-.991 9.574 4.796 6.877 9.62Z"></path><path fill="url(#IconifyId1813088fe1fbc01fb467)" d="M185.432.063L96.44 17.501a3.268 3.268 0 0 0-2.634 3.014l-5.474 92.456a3.268 3.268 0 0 0 3.997 3.378l24.777-5.718c2.318-.535 4.413 1.507 3.936 3.838l-7.361 36.047c-.495 2.426 1.782 4.5 4.151 3.78l15.304-4.649c2.372-.72 4.652 1.36 4.15 3.788l-11.698 56.621c-.732 3.542 3.979 5.473 5.943 2.437l1.313-2.028l72.516-144.72c1.215-2.423-.88-5.186-3.54-4.672l-25.505 4.922c-2.396.462-4.435-1.77-3.759-4.114l16.646-57.705c.677-2.35-1.37-4.583-3.769-4.113Z"></path></svg>
|
||||
|
After Width: | Height: | Size: 1.5 KiB |
@@ -0,0 +1,183 @@
|
||||
import { openai } from "@llamaindex/openai";
|
||||
import { liveEvents, LiveLLMSession, ModalityType } from "llamaindex";
|
||||
import { useEffect, useRef, useState } from "react";
|
||||
|
||||
const MicIcon = ({ isConnected }: { isConnected: boolean }) => (
|
||||
<svg
|
||||
xmlns="http://www.w3.org/2000/svg"
|
||||
viewBox="0 0 24 24"
|
||||
fill="currentColor"
|
||||
strokeWidth="2"
|
||||
strokeLinecap="round"
|
||||
strokeLinejoin="round"
|
||||
>
|
||||
{isConnected ? (
|
||||
<>
|
||||
<path d="M12 1a3 3 0 0 0-3 3v8a3 3 0 0 0 6 0V4a3 3 0 0 0-3-3z" />
|
||||
<path d="M19 10v2a7 7 0 0 1-14 0v-2" />
|
||||
<line x1="12" y1="19" x2="12" y2="23" />
|
||||
<line x1="8" y1="23" x2="16" y2="23" />
|
||||
</>
|
||||
) : (
|
||||
<>
|
||||
<path d="M12 1a3 3 0 0 0-3 3v8a3 3 0 0 0 6 0V4a3 3 0 0 0-3-3z" />
|
||||
<path d="M19 10v2a7 7 0 0 1-14 0v-2" />
|
||||
</>
|
||||
)}
|
||||
</svg>
|
||||
);
|
||||
|
||||
const WaveAnimation = () => (
|
||||
<div className="wave-animation">
|
||||
{[...Array(3)].map((_, i) => (
|
||||
<div key={i} className="wave" style={{ animationDelay: `${i * 0.2}s` }} />
|
||||
))}
|
||||
</div>
|
||||
);
|
||||
|
||||
export const AudioChat = () => {
|
||||
const [isConnected, setIsConnected] = useState(false);
|
||||
const [messages, setMessages] = useState<
|
||||
Array<{ role: string; content: string }>
|
||||
>([]);
|
||||
const [status, setStatus] = useState<string>("");
|
||||
const audioRef = useRef<HTMLAudioElement>(null);
|
||||
const sessionRef = useRef<LiveLLMSession | null>(null);
|
||||
const [stream, setStream] = useState<MediaStream | null>(null);
|
||||
const messagesEndRef = useRef<HTMLDivElement>(null);
|
||||
|
||||
const scrollToBottom = () => {
|
||||
messagesEndRef.current?.scrollIntoView({ behavior: "smooth" });
|
||||
};
|
||||
|
||||
useEffect(() => {
|
||||
scrollToBottom();
|
||||
}, [messages]);
|
||||
|
||||
useEffect(() => {
|
||||
return () => {
|
||||
if (stream) {
|
||||
stream.getTracks().forEach((track) => track.stop());
|
||||
}
|
||||
};
|
||||
}, [stream]);
|
||||
|
||||
const startChat = async () => {
|
||||
try {
|
||||
setStatus("Initializing microphone...");
|
||||
const userStream = await navigator.mediaDevices.getUserMedia({
|
||||
audio: true,
|
||||
});
|
||||
setStream(userStream);
|
||||
|
||||
setStatus("Connecting to AI...");
|
||||
const apiKey = prompt("Please enter your OpenAI API key:");
|
||||
if (!apiKey) {
|
||||
throw new Error("API key is required");
|
||||
}
|
||||
|
||||
// move this call to the server side for security reasons
|
||||
// Do not store the API key in the frontend!
|
||||
const serverllm = openai({
|
||||
apiKey: apiKey,
|
||||
model: "gpt-4o-realtime-preview-2025-06-03",
|
||||
});
|
||||
|
||||
const tempKey = await serverllm.live.getEphemeralKey();
|
||||
|
||||
const llm = openai({
|
||||
apiKey: tempKey,
|
||||
model: "gpt-4o-realtime-preview-2025-06-03",
|
||||
});
|
||||
const session = await llm.live.connect({
|
||||
systemInstruction: "You are a helpful assistant who speaks naturally.",
|
||||
responseModality: [ModalityType.TEXT, ModalityType.AUDIO],
|
||||
audioConfig: {
|
||||
stream: userStream,
|
||||
onTrack: (remoteStream) => {
|
||||
if (audioRef.current && remoteStream) {
|
||||
audioRef.current.srcObject = remoteStream;
|
||||
audioRef.current.play().catch(console.error);
|
||||
}
|
||||
},
|
||||
},
|
||||
});
|
||||
|
||||
sessionRef.current = session;
|
||||
setIsConnected(true);
|
||||
setStatus("Connected! Listening...");
|
||||
|
||||
for await (const event of session.streamEvents()) {
|
||||
if (liveEvents.open.include(event)) {
|
||||
setMessages((prev) => [
|
||||
...prev,
|
||||
{
|
||||
role: "user",
|
||||
content: "Hello, I'm ready to chat!",
|
||||
},
|
||||
]);
|
||||
session.sendMessage({
|
||||
content: "Hello, I'm ready to chat!",
|
||||
role: "user",
|
||||
});
|
||||
} else if (liveEvents.text.include(event)) {
|
||||
setMessages((prev) => [
|
||||
...prev,
|
||||
{
|
||||
role: "assistant",
|
||||
content: event.text,
|
||||
},
|
||||
]);
|
||||
}
|
||||
}
|
||||
} catch (error) {
|
||||
console.error("Error starting chat:", error);
|
||||
setStatus("Error connecting. Please try again.");
|
||||
setIsConnected(false);
|
||||
}
|
||||
};
|
||||
|
||||
const stopChat = async () => {
|
||||
setStatus("Disconnecting...");
|
||||
if (sessionRef.current) {
|
||||
await sessionRef.current.disconnect();
|
||||
sessionRef.current = null;
|
||||
}
|
||||
if (stream) {
|
||||
stream.getTracks().forEach((track) => track.stop());
|
||||
setStream(null);
|
||||
}
|
||||
if (audioRef.current) {
|
||||
audioRef.current.srcObject = null;
|
||||
}
|
||||
setIsConnected(false);
|
||||
setStatus("");
|
||||
};
|
||||
|
||||
return (
|
||||
<div className="audio-chat-container">
|
||||
<h1>AI Voice Chat</h1>
|
||||
<div className="messages-container">
|
||||
{messages.map((msg, idx) => (
|
||||
<div key={idx} className={`message ${msg.role}`}>
|
||||
{msg.content}
|
||||
</div>
|
||||
))}
|
||||
<div ref={messagesEndRef} />
|
||||
</div>
|
||||
|
||||
<div className="controls">
|
||||
{status && <div className="status-indicator">{status}</div>}
|
||||
<button
|
||||
className={`mic-button ${isConnected ? "connected" : ""}`}
|
||||
onClick={isConnected ? stopChat : startChat}
|
||||
title={isConnected ? "Stop Chat" : "Start Chat"}
|
||||
>
|
||||
<MicIcon isConnected={isConnected} />
|
||||
{isConnected && <WaveAnimation />}
|
||||
</button>
|
||||
<audio ref={audioRef} style={{ display: "none" }} />
|
||||
</div>
|
||||
</div>
|
||||
);
|
||||
};
|
||||
@@ -0,0 +1,322 @@
|
||||
:root {
|
||||
--primary-color: #646cff;
|
||||
--secondary-color: #535bf2;
|
||||
--background-dark: #1a1a1a;
|
||||
--chat-bg: #242424;
|
||||
--text-primary: #ffffff;
|
||||
--text-secondary: #888888;
|
||||
--success-color: #4caf50;
|
||||
--error-color: #f44336;
|
||||
--gradient-start: #4776e6;
|
||||
--gradient-end: #8e54e9;
|
||||
}
|
||||
|
||||
body {
|
||||
background-color: var(--background-dark);
|
||||
color: var(--text-primary);
|
||||
margin: 0;
|
||||
min-height: 100vh;
|
||||
font-family:
|
||||
-apple-system, BlinkMacSystemFont, "Segoe UI", Roboto, Oxygen, Ubuntu,
|
||||
Cantarell, "Open Sans", "Helvetica Neue", sans-serif;
|
||||
}
|
||||
|
||||
#root {
|
||||
max-width: 1280px;
|
||||
height: 100vh;
|
||||
margin: 0 auto;
|
||||
padding: 2rem;
|
||||
display: flex;
|
||||
align-items: center;
|
||||
justify-content: center;
|
||||
}
|
||||
|
||||
.logo {
|
||||
height: 6em;
|
||||
padding: 1.5em;
|
||||
will-change: filter;
|
||||
transition: filter 300ms;
|
||||
}
|
||||
.logo:hover {
|
||||
filter: drop-shadow(0 0 2em #646cffaa);
|
||||
}
|
||||
.logo.react:hover {
|
||||
filter: drop-shadow(0 0 2em #61dafbaa);
|
||||
}
|
||||
|
||||
@keyframes logo-spin {
|
||||
from {
|
||||
transform: rotate(0deg);
|
||||
}
|
||||
to {
|
||||
transform: rotate(360deg);
|
||||
}
|
||||
}
|
||||
|
||||
@media (prefers-reduced-motion: no-preference) {
|
||||
a:nth-of-type(2) .logo {
|
||||
animation: logo-spin infinite 20s linear;
|
||||
}
|
||||
}
|
||||
|
||||
.card {
|
||||
padding: 2em;
|
||||
}
|
||||
|
||||
.read-the-docs {
|
||||
color: #888;
|
||||
}
|
||||
|
||||
.audio-chat-container {
|
||||
display: flex;
|
||||
flex-direction: column;
|
||||
gap: 2rem;
|
||||
width: 100%;
|
||||
max-width: 800px;
|
||||
height: 80vh;
|
||||
margin: 0 auto;
|
||||
padding: 2rem;
|
||||
background: var(--chat-bg);
|
||||
border-radius: 24px;
|
||||
box-shadow: 0 8px 32px rgba(0, 0, 0, 0.2);
|
||||
position: relative;
|
||||
overflow: hidden;
|
||||
}
|
||||
|
||||
.audio-chat-container::before {
|
||||
content: "";
|
||||
position: absolute;
|
||||
top: 0;
|
||||
left: 0;
|
||||
right: 0;
|
||||
height: 4px;
|
||||
background: linear-gradient(
|
||||
to right,
|
||||
var(--gradient-start),
|
||||
var(--gradient-end)
|
||||
);
|
||||
}
|
||||
|
||||
.audio-chat-container h1 {
|
||||
font-size: 2.5rem;
|
||||
margin: 0;
|
||||
background: linear-gradient(
|
||||
to right,
|
||||
var(--gradient-start),
|
||||
var(--gradient-end)
|
||||
);
|
||||
-webkit-background-clip: text;
|
||||
background-clip: text;
|
||||
color: transparent;
|
||||
text-align: center;
|
||||
}
|
||||
|
||||
.messages-container {
|
||||
display: flex;
|
||||
flex-direction: column;
|
||||
gap: 1rem;
|
||||
flex: 1;
|
||||
overflow-y: auto;
|
||||
padding: 1rem;
|
||||
border-radius: 16px;
|
||||
background: rgba(255, 255, 255, 0.05);
|
||||
backdrop-filter: blur(10px);
|
||||
margin: 1rem 0;
|
||||
}
|
||||
|
||||
.message {
|
||||
padding: 1rem 1.5rem;
|
||||
border-radius: 16px;
|
||||
max-width: 80%;
|
||||
text-align: left;
|
||||
animation: messageSlide 0.3s ease-out;
|
||||
box-shadow: 0 2px 4px rgba(0, 0, 0, 0.1);
|
||||
}
|
||||
|
||||
@keyframes messageSlide {
|
||||
from {
|
||||
opacity: 0;
|
||||
transform: translateY(20px);
|
||||
}
|
||||
to {
|
||||
opacity: 1;
|
||||
transform: translateY(0);
|
||||
}
|
||||
}
|
||||
|
||||
.message.user {
|
||||
background: linear-gradient(
|
||||
135deg,
|
||||
var(--gradient-start),
|
||||
var(--gradient-end)
|
||||
);
|
||||
align-self: flex-end;
|
||||
margin-left: 20%;
|
||||
color: white;
|
||||
}
|
||||
|
||||
.message.assistant {
|
||||
background: rgba(255, 255, 255, 0.1);
|
||||
align-self: flex-start;
|
||||
margin-right: 20%;
|
||||
border: 1px solid rgba(255, 255, 255, 0.1);
|
||||
}
|
||||
|
||||
.controls {
|
||||
display: flex;
|
||||
justify-content: center;
|
||||
align-items: center;
|
||||
padding: 2rem;
|
||||
position: relative;
|
||||
}
|
||||
|
||||
.mic-button {
|
||||
width: 80px;
|
||||
height: 80px;
|
||||
border-radius: 50%;
|
||||
border: none;
|
||||
background: linear-gradient(
|
||||
135deg,
|
||||
var(--gradient-start),
|
||||
var(--gradient-end)
|
||||
);
|
||||
color: white;
|
||||
cursor: pointer;
|
||||
transition: all 0.3s cubic-bezier(0.4, 0, 0.2, 1);
|
||||
display: flex;
|
||||
align-items: center;
|
||||
justify-content: center;
|
||||
box-shadow: 0 4px 15px rgba(0, 0, 0, 0.2);
|
||||
position: relative;
|
||||
overflow: hidden;
|
||||
}
|
||||
|
||||
.mic-button::before {
|
||||
content: "";
|
||||
position: absolute;
|
||||
top: 0;
|
||||
left: 0;
|
||||
right: 0;
|
||||
bottom: 0;
|
||||
background: linear-gradient(
|
||||
135deg,
|
||||
rgba(255, 255, 255, 0.1),
|
||||
rgba(255, 255, 255, 0)
|
||||
);
|
||||
border-radius: 50%;
|
||||
transition: transform 0.3s ease;
|
||||
}
|
||||
|
||||
.mic-button:hover {
|
||||
transform: scale(1.05);
|
||||
box-shadow: 0 8px 25px rgba(0, 0, 0, 0.3);
|
||||
}
|
||||
|
||||
.mic-button:hover::before {
|
||||
transform: translateY(-100%);
|
||||
}
|
||||
|
||||
.mic-button.connected {
|
||||
background: var(--error-color);
|
||||
animation: pulseError 2s infinite;
|
||||
}
|
||||
|
||||
.mic-button svg {
|
||||
width: 32px;
|
||||
height: 32px;
|
||||
filter: drop-shadow(0 2px 4px rgba(0, 0, 0, 0.2));
|
||||
transition: transform 0.3s ease;
|
||||
}
|
||||
|
||||
.mic-button:hover svg {
|
||||
transform: scale(1.1);
|
||||
}
|
||||
|
||||
@keyframes pulseError {
|
||||
0% {
|
||||
box-shadow: 0 0 0 0 rgba(244, 67, 54, 0.4);
|
||||
}
|
||||
70% {
|
||||
box-shadow: 0 0 0 20px rgba(244, 67, 54, 0);
|
||||
}
|
||||
100% {
|
||||
box-shadow: 0 0 0 0 rgba(244, 67, 54, 0);
|
||||
}
|
||||
}
|
||||
|
||||
/* Status indicator */
|
||||
.status-indicator {
|
||||
position: absolute;
|
||||
top: -30px;
|
||||
left: 50%;
|
||||
transform: translateX(-50%);
|
||||
font-size: 0.9rem;
|
||||
color: var(--text-secondary);
|
||||
opacity: 0;
|
||||
transition: opacity 0.3s ease;
|
||||
}
|
||||
|
||||
.controls:hover .status-indicator {
|
||||
opacity: 1;
|
||||
}
|
||||
|
||||
/* Scrollbar styling */
|
||||
.messages-container::-webkit-scrollbar {
|
||||
width: 8px;
|
||||
}
|
||||
|
||||
.messages-container::-webkit-scrollbar-track {
|
||||
background: rgba(255, 255, 255, 0.05);
|
||||
border-radius: 4px;
|
||||
}
|
||||
|
||||
.messages-container::-webkit-scrollbar-thumb {
|
||||
background: linear-gradient(var(--gradient-start), var(--gradient-end));
|
||||
border-radius: 4px;
|
||||
}
|
||||
|
||||
.messages-container::-webkit-scrollbar-thumb:hover {
|
||||
background: linear-gradient(var(--gradient-end), var(--gradient-start));
|
||||
}
|
||||
|
||||
/* Wave Animation */
|
||||
.wave-animation {
|
||||
position: absolute;
|
||||
bottom: -15px;
|
||||
left: 50%;
|
||||
transform: translateX(-50%);
|
||||
display: flex;
|
||||
gap: 4px;
|
||||
}
|
||||
|
||||
.wave {
|
||||
width: 4px;
|
||||
height: 15px;
|
||||
background: currentColor;
|
||||
border-radius: 2px;
|
||||
animation: wave 0.5s ease-in-out infinite;
|
||||
}
|
||||
|
||||
@keyframes wave {
|
||||
0%,
|
||||
100% {
|
||||
transform: scaleY(0.5);
|
||||
}
|
||||
50% {
|
||||
transform: scaleY(1.5);
|
||||
}
|
||||
}
|
||||
|
||||
/* Loading state */
|
||||
.mic-button.loading {
|
||||
animation: rotate 1s linear infinite;
|
||||
}
|
||||
|
||||
@keyframes rotate {
|
||||
from {
|
||||
transform: rotate(0deg);
|
||||
}
|
||||
to {
|
||||
transform: rotate(360deg);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,10 @@
|
||||
import { StrictMode } from "react";
|
||||
import { createRoot } from "react-dom/client";
|
||||
import { AudioChat } from "./audio-chat.tsx";
|
||||
import "./index.css";
|
||||
|
||||
createRoot(document.getElementById("root")!).render(
|
||||
<StrictMode>
|
||||
<AudioChat />
|
||||
</StrictMode>,
|
||||
);
|
||||
@@ -0,0 +1 @@
|
||||
/// <reference types="vite/client" />
|
||||
@@ -0,0 +1,26 @@
|
||||
{
|
||||
"compilerOptions": {
|
||||
"tsBuildInfoFile": "./node_modules/.tmp/tsconfig.app.tsbuildinfo",
|
||||
"target": "ES2020",
|
||||
"useDefineForClassFields": true,
|
||||
"lib": ["ES2020", "DOM", "DOM.Iterable"],
|
||||
"module": "ESNext",
|
||||
"skipLibCheck": true,
|
||||
|
||||
/* Bundler mode */
|
||||
"moduleResolution": "bundler",
|
||||
"allowImportingTsExtensions": true,
|
||||
"verbatimModuleSyntax": true,
|
||||
"moduleDetection": "force",
|
||||
"noEmit": true,
|
||||
"jsx": "react-jsx",
|
||||
|
||||
/* Linting */
|
||||
"strict": true,
|
||||
"noUnusedLocals": true,
|
||||
"noUnusedParameters": true,
|
||||
"noFallthroughCasesInSwitch": true,
|
||||
"noUncheckedSideEffectImports": true
|
||||
},
|
||||
"include": ["src"]
|
||||
}
|
||||
@@ -0,0 +1,4 @@
|
||||
{
|
||||
"files": [],
|
||||
"references": [{ "path": "./tsconfig.app.json" }]
|
||||
}
|
||||
@@ -0,0 +1,7 @@
|
||||
import react from "@vitejs/plugin-react";
|
||||
import { defineConfig } from "vite";
|
||||
|
||||
// https://vite.dev/config/
|
||||
export default defineConfig({
|
||||
plugins: [react()],
|
||||
});
|
||||
@@ -18,5 +18,11 @@
|
||||
"module": "commonjs"
|
||||
}
|
||||
},
|
||||
"include": ["./**/*.ts"]
|
||||
"include": ["./**/*.ts"],
|
||||
"exclude": [
|
||||
"node_modules",
|
||||
"dist",
|
||||
"models/openai/live/browser/open-ai-realtime",
|
||||
"**/browser/**"
|
||||
]
|
||||
}
|
||||
|
||||
@@ -1,6 +1,9 @@
|
||||
export { BaseLLM, ToolCallLLM } from "./base";
|
||||
export { LiveLLM, LiveLLMSession, liveEvents, type LiveEvent } from "./live";
|
||||
export { LiveLLM, LiveLLMCapability, LiveLLMSession } from "./live/live";
|
||||
export { liveEvents, type LiveEvent } from "./live/live-types";
|
||||
export type { MessageSender } from "./live/sender";
|
||||
export type {
|
||||
AudioConfig,
|
||||
BaseTool,
|
||||
BaseToolWithCall,
|
||||
ChatMessage,
|
||||
|
||||
@@ -1,9 +1,7 @@
|
||||
import type {
|
||||
ChatMessage,
|
||||
LiveConnectConfig,
|
||||
MessageContentAudioDetail,
|
||||
MessageContentTextDetail,
|
||||
} from "./type";
|
||||
} from "../type";
|
||||
|
||||
export type OpenEvent = { type: "open" };
|
||||
|
||||
@@ -63,45 +61,3 @@ export const liveEvents = {
|
||||
e.type === "turnComplete",
|
||||
},
|
||||
};
|
||||
export abstract class LiveLLMSession {
|
||||
protected eventQueue: LiveEvent[] = [];
|
||||
protected eventResolvers: ((value: LiveEvent) => void)[] = [];
|
||||
protected closed = false;
|
||||
abstract sendMessage(message: ChatMessage): void;
|
||||
async *streamEvents(): AsyncIterable<LiveEvent> {
|
||||
while (true) {
|
||||
const event = await this.nextEvent();
|
||||
if (event === undefined) {
|
||||
break;
|
||||
}
|
||||
yield event;
|
||||
}
|
||||
}
|
||||
abstract disconnect(): Promise<void>;
|
||||
|
||||
protected async nextEvent(): Promise<LiveEvent | undefined> {
|
||||
if (this.eventQueue.length) {
|
||||
return Promise.resolve(this.eventQueue.shift());
|
||||
}
|
||||
|
||||
return new Promise((resolve) => {
|
||||
this.eventResolvers.push(resolve);
|
||||
});
|
||||
}
|
||||
|
||||
//Uses an async queue to send events to the client
|
||||
// if the consumer is waiting for an event, it will be resolved immediately
|
||||
// otherwise, the event will be queued up and sent when the consumer is ready
|
||||
pushEventToQueue(event: LiveEvent) {
|
||||
if (this.eventResolvers.length) {
|
||||
//resolving the promise with the event
|
||||
this.eventResolvers.shift()!(event);
|
||||
} else {
|
||||
this.eventQueue.push(event);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
export abstract class LiveLLM {
|
||||
abstract connect(config?: LiveConnectConfig): Promise<LiveLLMSession>;
|
||||
}
|
||||
@@ -0,0 +1,124 @@
|
||||
import type {
|
||||
ChatMessage,
|
||||
LiveConnectConfig,
|
||||
MessageContentAudioDetail,
|
||||
MessageContentDetail,
|
||||
MessageContentImageDataDetail,
|
||||
MessageContentVideoDetail,
|
||||
} from "../type";
|
||||
import type { LiveEvent } from "./live-types";
|
||||
import type { MessageSender } from "./sender";
|
||||
|
||||
export enum LiveLLMCapability {
|
||||
EPHEMERAL_KEY = "ephemeral_key",
|
||||
AUDIO_CONFIG = "audio_config",
|
||||
}
|
||||
|
||||
export abstract class LiveLLMSession {
|
||||
protected eventQueue: LiveEvent[] = [];
|
||||
protected eventResolvers: ((value: LiveEvent) => void)[] = [];
|
||||
closed = false;
|
||||
abstract get messageSender(): MessageSender;
|
||||
|
||||
private isTextMessage(content: MessageContentDetail) {
|
||||
return content.type === "text";
|
||||
}
|
||||
|
||||
private isAudioMessage(
|
||||
content: MessageContentDetail,
|
||||
): content is MessageContentAudioDetail {
|
||||
return content.type === "audio";
|
||||
}
|
||||
|
||||
private isImageMessage(
|
||||
content: MessageContentDetail,
|
||||
): content is MessageContentImageDataDetail {
|
||||
return content.type === "image";
|
||||
}
|
||||
|
||||
private isVideoMessage(
|
||||
content: MessageContentDetail,
|
||||
): content is MessageContentVideoDetail {
|
||||
return content.type === "video";
|
||||
}
|
||||
|
||||
sendMessage(message: ChatMessage) {
|
||||
const { content, role } = message;
|
||||
if (!Array.isArray(content)) {
|
||||
this.messageSender.sendTextMessage(content, role);
|
||||
} else {
|
||||
for (const item of content) {
|
||||
this.processMessage(item, role);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private processMessage(message: MessageContentDetail, role?: string) {
|
||||
if (this.isTextMessage(message)) {
|
||||
this.messageSender.sendTextMessage(message.text, role);
|
||||
} else if (
|
||||
this.isAudioMessage(message) &&
|
||||
this.messageSender.sendAudioMessage
|
||||
) {
|
||||
this.messageSender.sendAudioMessage(message, role);
|
||||
} else if (
|
||||
this.isImageMessage(message) &&
|
||||
this.messageSender.sendImageMessage
|
||||
) {
|
||||
this.messageSender.sendImageMessage(message, role);
|
||||
} else if (
|
||||
this.isVideoMessage(message) &&
|
||||
this.messageSender.sendVideoMessage
|
||||
) {
|
||||
this.messageSender.sendVideoMessage(message, role);
|
||||
}
|
||||
}
|
||||
|
||||
async *streamEvents(): AsyncIterable<LiveEvent> {
|
||||
while (true) {
|
||||
const event = await this.nextEvent();
|
||||
if (event === undefined) {
|
||||
break;
|
||||
}
|
||||
yield event;
|
||||
}
|
||||
}
|
||||
abstract disconnect(): Promise<void>;
|
||||
|
||||
protected async nextEvent(): Promise<LiveEvent | undefined> {
|
||||
if (this.eventQueue.length) {
|
||||
return Promise.resolve(this.eventQueue.shift());
|
||||
}
|
||||
|
||||
return new Promise((resolve) => {
|
||||
this.eventResolvers.push(resolve);
|
||||
});
|
||||
}
|
||||
|
||||
//Uses an async queue to send events to the client
|
||||
// if the consumer is waiting for an event, it will be resolved immediately
|
||||
// otherwise, the event will be queued up and sent when the consumer is ready
|
||||
pushEventToQueue(event: LiveEvent) {
|
||||
if (this.eventResolvers.length) {
|
||||
//resolving the promise with the event
|
||||
this.eventResolvers.shift()!(event);
|
||||
} else {
|
||||
this.eventQueue.push(event);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
export abstract class LiveLLM {
|
||||
/**
|
||||
* Set of capabilities supported by this implementation.
|
||||
* Override in subclasses as needed.
|
||||
*/
|
||||
capabilities: Set<LiveLLMCapability> = new Set();
|
||||
|
||||
abstract connect(config?: LiveConnectConfig): Promise<LiveLLMSession>;
|
||||
abstract getEphemeralKey(): Promise<string | undefined>;
|
||||
|
||||
hasCapability(capability: LiveLLMCapability): boolean {
|
||||
return this.capabilities.has(capability);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,15 @@
|
||||
import type {
|
||||
MessageContentAudioDetail,
|
||||
MessageContentImageDataDetail,
|
||||
MessageContentVideoDetail,
|
||||
} from "../type";
|
||||
|
||||
export interface MessageSender {
|
||||
sendTextMessage(message: string, role?: string): void;
|
||||
sendAudioMessage?(content: MessageContentAudioDetail, role?: string): void;
|
||||
sendImageMessage?(
|
||||
content: MessageContentImageDataDetail,
|
||||
role?: string,
|
||||
): void;
|
||||
sendVideoMessage?(content: MessageContentVideoDetail, role?: string): void;
|
||||
}
|
||||
@@ -290,8 +290,14 @@ export type ToolOutput = {
|
||||
isError: boolean;
|
||||
};
|
||||
|
||||
export interface AudioConfig {
|
||||
stream?: MediaStream;
|
||||
onTrack?: (track: MediaStream | null) => void;
|
||||
}
|
||||
|
||||
export interface LiveConnectConfig {
|
||||
tools?: BaseTool[];
|
||||
responseModality?: ModalityType[];
|
||||
systemInstruction?: string;
|
||||
audioConfig?: AudioConfig;
|
||||
}
|
||||
|
||||
@@ -12,14 +12,11 @@ import {
|
||||
LiveLLM,
|
||||
LiveLLMSession,
|
||||
type BaseTool,
|
||||
type ChatMessage,
|
||||
type LiveConnectConfig,
|
||||
type MessageContentAudioDetail,
|
||||
type MessageContentDetail,
|
||||
type MessageContentImageDataDetail,
|
||||
type MessageContentVideoDetail,
|
||||
type MessageSender,
|
||||
} from "@llamaindex/core/llms";
|
||||
import { getEnv, uint8ArrayToBase64 } from "@llamaindex/env";
|
||||
import { getEnv } from "@llamaindex/env";
|
||||
import { GeminiMessageSender } from "./message-sender";
|
||||
import { GEMINI_MODEL, type GeminiVoiceName } from "./types";
|
||||
import {
|
||||
mapBaseToolToGeminiLiveFunctionDeclaration,
|
||||
@@ -40,6 +37,10 @@ export class GeminiLiveSession extends LiveLLMSession {
|
||||
super();
|
||||
}
|
||||
|
||||
get messageSender(): MessageSender {
|
||||
return new GeminiMessageSender(this);
|
||||
}
|
||||
|
||||
private isInterruptedEvent(event: LiveServerMessage): boolean {
|
||||
return event.serverContent?.interrupted === true;
|
||||
}
|
||||
@@ -70,22 +71,6 @@ export class GeminiLiveSession extends LiveLLMSession {
|
||||
return event.setupComplete !== undefined;
|
||||
}
|
||||
|
||||
private isTextMessage(content: MessageContentDetail) {
|
||||
return content.type === "text";
|
||||
}
|
||||
|
||||
private isAudioMessage(content: MessageContentDetail) {
|
||||
return content.type === "audio";
|
||||
}
|
||||
|
||||
private isImageMessage(content: MessageContentDetail) {
|
||||
return content.type === "image";
|
||||
}
|
||||
|
||||
private isVideoMessage(content: MessageContentDetail) {
|
||||
return content.type === "video";
|
||||
}
|
||||
|
||||
//for the tool call event, we need to return the response with function responses
|
||||
private async handleToolCallEvent(
|
||||
event: LiveServerMessage,
|
||||
@@ -195,105 +180,11 @@ export class GeminiLiveSession extends LiveLLMSession {
|
||||
}
|
||||
}
|
||||
|
||||
private sendTextMessage(message: string, role?: string) {
|
||||
this.session?.sendClientContent({
|
||||
turns: [
|
||||
{
|
||||
parts: [{ text: message }],
|
||||
...(role ? { role } : {}),
|
||||
},
|
||||
],
|
||||
});
|
||||
}
|
||||
|
||||
private sendAudioMessage(content: MessageContentAudioDetail, role?: string) {
|
||||
if (typeof content.data === "string") {
|
||||
this.session?.sendRealtimeInput({
|
||||
audio: {
|
||||
data: content.data,
|
||||
mimeType: content.mimeType,
|
||||
},
|
||||
});
|
||||
} else {
|
||||
this.session?.sendRealtimeInput({
|
||||
audio: {
|
||||
data: uint8ArrayToBase64(content.data),
|
||||
mimeType: content.mimeType,
|
||||
},
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
private sendImageMessage(
|
||||
content: MessageContentImageDataDetail,
|
||||
role?: string,
|
||||
) {
|
||||
if (typeof content.data === "string") {
|
||||
this.session?.sendRealtimeInput({
|
||||
media: {
|
||||
data: content.data,
|
||||
mimeType: content.mimeType,
|
||||
},
|
||||
});
|
||||
} else {
|
||||
this.session?.sendRealtimeInput({
|
||||
media: {
|
||||
data: uint8ArrayToBase64(content.data),
|
||||
mimeType: content.mimeType,
|
||||
},
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
private sendVideoMessage(content: MessageContentVideoDetail, role?: string) {
|
||||
if (typeof content.data === "string") {
|
||||
this.session?.sendRealtimeInput({
|
||||
video: {
|
||||
data: content.data,
|
||||
mimeType: content.mimeType,
|
||||
},
|
||||
});
|
||||
} else {
|
||||
this.session?.sendRealtimeInput({
|
||||
video: {
|
||||
data: uint8ArrayToBase64(content.data),
|
||||
mimeType: content.mimeType,
|
||||
},
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
private handleUserInput(message: ChatMessage) {
|
||||
const { content, role } = message;
|
||||
|
||||
if (!Array.isArray(content)) {
|
||||
this.sendTextMessage(content, role);
|
||||
} else {
|
||||
for (const item of content) {
|
||||
if (this.isTextMessage(item)) {
|
||||
this.sendTextMessage(item.text, role);
|
||||
} else if (this.isAudioMessage(item)) {
|
||||
this.sendAudioMessage(item as MessageContentAudioDetail, role);
|
||||
} else if (this.isImageMessage(item)) {
|
||||
this.sendImageMessage(item as MessageContentImageDataDetail, role);
|
||||
} else if (this.isVideoMessage(item)) {
|
||||
this.sendVideoMessage(item as MessageContentVideoDetail, role);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
sendMessage(message: ChatMessage) {
|
||||
if (!this.session) {
|
||||
throw new Error("Session not connected");
|
||||
}
|
||||
this.handleUserInput(message);
|
||||
}
|
||||
|
||||
async disconnect() {
|
||||
if (!this.session) {
|
||||
throw new Error("Session not connected");
|
||||
}
|
||||
|
||||
this.session.close();
|
||||
}
|
||||
}
|
||||
@@ -322,6 +213,10 @@ export class GeminiLive extends LiveLLM {
|
||||
}
|
||||
}
|
||||
|
||||
async getEphemeralKey(): Promise<string | undefined> {
|
||||
throw new Error("Ephemeral key is not supported for Gemini Live");
|
||||
}
|
||||
|
||||
async connect(config?: LiveConnectConfig) {
|
||||
const liveConfig: GoogleLiveConnectConfig = {
|
||||
responseModalities: config?.responseModality
|
||||
@@ -356,6 +251,12 @@ export class GeminiLive extends LiveLLM {
|
||||
};
|
||||
}
|
||||
|
||||
if (config?.audioConfig) {
|
||||
throw new Error(
|
||||
"Audio config is not supported for Gemini Live, directly send and recieve audio events instead",
|
||||
);
|
||||
}
|
||||
|
||||
const geminiLiveSession = new GeminiLiveSession();
|
||||
|
||||
geminiLiveSession.session = await this.client.live.connect({
|
||||
|
||||
@@ -0,0 +1,53 @@
|
||||
import type {
|
||||
MessageContentAudioDetail,
|
||||
MessageContentImageDataDetail,
|
||||
MessageContentVideoDetail,
|
||||
MessageSender,
|
||||
} from "@llamaindex/core/llms";
|
||||
import type { GeminiLiveSession } from "./live";
|
||||
|
||||
export class GeminiMessageSender implements MessageSender {
|
||||
private geminiSession: GeminiLiveSession;
|
||||
|
||||
constructor(session: GeminiLiveSession) {
|
||||
this.geminiSession = session;
|
||||
}
|
||||
|
||||
sendTextMessage(message: string, role?: string) {
|
||||
this.geminiSession.session?.sendClientContent({
|
||||
turns: [
|
||||
{
|
||||
parts: [{ text: message }],
|
||||
...(role ? { role } : {}),
|
||||
},
|
||||
],
|
||||
});
|
||||
}
|
||||
|
||||
sendAudioMessage(content: MessageContentAudioDetail, role?: string) {
|
||||
this.geminiSession.session?.sendRealtimeInput({
|
||||
audio: {
|
||||
data: content.data,
|
||||
mimeType: content.mimeType,
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
sendImageMessage(content: MessageContentImageDataDetail, role?: string) {
|
||||
this.geminiSession.session?.sendRealtimeInput({
|
||||
media: {
|
||||
data: content.data,
|
||||
mimeType: content.mimeType,
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
sendVideoMessage(content: MessageContentVideoDetail, role?: string) {
|
||||
this.geminiSession.session?.sendRealtimeInput({
|
||||
video: {
|
||||
data: content.data,
|
||||
mimeType: content.mimeType,
|
||||
},
|
||||
});
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,182 @@
|
||||
import type {
|
||||
AudioConfig,
|
||||
BaseTool,
|
||||
MessageSender,
|
||||
} from "@llamaindex/core/llms";
|
||||
|
||||
import { LiveLLMSession } from "@llamaindex/core/llms";
|
||||
import { OpenAIMessageSender } from "./message-sender";
|
||||
|
||||
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
||||
type ServerEvent = Record<string, any>;
|
||||
export interface FunctionCall {
|
||||
name: string;
|
||||
call_id: string;
|
||||
args: Record<string, unknown>;
|
||||
}
|
||||
|
||||
export class OpenAILiveSession extends LiveLLMSession {
|
||||
closed = false;
|
||||
audioStream: MediaStream | undefined;
|
||||
peerConnection: RTCPeerConnection | undefined;
|
||||
dataChannel: RTCDataChannel | undefined;
|
||||
|
||||
get messageSender(): MessageSender {
|
||||
return new OpenAIMessageSender(this);
|
||||
}
|
||||
|
||||
private isResponseDoneEvent(event: ServerEvent) {
|
||||
return event.type === "response.done";
|
||||
}
|
||||
|
||||
private isTextEvent(event: ServerEvent) {
|
||||
return (
|
||||
event.response.output &&
|
||||
event.response?.output[0]?.type === "message" &&
|
||||
event.response?.output[0]?.content[0]?.type === "text"
|
||||
);
|
||||
}
|
||||
|
||||
private isFunctionCallEvent(event: ServerEvent) {
|
||||
return (
|
||||
event.response.output &&
|
||||
event.response?.output[0]?.type === "function_call"
|
||||
);
|
||||
}
|
||||
|
||||
private isAudioDeltaEvent(event: ServerEvent) {
|
||||
return event.type === "response.audio.delta";
|
||||
}
|
||||
|
||||
private isInterruptedEvent(event: ServerEvent) {
|
||||
return event.type === "output_audio_buffer.cleared";
|
||||
}
|
||||
|
||||
setupAudioTracks(config?: AudioConfig) {
|
||||
if (!this.peerConnection) return;
|
||||
|
||||
this.peerConnection.ontrack = (event) => {
|
||||
if (config?.onTrack) {
|
||||
config.onTrack(event.streams[0] || null);
|
||||
}
|
||||
};
|
||||
|
||||
if (config?.stream) {
|
||||
this.audioStream = config.stream;
|
||||
this.audioStream.getAudioTracks().forEach((track) => {
|
||||
this.peerConnection?.addTrack(track, this.audioStream!);
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
async handleEvents(event: ServerEvent, toolCalls: BaseTool[]) {
|
||||
if (this.isAudioDeltaEvent(event)) {
|
||||
this.pushEventToQueue({
|
||||
type: "audio",
|
||||
data: event.delta,
|
||||
mimeType: "audio/wav",
|
||||
});
|
||||
}
|
||||
if (this.isResponseDoneEvent(event)) {
|
||||
if (this.isTextEvent(event)) {
|
||||
this.pushEventToQueue({
|
||||
type: "text",
|
||||
text: event.response?.output[0]?.content[0]?.text,
|
||||
});
|
||||
}
|
||||
if (this.isFunctionCallEvent(event)) {
|
||||
await this.handleFunctionCall(event, toolCalls);
|
||||
}
|
||||
if (this.isInterruptedEvent(event)) {
|
||||
this.pushEventToQueue({
|
||||
type: "interrupted",
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private async sendToolCallResponses(
|
||||
eventToolCalls: FunctionCall[],
|
||||
toolCalls: BaseTool[],
|
||||
) {
|
||||
for (const toolCall of eventToolCalls) {
|
||||
const tool = toolCalls.find((t) => t.metadata.name === toolCall.name);
|
||||
if (tool && tool.call) {
|
||||
const response = await tool.call(toolCall.args);
|
||||
this.sendToolCallResponse(toolCall.call_id, response);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
async sendResponseCreateEvent() {
|
||||
this.dataChannel?.send(
|
||||
JSON.stringify({
|
||||
type: "response.create",
|
||||
}),
|
||||
);
|
||||
}
|
||||
|
||||
private sendToolCallResponse(callId: string, response: unknown) {
|
||||
if (!this.dataChannel) {
|
||||
throw new Error("Data channel not connected");
|
||||
}
|
||||
const event = {
|
||||
type: "conversation.item.create",
|
||||
item: {
|
||||
type: "function_call_output",
|
||||
call_id: callId,
|
||||
output: JSON.stringify(response),
|
||||
},
|
||||
};
|
||||
|
||||
this.dataChannel.send(JSON.stringify(event));
|
||||
this.sendResponseCreateEvent();
|
||||
}
|
||||
|
||||
private async handleFunctionCall(event: ServerEvent, toolCalls: BaseTool[]) {
|
||||
const eventToolCalls: FunctionCall[] = event.response.output.map(
|
||||
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
||||
(output: Record<string, any>) => {
|
||||
return {
|
||||
name: output.name,
|
||||
call_id: output.call_id,
|
||||
args:
|
||||
typeof output.arguments === "string"
|
||||
? JSON.parse(output.arguments)
|
||||
: output.arguments,
|
||||
};
|
||||
},
|
||||
);
|
||||
|
||||
if (eventToolCalls && eventToolCalls.length > 0) {
|
||||
await this.sendToolCallResponses(eventToolCalls, toolCalls);
|
||||
}
|
||||
}
|
||||
|
||||
async disconnect() {
|
||||
if (this.dataChannel) {
|
||||
this.dataChannel.close();
|
||||
this.dataChannel = undefined;
|
||||
}
|
||||
if (this.peerConnection) {
|
||||
this.peerConnection.close();
|
||||
this.peerConnection = undefined;
|
||||
}
|
||||
this.closed = true;
|
||||
|
||||
if (this.audioStream) {
|
||||
this.audioStream.getTracks().forEach((track) => {
|
||||
track.stop();
|
||||
});
|
||||
this.audioStream = undefined;
|
||||
}
|
||||
}
|
||||
|
||||
setPeerConnection(pc: RTCPeerConnection) {
|
||||
this.peerConnection = pc;
|
||||
}
|
||||
|
||||
setDataChannel(dc: RTCDataChannel) {
|
||||
this.dataChannel = dc;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,165 @@
|
||||
import {
|
||||
LiveLLM,
|
||||
LiveLLMCapability,
|
||||
type LiveConnectConfig,
|
||||
} from "@llamaindex/core/llms";
|
||||
import type { ChatModel } from "openai/resources.mjs";
|
||||
import { OpenAILiveSession } from "./live-session";
|
||||
import {
|
||||
mapModalityToOpenAIModality,
|
||||
toOpenAILiveTool,
|
||||
type OpenAILiveConfig,
|
||||
type OpenAIVoiceNames,
|
||||
} from "./utils";
|
||||
|
||||
const REALTIME_MODELS = [
|
||||
"gpt-4o-realtime-preview-2025-06-03",
|
||||
"gpt-4o-realtime-preview-2024-12-17",
|
||||
"gpt-4o-realtime-preview-2024-10-01",
|
||||
];
|
||||
export class OpenAILive extends LiveLLM {
|
||||
private apiKey: string | undefined;
|
||||
private model: ChatModel | (string & {});
|
||||
voiceName?: OpenAIVoiceNames | undefined;
|
||||
private baseURL: string;
|
||||
|
||||
constructor(init?: OpenAILiveConfig) {
|
||||
super();
|
||||
this.apiKey = init?.apiKey;
|
||||
this.model = init?.model ?? "gpt-4o-realtime-preview-2025-06-03";
|
||||
this.voiceName = init?.voiceName;
|
||||
this.baseURL = "https://api.openai.com/v1/realtime";
|
||||
if (!this.apiKey) {
|
||||
throw new Error("OPENAI_API_KEY is not set");
|
||||
}
|
||||
this.capabilities.add(LiveLLMCapability.EPHEMERAL_KEY);
|
||||
this.capabilities.add(LiveLLMCapability.AUDIO_CONFIG);
|
||||
}
|
||||
|
||||
async getEphemeralKey() {
|
||||
if (!REALTIME_MODELS.includes(this.model)) {
|
||||
throw new Error(
|
||||
"Ephemeral key is only supported for gpt-4o-realtime-preview models",
|
||||
);
|
||||
}
|
||||
const response = await fetch(`${this.baseURL}/sessions`, {
|
||||
method: "POST",
|
||||
headers: {
|
||||
Authorization: `Bearer ${this.apiKey}`,
|
||||
"Content-Type": "application/json",
|
||||
},
|
||||
body: JSON.stringify({
|
||||
model: this.model,
|
||||
voice: this.voiceName,
|
||||
}),
|
||||
});
|
||||
|
||||
const data = await response.json();
|
||||
return data.client_secret.value;
|
||||
}
|
||||
|
||||
private async getSDPResponse(offer: RTCSessionDescriptionInit) {
|
||||
const sdpResponse = await fetch(`${this.baseURL}?model=${this.model}`, {
|
||||
method: "POST",
|
||||
body: offer.sdp!,
|
||||
headers: {
|
||||
Authorization: `Bearer ${this.apiKey}`,
|
||||
"Content-Type": "application/sdp",
|
||||
},
|
||||
});
|
||||
|
||||
return sdpResponse;
|
||||
}
|
||||
|
||||
private async establishSDPConnection(session: OpenAILiveSession) {
|
||||
const offer = await session.peerConnection!.createOffer();
|
||||
await session.peerConnection!.setLocalDescription(offer);
|
||||
|
||||
if (!offer.sdp) {
|
||||
throw new Error("Failed to create SDP offer");
|
||||
}
|
||||
|
||||
const sdpResponse = await this.getSDPResponse(offer);
|
||||
|
||||
const answer = {
|
||||
sdp: await sdpResponse.text(),
|
||||
type: "answer" as RTCSdpType,
|
||||
};
|
||||
|
||||
await session.peerConnection!.setRemoteDescription(answer);
|
||||
}
|
||||
|
||||
private async initializeRTCPeerConnectionAndDataChannel(
|
||||
session: OpenAILiveSession,
|
||||
config?: LiveConnectConfig,
|
||||
) {
|
||||
session.peerConnection = new RTCPeerConnection();
|
||||
session.setupAudioTracks(config?.audioConfig);
|
||||
session.dataChannel =
|
||||
session.peerConnection.createDataChannel("oai-events");
|
||||
}
|
||||
|
||||
private async setupWebRTC(
|
||||
session: OpenAILiveSession,
|
||||
config?: LiveConnectConfig,
|
||||
) {
|
||||
this.initializeRTCPeerConnectionAndDataChannel(session, config);
|
||||
await this.establishSDPConnection(session);
|
||||
}
|
||||
|
||||
async connect(config?: LiveConnectConfig): Promise<OpenAILiveSession> {
|
||||
const session = new OpenAILiveSession();
|
||||
await this.setupWebRTC(session, config);
|
||||
this.setupEventListeners(session, config);
|
||||
return session;
|
||||
}
|
||||
|
||||
private setupEventListeners(
|
||||
session: OpenAILiveSession,
|
||||
config?: LiveConnectConfig,
|
||||
) {
|
||||
this.messageEventListener(session, config);
|
||||
this.openEventListener(session, config);
|
||||
this.errorEventListener(session);
|
||||
}
|
||||
|
||||
private messageEventListener(
|
||||
session: OpenAILiveSession,
|
||||
config?: LiveConnectConfig,
|
||||
) {
|
||||
session.dataChannel?.addEventListener("message", (event) => {
|
||||
session.handleEvents(JSON.parse(event.data), config?.tools ?? []);
|
||||
});
|
||||
}
|
||||
|
||||
private openEventListener(
|
||||
session: OpenAILiveSession,
|
||||
config?: LiveConnectConfig,
|
||||
) {
|
||||
session.dataChannel?.addEventListener("open", () => {
|
||||
const event = {
|
||||
type: "session.update",
|
||||
session: {
|
||||
voice: this.voiceName,
|
||||
instructions: config?.systemInstruction,
|
||||
tools: config?.tools?.map(toOpenAILiveTool),
|
||||
modalities: config?.responseModality?.map(
|
||||
mapModalityToOpenAIModality,
|
||||
),
|
||||
},
|
||||
};
|
||||
|
||||
session.dataChannel?.send(JSON.stringify(event));
|
||||
session.pushEventToQueue({ type: "open" });
|
||||
});
|
||||
}
|
||||
|
||||
private errorEventListener(session: OpenAILiveSession) {
|
||||
session.dataChannel?.addEventListener("error", (event) => {
|
||||
session.pushEventToQueue({
|
||||
type: "error",
|
||||
error: event,
|
||||
});
|
||||
});
|
||||
}
|
||||
}
|
||||
@@ -36,6 +36,7 @@ import type {
|
||||
ResponseFormatJSONObject,
|
||||
ResponseFormatJSONSchema,
|
||||
} from "openai/resources/index.js";
|
||||
import { OpenAILive } from "./live.js";
|
||||
import {
|
||||
ALL_AVAILABLE_OPENAI_MODELS,
|
||||
isFunctionCallingModel,
|
||||
@@ -44,6 +45,7 @@ import {
|
||||
type LLMInstance,
|
||||
type OpenAIAdditionalChatOptions,
|
||||
type OpenAIAdditionalMetadata,
|
||||
type OpenAIVoiceNames,
|
||||
} from "./utils.js";
|
||||
|
||||
export class OpenAI extends ToolCallLLM<OpenAIAdditionalChatOptions> {
|
||||
@@ -65,6 +67,8 @@ export class OpenAI extends ToolCallLLM<OpenAIAdditionalChatOptions> {
|
||||
additionalSessionOptions?:
|
||||
| undefined
|
||||
| Omit<Partial<OpenAIClientOptions>, "apiKey" | "maxRetries" | "timeout">;
|
||||
private voiceName?: OpenAIVoiceNames | undefined;
|
||||
private _live: OpenAILive | undefined;
|
||||
|
||||
// use lazy here to avoid check OPENAI_API_KEY immediately
|
||||
lazySession: () => Promise<LLMInstance>;
|
||||
@@ -79,6 +83,7 @@ export class OpenAI extends ToolCallLLM<OpenAIAdditionalChatOptions> {
|
||||
constructor(
|
||||
init?: Omit<Partial<OpenAI>, "session"> & {
|
||||
session?: LLMInstance | undefined;
|
||||
voiceName?: OpenAIVoiceNames | undefined;
|
||||
},
|
||||
) {
|
||||
super();
|
||||
@@ -97,7 +102,7 @@ export class OpenAI extends ToolCallLLM<OpenAIAdditionalChatOptions> {
|
||||
this.additionalSessionOptions = init?.additionalSessionOptions;
|
||||
this.apiKey = init?.session?.apiKey ?? init?.apiKey;
|
||||
this.baseURL = init?.session?.baseURL ?? init?.baseURL;
|
||||
|
||||
this.voiceName = init?.voiceName;
|
||||
this.lazySession = async () =>
|
||||
init?.session ??
|
||||
import("openai").then(({ OpenAI }) => {
|
||||
@@ -115,6 +120,17 @@ export class OpenAI extends ToolCallLLM<OpenAIAdditionalChatOptions> {
|
||||
return isFunctionCallingModel(this);
|
||||
}
|
||||
|
||||
get live(): OpenAILive {
|
||||
if (!this._live) {
|
||||
this._live = new OpenAILive({
|
||||
apiKey: this.apiKey,
|
||||
voiceName: this.voiceName,
|
||||
model: this.model as ChatModel,
|
||||
});
|
||||
}
|
||||
return this._live;
|
||||
}
|
||||
|
||||
get metadata(): LLMMetadata & OpenAIAdditionalMetadata {
|
||||
const contextWindow =
|
||||
ALL_AVAILABLE_OPENAI_MODELS[
|
||||
|
||||
@@ -0,0 +1,51 @@
|
||||
import {
|
||||
type MessageContentAudioDetail,
|
||||
type MessageSender,
|
||||
} from "@llamaindex/core/llms";
|
||||
import type { OpenAILiveSession } from "./live-session";
|
||||
|
||||
export class OpenAIMessageSender implements MessageSender {
|
||||
private openaiSession: OpenAILiveSession;
|
||||
|
||||
constructor(session: OpenAILiveSession) {
|
||||
this.openaiSession = session;
|
||||
}
|
||||
|
||||
sendTextMessage(message: string, role?: string) {
|
||||
const event = {
|
||||
type: "conversation.item.create",
|
||||
item: {
|
||||
type: "message",
|
||||
role: "user",
|
||||
content: [
|
||||
{
|
||||
type: "input_text",
|
||||
text: message,
|
||||
},
|
||||
],
|
||||
},
|
||||
};
|
||||
|
||||
this.openaiSession.dataChannel?.send(JSON.stringify(event));
|
||||
this.openaiSession.sendResponseCreateEvent();
|
||||
}
|
||||
|
||||
sendAudioMessage(content: MessageContentAudioDetail, role?: string) {
|
||||
const event = {
|
||||
type: "conversation.item.create",
|
||||
item: {
|
||||
type: "message",
|
||||
role: "user",
|
||||
content: [
|
||||
{
|
||||
type: "input_audio",
|
||||
audio: content.data,
|
||||
},
|
||||
],
|
||||
},
|
||||
};
|
||||
|
||||
this.openaiSession.dataChannel?.send(JSON.stringify(event));
|
||||
this.openaiSession.sendResponseCreateEvent();
|
||||
}
|
||||
}
|
||||
@@ -1,4 +1,5 @@
|
||||
import type { LLM, PartialToolCall } from "@llamaindex/core/llms";
|
||||
import type { BaseTool, LLM, PartialToolCall } from "@llamaindex/core/llms";
|
||||
import { ModalityType } from "@llamaindex/core/schema";
|
||||
import { OpenAI as OpenAILLM } from "openai";
|
||||
import type { ChatModel } from "openai/resources.mjs";
|
||||
import { OpenAI } from "./llm";
|
||||
@@ -34,6 +35,12 @@ export const GPT4_MODELS = {
|
||||
"gpt-4o-realtime-preview-2024-10-01": {
|
||||
contextWindow: 128000,
|
||||
},
|
||||
"gpt-4o-realtime-preview-2024-12-17": {
|
||||
contextWindow: 128000,
|
||||
},
|
||||
"gpt-4o-realtime-preview-2025-06-03": {
|
||||
contextWindow: 128000,
|
||||
},
|
||||
"gpt-4o-audio-preview": {
|
||||
contextWindow: 128000,
|
||||
},
|
||||
@@ -54,6 +61,7 @@ export const GPT4_MODELS = {
|
||||
},
|
||||
"gpt-4o-search-preview": { contextWindow: 128000 },
|
||||
"gpt-4o-mini-search-preview": { contextWindow: 128000 },
|
||||
"gpt-4o-mini-realtime-preview-2024-12-17": { contextWindow: 128000 },
|
||||
"gpt-4o-search-preview-2025-03-11": { contextWindow: 128000 },
|
||||
"gpt-4o-mini-search-preview-2025-03-11": { contextWindow: 128000 },
|
||||
"gpt-4.1": { contextWindow: 10 ** 6 },
|
||||
@@ -243,3 +251,45 @@ export type ResponsesMessageContentDetail =
|
||||
export type ResponseMessageContent = string | ResponsesMessageContentDetail[];
|
||||
|
||||
export type OpenAIResponsesRole = "user" | "assistant" | "system" | "developer";
|
||||
export type Modality = "text" | "audio" | "video";
|
||||
|
||||
export interface OpenAILiveConfig {
|
||||
apiKey?: string | undefined;
|
||||
model?: ChatModel | undefined;
|
||||
voiceName?: OpenAIVoiceNames | undefined;
|
||||
}
|
||||
|
||||
export function mapModalityToOpenAIModality(
|
||||
responseModalities: ModalityType,
|
||||
): Modality {
|
||||
return responseModalities === ModalityType.TEXT
|
||||
? "text"
|
||||
: responseModalities === ModalityType.AUDIO
|
||||
? "audio"
|
||||
: "video";
|
||||
}
|
||||
|
||||
export type OpenAIVoiceNames =
|
||||
| "alloy"
|
||||
| "ash"
|
||||
| "echo"
|
||||
| "fable"
|
||||
| "onyx"
|
||||
| "nova"
|
||||
| "shimmer";
|
||||
|
||||
export type OpenAILiveTool = {
|
||||
type: "function";
|
||||
name: string;
|
||||
description: string;
|
||||
parameters: Record<string, unknown>;
|
||||
};
|
||||
|
||||
export const toOpenAILiveTool = (tool: BaseTool): OpenAILiveTool => {
|
||||
return {
|
||||
type: "function",
|
||||
name: tool.metadata.name,
|
||||
description: tool.metadata.description,
|
||||
parameters: tool.metadata.parameters ?? {},
|
||||
};
|
||||
};
|
||||
|
||||
Generated
+797
-479
File diff suppressed because it is too large
Load Diff
Reference in New Issue
Block a user