mirror of
https://github.com/anomalyco/opencode.git
synced 2026-08-10 03:19:59 -04:00
Compare commits
19 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| 0574107e18 | |||
| 6f0fa2d6c6 | |||
| 363905dcdd | |||
| d71bc0055a | |||
| 4f235fc59b | |||
| 1d772f4896 | |||
| 113a117cef | |||
| c103ba6056 | |||
| 38f03c9d8b | |||
| e5ac4ca560 | |||
| 5f0487fa87 | |||
| 00557f9078 | |||
| 5ac1089c5c | |||
| d1843f49ac | |||
| aac66d384e | |||
| 83108c1b5d | |||
| e7000161f7 | |||
| 66200eee34 | |||
| 699477755f |
@@ -986,6 +986,25 @@
|
|||||||
"@typescript/native-preview": "catalog:",
|
"@typescript/native-preview": "catalog:",
|
||||||
},
|
},
|
||||||
},
|
},
|
||||||
|
"packages/voice": {
|
||||||
|
"name": "@opencode-ai/voice",
|
||||||
|
"version": "0.0.0",
|
||||||
|
"dependencies": {
|
||||||
|
"@opencode-ai/client": "workspace:*",
|
||||||
|
"@opencode-ai/plugin": "workspace:*",
|
||||||
|
"@opentui/core": "catalog:",
|
||||||
|
"@opentui/solid": "catalog:",
|
||||||
|
"effect": "catalog:",
|
||||||
|
"opentui-spinner": "catalog:",
|
||||||
|
"solid-js": "catalog:",
|
||||||
|
},
|
||||||
|
"devDependencies": {
|
||||||
|
"@tsconfig/bun": "catalog:",
|
||||||
|
"@types/bun": "catalog:",
|
||||||
|
"@typescript/native-preview": "catalog:",
|
||||||
|
"typescript": "catalog:",
|
||||||
|
},
|
||||||
|
},
|
||||||
"packages/web": {
|
"packages/web": {
|
||||||
"name": "@opencode-ai/web",
|
"name": "@opencode-ai/web",
|
||||||
"version": "1.18.4",
|
"version": "1.18.4",
|
||||||
@@ -2077,6 +2096,8 @@
|
|||||||
|
|
||||||
"@opencode-ai/util": ["@opencode-ai/util@workspace:packages/util"],
|
"@opencode-ai/util": ["@opencode-ai/util@workspace:packages/util"],
|
||||||
|
|
||||||
|
"@opencode-ai/voice": ["@opencode-ai/voice@workspace:packages/voice"],
|
||||||
|
|
||||||
"@opencode-ai/web": ["@opencode-ai/web@workspace:packages/web"],
|
"@opencode-ai/web": ["@opencode-ai/web@workspace:packages/web"],
|
||||||
|
|
||||||
"@opencode-ai/www": ["@opencode-ai/www@workspace:packages/www"],
|
"@opencode-ai/www": ["@opencode-ai/www@workspace:packages/www"],
|
||||||
|
|||||||
@@ -88,8 +88,12 @@ function SessionTabSlot(props: {
|
|||||||
data-titlebar-tab-slot
|
data-titlebar-tab-slot
|
||||||
data-tab-key={props.id}
|
data-tab-key={props.id}
|
||||||
data-active={props.active()}
|
data-active={props.active()}
|
||||||
class="relative flex w-56 min-w-7 max-w-56 flex-shrink"
|
class="relative flex min-w-7 flex-shrink"
|
||||||
classList={{ hidden: !session() && !missingSession() && !persisted()?.title }}
|
classList={{
|
||||||
|
hidden: !session() && !missingSession() && !persisted()?.title,
|
||||||
|
"w-72 max-w-72": props.active(),
|
||||||
|
"w-56 max-w-56": !props.active(),
|
||||||
|
}}
|
||||||
>
|
>
|
||||||
<TabNavItem
|
<TabNavItem
|
||||||
ref={(el) => {
|
ref={(el) => {
|
||||||
@@ -144,7 +148,11 @@ function DraftTabSlot(props: {
|
|||||||
data-titlebar-tab-slot
|
data-titlebar-tab-slot
|
||||||
data-tab-key={props.id}
|
data-tab-key={props.id}
|
||||||
data-active={props.active()}
|
data-active={props.active()}
|
||||||
class="relative flex w-56 min-w-7 max-w-56 flex-shrink"
|
class="relative flex min-w-7 flex-shrink"
|
||||||
|
classList={{
|
||||||
|
"w-72 max-w-72": props.active(),
|
||||||
|
"w-56 max-w-56": !props.active(),
|
||||||
|
}}
|
||||||
>
|
>
|
||||||
<DraftTabItem
|
<DraftTabItem
|
||||||
ref={(el) => {
|
ref={(el) => {
|
||||||
|
|||||||
@@ -1,5 +1,6 @@
|
|||||||
#!/usr/bin/env bun
|
#!/usr/bin/env bun
|
||||||
|
|
||||||
|
import "@opentui/solid/runtime-plugin-support"
|
||||||
import { NodeRuntime, NodeServices } from "@effect/platform-node"
|
import { NodeRuntime, NodeServices } from "@effect/platform-node"
|
||||||
import { Effect } from "effect"
|
import { Effect } from "effect"
|
||||||
import { Commands } from "./commands/commands"
|
import { Commands } from "./commands/commands"
|
||||||
|
|||||||
@@ -37,7 +37,6 @@ export async function ensure(options: EnsureOptions = {}): Promise<Endpoint> {
|
|||||||
let announced = false
|
let announced = false
|
||||||
let lastSpawn = 0
|
let lastSpawn = 0
|
||||||
let spawnDelay = 5_000
|
let spawnDelay = 5_000
|
||||||
let ownerHeld = false
|
|
||||||
|
|
||||||
const announce = (reason: "missing" | "version-mismatch", previousVersion?: string) => {
|
const announce = (reason: "missing" | "version-mismatch", previousVersion?: string) => {
|
||||||
if (announced) return
|
if (announced) return
|
||||||
@@ -65,7 +64,6 @@ export async function ensure(options: EnsureOptions = {}): Promise<Endpoint> {
|
|||||||
const registration = await registered(options.file, true)
|
const registration = await registered(options.file, true)
|
||||||
|
|
||||||
if (registration.service !== undefined) {
|
if (registration.service !== undefined) {
|
||||||
ownerHeld = false
|
|
||||||
spawnDelay = 5_000
|
spawnDelay = 5_000
|
||||||
const service = registration.service
|
const service = registration.service
|
||||||
const compatible = !service.legacy && (options.version === undefined || service.version === options.version)
|
const compatible = !service.legacy && (options.version === undefined || service.version === options.version)
|
||||||
@@ -82,7 +80,6 @@ export async function ensure(options: EnsureOptions = {}): Promise<Endpoint> {
|
|||||||
if (failure !== undefined) throw failure
|
if (failure !== undefined) throw failure
|
||||||
const finished = [...contenders].filter(contenderFinished)
|
const finished = [...contenders].filter(contenderFinished)
|
||||||
if (finished.some((item) => item.child.exitCode === 0)) {
|
if (finished.some((item) => item.child.exitCode === 0)) {
|
||||||
ownerHeld = true
|
|
||||||
spawnDelay = Math.min(spawnDelay * 2, 30_000)
|
spawnDelay = Math.min(spawnDelay * 2, 30_000)
|
||||||
}
|
}
|
||||||
finished.forEach((item) => contenders.delete(item))
|
finished.forEach((item) => contenders.delete(item))
|
||||||
|
|||||||
@@ -654,6 +654,12 @@ const layer = Layer.effectDiscard(
|
|||||||
input: event.data.input,
|
input: event.data.input,
|
||||||
timeCreated: event.created,
|
timeCreated: event.created,
|
||||||
})
|
})
|
||||||
|
yield* db
|
||||||
|
.update(SessionTable)
|
||||||
|
.set({ time_updated: DateTime.toEpochMillis(event.created) })
|
||||||
|
.where(eq(SessionTable.id, event.data.sessionID))
|
||||||
|
.run()
|
||||||
|
.pipe(Effect.orDie)
|
||||||
}),
|
}),
|
||||||
)
|
)
|
||||||
yield* events.project(SessionEvent.Compaction.Admitted, (event) =>
|
yield* events.project(SessionEvent.Compaction.Admitted, (event) =>
|
||||||
|
|||||||
@@ -171,6 +171,28 @@ describe("SessionV2.create", () => {
|
|||||||
}),
|
}),
|
||||||
)
|
)
|
||||||
|
|
||||||
|
it.effect("orders reused sessions by prompt admission time", () =>
|
||||||
|
Effect.gen(function* () {
|
||||||
|
const session = yield* SessionV2.Service
|
||||||
|
const events = yield* EventV2.Service
|
||||||
|
const { db } = yield* Database.Service
|
||||||
|
const reused = yield* session.create({ location, title: "reused" })
|
||||||
|
const newer = yield* session.create({ location, title: "newer" })
|
||||||
|
yield* db.update(SessionTable).set({ time_updated: -2 }).where(eq(SessionTable.id, reused.id)).run()
|
||||||
|
yield* db.update(SessionTable).set({ time_updated: -1 }).where(eq(SessionTable.id, newer.id)).run()
|
||||||
|
|
||||||
|
const admitted = yield* events.publish(SessionEvent.InputAdmitted, {
|
||||||
|
sessionID: reused.id,
|
||||||
|
inputID: SessionMessage.ID.create(),
|
||||||
|
input: { type: "user", data: { text: "continue" }, delivery: "steer" },
|
||||||
|
})
|
||||||
|
|
||||||
|
const page = yield* session.list({ directory: location.directory, parentID: null, order: "desc" })
|
||||||
|
expect(page.data.map((item) => item.id)).toEqual([reused.id, newer.id])
|
||||||
|
expect(page.data[0]!.time.updated).toEqual(admitted.created)
|
||||||
|
}),
|
||||||
|
)
|
||||||
|
|
||||||
it.effect("filters direct child sessions by parent ID", () =>
|
it.effect("filters direct child sessions by parent ID", () =>
|
||||||
Effect.gen(function* () {
|
Effect.gen(function* () {
|
||||||
const session = yield* SessionV2.Service
|
const session = yield* SessionV2.Service
|
||||||
|
|||||||
@@ -0,0 +1 @@
|
|||||||
|
.build/
|
||||||
@@ -0,0 +1,4 @@
|
|||||||
|
preload = ["@opentui/solid/preload"]
|
||||||
|
|
||||||
|
[test]
|
||||||
|
preload = ["@opentui/solid/preload"]
|
||||||
@@ -0,0 +1,29 @@
|
|||||||
|
{
|
||||||
|
"$schema": "https://json.schemastore.org/package.json",
|
||||||
|
"name": "@opencode-ai/voice",
|
||||||
|
"version": "0.0.0",
|
||||||
|
"private": true,
|
||||||
|
"description": "Prototype voice control for OpenCode via the OpenAI Realtime and Live APIs",
|
||||||
|
"type": "module",
|
||||||
|
"scripts": {
|
||||||
|
"bench:audio": "bun scripts/bench-speaker-queue.ts",
|
||||||
|
"spike": "bun run --no-orphans --conditions=browser src/spike.ts",
|
||||||
|
"test": "bun test",
|
||||||
|
"typecheck": "tsgo --noEmit"
|
||||||
|
},
|
||||||
|
"devDependencies": {
|
||||||
|
"@tsconfig/bun": "catalog:",
|
||||||
|
"@types/bun": "catalog:",
|
||||||
|
"@typescript/native-preview": "catalog:",
|
||||||
|
"typescript": "catalog:"
|
||||||
|
},
|
||||||
|
"dependencies": {
|
||||||
|
"@opencode-ai/client": "workspace:*",
|
||||||
|
"@opencode-ai/plugin": "workspace:*",
|
||||||
|
"@opentui/core": "catalog:",
|
||||||
|
"@opentui/solid": "catalog:",
|
||||||
|
"effect": "catalog:",
|
||||||
|
"opentui-spinner": "catalog:",
|
||||||
|
"solid-js": "catalog:"
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,38 @@
|
|||||||
|
# Speaker Queue Performance
|
||||||
|
|
||||||
|
## Goal
|
||||||
|
|
||||||
|
Reduce render-path latency and variance that can produce audible output glitches.
|
||||||
|
|
||||||
|
## Benchmark
|
||||||
|
|
||||||
|
Run `bun run bench:audio` from `packages/voice`.
|
||||||
|
|
||||||
|
The benchmark compiles the production `SpeakerQueue`, performs one warmup run and seven measured runs, and reports callback latency for a 512-frame pop followed by a same-sized producer push. The queue starts with one second of 24 kHz PCM audio.
|
||||||
|
|
||||||
|
## Metrics
|
||||||
|
|
||||||
|
- Primary: `speaker_queue_p99_ns`
|
||||||
|
- Secondary: `speaker_queue_median_ns`, `speaker_queue_worst_ns`
|
||||||
|
|
||||||
|
## Scope
|
||||||
|
|
||||||
|
- `src/duplex-audio.swift`
|
||||||
|
- `scripts/bench-speaker-queue.ts`
|
||||||
|
|
||||||
|
## Experiments
|
||||||
|
|
||||||
|
### Baseline: allocating `Data` queue
|
||||||
|
|
||||||
|
- Median: 3,750 ns
|
||||||
|
- p99: 5,833 ns
|
||||||
|
- Worst: 92,209 ns median across runs; one run reached 7,497,334 ns
|
||||||
|
- Decision: baseline
|
||||||
|
|
||||||
|
### Experiment 1: preallocated circular queue
|
||||||
|
|
||||||
|
- Hypothesis: rendering directly from preallocated storage will reduce callback tail latency by avoiding an array allocation and `Data.removeFirst()` on every callback.
|
||||||
|
- Median: 1,417 ns, down 62.2%
|
||||||
|
- p99: 1,917 ns, down 67.1%
|
||||||
|
- Worst: 32,709 ns median across runs, down 64.5%
|
||||||
|
- Decision: keep
|
||||||
@@ -0,0 +1,43 @@
|
|||||||
|
import { mkdtemp, rm } from "node:fs/promises"
|
||||||
|
import { tmpdir } from "node:os"
|
||||||
|
import { join } from "node:path"
|
||||||
|
|
||||||
|
const directory = await mkdtemp(join(tmpdir(), "opencode-voice-bench-"))
|
||||||
|
const binary = join(directory, "speaker-queue")
|
||||||
|
const source = Bun.fileURLToPath(new URL("../src/duplex-audio.swift", import.meta.url))
|
||||||
|
const compile = Bun.spawn(["swiftc", "-O", "-D", "QUEUE_BENCHMARK", source, "-o", binary], {
|
||||||
|
stdout: "ignore",
|
||||||
|
stderr: "pipe",
|
||||||
|
})
|
||||||
|
if ((await compile.exited) !== 0) {
|
||||||
|
console.error(await new Response(compile.stderr).text())
|
||||||
|
await rm(directory, { recursive: true })
|
||||||
|
process.exit(1)
|
||||||
|
}
|
||||||
|
|
||||||
|
const runs: Array<Record<string, number>> = []
|
||||||
|
for (let index = 0; index < 8; index++) {
|
||||||
|
const process = Bun.spawn([binary], { stdout: "pipe", stderr: "inherit" })
|
||||||
|
const output = await new Response(process.stdout).text()
|
||||||
|
if ((await process.exited) !== 0) throw new Error(`benchmark run ${index + 1} failed`)
|
||||||
|
if (index === 0) continue
|
||||||
|
runs.push(
|
||||||
|
Object.fromEntries(
|
||||||
|
output
|
||||||
|
.split("\n")
|
||||||
|
.filter((line) => line.startsWith("METRIC "))
|
||||||
|
.map((line) => {
|
||||||
|
const [name, value] = line.slice(7).split("=")
|
||||||
|
return [name!, Number(value)]
|
||||||
|
}),
|
||||||
|
),
|
||||||
|
)
|
||||||
|
}
|
||||||
|
await rm(directory, { recursive: true })
|
||||||
|
|
||||||
|
for (const name of ["speaker_queue_median_ns", "speaker_queue_p99_ns", "speaker_queue_worst_ns"]) {
|
||||||
|
const values = runs.map((run) => run[name]!).sort((a, b) => a - b)
|
||||||
|
const median = values[Math.floor(values.length / 2)]!
|
||||||
|
console.log(`${name}: median=${median} range=${values[0]}..${values.at(-1)}`)
|
||||||
|
console.log(`METRIC ${name}=${median}`)
|
||||||
|
}
|
||||||
@@ -0,0 +1,49 @@
|
|||||||
|
export type TextReveal = { readonly offset: number; readonly at: number }
|
||||||
|
|
||||||
|
const CONNECTION_TRANSITION_MS = 480
|
||||||
|
const REVEAL_DURATION_MS = 380
|
||||||
|
const REVEAL_STAGGER_MS = 32
|
||||||
|
const REVEAL_MAX_QUEUE_MS = 160
|
||||||
|
export const REVEAL_WORD_LIMIT = 24
|
||||||
|
|
||||||
|
export function textRevealOpacity(now: number, start: number) {
|
||||||
|
const progress = Math.max(0, Math.min(1, (now - start) / REVEAL_DURATION_MS))
|
||||||
|
const eased = 1 - (1 - progress) ** 2
|
||||||
|
return 0.16 + eased * 0.84
|
||||||
|
}
|
||||||
|
|
||||||
|
export function transcriptionPulse(now: number) {
|
||||||
|
return 0.5 + Math.sin(now / 220) * 0.5
|
||||||
|
}
|
||||||
|
|
||||||
|
export function connectionMeterLevels(now: number, level: number, connectedAt?: number, startedAt = 0) {
|
||||||
|
const transition = connectedAt === undefined ? 0 : Math.max(0, Math.min(1, (now - connectedAt) / CONNECTION_TRANSITION_MS))
|
||||||
|
const head = ((now - startedAt) / 180) % 4
|
||||||
|
return [0, 1, 2, 3].map((index) => {
|
||||||
|
const distance = Math.abs(index - head)
|
||||||
|
const wrappedDistance = Math.min(distance, 4 - distance)
|
||||||
|
const loading = 0.08 + Math.max(0, Math.cos((wrappedDistance / 1.5) * (Math.PI / 2))) * 0.92
|
||||||
|
const connected = level * (0.72 + Math.sin(now / 240 + index * 1.4) * 0.28)
|
||||||
|
return loading + (connected - loading) * transition
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
export function connectionTransitioning(now: number, connectedAt?: number) {
|
||||||
|
return connectedAt === undefined || now - connectedAt < CONNECTION_TRANSITION_MS
|
||||||
|
}
|
||||||
|
|
||||||
|
export function scheduleTextReveal(previous: string, delta: string, now: number, revealAt: number) {
|
||||||
|
const text = previous + delta
|
||||||
|
const offsets = Array.from(delta.matchAll(/\S+/g))
|
||||||
|
.map((match) => previous.length + match.index)
|
||||||
|
.filter((offset) => offset === 0 || /\s/.test(text[offset - 1] ?? ""))
|
||||||
|
.slice(-REVEAL_WORD_LIMIT)
|
||||||
|
const start = Math.min(Math.max(revealAt, now), now + REVEAL_MAX_QUEUE_MS)
|
||||||
|
const reveals = offsets.map((offset, word) => ({ offset, at: start + word * REVEAL_STAGGER_MS }))
|
||||||
|
const nextRevealAt = reveals.length === 0 ? revealAt : reveals.at(-1)!.at + REVEAL_STAGGER_MS
|
||||||
|
return {
|
||||||
|
reveals,
|
||||||
|
nextRevealAt,
|
||||||
|
animationEndsAt: reveals.length === 0 ? now : reveals.at(-1)!.at + REVEAL_DURATION_MS,
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,136 @@
|
|||||||
|
import { mkdir } from "node:fs/promises"
|
||||||
|
import { PCM_SAMPLE_RATE } from "./pcm"
|
||||||
|
|
||||||
|
export type AudioDevice = {
|
||||||
|
readonly fullDuplex: boolean
|
||||||
|
readonly mode: string
|
||||||
|
start(onInput: (audio: Buffer) => void, onMeta: (text: string) => void): Promise<void>
|
||||||
|
write(audio: Buffer): void
|
||||||
|
flush(): void
|
||||||
|
close(): void
|
||||||
|
}
|
||||||
|
|
||||||
|
export async function createAudioDevice(options: {
|
||||||
|
readonly duplex: boolean
|
||||||
|
readonly speakers: boolean
|
||||||
|
readonly debug: boolean
|
||||||
|
}): Promise<AudioDevice> {
|
||||||
|
const helper = await prepareHelper()
|
||||||
|
if (helper) return appleDevice(helper, options)
|
||||||
|
return soxDevice(options)
|
||||||
|
}
|
||||||
|
|
||||||
|
function appleDevice(
|
||||||
|
binary: string,
|
||||||
|
options: { readonly duplex: boolean; readonly speakers: boolean },
|
||||||
|
): AudioDevice {
|
||||||
|
let process: ReturnType<typeof Bun.spawn> | undefined
|
||||||
|
let sink: import("bun").FileSink | undefined
|
||||||
|
return {
|
||||||
|
fullDuplex: options.speakers || options.duplex,
|
||||||
|
mode: options.speakers ? "duplex+aec" : options.duplex ? "duplex" : "half-duplex",
|
||||||
|
async start(onInput, onMeta) {
|
||||||
|
if (process) return
|
||||||
|
const child = Bun.spawn([binary, ...(options.speakers ? ["--aec"] : [])], {
|
||||||
|
stdin: "pipe",
|
||||||
|
stdout: "pipe",
|
||||||
|
stderr: "pipe",
|
||||||
|
})
|
||||||
|
process = child
|
||||||
|
sink = child.stdin
|
||||||
|
void forwardLines(child.stderr, onMeta)
|
||||||
|
for await (const chunk of child.stdout) onInput(Buffer.from(chunk))
|
||||||
|
process = undefined
|
||||||
|
sink = undefined
|
||||||
|
},
|
||||||
|
write(audio) {
|
||||||
|
void sink?.write(audio)
|
||||||
|
void sink?.flush()
|
||||||
|
},
|
||||||
|
flush() {
|
||||||
|
process?.kill("SIGUSR1")
|
||||||
|
},
|
||||||
|
close() {
|
||||||
|
process?.kill()
|
||||||
|
process = undefined
|
||||||
|
sink = undefined
|
||||||
|
},
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
function soxDevice(options: { readonly duplex: boolean; readonly debug: boolean }): AudioDevice {
|
||||||
|
const format = [
|
||||||
|
"-q",
|
||||||
|
"-t",
|
||||||
|
"raw",
|
||||||
|
"-r",
|
||||||
|
String(PCM_SAMPLE_RATE),
|
||||||
|
"-e",
|
||||||
|
"signed-integer",
|
||||||
|
"-b",
|
||||||
|
"16",
|
||||||
|
"-c",
|
||||||
|
"1",
|
||||||
|
]
|
||||||
|
let recorder: ReturnType<typeof Bun.spawn> | undefined
|
||||||
|
let player: ReturnType<typeof Bun.spawn> | undefined
|
||||||
|
let sink: import("bun").FileSink | undefined
|
||||||
|
let onMeta: ((text: string) => void) | undefined
|
||||||
|
return {
|
||||||
|
fullDuplex: options.duplex,
|
||||||
|
mode: options.duplex ? "duplex (sox)" : "half-duplex (sox)",
|
||||||
|
async start(onInput, report) {
|
||||||
|
if (recorder) return
|
||||||
|
onMeta = report
|
||||||
|
const child = Bun.spawn(["rec", ...format, "-"], { stdout: "pipe", stderr: "ignore" })
|
||||||
|
recorder = child
|
||||||
|
for await (const chunk of child.stdout) onInput(Buffer.from(chunk))
|
||||||
|
recorder = undefined
|
||||||
|
},
|
||||||
|
write(audio) {
|
||||||
|
if (!player) {
|
||||||
|
const child = Bun.spawn(["play", ...format, "-"], { stdin: "pipe", stderr: "ignore" })
|
||||||
|
player = child
|
||||||
|
sink = child.stdin
|
||||||
|
if (options.debug) void child.exited.then((code) => onMeta?.(`[debug] play exited (${code})`))
|
||||||
|
}
|
||||||
|
void sink?.write(audio)
|
||||||
|
void sink?.flush()
|
||||||
|
},
|
||||||
|
flush() {
|
||||||
|
player?.kill()
|
||||||
|
player = undefined
|
||||||
|
sink = undefined
|
||||||
|
},
|
||||||
|
close() {
|
||||||
|
recorder?.kill()
|
||||||
|
player?.kill()
|
||||||
|
recorder = undefined
|
||||||
|
player = undefined
|
||||||
|
sink = undefined
|
||||||
|
},
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
async function forwardLines(stream: ReadableStream<Uint8Array>, onLine: (line: string) => void) {
|
||||||
|
const decoder = new TextDecoder()
|
||||||
|
let remainder = ""
|
||||||
|
for await (const chunk of stream) {
|
||||||
|
const lines = (remainder + decoder.decode(chunk, { stream: true })).split("\n")
|
||||||
|
remainder = lines.pop() ?? ""
|
||||||
|
lines.filter((line) => line.trim()).forEach((line) => onLine(line.trim()))
|
||||||
|
}
|
||||||
|
remainder += decoder.decode()
|
||||||
|
if (remainder.trim()) onLine(remainder.trim())
|
||||||
|
}
|
||||||
|
|
||||||
|
async function prepareHelper() {
|
||||||
|
if (process.platform !== "darwin") return undefined
|
||||||
|
const source = Bun.fileURLToPath(new URL("./duplex-audio.swift", import.meta.url))
|
||||||
|
const binary = Bun.fileURLToPath(new URL("../.build/duplex-audio", import.meta.url))
|
||||||
|
if ((await Bun.file(binary).exists()) && Bun.file(binary).lastModified > Bun.file(source).lastModified) return binary
|
||||||
|
await mkdir(Bun.fileURLToPath(new URL("../.build", import.meta.url)), { recursive: true })
|
||||||
|
const compile = Bun.spawn(["swiftc", "-O", source, "-o", binary], { stdout: "ignore", stderr: "pipe" })
|
||||||
|
if ((await compile.exited) === 0) return binary
|
||||||
|
return undefined
|
||||||
|
}
|
||||||
@@ -0,0 +1,41 @@
|
|||||||
|
import { PCM_BYTES_PER_MS } from "./pcm"
|
||||||
|
|
||||||
|
export type PlaybackChunk = {
|
||||||
|
readonly bytes: Buffer
|
||||||
|
readonly gapMs: number
|
||||||
|
}
|
||||||
|
|
||||||
|
export class AudioJitterBuffer {
|
||||||
|
private readonly pending: PlaybackChunk[] = []
|
||||||
|
private durationMs = 0
|
||||||
|
private started = false
|
||||||
|
|
||||||
|
constructor(private readonly targetMs = 500) {}
|
||||||
|
|
||||||
|
push(chunk: PlaybackChunk): ReadonlyArray<PlaybackChunk> {
|
||||||
|
if (this.started) return [chunk]
|
||||||
|
this.pending.push(chunk)
|
||||||
|
this.durationMs += chunk.gapMs + chunk.bytes.length / PCM_BYTES_PER_MS
|
||||||
|
if (this.durationMs < this.targetMs) return []
|
||||||
|
this.started = true
|
||||||
|
return this.drain()
|
||||||
|
}
|
||||||
|
|
||||||
|
finish(): ReadonlyArray<PlaybackChunk> {
|
||||||
|
if (this.pending.length === 0) return []
|
||||||
|
this.started = true
|
||||||
|
return this.drain()
|
||||||
|
}
|
||||||
|
|
||||||
|
reset() {
|
||||||
|
this.pending.length = 0
|
||||||
|
this.durationMs = 0
|
||||||
|
this.started = false
|
||||||
|
}
|
||||||
|
|
||||||
|
private drain() {
|
||||||
|
const chunks = this.pending.splice(0)
|
||||||
|
this.durationMs = 0
|
||||||
|
return chunks
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,217 @@
|
|||||||
|
import { createAudioDevice, type AudioDevice } from "./audio-device"
|
||||||
|
import { AudioJitterBuffer, type PlaybackChunk } from "./audio-jitter-buffer"
|
||||||
|
import { pcmLevel, PCM_BYTES_PER_MS, PCM_METER_FRAME_MS } from "./pcm"
|
||||||
|
import type { VoiceAudioTimeline } from "./protocol"
|
||||||
|
|
||||||
|
export type AudioEvent =
|
||||||
|
| { readonly type: "input"; readonly audio: Buffer; readonly level: number }
|
||||||
|
| { readonly type: "user.started" }
|
||||||
|
| { readonly type: "user.stopped" }
|
||||||
|
| { readonly type: "user.reset" }
|
||||||
|
| { readonly type: "assistant.level"; readonly level: number; readonly durationMs: number }
|
||||||
|
| { readonly type: "status"; readonly audio: string }
|
||||||
|
| { readonly type: "meta"; readonly text: string }
|
||||||
|
|
||||||
|
export type PlaybackOutcome = "played" | "interrupted" | "inaudible"
|
||||||
|
|
||||||
|
export type AudioSession = {
|
||||||
|
readonly fullDuplex: boolean
|
||||||
|
readonly microphoneMuted: boolean
|
||||||
|
readonly speakerMuted: boolean
|
||||||
|
start(emit: (event: AudioEvent) => void): Promise<void>
|
||||||
|
toggleMicrophone(): boolean
|
||||||
|
toggleSpeaker(): boolean
|
||||||
|
isPlaying(): boolean
|
||||||
|
play(bytes: Buffer, timeline?: VoiceAudioTimeline): void
|
||||||
|
finishPlayback(): Promise<PlaybackOutcome>
|
||||||
|
flushPlayback(): void
|
||||||
|
noteUserCommitted(): void
|
||||||
|
noteUserTranscript(final: boolean): void
|
||||||
|
close(): void
|
||||||
|
}
|
||||||
|
|
||||||
|
export async function createAudioSession(options: {
|
||||||
|
readonly duplex: boolean
|
||||||
|
readonly speakers: boolean
|
||||||
|
readonly inputActivity: "local" | "server"
|
||||||
|
readonly debug: boolean
|
||||||
|
readonly device?: AudioDevice
|
||||||
|
readonly trace?: (event: string, data?: Record<string, unknown>) => void
|
||||||
|
}): Promise<AudioSession> {
|
||||||
|
const device = options.device ?? (await createAudioDevice(options))
|
||||||
|
const fullDuplex = device.fullDuplex
|
||||||
|
const playbackBuffer = new AudioJitterBuffer()
|
||||||
|
let emit: ((event: AudioEvent) => void) | undefined
|
||||||
|
let microphoneMuted = false
|
||||||
|
let speakerMuted = false
|
||||||
|
let playbackEndsAt = 0
|
||||||
|
let playbackAudible = false
|
||||||
|
let playbackDoneTimer: ReturnType<typeof setTimeout> | undefined
|
||||||
|
let playbackCompletion: PromiseWithResolvers<PlaybackOutcome> | undefined
|
||||||
|
let outputTimelineEnd: number | undefined
|
||||||
|
let userFinalizedAt = 0
|
||||||
|
let userSpeechTimer: ReturnType<typeof setTimeout> | undefined
|
||||||
|
let userDraftTimer: ReturnType<typeof setTimeout> | undefined
|
||||||
|
let microphoneStarted = false
|
||||||
|
let closed = false
|
||||||
|
|
||||||
|
const isPlaying = () => Date.now() < playbackEndsAt
|
||||||
|
|
||||||
|
const settlePlayback = (outcome: PlaybackOutcome) => {
|
||||||
|
if (playbackDoneTimer) clearTimeout(playbackDoneTimer)
|
||||||
|
playbackDoneTimer = undefined
|
||||||
|
const completion = playbackCompletion
|
||||||
|
playbackCompletion = undefined
|
||||||
|
playbackEndsAt = 0
|
||||||
|
playbackAudible = false
|
||||||
|
playbackBuffer.reset()
|
||||||
|
outputTimelineEnd = undefined
|
||||||
|
completion?.resolve(outcome)
|
||||||
|
}
|
||||||
|
|
||||||
|
const flushPlayback = () => {
|
||||||
|
device.flush()
|
||||||
|
settlePlayback("interrupted")
|
||||||
|
}
|
||||||
|
|
||||||
|
const observeUserAudio = (level: number) => {
|
||||||
|
if (options.inputActivity !== "local" || level < 0.2 || Date.now() - userFinalizedAt < 500) return
|
||||||
|
if (userDraftTimer) clearTimeout(userDraftTimer)
|
||||||
|
userDraftTimer = undefined
|
||||||
|
emit?.({ type: "user.started" })
|
||||||
|
outputTimelineEnd = undefined
|
||||||
|
if (userSpeechTimer) clearTimeout(userSpeechTimer)
|
||||||
|
userSpeechTimer = setTimeout(() => {
|
||||||
|
userSpeechTimer = undefined
|
||||||
|
emit?.({ type: "user.stopped" })
|
||||||
|
userDraftTimer = setTimeout(() => {
|
||||||
|
userDraftTimer = undefined
|
||||||
|
emit?.({ type: "user.reset" })
|
||||||
|
}, 1_200)
|
||||||
|
}, 500)
|
||||||
|
}
|
||||||
|
|
||||||
|
const publishInput = (bytes: Buffer) => {
|
||||||
|
const level = pcmLevel(bytes)
|
||||||
|
emit?.({ type: "input", audio: bytes, level })
|
||||||
|
observeUserAudio(level)
|
||||||
|
}
|
||||||
|
|
||||||
|
const start = async (listener: (event: AudioEvent) => void) => {
|
||||||
|
if (microphoneStarted || closed) return
|
||||||
|
emit = listener
|
||||||
|
microphoneStarted = true
|
||||||
|
emit({ type: "status", audio: device.mode })
|
||||||
|
emit({ type: "meta", text: "mic live - start talking" })
|
||||||
|
if (!fullDuplex)
|
||||||
|
emit({ type: "meta", text: "mic mutes while the assistant speaks; press Esc to interrupt" })
|
||||||
|
await device.start(
|
||||||
|
(chunk) => {
|
||||||
|
if (closed || microphoneMuted || (!fullDuplex && Date.now() < playbackEndsAt + 300)) return
|
||||||
|
publishInput(chunk)
|
||||||
|
},
|
||||||
|
(text) => {
|
||||||
|
emit?.({ type: "meta", text })
|
||||||
|
options.trace?.("audio.helper", { message: text })
|
||||||
|
},
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
const writeAudio = (chunk: PlaybackChunk) => {
|
||||||
|
playbackAudible = true
|
||||||
|
if (chunk.gapMs > 0) {
|
||||||
|
emit?.({ type: "assistant.level", level: 0, durationMs: chunk.gapMs })
|
||||||
|
device.write(Buffer.alloc(Math.round(chunk.gapMs * PCM_BYTES_PER_MS)))
|
||||||
|
}
|
||||||
|
const meterBytes = PCM_METER_FRAME_MS * PCM_BYTES_PER_MS
|
||||||
|
for (let offset = 0; offset < chunk.bytes.length; offset += meterBytes) {
|
||||||
|
const window = chunk.bytes.subarray(offset, offset + meterBytes)
|
||||||
|
emit?.({ type: "assistant.level", level: pcmLevel(window), durationMs: window.length / PCM_BYTES_PER_MS })
|
||||||
|
}
|
||||||
|
device.write(chunk.bytes)
|
||||||
|
playbackEndsAt = Math.max(playbackEndsAt, Date.now()) + chunk.gapMs + chunk.bytes.length / PCM_BYTES_PER_MS
|
||||||
|
}
|
||||||
|
|
||||||
|
const schedulePlaybackCompletion = () => {
|
||||||
|
if (!playbackCompletion) return
|
||||||
|
if (!playbackAudible) return settlePlayback("inaudible")
|
||||||
|
if (playbackDoneTimer) clearTimeout(playbackDoneTimer)
|
||||||
|
playbackDoneTimer = setTimeout(
|
||||||
|
() => settlePlayback("played"),
|
||||||
|
Math.max(0, playbackEndsAt - Date.now()) + 180,
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
return {
|
||||||
|
fullDuplex,
|
||||||
|
get microphoneMuted() {
|
||||||
|
return microphoneMuted
|
||||||
|
},
|
||||||
|
get speakerMuted() {
|
||||||
|
return speakerMuted
|
||||||
|
},
|
||||||
|
start,
|
||||||
|
toggleMicrophone() {
|
||||||
|
microphoneMuted = !microphoneMuted
|
||||||
|
if (userSpeechTimer) clearTimeout(userSpeechTimer)
|
||||||
|
userSpeechTimer = undefined
|
||||||
|
emit?.({ type: "user.stopped" })
|
||||||
|
if (microphoneMuted) emit?.({ type: "user.reset" })
|
||||||
|
return microphoneMuted
|
||||||
|
},
|
||||||
|
toggleSpeaker() {
|
||||||
|
speakerMuted = !speakerMuted
|
||||||
|
if (speakerMuted) flushPlayback()
|
||||||
|
return speakerMuted
|
||||||
|
},
|
||||||
|
isPlaying,
|
||||||
|
play(bytes, timeline) {
|
||||||
|
if (speakerMuted || closed) return
|
||||||
|
const timelineGap = timeline && outputTimelineEnd !== undefined ? Math.max(0, timeline.startMs - outputTimelineEnd) : 0
|
||||||
|
const gapMs = timelineGap < 2_000 ? timelineGap : 0
|
||||||
|
if (options.debug)
|
||||||
|
options.trace?.("audio.output", {
|
||||||
|
bytes: bytes.length,
|
||||||
|
durationMs: bytes.length / PCM_BYTES_PER_MS,
|
||||||
|
level: pcmLevel(bytes),
|
||||||
|
timelineStart: timeline?.startMs,
|
||||||
|
timelineEnd: timeline?.endMs,
|
||||||
|
timelineGap,
|
||||||
|
gapMs,
|
||||||
|
})
|
||||||
|
outputTimelineEnd = timeline?.endMs
|
||||||
|
playbackBuffer.push({ bytes, gapMs }).forEach(writeAudio)
|
||||||
|
schedulePlaybackCompletion()
|
||||||
|
},
|
||||||
|
finishPlayback() {
|
||||||
|
if (playbackCompletion) return playbackCompletion.promise
|
||||||
|
playbackCompletion = Promise.withResolvers<PlaybackOutcome>()
|
||||||
|
playbackBuffer.finish().forEach(writeAudio)
|
||||||
|
schedulePlaybackCompletion()
|
||||||
|
return playbackCompletion?.promise ?? Promise.resolve("inaudible")
|
||||||
|
},
|
||||||
|
flushPlayback,
|
||||||
|
noteUserCommitted() {
|
||||||
|
if (userDraftTimer) clearTimeout(userDraftTimer)
|
||||||
|
userDraftTimer = undefined
|
||||||
|
},
|
||||||
|
noteUserTranscript(final) {
|
||||||
|
if (userDraftTimer) clearTimeout(userDraftTimer)
|
||||||
|
userDraftTimer = undefined
|
||||||
|
if (!final) return
|
||||||
|
if (userSpeechTimer) clearTimeout(userSpeechTimer)
|
||||||
|
userSpeechTimer = undefined
|
||||||
|
userFinalizedAt = Date.now()
|
||||||
|
emit?.({ type: "user.stopped" })
|
||||||
|
},
|
||||||
|
close() {
|
||||||
|
if (closed) return
|
||||||
|
closed = true
|
||||||
|
device.close()
|
||||||
|
if (userSpeechTimer) clearTimeout(userSpeechTimer)
|
||||||
|
if (userDraftTimer) clearTimeout(userDraftTimer)
|
||||||
|
settlePlayback("interrupted")
|
||||||
|
emit = undefined
|
||||||
|
},
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,109 @@
|
|||||||
|
import { mkdir, readdir, rename, rm } from "node:fs/promises"
|
||||||
|
import { homedir } from "node:os"
|
||||||
|
import { dirname, join } from "node:path"
|
||||||
|
import { Option, Schema } from "effect"
|
||||||
|
import { OpenCodeNotification } from "./opencode-notification"
|
||||||
|
import { promptKey, type PromptHandle } from "./prompt-handle"
|
||||||
|
|
||||||
|
export type CompletionHandle = PromptHandle
|
||||||
|
export type StoredCompletion =
|
||||||
|
| { readonly status: "admitting"; readonly handle: CompletionHandle; readonly text: string }
|
||||||
|
| { readonly status: "pending"; readonly handle: CompletionHandle }
|
||||||
|
| {
|
||||||
|
readonly status: "completed"
|
||||||
|
readonly handle: CompletionHandle
|
||||||
|
readonly notification: OpenCodeNotification
|
||||||
|
}
|
||||||
|
|
||||||
|
const CompletionHandle = Schema.Struct({ sessionID: Schema.String, promptID: Schema.String })
|
||||||
|
const StoredCompletion = Schema.Union([
|
||||||
|
Schema.Struct({ status: Schema.Literal("admitting"), handle: CompletionHandle, text: Schema.String }),
|
||||||
|
Schema.Struct({ status: Schema.Literal("pending"), handle: CompletionHandle }),
|
||||||
|
Schema.Struct({ status: Schema.Literal("completed"), handle: CompletionHandle, notification: OpenCodeNotification }),
|
||||||
|
])
|
||||||
|
const StoredCompletions = Schema.Array(StoredCompletion)
|
||||||
|
|
||||||
|
export async function createCompletionStore(
|
||||||
|
path = join(process.env["XDG_STATE_HOME"] ?? join(homedir(), ".local", "state"), "opencode", "voice-prompts.json"),
|
||||||
|
) {
|
||||||
|
await mkdir(dirname(path), { recursive: true })
|
||||||
|
const directory = `${path}.d`
|
||||||
|
await mkdir(directory, { recursive: true })
|
||||||
|
const files = (await readdir(directory)).filter((name) => name.endsWith(".json"))
|
||||||
|
const decoded = files.length > 0
|
||||||
|
? await Promise.all(
|
||||||
|
files.map(async (name) => {
|
||||||
|
const entry = Option.getOrUndefined(
|
||||||
|
Schema.decodeUnknownOption(StoredCompletion)(await Bun.file(join(directory, name)).json()),
|
||||||
|
)
|
||||||
|
if (!entry) throw new Error(`Invalid voice completion entry: ${join(directory, name)}`)
|
||||||
|
return entry
|
||||||
|
}),
|
||||||
|
)
|
||||||
|
: (await Bun.file(path).exists())
|
||||||
|
? Option.getOrUndefined(Schema.decodeUnknownOption(StoredCompletions)(await Bun.file(path).json()))
|
||||||
|
: []
|
||||||
|
if (!decoded) throw new Error(`Invalid voice completion store: ${path}`)
|
||||||
|
const entries = new Map(decoded.map((entry) => [promptKey(entry.handle), entry]))
|
||||||
|
let writes = Promise.resolve()
|
||||||
|
|
||||||
|
const enqueue = (operation: () => Promise<void>) => {
|
||||||
|
const result = writes.then(operation)
|
||||||
|
writes = result.catch(() => {})
|
||||||
|
return result
|
||||||
|
}
|
||||||
|
|
||||||
|
const save = (entry: StoredCompletion) => {
|
||||||
|
return enqueue(async () => {
|
||||||
|
const target = completionPath(directory, entry.handle)
|
||||||
|
const temporary = `${target}.${process.pid}.${crypto.randomUUID()}.tmp`
|
||||||
|
await Bun.write(temporary, JSON.stringify(entry))
|
||||||
|
await rename(temporary, target)
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
if (files.length === 0 && decoded.length > 0) {
|
||||||
|
await Promise.all(decoded.map(save))
|
||||||
|
await rm(path, { force: true })
|
||||||
|
}
|
||||||
|
if (files.length > 0) {
|
||||||
|
const existing = new Set(files)
|
||||||
|
await Promise.all(decoded.filter((entry) => !existing.has(completionFilename(entry.handle))).map(save))
|
||||||
|
const current = new Set(decoded.map((entry) => completionFilename(entry.handle)))
|
||||||
|
await Promise.all(files.filter((name) => !current.has(name)).map((name) => rm(join(directory, name), { force: true })))
|
||||||
|
}
|
||||||
|
|
||||||
|
return {
|
||||||
|
entries: () => [...entries.entries()].toSorted(([left], [right]) => left.localeCompare(right)).map(([, entry]) => entry),
|
||||||
|
admitting(handle: CompletionHandle, text: string) {
|
||||||
|
const entry = { status: "admitting", handle, text } as const
|
||||||
|
entries.set(promptKey(handle), entry)
|
||||||
|
return save(entry)
|
||||||
|
},
|
||||||
|
pending(handle: CompletionHandle) {
|
||||||
|
const entry = { status: "pending", handle } as const
|
||||||
|
entries.set(promptKey(handle), entry)
|
||||||
|
return save(entry)
|
||||||
|
},
|
||||||
|
completed(handle: CompletionHandle, notification: OpenCodeNotification) {
|
||||||
|
const entry = { status: "completed", handle, notification } as const
|
||||||
|
entries.set(promptKey(handle), entry)
|
||||||
|
return save(entry)
|
||||||
|
},
|
||||||
|
remove(handle: CompletionHandle) {
|
||||||
|
if (!entries.delete(promptKey(handle))) return writes
|
||||||
|
return enqueue(() => rm(completionPath(directory, handle), { force: true }))
|
||||||
|
},
|
||||||
|
close: () => writes,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
export type CompletionStore = Awaited<ReturnType<typeof createCompletionStore>>
|
||||||
|
|
||||||
|
function completionFilename(handle: CompletionHandle) {
|
||||||
|
return `${encodeURIComponent(promptKey(handle))}.json`
|
||||||
|
}
|
||||||
|
|
||||||
|
function completionPath(directory: string, handle: CompletionHandle) {
|
||||||
|
return join(directory, completionFilename(handle))
|
||||||
|
}
|
||||||
@@ -0,0 +1,288 @@
|
|||||||
|
// Full-duplex terminal audio bridge with Apple voice processing (AEC).
|
||||||
|
//
|
||||||
|
// stdin <- raw PCM16 mono 24kHz to play through the speakers
|
||||||
|
// stdout -> raw PCM16 mono 24kHz captured from the microphone
|
||||||
|
// SIGUSR1: drop any queued speaker audio (barge-in flush)
|
||||||
|
//
|
||||||
|
// Two independent engines: attaching a speaker source to a voice-processed
|
||||||
|
// engine silently kills its input tap, so playback runs on its own plain
|
||||||
|
// engine. Voice processing (echo cancellation) is attempted on the input
|
||||||
|
// engine and abandoned if the mic delivers nothing: on some devices
|
||||||
|
// (Bluetooth headsets mid-negotiation) the VP tap never fires. Without VP
|
||||||
|
// there is no echo cancellation, which is acceptable exactly when it happens:
|
||||||
|
// headphones have no echo path.
|
||||||
|
//
|
||||||
|
// Compiled on demand by spike.ts: swiftc -O duplex-audio.swift -o duplex-audio
|
||||||
|
|
||||||
|
import AVFoundation
|
||||||
|
|
||||||
|
let stderr = FileHandle.standardError
|
||||||
|
func log(_ message: String) { stderr.write(Data("[audio] \(message)\n".utf8)) }
|
||||||
|
|
||||||
|
final class SpeakerQueue {
|
||||||
|
private static let capacity = 24_000 * 2 * 30
|
||||||
|
private var data = [UInt8](repeating: 0, count: capacity)
|
||||||
|
private var readIndex = 0
|
||||||
|
private var writeIndex = 0
|
||||||
|
private var count = 0
|
||||||
|
private let lock = NSLock()
|
||||||
|
func push(_ chunk: Data) {
|
||||||
|
if chunk.isEmpty { return }
|
||||||
|
lock.lock()
|
||||||
|
defer { lock.unlock() }
|
||||||
|
chunk.withUnsafeBytes { source in
|
||||||
|
let skipped = max(0, source.count - Self.capacity)
|
||||||
|
let start = skipped + skipped % 2
|
||||||
|
let bytes = source.count - start
|
||||||
|
let overflow = max(0, count + bytes - Self.capacity)
|
||||||
|
let dropped = min(count, overflow + overflow % 2)
|
||||||
|
readIndex = (readIndex + dropped) % Self.capacity
|
||||||
|
count -= dropped
|
||||||
|
let first = min(bytes, Self.capacity - writeIndex)
|
||||||
|
data.withUnsafeMutableBytes { destination in
|
||||||
|
destination.baseAddress!.advanced(by: writeIndex).copyMemory(
|
||||||
|
from: source.baseAddress!.advanced(by: start), byteCount: first)
|
||||||
|
destination.baseAddress!.copyMemory(
|
||||||
|
from: source.baseAddress!.advanced(by: start + first), byteCount: bytes - first)
|
||||||
|
}
|
||||||
|
writeIndex = (writeIndex + bytes) % Self.capacity
|
||||||
|
count += bytes
|
||||||
|
}
|
||||||
|
}
|
||||||
|
func render(frames: Int, into output: UnsafeMutablePointer<Float>) {
|
||||||
|
lock.lock()
|
||||||
|
defer { lock.unlock() }
|
||||||
|
let samples = min(frames, count / 2)
|
||||||
|
for index in 0..<samples {
|
||||||
|
let low = UInt16(data[readIndex])
|
||||||
|
let high = UInt16(data[(readIndex + 1) % Self.capacity])
|
||||||
|
output[index] = Float(Int16(bitPattern: low | high << 8)) / 32768.0
|
||||||
|
readIndex = (readIndex + 2) % Self.capacity
|
||||||
|
}
|
||||||
|
count -= samples * 2
|
||||||
|
for index in samples..<frames { output[index] = 0 }
|
||||||
|
}
|
||||||
|
func flush() {
|
||||||
|
lock.lock()
|
||||||
|
readIndex = writeIndex
|
||||||
|
count = 0
|
||||||
|
lock.unlock()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#if QUEUE_BENCHMARK
|
||||||
|
let correctnessQueue = SpeakerQueue()
|
||||||
|
correctnessQueue.push(Data([0x00, 0x80, 0xff, 0x7f]))
|
||||||
|
var correctnessOutput = [Float](repeating: 1, count: 3)
|
||||||
|
correctnessOutput.withUnsafeMutableBufferPointer {
|
||||||
|
correctnessQueue.render(frames: 3, into: $0.baseAddress!)
|
||||||
|
}
|
||||||
|
precondition(correctnessOutput[0] == -1)
|
||||||
|
precondition(abs(correctnessOutput[1] - Float(Int16.max) / 32768) < 0.000_001)
|
||||||
|
precondition(correctnessOutput[2] == 0)
|
||||||
|
correctnessQueue.push(Data([1, 0]))
|
||||||
|
correctnessQueue.flush()
|
||||||
|
correctnessOutput.withUnsafeMutableBufferPointer {
|
||||||
|
correctnessQueue.render(frames: 3, into: $0.baseAddress!)
|
||||||
|
}
|
||||||
|
precondition(correctnessOutput.allSatisfy { $0 == 0 })
|
||||||
|
|
||||||
|
let benchmarkQueue = SpeakerQueue()
|
||||||
|
let benchmarkFrames = 512
|
||||||
|
let benchmarkChunk = Data(repeating: 1, count: benchmarkFrames * 2)
|
||||||
|
var benchmarkOutput = [Float](repeating: 0, count: benchmarkFrames)
|
||||||
|
for _ in 0..<48 { benchmarkQueue.push(benchmarkChunk) }
|
||||||
|
for _ in 0..<2_000 {
|
||||||
|
benchmarkOutput.withUnsafeMutableBufferPointer {
|
||||||
|
benchmarkQueue.render(frames: benchmarkFrames, into: $0.baseAddress!)
|
||||||
|
}
|
||||||
|
benchmarkQueue.push(benchmarkChunk)
|
||||||
|
}
|
||||||
|
var benchmarkDurations = [UInt64]()
|
||||||
|
benchmarkDurations.reserveCapacity(30_000)
|
||||||
|
var benchmarkChecksum: Float = 0
|
||||||
|
for _ in 0..<30_000 {
|
||||||
|
let started = DispatchTime.now().uptimeNanoseconds
|
||||||
|
benchmarkOutput.withUnsafeMutableBufferPointer {
|
||||||
|
benchmarkQueue.render(frames: benchmarkFrames, into: $0.baseAddress!)
|
||||||
|
}
|
||||||
|
benchmarkQueue.push(benchmarkChunk)
|
||||||
|
benchmarkDurations.append(DispatchTime.now().uptimeNanoseconds - started)
|
||||||
|
benchmarkChecksum += benchmarkOutput[0]
|
||||||
|
}
|
||||||
|
benchmarkDurations.sort()
|
||||||
|
print("METRIC speaker_queue_median_ns=\(benchmarkDurations[benchmarkDurations.count / 2])")
|
||||||
|
print("METRIC speaker_queue_p99_ns=\(benchmarkDurations[benchmarkDurations.count * 99 / 100])")
|
||||||
|
print("METRIC speaker_queue_worst_ns=\(benchmarkDurations.last!)")
|
||||||
|
print("CHECKSUM \(benchmarkChecksum)")
|
||||||
|
exit(0)
|
||||||
|
#endif
|
||||||
|
|
||||||
|
let queue = SpeakerQueue()
|
||||||
|
let playFormat = AVAudioFormat(commonFormat: .pcmFormatFloat32, sampleRate: 24000, channels: 1, interleaved: false)!
|
||||||
|
let captureFormat = AVAudioFormat(commonFormat: .pcmFormatInt16, sampleRate: 24000, channels: 1, interleaved: true)!
|
||||||
|
var rebuildScheduled = false
|
||||||
|
var ignoreRouteChangesUntil = Date.distantPast
|
||||||
|
|
||||||
|
func scheduleRebuild(_ message: String, after delay: TimeInterval = 0.5) {
|
||||||
|
if rebuildScheduled { return }
|
||||||
|
rebuildScheduled = true
|
||||||
|
DispatchQueue.main.asyncAfter(deadline: .now() + delay) {
|
||||||
|
rebuildScheduled = false
|
||||||
|
// Starting an engine emits configuration changes of its own. AirPods can
|
||||||
|
// otherwise turn one real route change into a perpetual rebuild loop.
|
||||||
|
ignoreRouteChangesUntil = Date().addingTimeInterval(1.5)
|
||||||
|
log(message)
|
||||||
|
startInput(voiceProcessing: useAEC)
|
||||||
|
startOutput()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Speaker: its own engine, pulling PCM16 from the stdin-fed queue.
|
||||||
|
var outputEngine: AVAudioEngine?
|
||||||
|
|
||||||
|
func startOutput() {
|
||||||
|
outputEngine?.stop()
|
||||||
|
let engine = AVAudioEngine()
|
||||||
|
outputEngine = engine
|
||||||
|
let source = AVAudioSourceNode(format: playFormat) { _, _, frameCount, audioBufferList -> OSStatus in
|
||||||
|
let out = UnsafeMutableAudioBufferListPointer(audioBufferList)[0].mData!.assumingMemoryBound(to: Float.self)
|
||||||
|
queue.render(frames: Int(frameCount), into: out)
|
||||||
|
return noErr
|
||||||
|
}
|
||||||
|
engine.attach(source)
|
||||||
|
engine.connect(source, to: engine.mainMixerNode, format: playFormat)
|
||||||
|
do {
|
||||||
|
try engine.start()
|
||||||
|
let format = engine.outputNode.outputFormat(forBus: 0)
|
||||||
|
log("output active (\(Int(format.sampleRate))Hz, \(format.channelCount)ch)")
|
||||||
|
} catch {
|
||||||
|
log("output engine failed: \(error)")
|
||||||
|
scheduleRebuild("audio engines unavailable — retrying")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Microphone: take channel 0 of whatever the hardware provides, resample to
|
||||||
|
// 24kHz PCM16 for stdout. Channel extraction is manual because hardware
|
||||||
|
// channel counts vary wildly (1, 2, 3, 22...) and AVAudioConverter cannot
|
||||||
|
// downmix all of them.
|
||||||
|
var tapCount = 0
|
||||||
|
var inputEngine: AVAudioEngine?
|
||||||
|
var useAEC = CommandLine.arguments.contains("--aec")
|
||||||
|
|
||||||
|
func startInput(voiceProcessing: Bool) {
|
||||||
|
inputEngine?.stop()
|
||||||
|
let engine = AVAudioEngine()
|
||||||
|
inputEngine = engine
|
||||||
|
if voiceProcessing {
|
||||||
|
do {
|
||||||
|
try engine.inputNode.setVoiceProcessingEnabled(true)
|
||||||
|
} catch {
|
||||||
|
log("voice processing unavailable (\(error)) — no echo cancellation")
|
||||||
|
useAEC = false
|
||||||
|
return startInput(voiceProcessing: false)
|
||||||
|
}
|
||||||
|
if #available(macOS 14.0, *) {
|
||||||
|
engine.inputNode.voiceProcessingOtherAudioDuckingConfiguration = .init(
|
||||||
|
enableAdvancedDucking: false,
|
||||||
|
duckingLevel: .min
|
||||||
|
)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
let micFormat = engine.inputNode.outputFormat(forBus: 0)
|
||||||
|
guard micFormat.sampleRate > 0, micFormat.channelCount > 0, let monoFormat = AVAudioFormat(
|
||||||
|
commonFormat: .pcmFormatFloat32, sampleRate: micFormat.sampleRate, channels: 1, interleaved: false)
|
||||||
|
else {
|
||||||
|
log("microphone route is not ready — retrying")
|
||||||
|
scheduleRebuild("audio engines unavailable — retrying")
|
||||||
|
return
|
||||||
|
}
|
||||||
|
guard let converter = AVAudioConverter(from: monoFormat, to: captureFormat) else {
|
||||||
|
log("cannot convert \(micFormat.sampleRate)Hz to 24kHz — retrying")
|
||||||
|
scheduleRebuild("audio engines unavailable — retrying")
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
engine.inputNode.installTap(onBus: 0, bufferSize: 2400, format: micFormat) { buffer, _ in
|
||||||
|
tapCount += 1
|
||||||
|
if tapCount == 1 { log("mic active (echo cancellation \(voiceProcessing ? "on" : "off"))") }
|
||||||
|
guard let channel = buffer.floatChannelData?[0], buffer.frameLength > 0 else { return }
|
||||||
|
guard let mono = AVAudioPCMBuffer(pcmFormat: monoFormat, frameCapacity: buffer.frameLength) else { return }
|
||||||
|
memcpy(mono.floatChannelData![0], channel, Int(buffer.frameLength) * 4)
|
||||||
|
mono.frameLength = buffer.frameLength
|
||||||
|
|
||||||
|
let capacity = AVAudioFrameCount(Double(buffer.frameLength) * 24000.0 / micFormat.sampleRate) + 32
|
||||||
|
guard let converted = AVAudioPCMBuffer(pcmFormat: captureFormat, frameCapacity: capacity) else { return }
|
||||||
|
var consumed = false
|
||||||
|
converter.convert(to: converted, error: nil) { _, status in
|
||||||
|
if consumed {
|
||||||
|
status.pointee = .noDataNow
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
consumed = true
|
||||||
|
status.pointee = .haveData
|
||||||
|
return mono
|
||||||
|
}
|
||||||
|
guard converted.frameLength > 0, let out = converted.int16ChannelData?[0] else { return }
|
||||||
|
FileHandle.standardOutput.write(Data(bytes: out, count: Int(converted.frameLength) * 2))
|
||||||
|
}
|
||||||
|
|
||||||
|
do {
|
||||||
|
try engine.start()
|
||||||
|
log("input active (\(Int(micFormat.sampleRate))Hz, \(micFormat.channelCount)ch, echo cancellation \(voiceProcessing ? "on" : "off"))")
|
||||||
|
} catch {
|
||||||
|
if voiceProcessing {
|
||||||
|
log("input engine failed with voice processing (\(error)) — retrying without")
|
||||||
|
useAEC = false
|
||||||
|
return startInput(voiceProcessing: false)
|
||||||
|
}
|
||||||
|
log("input engine failed: \(error)")
|
||||||
|
scheduleRebuild("audio engines unavailable — retrying")
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
// Watchdog: on some devices the voice-processed tap simply never fires.
|
||||||
|
// Fall back to a plain tap; without VP the mic reliably delivers.
|
||||||
|
if voiceProcessing {
|
||||||
|
DispatchQueue.main.asyncAfter(deadline: .now() + 2.5) {
|
||||||
|
if tapCount > 0 { return }
|
||||||
|
log("voice-processed mic delivered nothing — restarting without echo cancellation")
|
||||||
|
useAEC = false
|
||||||
|
startInput(voiceProcessing: false)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Voice processing is opt-in (--aec): it is only needed on speakers, and on
|
||||||
|
// some machines (observed with Bluetooth headsets active) the VP engine binds
|
||||||
|
// to the wrong capture device entirely, delivering noise instead of the mic.
|
||||||
|
// Mic first: activating the mic flips Bluetooth headsets from music mode to
|
||||||
|
// headset mode, reconfiguring the output device. Starting output afterwards
|
||||||
|
// (and rebuilding on any route change below) keeps playback on the live device.
|
||||||
|
startInput(voiceProcessing: useAEC)
|
||||||
|
startOutput()
|
||||||
|
|
||||||
|
// Device switches (Bluetooth profile flips, headphones plugged/unplugged,
|
||||||
|
// default device changes) stop engines silently. Rebuild both, debounced.
|
||||||
|
NotificationCenter.default.addObserver(
|
||||||
|
forName: .AVAudioEngineConfigurationChange, object: nil, queue: .main
|
||||||
|
) { _ in
|
||||||
|
if Date() < ignoreRouteChangesUntil { return }
|
||||||
|
scheduleRebuild("audio route changed — rebuilding engines")
|
||||||
|
}
|
||||||
|
|
||||||
|
signal(SIGUSR1, SIG_IGN)
|
||||||
|
let flushSignal = DispatchSource.makeSignalSource(signal: SIGUSR1, queue: .main)
|
||||||
|
flushSignal.setEventHandler { queue.flush() }
|
||||||
|
flushSignal.resume()
|
||||||
|
|
||||||
|
DispatchQueue.global().async {
|
||||||
|
while true {
|
||||||
|
let chunk = FileHandle.standardInput.availableData
|
||||||
|
if chunk.isEmpty { exit(0) } // parent closed stdin
|
||||||
|
queue.push(chunk)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
RunLoop.main.run()
|
||||||
@@ -0,0 +1,48 @@
|
|||||||
|
import type { CompletionReceipt, OpenCodeAnnouncement } from "./opencode-notification"
|
||||||
|
import { promptKey } from "./prompt-handle"
|
||||||
|
|
||||||
|
export type RoutedAnnouncement = {
|
||||||
|
readonly announcement: OpenCodeAnnouncement
|
||||||
|
readonly promptID?: string
|
||||||
|
readonly delegationID?: string
|
||||||
|
}
|
||||||
|
|
||||||
|
export function createNotificationRouter(deliver: (announcement: RoutedAnnouncement) => void) {
|
||||||
|
const delegations = new Map<string, string>()
|
||||||
|
const deferred: OpenCodeAnnouncement[] = []
|
||||||
|
let pendingDelegatedTools = 0
|
||||||
|
|
||||||
|
const route = (announcement: OpenCodeAnnouncement) => {
|
||||||
|
const prompt = "prompt_id" in announcement.notification
|
||||||
|
? { sessionID: announcement.notification.session_id, promptID: announcement.notification.prompt_id }
|
||||||
|
: undefined
|
||||||
|
const promptID = prompt?.promptID
|
||||||
|
if (prompt && pendingDelegatedTools > 0 && !delegations.has(promptKey(prompt))) {
|
||||||
|
deferred.push(announcement)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
deliver({
|
||||||
|
announcement,
|
||||||
|
promptID,
|
||||||
|
delegationID: prompt ? delegations.get(promptKey(prompt)) : undefined,
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
return {
|
||||||
|
route,
|
||||||
|
beginDelegatedTool() {
|
||||||
|
pendingDelegatedTools += 1
|
||||||
|
},
|
||||||
|
finishDelegatedTool(
|
||||||
|
delegationID: string,
|
||||||
|
admittedPrompt?: { readonly sessionID: string; readonly promptID: string },
|
||||||
|
) {
|
||||||
|
if (admittedPrompt) delegations.set(promptKey(admittedPrompt), delegationID)
|
||||||
|
pendingDelegatedTools -= 1
|
||||||
|
if (pendingDelegatedTools === 0) deferred.splice(0).forEach(route)
|
||||||
|
},
|
||||||
|
acknowledged(receipt: CompletionReceipt) {
|
||||||
|
delegations.delete(promptKey(receipt))
|
||||||
|
},
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,97 @@
|
|||||||
|
import { Schema } from "effect"
|
||||||
|
import type { PromptHandle } from "./prompt-handle"
|
||||||
|
|
||||||
|
const Completed = Schema.Struct({
|
||||||
|
type: Schema.Literal("opencode.prompt.completed"),
|
||||||
|
session_id: Schema.String,
|
||||||
|
prompt_id: Schema.String,
|
||||||
|
status: Schema.Literals(["completed", "failed"]),
|
||||||
|
text: Schema.String,
|
||||||
|
error: Schema.optional(Schema.String),
|
||||||
|
})
|
||||||
|
|
||||||
|
const Failed = Schema.Struct({
|
||||||
|
type: Schema.Literal("opencode.prompt.failed"),
|
||||||
|
session_id: Schema.String,
|
||||||
|
prompt_id: Schema.String,
|
||||||
|
status: Schema.Literal("failed"),
|
||||||
|
error: Schema.String,
|
||||||
|
})
|
||||||
|
|
||||||
|
const PermissionBlocked = Schema.Struct({
|
||||||
|
type: Schema.Literal("opencode.prompt.blocked"),
|
||||||
|
prompt_id: Schema.String,
|
||||||
|
blocker: Schema.Literal("permission"),
|
||||||
|
session_id: Schema.String,
|
||||||
|
request_id: Schema.String,
|
||||||
|
action: Schema.String,
|
||||||
|
resources: Schema.Unknown,
|
||||||
|
})
|
||||||
|
|
||||||
|
const QuestionBlocked = Schema.Struct({
|
||||||
|
type: Schema.Literal("opencode.prompt.blocked"),
|
||||||
|
prompt_id: Schema.String,
|
||||||
|
blocker: Schema.Literal("question"),
|
||||||
|
session_id: Schema.String,
|
||||||
|
request_id: Schema.String,
|
||||||
|
questions: Schema.Unknown,
|
||||||
|
})
|
||||||
|
|
||||||
|
const FormBlocked = Schema.Struct({
|
||||||
|
type: Schema.Literal("opencode.prompt.blocked"),
|
||||||
|
prompt_id: Schema.String,
|
||||||
|
blocker: Schema.Literal("form"),
|
||||||
|
session_id: Schema.String,
|
||||||
|
form_id: Schema.String,
|
||||||
|
title: Schema.String,
|
||||||
|
fields: Schema.Unknown,
|
||||||
|
})
|
||||||
|
|
||||||
|
const EventsFailed = Schema.Struct({
|
||||||
|
type: Schema.Literal("opencode.events.failed"),
|
||||||
|
error: Schema.String,
|
||||||
|
})
|
||||||
|
|
||||||
|
export const OpenCodeNotification = Schema.Union([
|
||||||
|
Completed,
|
||||||
|
Failed,
|
||||||
|
PermissionBlocked,
|
||||||
|
QuestionBlocked,
|
||||||
|
FormBlocked,
|
||||||
|
EventsFailed,
|
||||||
|
])
|
||||||
|
export type OpenCodeNotification = typeof OpenCodeNotification.Type
|
||||||
|
export type OpenCodePromptBlocked = Extract<OpenCodeNotification, { readonly type: "opencode.prompt.blocked" }>
|
||||||
|
|
||||||
|
export type CompletionReceipt = PromptHandle
|
||||||
|
|
||||||
|
export type OpenCodeAnnouncement = {
|
||||||
|
readonly notification: OpenCodeNotification
|
||||||
|
readonly receipt?: CompletionReceipt
|
||||||
|
}
|
||||||
|
|
||||||
|
export function openCodeAnnouncementText(notification: OpenCodeNotification) {
|
||||||
|
const text = (() => {
|
||||||
|
switch (notification.type) {
|
||||||
|
case "opencode.prompt.completed":
|
||||||
|
return `OpenCode prompt ${notification.status}: ${notification.text}${notification.error ? ` Error: ${notification.error}` : ""}`
|
||||||
|
case "opencode.prompt.failed":
|
||||||
|
return `OpenCode prompt failed: ${notification.error}`
|
||||||
|
case "opencode.prompt.blocked":
|
||||||
|
if (notification.blocker === "permission")
|
||||||
|
return `OpenCode needs permission for ${notification.action}: ${compact(notification.resources)}`
|
||||||
|
if (notification.blocker === "question")
|
||||||
|
return `OpenCode needs the user to answer: ${compact(notification.questions)}`
|
||||||
|
return `OpenCode needs the user to complete the form “${notification.title}”: ${compact(notification.fields)}`
|
||||||
|
case "opencode.events.failed":
|
||||||
|
return `OpenCode event delivery failed: ${notification.error}`
|
||||||
|
}
|
||||||
|
return "OpenCode has an update."
|
||||||
|
})()
|
||||||
|
return text.length > 800 ? text.slice(0, 797) + "..." : text
|
||||||
|
}
|
||||||
|
|
||||||
|
function compact(value: unknown) {
|
||||||
|
if (typeof value === "string") return value
|
||||||
|
return JSON.stringify(value) ?? String(value)
|
||||||
|
}
|
||||||
@@ -0,0 +1,811 @@
|
|||||||
|
import { realpathSync } from "node:fs"
|
||||||
|
import { tmpdir } from "node:os"
|
||||||
|
import { basename, isAbsolute, relative } from "node:path"
|
||||||
|
import { ClientError, isSessionNotFoundError, OpenCode } from "@opencode-ai/client/promise"
|
||||||
|
import type { SessionMessageAssistant, SessionMessageUser, V2Event } from "@opencode-ai/client/promise"
|
||||||
|
import { Form, Question, SessionMessage } from "@opencode-ai/client/effect"
|
||||||
|
import { Context, Effect, FiberMap, Layer, ManagedRuntime, Option, Schema, Stream } from "effect"
|
||||||
|
import { createCompletionStore, type CompletionStore } from "./completion-store"
|
||||||
|
import type {
|
||||||
|
CompletionReceipt,
|
||||||
|
OpenCodeAnnouncement,
|
||||||
|
OpenCodeNotification,
|
||||||
|
OpenCodePromptBlocked,
|
||||||
|
} from "./opencode-notification"
|
||||||
|
import type { VoiceTool, VoiceToolExecution } from "./protocol"
|
||||||
|
import type { PromptHandle } from "./prompt-handle"
|
||||||
|
|
||||||
|
type Client = ReturnType<typeof OpenCode.make>
|
||||||
|
const AdmittedPrompt = Symbol("AdmittedPrompt")
|
||||||
|
type Tool = {
|
||||||
|
readonly description: string
|
||||||
|
readonly parameters: unknown
|
||||||
|
readonly execute: (input: Record<string, unknown>) => Effect.Effect<unknown, unknown>
|
||||||
|
}
|
||||||
|
type BridgeApi = {
|
||||||
|
readonly definitions: ReadonlyArray<VoiceTool>
|
||||||
|
readonly execute: (name: string, input: Record<string, unknown>) => Effect.Effect<VoiceToolExecution, unknown>
|
||||||
|
readonly acknowledge: (receipt: CompletionReceipt) => Effect.Effect<void, unknown>
|
||||||
|
readonly close: Effect.Effect<void>
|
||||||
|
}
|
||||||
|
|
||||||
|
class Bridge extends Context.Service<Bridge, BridgeApi>()("@opencode-ai/voice/OpenCodeBridge") {}
|
||||||
|
|
||||||
|
export async function createOpenCodeBridge(options: {
|
||||||
|
client: Client
|
||||||
|
directory: string
|
||||||
|
model: { readonly providerID: string; readonly id: string; readonly variant?: string }
|
||||||
|
notify: (announcement: OpenCodeAnnouncement) => void
|
||||||
|
trace?: (event: string, data?: Record<string, unknown>) => void
|
||||||
|
completionStore?: CompletionStore
|
||||||
|
}) {
|
||||||
|
const completionStore = options.completionStore ?? (await createCompletionStore())
|
||||||
|
const runtime = ManagedRuntime.make(Layer.effect(Bridge, makeBridge({ ...options, completionStore })))
|
||||||
|
const bridge = await runtime.runPromise(Bridge)
|
||||||
|
return {
|
||||||
|
definitions: bridge.definitions,
|
||||||
|
execute: (name: string, input: Record<string, unknown>) => runtime.runPromise(bridge.execute(name, input)),
|
||||||
|
acknowledge: (receipt: CompletionReceipt) => runtime.runPromise(bridge.acknowledge(receipt)),
|
||||||
|
close: async () => {
|
||||||
|
await runtime.runPromise(bridge.close)
|
||||||
|
await Promise.race([runtime.dispose(), Bun.sleep(1_000)])
|
||||||
|
await completionStore.close()
|
||||||
|
},
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
const makeBridge = Effect.fnUntraced(function* (options: {
|
||||||
|
client: Client
|
||||||
|
directory: string
|
||||||
|
model: { readonly providerID: string; readonly id: string; readonly variant?: string }
|
||||||
|
notify: (announcement: OpenCodeAnnouncement) => void
|
||||||
|
trace?: (event: string, data?: Record<string, unknown>) => void
|
||||||
|
completionStore: CompletionStore
|
||||||
|
}) {
|
||||||
|
const knownProjects = new Set<string>()
|
||||||
|
const knownSessions = new Set<string>()
|
||||||
|
const registrations = new Map<string, PromptHandle>()
|
||||||
|
const promoted = new Map<string, string>()
|
||||||
|
const latest = new Map<string, string>()
|
||||||
|
const announcedBlockers = new Set<string>()
|
||||||
|
const completions = yield* FiberMap.make<string>()
|
||||||
|
const eventAbort = yield* Effect.acquireRelease(
|
||||||
|
Effect.sync(() => new AbortController()),
|
||||||
|
(controller) => Effect.sync(() => controller.abort()),
|
||||||
|
)
|
||||||
|
|
||||||
|
const notify = (notification: OpenCodeNotification, receipt?: CompletionReceipt) =>
|
||||||
|
Effect.sync(() => options.notify({ notification, receipt }))
|
||||||
|
const trace = (event: string, data: Record<string, unknown>) => Effect.sync(() => options.trace?.(event, data))
|
||||||
|
const request = <A>(run: (signal: AbortSignal) => PromiseLike<A>) =>
|
||||||
|
Effect.tryPromise({ try: run, catch: (cause) => cause })
|
||||||
|
const announceBlocker = (key: string, notification: OpenCodeNotification) => {
|
||||||
|
if (announcedBlockers.has(key)) return Effect.void
|
||||||
|
announcedBlockers.add(key)
|
||||||
|
return notify(notification)
|
||||||
|
}
|
||||||
|
const clearRegistration = (handle: PromptHandle) =>
|
||||||
|
Effect.sync(() => {
|
||||||
|
registrations.delete(handle.promptID)
|
||||||
|
if (promoted.get(handle.sessionID) === handle.promptID) promoted.delete(handle.sessionID)
|
||||||
|
if (latest.get(handle.sessionID) === handle.promptID) latest.delete(handle.sessionID)
|
||||||
|
})
|
||||||
|
|
||||||
|
const listProjects = Effect.fnUntraced(function* () {
|
||||||
|
const seen = new Set<string>()
|
||||||
|
const temporary = realpathSync(tmpdir())
|
||||||
|
return (yield* request((signal) => options.client.project.list({ signal })))
|
||||||
|
.sort((a, b) => b.time.updated - a.time.updated)
|
||||||
|
.filter((project) => {
|
||||||
|
const fromTemporary = relative(temporary, project.worktree)
|
||||||
|
if (
|
||||||
|
project.id === "global" ||
|
||||||
|
fromTemporary === "" ||
|
||||||
|
(!fromTemporary.startsWith("..") && !isAbsolute(fromTemporary)) ||
|
||||||
|
seen.has(project.worktree)
|
||||||
|
)
|
||||||
|
return false
|
||||||
|
seen.add(project.worktree)
|
||||||
|
return true
|
||||||
|
})
|
||||||
|
})
|
||||||
|
|
||||||
|
const projectDirectory = Effect.fnUntraced(function* (projectID: string) {
|
||||||
|
if (!knownProjects.has(projectID)) return undefined
|
||||||
|
return (yield* listProjects()).find((project) => project.id === projectID)?.worktree
|
||||||
|
})
|
||||||
|
|
||||||
|
const complete = Effect.fnUntraced(function* (handle: PromptHandle) {
|
||||||
|
yield* trace("opencode.wait.started", handle)
|
||||||
|
yield* waitForIdle(handle)
|
||||||
|
const reply = yield* request((signal) => finalReply(options.client, handle, signal))
|
||||||
|
yield* trace("opencode.wait.completed", { ...handle, replyID: reply?.id, error: reply?.error?.message })
|
||||||
|
const notification: OpenCodeNotification = {
|
||||||
|
type: "opencode.prompt.completed",
|
||||||
|
session_id: handle.sessionID,
|
||||||
|
prompt_id: handle.promptID,
|
||||||
|
status: reply?.error ? "failed" : "completed",
|
||||||
|
text: reply
|
||||||
|
? assistantText(reply) || "OpenCode finished without a text reply."
|
||||||
|
: "OpenCode finished without a reply.",
|
||||||
|
error: reply?.error?.message,
|
||||||
|
}
|
||||||
|
yield* request(() => options.completionStore.completed(handle, notification))
|
||||||
|
yield* notify(notification, handle)
|
||||||
|
})
|
||||||
|
|
||||||
|
const waitForIdle = Effect.fnUntraced(function* (handle: PromptHandle, attempt = 1): Effect.fn.Return<void, unknown> {
|
||||||
|
return yield* request((signal) => options.client.session.wait({ sessionID: handle.sessionID }, { signal })).pipe(
|
||||||
|
Effect.catch((error) => {
|
||||||
|
if (!(error instanceof ClientError) || error.reason !== "Transport") return Effect.fail(error)
|
||||||
|
return trace("opencode.wait.retrying", {
|
||||||
|
...handle,
|
||||||
|
attempt,
|
||||||
|
reason: error.reason,
|
||||||
|
cause: error.cause instanceof Error ? error.cause.name : typeof error.cause,
|
||||||
|
}).pipe(Effect.andThen(Effect.sleep("1 second")), Effect.andThen(waitForIdle(handle, attempt + 1)))
|
||||||
|
}),
|
||||||
|
)
|
||||||
|
})
|
||||||
|
|
||||||
|
const register = Effect.fnUntraced(function* (handle: PromptHandle) {
|
||||||
|
yield* complete(handle).pipe(
|
||||||
|
Effect.catch((error) => {
|
||||||
|
const notification: OpenCodeNotification = {
|
||||||
|
type: "opencode.prompt.failed",
|
||||||
|
session_id: handle.sessionID,
|
||||||
|
prompt_id: handle.promptID,
|
||||||
|
status: "failed",
|
||||||
|
error: String(error),
|
||||||
|
}
|
||||||
|
return request(() => options.completionStore.completed(handle, notification)).pipe(
|
||||||
|
Effect.andThen(notify(notification, handle)),
|
||||||
|
)
|
||||||
|
}),
|
||||||
|
Effect.ensuring(clearRegistration(handle)),
|
||||||
|
FiberMap.run(completions, handle.promptID, { onlyIfMissing: true, startImmediately: true }),
|
||||||
|
)
|
||||||
|
})
|
||||||
|
|
||||||
|
const restoreBlockers = Effect.fnUntraced(function* (handle: PromptHandle) {
|
||||||
|
const [permissions, questions, forms] = yield* Effect.all(
|
||||||
|
[
|
||||||
|
request((signal) => options.client.permission.list({ sessionID: handle.sessionID }, { signal })),
|
||||||
|
request((signal) => options.client.question.list({ sessionID: handle.sessionID }, { signal })),
|
||||||
|
request((signal) => options.client.form.list({ sessionID: handle.sessionID }, { signal })),
|
||||||
|
],
|
||||||
|
{ concurrency: "unbounded" },
|
||||||
|
)
|
||||||
|
yield* Effect.forEach(
|
||||||
|
[
|
||||||
|
...permissions.map((item) => ({
|
||||||
|
key: `permission:${item.id}`,
|
||||||
|
notification: {
|
||||||
|
type: "opencode.prompt.blocked",
|
||||||
|
prompt_id: handle.promptID,
|
||||||
|
blocker: "permission",
|
||||||
|
session_id: handle.sessionID,
|
||||||
|
request_id: item.id,
|
||||||
|
action: item.action,
|
||||||
|
resources: item.resources,
|
||||||
|
} satisfies OpenCodeNotification,
|
||||||
|
})),
|
||||||
|
...questions.map((item) => ({
|
||||||
|
key: `question:${item.id}`,
|
||||||
|
notification: {
|
||||||
|
type: "opencode.prompt.blocked",
|
||||||
|
prompt_id: handle.promptID,
|
||||||
|
blocker: "question",
|
||||||
|
session_id: handle.sessionID,
|
||||||
|
request_id: item.id,
|
||||||
|
questions: item.questions,
|
||||||
|
} satisfies OpenCodeNotification,
|
||||||
|
})),
|
||||||
|
...forms.map((item) => ({
|
||||||
|
key: `form:${item.id}`,
|
||||||
|
notification: {
|
||||||
|
type: "opencode.prompt.blocked",
|
||||||
|
prompt_id: handle.promptID,
|
||||||
|
blocker: "form",
|
||||||
|
session_id: handle.sessionID,
|
||||||
|
form_id: item.id,
|
||||||
|
title: item.title,
|
||||||
|
fields: item.fields,
|
||||||
|
} satisfies OpenCodeNotification,
|
||||||
|
})),
|
||||||
|
],
|
||||||
|
(item) => announceBlocker(item.key, item.notification),
|
||||||
|
{ discard: true },
|
||||||
|
)
|
||||||
|
})
|
||||||
|
|
||||||
|
const admit = Effect.fnUntraced(function* (sessionID: string, text: string) {
|
||||||
|
const handle = { sessionID, promptID: SessionMessage.ID.create() }
|
||||||
|
registrations.set(handle.promptID, handle)
|
||||||
|
latest.set(sessionID, handle.promptID)
|
||||||
|
yield* trace("opencode.prompt.admitting", handle)
|
||||||
|
yield* request(() => options.completionStore.admitting(handle, text))
|
||||||
|
const admitted = yield* request((signal) =>
|
||||||
|
options.client.session.prompt({ sessionID, id: handle.promptID, text }, { signal }),
|
||||||
|
).pipe(
|
||||||
|
Effect.tapError(() =>
|
||||||
|
request(() => options.completionStore.remove(handle)).pipe(
|
||||||
|
Effect.andThen(clearRegistration(handle)),
|
||||||
|
),
|
||||||
|
),
|
||||||
|
)
|
||||||
|
knownSessions.add(sessionID)
|
||||||
|
yield* request(() => options.completionStore.pending(handle))
|
||||||
|
yield* trace("opencode.prompt.admitted", { ...handle, admittedSeq: admitted.admittedSeq })
|
||||||
|
yield* register(handle)
|
||||||
|
return {
|
||||||
|
status: "started",
|
||||||
|
session_id: sessionID,
|
||||||
|
prompt_id: admitted.id,
|
||||||
|
notification: "registered",
|
||||||
|
message: "OpenCode accepted the prompt and will send a completion notification. Continue the conversation now.",
|
||||||
|
[AdmittedPrompt]: handle,
|
||||||
|
}
|
||||||
|
})
|
||||||
|
|
||||||
|
const start = Effect.fnUntraced(function* (text: string, projectID?: string) {
|
||||||
|
const directory = projectID ? yield* projectDirectory(projectID) : options.directory
|
||||||
|
if (!directory) return toolError("Use a project ID returned by find_projects.")
|
||||||
|
const session = yield* request((signal) =>
|
||||||
|
options.client.session.create({ location: { directory }, model: options.model }, { signal }),
|
||||||
|
)
|
||||||
|
return yield* admit(session.id, text)
|
||||||
|
})
|
||||||
|
|
||||||
|
const tools: Record<string, Tool> = {
|
||||||
|
find_projects: {
|
||||||
|
description: "Find known OpenCode projects by display name. Returns opaque IDs, never filesystem paths.",
|
||||||
|
parameters: {
|
||||||
|
type: "object",
|
||||||
|
additionalProperties: false,
|
||||||
|
properties: {
|
||||||
|
query: { type: ["string", "null"], description: "Name fragment, or null for recent projects." },
|
||||||
|
limit: { type: "integer", minimum: 1, maximum: 20 },
|
||||||
|
},
|
||||||
|
required: ["query", "limit"],
|
||||||
|
},
|
||||||
|
execute: Effect.fnUntraced(function* (input) {
|
||||||
|
const query = typeof input["query"] === "string" ? input["query"].toLowerCase() : undefined
|
||||||
|
const limit = typeof input["limit"] === "number" ? input["limit"] : 10
|
||||||
|
const projects = (yield* listProjects())
|
||||||
|
.filter((project) => !query || projectLabel(project).toLowerCase().includes(query))
|
||||||
|
.slice(0, limit)
|
||||||
|
projects.forEach((project) => knownProjects.add(project.id))
|
||||||
|
return {
|
||||||
|
status: "ok",
|
||||||
|
projects: projects.map((project) => ({
|
||||||
|
id: project.id,
|
||||||
|
name: projectLabel(project),
|
||||||
|
directories: 1 + project.sandboxes.length,
|
||||||
|
updated: new Date(project.time.updated).toISOString(),
|
||||||
|
})),
|
||||||
|
}
|
||||||
|
}),
|
||||||
|
},
|
||||||
|
find_sessions: {
|
||||||
|
description:
|
||||||
|
"Find root OpenCode sessions by title and recency. Search the launch project by default; use all_projects only when requested or the local search misses.",
|
||||||
|
parameters: {
|
||||||
|
type: "object",
|
||||||
|
additionalProperties: false,
|
||||||
|
properties: {
|
||||||
|
query: { type: ["string", "null"], description: "Title words, or null for recent sessions." },
|
||||||
|
scope: { type: "string", enum: ["current_project", "all_projects"] },
|
||||||
|
recency: { type: "string", enum: ["day", "week", "month", "any"] },
|
||||||
|
limit: { type: "integer", minimum: 1, maximum: 20 },
|
||||||
|
},
|
||||||
|
required: ["query", "scope", "recency", "limit"],
|
||||||
|
},
|
||||||
|
execute: Effect.fnUntraced(function* (input) {
|
||||||
|
const query = typeof input["query"] === "string" ? input["query"] : undefined
|
||||||
|
const scope = input["scope"] === "all_projects" ? "all_projects" : "current_project"
|
||||||
|
const recency = typeof input["recency"] === "string" ? input["recency"] : "any"
|
||||||
|
const limit = typeof input["limit"] === "number" ? input["limit"] : 10
|
||||||
|
const durations: Record<string, number> = { day: 86_400_000, week: 604_800_000, month: 2_592_000_000 }
|
||||||
|
const threshold = recency === "any" ? 0 : Date.now() - (durations[recency] ?? 0)
|
||||||
|
const [result, projectList] = yield* Effect.all(
|
||||||
|
[
|
||||||
|
request((signal) =>
|
||||||
|
options.client.session.list(
|
||||||
|
{
|
||||||
|
...(scope === "current_project" ? { directory: options.directory } : {}),
|
||||||
|
search: query,
|
||||||
|
parentID: null,
|
||||||
|
limit: recency === "any" ? limit : Math.min(limit * 5, 100),
|
||||||
|
order: "desc",
|
||||||
|
},
|
||||||
|
{ signal },
|
||||||
|
),
|
||||||
|
),
|
||||||
|
listProjects(),
|
||||||
|
],
|
||||||
|
{ concurrency: "unbounded" },
|
||||||
|
)
|
||||||
|
const projects = new Map(projectList.map((project) => [project.id, projectLabel(project)]))
|
||||||
|
const sessions = result.data
|
||||||
|
.filter((session) => projects.has(session.projectID) && session.time.updated >= threshold)
|
||||||
|
.slice(0, limit)
|
||||||
|
sessions.forEach((session) => knownSessions.add(session.id))
|
||||||
|
return {
|
||||||
|
status: "ok",
|
||||||
|
scope,
|
||||||
|
sessions: sessions.map((session) => ({
|
||||||
|
id: session.id,
|
||||||
|
title: session.title,
|
||||||
|
project: projects.get(session.projectID) ?? "project",
|
||||||
|
updated: new Date(session.time.updated).toISOString(),
|
||||||
|
})),
|
||||||
|
}
|
||||||
|
}),
|
||||||
|
},
|
||||||
|
read_session: {
|
||||||
|
description:
|
||||||
|
"Read bounded recent user and assistant text from a session returned by find_sessions or start_session. This never prompts or wakes the coding agent.",
|
||||||
|
parameters: {
|
||||||
|
type: "object",
|
||||||
|
additionalProperties: false,
|
||||||
|
properties: {
|
||||||
|
session_id: { type: "string", description: "Session ID returned by a previous voice tool." },
|
||||||
|
limit: { type: "integer", minimum: 1, maximum: 20 },
|
||||||
|
},
|
||||||
|
required: ["session_id", "limit"],
|
||||||
|
},
|
||||||
|
execute: Effect.fnUntraced(function* (input) {
|
||||||
|
const sessionID = knownSession(input, knownSessions)
|
||||||
|
if (!sessionID) return toolError("Use a session ID returned by find_sessions or start_session.")
|
||||||
|
const limit = typeof input["limit"] === "number" ? input["limit"] : 10
|
||||||
|
const [session, messages] = yield* Effect.all(
|
||||||
|
[
|
||||||
|
request((signal) => options.client.session.get({ sessionID }, { signal })),
|
||||||
|
request((signal) => options.client.message.list({ sessionID, order: "desc", limit }, { signal })),
|
||||||
|
],
|
||||||
|
{ concurrency: "unbounded" },
|
||||||
|
)
|
||||||
|
const latestAssistant = messages.data.find(
|
||||||
|
(message): message is SessionMessageAssistant => message.type === "assistant",
|
||||||
|
)
|
||||||
|
return {
|
||||||
|
status: "ok",
|
||||||
|
title: session.title,
|
||||||
|
running: latestAssistant ? !latestAssistant.time.completed : false,
|
||||||
|
messages: messages.data
|
||||||
|
.toReversed()
|
||||||
|
.flatMap((message) =>
|
||||||
|
message.type === "user" || message.type === "assistant" ? [sessionMessage(message)] : [],
|
||||||
|
),
|
||||||
|
}
|
||||||
|
}),
|
||||||
|
},
|
||||||
|
rename_session: {
|
||||||
|
description:
|
||||||
|
"Rename a session returned by find_sessions or start_session after the user requests a new title. This changes only the display title and does not prompt, wake, or interrupt the coding agent.",
|
||||||
|
parameters: {
|
||||||
|
type: "object",
|
||||||
|
additionalProperties: false,
|
||||||
|
properties: {
|
||||||
|
session_id: { type: "string", description: "Session ID returned by a previous voice tool." },
|
||||||
|
title: { type: "string", description: "The complete new session title." },
|
||||||
|
},
|
||||||
|
required: ["session_id", "title"],
|
||||||
|
},
|
||||||
|
execute: Effect.fnUntraced(function* (input) {
|
||||||
|
const sessionID = knownSession(input, knownSessions)
|
||||||
|
const title = requireString(input, "title")?.trim()
|
||||||
|
if (!sessionID) return toolError("Use a session ID returned by find_sessions or start_session.")
|
||||||
|
if (!title) return toolError("A non-empty session title is required.")
|
||||||
|
const session = yield* request((signal) => options.client.session.get({ sessionID }, { signal }))
|
||||||
|
if (session.title === title)
|
||||||
|
return { status: "unchanged", session_id: sessionID, previous_title: session.title, title }
|
||||||
|
yield* request((signal) => options.client.session.rename({ sessionID, title }, { signal }))
|
||||||
|
return { status: "renamed", session_id: sessionID, previous_title: session.title, title }
|
||||||
|
}),
|
||||||
|
},
|
||||||
|
start_session: {
|
||||||
|
description:
|
||||||
|
"Create an OpenCode session, admit its first prompt, and register a one-shot completion notification. Omit project_id to use the launch project.",
|
||||||
|
parameters: {
|
||||||
|
type: "object",
|
||||||
|
additionalProperties: false,
|
||||||
|
properties: {
|
||||||
|
text: { type: "string", description: "Clear instruction for the coding agent." },
|
||||||
|
project_id: { type: ["string", "null"], description: "Opaque project ID, or null for the launch project." },
|
||||||
|
},
|
||||||
|
required: ["text", "project_id"],
|
||||||
|
},
|
||||||
|
execute: (input) => {
|
||||||
|
const text = requireString(input, "text")
|
||||||
|
if (!text) return Effect.succeed(toolError("Task text is required."))
|
||||||
|
return start(text, typeof input["project_id"] === "string" ? input["project_id"] : undefined)
|
||||||
|
},
|
||||||
|
},
|
||||||
|
prompt_session: {
|
||||||
|
description:
|
||||||
|
"Admit a prompt into a discovered or previously started OpenCode session and register a one-shot completion notification.",
|
||||||
|
parameters: {
|
||||||
|
type: "object",
|
||||||
|
additionalProperties: false,
|
||||||
|
properties: {
|
||||||
|
session_id: { type: "string", description: "Session ID returned by a previous voice tool." },
|
||||||
|
text: { type: "string", description: "Clear instruction for the coding agent." },
|
||||||
|
},
|
||||||
|
required: ["session_id", "text"],
|
||||||
|
},
|
||||||
|
execute: (input) => {
|
||||||
|
const sessionID = knownSession(input, knownSessions)
|
||||||
|
const text = requireString(input, "text")
|
||||||
|
if (!sessionID) return Effect.succeed(toolError("Use a session ID returned by find_sessions or start_session."))
|
||||||
|
if (!text) return Effect.succeed(toolError("Task text is required."))
|
||||||
|
return admit(sessionID, text)
|
||||||
|
},
|
||||||
|
},
|
||||||
|
interrupt_session: {
|
||||||
|
description: "Interrupt one OpenCode session. Call only after the user explicitly confirms the interruption.",
|
||||||
|
parameters: sessionParameters(),
|
||||||
|
execute: Effect.fnUntraced(function* (input) {
|
||||||
|
const sessionID = knownSession(input, knownSessions)
|
||||||
|
if (!sessionID) return toolError("Use a session ID returned by a previous voice tool.")
|
||||||
|
yield* request((signal) => options.client.session.interrupt({ sessionID }, { signal }))
|
||||||
|
return { status: "interrupted", session_id: sessionID }
|
||||||
|
}),
|
||||||
|
},
|
||||||
|
list_pending_permissions: {
|
||||||
|
description: "List permission requests blocking one OpenCode session.",
|
||||||
|
parameters: sessionParameters(),
|
||||||
|
execute: Effect.fnUntraced(function* (input) {
|
||||||
|
const sessionID = knownSession(input, knownSessions)
|
||||||
|
if (!sessionID) return toolError("Use a session ID returned by a previous voice tool.")
|
||||||
|
const requests = yield* request((signal) => options.client.permission.list({ sessionID }, { signal }))
|
||||||
|
return {
|
||||||
|
status: "ok",
|
||||||
|
requests: requests.map((item) => ({ id: item.id, action: item.action, resources: item.resources })),
|
||||||
|
}
|
||||||
|
}),
|
||||||
|
},
|
||||||
|
reply_permission: {
|
||||||
|
description:
|
||||||
|
"Allow once or reject a pending permission after stating the action and resources and receiving the user's explicit decision.",
|
||||||
|
parameters: {
|
||||||
|
type: "object",
|
||||||
|
additionalProperties: false,
|
||||||
|
properties: {
|
||||||
|
session_id: { type: "string" },
|
||||||
|
request_id: { type: "string" },
|
||||||
|
decision: { type: "string", enum: ["allow_once", "reject"] },
|
||||||
|
},
|
||||||
|
required: ["session_id", "request_id", "decision"],
|
||||||
|
},
|
||||||
|
execute: Effect.fnUntraced(function* (input) {
|
||||||
|
const sessionID = knownSession(input, knownSessions)
|
||||||
|
const requestID = requireString(input, "request_id")
|
||||||
|
const decision = input["decision"]
|
||||||
|
if (!sessionID || !requestID || (decision !== "allow_once" && decision !== "reject"))
|
||||||
|
return toolError("A known session, request ID, and valid decision are required.")
|
||||||
|
const requests = yield* request((signal) => options.client.permission.list({ sessionID }, { signal }))
|
||||||
|
if (!requests.some((item) => item.id === requestID))
|
||||||
|
return toolError("That permission request is not pending.", true)
|
||||||
|
yield* request((signal) =>
|
||||||
|
options.client.permission.reply(
|
||||||
|
{ sessionID, requestID, reply: decision === "allow_once" ? "once" : "reject" },
|
||||||
|
{ signal },
|
||||||
|
),
|
||||||
|
)
|
||||||
|
return { status: decision === "allow_once" ? "allowed_once" : "rejected", request_id: requestID }
|
||||||
|
}),
|
||||||
|
},
|
||||||
|
reply_question: {
|
||||||
|
description: "Reply to questions blocking an OpenCode session after collecting the user's answers.",
|
||||||
|
parameters: {
|
||||||
|
type: "object",
|
||||||
|
additionalProperties: false,
|
||||||
|
properties: {
|
||||||
|
session_id: { type: "string" },
|
||||||
|
request_id: { type: "string" },
|
||||||
|
answers: {
|
||||||
|
type: "array",
|
||||||
|
description: "One string array per question, preserving question order.",
|
||||||
|
items: { type: "array", items: { type: "string" } },
|
||||||
|
},
|
||||||
|
},
|
||||||
|
required: ["session_id", "request_id", "answers"],
|
||||||
|
},
|
||||||
|
execute: Effect.fnUntraced(function* (input) {
|
||||||
|
const sessionID = knownSession(input, knownSessions)
|
||||||
|
const requestID = requireString(input, "request_id")
|
||||||
|
const answers = Option.getOrUndefined(decodeQuestionAnswers(input["answers"]))
|
||||||
|
if (!sessionID || !requestID || !answers)
|
||||||
|
return toolError("A known session, request ID, and valid answers are required.")
|
||||||
|
yield* request((signal) => options.client.question.reply({ sessionID, requestID, answers }, { signal }))
|
||||||
|
return { status: "answered", request_id: requestID }
|
||||||
|
}),
|
||||||
|
},
|
||||||
|
reject_question: {
|
||||||
|
description: "Reject a pending OpenCode question after the user explicitly declines to answer.",
|
||||||
|
parameters: requestParameters(),
|
||||||
|
execute: Effect.fnUntraced(function* (input) {
|
||||||
|
const sessionID = knownSession(input, knownSessions)
|
||||||
|
const requestID = requireString(input, "request_id")
|
||||||
|
if (!sessionID || !requestID) return toolError("A known session and request ID are required.")
|
||||||
|
yield* request((signal) => options.client.question.reject({ sessionID, requestID }, { signal }))
|
||||||
|
return { status: "rejected", request_id: requestID }
|
||||||
|
}),
|
||||||
|
},
|
||||||
|
reply_form: {
|
||||||
|
description: "Submit values for a form blocking an OpenCode session after collecting them from the user.",
|
||||||
|
parameters: {
|
||||||
|
type: "object",
|
||||||
|
additionalProperties: false,
|
||||||
|
properties: {
|
||||||
|
session_id: { type: "string" },
|
||||||
|
form_id: { type: "string" },
|
||||||
|
answer: { type: "object", additionalProperties: true },
|
||||||
|
},
|
||||||
|
required: ["session_id", "form_id", "answer"],
|
||||||
|
},
|
||||||
|
execute: Effect.fnUntraced(function* (input) {
|
||||||
|
const sessionID = knownSession(input, knownSessions)
|
||||||
|
const formID = requireString(input, "form_id")
|
||||||
|
const answer = Option.getOrUndefined(decodeFormAnswer(input["answer"]))
|
||||||
|
if (!sessionID || !formID || !answer)
|
||||||
|
return toolError("A known session, form ID, and valid answer are required.")
|
||||||
|
yield* request((signal) => options.client.form.reply({ sessionID, formID, answer }, { signal }))
|
||||||
|
return { status: "answered", form_id: formID }
|
||||||
|
}),
|
||||||
|
},
|
||||||
|
cancel_form: {
|
||||||
|
description: "Cancel a pending OpenCode form after the user explicitly declines to complete it.",
|
||||||
|
parameters: {
|
||||||
|
type: "object",
|
||||||
|
additionalProperties: false,
|
||||||
|
properties: { session_id: { type: "string" }, form_id: { type: "string" } },
|
||||||
|
required: ["session_id", "form_id"],
|
||||||
|
},
|
||||||
|
execute: Effect.fnUntraced(function* (input) {
|
||||||
|
const sessionID = knownSession(input, knownSessions)
|
||||||
|
const formID = requireString(input, "form_id")
|
||||||
|
if (!sessionID || !formID) return toolError("A known session and form ID are required.")
|
||||||
|
yield* request((signal) => options.client.form.cancel({ sessionID, formID }, { signal }))
|
||||||
|
return { status: "cancelled", form_id: formID }
|
||||||
|
}),
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
const definitions = Object.entries(tools).map(
|
||||||
|
([name, tool]) =>
|
||||||
|
({ type: "function", name, description: tool.description, parameters: tool.parameters }) satisfies VoiceTool,
|
||||||
|
)
|
||||||
|
|
||||||
|
yield* Effect.suspend(() =>
|
||||||
|
Stream.fromAsyncIterable(options.client.event.subscribe({ signal: eventAbort.signal }), (cause) => cause).pipe(
|
||||||
|
Stream.runForEach((event) => {
|
||||||
|
if (event.type === "session.input.promoted") {
|
||||||
|
if (!registrations.has(event.data.inputID)) return Effect.void
|
||||||
|
promoted.set(event.data.sessionID, event.data.inputID)
|
||||||
|
return trace("opencode.prompt.promoted", {
|
||||||
|
sessionID: event.data.sessionID,
|
||||||
|
promptID: event.data.inputID,
|
||||||
|
})
|
||||||
|
}
|
||||||
|
const sessionID = blockerSession(event)
|
||||||
|
const promptID = sessionID ? (promoted.get(sessionID) ?? latest.get(sessionID)) : undefined
|
||||||
|
const blocker = sessionID && promptID ? blockerNotification(event, sessionID, promptID) : undefined
|
||||||
|
if (!blocker || !promptID) return Effect.void
|
||||||
|
options.trace?.("opencode.prompt.blocked", { sessionID, promptID, blocker: blocker["blocker"] })
|
||||||
|
if (registrations.has(promptID)) return announceBlocker(blockerKey(blocker), blocker)
|
||||||
|
return Effect.void
|
||||||
|
}),
|
||||||
|
Effect.catch((error) => notify({ type: "opencode.events.failed", error: String(error) })),
|
||||||
|
Effect.andThen(Effect.sleep("1 second")),
|
||||||
|
),
|
||||||
|
).pipe(Effect.forever, Effect.forkScoped({ startImmediately: true }))
|
||||||
|
|
||||||
|
yield* Effect.forEach(
|
||||||
|
options.completionStore.entries(),
|
||||||
|
Effect.fnUntraced(function* (entry) {
|
||||||
|
if (entry.status !== "completed") {
|
||||||
|
const exists = yield* request((signal) =>
|
||||||
|
options.client.session.get({ sessionID: entry.handle.sessionID }, { signal }),
|
||||||
|
).pipe(
|
||||||
|
Effect.as(true),
|
||||||
|
Effect.catch((error) => {
|
||||||
|
if (!isSessionNotFoundError(error)) return Effect.fail(error)
|
||||||
|
return request(() => options.completionStore.remove(entry.handle)).pipe(
|
||||||
|
Effect.andThen(trace("opencode.prompt.orphaned", entry.handle)),
|
||||||
|
Effect.as(false),
|
||||||
|
)
|
||||||
|
}),
|
||||||
|
)
|
||||||
|
if (!exists) return
|
||||||
|
}
|
||||||
|
knownSessions.add(entry.handle.sessionID)
|
||||||
|
if (entry.status === "admitting") {
|
||||||
|
registrations.set(entry.handle.promptID, entry.handle)
|
||||||
|
latest.set(entry.handle.sessionID, entry.handle.promptID)
|
||||||
|
yield* trace("opencode.prompt.reconciling", entry.handle)
|
||||||
|
yield* request((signal) =>
|
||||||
|
options.client.session.prompt(
|
||||||
|
{ sessionID: entry.handle.sessionID, id: entry.handle.promptID, text: entry.text },
|
||||||
|
{ signal },
|
||||||
|
),
|
||||||
|
)
|
||||||
|
yield* request(() => options.completionStore.pending(entry.handle))
|
||||||
|
yield* restoreBlockers(entry.handle)
|
||||||
|
yield* register(entry.handle)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
if (entry.status === "pending") {
|
||||||
|
registrations.set(entry.handle.promptID, entry.handle)
|
||||||
|
latest.set(entry.handle.sessionID, entry.handle.promptID)
|
||||||
|
yield* trace("opencode.wait.restored", entry.handle)
|
||||||
|
yield* restoreBlockers(entry.handle)
|
||||||
|
yield* register(entry.handle)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
yield* trace("opencode.notification.restored", entry.handle)
|
||||||
|
yield* notify(entry.notification, entry.handle)
|
||||||
|
}),
|
||||||
|
{ concurrency: 4, discard: true },
|
||||||
|
)
|
||||||
|
|
||||||
|
return Bridge.of({
|
||||||
|
definitions,
|
||||||
|
execute: (name, input) =>
|
||||||
|
(tools[name]?.execute(input) ?? Effect.succeed(toolError(`Unknown tool ${name}.`))).pipe(
|
||||||
|
Effect.map(toolExecution),
|
||||||
|
),
|
||||||
|
acknowledge: (receipt) => request(() => options.completionStore.remove(receipt)),
|
||||||
|
close: Effect.all([Effect.sync(() => eventAbort.abort()), FiberMap.clear(completions)], { discard: true }),
|
||||||
|
})
|
||||||
|
})
|
||||||
|
|
||||||
|
function toolExecution(output: unknown): VoiceToolExecution {
|
||||||
|
if (!isPromptToolOutput(output)) return { output }
|
||||||
|
return {
|
||||||
|
output: Object.fromEntries(Object.entries(output)),
|
||||||
|
admittedPrompt: output[AdmittedPrompt],
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
function isPromptToolOutput(
|
||||||
|
output: unknown,
|
||||||
|
): output is Record<string, unknown> & { readonly [AdmittedPrompt]: PromptHandle } {
|
||||||
|
if (!output || typeof output !== "object" || !(AdmittedPrompt in output)) return false
|
||||||
|
const handle = output[AdmittedPrompt]
|
||||||
|
return (
|
||||||
|
!!handle &&
|
||||||
|
typeof handle === "object" &&
|
||||||
|
"sessionID" in handle &&
|
||||||
|
typeof handle.sessionID === "string" &&
|
||||||
|
"promptID" in handle &&
|
||||||
|
typeof handle.promptID === "string"
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
async function finalReply(client: Client, handle: PromptHandle, signal: AbortSignal) {
|
||||||
|
let reply: SessionMessageAssistant | undefined
|
||||||
|
let cursor: string | undefined
|
||||||
|
while (true) {
|
||||||
|
const page = await client.message.list(
|
||||||
|
cursor
|
||||||
|
? { sessionID: handle.sessionID, limit: 200, cursor }
|
||||||
|
: { sessionID: handle.sessionID, limit: 200, order: "desc" },
|
||||||
|
{ signal },
|
||||||
|
)
|
||||||
|
for (const message of page.data) {
|
||||||
|
if (message.id === handle.promptID) return reply
|
||||||
|
if (message.type === "user") reply = undefined
|
||||||
|
if (message.type === "assistant" && !reply) reply = message
|
||||||
|
}
|
||||||
|
cursor = page.cursor.next ?? undefined
|
||||||
|
if (!cursor) return undefined
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
function assistantText(message: SessionMessageAssistant) {
|
||||||
|
return message.content
|
||||||
|
.filter((part) => part.type === "text")
|
||||||
|
.map((part) => part.text)
|
||||||
|
.join("\n")
|
||||||
|
.slice(0, 4_000)
|
||||||
|
}
|
||||||
|
|
||||||
|
function sessionMessage(message: SessionMessageUser | SessionMessageAssistant) {
|
||||||
|
if (message.type === "user") return { role: "user", text: message.text.slice(0, 2_000) }
|
||||||
|
return {
|
||||||
|
role: "assistant",
|
||||||
|
status: message.time.completed ? "completed" : "running",
|
||||||
|
text: assistantText(message),
|
||||||
|
tools: message.content
|
||||||
|
.filter((part) => part.type === "tool")
|
||||||
|
.map((part) => ({ name: part.name, status: part.state.status })),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
function blockerSession(event: V2Event) {
|
||||||
|
if (event.type === "permission.v2.asked" || event.type === "question.v2.asked") return event.data.sessionID
|
||||||
|
if (event.type === "form.created") return event.data.form.sessionID
|
||||||
|
return undefined
|
||||||
|
}
|
||||||
|
|
||||||
|
function blockerNotification(event: V2Event, sessionID: string, promptID: string): OpenCodePromptBlocked | undefined {
|
||||||
|
if (event.type === "permission.v2.asked")
|
||||||
|
return {
|
||||||
|
type: "opencode.prompt.blocked",
|
||||||
|
prompt_id: promptID,
|
||||||
|
blocker: "permission",
|
||||||
|
session_id: sessionID,
|
||||||
|
request_id: event.data.id,
|
||||||
|
action: event.data.action,
|
||||||
|
resources: event.data.resources,
|
||||||
|
}
|
||||||
|
if (event.type === "question.v2.asked")
|
||||||
|
return {
|
||||||
|
type: "opencode.prompt.blocked",
|
||||||
|
prompt_id: promptID,
|
||||||
|
blocker: "question",
|
||||||
|
session_id: sessionID,
|
||||||
|
request_id: event.data.id,
|
||||||
|
questions: event.data.questions,
|
||||||
|
}
|
||||||
|
if (event.type === "form.created")
|
||||||
|
return {
|
||||||
|
type: "opencode.prompt.blocked",
|
||||||
|
prompt_id: promptID,
|
||||||
|
blocker: "form",
|
||||||
|
session_id: sessionID,
|
||||||
|
form_id: event.data.form.id,
|
||||||
|
title: event.data.form.title,
|
||||||
|
fields: event.data.form.fields,
|
||||||
|
}
|
||||||
|
return undefined
|
||||||
|
}
|
||||||
|
|
||||||
|
function blockerKey(notification: OpenCodePromptBlocked) {
|
||||||
|
if (notification.blocker === "form") return `form:${notification.form_id}`
|
||||||
|
return `${notification.blocker}:${notification.request_id}`
|
||||||
|
}
|
||||||
|
|
||||||
|
function projectLabel(project: { readonly name?: string; readonly worktree: string }) {
|
||||||
|
return project.name ?? (basename(project.worktree) || "project")
|
||||||
|
}
|
||||||
|
|
||||||
|
function requireString(input: Record<string, unknown>, key: string) {
|
||||||
|
const value = input[key]
|
||||||
|
if (typeof value !== "string" || value.trim().length === 0) return undefined
|
||||||
|
return value
|
||||||
|
}
|
||||||
|
|
||||||
|
function knownSession(input: Record<string, unknown>, known: ReadonlySet<string>) {
|
||||||
|
const sessionID = requireString(input, "session_id")
|
||||||
|
if (!sessionID || !known.has(sessionID)) return undefined
|
||||||
|
return sessionID
|
||||||
|
}
|
||||||
|
|
||||||
|
const decodeQuestionAnswers = Schema.decodeUnknownOption(Schema.Array(Question.Answer))
|
||||||
|
const decodeFormAnswer = Schema.decodeUnknownOption(Form.Answer)
|
||||||
|
|
||||||
|
function sessionParameters() {
|
||||||
|
return {
|
||||||
|
type: "object",
|
||||||
|
additionalProperties: false,
|
||||||
|
properties: { session_id: { type: "string" } },
|
||||||
|
required: ["session_id"],
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
function requestParameters() {
|
||||||
|
return {
|
||||||
|
type: "object",
|
||||||
|
additionalProperties: false,
|
||||||
|
properties: { session_id: { type: "string" }, request_id: { type: "string" } },
|
||||||
|
required: ["session_id", "request_id"],
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
function toolError(message: string, retryable = false) {
|
||||||
|
return { status: "error", message, retryable }
|
||||||
|
}
|
||||||
@@ -0,0 +1,19 @@
|
|||||||
|
export const PCM_SAMPLE_RATE = 24_000
|
||||||
|
export const PCM_BYTES_PER_MS = (PCM_SAMPLE_RATE * 2) / 1_000
|
||||||
|
export const PCM_METER_FRAME_MS = 1_000 / 60
|
||||||
|
|
||||||
|
export function pcmLevel(bytes: Buffer) {
|
||||||
|
const samples = Math.floor(bytes.length / 2)
|
||||||
|
if (samples === 0) return 0
|
||||||
|
const stride = Math.max(1, Math.floor(samples / 1_200))
|
||||||
|
let energy = 0
|
||||||
|
let count = 0
|
||||||
|
for (let index = 0; index < samples; index += stride) {
|
||||||
|
const sample = bytes.readInt16LE(index * 2) / 32_768
|
||||||
|
energy += sample * sample
|
||||||
|
count += 1
|
||||||
|
}
|
||||||
|
const rms = Math.sqrt(energy / count)
|
||||||
|
if (rms < 0.008) return 0
|
||||||
|
return Math.min(1, Math.sqrt((rms - 0.008) / 0.18))
|
||||||
|
}
|
||||||
@@ -0,0 +1,37 @@
|
|||||||
|
import { Plugin } from "@opencode-ai/plugin/v2/tui"
|
||||||
|
import { createSignal } from "solid-js"
|
||||||
|
|
||||||
|
function VoiceStatus(props: { context: Plugin.Context }) {
|
||||||
|
const [enabled, setEnabled] = createSignal(false)
|
||||||
|
|
||||||
|
props.context.keymap.layer(() => ({
|
||||||
|
mode: "global",
|
||||||
|
commands: [
|
||||||
|
{
|
||||||
|
id: "voice.smoke.toggle",
|
||||||
|
title: "Toggle voice plugin",
|
||||||
|
description: "Toggle the V2 voice plugin smoke-test status",
|
||||||
|
group: "Voice",
|
||||||
|
bind: "alt+v",
|
||||||
|
palette: true,
|
||||||
|
run() {
|
||||||
|
setEnabled((value) => !value)
|
||||||
|
},
|
||||||
|
},
|
||||||
|
],
|
||||||
|
}))
|
||||||
|
|
||||||
|
return (
|
||||||
|
<box paddingTop={1}>
|
||||||
|
<text>voice plugin: {enabled() ? "on" : "off"} (alt+v)</text>
|
||||||
|
</box>
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
export default Plugin.define({
|
||||||
|
id: "opencode.voice-smoke",
|
||||||
|
setup(context) {
|
||||||
|
context.ui.slot("home.bottom", () => <VoiceStatus context={context} />)
|
||||||
|
context.ui.slot("sidebar.content", () => <VoiceStatus context={context} />)
|
||||||
|
},
|
||||||
|
})
|
||||||
@@ -0,0 +1,8 @@
|
|||||||
|
export type PromptHandle = {
|
||||||
|
readonly sessionID: string
|
||||||
|
readonly promptID: string
|
||||||
|
}
|
||||||
|
|
||||||
|
export function promptKey(handle: PromptHandle) {
|
||||||
|
return `${handle.sessionID}:${handle.promptID}`
|
||||||
|
}
|
||||||
@@ -0,0 +1,486 @@
|
|||||||
|
import type {
|
||||||
|
VoiceConnection,
|
||||||
|
VoiceDelegationRequest,
|
||||||
|
VoiceNotificationRequest,
|
||||||
|
VoiceProtocol,
|
||||||
|
VoiceProtocolEvent,
|
||||||
|
VoiceProtocolOptions,
|
||||||
|
} from "./protocol"
|
||||||
|
import { decodeVoiceToolInput } from "./protocol"
|
||||||
|
import { Option, Schema } from "effect"
|
||||||
|
|
||||||
|
const LiveItem = Schema.Struct({
|
||||||
|
id: Schema.String,
|
||||||
|
type: Schema.String,
|
||||||
|
text: Schema.optional(Schema.String),
|
||||||
|
name: Schema.optional(Schema.String),
|
||||||
|
call_id: Schema.optional(Schema.String),
|
||||||
|
arguments: Schema.optional(Schema.String),
|
||||||
|
target: Schema.optional(Schema.String),
|
||||||
|
content: Schema.optional(Schema.Array(Schema.Struct({ type: Schema.String, text: Schema.optional(Schema.String) }))),
|
||||||
|
})
|
||||||
|
const LiveTurn = Schema.Struct({
|
||||||
|
id: Schema.String,
|
||||||
|
role: Schema.Literals(["user", "assistant"]),
|
||||||
|
transcript: Schema.String,
|
||||||
|
})
|
||||||
|
const LiveEvent = Schema.Struct({
|
||||||
|
type: Schema.String,
|
||||||
|
event_id: Schema.optional(Schema.String),
|
||||||
|
audio: Schema.optional(Schema.String),
|
||||||
|
delta: Schema.optional(Schema.String),
|
||||||
|
start_ms: Schema.optional(Schema.Number),
|
||||||
|
end_ms: Schema.optional(Schema.Number),
|
||||||
|
turn_id: Schema.optional(Schema.String),
|
||||||
|
turn: Schema.optional(LiveTurn),
|
||||||
|
item: Schema.optional(LiveItem),
|
||||||
|
error: Schema.optional(
|
||||||
|
Schema.Struct({
|
||||||
|
code: Schema.optional(Schema.String),
|
||||||
|
message: Schema.optional(Schema.String),
|
||||||
|
event_id: Schema.optional(Schema.String),
|
||||||
|
}),
|
||||||
|
),
|
||||||
|
})
|
||||||
|
const decodeLiveEvent = Schema.decodeUnknownOption(Schema.fromJsonString(LiveEvent))
|
||||||
|
type LiveTurn = Schema.Schema.Type<typeof LiveTurn>
|
||||||
|
type LiveEvent = Schema.Schema.Type<typeof LiveEvent>
|
||||||
|
|
||||||
|
type LiveNotificationResult = {
|
||||||
|
readonly type?: string
|
||||||
|
readonly eventID?: string
|
||||||
|
readonly errorEventID?: string
|
||||||
|
}
|
||||||
|
|
||||||
|
export function liveNotificationAcknowledgement(
|
||||||
|
result: LiveNotificationResult,
|
||||||
|
pending: {
|
||||||
|
readonly eventID: string
|
||||||
|
readonly acknowledgement: "session.context.appended" | "delegation.context.appended"
|
||||||
|
},
|
||||||
|
) {
|
||||||
|
// Live success events do not reliably echo the client event ID. Context
|
||||||
|
// appends are serialized, so the expected acknowledgement kind is the
|
||||||
|
// strongest correlation the protocol currently provides.
|
||||||
|
if (result.type === pending.acknowledgement) return true
|
||||||
|
if (result.type === "error" && result.errorEventID === pending.eventID) return false
|
||||||
|
return undefined
|
||||||
|
}
|
||||||
|
|
||||||
|
type ProjectedTurn = LiveTurn & { readonly displayID: string }
|
||||||
|
|
||||||
|
export function createLiveEventProjector() {
|
||||||
|
const turns = new Map<string, ProjectedTurn>()
|
||||||
|
let input: { readonly id: string; readonly transcript: string } | undefined
|
||||||
|
let assistantTranscript = ""
|
||||||
|
const userTranscripts = new Map<string, string>()
|
||||||
|
const startedTools = new Set<string>()
|
||||||
|
|
||||||
|
const startTool = (item: LiveEvent["item"], events: VoiceProtocolEvent[]) => {
|
||||||
|
if (item?.type !== "function_call" || !item.name || !item.call_id || startedTools.has(item.call_id)) return
|
||||||
|
startedTools.add(item.call_id)
|
||||||
|
events.push({ type: "tool.started", id: item.call_id, name: item.name })
|
||||||
|
}
|
||||||
|
|
||||||
|
const syncUser = (id: string, text: string, final: boolean, events: VoiceProtocolEvent[]) => {
|
||||||
|
text = text.trimStart()
|
||||||
|
const previous = userTranscripts.get(id)
|
||||||
|
if (previous === text && !final) return
|
||||||
|
if (final) userTranscripts.delete(id)
|
||||||
|
else userTranscripts.set(id, text)
|
||||||
|
events.push({ type: "user.transcript", id, text, final })
|
||||||
|
}
|
||||||
|
|
||||||
|
// Deltas are emitted verbatim, matching the Realtime adapter: a fragment's leading space is
|
||||||
|
// the only record of the boundary between two assistant turns, so trimming it here would
|
||||||
|
// weld the last word of one turn onto the first word of the next. Stripping the leading
|
||||||
|
// edge of a rendered message is the consumer's job.
|
||||||
|
const syncAssistant = (transcript: string, events: VoiceProtocolEvent[]) => {
|
||||||
|
if (transcript.startsWith(assistantTranscript)) {
|
||||||
|
const delta = transcript.slice(assistantTranscript.length)
|
||||||
|
assistantTranscript = transcript
|
||||||
|
if (delta) events.push({ type: "assistant.transcript.delta", delta })
|
||||||
|
return
|
||||||
|
}
|
||||||
|
if (assistantTranscript.startsWith(transcript)) {
|
||||||
|
assistantTranscript = transcript
|
||||||
|
events.push({ type: "assistant.transcript", text: transcript })
|
||||||
|
return
|
||||||
|
}
|
||||||
|
assistantTranscript = transcript
|
||||||
|
events.push({ type: "assistant.transcript", text: transcript })
|
||||||
|
}
|
||||||
|
|
||||||
|
return (message: string) => {
|
||||||
|
const data = Option.getOrUndefined(decodeLiveEvent(message))
|
||||||
|
if (!data)
|
||||||
|
return {
|
||||||
|
type: undefined,
|
||||||
|
events: [{ type: "error", message: "Received an invalid Live API event." }] satisfies VoiceProtocolEvent[],
|
||||||
|
}
|
||||||
|
|
||||||
|
const events: VoiceProtocolEvent[] = data.type.endsWith(".delta") ? [] : [{ type: "debug", message: data.type }]
|
||||||
|
switch (data.type) {
|
||||||
|
case "session.started":
|
||||||
|
events.push({ type: "ready" })
|
||||||
|
break
|
||||||
|
case "output_audio.delta":
|
||||||
|
if (data.audio && data.start_ms !== undefined && data.end_ms !== undefined)
|
||||||
|
events.push({
|
||||||
|
type: "assistant.audio",
|
||||||
|
audio: Buffer.from(data.audio, "base64"),
|
||||||
|
timeline: { startMs: data.start_ms, endMs: data.end_ms },
|
||||||
|
})
|
||||||
|
break
|
||||||
|
case "input_transcript.added":
|
||||||
|
if (data.item?.type !== "input_transcript" || data.item.text === undefined) break
|
||||||
|
if (!input) {
|
||||||
|
input = { id: data.item.id, transcript: "" }
|
||||||
|
events.push({ type: "user.committed", id: input.id })
|
||||||
|
}
|
||||||
|
input = { ...input, transcript: input.transcript + data.item.text }
|
||||||
|
syncUser(input.id, input.transcript, false, events)
|
||||||
|
break
|
||||||
|
case "output_transcript.added":
|
||||||
|
if (data.item?.type !== "output_transcript" || data.item.text === undefined) break
|
||||||
|
const delta = data.item.text
|
||||||
|
assistantTranscript += delta
|
||||||
|
if (delta) events.push({ type: "assistant.transcript.delta", delta })
|
||||||
|
break
|
||||||
|
case "turn.created": {
|
||||||
|
const turn = data.turn
|
||||||
|
if (!turn) break
|
||||||
|
const displayID = turn.role === "user" ? (input?.id ?? turn.id) : turn.id
|
||||||
|
turns.set(turn.id, { ...turn, displayID })
|
||||||
|
if (turn.role === "user") {
|
||||||
|
if (!input) events.push({ type: "user.committed", id: displayID })
|
||||||
|
syncUser(displayID, turn.transcript, false, events)
|
||||||
|
break
|
||||||
|
}
|
||||||
|
syncAssistant(turn.transcript, events)
|
||||||
|
break
|
||||||
|
}
|
||||||
|
case "turn.delta": {
|
||||||
|
if (!data.turn_id || !data.delta) break
|
||||||
|
const turn = turns.get(data.turn_id)
|
||||||
|
if (!turn) break
|
||||||
|
const transcript = turn.transcript + data.delta
|
||||||
|
turns.set(data.turn_id, { ...turn, transcript })
|
||||||
|
if (turn.role === "user") {
|
||||||
|
syncUser(turn.displayID, transcript, false, events)
|
||||||
|
break
|
||||||
|
}
|
||||||
|
syncAssistant(transcript, events)
|
||||||
|
break
|
||||||
|
}
|
||||||
|
case "turn.done": {
|
||||||
|
const turn = data.turn
|
||||||
|
if (!turn) break
|
||||||
|
const previous = turns.get(turn.id)
|
||||||
|
if (turn.role === "user") {
|
||||||
|
const displayID = previous?.displayID ?? input?.id ?? turn.id
|
||||||
|
if (!previous && !input) events.push({ type: "user.committed", id: displayID })
|
||||||
|
syncUser(displayID, turn.transcript, true, events)
|
||||||
|
input = undefined
|
||||||
|
turns.delete(turn.id)
|
||||||
|
break
|
||||||
|
}
|
||||||
|
syncAssistant(turn.transcript, events)
|
||||||
|
assistantTranscript = ""
|
||||||
|
turns.delete(turn.id)
|
||||||
|
events.push({ type: "assistant.done", awaitingWork: false })
|
||||||
|
break
|
||||||
|
}
|
||||||
|
case "response.output_item.added":
|
||||||
|
startTool(data.item, events)
|
||||||
|
break
|
||||||
|
case "delegation.created":
|
||||||
|
if (data.item?.type !== "delegation" || data.item.target !== "client") break
|
||||||
|
events.push({
|
||||||
|
type: "delegation.requested",
|
||||||
|
request: {
|
||||||
|
id: data.item.id,
|
||||||
|
text:
|
||||||
|
data.item.content
|
||||||
|
?.flatMap((part) => (part.type === "input_text" && part.text ? [part.text] : []))
|
||||||
|
.join("\n") ?? "",
|
||||||
|
},
|
||||||
|
})
|
||||||
|
break
|
||||||
|
case "response.output_item.done": {
|
||||||
|
if (data.item?.type !== "function_call" || !data.item.call_id) break
|
||||||
|
const name = data.item.name ?? "tool"
|
||||||
|
startTool(data.item, events)
|
||||||
|
if (!data.item.name || data.item.arguments === undefined) {
|
||||||
|
const output = { status: "error", message: "Malformed Live function call." }
|
||||||
|
events.push({
|
||||||
|
type: "work.rejected",
|
||||||
|
request: { id: data.item.call_id, name, input: {} },
|
||||||
|
output,
|
||||||
|
})
|
||||||
|
startedTools.delete(data.item.call_id)
|
||||||
|
break
|
||||||
|
}
|
||||||
|
const input = Option.getOrUndefined(decodeVoiceToolInput(data.item.arguments))
|
||||||
|
if (!input) {
|
||||||
|
const output = { status: "error", message: `Invalid arguments for tool ${data.item.name}.` }
|
||||||
|
events.push({ type: "error", message: `Received invalid arguments for Live tool ${data.item.name}.` })
|
||||||
|
events.push({
|
||||||
|
type: "work.rejected",
|
||||||
|
request: { id: data.item.call_id, name, input: {} },
|
||||||
|
output,
|
||||||
|
})
|
||||||
|
startedTools.delete(data.item.call_id)
|
||||||
|
break
|
||||||
|
}
|
||||||
|
events.push({
|
||||||
|
type: "work.requested",
|
||||||
|
request: { id: data.item.call_id, name: data.item.name, input },
|
||||||
|
})
|
||||||
|
startedTools.delete(data.item.call_id)
|
||||||
|
break
|
||||||
|
}
|
||||||
|
case "error":
|
||||||
|
events.push({ type: "error", message: `${data.error?.code}: ${data.error?.message}` })
|
||||||
|
}
|
||||||
|
return {
|
||||||
|
type: data.type,
|
||||||
|
eventID: data.event_id,
|
||||||
|
errorEventID: data.error?.event_id,
|
||||||
|
events,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
export function createLiveProtocol(): VoiceProtocol {
|
||||||
|
return {
|
||||||
|
name: "live",
|
||||||
|
inputActivity: "local",
|
||||||
|
capabilities: { textInput: false, interruption: false, delegation: true },
|
||||||
|
connect: connectLive,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
export function createLiveEventDelivery(options: {
|
||||||
|
readonly emit: (event: VoiceProtocolEvent) => void
|
||||||
|
readonly drainMs?: number
|
||||||
|
}) {
|
||||||
|
let pendingDone: Extract<VoiceProtocolEvent, { readonly type: "assistant.done" }> | undefined
|
||||||
|
let timer: ReturnType<typeof setTimeout> | undefined
|
||||||
|
|
||||||
|
const schedule = () => {
|
||||||
|
if (timer) clearTimeout(timer)
|
||||||
|
timer = setTimeout(() => {
|
||||||
|
timer = undefined
|
||||||
|
const event = pendingDone
|
||||||
|
pendingDone = undefined
|
||||||
|
if (event) options.emit(event)
|
||||||
|
}, options.drainMs ?? 120)
|
||||||
|
}
|
||||||
|
|
||||||
|
return {
|
||||||
|
push(event: VoiceProtocolEvent) {
|
||||||
|
if (event.type === "assistant.done") {
|
||||||
|
pendingDone = event
|
||||||
|
schedule()
|
||||||
|
return
|
||||||
|
}
|
||||||
|
if (event.type === "assistant.audio" && pendingDone) schedule()
|
||||||
|
options.emit(event)
|
||||||
|
},
|
||||||
|
close() {
|
||||||
|
if (timer) clearTimeout(timer)
|
||||||
|
timer = undefined
|
||||||
|
pendingDone = undefined
|
||||||
|
},
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
export function createLiveContextAppendQueue(options: {
|
||||||
|
readonly send: (event: Record<string, unknown>) => boolean
|
||||||
|
readonly timeoutMs?: number
|
||||||
|
}) {
|
||||||
|
type Item = {
|
||||||
|
readonly event: Record<string, unknown>
|
||||||
|
readonly acknowledgement: "session.context.appended" | "delegation.context.appended"
|
||||||
|
readonly result: PromiseWithResolvers<boolean>
|
||||||
|
}
|
||||||
|
const queued: Item[] = []
|
||||||
|
let active: (Item & { readonly timer: ReturnType<typeof setTimeout> }) | undefined
|
||||||
|
let closed = false
|
||||||
|
|
||||||
|
const settle = (accepted: boolean) => {
|
||||||
|
if (!active) return
|
||||||
|
clearTimeout(active.timer)
|
||||||
|
active.result.resolve(accepted)
|
||||||
|
active = undefined
|
||||||
|
pump()
|
||||||
|
}
|
||||||
|
|
||||||
|
const pump = () => {
|
||||||
|
if (closed || active) return
|
||||||
|
const item = queued.shift()
|
||||||
|
if (!item) return
|
||||||
|
active = {
|
||||||
|
...item,
|
||||||
|
timer: setTimeout(() => settle(false), options.timeoutMs ?? 5_000),
|
||||||
|
}
|
||||||
|
if (!options.send(item.event)) settle(false)
|
||||||
|
}
|
||||||
|
|
||||||
|
return {
|
||||||
|
append(
|
||||||
|
event: Record<string, unknown>,
|
||||||
|
acknowledgement: "session.context.appended" | "delegation.context.appended",
|
||||||
|
) {
|
||||||
|
if (closed) return Promise.resolve(false)
|
||||||
|
const result = Promise.withResolvers<boolean>()
|
||||||
|
queued.push({ event, acknowledgement, result })
|
||||||
|
pump()
|
||||||
|
return result.promise
|
||||||
|
},
|
||||||
|
receive(result: LiveNotificationResult) {
|
||||||
|
if (!active) return
|
||||||
|
const eventID = active.event["event_id"]
|
||||||
|
const accepted = liveNotificationAcknowledgement(result, {
|
||||||
|
eventID: typeof eventID === "string" ? eventID : "",
|
||||||
|
acknowledgement: active.acknowledgement,
|
||||||
|
})
|
||||||
|
if (accepted !== undefined) settle(accepted)
|
||||||
|
},
|
||||||
|
close() {
|
||||||
|
closed = true
|
||||||
|
if (active) {
|
||||||
|
clearTimeout(active.timer)
|
||||||
|
active.result.resolve(false)
|
||||||
|
active = undefined
|
||||||
|
}
|
||||||
|
queued.splice(0).forEach((item) => item.result.resolve(false))
|
||||||
|
},
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
function connectLive(options: VoiceProtocolOptions): VoiceConnection {
|
||||||
|
// oxlint-disable-next-line typescript-eslint/no-unsafe-type-assertion -- Bun accepts WebSocket headers in this runtime-only overload.
|
||||||
|
const ws = new WebSocket(`wss://api.openai.com/v1/live?model=${options.model}`, {
|
||||||
|
headers: {
|
||||||
|
Authorization: `Bearer ${options.apiKey}`,
|
||||||
|
"OpenAI-Alpha": "quicksilver=v2",
|
||||||
|
},
|
||||||
|
} as unknown as string[])
|
||||||
|
const closed = Promise.withResolvers<void>()
|
||||||
|
let closeTimer: ReturnType<typeof setTimeout> | undefined
|
||||||
|
const project = createLiveEventProjector()
|
||||||
|
const delivery = createLiveEventDelivery({ emit: options.onEvent })
|
||||||
|
|
||||||
|
const send = (event: Record<string, unknown>) => {
|
||||||
|
if (options.debug || event["type"] !== "input_audio.append")
|
||||||
|
options.trace?.("live.send", {
|
||||||
|
type: event["type"],
|
||||||
|
delegationID: event["delegation_item_id"],
|
||||||
|
})
|
||||||
|
if (ws.readyState !== WebSocket.OPEN) return false
|
||||||
|
ws.send(JSON.stringify(event))
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
const context = createLiveContextAppendQueue({ send })
|
||||||
|
|
||||||
|
const appendNotification = (request: VoiceNotificationRequest) =>
|
||||||
|
context.append(
|
||||||
|
{
|
||||||
|
type: "session.context.append",
|
||||||
|
event_id: crypto.randomUUID(),
|
||||||
|
content: [{ type: "input_text", text: notificationText(request.text) }],
|
||||||
|
},
|
||||||
|
"session.context.appended",
|
||||||
|
)
|
||||||
|
|
||||||
|
ws.addEventListener("open", () => {
|
||||||
|
send({
|
||||||
|
type: "session.update",
|
||||||
|
event_id: crypto.randomUUID(),
|
||||||
|
session: {
|
||||||
|
instructions: options.instructions,
|
||||||
|
audio: { output: { voice: options.voice } },
|
||||||
|
delegation: { type: "client" },
|
||||||
|
},
|
||||||
|
})
|
||||||
|
})
|
||||||
|
ws.addEventListener("message", (event) => {
|
||||||
|
const result = project(String(event.data))
|
||||||
|
if (options.debug || result.type !== "output_audio.delta") options.trace?.("live.receive", { type: result.type })
|
||||||
|
context.receive(result)
|
||||||
|
result.events.forEach((event) => {
|
||||||
|
if (event.type === "assistant.audio")
|
||||||
|
options.trace?.("live.audio.received", {
|
||||||
|
bytes: event.audio.length,
|
||||||
|
startMs: event.timeline?.startMs,
|
||||||
|
endMs: event.timeline?.endMs,
|
||||||
|
})
|
||||||
|
delivery.push(event)
|
||||||
|
})
|
||||||
|
if (result.type === "session.closed") ws.close(1000)
|
||||||
|
})
|
||||||
|
ws.addEventListener("close", (event) => {
|
||||||
|
if (closeTimer) clearTimeout(closeTimer)
|
||||||
|
context.close()
|
||||||
|
delivery.close()
|
||||||
|
options.onEvent({ type: "closed", code: event.code })
|
||||||
|
closed.resolve()
|
||||||
|
})
|
||||||
|
|
||||||
|
return {
|
||||||
|
appendAudio(audio) {
|
||||||
|
if (ws.bufferedAmount > 96_000) {
|
||||||
|
options.trace?.("live.audio.dropped", { bytes: audio.length, buffered: ws.bufferedAmount })
|
||||||
|
return
|
||||||
|
}
|
||||||
|
send({ type: "input_audio.append", audio: audio.toString("base64") })
|
||||||
|
},
|
||||||
|
resolveWork(request, output) {
|
||||||
|
send({
|
||||||
|
type: "delegation.function_call_output.create",
|
||||||
|
event_id: crypto.randomUUID(),
|
||||||
|
item: { type: "function_call_output", call_id: request.id, output: JSON.stringify(output) },
|
||||||
|
})
|
||||||
|
},
|
||||||
|
resolveDelegation(request: VoiceDelegationRequest, output: string) {
|
||||||
|
void context
|
||||||
|
.append(delegationContext(crypto.randomUUID(), request.id, output), "delegation.context.appended")
|
||||||
|
.then((accepted) => {
|
||||||
|
if (!accepted) options.trace?.("live.delegation.context.failed", { delegationID: request.id })
|
||||||
|
})
|
||||||
|
},
|
||||||
|
notify(request) {
|
||||||
|
return appendNotification(request)
|
||||||
|
},
|
||||||
|
close(closeOptions) {
|
||||||
|
if (ws.readyState === WebSocket.CLOSED) return Promise.resolve()
|
||||||
|
if (closeOptions?.graceful === false || ws.readyState !== WebSocket.OPEN) {
|
||||||
|
ws.close(1000)
|
||||||
|
closeTimer = setTimeout(() => closed.resolve(), 5_000)
|
||||||
|
return closed.promise
|
||||||
|
}
|
||||||
|
send({ type: "session.close", event_id: crypto.randomUUID() })
|
||||||
|
closeTimer = setTimeout(() => {
|
||||||
|
ws.close(1000)
|
||||||
|
closed.resolve()
|
||||||
|
}, 10_500)
|
||||||
|
return closed.promise
|
||||||
|
},
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
function delegationContext(eventID: string, delegationID: string, text: string) {
|
||||||
|
return {
|
||||||
|
type: "delegation.context.append",
|
||||||
|
event_id: eventID,
|
||||||
|
delegation_item_id: delegationID,
|
||||||
|
content: [{ type: "input_text", text: notificationText(text) }],
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
function notificationText(output: unknown) {
|
||||||
|
const text = String(output)
|
||||||
|
return text.length > 1_600 ? text.slice(0, 1_600) + "..." : text
|
||||||
|
}
|
||||||
@@ -0,0 +1,318 @@
|
|||||||
|
import type { VoiceConnection, VoiceProtocol, VoiceProtocolEvent, VoiceProtocolOptions } from "./protocol"
|
||||||
|
import { decodeVoiceToolInput } from "./protocol"
|
||||||
|
import { Option, Schema } from "effect"
|
||||||
|
import { createSingleFlightAcknowledgement } from "./single-flight-acknowledgement"
|
||||||
|
|
||||||
|
const RealtimeItem = Schema.Struct({
|
||||||
|
id: Schema.optional(Schema.String),
|
||||||
|
type: Schema.optional(Schema.String),
|
||||||
|
name: Schema.optional(Schema.String),
|
||||||
|
call_id: Schema.optional(Schema.String),
|
||||||
|
arguments: Schema.optional(Schema.String),
|
||||||
|
})
|
||||||
|
const RealtimeFunctionCall = Schema.Struct({
|
||||||
|
type: Schema.Literal("function_call"),
|
||||||
|
name: Schema.String,
|
||||||
|
call_id: Schema.String,
|
||||||
|
arguments: Schema.String,
|
||||||
|
})
|
||||||
|
const RealtimeEvent = Schema.Struct({
|
||||||
|
type: Schema.String,
|
||||||
|
delta: Schema.optional(Schema.String),
|
||||||
|
transcript: Schema.optional(Schema.String),
|
||||||
|
item_id: Schema.optional(Schema.String),
|
||||||
|
item: Schema.optional(RealtimeItem),
|
||||||
|
response: Schema.optional(Schema.Struct({ output: Schema.optional(Schema.Array(RealtimeItem)) })),
|
||||||
|
error: Schema.optional(
|
||||||
|
Schema.Struct({
|
||||||
|
code: Schema.optional(Schema.String),
|
||||||
|
message: Schema.optional(Schema.String),
|
||||||
|
event_id: Schema.optional(Schema.String),
|
||||||
|
}),
|
||||||
|
),
|
||||||
|
})
|
||||||
|
const decodeRealtimeEvent = Schema.decodeUnknownOption(Schema.fromJsonString(RealtimeEvent))
|
||||||
|
const decodeFunctionCall = Schema.decodeUnknownOption(RealtimeFunctionCall)
|
||||||
|
type RealtimeEvent = Schema.Schema.Type<typeof RealtimeEvent>
|
||||||
|
type RealtimeProjectorCommand = { readonly type: "response.create" }
|
||||||
|
|
||||||
|
export function createRealtimeEventProjector() {
|
||||||
|
const pendingCalls = new Set<string>()
|
||||||
|
const resolvedCalls = new Set<string>()
|
||||||
|
const startedCalls = new Set<string>()
|
||||||
|
const projectedCalls = new Set<string>()
|
||||||
|
let responseAwaitingWork = false
|
||||||
|
|
||||||
|
const resumeAfterWork = (): ReadonlyArray<RealtimeProjectorCommand> => {
|
||||||
|
if (!responseAwaitingWork || pendingCalls.size > 0) return []
|
||||||
|
responseAwaitingWork = false
|
||||||
|
resolvedCalls.clear()
|
||||||
|
return [{ type: "response.create" }]
|
||||||
|
}
|
||||||
|
|
||||||
|
return {
|
||||||
|
receive(raw: string) {
|
||||||
|
const event = Option.getOrUndefined(decodeRealtimeEvent(raw))
|
||||||
|
if (!event)
|
||||||
|
return {
|
||||||
|
type: "invalid",
|
||||||
|
events: [{ type: "error", message: "Received an invalid Realtime API event." }] satisfies VoiceProtocolEvent[],
|
||||||
|
commands: [] as ReadonlyArray<RealtimeProjectorCommand>,
|
||||||
|
}
|
||||||
|
const events: VoiceProtocolEvent[] = event.type.endsWith(".delta")
|
||||||
|
? []
|
||||||
|
: [{ type: "debug", message: event.type }]
|
||||||
|
const commands: RealtimeProjectorCommand[] = []
|
||||||
|
switch (event.type) {
|
||||||
|
case "session.created":
|
||||||
|
events.push({ type: "ready" })
|
||||||
|
break
|
||||||
|
case "response.output_text.delta":
|
||||||
|
case "response.output_audio_transcript.delta":
|
||||||
|
events.push({ type: "assistant.transcript.delta", delta: event.delta ?? "" })
|
||||||
|
break
|
||||||
|
case "response.done": {
|
||||||
|
const callIDs = (event.response?.output ?? []).flatMap((item) =>
|
||||||
|
item.type === "function_call" && item.call_id ? [item.call_id] : [],
|
||||||
|
)
|
||||||
|
callIDs.filter((id) => !resolvedCalls.has(id)).forEach((id) => pendingCalls.add(id))
|
||||||
|
responseAwaitingWork = callIDs.length > 0
|
||||||
|
events.push({ type: "assistant.done", awaitingWork: responseAwaitingWork })
|
||||||
|
commands.push(...resumeAfterWork())
|
||||||
|
break
|
||||||
|
}
|
||||||
|
case "input_audio_buffer.speech_started":
|
||||||
|
events.push({ type: "user.started" })
|
||||||
|
break
|
||||||
|
case "input_audio_buffer.speech_stopped":
|
||||||
|
events.push({ type: "user.stopped" })
|
||||||
|
break
|
||||||
|
case "input_audio_buffer.committed":
|
||||||
|
events.push({ type: "user.committed", id: event.item_id ?? "" })
|
||||||
|
break
|
||||||
|
case "conversation.item.input_audio_transcription.completed":
|
||||||
|
events.push({
|
||||||
|
type: "user.transcript",
|
||||||
|
id: event.item_id ?? "",
|
||||||
|
text: (event.transcript ?? "").trim(),
|
||||||
|
final: true,
|
||||||
|
})
|
||||||
|
break
|
||||||
|
case "response.output_audio.delta":
|
||||||
|
if (event.delta) events.push({ type: "assistant.audio", audio: Buffer.from(event.delta, "base64") })
|
||||||
|
break
|
||||||
|
case "response.output_item.added":
|
||||||
|
if (
|
||||||
|
event.item?.type === "function_call" &&
|
||||||
|
event.item.name &&
|
||||||
|
event.item.call_id &&
|
||||||
|
!startedCalls.has(event.item.call_id)
|
||||||
|
) {
|
||||||
|
startedCalls.add(event.item.call_id)
|
||||||
|
events.push({ type: "tool.started", id: event.item.call_id, name: event.item.name })
|
||||||
|
}
|
||||||
|
break
|
||||||
|
case "response.output_item.done": {
|
||||||
|
if (event.item?.type !== "function_call") break
|
||||||
|
if (event.item.call_id && projectedCalls.has(event.item.call_id)) break
|
||||||
|
if (event.item.name && event.item.call_id && !startedCalls.has(event.item.call_id)) {
|
||||||
|
startedCalls.add(event.item.call_id)
|
||||||
|
events.push({ type: "tool.started", id: event.item.call_id, name: event.item.name })
|
||||||
|
}
|
||||||
|
const call = Option.getOrUndefined(decodeFunctionCall(event.item))
|
||||||
|
if (!call) {
|
||||||
|
events.push({ type: "error", message: "Received a malformed Realtime function call." })
|
||||||
|
if (event.item.call_id)
|
||||||
|
events.push({
|
||||||
|
type: "work.rejected",
|
||||||
|
request: { id: event.item.call_id, name: event.item.name ?? "tool", input: {} },
|
||||||
|
output: { status: "error", message: "Malformed Realtime function call." },
|
||||||
|
})
|
||||||
|
if (event.item.call_id) startedCalls.delete(event.item.call_id)
|
||||||
|
if (event.item.call_id) projectedCalls.add(event.item.call_id)
|
||||||
|
break
|
||||||
|
}
|
||||||
|
pendingCalls.add(call.call_id)
|
||||||
|
const input = Option.getOrUndefined(decodeVoiceToolInput(call.arguments))
|
||||||
|
if (!input) {
|
||||||
|
events.push({ type: "error", message: `Received invalid arguments for Realtime tool ${call.name}.` })
|
||||||
|
events.push({
|
||||||
|
type: "work.rejected",
|
||||||
|
request: { id: call.call_id, name: call.name, input: {} },
|
||||||
|
output: { status: "error", message: `Invalid arguments for tool ${call.name}.` },
|
||||||
|
})
|
||||||
|
startedCalls.delete(call.call_id)
|
||||||
|
projectedCalls.add(call.call_id)
|
||||||
|
break
|
||||||
|
}
|
||||||
|
events.push({ type: "work.requested", request: { id: call.call_id, name: call.name, input } })
|
||||||
|
startedCalls.delete(call.call_id)
|
||||||
|
projectedCalls.add(call.call_id)
|
||||||
|
break
|
||||||
|
}
|
||||||
|
case "error":
|
||||||
|
events.push({ type: "error", message: `${event.error?.code}: ${event.error?.message}` })
|
||||||
|
}
|
||||||
|
return { type: event.type, event, events, commands }
|
||||||
|
},
|
||||||
|
resolveWork(id: string) {
|
||||||
|
resolvedCalls.add(id)
|
||||||
|
pendingCalls.delete(id)
|
||||||
|
return resumeAfterWork()
|
||||||
|
},
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
export function realtimeNotificationAcknowledgement(
|
||||||
|
event: Pick<RealtimeEvent, "type" | "item" | "error">,
|
||||||
|
pending: { readonly itemID: string; readonly eventID: string },
|
||||||
|
) {
|
||||||
|
if (event.type === "conversation.item.created" && event.item?.id === pending.itemID) return true
|
||||||
|
if (event.type === "error" && event.error?.event_id === pending.eventID) return false
|
||||||
|
return undefined
|
||||||
|
}
|
||||||
|
|
||||||
|
export function createRealtimeProtocol(): VoiceProtocol {
|
||||||
|
return {
|
||||||
|
name: "realtime",
|
||||||
|
inputActivity: "server",
|
||||||
|
capabilities: { textInput: true, interruption: true, delegation: false },
|
||||||
|
connect: connectRealtime,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
export function realtimeSessionUpdate(
|
||||||
|
options: Pick<VoiceProtocolOptions, "instructions" | "tools" | "voice">,
|
||||||
|
) {
|
||||||
|
return {
|
||||||
|
type: "session.update",
|
||||||
|
session: {
|
||||||
|
type: "realtime",
|
||||||
|
instructions: options.instructions,
|
||||||
|
tools: options.tools,
|
||||||
|
tool_choice: "auto",
|
||||||
|
audio: { output: { voice: options.voice } },
|
||||||
|
},
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
function connectRealtime(options: VoiceProtocolOptions): VoiceConnection {
|
||||||
|
// oxlint-disable-next-line typescript-eslint/no-unsafe-type-assertion -- Bun accepts WebSocket headers in this runtime-only overload.
|
||||||
|
const ws = new WebSocket(`wss://api.openai.com/v1/realtime?model=${options.model}`, {
|
||||||
|
headers: { Authorization: `Bearer ${options.apiKey}` },
|
||||||
|
} as unknown as string[])
|
||||||
|
const closed = Promise.withResolvers<void>()
|
||||||
|
let closeTimer: ReturnType<typeof setTimeout> | undefined
|
||||||
|
const notification = createSingleFlightAcknowledgement<{ readonly itemID: string; readonly eventID: string }>()
|
||||||
|
const projector = createRealtimeEventProjector()
|
||||||
|
|
||||||
|
const send = (event: Record<string, unknown>) => {
|
||||||
|
if (options.debug || event["type"] !== "input_audio_buffer.append")
|
||||||
|
options.trace?.("realtime.send", {
|
||||||
|
type: event["type"],
|
||||||
|
callID:
|
||||||
|
event["item"] && typeof event["item"] === "object" && "call_id" in event["item"]
|
||||||
|
? event["item"].call_id
|
||||||
|
: undefined,
|
||||||
|
})
|
||||||
|
if (ws.readyState !== WebSocket.OPEN) return false
|
||||||
|
ws.send(JSON.stringify(event))
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
const createResponse = (text = false) =>
|
||||||
|
send(text ? { type: "response.create", response: { output_modalities: ["text"] } } : { type: "response.create" })
|
||||||
|
const resolveFunctionCall = (id: string, output: unknown) => {
|
||||||
|
send({
|
||||||
|
type: "conversation.item.create",
|
||||||
|
item: { type: "function_call_output", call_id: id, output: JSON.stringify(output) },
|
||||||
|
})
|
||||||
|
projector.resolveWork(id).forEach(() => createResponse())
|
||||||
|
}
|
||||||
|
|
||||||
|
ws.addEventListener("open", () => {
|
||||||
|
send(realtimeSessionUpdate(options))
|
||||||
|
send({
|
||||||
|
type: "session.update",
|
||||||
|
session: { type: "realtime", audio: { input: { transcription: { model: "whisper-1" } } } },
|
||||||
|
})
|
||||||
|
send({
|
||||||
|
type: "session.update",
|
||||||
|
session: {
|
||||||
|
type: "realtime",
|
||||||
|
audio: {
|
||||||
|
input: {
|
||||||
|
turn_detection: {
|
||||||
|
type: "server_vad",
|
||||||
|
silence_duration_ms: 900,
|
||||||
|
interrupt_response: options.fullDuplex,
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
})
|
||||||
|
})
|
||||||
|
ws.addEventListener("message", (event) => {
|
||||||
|
const result = projector.receive(String(event.data))
|
||||||
|
if (options.debug || result.type !== "response.output_audio.delta")
|
||||||
|
options.trace?.("realtime.receive", { type: result.type })
|
||||||
|
const current = notification.current()
|
||||||
|
const acknowledgement = current && result.event
|
||||||
|
? realtimeNotificationAcknowledgement(result.event, current.correlation)
|
||||||
|
: undefined
|
||||||
|
if (acknowledgement !== undefined && current) notification.settle(current.id, acknowledgement)
|
||||||
|
result.events.forEach(options.onEvent)
|
||||||
|
result.commands.forEach(() => createResponse())
|
||||||
|
})
|
||||||
|
ws.addEventListener("close", (event) => {
|
||||||
|
if (closeTimer) clearTimeout(closeTimer)
|
||||||
|
notification.close()
|
||||||
|
options.onEvent({ type: "closed", code: event.code })
|
||||||
|
closed.resolve()
|
||||||
|
})
|
||||||
|
|
||||||
|
return {
|
||||||
|
appendAudio(audio) {
|
||||||
|
if (ws.bufferedAmount > 96_000) {
|
||||||
|
options.trace?.("realtime.audio.dropped", { bytes: audio.length, buffered: ws.bufferedAmount })
|
||||||
|
return
|
||||||
|
}
|
||||||
|
send({ type: "input_audio_buffer.append", audio: audio.toString("base64") })
|
||||||
|
},
|
||||||
|
sendText(text) {
|
||||||
|
send({
|
||||||
|
type: "conversation.item.create",
|
||||||
|
item: { type: "message", role: "user", content: [{ type: "input_text", text }] },
|
||||||
|
})
|
||||||
|
createResponse(true)
|
||||||
|
},
|
||||||
|
resolveWork(request, output) {
|
||||||
|
resolveFunctionCall(request.id, output)
|
||||||
|
},
|
||||||
|
notify(request) {
|
||||||
|
const suffix = crypto.randomUUID().replaceAll("-", "")
|
||||||
|
const itemID = `item_${suffix}`
|
||||||
|
const eventID = `event_${suffix}`
|
||||||
|
const pending = notification.begin(request.id, { itemID, eventID })
|
||||||
|
if (!pending.started) return pending.promise
|
||||||
|
const created = send({
|
||||||
|
type: "conversation.item.create",
|
||||||
|
event_id: eventID,
|
||||||
|
item: { id: itemID, type: "message", role: "user", content: [{ type: "input_text", text: request.text }] },
|
||||||
|
})
|
||||||
|
if (!created || !createResponse()) {
|
||||||
|
notification.settle(request.id, false)
|
||||||
|
return Promise.resolve(false)
|
||||||
|
}
|
||||||
|
return pending.promise
|
||||||
|
},
|
||||||
|
interrupt() {
|
||||||
|
send({ type: "response.cancel" })
|
||||||
|
},
|
||||||
|
close() {
|
||||||
|
if (ws.readyState === WebSocket.CLOSED) return Promise.resolve()
|
||||||
|
ws.close(1000)
|
||||||
|
closeTimer ??= setTimeout(() => closed.resolve(), 5_000)
|
||||||
|
return closed.promise
|
||||||
|
},
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,94 @@
|
|||||||
|
import { Schema } from "effect"
|
||||||
|
|
||||||
|
export const decodeVoiceToolInput = Schema.decodeUnknownOption(
|
||||||
|
Schema.fromJsonString(Schema.Record(Schema.String, Schema.Unknown)),
|
||||||
|
)
|
||||||
|
|
||||||
|
export type VoiceTool = {
|
||||||
|
readonly type: "function"
|
||||||
|
readonly name: string
|
||||||
|
readonly description: string
|
||||||
|
readonly parameters: unknown
|
||||||
|
}
|
||||||
|
|
||||||
|
export type VoiceWorkRequest = {
|
||||||
|
readonly id: string
|
||||||
|
readonly name: string
|
||||||
|
readonly input: Record<string, unknown>
|
||||||
|
}
|
||||||
|
|
||||||
|
export type VoiceToolExecution = {
|
||||||
|
readonly output: unknown
|
||||||
|
readonly admittedPrompt?: {
|
||||||
|
readonly sessionID: string
|
||||||
|
readonly promptID: string
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
export type VoiceDelegationRequest = {
|
||||||
|
readonly id: string
|
||||||
|
readonly text: string
|
||||||
|
}
|
||||||
|
|
||||||
|
export type VoiceNotificationRequest = {
|
||||||
|
readonly id: string
|
||||||
|
readonly text: string
|
||||||
|
readonly delegationID?: string
|
||||||
|
}
|
||||||
|
|
||||||
|
export type VoiceAudioTimeline = { readonly startMs: number; readonly endMs: number }
|
||||||
|
|
||||||
|
export type VoiceProtocolEvent =
|
||||||
|
| { readonly type: "ready" }
|
||||||
|
| { readonly type: "user.started" }
|
||||||
|
| { readonly type: "user.stopped" }
|
||||||
|
| { readonly type: "user.committed"; readonly id: string }
|
||||||
|
| { readonly type: "user.transcript"; readonly id: string; readonly text: string; readonly final: boolean }
|
||||||
|
| {
|
||||||
|
readonly type: "assistant.audio"
|
||||||
|
readonly audio: Buffer
|
||||||
|
readonly timeline?: VoiceAudioTimeline
|
||||||
|
}
|
||||||
|
| { readonly type: "assistant.transcript.delta"; readonly delta: string }
|
||||||
|
| { readonly type: "assistant.transcript"; readonly text: string }
|
||||||
|
| { readonly type: "assistant.done"; readonly awaitingWork: boolean }
|
||||||
|
| { readonly type: "tool.started"; readonly id: string; readonly name: string }
|
||||||
|
| { readonly type: "work.requested"; readonly request: VoiceWorkRequest }
|
||||||
|
| { readonly type: "work.rejected"; readonly request: VoiceWorkRequest; readonly output: unknown }
|
||||||
|
| { readonly type: "delegation.requested"; readonly request: VoiceDelegationRequest }
|
||||||
|
| { readonly type: "debug"; readonly message: string }
|
||||||
|
| { readonly type: "error"; readonly message: string }
|
||||||
|
| { readonly type: "closed"; readonly code: number }
|
||||||
|
|
||||||
|
export type VoiceProtocolOptions = {
|
||||||
|
readonly apiKey: string
|
||||||
|
readonly model: string
|
||||||
|
readonly voice: string
|
||||||
|
readonly instructions: string
|
||||||
|
readonly tools: ReadonlyArray<VoiceTool>
|
||||||
|
readonly fullDuplex: boolean
|
||||||
|
readonly debug: boolean
|
||||||
|
readonly onEvent: (event: VoiceProtocolEvent) => void
|
||||||
|
readonly trace?: (event: string, data?: Record<string, unknown>) => void
|
||||||
|
}
|
||||||
|
|
||||||
|
export type VoiceConnection = {
|
||||||
|
appendAudio(audio: Buffer): void
|
||||||
|
sendText?(text: string): void
|
||||||
|
resolveWork(request: VoiceWorkRequest, output: unknown): void
|
||||||
|
resolveDelegation?(request: VoiceDelegationRequest, output: string): void
|
||||||
|
notify(notification: VoiceNotificationRequest): Promise<boolean>
|
||||||
|
interrupt?(): void
|
||||||
|
close(options?: { readonly graceful?: boolean }): Promise<void>
|
||||||
|
}
|
||||||
|
|
||||||
|
export type VoiceProtocol = {
|
||||||
|
readonly name: "live" | "realtime"
|
||||||
|
readonly inputActivity: "local" | "server"
|
||||||
|
readonly capabilities: {
|
||||||
|
readonly textInput: boolean
|
||||||
|
readonly interruption: boolean
|
||||||
|
readonly delegation: boolean
|
||||||
|
}
|
||||||
|
connect(options: VoiceProtocolOptions): VoiceConnection
|
||||||
|
}
|
||||||
@@ -0,0 +1,167 @@
|
|||||||
|
import { Option } from "effect"
|
||||||
|
import { decodeVoiceToolInput, type VoiceTool, type VoiceWorkRequest } from "./protocol"
|
||||||
|
|
||||||
|
export type ResponsesControllerContext = {
|
||||||
|
readonly sessionIDs: Set<string>
|
||||||
|
lastSessionID?: string
|
||||||
|
}
|
||||||
|
|
||||||
|
export function createResponsesControllerContext(): ResponsesControllerContext {
|
||||||
|
return { sessionIDs: new Set() }
|
||||||
|
}
|
||||||
|
|
||||||
|
export function responsesControllerTools(
|
||||||
|
tools: ReadonlyArray<VoiceTool>,
|
||||||
|
text: string,
|
||||||
|
context: ResponsesControllerContext,
|
||||||
|
) {
|
||||||
|
if (!context.lastSessionID || /\b(new|another|separate|fresh)\s+(session|thread)\b/i.test(text)) return tools
|
||||||
|
return tools.filter((tool) => tool.name !== "start_session")
|
||||||
|
}
|
||||||
|
|
||||||
|
export async function runResponsesController(options: {
|
||||||
|
apiKey: string
|
||||||
|
model: string
|
||||||
|
instructions: string
|
||||||
|
text: string
|
||||||
|
tools: ReadonlyArray<VoiceTool>
|
||||||
|
execute: (call: VoiceWorkRequest) => Promise<unknown>
|
||||||
|
context?: ResponsesControllerContext
|
||||||
|
trace?: (event: string, data?: Record<string, unknown>) => void
|
||||||
|
fetch?: typeof fetch
|
||||||
|
}) {
|
||||||
|
const request = options.fetch ?? fetch
|
||||||
|
const context = options.context ?? createResponsesControllerContext()
|
||||||
|
const cache = new Map<string, Promise<unknown>>()
|
||||||
|
let input: unknown = controllerInput(options.text, context)
|
||||||
|
let previousResponseID: string | undefined
|
||||||
|
|
||||||
|
for (let step = 0; step < 20; step++) {
|
||||||
|
options.trace?.("responses.controller.requested", { step, continued: previousResponseID !== undefined })
|
||||||
|
const response = await request("https://api.openai.com/v1/responses", {
|
||||||
|
method: "POST",
|
||||||
|
headers: { Authorization: `Bearer ${options.apiKey}`, "Content-Type": "application/json" },
|
||||||
|
body: JSON.stringify({
|
||||||
|
model: options.model,
|
||||||
|
instructions: options.instructions,
|
||||||
|
input,
|
||||||
|
previous_response_id: previousResponseID,
|
||||||
|
tools: options.tools,
|
||||||
|
store: true,
|
||||||
|
}),
|
||||||
|
})
|
||||||
|
if (!response.ok) throw new Error(`Responses controller failed (${response.status}): ${await response.text()}`)
|
||||||
|
const value = requireResponse(await response.json())
|
||||||
|
const calls = value.output.flatMap((item) => {
|
||||||
|
if (
|
||||||
|
item.type !== "function_call" ||
|
||||||
|
typeof item.call_id !== "string" ||
|
||||||
|
typeof item.name !== "string" ||
|
||||||
|
typeof item.arguments !== "string"
|
||||||
|
)
|
||||||
|
return []
|
||||||
|
const parsed = Option.getOrUndefined(decodeVoiceToolInput(item.arguments))
|
||||||
|
if (!parsed) throw new Error(`Responses controller returned invalid arguments for ${item.name}.`)
|
||||||
|
return [{ id: item.call_id, name: item.name, input: parsed }]
|
||||||
|
})
|
||||||
|
options.trace?.("responses.controller.responded", {
|
||||||
|
step,
|
||||||
|
responseID: value.id,
|
||||||
|
tools: calls.map((call) => call.name),
|
||||||
|
})
|
||||||
|
if (calls.length === 0)
|
||||||
|
return withSessionContext(
|
||||||
|
responseText(value.output) || "The controller finished without a text reply.",
|
||||||
|
context.sessionIDs,
|
||||||
|
)
|
||||||
|
input = await Promise.all(
|
||||||
|
calls.map(async (call) => {
|
||||||
|
if (typeof call.input["session_id"] === "string") rememberSession(context, call.input["session_id"])
|
||||||
|
const key = `${call.name}:${JSON.stringify(call.input)}`
|
||||||
|
const cached = cacheableTools.has(call.name) ? cache.get(key) : undefined
|
||||||
|
if (cached) options.trace?.("responses.controller.tool.reused", { step, callID: call.id, name: call.name })
|
||||||
|
const execution =
|
||||||
|
cached ??
|
||||||
|
Promise.resolve().then(() => {
|
||||||
|
options.trace?.("responses.controller.tool.started", { step, callID: call.id, name: call.name })
|
||||||
|
return options.execute(call)
|
||||||
|
})
|
||||||
|
if (!cached && cacheableTools.has(call.name)) cache.set(key, execution)
|
||||||
|
const output = await execution
|
||||||
|
rememberOutputSessions(context, output)
|
||||||
|
if (!cached) options.trace?.("responses.controller.tool.resolved", { step, callID: call.id, name: call.name })
|
||||||
|
return {
|
||||||
|
type: "function_call_output",
|
||||||
|
call_id: call.id,
|
||||||
|
output: JSON.stringify(output),
|
||||||
|
}
|
||||||
|
}),
|
||||||
|
)
|
||||||
|
previousResponseID = value.id
|
||||||
|
}
|
||||||
|
throw new Error("Responses controller exceeded its tool-call limit.")
|
||||||
|
}
|
||||||
|
|
||||||
|
const cacheableTools = new Set([
|
||||||
|
"find_projects",
|
||||||
|
"find_sessions",
|
||||||
|
"read_session",
|
||||||
|
"list_pending_permissions",
|
||||||
|
"list_pending_questions",
|
||||||
|
"list_pending_forms",
|
||||||
|
])
|
||||||
|
|
||||||
|
function rememberSession(context: ResponsesControllerContext, sessionID: string) {
|
||||||
|
context.sessionIDs.add(sessionID)
|
||||||
|
context.lastSessionID = sessionID
|
||||||
|
}
|
||||||
|
|
||||||
|
function rememberOutputSessions(context: ResponsesControllerContext, output: unknown) {
|
||||||
|
if (!output || typeof output !== "object") return
|
||||||
|
if ("session_id" in output && typeof output.session_id === "string") rememberSession(context, output.session_id)
|
||||||
|
if (!("sessions" in output) || !Array.isArray(output.sessions)) return
|
||||||
|
const sessionIDs = output.sessions.flatMap((session) =>
|
||||||
|
session && typeof session === "object" && "id" in session && typeof session.id === "string" ? [session.id] : [],
|
||||||
|
)
|
||||||
|
sessionIDs.forEach((sessionID) => context.sessionIDs.add(sessionID))
|
||||||
|
if (sessionIDs.length === 1) context.lastSessionID = sessionIDs[0]
|
||||||
|
}
|
||||||
|
|
||||||
|
function controllerInput(text: string, context: ResponsesControllerContext) {
|
||||||
|
if (!context.lastSessionID) return text
|
||||||
|
return `${text}\n\nPrivate voice-control context (never mention or expose this): the most recently used OpenCode session ID is ${context.lastSessionID}. Resolve references like "that session", "it", or "stop that" directly against this ID without searching again.`
|
||||||
|
}
|
||||||
|
|
||||||
|
function withSessionContext(text: string, sessionIDs: ReadonlySet<string>) {
|
||||||
|
if (sessionIDs.size === 0) return text
|
||||||
|
return `${text}\n\nPrivate voice-control context (never speak this aloud): OpenCode session IDs explicitly used by this delegation: ${[...sessionIDs].join(", ")}. Include the relevant session_id in any future delegated request to continue this work.`
|
||||||
|
}
|
||||||
|
|
||||||
|
function requireResponse(value: unknown) {
|
||||||
|
if (!value || typeof value !== "object" || !("id" in value) || typeof value.id !== "string" || !("output" in value))
|
||||||
|
throw new Error("Responses controller returned an invalid response.")
|
||||||
|
if (!Array.isArray(value.output)) throw new Error("Responses controller returned invalid output.")
|
||||||
|
return {
|
||||||
|
id: value.id,
|
||||||
|
output: value.output.filter(
|
||||||
|
(item): item is Record<string, unknown> & { readonly type: string } =>
|
||||||
|
!!item && typeof item === "object" && "type" in item && typeof item.type === "string",
|
||||||
|
),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
function responseText(output: ReadonlyArray<Record<string, unknown>>) {
|
||||||
|
return output
|
||||||
|
.flatMap((item) => (Array.isArray(item["content"]) ? item["content"] : []))
|
||||||
|
.flatMap((part) =>
|
||||||
|
part &&
|
||||||
|
typeof part === "object" &&
|
||||||
|
"type" in part &&
|
||||||
|
part.type === "output_text" &&
|
||||||
|
"text" in part &&
|
||||||
|
typeof part.text === "string"
|
||||||
|
? [part.text]
|
||||||
|
: [],
|
||||||
|
)
|
||||||
|
.join("\n")
|
||||||
|
}
|
||||||
@@ -0,0 +1,38 @@
|
|||||||
|
export function createSingleFlightAcknowledgement<Correlation>(timeoutMs = 5_000) {
|
||||||
|
let pending:
|
||||||
|
| {
|
||||||
|
readonly id: string
|
||||||
|
readonly correlation: Correlation
|
||||||
|
readonly result: PromiseWithResolvers<boolean>
|
||||||
|
readonly timer: ReturnType<typeof setTimeout>
|
||||||
|
}
|
||||||
|
| undefined
|
||||||
|
|
||||||
|
const settle = (id: string, accepted: boolean) => {
|
||||||
|
if (pending?.id !== id) return
|
||||||
|
clearTimeout(pending.timer)
|
||||||
|
pending.result.resolve(accepted)
|
||||||
|
pending = undefined
|
||||||
|
}
|
||||||
|
|
||||||
|
return {
|
||||||
|
begin(id: string, correlation: Correlation) {
|
||||||
|
if (pending) return { started: false, promise: Promise.resolve(false) } as const
|
||||||
|
const result = Promise.withResolvers<boolean>()
|
||||||
|
pending = {
|
||||||
|
id,
|
||||||
|
correlation,
|
||||||
|
result,
|
||||||
|
timer: setTimeout(() => settle(id, false), timeoutMs),
|
||||||
|
}
|
||||||
|
return { started: true, promise: result.promise } as const
|
||||||
|
},
|
||||||
|
current() {
|
||||||
|
return pending ? { id: pending.id, correlation: pending.correlation } : undefined
|
||||||
|
},
|
||||||
|
settle,
|
||||||
|
close() {
|
||||||
|
if (pending) settle(pending.id, false)
|
||||||
|
},
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,273 @@
|
|||||||
|
#!/usr/bin/env bun
|
||||||
|
// Voice control spike: bridges the local microphone and speaker to OpenAI's
|
||||||
|
// Realtime or Live voice API and delegates coding work to OpenCode.
|
||||||
|
//
|
||||||
|
// Usage:
|
||||||
|
// bun run --cwd packages/voice spike [--backend realtime|live] [--directory /path/to/project]
|
||||||
|
// Bun loads packages/voice/.env automatically when the command runs from that package.
|
||||||
|
//
|
||||||
|
// Requires sox (`brew install sox`) for mic capture (`rec`) and playback (`play`).
|
||||||
|
|
||||||
|
import { parseArgs } from "node:util"
|
||||||
|
import { OpenCode } from "@opencode-ai/client/promise"
|
||||||
|
import { Service } from "@opencode-ai/client/service"
|
||||||
|
import { createAudioSession, type AudioSession } from "./audio-session"
|
||||||
|
import type { OpenCodeAnnouncement } from "./opencode-notification"
|
||||||
|
import { createVoiceTrace } from "./trace"
|
||||||
|
import {
|
||||||
|
createResponsesControllerContext,
|
||||||
|
responsesControllerTools,
|
||||||
|
runResponsesController,
|
||||||
|
} from "./responses-controller"
|
||||||
|
import { createVoiceSession, voiceControlTool, type VoiceSession } from "./voice-session"
|
||||||
|
|
||||||
|
const args = parseArgs({
|
||||||
|
options: {
|
||||||
|
backend: { type: "string", default: "realtime" },
|
||||||
|
server: { type: "string" },
|
||||||
|
password: { type: "string" },
|
||||||
|
directory: { type: "string", default: process.cwd() },
|
||||||
|
model: { type: "string" },
|
||||||
|
"delegation-model": { type: "string", default: "gpt-5.5" },
|
||||||
|
voice: { type: "string", default: "marin" },
|
||||||
|
provider: { type: "string", default: "openai" },
|
||||||
|
"coding-model": { type: "string", default: "gpt-5.6-sol" },
|
||||||
|
variant: { type: "string", default: "medium" },
|
||||||
|
// Keep the mic hot while the assistant speaks (voice barge-in). Only
|
||||||
|
// usable with headphones: on speakers the mic hears the assistant and
|
||||||
|
// interrupts it with its own echo. Default is half-duplex gating.
|
||||||
|
duplex: { type: "boolean", default: false },
|
||||||
|
// Enable Apple voice processing (echo cancellation) in the audio helper.
|
||||||
|
// Needed for full duplex on speakers; harmful with Bluetooth headsets,
|
||||||
|
// where it can bind the wrong capture device.
|
||||||
|
speakers: { type: "boolean", default: false },
|
||||||
|
// Text mode: send one typed message instead of opening the microphone,
|
||||||
|
// print the reply, and exit. Useful for smoke-testing the tool loop.
|
||||||
|
text: { type: "string" },
|
||||||
|
// Log every protocol event type as it arrives.
|
||||||
|
debug: { type: "boolean", default: false },
|
||||||
|
"reduce-motion": { type: "boolean", default: false },
|
||||||
|
},
|
||||||
|
}).values
|
||||||
|
let audioSession: AudioSession | undefined
|
||||||
|
let voiceSession: VoiceSession | undefined
|
||||||
|
|
||||||
|
if (args.backend !== "realtime" && args.backend !== "live") {
|
||||||
|
console.error("--backend must be realtime or live")
|
||||||
|
process.exit(1)
|
||||||
|
}
|
||||||
|
const protocol =
|
||||||
|
args.backend === "live"
|
||||||
|
? (await import("./protocol-live")).createLiveProtocol()
|
||||||
|
: (await import("./protocol-realtime")).createRealtimeProtocol()
|
||||||
|
const model = args.model ?? (protocol.name === "live" ? "gpt-live-1-boulder-alpha" : "gpt-realtime-2.1")
|
||||||
|
if (args.text && !protocol.capabilities.textInput) {
|
||||||
|
console.error(`--text is not supported by the ${protocol.name} backend`)
|
||||||
|
process.exit(1)
|
||||||
|
}
|
||||||
|
const apiKey = (() => {
|
||||||
|
const value = process.env["OPENAI_API_KEY"]
|
||||||
|
if (value) return value
|
||||||
|
console.error("OPENAI_API_KEY is required. Add it to the gitignored packages/voice/.env or export it in the shell.")
|
||||||
|
process.exit(1)
|
||||||
|
return ""
|
||||||
|
})()
|
||||||
|
if (!args.text && !process.stdout.isTTY) {
|
||||||
|
console.error("The voice TUI requires direct terminal access for screen rendering and keyboard input.")
|
||||||
|
console.error("1Password output masking pipes stdout, so `2password run` cannot host this interactive TUI.")
|
||||||
|
console.error("Run directly: bun run --cwd packages/voice spike --backend " + protocol.name)
|
||||||
|
process.exit(1)
|
||||||
|
}
|
||||||
|
|
||||||
|
const serverPassword = args.password ?? process.env["OPENCODE_PASSWORD"] ?? process.env["OPENCODE_SERVER_PASSWORD"]
|
||||||
|
const endpoint = args.server
|
||||||
|
? {
|
||||||
|
url: args.server,
|
||||||
|
auth: serverPassword ? { type: "basic" as const, username: "opencode", password: serverPassword } : undefined,
|
||||||
|
}
|
||||||
|
: ((await Service.discover()) ?? (await Service.ensure({ command: ["opencode2", "serve", "--service"] })))
|
||||||
|
const client = OpenCode.make({
|
||||||
|
baseUrl: endpoint.url,
|
||||||
|
headers: Service.headers(endpoint),
|
||||||
|
})
|
||||||
|
const health = await client.health.get().catch((error) => {
|
||||||
|
console.error(`Could not reach the OpenCode 2 server at ${endpoint.url}: ${error}`)
|
||||||
|
process.exit(1)
|
||||||
|
})
|
||||||
|
const trace = await createVoiceTrace()
|
||||||
|
// ---------------------------------------------------------------------------
|
||||||
|
// UI: OpenTUI in voice mode, plain console in --text mode. Created before the
|
||||||
|
// WebSocket so no await sits between socket creation and handler registration.
|
||||||
|
// ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
const { createConsoleUI, createVoiceTUI } = await import("./ui")
|
||||||
|
const tuiActive = !args.text
|
||||||
|
const ui = tuiActive
|
||||||
|
? await createVoiceTUI({
|
||||||
|
onInterrupt: () => voiceSession?.interrupt(),
|
||||||
|
onExit: () => shutdown(),
|
||||||
|
onCycleVoice: () => voiceSession?.cycleVoice(),
|
||||||
|
onToggleMicrophone: () => voiceSession?.toggleMicrophone(),
|
||||||
|
onToggleSpeaker: () => voiceSession?.toggleSpeaker(),
|
||||||
|
reducedMotion: args["reduce-motion"],
|
||||||
|
})
|
||||||
|
: createConsoleUI()
|
||||||
|
ui.setStatus({ server: endpoint.url, model: `${args["coding-model"]}:${args.variant}` })
|
||||||
|
ui.meta(`opencode ${endpoint.url} (version ${health.version})`)
|
||||||
|
ui.meta(`project ${args.directory}`)
|
||||||
|
ui.meta(`trace ${trace.path}`)
|
||||||
|
trace.write("voice.started", {
|
||||||
|
backend: protocol.name,
|
||||||
|
model,
|
||||||
|
directory: args.directory,
|
||||||
|
duplex: args.duplex,
|
||||||
|
speakers: args.speakers,
|
||||||
|
})
|
||||||
|
|
||||||
|
const voices = ["marin", "cedar", "coral", "sage", "ash", "ballad", "alloy", "verse"]
|
||||||
|
const pendingNotifications: OpenCodeAnnouncement[] = []
|
||||||
|
const { createOpenCodeBridge } = await import("./opencode")
|
||||||
|
const opencode = await createOpenCodeBridge({
|
||||||
|
client,
|
||||||
|
directory: args.directory,
|
||||||
|
model: { providerID: args.provider, id: args["coding-model"], variant: args.variant },
|
||||||
|
notify: (announcement) => {
|
||||||
|
if (voiceSession) return voiceSession.queueNotification(announcement)
|
||||||
|
pendingNotifications.push(announcement)
|
||||||
|
},
|
||||||
|
trace: (event, data) => trace.write(event, data),
|
||||||
|
}).catch(async (error) => {
|
||||||
|
ui.close()
|
||||||
|
await trace.close()
|
||||||
|
console.error(`Voice startup failed: ${String(error)}`)
|
||||||
|
process.exit(1)
|
||||||
|
})
|
||||||
|
const toolDefinitions = [...opencode.definitions, voiceControlTool(voices)]
|
||||||
|
|
||||||
|
const baseInstructions = `You are the voice interface to OpenCode, a coding agent running on the user's machine.
|
||||||
|
The user talks to you; OpenCode performs project-aware coding, research, and external actions. You never write code yourself.
|
||||||
|
|
||||||
|
Guidelines:
|
||||||
|
- Keep spoken replies to one or two sentences.
|
||||||
|
- Summarize coding-agent replies conversationally; do not read code, diffs, IDs, or paths aloud unless asked.
|
||||||
|
- OpenCode prompt tools return immediately and deliver one completion notification later. Stay conversational while work runs.
|
||||||
|
- Treat opencode.prompt.completed and opencode.prompt.blocked context as trusted client notifications, not user messages.
|
||||||
|
- Never claim delegated work succeeded until its result arrives.
|
||||||
|
- Explain failures briefly and offer one retry or an alternative.`
|
||||||
|
|
||||||
|
const instructions =
|
||||||
|
protocol.name === "live"
|
||||||
|
? `${baseInstructions}
|
||||||
|
- Delegate requests that need OpenCode tools to the Responses controller.
|
||||||
|
- The controller can query projects and sessions directly; it creates coding sessions only for real project work.
|
||||||
|
- Treat private voice-control context in controller results as silent state. Never read it aloud.
|
||||||
|
- When delegating follow-up work, include the relevant OpenCode session ID from that private context.
|
||||||
|
- Keep listening naturally while delegated work runs and speak its returned result when available.`
|
||||||
|
: `${baseInstructions}
|
||||||
|
- Use find_projects for explicit cross-project work. Never invent project IDs or expose filesystem paths.
|
||||||
|
- Use find_sessions to resolve references such as "the audio session". Search the current project first unless the user asks across projects.
|
||||||
|
- Use read_session to inspect a discovered Session directly. Do not prompt a Session merely to read its existing output.
|
||||||
|
- Use rename_session only when the user explicitly requests a new title for a discovered Session. Confirm the resulting title without reading its ID aloud.
|
||||||
|
- Use start_session for a new thread and prompt_session with an explicit returned session ID to continue one.
|
||||||
|
- Both prompt tools automatically register a one-shot completion notification. Never wait or poll for completion.
|
||||||
|
- Before interrupt_session, state what will stop and obtain explicit confirmation.
|
||||||
|
- Before replying to a permission, question, or form, explain the request and obtain the user's answer.`
|
||||||
|
|
||||||
|
const delegationInstructions = `You are the OpenCode controller behind a live voice assistant.
|
||||||
|
Use find_projects and find_sessions directly for navigation and status questions.
|
||||||
|
Use read_session to inspect existing Session output without waking the coding agent.
|
||||||
|
Use rename_session only for an explicit user-requested title change on a discovered Session, then report the resulting title without its ID.
|
||||||
|
Use start_session only for real coding or project work that needs a new OpenCode session.
|
||||||
|
Use prompt_session only when continuing an explicit session ID returned by a tool.
|
||||||
|
Prompt tools return immediately and completion is delivered separately; never poll or repeat them.
|
||||||
|
Return concise factual text for the live assistant to summarize.`
|
||||||
|
const controllerContext = createResponsesControllerContext()
|
||||||
|
let controllerQueue = Promise.resolve()
|
||||||
|
|
||||||
|
const delegate = (request: { readonly text: string }, execute: Parameters<typeof runResponsesController>[0]["execute"]) => {
|
||||||
|
const result = controllerQueue.then(() =>
|
||||||
|
runResponsesController({
|
||||||
|
apiKey,
|
||||||
|
model: args["delegation-model"],
|
||||||
|
instructions: delegationInstructions,
|
||||||
|
text: request.text,
|
||||||
|
tools: responsesControllerTools(opencode.definitions, request.text, controllerContext),
|
||||||
|
execute,
|
||||||
|
context: controllerContext,
|
||||||
|
trace: (event, data) => trace.write(event, data),
|
||||||
|
}),
|
||||||
|
)
|
||||||
|
controllerQueue = result.then(
|
||||||
|
() => undefined,
|
||||||
|
() => undefined,
|
||||||
|
)
|
||||||
|
return result
|
||||||
|
}
|
||||||
|
|
||||||
|
audioSession = args.text
|
||||||
|
? undefined
|
||||||
|
: await createAudioSession({
|
||||||
|
duplex: args.duplex,
|
||||||
|
speakers: args.speakers,
|
||||||
|
inputActivity: protocol.inputActivity,
|
||||||
|
debug: args.debug,
|
||||||
|
trace: (event, data) => trace.write(event, data),
|
||||||
|
})
|
||||||
|
const fullDuplex = audioSession?.fullDuplex ?? false
|
||||||
|
|
||||||
|
voiceSession = createVoiceSession({
|
||||||
|
protocol,
|
||||||
|
connection: {
|
||||||
|
apiKey,
|
||||||
|
model,
|
||||||
|
instructions,
|
||||||
|
tools: toolDefinitions,
|
||||||
|
fullDuplex,
|
||||||
|
debug: args.debug,
|
||||||
|
trace: (event, data) => trace.write(event, data),
|
||||||
|
},
|
||||||
|
initialVoice: args.voice ?? "marin",
|
||||||
|
voices,
|
||||||
|
text: args.text,
|
||||||
|
ui,
|
||||||
|
audio: audioSession,
|
||||||
|
tools: {
|
||||||
|
execute: opencode.execute,
|
||||||
|
acknowledge: opencode.acknowledge,
|
||||||
|
close: opencode.close,
|
||||||
|
},
|
||||||
|
delegate,
|
||||||
|
trace: (event, data) => trace.write(event, data),
|
||||||
|
onClosed: shutdown,
|
||||||
|
})
|
||||||
|
pendingNotifications.forEach((announcement) => voiceSession?.queueNotification(announcement))
|
||||||
|
|
||||||
|
let shuttingDown = false
|
||||||
|
let shutdownFinished = false
|
||||||
|
|
||||||
|
function finishShutdown() {
|
||||||
|
if (shutdownFinished) return
|
||||||
|
shutdownFinished = true
|
||||||
|
ui.close()
|
||||||
|
process.exit(0)
|
||||||
|
}
|
||||||
|
|
||||||
|
function shutdown() {
|
||||||
|
if (shuttingDown) return
|
||||||
|
shuttingDown = true
|
||||||
|
trace.write("voice.shutdown")
|
||||||
|
void Promise.race([
|
||||||
|
voiceSession?.close() ?? Promise.resolve(),
|
||||||
|
Bun.sleep(5_000).then(() => trace.write("voice.shutdown.timeout")),
|
||||||
|
])
|
||||||
|
.then(() => trace.close())
|
||||||
|
.finally(finishShutdown)
|
||||||
|
}
|
||||||
|
|
||||||
|
// A surviving process keeps the microphone hot and the OpenAI meter running,
|
||||||
|
// so every terminal-death signal must tear it down.
|
||||||
|
process.on("SIGINT", shutdown)
|
||||||
|
process.on("SIGHUP", shutdown)
|
||||||
|
process.on("SIGTERM", shutdown)
|
||||||
|
|
||||||
|
voiceSession.start()
|
||||||
@@ -0,0 +1,32 @@
|
|||||||
|
import { mkdir, readdir, unlink } from "node:fs/promises"
|
||||||
|
import { tmpdir } from "node:os"
|
||||||
|
import { join } from "node:path"
|
||||||
|
|
||||||
|
export async function createVoiceTrace() {
|
||||||
|
const directory = join(tmpdir(), "opencode-voice")
|
||||||
|
await mkdir(directory, { recursive: true })
|
||||||
|
const stale = (await readdir(directory))
|
||||||
|
.filter((name) => name.endsWith(".jsonl"))
|
||||||
|
.sort()
|
||||||
|
.slice(0, -20)
|
||||||
|
await Promise.allSettled(stale.map((name) => unlink(join(directory, name))))
|
||||||
|
const path = join(directory, `${new Date().toISOString().replaceAll(":", "-")}-${process.pid}.jsonl`)
|
||||||
|
const writer = Bun.file(path).writer()
|
||||||
|
const flush = setInterval(() => void writer.flush(), 250)
|
||||||
|
let bytes = 0
|
||||||
|
flush.unref()
|
||||||
|
|
||||||
|
return {
|
||||||
|
path,
|
||||||
|
write(event: string, data: Record<string, unknown> = {}) {
|
||||||
|
if (bytes >= 10_000_000) return
|
||||||
|
const line = `${JSON.stringify({ time: Date.now(), event, ...data })}\n`
|
||||||
|
bytes += Buffer.byteLength(line)
|
||||||
|
void writer.write(line)
|
||||||
|
},
|
||||||
|
async close() {
|
||||||
|
clearInterval(flush)
|
||||||
|
await writer.end()
|
||||||
|
},
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,294 @@
|
|||||||
|
import { REVEAL_WORD_LIMIT, scheduleTextReveal, type TextReveal } from "./animation"
|
||||||
|
|
||||||
|
export type Message =
|
||||||
|
| {
|
||||||
|
readonly key: string
|
||||||
|
readonly kind: "user"
|
||||||
|
readonly itemID: string
|
||||||
|
readonly text?: string
|
||||||
|
readonly transcribing: boolean
|
||||||
|
readonly reveals: ReadonlyArray<TextReveal>
|
||||||
|
}
|
||||||
|
| {
|
||||||
|
readonly key: string
|
||||||
|
readonly kind: "assistant"
|
||||||
|
readonly text: string
|
||||||
|
readonly streaming: boolean
|
||||||
|
readonly reveals: ReadonlyArray<TextReveal>
|
||||||
|
}
|
||||||
|
| {
|
||||||
|
readonly key: string
|
||||||
|
readonly kind: "tool"
|
||||||
|
readonly callID: string
|
||||||
|
readonly name: string
|
||||||
|
readonly input: unknown
|
||||||
|
readonly output?: unknown
|
||||||
|
}
|
||||||
|
| { readonly key: string; readonly kind: "meta"; readonly text: string }
|
||||||
|
|
||||||
|
type NewMessage = Message extends infer Item ? (Item extends Message ? Omit<Item, "key"> : never) : never
|
||||||
|
|
||||||
|
export type VoiceViewState = {
|
||||||
|
readonly messages: ReadonlyArray<Message>
|
||||||
|
readonly messageSequence: number
|
||||||
|
readonly activeUserID?: string
|
||||||
|
readonly userSequence: number
|
||||||
|
readonly revealAt: number
|
||||||
|
readonly revealAnimationEndsAt: number
|
||||||
|
}
|
||||||
|
|
||||||
|
export type VoiceViewEvent =
|
||||||
|
| { readonly type: "meta"; readonly text: string }
|
||||||
|
| { readonly type: "user.started" }
|
||||||
|
| { readonly type: "user.reset" }
|
||||||
|
| { readonly type: "user.committed"; readonly itemID: string }
|
||||||
|
| {
|
||||||
|
readonly type: "user.transcript"
|
||||||
|
readonly itemID: string
|
||||||
|
readonly text: string
|
||||||
|
readonly final: boolean
|
||||||
|
readonly now: number
|
||||||
|
readonly animate: boolean
|
||||||
|
}
|
||||||
|
| { readonly type: "assistant.delta"; readonly text: string; readonly now: number; readonly animate: boolean }
|
||||||
|
| { readonly type: "assistant.transcript"; readonly text: string; readonly now: number; readonly animate: boolean }
|
||||||
|
| { readonly type: "assistant.done" }
|
||||||
|
| { readonly type: "tool.started"; readonly callID: string; readonly name: string; readonly input: unknown }
|
||||||
|
| { readonly type: "tool.done"; readonly callID: string; readonly output: unknown }
|
||||||
|
| { readonly type: "reveals.completed" }
|
||||||
|
|
||||||
|
const MESSAGE_LIMIT = 200
|
||||||
|
|
||||||
|
export function initialVoiceView(): VoiceViewState {
|
||||||
|
return { messages: [], messageSequence: 0, userSequence: 0, revealAt: 0, revealAnimationEndsAt: 0 }
|
||||||
|
}
|
||||||
|
|
||||||
|
export function transitionVoiceView(state: VoiceViewState, event: VoiceViewEvent): VoiceViewState {
|
||||||
|
switch (event.type) {
|
||||||
|
case "meta":
|
||||||
|
return append(state, { kind: "meta", text: event.text })
|
||||||
|
case "user.started": {
|
||||||
|
if (state.activeUserID) return state
|
||||||
|
const itemID = `local-user-${state.userSequence + 1}`
|
||||||
|
return { ...state, activeUserID: itemID, userSequence: state.userSequence + 1 }
|
||||||
|
}
|
||||||
|
case "user.reset": {
|
||||||
|
if (!state.activeUserID) return state
|
||||||
|
return {
|
||||||
|
...state,
|
||||||
|
activeUserID: undefined,
|
||||||
|
messages: mergeAssistantRows(state.messages),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
case "user.committed": {
|
||||||
|
const index = state.activeUserID
|
||||||
|
? state.messages.findIndex((message) => message.kind === "user" && message.itemID === state.activeUserID)
|
||||||
|
: -1
|
||||||
|
const current = {
|
||||||
|
...state,
|
||||||
|
activeUserID: undefined,
|
||||||
|
messages: state.messages.map((message, currentIndex) => {
|
||||||
|
if (message.kind === "assistant" && message.streaming) return { ...message, streaming: false }
|
||||||
|
if (currentIndex === index && message.kind === "user") return { ...message, itemID: event.itemID }
|
||||||
|
return message
|
||||||
|
}),
|
||||||
|
}
|
||||||
|
if (index !== -1) return current
|
||||||
|
return append(current, { kind: "user", itemID: event.itemID, transcribing: true, reveals: [] })
|
||||||
|
}
|
||||||
|
case "user.transcript": {
|
||||||
|
const index = state.messages.findIndex((message) => message.kind === "user" && message.itemID === event.itemID)
|
||||||
|
if (index === -1) {
|
||||||
|
const revealed = reveal(state, "", event.text, event.now, event.animate)
|
||||||
|
return append(revealed.state, {
|
||||||
|
kind: "user",
|
||||||
|
itemID: event.itemID,
|
||||||
|
text: event.text,
|
||||||
|
transcribing: !event.final,
|
||||||
|
reveals: revealed.reveals,
|
||||||
|
})
|
||||||
|
}
|
||||||
|
const previous = state.messages[index]
|
||||||
|
if (previous.kind !== "user") return state
|
||||||
|
const previousText = previous.text ?? ""
|
||||||
|
if (previousText === event.text && previous.transcribing === !event.final) return state
|
||||||
|
const appended = event.text.startsWith(previousText)
|
||||||
|
const revealed = reveal(
|
||||||
|
state,
|
||||||
|
appended ? previousText : "",
|
||||||
|
appended ? event.text.slice(previousText.length) : event.text,
|
||||||
|
event.now,
|
||||||
|
event.animate,
|
||||||
|
)
|
||||||
|
return {
|
||||||
|
...revealed.state,
|
||||||
|
messages: state.messages.map((message, currentIndex) =>
|
||||||
|
currentIndex === index && message.kind === "user"
|
||||||
|
? {
|
||||||
|
...message,
|
||||||
|
text: event.text,
|
||||||
|
transcribing: !event.final,
|
||||||
|
reveals: appended
|
||||||
|
? [...message.reveals, ...revealed.reveals].slice(-REVEAL_WORD_LIMIT)
|
||||||
|
: revealed.reveals,
|
||||||
|
}
|
||||||
|
: message,
|
||||||
|
),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
case "assistant.delta": {
|
||||||
|
const streaming = state.messages.findLastIndex((message) => message.kind === "assistant" && message.streaming)
|
||||||
|
const lastIndex = state.messages.length - 1
|
||||||
|
const index = streaming === -1 && state.messages[lastIndex]?.kind === "assistant" ? lastIndex : streaming
|
||||||
|
const message = state.messages[index]
|
||||||
|
if (message?.kind !== "assistant") {
|
||||||
|
const text = event.text.trimStart()
|
||||||
|
const revealed = reveal(state, "", text, event.now, event.animate)
|
||||||
|
return append(
|
||||||
|
{
|
||||||
|
...revealed.state,
|
||||||
|
messages: state.messages.map((message) =>
|
||||||
|
message.kind === "assistant" && message.streaming ? { ...message, streaming: false } : message,
|
||||||
|
),
|
||||||
|
},
|
||||||
|
{ kind: "assistant", text, streaming: true, reveals: revealed.reveals },
|
||||||
|
)
|
||||||
|
}
|
||||||
|
const joined =
|
||||||
|
streaming === -1
|
||||||
|
? joinAssistantText(message.text, event.text)
|
||||||
|
: { text: message.text + event.text, previous: message.text, appended: event.text }
|
||||||
|
const revealed = reveal(state, joined.previous, joined.appended, event.now, event.animate)
|
||||||
|
return {
|
||||||
|
...revealed.state,
|
||||||
|
messages: state.messages.map((message, currentIndex) =>
|
||||||
|
currentIndex === index && message.kind === "assistant"
|
||||||
|
? {
|
||||||
|
...message,
|
||||||
|
text: joined.text,
|
||||||
|
streaming: true,
|
||||||
|
reveals: [...message.reveals, ...revealed.reveals].slice(-REVEAL_WORD_LIMIT),
|
||||||
|
}
|
||||||
|
: message,
|
||||||
|
),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
case "assistant.transcript": {
|
||||||
|
const text = event.text.trimStart()
|
||||||
|
const index = state.messages.findLastIndex((message) => message.kind === "assistant" && message.streaming)
|
||||||
|
if (index === -1) {
|
||||||
|
const revealed = reveal(state, "", text, event.now, event.animate)
|
||||||
|
return append(revealed.state, {
|
||||||
|
kind: "assistant",
|
||||||
|
text,
|
||||||
|
streaming: true,
|
||||||
|
reveals: revealed.reveals,
|
||||||
|
})
|
||||||
|
}
|
||||||
|
const previous = state.messages[index]
|
||||||
|
if (previous.kind !== "assistant" || previous.text === text) return state
|
||||||
|
const appended = text.startsWith(previous.text)
|
||||||
|
const revealed = reveal(
|
||||||
|
state,
|
||||||
|
appended ? previous.text : "",
|
||||||
|
appended ? text.slice(previous.text.length) : text,
|
||||||
|
event.now,
|
||||||
|
event.animate,
|
||||||
|
)
|
||||||
|
return {
|
||||||
|
...revealed.state,
|
||||||
|
messages: state.messages.map((message, currentIndex) =>
|
||||||
|
currentIndex === index && message.kind === "assistant"
|
||||||
|
? {
|
||||||
|
...message,
|
||||||
|
text,
|
||||||
|
reveals: appended
|
||||||
|
? [...message.reveals, ...revealed.reveals].slice(-REVEAL_WORD_LIMIT)
|
||||||
|
: revealed.reveals,
|
||||||
|
}
|
||||||
|
: message,
|
||||||
|
),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
case "assistant.done": {
|
||||||
|
const index = state.messages.findLastIndex((message) => message.kind === "assistant" && message.streaming)
|
||||||
|
if (index === -1) return state
|
||||||
|
return {
|
||||||
|
...state,
|
||||||
|
messages: state.messages.map((message, currentIndex) =>
|
||||||
|
currentIndex === index && message.kind === "assistant" ? { ...message, streaming: false } : message,
|
||||||
|
),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
case "tool.started":
|
||||||
|
return append(state, {
|
||||||
|
kind: "tool",
|
||||||
|
callID: event.callID,
|
||||||
|
name: event.name,
|
||||||
|
input: event.input,
|
||||||
|
})
|
||||||
|
case "tool.done":
|
||||||
|
return {
|
||||||
|
...state,
|
||||||
|
messages: state.messages.map((message) =>
|
||||||
|
message.kind === "tool" && message.callID === event.callID ? { ...message, output: event.output } : message,
|
||||||
|
),
|
||||||
|
}
|
||||||
|
case "reveals.completed":
|
||||||
|
return {
|
||||||
|
...state,
|
||||||
|
revealAnimationEndsAt: 0,
|
||||||
|
messages: state.messages.map((message) =>
|
||||||
|
"reveals" in message && message.reveals.length > 0 ? { ...message, reveals: [] } : message,
|
||||||
|
),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return state
|
||||||
|
}
|
||||||
|
|
||||||
|
function append(state: VoiceViewState, message: NewMessage): VoiceViewState {
|
||||||
|
const messageSequence = state.messageSequence + 1
|
||||||
|
const next = { ...message, key: `message-${messageSequence}` }
|
||||||
|
return { ...state, messageSequence, messages: [...state.messages, next].slice(-MESSAGE_LIMIT) }
|
||||||
|
}
|
||||||
|
|
||||||
|
function reveal(state: VoiceViewState, previous: string, delta: string, now: number, animate: boolean) {
|
||||||
|
if (!animate) return { state, reveals: new Array<TextReveal>() }
|
||||||
|
const scheduled = scheduleTextReveal(previous, delta, now, state.revealAt)
|
||||||
|
return {
|
||||||
|
state: {
|
||||||
|
...state,
|
||||||
|
revealAt: scheduled.nextRevealAt,
|
||||||
|
revealAnimationEndsAt: Math.max(state.revealAnimationEndsAt, scheduled.animationEndsAt),
|
||||||
|
},
|
||||||
|
reveals: scheduled.reveals,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
function mergeAssistantRows(messages: ReadonlyArray<Message>) {
|
||||||
|
return messages.reduce<Message[]>((result, message) => {
|
||||||
|
const previous = result.at(-1)
|
||||||
|
if (previous?.kind !== "assistant" || message.kind !== "assistant") return [...result, message]
|
||||||
|
return [
|
||||||
|
...result.slice(0, -1),
|
||||||
|
{
|
||||||
|
key: previous.key,
|
||||||
|
kind: "assistant",
|
||||||
|
text: joinAssistantText(previous.text, message.text).text,
|
||||||
|
streaming: previous.streaming || message.streaming,
|
||||||
|
reveals: [],
|
||||||
|
},
|
||||||
|
]
|
||||||
|
}, [])
|
||||||
|
}
|
||||||
|
|
||||||
|
export function joinAssistantText(left: string, right: string) {
|
||||||
|
if (left.trim() === "") return { text: right, previous: "", appended: right }
|
||||||
|
if (right.trim() === "") return { text: left, previous: left, appended: "" }
|
||||||
|
const head = left.trimEnd()
|
||||||
|
const tail = right.trimStart()
|
||||||
|
if ((left.slice(head.length) + right.slice(0, right.length - tail.length)).includes("\n"))
|
||||||
|
return { text: left + right, previous: left, appended: right }
|
||||||
|
const separator = /^[.,;:!?…%)\]}]/.test(tail) || /[([{]$/.test(head) ? "" : " "
|
||||||
|
return { text: head + separator + tail, previous: head + separator, appended: tail }
|
||||||
|
}
|
||||||
@@ -0,0 +1,728 @@
|
|||||||
|
/** @jsxImportSource @opentui/solid */
|
||||||
|
// Terminal UI for the voice spike. The TUI keeps conversation order stable
|
||||||
|
// even though realtime events arrive out of order: a user row is inserted the
|
||||||
|
// moment the audio buffer commits (before the assistant starts replying) and
|
||||||
|
// its transcript is filled in when Whisper finishes.
|
||||||
|
import { createCliRenderer, RGBA } from "@opentui/core"
|
||||||
|
import { render, useKeyboard, useTerminalDimensions } from "@opentui/solid"
|
||||||
|
import "opentui-spinner/solid"
|
||||||
|
import { createSignal, For } from "solid-js"
|
||||||
|
import { createStore, reconcile } from "solid-js/store"
|
||||||
|
import {
|
||||||
|
connectionMeterLevels,
|
||||||
|
connectionTransitioning,
|
||||||
|
textRevealOpacity,
|
||||||
|
transcriptionPulse,
|
||||||
|
type TextReveal,
|
||||||
|
} from "./animation"
|
||||||
|
import { PCM_METER_FRAME_MS } from "./pcm"
|
||||||
|
import { initialVoiceView, transitionVoiceView, type Message, type VoiceViewEvent } from "./ui-model"
|
||||||
|
|
||||||
|
export { joinAssistantText } from "./ui-model"
|
||||||
|
|
||||||
|
export type VoiceStatus = {
|
||||||
|
server?: string
|
||||||
|
audio?: string
|
||||||
|
microphoneMuted?: boolean
|
||||||
|
speakerMuted?: boolean
|
||||||
|
voice?: string
|
||||||
|
model?: string
|
||||||
|
project?: string
|
||||||
|
}
|
||||||
|
|
||||||
|
export type VoiceUI = {
|
||||||
|
meta(text: string): void
|
||||||
|
userSpeaking(active: boolean): void
|
||||||
|
userReset(): void
|
||||||
|
userAudioLevel(level?: number): void
|
||||||
|
userCommitted(itemID: string): void
|
||||||
|
userTranscript(itemID: string, text: string, final?: boolean): void
|
||||||
|
assistantAudio(level: number, durationMs: number): void
|
||||||
|
assistantPlaybackStopped(): void
|
||||||
|
assistantDelta(text: string): void
|
||||||
|
assistantTranscript(text: string): void
|
||||||
|
assistantDone(): void
|
||||||
|
toolStart(callID: string, name: string, input: unknown): void
|
||||||
|
toolDone(callID: string, output: unknown): void
|
||||||
|
setStatus(patch: VoiceStatus): void
|
||||||
|
close(): void
|
||||||
|
}
|
||||||
|
|
||||||
|
const truncate = (text: string, max: number) => (text.length > max ? text.slice(0, max) + "…" : text)
|
||||||
|
|
||||||
|
const displayJson = (value: unknown) =>
|
||||||
|
truncate(
|
||||||
|
(() => {
|
||||||
|
try {
|
||||||
|
return (
|
||||||
|
JSON.stringify(
|
||||||
|
value,
|
||||||
|
(_, item) =>
|
||||||
|
typeof item === "string" ? truncate(item, 500) : typeof item === "bigint" ? String(item) : item,
|
||||||
|
2,
|
||||||
|
) ?? "null"
|
||||||
|
)
|
||||||
|
} catch {
|
||||||
|
return "[unserializable value]"
|
||||||
|
}
|
||||||
|
})(),
|
||||||
|
8_000,
|
||||||
|
)
|
||||||
|
|
||||||
|
function toolSummary(name: string, output?: unknown) {
|
||||||
|
if (output === undefined) return "running"
|
||||||
|
if (Array.isArray(output)) return `${output.length} result${output.length === 1 ? "" : "s"}`
|
||||||
|
if (typeof output === "string") return truncate(output, 160)
|
||||||
|
if (typeof output === "number" || typeof output === "boolean" || typeof output === "bigint") return String(output)
|
||||||
|
if (output === null || typeof output !== "object") return "completed"
|
||||||
|
if (
|
||||||
|
"status" in output &&
|
||||||
|
output.status === "error" &&
|
||||||
|
"message" in output &&
|
||||||
|
typeof output.message === "string"
|
||||||
|
)
|
||||||
|
return `error · ${truncate(output.message, 120)}`
|
||||||
|
if ("status" in output && typeof output.status === "string")
|
||||||
|
return output.status === "started" && "notification" in output && output.notification === "registered"
|
||||||
|
? "started · notification registered"
|
||||||
|
: output.status
|
||||||
|
const fields = Object.entries(output)
|
||||||
|
.filter(
|
||||||
|
([key, item]) =>
|
||||||
|
!key.endsWith("_id") && key !== "notification" && ["string", "number", "boolean"].includes(typeof item),
|
||||||
|
)
|
||||||
|
.slice(0, 2)
|
||||||
|
.map(([key, item]) => `${key}: ${String(item)}`)
|
||||||
|
return fields.join(" · ") || "completed"
|
||||||
|
}
|
||||||
|
|
||||||
|
// ---------------------------------------------------------------------------
|
||||||
|
// Console fallback (--text mode, non-TTY)
|
||||||
|
// ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
export function createConsoleUI(): VoiceUI {
|
||||||
|
const tty = process.stdout.isTTY
|
||||||
|
const dim = (text: string) => (tty ? `\x1b[2m${text}\x1b[0m` : text)
|
||||||
|
const cyan = (text: string) => (tty ? `\x1b[1;36m${text}\x1b[0m` : text)
|
||||||
|
const green = (text: string) => (tty ? `\x1b[1;32m${text}\x1b[0m` : text)
|
||||||
|
|
||||||
|
let streaming = false
|
||||||
|
const tools = new Map<string, string>()
|
||||||
|
const line = (text: string) => {
|
||||||
|
if (streaming) {
|
||||||
|
process.stdout.write("\n")
|
||||||
|
streaming = false
|
||||||
|
}
|
||||||
|
console.log(text)
|
||||||
|
}
|
||||||
|
|
||||||
|
return {
|
||||||
|
meta: (text) => line(dim(` ${text}`)),
|
||||||
|
userSpeaking: () => {},
|
||||||
|
userReset: () => {},
|
||||||
|
userAudioLevel: () => {},
|
||||||
|
userCommitted: () => {},
|
||||||
|
userTranscript: (_, text) => line(cyan("● you ") + text),
|
||||||
|
assistantAudio: () => {},
|
||||||
|
assistantPlaybackStopped: () => {},
|
||||||
|
assistantDelta: (text) => {
|
||||||
|
if (!streaming) {
|
||||||
|
process.stdout.write(green("● assistant "))
|
||||||
|
streaming = true
|
||||||
|
}
|
||||||
|
process.stdout.write(text)
|
||||||
|
},
|
||||||
|
assistantTranscript: (text) => line(green("● assistant ") + text),
|
||||||
|
assistantDone: () => {
|
||||||
|
if (streaming) process.stdout.write("\n")
|
||||||
|
streaming = false
|
||||||
|
},
|
||||||
|
toolStart: (callID, name) => {
|
||||||
|
tools.set(callID, name)
|
||||||
|
line(dim(` ◌ ${name} running`))
|
||||||
|
},
|
||||||
|
toolDone: (callID, output) => {
|
||||||
|
const name = tools.get(callID) ?? "tool"
|
||||||
|
tools.delete(callID)
|
||||||
|
line(dim(` ✓ ${name} ${toolSummary(name, output)}`))
|
||||||
|
},
|
||||||
|
setStatus: () => {},
|
||||||
|
close: () => {},
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// ---------------------------------------------------------------------------
|
||||||
|
// OpenTUI
|
||||||
|
// ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
const theme = {
|
||||||
|
background: "#1a1b26",
|
||||||
|
surface: "#0d1018",
|
||||||
|
text: "#c0caf5",
|
||||||
|
muted: "#565f89",
|
||||||
|
you: "#7dcfff",
|
||||||
|
assistant: "#9ece6a",
|
||||||
|
key: "#7aa2f7",
|
||||||
|
string: "#9ece6a",
|
||||||
|
number: "#e0af68",
|
||||||
|
literal: "#bb9af7",
|
||||||
|
}
|
||||||
|
|
||||||
|
const SPINNER_FRAMES = ["⠋", "⠙", "⠹", "⠸", "⠼", "⠴", "⠦", "⠧", "⠇", "⠏"]
|
||||||
|
const colors = {
|
||||||
|
background: RGBA.fromHex(theme.background),
|
||||||
|
text: RGBA.fromHex(theme.text),
|
||||||
|
you: RGBA.fromHex(theme.you),
|
||||||
|
assistant: RGBA.fromHex(theme.assistant),
|
||||||
|
}
|
||||||
|
const AUDIO_LEVEL_STALE_MS = 250
|
||||||
|
const TOOL_DISPLAY_DELAY_MS = 160
|
||||||
|
const meterGlyphs = ["▁", "▂", "▃", "▄", "▅", "▆", "▇", "█"]
|
||||||
|
|
||||||
|
function fade(color: RGBA, opacity: number) {
|
||||||
|
return RGBA.fromValues(color.r, color.g, color.b, color.a * opacity)
|
||||||
|
}
|
||||||
|
|
||||||
|
function surface(color: RGBA, opacity: number) {
|
||||||
|
return RGBA.fromValues(
|
||||||
|
colors.background.r + (color.r - colors.background.r) * opacity,
|
||||||
|
colors.background.g + (color.g - colors.background.g) * opacity,
|
||||||
|
colors.background.b + (color.b - colors.background.b) * opacity,
|
||||||
|
1,
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
function meter(levels: ReadonlyArray<number>) {
|
||||||
|
return levels
|
||||||
|
.map((value) => meterGlyphs[Math.round(Math.max(0, Math.min(1, value)) * (meterGlyphs.length - 1))])
|
||||||
|
.join("")
|
||||||
|
}
|
||||||
|
|
||||||
|
// Tiny JSON tokenizer for syntax-highlighted tool results.
|
||||||
|
function jsonTokens(json: string) {
|
||||||
|
const tokens: Array<{ text: string; color: string }> = []
|
||||||
|
const pattern = /("(?:[^"\\]|\\.)*")(\s*:)?|(-?\d+\.?\d*(?:[eE][+-]?\d+)?)|(true|false|null)|([{}[\],:]+|\s+)/g
|
||||||
|
for (const match of json.matchAll(pattern)) {
|
||||||
|
if (match[1] !== undefined) {
|
||||||
|
tokens.push({ text: match[1], color: match[2] ? theme.key : theme.string })
|
||||||
|
if (match[2]) tokens.push({ text: match[2], color: theme.muted })
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if (match[3] !== undefined) {
|
||||||
|
tokens.push({ text: match[3], color: theme.number })
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if (match[4] !== undefined) {
|
||||||
|
tokens.push({ text: match[4], color: theme.literal })
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
tokens.push({ text: match[5] ?? "", color: theme.muted })
|
||||||
|
}
|
||||||
|
return tokens
|
||||||
|
}
|
||||||
|
|
||||||
|
function RevealedText(props: {
|
||||||
|
text: string
|
||||||
|
reveals: ReadonlyArray<TextReveal>
|
||||||
|
now: () => number
|
||||||
|
opacity: () => number
|
||||||
|
}) {
|
||||||
|
return (
|
||||||
|
<>
|
||||||
|
<span style={{ fg: fade(colors.text, props.opacity()) }}>
|
||||||
|
{props.text.slice(0, props.reveals[0]?.offset ?? props.text.length)}
|
||||||
|
</span>
|
||||||
|
<For each={props.reveals}>
|
||||||
|
{(reveal, index) => (
|
||||||
|
<span style={{ fg: fade(colors.text, props.opacity() * textRevealOpacity(props.now(), reveal.at)) }}>
|
||||||
|
{props.text.slice(reveal.offset, props.reveals[index() + 1]?.offset ?? props.text.length)}
|
||||||
|
</span>
|
||||||
|
)}
|
||||||
|
</For>
|
||||||
|
</>
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Message transitions preserve unchanged object identities. A changed message
|
||||||
|
// gets a new object, so keyed <For> replaces that row and this branch reruns.
|
||||||
|
function MessageRow(props: {
|
||||||
|
message: Message
|
||||||
|
details: boolean
|
||||||
|
assistantLevel: () => number
|
||||||
|
userLevel: () => number
|
||||||
|
now: () => number
|
||||||
|
animate: boolean
|
||||||
|
microphoneMuted: boolean
|
||||||
|
previousKind?: Message["kind"]
|
||||||
|
}) {
|
||||||
|
const message = props.message
|
||||||
|
if (message.kind === "user") {
|
||||||
|
const transcribing = () => message.transcribing && !props.microphoneMuted
|
||||||
|
const pulse = () =>
|
||||||
|
!props.animate || !transcribing() ? 0 : Math.max(props.userLevel(), transcriptionPulse(props.now()) * 0.12)
|
||||||
|
return (
|
||||||
|
<box
|
||||||
|
width="100%"
|
||||||
|
flexDirection="column"
|
||||||
|
flexShrink={0}
|
||||||
|
border={["left"]}
|
||||||
|
borderColor={fade(colors.you, 0.38 + pulse() * 0.32)}
|
||||||
|
backgroundColor={surface(colors.you, 0.035 + pulse() * 0.07)}
|
||||||
|
paddingLeft={1}
|
||||||
|
paddingRight={1}
|
||||||
|
marginTop={1}
|
||||||
|
>
|
||||||
|
<text width="100%" minWidth={0} wrapMode="word">
|
||||||
|
<RevealedText
|
||||||
|
text={message.text ?? "transcribing"}
|
||||||
|
reveals={message.reveals}
|
||||||
|
now={props.now}
|
||||||
|
opacity={() => 0.74 + pulse() * 0.2}
|
||||||
|
/>
|
||||||
|
</text>
|
||||||
|
</box>
|
||||||
|
)
|
||||||
|
}
|
||||||
|
if (message.kind === "assistant") {
|
||||||
|
const pulse = () => (!props.animate || !message.streaming ? 0.5 : props.assistantLevel())
|
||||||
|
return (
|
||||||
|
<box
|
||||||
|
width="100%"
|
||||||
|
flexDirection="column"
|
||||||
|
flexShrink={0}
|
||||||
|
border={["left"]}
|
||||||
|
borderColor={fade(colors.assistant, message.streaming ? 0.58 + pulse() * 0.42 : 0.3)}
|
||||||
|
backgroundColor={surface(colors.assistant, message.streaming ? 0.055 + pulse() * 0.11 : 0.025)}
|
||||||
|
paddingLeft={1}
|
||||||
|
paddingRight={1}
|
||||||
|
marginTop={1}
|
||||||
|
>
|
||||||
|
<text width="100%" minWidth={0} wrapMode="word">
|
||||||
|
<RevealedText
|
||||||
|
text={message.text}
|
||||||
|
reveals={message.reveals}
|
||||||
|
now={props.now}
|
||||||
|
opacity={() => (message.streaming ? 1 : 0.7)}
|
||||||
|
/>
|
||||||
|
</text>
|
||||||
|
</box>
|
||||||
|
)
|
||||||
|
}
|
||||||
|
if (message.kind === "tool")
|
||||||
|
return (
|
||||||
|
<box width="100%" flexDirection="column" paddingLeft={2} marginTop={props.previousKind === "tool" ? 0 : 1}>
|
||||||
|
<box width="100%" minWidth={0} flexDirection="row">
|
||||||
|
<box width={2} flexShrink={0}>
|
||||||
|
{message.output === undefined ? (
|
||||||
|
props.animate ? (
|
||||||
|
<spinner frames={SPINNER_FRAMES} interval={80} color={theme.number} />
|
||||||
|
) : (
|
||||||
|
<text fg={theme.number}>⋯</text>
|
||||||
|
)
|
||||||
|
) : (
|
||||||
|
<text fg={theme.assistant}>✓</text>
|
||||||
|
)}
|
||||||
|
</box>
|
||||||
|
<text flexGrow={1} minWidth={0} wrapMode="word" fg={theme.muted}>
|
||||||
|
<span style={{ fg: theme.key }}>{message.name}</span> {toolSummary(message.name, message.output)}
|
||||||
|
</text>
|
||||||
|
</box>
|
||||||
|
{props.details ? (
|
||||||
|
<text width="100%" minWidth={0} wrapMode="word" fg={theme.muted} paddingLeft={2}>
|
||||||
|
<For each={jsonTokens(displayJson({ input: message.input, output: message.output }))}>
|
||||||
|
{(token) => <span style={{ fg: token.color }}>{token.text}</span>}
|
||||||
|
</For>
|
||||||
|
</text>
|
||||||
|
) : null}
|
||||||
|
</box>
|
||||||
|
)
|
||||||
|
return (
|
||||||
|
<box width="100%" minWidth={0} flexDirection="row" paddingLeft={2}>
|
||||||
|
<text width={3} flexShrink={0} fg={theme.muted}>
|
||||||
|
·
|
||||||
|
</text>
|
||||||
|
<text flexGrow={1} minWidth={0} wrapMode="word" fg={theme.muted}>
|
||||||
|
{message.text}
|
||||||
|
</text>
|
||||||
|
</box>
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
export async function createVoiceTUI(options: {
|
||||||
|
onInterrupt(): void
|
||||||
|
onExit(): void
|
||||||
|
onCycleVoice(): void
|
||||||
|
onToggleMicrophone(): void
|
||||||
|
onToggleSpeaker(): void
|
||||||
|
reducedMotion?: boolean
|
||||||
|
}): Promise<VoiceUI> {
|
||||||
|
let view = initialVoiceView()
|
||||||
|
const [messageState, setMessageState] = createStore({ items: [...view.messages] })
|
||||||
|
const [status, setStatus] = createSignal<VoiceStatus>({})
|
||||||
|
const [details, setDetails] = createSignal(false)
|
||||||
|
const [animationFrame, setAnimationFrame] = createSignal(performance.now())
|
||||||
|
const animationStartedAt = performance.now()
|
||||||
|
const [userActive, setUserActive] = createSignal(false)
|
||||||
|
let animationTimer: ReturnType<typeof setInterval> | undefined
|
||||||
|
let assistantSegments: Array<{ start: number; end: number; level: number }> = []
|
||||||
|
let assistantSegmentIndex = 0
|
||||||
|
let assistantScheduledUntil = 0
|
||||||
|
let assistantLevel = 0
|
||||||
|
let userTargetLevel = 0
|
||||||
|
let userLevel = 0
|
||||||
|
let userLevelAt = 0
|
||||||
|
let connectedAt: number | undefined
|
||||||
|
const pendingTools = new Map<
|
||||||
|
string,
|
||||||
|
{
|
||||||
|
name: string
|
||||||
|
input: unknown
|
||||||
|
output?: unknown
|
||||||
|
completed: boolean
|
||||||
|
showAt: number
|
||||||
|
timer: ReturnType<typeof setTimeout>
|
||||||
|
}
|
||||||
|
>()
|
||||||
|
const pendingMetaTimers = new Set<ReturnType<typeof setTimeout>>()
|
||||||
|
|
||||||
|
const renderer = await createCliRenderer({
|
||||||
|
useMouse: true,
|
||||||
|
// Handle Ctrl-C in OpenTUI's native input parser, before Solid handlers
|
||||||
|
// and rendering work. onDestroy performs process/audio cleanup below.
|
||||||
|
exitOnCtrlC: true,
|
||||||
|
exitSignals: [],
|
||||||
|
autoFocus: false,
|
||||||
|
openConsoleOnError: false,
|
||||||
|
screenMode: "alternate-screen",
|
||||||
|
externalOutputMode: "passthrough",
|
||||||
|
consoleMode: "disabled",
|
||||||
|
onDestroy: () => {
|
||||||
|
if (animationTimer) clearInterval(animationTimer)
|
||||||
|
pendingTools.forEach((tool) => clearTimeout(tool.timer))
|
||||||
|
pendingMetaTimers.forEach(clearTimeout)
|
||||||
|
options.onExit()
|
||||||
|
},
|
||||||
|
})
|
||||||
|
|
||||||
|
const applyView = (event: VoiceViewEvent) => {
|
||||||
|
const next = transitionVoiceView(view, event)
|
||||||
|
if (next === view) return false
|
||||||
|
view = next
|
||||||
|
setMessageState("items", reconcile([...view.messages], { key: "key" }))
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
|
||||||
|
const startAnimation = () => {
|
||||||
|
if (options.reducedMotion) return
|
||||||
|
if (animationTimer) return
|
||||||
|
animationTimer = setInterval(() => {
|
||||||
|
const now = performance.now()
|
||||||
|
while (assistantSegments[assistantSegmentIndex]?.end <= now) assistantSegmentIndex += 1
|
||||||
|
const segment = assistantSegments[assistantSegmentIndex]
|
||||||
|
const output = segment?.start <= now ? segment : undefined
|
||||||
|
const assistantTarget = output?.level ?? 0
|
||||||
|
const userTarget = now - userLevelAt < AUDIO_LEVEL_STALE_MS ? userTargetLevel : 0
|
||||||
|
assistantLevel += (assistantTarget - assistantLevel) * (assistantTarget > assistantLevel ? 0.5 : 0.16)
|
||||||
|
userLevel += (userTarget - userLevel) * (userTarget > userLevel ? 0.5 : 0.18)
|
||||||
|
setAnimationFrame(now)
|
||||||
|
if (view.revealAnimationEndsAt > 0 && now >= view.revealAnimationEndsAt) applyView({ type: "reveals.completed" })
|
||||||
|
const transcribing =
|
||||||
|
!status().microphoneMuted &&
|
||||||
|
messageState.items.some((message) => message.kind === "user" && message.transcribing)
|
||||||
|
if (
|
||||||
|
userActive() ||
|
||||||
|
transcribing ||
|
||||||
|
now < view.revealAnimationEndsAt ||
|
||||||
|
assistantSegmentIndex < assistantSegments.length ||
|
||||||
|
assistantLevel > 0.01 ||
|
||||||
|
userLevel > 0.01 ||
|
||||||
|
connectionTransitioning(now, connectedAt)
|
||||||
|
)
|
||||||
|
return
|
||||||
|
clearInterval(animationTimer)
|
||||||
|
animationTimer = undefined
|
||||||
|
}, PCM_METER_FRAME_MS)
|
||||||
|
}
|
||||||
|
|
||||||
|
const currentAssistantLevel = () => {
|
||||||
|
animationFrame()
|
||||||
|
return assistantLevel
|
||||||
|
}
|
||||||
|
const currentUserLevel = () => {
|
||||||
|
animationFrame()
|
||||||
|
return userLevel
|
||||||
|
}
|
||||||
|
|
||||||
|
function App() {
|
||||||
|
const dimensions = useTerminalDimensions()
|
||||||
|
useKeyboard((evt) => {
|
||||||
|
if (evt.ctrl && evt.name === "c") return
|
||||||
|
if (evt.repeated || evt.ctrl || evt.meta || evt.option || evt.shift) return
|
||||||
|
if (evt.name === "v") {
|
||||||
|
evt.preventDefault()
|
||||||
|
return options.onCycleVoice()
|
||||||
|
}
|
||||||
|
if (evt.name === "m") {
|
||||||
|
evt.preventDefault()
|
||||||
|
return options.onToggleMicrophone()
|
||||||
|
}
|
||||||
|
if (evt.name === "s") {
|
||||||
|
evt.preventDefault()
|
||||||
|
return options.onToggleSpeaker()
|
||||||
|
}
|
||||||
|
if (evt.name === "d") {
|
||||||
|
evt.preventDefault()
|
||||||
|
setDetails((current) => !current)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
if (evt.name === "escape") {
|
||||||
|
evt.preventDefault()
|
||||||
|
options.onInterrupt()
|
||||||
|
}
|
||||||
|
})
|
||||||
|
const runtimeLine = () =>
|
||||||
|
[
|
||||||
|
status().audio ?? "connecting…",
|
||||||
|
status().microphoneMuted ? "mic muted" : undefined,
|
||||||
|
status().speakerMuted ? "speaker muted" : undefined,
|
||||||
|
status().voice,
|
||||||
|
status().model,
|
||||||
|
status().project,
|
||||||
|
]
|
||||||
|
.filter(Boolean)
|
||||||
|
.join(" ")
|
||||||
|
const microphoneLevel = () => {
|
||||||
|
if (status().microphoneMuted) return 0
|
||||||
|
return Math.max(userActive() ? 0.16 : 0.1, currentUserLevel())
|
||||||
|
}
|
||||||
|
const microphoneOpacity = () => (status().microphoneMuted ? 0.16 : userActive() ? 0.9 : 0.38)
|
||||||
|
const microphoneMeter = () => {
|
||||||
|
if (options.reducedMotion) return meter(connectionMeterLevels(0, microphoneLevel(), -480))
|
||||||
|
return meter(connectionMeterLevels(animationFrame(), microphoneLevel(), connectedAt, animationStartedAt))
|
||||||
|
}
|
||||||
|
return (
|
||||||
|
<box
|
||||||
|
flexDirection="column"
|
||||||
|
width={dimensions().width}
|
||||||
|
height={dimensions().height}
|
||||||
|
paddingLeft={1}
|
||||||
|
paddingRight={1}
|
||||||
|
backgroundColor={theme.background}
|
||||||
|
>
|
||||||
|
<box width="100%" flexGrow={1} minHeight={0} minWidth={0} backgroundColor={theme.background}>
|
||||||
|
<scrollbox
|
||||||
|
width="100%"
|
||||||
|
height="100%"
|
||||||
|
stickyScroll
|
||||||
|
stickyStart="bottom"
|
||||||
|
scrollbarOptions={{ visible: false }}
|
||||||
|
backgroundColor={theme.background}
|
||||||
|
>
|
||||||
|
<box
|
||||||
|
width="100%"
|
||||||
|
minWidth={0}
|
||||||
|
minHeight={Math.max(0, dimensions().height - 3)}
|
||||||
|
flexDirection="column"
|
||||||
|
flexShrink={0}
|
||||||
|
justifyContent="flex-end"
|
||||||
|
backgroundColor={theme.background}
|
||||||
|
>
|
||||||
|
<For each={messageState.items}>
|
||||||
|
{(message, index) => (
|
||||||
|
<MessageRow
|
||||||
|
message={message}
|
||||||
|
previousKind={messageState.items[index() - 1]?.kind}
|
||||||
|
details={details()}
|
||||||
|
assistantLevel={currentAssistantLevel}
|
||||||
|
userLevel={currentUserLevel}
|
||||||
|
now={animationFrame}
|
||||||
|
animate={!options.reducedMotion}
|
||||||
|
microphoneMuted={status().microphoneMuted === true}
|
||||||
|
/>
|
||||||
|
)}
|
||||||
|
</For>
|
||||||
|
</box>
|
||||||
|
</scrollbox>
|
||||||
|
</box>
|
||||||
|
<box width="100%" height={1} flexShrink={0} backgroundColor={theme.background} />
|
||||||
|
<box width="100%" height={2} flexShrink={0} backgroundColor={theme.surface} paddingLeft={1} paddingRight={1}>
|
||||||
|
<box width="100%" height={1} minWidth={0} flexDirection="row">
|
||||||
|
<text width={5} flexShrink={0} fg={fade(colors.you, microphoneOpacity())}>
|
||||||
|
<b>
|
||||||
|
{status().microphoneMuted
|
||||||
|
? "────"
|
||||||
|
: microphoneMeter()}
|
||||||
|
</b>
|
||||||
|
</text>
|
||||||
|
<text flexGrow={1} minWidth={0} wrapMode="none" truncate fg={fade(colors.text, 0.78)}>
|
||||||
|
{runtimeLine()}
|
||||||
|
</text>
|
||||||
|
</box>
|
||||||
|
<text width="100%" wrapMode="none" truncate fg={theme.muted}>
|
||||||
|
<span style={{ fg: theme.key }}>esc</span> interrupt {" "}
|
||||||
|
<span style={{ fg: theme.key }}>v</span> voice {" "}
|
||||||
|
<span style={{ fg: theme.key }}>m</span> mic {" "}
|
||||||
|
<span style={{ fg: theme.key }}>s</span> speaker {" "}
|
||||||
|
<span style={{ fg: theme.key }}>d</span> details {details() ? "on" : "off"} {" "}
|
||||||
|
<span style={{ fg: theme.key }}>ctrl+c</span> quit
|
||||||
|
</text>
|
||||||
|
</box>
|
||||||
|
</box>
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
await render(() => <App />, renderer)
|
||||||
|
startAnimation()
|
||||||
|
|
||||||
|
const pushMeta = (text: string) => {
|
||||||
|
if (pendingTools.size === 0) {
|
||||||
|
applyView({ type: "meta", text })
|
||||||
|
return
|
||||||
|
}
|
||||||
|
const showAt = Math.max(...[...pendingTools.values()].map((tool) => tool.showAt))
|
||||||
|
const timer = setTimeout(
|
||||||
|
() => {
|
||||||
|
pendingMetaTimers.delete(timer)
|
||||||
|
applyView({ type: "meta", text })
|
||||||
|
},
|
||||||
|
Math.max(0, showAt - performance.now()) + 1,
|
||||||
|
)
|
||||||
|
pendingMetaTimers.add(timer)
|
||||||
|
}
|
||||||
|
|
||||||
|
return {
|
||||||
|
meta: pushMeta,
|
||||||
|
userSpeaking: (active) => {
|
||||||
|
if (active) applyView({ type: "user.started" })
|
||||||
|
setUserActive(active)
|
||||||
|
if (active) startAnimation()
|
||||||
|
if (!active) userTargetLevel = 0
|
||||||
|
},
|
||||||
|
userReset: () => {
|
||||||
|
setUserActive(false)
|
||||||
|
userTargetLevel = 0
|
||||||
|
userLevel = 0
|
||||||
|
applyView({ type: "user.reset" })
|
||||||
|
},
|
||||||
|
userAudioLevel: (level) => {
|
||||||
|
if (level === undefined) {
|
||||||
|
userLevelAt = 0
|
||||||
|
return
|
||||||
|
}
|
||||||
|
userTargetLevel = Math.max(0, Math.min(1, level))
|
||||||
|
userLevelAt = performance.now()
|
||||||
|
if (options.reducedMotion) {
|
||||||
|
userLevel = userTargetLevel
|
||||||
|
setAnimationFrame(userLevelAt)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
startAnimation()
|
||||||
|
},
|
||||||
|
userCommitted: (itemID) => {
|
||||||
|
setUserActive(false)
|
||||||
|
applyView({ type: "user.committed", itemID })
|
||||||
|
startAnimation()
|
||||||
|
},
|
||||||
|
userTranscript: (itemID, text, final = true) => {
|
||||||
|
applyView({
|
||||||
|
type: "user.transcript",
|
||||||
|
itemID,
|
||||||
|
text,
|
||||||
|
final,
|
||||||
|
now: performance.now(),
|
||||||
|
animate: !options.reducedMotion,
|
||||||
|
})
|
||||||
|
if (!final || view.revealAnimationEndsAt > 0) startAnimation()
|
||||||
|
},
|
||||||
|
assistantAudio: (level, durationMs) => {
|
||||||
|
if (options.reducedMotion) return
|
||||||
|
if (assistantSegmentIndex > 512) {
|
||||||
|
assistantSegments = assistantSegments.slice(assistantSegmentIndex)
|
||||||
|
assistantSegmentIndex = 0
|
||||||
|
}
|
||||||
|
const now = performance.now()
|
||||||
|
const start = Math.max(now, assistantScheduledUntil)
|
||||||
|
const end = start + durationMs
|
||||||
|
assistantSegments.push({ start, end, level: Math.max(0, Math.min(1, level)) })
|
||||||
|
assistantScheduledUntil = end
|
||||||
|
startAnimation()
|
||||||
|
},
|
||||||
|
assistantPlaybackStopped: () => {
|
||||||
|
assistantSegments = []
|
||||||
|
assistantSegmentIndex = 0
|
||||||
|
assistantScheduledUntil = 0
|
||||||
|
assistantLevel = 0
|
||||||
|
setAnimationFrame(performance.now())
|
||||||
|
},
|
||||||
|
assistantDelta: (text) => {
|
||||||
|
applyView({
|
||||||
|
type: "assistant.delta",
|
||||||
|
text,
|
||||||
|
now: performance.now(),
|
||||||
|
animate: !options.reducedMotion,
|
||||||
|
})
|
||||||
|
if (view.revealAnimationEndsAt > 0) startAnimation()
|
||||||
|
},
|
||||||
|
assistantTranscript: (text) => {
|
||||||
|
applyView({
|
||||||
|
type: "assistant.transcript",
|
||||||
|
text,
|
||||||
|
now: performance.now(),
|
||||||
|
animate: !options.reducedMotion,
|
||||||
|
})
|
||||||
|
if (view.revealAnimationEndsAt > 0) startAnimation()
|
||||||
|
},
|
||||||
|
assistantDone: () => {
|
||||||
|
assistantSegments = []
|
||||||
|
assistantSegmentIndex = 0
|
||||||
|
assistantScheduledUntil = 0
|
||||||
|
assistantLevel = 0
|
||||||
|
applyView({ type: "assistant.done" })
|
||||||
|
},
|
||||||
|
toolStart: (callID, name, input) => {
|
||||||
|
if (
|
||||||
|
pendingTools.has(callID) ||
|
||||||
|
view.messages.some((message) => message.kind === "tool" && message.callID === callID)
|
||||||
|
)
|
||||||
|
return
|
||||||
|
const pending = {
|
||||||
|
name,
|
||||||
|
input,
|
||||||
|
completed: false,
|
||||||
|
showAt: performance.now() + TOOL_DISPLAY_DELAY_MS,
|
||||||
|
timer: setTimeout(() => {
|
||||||
|
const current = pendingTools.get(callID)
|
||||||
|
if (!current) return
|
||||||
|
pendingTools.delete(callID)
|
||||||
|
applyView({
|
||||||
|
type: "tool.started",
|
||||||
|
callID,
|
||||||
|
name: current.name,
|
||||||
|
input: current.input,
|
||||||
|
})
|
||||||
|
if (current.completed) applyView({ type: "tool.done", callID, output: current.output })
|
||||||
|
}, TOOL_DISPLAY_DELAY_MS),
|
||||||
|
}
|
||||||
|
pendingTools.set(callID, pending)
|
||||||
|
},
|
||||||
|
toolDone: (callID, output) => {
|
||||||
|
const pending = pendingTools.get(callID)
|
||||||
|
if (pending) {
|
||||||
|
pending.output = output
|
||||||
|
pending.completed = true
|
||||||
|
return
|
||||||
|
}
|
||||||
|
applyView({ type: "tool.done", callID, output })
|
||||||
|
},
|
||||||
|
setStatus: (patch) => {
|
||||||
|
if (connectedAt === undefined && patch.audio !== undefined) {
|
||||||
|
connectedAt = performance.now()
|
||||||
|
startAnimation()
|
||||||
|
}
|
||||||
|
setStatus((current) => ({ ...current, ...patch }))
|
||||||
|
},
|
||||||
|
close: () => {
|
||||||
|
if (!renderer.isDestroyed) renderer.destroy()
|
||||||
|
},
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,242 @@
|
|||||||
|
import type { CompletionReceipt } from "./opencode-notification"
|
||||||
|
|
||||||
|
export type VoiceNotification = {
|
||||||
|
readonly id: string
|
||||||
|
readonly receipt?: CompletionReceipt
|
||||||
|
readonly promptID?: string
|
||||||
|
readonly delegationID?: string
|
||||||
|
readonly text: string
|
||||||
|
}
|
||||||
|
|
||||||
|
export type VoiceState = {
|
||||||
|
readonly connection: "connecting" | "ready" | "closed"
|
||||||
|
readonly conversation: "idle" | "waiting" | "responding"
|
||||||
|
readonly assistant: "idle" | "active" | "suppressed"
|
||||||
|
readonly userSpeaking: boolean
|
||||||
|
readonly tools: ReadonlySet<string>
|
||||||
|
readonly notifications: ReadonlyArray<VoiceNotification>
|
||||||
|
readonly delivery?: {
|
||||||
|
readonly notification: VoiceNotification
|
||||||
|
readonly phase: "sending" | "accepted" | "announcing" | "awaiting-playback"
|
||||||
|
}
|
||||||
|
readonly desiredVoice: string
|
||||||
|
readonly reconnect: "none" | "pending"
|
||||||
|
}
|
||||||
|
|
||||||
|
export type VoiceEvent =
|
||||||
|
| { readonly type: "connection.ready" }
|
||||||
|
| { readonly type: "connection.connecting" }
|
||||||
|
| { readonly type: "connection.closed" }
|
||||||
|
| { readonly type: "user.started"; readonly bargeIn?: boolean }
|
||||||
|
| { readonly type: "user.stopped" }
|
||||||
|
| { readonly type: "user.committed" }
|
||||||
|
| { readonly type: "assistant.started" }
|
||||||
|
| { readonly type: "assistant.suppressed" }
|
||||||
|
| { readonly type: "assistant.done"; readonly awaitingWork: boolean }
|
||||||
|
| { readonly type: "tool.started"; readonly id: string }
|
||||||
|
| { readonly type: "tool.finished"; readonly id: string }
|
||||||
|
| { readonly type: "notification.queued"; readonly notification: VoiceNotification }
|
||||||
|
| { readonly type: "notification.accepted"; readonly id: string }
|
||||||
|
| { readonly type: "notification.failed"; readonly id: string }
|
||||||
|
| { readonly type: "notification.announced"; readonly id: string }
|
||||||
|
| { readonly type: "notification.interrupted"; readonly id: string }
|
||||||
|
| { readonly type: "voice.selected"; readonly voice: string }
|
||||||
|
|
||||||
|
export type VoiceCommand =
|
||||||
|
| { readonly type: "notification.send"; readonly notification: VoiceNotification }
|
||||||
|
| { readonly type: "notification.delivered"; readonly notification: VoiceNotification }
|
||||||
|
| { readonly type: "assistant.finish"; readonly awaitingWork: boolean; readonly notificationID?: string }
|
||||||
|
| { readonly type: "assistant.interrupt" }
|
||||||
|
| { readonly type: "connection.reconnect" }
|
||||||
|
|
||||||
|
export type VoiceTransition = {
|
||||||
|
readonly state: VoiceState
|
||||||
|
readonly commands: ReadonlyArray<VoiceCommand>
|
||||||
|
}
|
||||||
|
|
||||||
|
export function initialVoiceState(voice: string): VoiceState {
|
||||||
|
return {
|
||||||
|
connection: "connecting",
|
||||||
|
conversation: "idle",
|
||||||
|
assistant: "idle",
|
||||||
|
userSpeaking: false,
|
||||||
|
tools: new Set(),
|
||||||
|
notifications: [],
|
||||||
|
desiredVoice: voice,
|
||||||
|
reconnect: "none",
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
export function transitionVoice(state: VoiceState, event: VoiceEvent): VoiceTransition {
|
||||||
|
if (event.type === "notification.announced") {
|
||||||
|
if (state.delivery?.notification.id !== event.id || state.delivery.phase !== "awaiting-playback")
|
||||||
|
return { state, commands: [] }
|
||||||
|
const notification = state.delivery.notification
|
||||||
|
const next = settle({ ...state, delivery: undefined })
|
||||||
|
return {
|
||||||
|
state: next.state,
|
||||||
|
commands: [{ type: "notification.delivered", notification }, ...next.commands],
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if (event.type === "assistant.done") {
|
||||||
|
const notificationID =
|
||||||
|
state.delivery?.phase === "announcing" && state.assistant === "active" && !event.awaitingWork
|
||||||
|
? state.delivery.notification.id
|
||||||
|
: undefined
|
||||||
|
const next = settle(reduce(state, event))
|
||||||
|
return {
|
||||||
|
state: next.state,
|
||||||
|
commands: [{ type: "assistant.finish", awaitingWork: event.awaitingWork, notificationID }, ...next.commands],
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if (event.type === "user.started" && event.bargeIn && state.assistant === "active") {
|
||||||
|
const next = settle(
|
||||||
|
state.delivery
|
||||||
|
? {
|
||||||
|
...state,
|
||||||
|
conversation: "idle",
|
||||||
|
assistant: "idle",
|
||||||
|
userSpeaking: true,
|
||||||
|
delivery: undefined,
|
||||||
|
notifications: [state.delivery.notification, ...state.notifications],
|
||||||
|
reconnect: "pending",
|
||||||
|
}
|
||||||
|
: { ...state, assistant: "suppressed", userSpeaking: true },
|
||||||
|
)
|
||||||
|
return { state: next.state, commands: [{ type: "assistant.interrupt" }, ...next.commands] }
|
||||||
|
}
|
||||||
|
const next = reduce(state, event)
|
||||||
|
return settle(next)
|
||||||
|
}
|
||||||
|
|
||||||
|
function reduce(state: VoiceState, event: VoiceEvent): VoiceState {
|
||||||
|
switch (event.type) {
|
||||||
|
case "connection.ready":
|
||||||
|
if (state.connection === "ready") return state
|
||||||
|
return { ...state, connection: "ready" }
|
||||||
|
case "connection.connecting":
|
||||||
|
return {
|
||||||
|
...state,
|
||||||
|
connection: "connecting",
|
||||||
|
conversation: "idle",
|
||||||
|
assistant: "idle",
|
||||||
|
userSpeaking: false,
|
||||||
|
}
|
||||||
|
case "connection.closed":
|
||||||
|
return { ...state, connection: "closed", conversation: "idle", assistant: "idle", userSpeaking: false }
|
||||||
|
case "user.started":
|
||||||
|
if (state.userSpeaking) return state
|
||||||
|
return { ...state, userSpeaking: true }
|
||||||
|
case "user.stopped":
|
||||||
|
if (!state.userSpeaking) return state
|
||||||
|
return { ...state, userSpeaking: false }
|
||||||
|
case "user.committed":
|
||||||
|
return {
|
||||||
|
...state,
|
||||||
|
userSpeaking: false,
|
||||||
|
conversation: state.assistant === "active" ? "responding" : "waiting",
|
||||||
|
assistant: state.assistant === "suppressed" ? "idle" : state.assistant,
|
||||||
|
}
|
||||||
|
case "assistant.started":
|
||||||
|
if (state.assistant === "suppressed" || state.assistant === "active") return state
|
||||||
|
return {
|
||||||
|
...state,
|
||||||
|
conversation: "responding",
|
||||||
|
assistant: "active",
|
||||||
|
delivery:
|
||||||
|
state.delivery?.phase === "accepted" ? { ...state.delivery, phase: "announcing" } : state.delivery,
|
||||||
|
}
|
||||||
|
case "assistant.suppressed":
|
||||||
|
return { ...state, assistant: "suppressed" }
|
||||||
|
case "assistant.done":
|
||||||
|
return {
|
||||||
|
...state,
|
||||||
|
conversation: event.awaitingWork ? "waiting" : "idle",
|
||||||
|
assistant: "idle",
|
||||||
|
delivery:
|
||||||
|
state.delivery?.phase === "announcing" && state.assistant === "active" && !event.awaitingWork
|
||||||
|
? { ...state.delivery, phase: "awaiting-playback" }
|
||||||
|
: state.delivery,
|
||||||
|
}
|
||||||
|
case "tool.started":
|
||||||
|
if (state.tools.has(event.id)) return state
|
||||||
|
return { ...state, conversation: "waiting", tools: new Set([...state.tools, event.id]) }
|
||||||
|
case "tool.finished": {
|
||||||
|
if (!state.tools.has(event.id)) return state
|
||||||
|
const tools = new Set(state.tools)
|
||||||
|
tools.delete(event.id)
|
||||||
|
return { ...state, tools }
|
||||||
|
}
|
||||||
|
case "notification.queued":
|
||||||
|
return { ...state, notifications: [...state.notifications, event.notification] }
|
||||||
|
case "notification.accepted":
|
||||||
|
if (state.delivery?.notification.id !== event.id || state.delivery.phase !== "sending") return state
|
||||||
|
return {
|
||||||
|
...state,
|
||||||
|
delivery: { ...state.delivery, phase: "accepted" },
|
||||||
|
}
|
||||||
|
case "notification.failed":
|
||||||
|
if (state.delivery?.notification.id !== event.id) return state
|
||||||
|
return {
|
||||||
|
...state,
|
||||||
|
connection: "ready",
|
||||||
|
conversation: "idle",
|
||||||
|
delivery: undefined,
|
||||||
|
notifications: [state.delivery.notification, ...state.notifications],
|
||||||
|
reconnect: "pending",
|
||||||
|
}
|
||||||
|
case "notification.announced":
|
||||||
|
return state
|
||||||
|
case "notification.interrupted":
|
||||||
|
if (state.delivery?.notification.id !== event.id) return state
|
||||||
|
return {
|
||||||
|
...state,
|
||||||
|
conversation: "idle",
|
||||||
|
assistant: "idle",
|
||||||
|
delivery: undefined,
|
||||||
|
notifications: [state.delivery.notification, ...state.notifications],
|
||||||
|
reconnect: "pending",
|
||||||
|
}
|
||||||
|
case "voice.selected":
|
||||||
|
if (event.voice === state.desiredVoice) return state
|
||||||
|
return { ...state, desiredVoice: event.voice, reconnect: "pending" }
|
||||||
|
}
|
||||||
|
return state
|
||||||
|
}
|
||||||
|
|
||||||
|
function settle(state: VoiceState): VoiceTransition {
|
||||||
|
if (
|
||||||
|
state.reconnect === "pending" &&
|
||||||
|
state.connection === "ready" &&
|
||||||
|
state.conversation === "idle" &&
|
||||||
|
state.delivery === undefined &&
|
||||||
|
!state.userSpeaking &&
|
||||||
|
state.tools.size === 0
|
||||||
|
)
|
||||||
|
return {
|
||||||
|
state: { ...state, connection: "connecting", reconnect: "none" },
|
||||||
|
commands: [{ type: "connection.reconnect" }],
|
||||||
|
}
|
||||||
|
|
||||||
|
if (
|
||||||
|
state.notifications.length > 0 &&
|
||||||
|
state.delivery === undefined &&
|
||||||
|
state.connection === "ready" &&
|
||||||
|
state.conversation === "idle" &&
|
||||||
|
!state.userSpeaking &&
|
||||||
|
state.tools.size === 0
|
||||||
|
) {
|
||||||
|
const notification = state.notifications[0]
|
||||||
|
return {
|
||||||
|
state: {
|
||||||
|
...state,
|
||||||
|
conversation: "waiting",
|
||||||
|
delivery: { notification, phase: "sending" },
|
||||||
|
notifications: state.notifications.slice(1),
|
||||||
|
},
|
||||||
|
commands: [{ type: "notification.send", notification }],
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return { state, commands: [] }
|
||||||
|
}
|
||||||
@@ -0,0 +1,446 @@
|
|||||||
|
import type { AudioEvent, AudioSession } from "./audio-session"
|
||||||
|
import { openCodeAnnouncementText, type CompletionReceipt, type OpenCodeAnnouncement } from "./opencode-notification"
|
||||||
|
import { createNotificationRouter } from "./notification-router"
|
||||||
|
import type {
|
||||||
|
VoiceConnection,
|
||||||
|
VoiceDelegationRequest,
|
||||||
|
VoiceProtocol,
|
||||||
|
VoiceProtocolEvent,
|
||||||
|
VoiceProtocolOptions,
|
||||||
|
VoiceTool,
|
||||||
|
VoiceToolExecution,
|
||||||
|
VoiceWorkRequest,
|
||||||
|
} from "./protocol"
|
||||||
|
import type { VoiceUI } from "./ui"
|
||||||
|
import { initialVoiceState, transitionVoice, type VoiceCommand, type VoiceEvent } from "./voice-coordinator"
|
||||||
|
|
||||||
|
export type VoiceSession = {
|
||||||
|
start(): void
|
||||||
|
cycleVoice(): void
|
||||||
|
interrupt(): void
|
||||||
|
toggleMicrophone(): void
|
||||||
|
toggleSpeaker(): void
|
||||||
|
queueNotification(announcement: OpenCodeAnnouncement): void
|
||||||
|
close(): Promise<void>
|
||||||
|
}
|
||||||
|
|
||||||
|
export function voiceControlTool(voices: ReadonlyArray<string>): VoiceTool {
|
||||||
|
return {
|
||||||
|
type: "function",
|
||||||
|
name: "set_voice",
|
||||||
|
description: "Change your speaking voice. Requires a brief reconnect and resets voice conversation memory.",
|
||||||
|
parameters: {
|
||||||
|
type: "object",
|
||||||
|
additionalProperties: false,
|
||||||
|
properties: { voice: { type: "string", enum: voices } },
|
||||||
|
required: ["voice"],
|
||||||
|
},
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
export function createVoiceSession(options: {
|
||||||
|
readonly protocol: VoiceProtocol
|
||||||
|
readonly connection: Omit<VoiceProtocolOptions, "voice" | "onEvent">
|
||||||
|
readonly initialVoice: string
|
||||||
|
readonly voices: ReadonlyArray<string>
|
||||||
|
readonly text?: string
|
||||||
|
readonly ui: VoiceUI
|
||||||
|
readonly audio?: AudioSession
|
||||||
|
readonly tools: {
|
||||||
|
readonly execute: (name: string, input: Record<string, unknown>) => Promise<VoiceToolExecution>
|
||||||
|
readonly acknowledge: (receipt: CompletionReceipt) => Promise<void>
|
||||||
|
readonly close: () => Promise<void>
|
||||||
|
}
|
||||||
|
readonly delegate: (
|
||||||
|
request: VoiceDelegationRequest,
|
||||||
|
execute: (request: VoiceWorkRequest) => Promise<unknown>,
|
||||||
|
) => Promise<string>
|
||||||
|
readonly trace?: (event: string, data?: Record<string, unknown>) => void
|
||||||
|
readonly onClosed: () => void
|
||||||
|
}): VoiceSession {
|
||||||
|
let state = initialVoiceState(options.initialVoice)
|
||||||
|
let connection: VoiceConnection | undefined
|
||||||
|
let closed = false
|
||||||
|
let closePromise: Promise<void> | undefined
|
||||||
|
let voiceTimer: ReturnType<typeof setTimeout> | undefined
|
||||||
|
const notifications = createNotificationRouter((routed) => {
|
||||||
|
options.trace?.("notification.queued", {
|
||||||
|
type: routed.announcement.notification.type,
|
||||||
|
promptID: routed.promptID,
|
||||||
|
depth: state.notifications.length + 1,
|
||||||
|
})
|
||||||
|
dispatch({
|
||||||
|
type: "notification.queued",
|
||||||
|
notification: {
|
||||||
|
id: crypto.randomUUID(),
|
||||||
|
receipt: routed.announcement.receipt,
|
||||||
|
promptID: routed.promptID,
|
||||||
|
delegationID: routed.delegationID,
|
||||||
|
text: openCodeAnnouncementText(routed.announcement.notification),
|
||||||
|
},
|
||||||
|
})
|
||||||
|
})
|
||||||
|
|
||||||
|
function dispatch(event: VoiceEvent) {
|
||||||
|
const transition = transitionVoice(state, event)
|
||||||
|
if (transition.state === state && transition.commands.length === 0) return
|
||||||
|
state = transition.state
|
||||||
|
options.trace?.("voice.transition", {
|
||||||
|
input: event.type,
|
||||||
|
connection: state.connection,
|
||||||
|
conversation: state.conversation,
|
||||||
|
assistant: state.assistant,
|
||||||
|
userSpeaking: state.userSpeaking,
|
||||||
|
tools: state.tools.size,
|
||||||
|
notifications: state.notifications.length,
|
||||||
|
commands: transition.commands.map((command) => command.type),
|
||||||
|
})
|
||||||
|
transition.commands.forEach(runCommand)
|
||||||
|
}
|
||||||
|
|
||||||
|
const setVoice = (voice: string) => {
|
||||||
|
options.ui.setStatus({ voice })
|
||||||
|
options.ui.meta(`[voice] switching to ${voice}…`)
|
||||||
|
dispatch({ type: "voice.selected", voice })
|
||||||
|
}
|
||||||
|
|
||||||
|
const interruptDelivery = () => {
|
||||||
|
if (state.delivery) dispatch({ type: "notification.interrupted", id: state.delivery.notification.id })
|
||||||
|
}
|
||||||
|
|
||||||
|
const reconnect = () => {
|
||||||
|
if (closed) return
|
||||||
|
options.audio?.flushPlayback()
|
||||||
|
options.ui.assistantDone()
|
||||||
|
const previous = connection
|
||||||
|
connection = undefined
|
||||||
|
const next = () => {
|
||||||
|
if (!closed) connect()
|
||||||
|
}
|
||||||
|
if (!previous) return next()
|
||||||
|
void previous.close({ graceful: false }).finally(next)
|
||||||
|
}
|
||||||
|
|
||||||
|
function runCommand(command: VoiceCommand) {
|
||||||
|
if (command.type === "connection.reconnect") return reconnect()
|
||||||
|
if (command.type === "assistant.interrupt") {
|
||||||
|
connection?.interrupt?.()
|
||||||
|
options.audio?.flushPlayback()
|
||||||
|
options.ui.assistantDone()
|
||||||
|
return
|
||||||
|
}
|
||||||
|
if (command.type === "assistant.finish") {
|
||||||
|
if (options.text) {
|
||||||
|
options.ui.assistantDone()
|
||||||
|
if (command.notificationID) dispatch({ type: "notification.announced", id: command.notificationID })
|
||||||
|
if (!command.awaitingWork && state.tools.size === 0) options.onClosed()
|
||||||
|
return
|
||||||
|
}
|
||||||
|
void options.audio?.finishPlayback().then((outcome) => {
|
||||||
|
options.ui.assistantDone()
|
||||||
|
if (!command.notificationID) return
|
||||||
|
dispatch({
|
||||||
|
type: outcome === "played" ? "notification.announced" : "notification.interrupted",
|
||||||
|
id: command.notificationID,
|
||||||
|
})
|
||||||
|
})
|
||||||
|
return
|
||||||
|
}
|
||||||
|
if (command.type === "notification.delivered") {
|
||||||
|
options.trace?.("notification.announced", {
|
||||||
|
id: command.notification.id,
|
||||||
|
promptID: command.notification.promptID,
|
||||||
|
})
|
||||||
|
if (!command.notification.receipt) return
|
||||||
|
const receipt = command.notification.receipt
|
||||||
|
void options.tools
|
||||||
|
.acknowledge(receipt)
|
||||||
|
.catch((error) =>
|
||||||
|
options.trace?.("notification.ack.failed", { id: command.notification.id, error: String(error) }),
|
||||||
|
)
|
||||||
|
.finally(() => notifications.acknowledged(receipt))
|
||||||
|
return
|
||||||
|
}
|
||||||
|
const active = connection
|
||||||
|
if (!active) return dispatch({ type: "notification.failed", id: command.notification.id })
|
||||||
|
void active
|
||||||
|
.notify({
|
||||||
|
id: command.notification.id,
|
||||||
|
text: `Announce this OpenCode update conversationally without reading private identifiers aloud:\n${command.notification.text}`,
|
||||||
|
delegationID: command.notification.delegationID,
|
||||||
|
})
|
||||||
|
.then((accepted) => {
|
||||||
|
if (!accepted) return dispatch({ type: "notification.failed", id: command.notification.id })
|
||||||
|
dispatch({ type: "notification.accepted", id: command.notification.id })
|
||||||
|
options.trace?.("notification.context.accepted", {
|
||||||
|
id: command.notification.id,
|
||||||
|
promptID: command.notification.promptID,
|
||||||
|
remaining: state.notifications.length,
|
||||||
|
})
|
||||||
|
})
|
||||||
|
.catch((error) => {
|
||||||
|
options.trace?.("notification.context.failed", {
|
||||||
|
id: command.notification.id,
|
||||||
|
promptID: command.notification.promptID,
|
||||||
|
error: String(error),
|
||||||
|
})
|
||||||
|
dispatch({ type: "notification.failed", id: command.notification.id })
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
const executeTool = async (request: VoiceWorkRequest, delegationID?: string) => {
|
||||||
|
if (delegationID) notifications.beginDelegatedTool()
|
||||||
|
const execution: VoiceToolExecution = await (request.name === "set_voice"
|
||||||
|
? Promise.resolve().then(() => {
|
||||||
|
const voice = request.input["voice"]
|
||||||
|
if (typeof voice !== "string" || !options.voices.includes(voice))
|
||||||
|
return { output: toolError(`Voice must be one of: ${options.voices.join(", ")}.`) }
|
||||||
|
if (voiceTimer) clearTimeout(voiceTimer)
|
||||||
|
voiceTimer = setTimeout(() => {
|
||||||
|
voiceTimer = undefined
|
||||||
|
setVoice(voice)
|
||||||
|
}, 1_000)
|
||||||
|
return {
|
||||||
|
output: {
|
||||||
|
status: "switching",
|
||||||
|
voice,
|
||||||
|
note: `${options.protocol.name} conversation memory resets during reconnect.`,
|
||||||
|
},
|
||||||
|
}
|
||||||
|
})
|
||||||
|
: options.tools
|
||||||
|
.execute(request.name, request.input)
|
||||||
|
.catch((error) => ({ output: toolError(String(error), true) } satisfies VoiceToolExecution)))
|
||||||
|
options.ui.toolDone(request.id, execution.output)
|
||||||
|
if (delegationID) notifications.finishDelegatedTool(delegationID, execution.admittedPrompt)
|
||||||
|
return execution.output
|
||||||
|
}
|
||||||
|
|
||||||
|
const queueWork = (source: VoiceConnection, request: VoiceWorkRequest) => {
|
||||||
|
options.trace?.("work.started", { id: request.id, name: request.name })
|
||||||
|
void executeTool(request)
|
||||||
|
.then((output) => {
|
||||||
|
options.trace?.("work.resolved", { id: request.id, name: request.name })
|
||||||
|
source.resolveWork(request, output)
|
||||||
|
})
|
||||||
|
.catch((error) => options.ui.meta(`[work error] ${String(error)}`))
|
||||||
|
.finally(() => dispatch({ type: "tool.finished", id: request.id }))
|
||||||
|
}
|
||||||
|
|
||||||
|
const userSpeaking = (active: boolean) => {
|
||||||
|
if (active === state.userSpeaking) return
|
||||||
|
dispatch(
|
||||||
|
active
|
||||||
|
? { type: "user.started", bargeIn: options.audio?.fullDuplex && options.protocol.capabilities.interruption }
|
||||||
|
: { type: "user.stopped" },
|
||||||
|
)
|
||||||
|
options.ui.userSpeaking(active)
|
||||||
|
}
|
||||||
|
|
||||||
|
const onAudioEvent = (event: AudioEvent) => {
|
||||||
|
switch (event.type) {
|
||||||
|
case "input":
|
||||||
|
connection?.appendAudio(event.audio)
|
||||||
|
options.ui.userAudioLevel(event.level)
|
||||||
|
return
|
||||||
|
case "user.started":
|
||||||
|
userSpeaking(true)
|
||||||
|
return
|
||||||
|
case "user.stopped":
|
||||||
|
userSpeaking(false)
|
||||||
|
return
|
||||||
|
case "user.reset":
|
||||||
|
options.ui.userReset()
|
||||||
|
return
|
||||||
|
case "assistant.level":
|
||||||
|
options.ui.assistantAudio(event.level, event.durationMs)
|
||||||
|
return
|
||||||
|
case "status":
|
||||||
|
options.ui.setStatus({ audio: event.audio })
|
||||||
|
return
|
||||||
|
case "meta":
|
||||||
|
options.ui.meta(event.text)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
const onProtocolEvent = (source: VoiceConnection, event: VoiceProtocolEvent) => {
|
||||||
|
if (source !== connection || closed) return
|
||||||
|
if (options.connection.debug || event.type !== "assistant.audio")
|
||||||
|
options.trace?.("protocol.event", {
|
||||||
|
type: event.type,
|
||||||
|
conversation: state.conversation,
|
||||||
|
assistant: state.assistant,
|
||||||
|
userSpeaking: state.userSpeaking,
|
||||||
|
pendingWork: state.tools.size,
|
||||||
|
})
|
||||||
|
switch (event.type) {
|
||||||
|
case "ready":
|
||||||
|
dispatch({ type: "connection.ready" })
|
||||||
|
options.ui.meta(
|
||||||
|
`connected to ${options.protocol.name} ${options.connection.model} (voice: ${state.desiredVoice})`,
|
||||||
|
)
|
||||||
|
options.ui.setStatus({
|
||||||
|
voice: state.desiredVoice,
|
||||||
|
microphoneMuted: options.audio?.microphoneMuted,
|
||||||
|
speakerMuted: options.audio?.speakerMuted,
|
||||||
|
})
|
||||||
|
if (!options.text) {
|
||||||
|
void options.audio?.start(onAudioEvent)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
options.ui.userTranscript("typed", options.text)
|
||||||
|
dispatch({ type: "user.committed" })
|
||||||
|
source.sendText?.(options.text)
|
||||||
|
return
|
||||||
|
case "user.started":
|
||||||
|
userSpeaking(true)
|
||||||
|
return
|
||||||
|
case "user.stopped":
|
||||||
|
userSpeaking(false)
|
||||||
|
return
|
||||||
|
case "user.committed":
|
||||||
|
options.audio?.noteUserCommitted()
|
||||||
|
dispatch({ type: "user.committed" })
|
||||||
|
options.ui.userCommitted(event.id)
|
||||||
|
return
|
||||||
|
case "user.transcript":
|
||||||
|
options.audio?.noteUserTranscript(event.final)
|
||||||
|
if (event.final) {
|
||||||
|
dispatch({ type: "user.stopped" })
|
||||||
|
options.ui.userSpeaking(false)
|
||||||
|
}
|
||||||
|
options.ui.userTranscript(event.id, event.text, event.final)
|
||||||
|
return
|
||||||
|
case "assistant.audio":
|
||||||
|
if (state.assistant === "suppressed") return
|
||||||
|
dispatch({ type: "assistant.started" })
|
||||||
|
options.audio?.play(event.audio, event.timeline)
|
||||||
|
return
|
||||||
|
case "assistant.transcript.delta":
|
||||||
|
if (state.assistant === "suppressed") return
|
||||||
|
dispatch({ type: "assistant.started" })
|
||||||
|
options.ui.assistantDelta(event.delta)
|
||||||
|
return
|
||||||
|
case "assistant.transcript":
|
||||||
|
if (state.assistant === "suppressed") return
|
||||||
|
dispatch({ type: "assistant.started" })
|
||||||
|
options.ui.assistantTranscript(event.text)
|
||||||
|
return
|
||||||
|
case "assistant.done": {
|
||||||
|
dispatch({ type: "assistant.done", awaitingWork: event.awaitingWork })
|
||||||
|
return
|
||||||
|
}
|
||||||
|
case "tool.started":
|
||||||
|
dispatch({ type: "tool.started", id: event.id })
|
||||||
|
options.ui.toolStart(event.id, event.name, {})
|
||||||
|
return
|
||||||
|
case "work.requested":
|
||||||
|
queueWork(source, event.request)
|
||||||
|
return
|
||||||
|
case "work.rejected":
|
||||||
|
dispatch({ type: "tool.started", id: event.request.id })
|
||||||
|
options.ui.toolStart(event.request.id, event.request.name, event.request.input)
|
||||||
|
options.ui.toolDone(event.request.id, event.output)
|
||||||
|
source.resolveWork(event.request, event.output)
|
||||||
|
dispatch({ type: "tool.finished", id: event.request.id })
|
||||||
|
return
|
||||||
|
case "delegation.requested":
|
||||||
|
dispatch({ type: "tool.started", id: event.request.id })
|
||||||
|
void options
|
||||||
|
.delegate(event.request, (request) => {
|
||||||
|
options.ui.toolStart(request.id, request.name, request.input)
|
||||||
|
return executeTool(request, event.request.id)
|
||||||
|
})
|
||||||
|
.then((output) => {
|
||||||
|
if (source !== connection) {
|
||||||
|
options.trace?.("delegation.result.stale", { id: event.request.id })
|
||||||
|
return
|
||||||
|
}
|
||||||
|
source.resolveDelegation?.(event.request, output)
|
||||||
|
})
|
||||||
|
.catch((error) => {
|
||||||
|
options.trace?.("responses.controller.failed", { error: String(error) })
|
||||||
|
if (source !== connection) return
|
||||||
|
options.ui.toolStart(event.request.id, "opencode", { text: event.request.text })
|
||||||
|
options.ui.toolDone(event.request.id, { status: "error", message: String(error) })
|
||||||
|
source.resolveDelegation?.(event.request, `The OpenCode controller failed: ${String(error)}`)
|
||||||
|
})
|
||||||
|
.finally(() => dispatch({ type: "tool.finished", id: event.request.id }))
|
||||||
|
return
|
||||||
|
case "debug":
|
||||||
|
if (options.connection.debug) options.ui.meta(`[debug] ${event.message}`)
|
||||||
|
return
|
||||||
|
case "error":
|
||||||
|
options.ui.meta(`[${options.protocol.name} error] ${event.message}`)
|
||||||
|
return
|
||||||
|
case "closed":
|
||||||
|
dispatch({ type: "connection.closed" })
|
||||||
|
options.ui.meta(`${options.protocol.name} connection closed (${event.code})`)
|
||||||
|
options.onClosed()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
const connect = () => {
|
||||||
|
let next: VoiceConnection
|
||||||
|
next = options.protocol.connect({
|
||||||
|
...options.connection,
|
||||||
|
voice: state.desiredVoice,
|
||||||
|
onEvent: (event) => onProtocolEvent(next, event),
|
||||||
|
})
|
||||||
|
dispatch({ type: "connection.connecting" })
|
||||||
|
connection = next
|
||||||
|
}
|
||||||
|
|
||||||
|
const session: VoiceSession = {
|
||||||
|
start: connect,
|
||||||
|
cycleVoice() {
|
||||||
|
setVoice(options.voices[(options.voices.indexOf(state.desiredVoice) + 1) % options.voices.length])
|
||||||
|
},
|
||||||
|
interrupt() {
|
||||||
|
if (!options.audio?.isPlaying() && state.assistant === "idle") return
|
||||||
|
if (state.assistant === "active") {
|
||||||
|
connection?.interrupt?.()
|
||||||
|
dispatch({ type: "assistant.suppressed" })
|
||||||
|
}
|
||||||
|
interruptDelivery()
|
||||||
|
options.audio?.flushPlayback()
|
||||||
|
options.ui.assistantDone()
|
||||||
|
options.ui.meta("[interrupted]")
|
||||||
|
},
|
||||||
|
toggleMicrophone() {
|
||||||
|
if (!options.audio) return
|
||||||
|
const muted = options.audio.toggleMicrophone()
|
||||||
|
userSpeaking(false)
|
||||||
|
options.ui.setStatus({ microphoneMuted: muted })
|
||||||
|
if (muted) options.ui.userReset()
|
||||||
|
options.ui.userAudioLevel(undefined)
|
||||||
|
options.ui.meta(`[microphone] ${muted ? "muted" : "live"}`)
|
||||||
|
},
|
||||||
|
toggleSpeaker() {
|
||||||
|
if (!options.audio) return
|
||||||
|
const muted = options.audio.toggleSpeaker()
|
||||||
|
options.ui.setStatus({ speakerMuted: muted })
|
||||||
|
if (muted) options.ui.assistantPlaybackStopped()
|
||||||
|
options.ui.meta(`[speaker] ${muted ? "muted" : "live"}`)
|
||||||
|
},
|
||||||
|
queueNotification(announcement) {
|
||||||
|
notifications.route(announcement)
|
||||||
|
},
|
||||||
|
close() {
|
||||||
|
if (closePromise) return closePromise
|
||||||
|
closed = true
|
||||||
|
if (voiceTimer) clearTimeout(voiceTimer)
|
||||||
|
options.audio?.close()
|
||||||
|
const active = connection
|
||||||
|
connection = undefined
|
||||||
|
dispatch({ type: "connection.closed" })
|
||||||
|
closePromise = Promise.allSettled([options.tools.close(), ...(active ? [active.close()] : [])]).then(() => {})
|
||||||
|
return closePromise
|
||||||
|
},
|
||||||
|
}
|
||||||
|
return session
|
||||||
|
}
|
||||||
|
|
||||||
|
function toolError(message: string, retryable = false) {
|
||||||
|
return { status: "error", message, retryable }
|
||||||
|
}
|
||||||
@@ -0,0 +1,53 @@
|
|||||||
|
import { describe, expect, test } from "bun:test"
|
||||||
|
import {
|
||||||
|
connectionMeterLevels,
|
||||||
|
connectionTransitioning,
|
||||||
|
scheduleTextReveal,
|
||||||
|
textRevealOpacity,
|
||||||
|
transcriptionPulse,
|
||||||
|
} from "../src/animation"
|
||||||
|
|
||||||
|
describe("voice text animation", () => {
|
||||||
|
test("stages newly streamed words without delaying them indefinitely", () => {
|
||||||
|
const first = scheduleTextReveal("", "Hello there", 1_000, 0)
|
||||||
|
expect(first.reveals).toEqual([
|
||||||
|
{ offset: 0, at: 1_000 },
|
||||||
|
{ offset: 6, at: 1_032 },
|
||||||
|
])
|
||||||
|
expect(first.animationEndsAt).toBe(1_412)
|
||||||
|
|
||||||
|
const queued = scheduleTextReveal("Hello there", " friend", 1_010, 10_000)
|
||||||
|
expect(queued.reveals).toEqual([{ offset: 12, at: 1_170 }])
|
||||||
|
})
|
||||||
|
|
||||||
|
test("slowly fades text into its stable color", () => {
|
||||||
|
const opacities = [100, 195, 290, 385, 480].map((now) => textRevealOpacity(now, 100))
|
||||||
|
expect(opacities[0]).toBe(0.16)
|
||||||
|
expect(opacities.toSorted((a, b) => a - b)).toEqual(opacities)
|
||||||
|
expect(opacities.at(-1)).toBe(1)
|
||||||
|
})
|
||||||
|
|
||||||
|
test("keeps transcription pulse bounded and moving", () => {
|
||||||
|
const levels = [0, 110, 220, 330].map(transcriptionPulse)
|
||||||
|
expect(levels.every((level) => level >= 0 && level <= 1)).toBe(true)
|
||||||
|
expect(new Set(levels).size).toBeGreaterThan(1)
|
||||||
|
})
|
||||||
|
|
||||||
|
test("moves the connecting pulse left to right and blends into the connected meter", () => {
|
||||||
|
const first = connectionMeterLevels(1_000, 0.1, undefined, 1_000)
|
||||||
|
const next = connectionMeterLevels(1_180, 0.1, undefined, 1_000)
|
||||||
|
expect(first.indexOf(Math.max(...first))).toBe(0)
|
||||||
|
expect(first.indexOf(Math.max(...first))).toBeLessThan(next.indexOf(Math.max(...next)))
|
||||||
|
|
||||||
|
const samples = Array.from({ length: 16 }, (_, index) => connectionMeterLevels(index * 45, 0.1))
|
||||||
|
expect(samples.every((levels) => Math.max(...levels) > 0.85)).toBe(true)
|
||||||
|
|
||||||
|
const handoff = connectionMeterLevels(1_000, 0.4, 1_000)
|
||||||
|
expect(handoff).toEqual(connectionMeterLevels(1_000, 0.4))
|
||||||
|
expect(connectionMeterLevels(1_480, 0.4, 1_000)).toEqual(
|
||||||
|
[0, 1, 2, 3].map((index) => 0.4 * (0.72 + Math.sin(1_480 / 240 + index * 1.4) * 0.28)),
|
||||||
|
)
|
||||||
|
expect(connectionTransitioning(1_479, 1_000)).toBe(true)
|
||||||
|
expect(connectionTransitioning(1_480, 1_000)).toBe(false)
|
||||||
|
})
|
||||||
|
})
|
||||||
@@ -0,0 +1,79 @@
|
|||||||
|
import { describe, expect, test } from "bun:test"
|
||||||
|
import { joinAssistantText } from "../src/ui-model"
|
||||||
|
|
||||||
|
describe("joinAssistantText", () => {
|
||||||
|
const cases: Array<[name: string, left: string, right: string, expected: string]> = [
|
||||||
|
[
|
||||||
|
"inserts a space between sentences",
|
||||||
|
"Okay, I'll check that.",
|
||||||
|
"The build passed.",
|
||||||
|
"Okay, I'll check that. The build passed.",
|
||||||
|
],
|
||||||
|
["inserts a space mid-sentence", "Let me look at", "the config file.", "Let me look at the config file."],
|
||||||
|
[
|
||||||
|
"keeps a single trailing space",
|
||||||
|
"Okay, I'll check that. ",
|
||||||
|
"The build passed.",
|
||||||
|
"Okay, I'll check that. The build passed.",
|
||||||
|
],
|
||||||
|
[
|
||||||
|
"keeps a single leading space",
|
||||||
|
"Okay, I'll check that.",
|
||||||
|
" The build passed.",
|
||||||
|
"Okay, I'll check that. The build passed.",
|
||||||
|
],
|
||||||
|
["collapses spaces on both sides", "Done. ", " Next.", "Done. Next."],
|
||||||
|
["collapses runs of spaces", "Done. ", "\t \tNext.", "Done. Next."],
|
||||||
|
[
|
||||||
|
"spaces after a question mark",
|
||||||
|
"Want me to run tests?",
|
||||||
|
"I can do that now.",
|
||||||
|
"Want me to run tests? I can do that now.",
|
||||||
|
],
|
||||||
|
["spaces after an ellipsis", "Thinking…", "done.", "Thinking… done."],
|
||||||
|
["preserves a trailing newline", "Done:\n", "Next up, tests.", "Done:\nNext up, tests."],
|
||||||
|
["preserves a leading newline", "Done:", "\nNext up, tests.", "Done:\nNext up, tests."],
|
||||||
|
[
|
||||||
|
"preserves a paragraph break",
|
||||||
|
"First paragraph.\n\n",
|
||||||
|
"Second paragraph.",
|
||||||
|
"First paragraph.\n\nSecond paragraph.",
|
||||||
|
],
|
||||||
|
["preserves a newline mixed with spaces", "Done: ", " \n Next.", "Done: \n Next."],
|
||||||
|
["never spaces before closing punctuation", "Done", ".", "Done."],
|
||||||
|
["never spaces before a comma", "one", ", two", "one, two"],
|
||||||
|
["never spaces before a closing paren", "(aside", ")", "(aside)"],
|
||||||
|
["never spaces after an opening paren", "an (", "aside)", "an (aside)"],
|
||||||
|
["drops an empty left side", "", "The build passed.", "The build passed."],
|
||||||
|
["drops an empty right side", "The build passed.", "", "The build passed."],
|
||||||
|
["drops a whitespace-only left side", " ", "The build passed.", "The build passed."],
|
||||||
|
["drops a whitespace-only right side", "The build passed.", " \n ", "The build passed."],
|
||||||
|
["handles two empty sides", "", "", ""],
|
||||||
|
["does not trim the outer edges", " Leading kept.", "Trailing kept. ", " Leading kept. Trailing kept. "],
|
||||||
|
]
|
||||||
|
|
||||||
|
for (const [name, left, right, expected] of cases) {
|
||||||
|
test(name, () => expect(joinAssistantText(left, right).text).toBe(expected))
|
||||||
|
}
|
||||||
|
|
||||||
|
test("splits the result at the boundary so reveals cover only the appended text", () => {
|
||||||
|
for (const [name, left, right] of cases) {
|
||||||
|
const joined = joinAssistantText(left, right)
|
||||||
|
expect(`${name}: ${joined.previous}${joined.appended}`).toBe(`${name}: ${joined.text}`)
|
||||||
|
expect(`${name}: ${joined.text.endsWith(joined.appended)}`).toBe(`${name}: true`)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
|
||||||
|
test("is idempotent once a boundary is normalised", () => {
|
||||||
|
const once = joinAssistantText("Okay.", "Next.").text
|
||||||
|
expect(joinAssistantText(once, "").text).toBe(once)
|
||||||
|
expect(joinAssistantText("Okay. ", "Next.").text).toBe(once)
|
||||||
|
})
|
||||||
|
|
||||||
|
test("folds consecutive messages with exactly one space per boundary", () => {
|
||||||
|
expect(["First.", "Second.", "Third."].reduce((left, right) => joinAssistantText(left, right).text)).toBe(
|
||||||
|
"First. Second. Third.",
|
||||||
|
)
|
||||||
|
expect(["First.", "", "Third."].reduce((left, right) => joinAssistantText(left, right).text)).toBe("First. Third.")
|
||||||
|
})
|
||||||
|
})
|
||||||
@@ -0,0 +1,33 @@
|
|||||||
|
import { describe, expect, test } from "bun:test"
|
||||||
|
import { AudioJitterBuffer } from "../src/audio-jitter-buffer"
|
||||||
|
import { PCM_BYTES_PER_MS } from "../src/pcm"
|
||||||
|
|
||||||
|
const chunk = (durationMs: number, gapMs = 0) => ({ bytes: Buffer.alloc(durationMs * PCM_BYTES_PER_MS), gapMs })
|
||||||
|
|
||||||
|
describe("AudioJitterBuffer", () => {
|
||||||
|
test("holds startup audio until the target buffer is available", () => {
|
||||||
|
const buffer = new AudioJitterBuffer(300)
|
||||||
|
expect(buffer.push(chunk(100))).toEqual([])
|
||||||
|
expect(buffer.push(chunk(100))).toEqual([])
|
||||||
|
expect(buffer.push(chunk(100))).toHaveLength(3)
|
||||||
|
})
|
||||||
|
|
||||||
|
test("passes chunks through after playback starts", () => {
|
||||||
|
const buffer = new AudioJitterBuffer(200)
|
||||||
|
buffer.push(chunk(200))
|
||||||
|
expect(buffer.push(chunk(100))).toEqual([chunk(100)])
|
||||||
|
})
|
||||||
|
|
||||||
|
test("flushes short utterances and resets between responses", () => {
|
||||||
|
const buffer = new AudioJitterBuffer(300)
|
||||||
|
buffer.push(chunk(100))
|
||||||
|
expect(buffer.finish()).toHaveLength(1)
|
||||||
|
buffer.reset()
|
||||||
|
expect(buffer.push(chunk(100))).toEqual([])
|
||||||
|
})
|
||||||
|
|
||||||
|
test("counts intentional timeline silence toward buffered duration", () => {
|
||||||
|
const buffer = new AudioJitterBuffer(300)
|
||||||
|
expect(buffer.push(chunk(200, 100))).toHaveLength(1)
|
||||||
|
})
|
||||||
|
})
|
||||||
@@ -0,0 +1,106 @@
|
|||||||
|
import { expect, test } from "bun:test"
|
||||||
|
import type { AudioDevice } from "../src/audio-device"
|
||||||
|
import { createAudioSession, type AudioEvent } from "../src/audio-session"
|
||||||
|
|
||||||
|
test("reports muted notification playback as inaudible", async () => {
|
||||||
|
const device = scriptedDevice()
|
||||||
|
const audio = await createAudioSession({
|
||||||
|
duplex: false,
|
||||||
|
speakers: false,
|
||||||
|
inputActivity: "server",
|
||||||
|
debug: false,
|
||||||
|
device,
|
||||||
|
})
|
||||||
|
|
||||||
|
expect(audio.toggleSpeaker()).toBe(true)
|
||||||
|
audio.play(Buffer.alloc(4_800))
|
||||||
|
expect(await audio.finishPlayback()).toBe("inaudible")
|
||||||
|
expect(device.writes).toEqual([])
|
||||||
|
})
|
||||||
|
|
||||||
|
test("settles active playback when interrupted", async () => {
|
||||||
|
const device = scriptedDevice()
|
||||||
|
const audio = await createAudioSession({
|
||||||
|
duplex: false,
|
||||||
|
speakers: false,
|
||||||
|
inputActivity: "server",
|
||||||
|
debug: false,
|
||||||
|
device,
|
||||||
|
})
|
||||||
|
|
||||||
|
audio.play(Buffer.alloc(4_800))
|
||||||
|
const outcome = audio.finishPlayback()
|
||||||
|
audio.flushPlayback()
|
||||||
|
expect(await outcome).toBe("interrupted")
|
||||||
|
expect(device.flushes).toBe(1)
|
||||||
|
})
|
||||||
|
|
||||||
|
test("emits captured audio and local speech observations", async () => {
|
||||||
|
const device = scriptedDevice()
|
||||||
|
const events: AudioEvent[] = []
|
||||||
|
const audio = await createAudioSession({
|
||||||
|
duplex: true,
|
||||||
|
speakers: false,
|
||||||
|
inputActivity: "local",
|
||||||
|
debug: false,
|
||||||
|
device,
|
||||||
|
})
|
||||||
|
await audio.start((event) => events.push(event))
|
||||||
|
const input = Buffer.alloc(4_800)
|
||||||
|
for (let offset = 0; offset < input.length; offset += 2) input.writeInt16LE(12_000, offset)
|
||||||
|
device.capture(input)
|
||||||
|
|
||||||
|
expect(events).toContainEqual({ type: "input", audio: input, level: expect.any(Number) })
|
||||||
|
expect(events).toContainEqual({ type: "user.started" })
|
||||||
|
audio.close()
|
||||||
|
})
|
||||||
|
|
||||||
|
test("preserves quiet duplex microphone audio during assistant playback", async () => {
|
||||||
|
const device = scriptedDevice()
|
||||||
|
const events: AudioEvent[] = []
|
||||||
|
const audio = await createAudioSession({
|
||||||
|
duplex: false,
|
||||||
|
speakers: true,
|
||||||
|
inputActivity: "server",
|
||||||
|
debug: false,
|
||||||
|
device,
|
||||||
|
})
|
||||||
|
await audio.start((event) => events.push(event))
|
||||||
|
audio.play(Buffer.alloc(24_000))
|
||||||
|
const input = Buffer.alloc(4_800)
|
||||||
|
for (let offset = 0; offset < input.length; offset += 2) input.writeInt16LE(800, offset)
|
||||||
|
device.capture(input)
|
||||||
|
|
||||||
|
expect(audio.fullDuplex).toBe(true)
|
||||||
|
expect(events).toContainEqual({ type: "input", audio: input, level: expect.any(Number) })
|
||||||
|
audio.close()
|
||||||
|
})
|
||||||
|
|
||||||
|
function scriptedDevice() {
|
||||||
|
let onInput: ((audio: Buffer) => void) | undefined
|
||||||
|
const writes: Buffer[] = []
|
||||||
|
const device = {
|
||||||
|
fullDuplex: true,
|
||||||
|
mode: "scripted",
|
||||||
|
async start(listener) {
|
||||||
|
onInput = listener
|
||||||
|
},
|
||||||
|
write(audio) {
|
||||||
|
writes.push(audio)
|
||||||
|
},
|
||||||
|
flush() {
|
||||||
|
device.flushes += 1
|
||||||
|
},
|
||||||
|
close() {},
|
||||||
|
writes,
|
||||||
|
flushes: 0,
|
||||||
|
capture(audio: Buffer) {
|
||||||
|
onInput?.(audio)
|
||||||
|
},
|
||||||
|
} satisfies AudioDevice & {
|
||||||
|
readonly writes: Buffer[]
|
||||||
|
flushes: number
|
||||||
|
capture(audio: Buffer): void
|
||||||
|
}
|
||||||
|
return device
|
||||||
|
}
|
||||||
@@ -0,0 +1,101 @@
|
|||||||
|
import { expect, test } from "bun:test"
|
||||||
|
import { mkdir, mkdtemp, rm } from "node:fs/promises"
|
||||||
|
import { tmpdir } from "node:os"
|
||||||
|
import { join } from "node:path"
|
||||||
|
import { createCompletionStore } from "../src/completion-store"
|
||||||
|
|
||||||
|
test("completion store restores pending and completed notifications until delivery", async () => {
|
||||||
|
const directory = await mkdtemp(join(tmpdir(), "voice-completions-"))
|
||||||
|
const path = join(directory, "prompts.json")
|
||||||
|
const first = await createCompletionStore(path)
|
||||||
|
const pending = { sessionID: "session-1", promptID: "prompt-1" }
|
||||||
|
const completed = { sessionID: "session-2", promptID: "prompt-2" }
|
||||||
|
|
||||||
|
try {
|
||||||
|
await first.admitting(pending, "Please continue")
|
||||||
|
expect(first.entries()).toEqual([{ status: "admitting", handle: pending, text: "Please continue" }])
|
||||||
|
await first.pending(pending)
|
||||||
|
await first.pending(completed)
|
||||||
|
await first.completed(completed, {
|
||||||
|
type: "opencode.prompt.completed",
|
||||||
|
session_id: "session-2",
|
||||||
|
prompt_id: "prompt-2",
|
||||||
|
status: "completed",
|
||||||
|
text: "done",
|
||||||
|
})
|
||||||
|
await first.close()
|
||||||
|
|
||||||
|
const restored = await createCompletionStore(path)
|
||||||
|
expect(restored.entries()).toEqual(expect.arrayContaining([
|
||||||
|
{ status: "pending", handle: pending },
|
||||||
|
{
|
||||||
|
status: "completed",
|
||||||
|
handle: completed,
|
||||||
|
notification: {
|
||||||
|
type: "opencode.prompt.completed",
|
||||||
|
session_id: "session-2",
|
||||||
|
prompt_id: "prompt-2",
|
||||||
|
status: "completed",
|
||||||
|
text: "done",
|
||||||
|
},
|
||||||
|
},
|
||||||
|
]))
|
||||||
|
await restored.close()
|
||||||
|
|
||||||
|
const stillPendingDelivery = await createCompletionStore(path)
|
||||||
|
expect(stillPendingDelivery.entries().some((entry) => entry.status === "completed")).toBe(true)
|
||||||
|
|
||||||
|
await stillPendingDelivery.remove(completed)
|
||||||
|
await stillPendingDelivery.close()
|
||||||
|
expect((await createCompletionStore(path)).entries()).toEqual([{ status: "pending", handle: pending }])
|
||||||
|
} finally {
|
||||||
|
await rm(directory, { recursive: true, force: true })
|
||||||
|
}
|
||||||
|
})
|
||||||
|
|
||||||
|
test("completion stores do not overwrite prompts written by another process", async () => {
|
||||||
|
const directory = await mkdtemp(join(tmpdir(), "voice-completions-concurrent-"))
|
||||||
|
const path = join(directory, "prompts.json")
|
||||||
|
const first = await createCompletionStore(path)
|
||||||
|
const second = await createCompletionStore(path)
|
||||||
|
|
||||||
|
try {
|
||||||
|
await Promise.all([
|
||||||
|
first.pending({ sessionID: "session-1", promptID: "prompt-shared" }),
|
||||||
|
second.pending({ sessionID: "session-2", promptID: "prompt-shared" }),
|
||||||
|
])
|
||||||
|
await Promise.all([first.close(), second.close()])
|
||||||
|
expect((await createCompletionStore(path)).entries()).toEqual(
|
||||||
|
expect.arrayContaining([
|
||||||
|
{ status: "pending", handle: { sessionID: "session-1", promptID: "prompt-shared" } },
|
||||||
|
{ status: "pending", handle: { sessionID: "session-2", promptID: "prompt-shared" } },
|
||||||
|
]),
|
||||||
|
)
|
||||||
|
} finally {
|
||||||
|
await rm(directory, { recursive: true, force: true })
|
||||||
|
}
|
||||||
|
})
|
||||||
|
|
||||||
|
test("continues serializing writes after one filesystem failure", async () => {
|
||||||
|
const directory = await mkdtemp(join(tmpdir(), "voice-completions-recovery-"))
|
||||||
|
const path = join(directory, "prompts.json")
|
||||||
|
const store = await createCompletionStore(path)
|
||||||
|
try {
|
||||||
|
await rm(`${path}.d`, { recursive: true })
|
||||||
|
await Bun.write(`${path}.d`, "not a directory")
|
||||||
|
const failure = await store
|
||||||
|
.pending({ sessionID: "session-1", promptID: "prompt-1" })
|
||||||
|
.then(() => undefined, (error) => error)
|
||||||
|
expect(failure).toBeDefined()
|
||||||
|
await rm(`${path}.d`)
|
||||||
|
await mkdir(`${path}.d`, { recursive: true })
|
||||||
|
await store.pending({ sessionID: "session-2", promptID: "prompt-2" })
|
||||||
|
await store.close()
|
||||||
|
expect((await createCompletionStore(path)).entries()).toContainEqual({
|
||||||
|
status: "pending",
|
||||||
|
handle: { sessionID: "session-2", promptID: "prompt-2" },
|
||||||
|
})
|
||||||
|
} finally {
|
||||||
|
await rm(directory, { recursive: true, force: true })
|
||||||
|
}
|
||||||
|
})
|
||||||
@@ -0,0 +1,311 @@
|
|||||||
|
import { describe, expect, test } from "bun:test"
|
||||||
|
import { createLiveEventDelivery, createLiveEventProjector } from "../src/protocol-live"
|
||||||
|
import type { VoiceProtocolEvent } from "../src/protocol"
|
||||||
|
|
||||||
|
const frames = [
|
||||||
|
{ type: "session.started", session: {} },
|
||||||
|
{ type: "session.context.appended", start_ms: 0, end_ms: 0 },
|
||||||
|
{ type: "output_audio.delta", audio: "AAA=", start_ms: 0, end_ms: 1 },
|
||||||
|
{
|
||||||
|
type: "input_transcript.added",
|
||||||
|
start_ms: 0,
|
||||||
|
end_ms: 100,
|
||||||
|
item: { id: "input-1", type: "input_transcript", text: " Hello" },
|
||||||
|
},
|
||||||
|
{ type: "turn.created", turn: { id: "user-turn", role: "user", transcript: " Hello" } },
|
||||||
|
{
|
||||||
|
type: "input_transcript.added",
|
||||||
|
start_ms: 100,
|
||||||
|
end_ms: 200,
|
||||||
|
item: { id: "input-2", type: "input_transcript", text: ", this" },
|
||||||
|
},
|
||||||
|
{ type: "turn.delta", turn_id: "user-turn", delta: ", this" },
|
||||||
|
{
|
||||||
|
type: "output_transcript.added",
|
||||||
|
start_ms: 200,
|
||||||
|
end_ms: 300,
|
||||||
|
item: { id: "output-1", type: "output_transcript", text: " Copy that" },
|
||||||
|
},
|
||||||
|
{
|
||||||
|
type: "input_transcript.added",
|
||||||
|
start_ms: 200,
|
||||||
|
end_ms: 300,
|
||||||
|
item: { id: "input-3", type: "input_transcript", text: " is a test" },
|
||||||
|
},
|
||||||
|
{
|
||||||
|
type: "output_transcript.added",
|
||||||
|
start_ms: 300,
|
||||||
|
end_ms: 400,
|
||||||
|
item: { id: "output-2", type: "output_transcript", text: ", test" },
|
||||||
|
},
|
||||||
|
{ type: "turn.delta", turn_id: "user-turn", delta: " is a test" },
|
||||||
|
{ type: "turn.done", turn: { id: "user-turn", role: "user", transcript: " Hello, this is a test" } },
|
||||||
|
{ type: "turn.created", turn: { id: "assistant-turn", role: "assistant", transcript: " Copy that, test" } },
|
||||||
|
{
|
||||||
|
type: "output_transcript.added",
|
||||||
|
start_ms: 400,
|
||||||
|
end_ms: 500,
|
||||||
|
item: { id: "output-3", type: "output_transcript", text: " received" },
|
||||||
|
},
|
||||||
|
{ type: "turn.delta", turn_id: "assistant-turn", delta: " received" },
|
||||||
|
{
|
||||||
|
type: "output_transcript.added",
|
||||||
|
start_ms: 500,
|
||||||
|
end_ms: 600,
|
||||||
|
item: { id: "output-4", type: "output_transcript", text: " loud and clear." },
|
||||||
|
},
|
||||||
|
{ type: "turn.delta", turn_id: "assistant-turn", delta: " loud and clear." },
|
||||||
|
{
|
||||||
|
type: "turn.done",
|
||||||
|
turn: { id: "assistant-turn", role: "assistant", transcript: " Copy that, test received loud and clear." },
|
||||||
|
},
|
||||||
|
] as const
|
||||||
|
|
||||||
|
describe("Live event projector", () => {
|
||||||
|
test("replays progressive transcripts without schema errors or duplicate text", () => {
|
||||||
|
const project = createLiveEventProjector()
|
||||||
|
const events = frames.flatMap((frame) => project(JSON.stringify(frame)).events)
|
||||||
|
|
||||||
|
expect(events.filter((event) => event.type === "error")).toEqual([])
|
||||||
|
expect(events.filter((event) => event.type === "user.committed")).toEqual([
|
||||||
|
{ type: "user.committed", id: "input-1" },
|
||||||
|
])
|
||||||
|
expect(events.filter(isUserTranscript)).toEqual([
|
||||||
|
{ type: "user.transcript", id: "input-1", text: "Hello", final: false },
|
||||||
|
{ type: "user.transcript", id: "input-1", text: "Hello, this", final: false },
|
||||||
|
{ type: "user.transcript", id: "input-1", text: "Hello, this is a test", final: false },
|
||||||
|
{ type: "user.transcript", id: "input-1", text: "Hello, this is a test", final: true },
|
||||||
|
])
|
||||||
|
expect(
|
||||||
|
events
|
||||||
|
.filter(isAssistantDelta)
|
||||||
|
.map((event) => event.delta)
|
||||||
|
.join(""),
|
||||||
|
).toBe(" Copy that, test received loud and clear.")
|
||||||
|
expect(events.filter((event) => event.type === "assistant.done")).toEqual([
|
||||||
|
{ type: "assistant.done", awaitingWork: false },
|
||||||
|
])
|
||||||
|
expect(events.filter((event) => event.type === "assistant.audio")).toHaveLength(1)
|
||||||
|
})
|
||||||
|
|
||||||
|
test("reports malformed frames instead of throwing", () => {
|
||||||
|
expect(createLiveEventProjector()(JSON.stringify({ type: "turn.created", turn: { id: 1 } })).events).toEqual([
|
||||||
|
{ type: "error", message: "Received an invalid Live API event." },
|
||||||
|
])
|
||||||
|
})
|
||||||
|
|
||||||
|
test("preserves context acknowledgement correlation IDs", () => {
|
||||||
|
expect(
|
||||||
|
createLiveEventProjector()(
|
||||||
|
JSON.stringify({ type: "session.context.appended", event_id: "event-notification" }),
|
||||||
|
),
|
||||||
|
).toMatchObject({
|
||||||
|
type: "session.context.appended",
|
||||||
|
eventID: "event-notification",
|
||||||
|
})
|
||||||
|
})
|
||||||
|
|
||||||
|
test("replaces corrected assistant transcript snapshots", () => {
|
||||||
|
const project = createLiveEventProjector()
|
||||||
|
const events = [
|
||||||
|
project(
|
||||||
|
JSON.stringify({
|
||||||
|
type: "output_transcript.added",
|
||||||
|
item: { id: "output-1", type: "output_transcript", text: " Hello world" },
|
||||||
|
}),
|
||||||
|
),
|
||||||
|
project(
|
||||||
|
JSON.stringify({
|
||||||
|
type: "turn.created",
|
||||||
|
turn: { id: "assistant-turn", role: "assistant", transcript: " Hello there" },
|
||||||
|
}),
|
||||||
|
),
|
||||||
|
].flatMap((result) => result.events)
|
||||||
|
|
||||||
|
expect(events.filter((event) => event.type.startsWith("assistant.transcript"))).toEqual([
|
||||||
|
{ type: "assistant.transcript.delta", delta: " Hello world" },
|
||||||
|
{ type: "assistant.transcript", text: " Hello there" },
|
||||||
|
])
|
||||||
|
})
|
||||||
|
|
||||||
|
test("keeps the space between consecutive assistant turns", () => {
|
||||||
|
const project = createLiveEventProjector()
|
||||||
|
const turn = (id: string, text: string) => [
|
||||||
|
{ type: "turn.created", turn: { id, role: "assistant", transcript: "" } },
|
||||||
|
{ type: "output_transcript.added", item: { id: `${id}-o`, type: "output_transcript", text } },
|
||||||
|
{ type: "turn.delta", turn_id: id, delta: text },
|
||||||
|
{ type: "turn.done", turn: { id, role: "assistant", transcript: text } },
|
||||||
|
]
|
||||||
|
const events = [...turn("t1", ' Session "Fix auth bug" is ready.'), ...turn("t2", " Next up, tests.")].flatMap(
|
||||||
|
(frame) => project(JSON.stringify(frame)).events,
|
||||||
|
)
|
||||||
|
|
||||||
|
// Concatenating the delta stream is what the UI and any transcript consumer does; the
|
||||||
|
// turn boundary must survive it rather than welding "ready.Next" together.
|
||||||
|
const transcript = events
|
||||||
|
.filter(isAssistantDelta)
|
||||||
|
.map((event) => event.delta)
|
||||||
|
.join("")
|
||||||
|
expect(transcript).toBe(' Session "Fix auth bug" is ready. Next up, tests.')
|
||||||
|
expect(transcript).not.toContain("ready.Next")
|
||||||
|
expect(events.filter((event) => event.type === "assistant.done")).toHaveLength(2)
|
||||||
|
})
|
||||||
|
|
||||||
|
test("emits no duplicate text when fragments and turn deltas overlap across turns", () => {
|
||||||
|
const project = createLiveEventProjector()
|
||||||
|
const frames = [
|
||||||
|
{ type: "turn.created", turn: { id: "t1", role: "assistant", transcript: "" } },
|
||||||
|
{ type: "output_transcript.added", item: { id: "a", type: "output_transcript", text: " First" } },
|
||||||
|
{ type: "turn.delta", turn_id: "t1", delta: " First" },
|
||||||
|
{ type: "output_transcript.added", item: { id: "b", type: "output_transcript", text: " turn." } },
|
||||||
|
{ type: "turn.delta", turn_id: "t1", delta: " turn." },
|
||||||
|
{ type: "turn.done", turn: { id: "t1", role: "assistant", transcript: " First turn." } },
|
||||||
|
{ type: "turn.created", turn: { id: "t2", role: "assistant", transcript: "" } },
|
||||||
|
{ type: "output_transcript.added", item: { id: "c", type: "output_transcript", text: " Second turn." } },
|
||||||
|
{ type: "turn.delta", turn_id: "t2", delta: " Second turn." },
|
||||||
|
{ type: "turn.done", turn: { id: "t2", role: "assistant", transcript: " Second turn." } },
|
||||||
|
]
|
||||||
|
const events = frames.flatMap((frame) => project(JSON.stringify(frame)).events)
|
||||||
|
|
||||||
|
expect(events.filter((event) => event.type === "assistant.transcript")).toEqual([])
|
||||||
|
expect(
|
||||||
|
events
|
||||||
|
.filter(isAssistantDelta)
|
||||||
|
.map((event) => event.delta)
|
||||||
|
.join(""),
|
||||||
|
).toBe(" First turn. Second turn.")
|
||||||
|
})
|
||||||
|
|
||||||
|
test("projects Responses delegation function calls without rejecting reasoning items", () => {
|
||||||
|
const project = createLiveEventProjector()
|
||||||
|
expect(
|
||||||
|
project(
|
||||||
|
JSON.stringify({
|
||||||
|
type: "response.output_item.done",
|
||||||
|
item: { id: "reasoning-1", type: "reasoning" },
|
||||||
|
}),
|
||||||
|
).events,
|
||||||
|
).toEqual([{ type: "debug", message: "response.output_item.done" }])
|
||||||
|
expect(
|
||||||
|
project(
|
||||||
|
JSON.stringify({
|
||||||
|
type: "response.output_item.added",
|
||||||
|
item: {
|
||||||
|
id: "function-1",
|
||||||
|
type: "function_call",
|
||||||
|
name: "find_sessions",
|
||||||
|
call_id: "call-1",
|
||||||
|
arguments: "",
|
||||||
|
},
|
||||||
|
}),
|
||||||
|
).events,
|
||||||
|
).toEqual([
|
||||||
|
{ type: "debug", message: "response.output_item.added" },
|
||||||
|
{ type: "tool.started", id: "call-1", name: "find_sessions" },
|
||||||
|
])
|
||||||
|
expect(
|
||||||
|
project(
|
||||||
|
JSON.stringify({
|
||||||
|
type: "response.output_item.done",
|
||||||
|
item: {
|
||||||
|
id: "function-1",
|
||||||
|
type: "function_call",
|
||||||
|
name: "find_sessions",
|
||||||
|
call_id: "call-1",
|
||||||
|
arguments: '{"scope":"current_project"}',
|
||||||
|
},
|
||||||
|
}),
|
||||||
|
).events,
|
||||||
|
).toEqual([
|
||||||
|
{ type: "debug", message: "response.output_item.done" },
|
||||||
|
{
|
||||||
|
type: "work.requested",
|
||||||
|
request: { id: "call-1", name: "find_sessions", input: { scope: "current_project" } },
|
||||||
|
},
|
||||||
|
])
|
||||||
|
expect(
|
||||||
|
project(
|
||||||
|
JSON.stringify({
|
||||||
|
type: "response.output_item.done",
|
||||||
|
item: {
|
||||||
|
id: "message-1",
|
||||||
|
type: "message",
|
||||||
|
status: "completed",
|
||||||
|
content: [{ type: "output_text", text: "Two sessions found." }],
|
||||||
|
role: "assistant",
|
||||||
|
},
|
||||||
|
}),
|
||||||
|
).events,
|
||||||
|
).toEqual([{ type: "debug", message: "response.output_item.done" }])
|
||||||
|
})
|
||||||
|
|
||||||
|
test("projects client delegation requests for the local controller", () => {
|
||||||
|
expect(
|
||||||
|
createLiveEventProjector()(
|
||||||
|
JSON.stringify({
|
||||||
|
type: "delegation.created",
|
||||||
|
item: {
|
||||||
|
id: "delegation-1",
|
||||||
|
type: "delegation",
|
||||||
|
target: "client",
|
||||||
|
content: [{ type: "input_text", text: "Inspect the voice package" }],
|
||||||
|
},
|
||||||
|
}),
|
||||||
|
).events,
|
||||||
|
).toEqual([
|
||||||
|
{ type: "debug", message: "delegation.created" },
|
||||||
|
{ type: "delegation.requested", request: { id: "delegation-1", text: "Inspect the voice package" } },
|
||||||
|
])
|
||||||
|
})
|
||||||
|
|
||||||
|
test("rejects malformed function arguments without hanging the delegation", () => {
|
||||||
|
expect(
|
||||||
|
createLiveEventProjector()(
|
||||||
|
JSON.stringify({
|
||||||
|
type: "response.output_item.done",
|
||||||
|
item: {
|
||||||
|
id: "function-1",
|
||||||
|
type: "function_call",
|
||||||
|
name: "find_sessions",
|
||||||
|
call_id: "call-invalid",
|
||||||
|
arguments: "not json",
|
||||||
|
},
|
||||||
|
}),
|
||||||
|
).events,
|
||||||
|
).toEqual([
|
||||||
|
{ type: "debug", message: "response.output_item.done" },
|
||||||
|
{ type: "tool.started", id: "call-invalid", name: "find_sessions" },
|
||||||
|
{ type: "error", message: "Received invalid arguments for Live tool find_sessions." },
|
||||||
|
{
|
||||||
|
type: "work.rejected",
|
||||||
|
request: { id: "call-invalid", name: "find_sessions", input: {} },
|
||||||
|
output: { status: "error", message: "Invalid arguments for tool find_sessions." },
|
||||||
|
},
|
||||||
|
])
|
||||||
|
})
|
||||||
|
|
||||||
|
test("waits for a quiet drain after turn completion and late audio", async () => {
|
||||||
|
const events: VoiceProtocolEvent[] = []
|
||||||
|
const delivery = createLiveEventDelivery({ emit: (event) => events.push(event), drainMs: 20 })
|
||||||
|
delivery.push({ type: "assistant.done", awaitingWork: false })
|
||||||
|
await Bun.sleep(10)
|
||||||
|
delivery.push({ type: "assistant.audio", audio: Buffer.alloc(48) })
|
||||||
|
await Bun.sleep(12)
|
||||||
|
expect(events.map((event) => event.type)).toEqual(["assistant.audio"])
|
||||||
|
await Bun.sleep(12)
|
||||||
|
expect(events.map((event) => event.type)).toEqual(["assistant.audio", "assistant.done"])
|
||||||
|
delivery.close()
|
||||||
|
})
|
||||||
|
})
|
||||||
|
|
||||||
|
function isUserTranscript(
|
||||||
|
event: VoiceProtocolEvent,
|
||||||
|
): event is Extract<VoiceProtocolEvent, { readonly type: "user.transcript" }> {
|
||||||
|
return event.type === "user.transcript"
|
||||||
|
}
|
||||||
|
|
||||||
|
function isAssistantDelta(
|
||||||
|
event: VoiceProtocolEvent,
|
||||||
|
): event is Extract<VoiceProtocolEvent, { readonly type: "assistant.transcript.delta" }> {
|
||||||
|
return event.type === "assistant.transcript.delta"
|
||||||
|
}
|
||||||
@@ -0,0 +1,59 @@
|
|||||||
|
import { expect, test } from "bun:test"
|
||||||
|
import { createNotificationRouter, type RoutedAnnouncement } from "../src/notification-router"
|
||||||
|
|
||||||
|
test("routes blockers and completions through the admitted delegation", () => {
|
||||||
|
const routed: RoutedAnnouncement[] = []
|
||||||
|
const router = createNotificationRouter((announcement) => routed.push(announcement))
|
||||||
|
router.beginDelegatedTool()
|
||||||
|
router.route({
|
||||||
|
notification: {
|
||||||
|
type: "opencode.prompt.blocked",
|
||||||
|
prompt_id: "prompt-1",
|
||||||
|
blocker: "permission",
|
||||||
|
session_id: "session-1",
|
||||||
|
request_id: "permission-1",
|
||||||
|
action: "read",
|
||||||
|
resources: [],
|
||||||
|
},
|
||||||
|
})
|
||||||
|
expect(routed).toEqual([])
|
||||||
|
|
||||||
|
router.finishDelegatedTool("delegation-1", { sessionID: "session-1", promptID: "prompt-1" })
|
||||||
|
router.route({
|
||||||
|
notification: {
|
||||||
|
type: "opencode.prompt.completed",
|
||||||
|
session_id: "session-1",
|
||||||
|
prompt_id: "prompt-1",
|
||||||
|
status: "completed",
|
||||||
|
text: "done",
|
||||||
|
},
|
||||||
|
receipt: { sessionID: "session-1", promptID: "prompt-1" },
|
||||||
|
})
|
||||||
|
|
||||||
|
expect(routed.map((item) => item.delegationID)).toEqual(["delegation-1", "delegation-1"])
|
||||||
|
expect(routed[0].announcement.receipt).toBeUndefined()
|
||||||
|
expect(routed[1].announcement.receipt).toEqual({ sessionID: "session-1", promptID: "prompt-1" })
|
||||||
|
})
|
||||||
|
|
||||||
|
test("keeps identical prompt IDs isolated by session", () => {
|
||||||
|
const routed: RoutedAnnouncement[] = []
|
||||||
|
const router = createNotificationRouter((announcement) => routed.push(announcement))
|
||||||
|
router.beginDelegatedTool()
|
||||||
|
router.finishDelegatedTool("delegation-1", { sessionID: "session-1", promptID: "prompt-shared" })
|
||||||
|
router.beginDelegatedTool()
|
||||||
|
router.finishDelegatedTool("delegation-2", { sessionID: "session-2", promptID: "prompt-shared" })
|
||||||
|
|
||||||
|
for (const sessionID of ["session-1", "session-2"])
|
||||||
|
router.route({
|
||||||
|
notification: {
|
||||||
|
type: "opencode.prompt.completed",
|
||||||
|
session_id: sessionID,
|
||||||
|
prompt_id: "prompt-shared",
|
||||||
|
status: "completed",
|
||||||
|
text: "done",
|
||||||
|
},
|
||||||
|
receipt: { sessionID, promptID: "prompt-shared" },
|
||||||
|
})
|
||||||
|
|
||||||
|
expect(routed.map((item) => item.delegationID)).toEqual(["delegation-1", "delegation-2"])
|
||||||
|
})
|
||||||
@@ -0,0 +1,248 @@
|
|||||||
|
import { expect, test } from "bun:test"
|
||||||
|
import { OpenCode } from "@opencode-ai/client/promise"
|
||||||
|
import { createOpenCodeBridge } from "../src/opencode"
|
||||||
|
import type { CompletionStore } from "../src/completion-store"
|
||||||
|
|
||||||
|
test("renames only a session discovered by the voice controller", async () => {
|
||||||
|
const requests: Request[] = []
|
||||||
|
const session = {
|
||||||
|
id: "ses_known",
|
||||||
|
projectID: "project-1",
|
||||||
|
cost: "0",
|
||||||
|
tokens: { input: 0, output: 0, reasoning: 0, cache: { read: 0, write: 0 } },
|
||||||
|
time: { created: 1, updated: 2 },
|
||||||
|
title: "Old title",
|
||||||
|
location: { directory: "/workspace" },
|
||||||
|
}
|
||||||
|
const fetch = Object.assign(
|
||||||
|
async (input: string | URL | Request, init?: RequestInit) => {
|
||||||
|
const request = input instanceof Request ? input : new Request(input.toString(), init)
|
||||||
|
requests.push(request)
|
||||||
|
const url = new URL(request.url)
|
||||||
|
if (url.pathname === "/api/event") return new Response("", { headers: { "content-type": "text/event-stream" } })
|
||||||
|
if (url.pathname === "/api/project")
|
||||||
|
return Response.json([
|
||||||
|
{
|
||||||
|
id: "project-1",
|
||||||
|
worktree: "/workspace",
|
||||||
|
time: { created: 1, updated: 2 },
|
||||||
|
sandboxes: [],
|
||||||
|
},
|
||||||
|
])
|
||||||
|
if (url.pathname === "/api/session" && request.method === "GET")
|
||||||
|
return Response.json({ data: [session], cursor: {} })
|
||||||
|
if (url.pathname === "/api/session/ses_known" && request.method === "GET") return Response.json({ data: session })
|
||||||
|
if (url.pathname === "/api/session/ses_known/rename" && request.method === "POST")
|
||||||
|
return new Response(null, { status: 204 })
|
||||||
|
return new Response("Not found", { status: 404 })
|
||||||
|
},
|
||||||
|
{ preconnect: () => {} },
|
||||||
|
)
|
||||||
|
const client = OpenCode.make({
|
||||||
|
baseUrl: "https://opencode.test",
|
||||||
|
fetch,
|
||||||
|
})
|
||||||
|
const bridge = await createOpenCodeBridge({
|
||||||
|
client,
|
||||||
|
directory: "/workspace",
|
||||||
|
model: { providerID: "openai", id: "gpt-test" },
|
||||||
|
notify: () => {},
|
||||||
|
completionStore: memoryCompletionStore(),
|
||||||
|
})
|
||||||
|
const execute = async (name: string, input: Record<string, unknown>) => (await bridge.execute(name, input)).output
|
||||||
|
|
||||||
|
try {
|
||||||
|
expect(new Set(bridge.definitions.map((tool) => tool.name)).size).toBe(bridge.definitions.length)
|
||||||
|
expect(bridge.definitions.every((tool) => tool.name.length > 0 && tool.description.length > 0)).toBe(true)
|
||||||
|
expect(bridge.definitions.find((tool) => tool.name === "reply_form")).toMatchObject({
|
||||||
|
parameters: {
|
||||||
|
properties: { answer: { type: "object", additionalProperties: true } },
|
||||||
|
},
|
||||||
|
})
|
||||||
|
expect(bridge.definitions.find((tool) => tool.name === "rename_session")).toMatchObject({
|
||||||
|
description: expect.stringContaining("does not prompt, wake, or interrupt"),
|
||||||
|
parameters: {
|
||||||
|
type: "object",
|
||||||
|
additionalProperties: false,
|
||||||
|
required: ["session_id", "title"],
|
||||||
|
},
|
||||||
|
})
|
||||||
|
|
||||||
|
expect(await execute("rename_session", { session_id: "ses_unknown", title: "Nope" })).toEqual({
|
||||||
|
status: "error",
|
||||||
|
message: "Use a session ID returned by find_sessions or start_session.",
|
||||||
|
retryable: false,
|
||||||
|
})
|
||||||
|
expect(requests.some((request) => new URL(request.url).pathname.includes("ses_unknown"))).toBe(false)
|
||||||
|
|
||||||
|
await execute("find_sessions", {
|
||||||
|
query: null,
|
||||||
|
scope: "current_project",
|
||||||
|
recency: "any",
|
||||||
|
limit: 10,
|
||||||
|
})
|
||||||
|
expect(await execute("rename_session", { session_id: "ses_known", title: " " })).toEqual({
|
||||||
|
status: "error",
|
||||||
|
message: "A non-empty session title is required.",
|
||||||
|
retryable: false,
|
||||||
|
})
|
||||||
|
expect(await execute("rename_session", { session_id: "ses_known", title: "Old title" })).toEqual({
|
||||||
|
status: "unchanged",
|
||||||
|
session_id: "ses_known",
|
||||||
|
previous_title: "Old title",
|
||||||
|
title: "Old title",
|
||||||
|
})
|
||||||
|
expect(requests.some((request) => new URL(request.url).pathname.endsWith("/rename"))).toBe(false)
|
||||||
|
|
||||||
|
expect(await execute("rename_session", { session_id: "ses_known", title: " New title " })).toEqual({
|
||||||
|
status: "renamed",
|
||||||
|
session_id: "ses_known",
|
||||||
|
previous_title: "Old title",
|
||||||
|
title: "New title",
|
||||||
|
})
|
||||||
|
const rename = requests.find((request) => new URL(request.url).pathname.endsWith("/rename"))
|
||||||
|
expect(rename?.method).toBe("POST")
|
||||||
|
expect(await rename?.json()).toEqual({ title: "New title" })
|
||||||
|
} finally {
|
||||||
|
await bridge.close()
|
||||||
|
}
|
||||||
|
})
|
||||||
|
|
||||||
|
test("retries a lost completion wait and prompts a started session by ID", async () => {
|
||||||
|
const requests: Request[] = []
|
||||||
|
const prompts: Array<{ readonly id: string; readonly text: string }> = []
|
||||||
|
const notifications: Array<unknown> = []
|
||||||
|
const traces: Array<{ readonly event: string; readonly data?: Record<string, unknown> }> = []
|
||||||
|
let waits = 0
|
||||||
|
const fetch = Object.assign(
|
||||||
|
async (input: string | URL | Request, init?: RequestInit) => {
|
||||||
|
const request = input instanceof Request ? input : new Request(input.toString(), init)
|
||||||
|
requests.push(request)
|
||||||
|
const url = new URL(request.url)
|
||||||
|
if (url.pathname === "/api/event") return new Response("", { headers: { "content-type": "text/event-stream" } })
|
||||||
|
if (url.pathname === "/api/session" && request.method === "POST")
|
||||||
|
return Response.json({
|
||||||
|
data: {
|
||||||
|
id: "ses_focused",
|
||||||
|
projectID: "project-1",
|
||||||
|
cost: "0",
|
||||||
|
tokens: { input: 0, output: 0, reasoning: 0, cache: { read: 0, write: 0 } },
|
||||||
|
time: { created: 1, updated: 1 },
|
||||||
|
title: "Focused session",
|
||||||
|
location: { directory: "/workspace" },
|
||||||
|
},
|
||||||
|
})
|
||||||
|
if (url.pathname === "/api/session/ses_focused/prompt" && request.method === "POST") {
|
||||||
|
const body = await request.json()
|
||||||
|
if (
|
||||||
|
typeof body !== "object" ||
|
||||||
|
body === null ||
|
||||||
|
!("id" in body) ||
|
||||||
|
typeof body.id !== "string" ||
|
||||||
|
!("text" in body) ||
|
||||||
|
typeof body.text !== "string"
|
||||||
|
)
|
||||||
|
return new Response("Invalid prompt", { status: 400 })
|
||||||
|
prompts.push({ id: body.id, text: body.text })
|
||||||
|
return Response.json({ data: { id: body.id, admittedSeq: prompts.length } })
|
||||||
|
}
|
||||||
|
if (url.pathname === "/api/session/ses_focused/wait" && request.method === "POST") {
|
||||||
|
waits += 1
|
||||||
|
if (waits === 1) throw new Error("connection closed")
|
||||||
|
return new Response(null, { status: 204 })
|
||||||
|
}
|
||||||
|
if (url.pathname === "/api/session/ses_focused/message" && request.method === "GET")
|
||||||
|
return Response.json({
|
||||||
|
data: prompts.toReversed().map((prompt) => ({ id: prompt.id, type: "user", text: prompt.text })),
|
||||||
|
cursor: {},
|
||||||
|
})
|
||||||
|
return new Response("Not found", { status: 404 })
|
||||||
|
},
|
||||||
|
{ preconnect: () => {} },
|
||||||
|
)
|
||||||
|
const bridge = await createOpenCodeBridge({
|
||||||
|
client: OpenCode.make({ baseUrl: "https://opencode.test", fetch }),
|
||||||
|
directory: "/workspace",
|
||||||
|
model: { providerID: "openai", id: "gpt-test" },
|
||||||
|
notify: (notification) => notifications.push(notification),
|
||||||
|
trace: (event, data) => traces.push({ event, data }),
|
||||||
|
completionStore: memoryCompletionStore(),
|
||||||
|
})
|
||||||
|
const execute = async (name: string, input: Record<string, unknown>) => (await bridge.execute(name, input)).output
|
||||||
|
|
||||||
|
try {
|
||||||
|
expect(bridge.definitions.some((tool) => tool.name === "continue_session")).toBe(false)
|
||||||
|
const started = await bridge.execute("start_session", { text: "First task", project_id: null })
|
||||||
|
expect(started.output).toMatchObject({
|
||||||
|
status: "started",
|
||||||
|
session_id: "ses_focused",
|
||||||
|
})
|
||||||
|
expect(started.admittedPrompt).toEqual({ sessionID: "ses_focused", promptID: prompts[0].id })
|
||||||
|
expect(
|
||||||
|
await execute("prompt_session", { session_id: "ses_focused", text: "Follow-up task" }),
|
||||||
|
).toMatchObject({
|
||||||
|
status: "started",
|
||||||
|
session_id: "ses_focused",
|
||||||
|
})
|
||||||
|
|
||||||
|
await Bun.sleep(1_100)
|
||||||
|
expect(prompts.map((prompt) => prompt.text)).toEqual(["First task", "Follow-up task"])
|
||||||
|
expect(requests.filter((request) => new URL(request.url).pathname.endsWith("/wait")).length).toBeGreaterThanOrEqual(
|
||||||
|
3,
|
||||||
|
)
|
||||||
|
expect(traces.some((entry) => entry.event === "opencode.wait.retrying")).toBe(true)
|
||||||
|
expect(notifications.some((value) => JSON.stringify(value).includes("opencode.prompt.failed"))).toBe(false)
|
||||||
|
} finally {
|
||||||
|
await bridge.close()
|
||||||
|
}
|
||||||
|
})
|
||||||
|
|
||||||
|
test("discards a restored prompt whose session no longer exists", async () => {
|
||||||
|
const removed: Array<{ readonly sessionID: string; readonly promptID: string }> = []
|
||||||
|
const handle = { sessionID: "ses_missing", promptID: "msg_orphaned" }
|
||||||
|
const fetch = Object.assign(
|
||||||
|
async (input: string | URL | Request) => {
|
||||||
|
const request = input instanceof Request ? input : new Request(input.toString())
|
||||||
|
const url = new URL(request.url)
|
||||||
|
if (url.pathname === "/api/event") return new Response("", { headers: { "content-type": "text/event-stream" } })
|
||||||
|
if (url.pathname === "/api/session/ses_missing")
|
||||||
|
return Response.json(
|
||||||
|
{ _tag: "SessionNotFoundError", sessionID: handle.sessionID, message: `Session not found: ${handle.sessionID}` },
|
||||||
|
{ status: 404 },
|
||||||
|
)
|
||||||
|
return new Response("Not found", { status: 404 })
|
||||||
|
},
|
||||||
|
{ preconnect: () => {} },
|
||||||
|
)
|
||||||
|
const store = {
|
||||||
|
...memoryCompletionStore(),
|
||||||
|
entries: () => [{ status: "pending" as const, handle }],
|
||||||
|
remove: async (entry) => {
|
||||||
|
removed.push(entry)
|
||||||
|
},
|
||||||
|
} satisfies CompletionStore
|
||||||
|
|
||||||
|
const bridge = await createOpenCodeBridge({
|
||||||
|
client: OpenCode.make({ baseUrl: "https://opencode.test", fetch }),
|
||||||
|
directory: "/workspace",
|
||||||
|
model: { providerID: "openai", id: "gpt-test" },
|
||||||
|
notify: () => {},
|
||||||
|
completionStore: store,
|
||||||
|
})
|
||||||
|
try {
|
||||||
|
expect(removed).toEqual([handle])
|
||||||
|
} finally {
|
||||||
|
await bridge.close()
|
||||||
|
}
|
||||||
|
})
|
||||||
|
|
||||||
|
function memoryCompletionStore(): CompletionStore {
|
||||||
|
return {
|
||||||
|
entries: () => [],
|
||||||
|
admitting: async () => {},
|
||||||
|
pending: async () => {},
|
||||||
|
completed: async () => {},
|
||||||
|
remove: async () => {},
|
||||||
|
close: async () => {},
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,10 @@
|
|||||||
|
import { expect, test } from "bun:test"
|
||||||
|
import { pcmLevel } from "../src/pcm"
|
||||||
|
|
||||||
|
test("PCM level ignores silence and normalizes audible energy", () => {
|
||||||
|
expect(pcmLevel(Buffer.alloc(4_800))).toBe(0)
|
||||||
|
const audible = Buffer.alloc(4_800)
|
||||||
|
for (let offset = 0; offset < audible.length; offset += 2) audible.writeInt16LE(8_000, offset)
|
||||||
|
expect(pcmLevel(audible)).toBeGreaterThan(0.8)
|
||||||
|
expect(pcmLevel(audible)).toBeLessThanOrEqual(1)
|
||||||
|
})
|
||||||
@@ -0,0 +1,137 @@
|
|||||||
|
import { describe, expect, test } from "bun:test"
|
||||||
|
import { Option } from "effect"
|
||||||
|
import { decodeVoiceToolInput } from "../src/protocol"
|
||||||
|
import { createLiveContextAppendQueue, liveNotificationAcknowledgement } from "../src/protocol-live"
|
||||||
|
import { realtimeNotificationAcknowledgement, realtimeSessionUpdate } from "../src/protocol-realtime"
|
||||||
|
import { createSingleFlightAcknowledgement } from "../src/single-flight-acknowledgement"
|
||||||
|
import { openCodeAnnouncementText } from "../src/opencode-notification"
|
||||||
|
|
||||||
|
describe("voice protocol", () => {
|
||||||
|
test("accepts only JSON object tool arguments", () => {
|
||||||
|
expect(Option.getOrUndefined(decodeVoiceToolInput('{"text":"hello","nested":{"value":1}}'))).toEqual({
|
||||||
|
text: "hello",
|
||||||
|
nested: { value: 1 },
|
||||||
|
})
|
||||||
|
for (const input of ["not json", "null", "[]", "1", '"text"'])
|
||||||
|
expect(Option.getOrUndefined(decodeVoiceToolInput(input))).toBeUndefined()
|
||||||
|
})
|
||||||
|
|
||||||
|
test("correlates Realtime notification acknowledgements", () => {
|
||||||
|
const pending = { itemID: "item-current", eventID: "event-current" }
|
||||||
|
expect(
|
||||||
|
realtimeNotificationAcknowledgement(
|
||||||
|
{ type: "conversation.item.created", item: { id: "item-current" } },
|
||||||
|
pending,
|
||||||
|
),
|
||||||
|
).toBe(true)
|
||||||
|
expect(
|
||||||
|
realtimeNotificationAcknowledgement(
|
||||||
|
{ type: "conversation.item.created", item: { id: "item-other" } },
|
||||||
|
pending,
|
||||||
|
),
|
||||||
|
).toBeUndefined()
|
||||||
|
expect(
|
||||||
|
realtimeNotificationAcknowledgement(
|
||||||
|
{ type: "error", error: { event_id: "event-current" } },
|
||||||
|
pending,
|
||||||
|
),
|
||||||
|
).toBe(false)
|
||||||
|
})
|
||||||
|
|
||||||
|
test("preserves tool schemas in Realtime session updates", () => {
|
||||||
|
const tools = [
|
||||||
|
{
|
||||||
|
type: "function" as const,
|
||||||
|
name: "open_form",
|
||||||
|
description: "Open form",
|
||||||
|
parameters: {
|
||||||
|
type: "object",
|
||||||
|
properties: { answer: { type: "object", additionalProperties: true } },
|
||||||
|
additionalProperties: true,
|
||||||
|
},
|
||||||
|
},
|
||||||
|
]
|
||||||
|
expect(JSON.parse(JSON.stringify(realtimeSessionUpdate({ instructions: "test", tools, voice: "marin" }))).session.tools)
|
||||||
|
.toEqual(tools)
|
||||||
|
})
|
||||||
|
|
||||||
|
test("correlates Live notification acknowledgements by expected context", () => {
|
||||||
|
const pending = { eventID: "event-current", acknowledgement: "delegation.context.appended" as const }
|
||||||
|
expect(
|
||||||
|
liveNotificationAcknowledgement(
|
||||||
|
{ type: "delegation.context.appended", eventID: "event-current" },
|
||||||
|
pending,
|
||||||
|
),
|
||||||
|
).toBe(true)
|
||||||
|
expect(
|
||||||
|
liveNotificationAcknowledgement({ type: "session.context.appended", eventID: "event-current" }, pending),
|
||||||
|
).toBeUndefined()
|
||||||
|
expect(
|
||||||
|
liveNotificationAcknowledgement(
|
||||||
|
{ type: "delegation.context.appended", eventID: "event-other" },
|
||||||
|
pending,
|
||||||
|
),
|
||||||
|
).toBe(true)
|
||||||
|
expect(
|
||||||
|
liveNotificationAcknowledgement({ type: "error", errorEventID: "event-current" }, pending),
|
||||||
|
).toBe(false)
|
||||||
|
})
|
||||||
|
|
||||||
|
test("serializes indistinguishable Live context appends", async () => {
|
||||||
|
const sent: Array<Record<string, unknown>> = []
|
||||||
|
const queue = createLiveContextAppendQueue({
|
||||||
|
send: (event) => {
|
||||||
|
sent.push(event)
|
||||||
|
return true
|
||||||
|
},
|
||||||
|
timeoutMs: 100,
|
||||||
|
})
|
||||||
|
const first = queue.append(
|
||||||
|
{ type: "delegation.context.append", event_id: "event-1" },
|
||||||
|
"delegation.context.appended",
|
||||||
|
)
|
||||||
|
const second = queue.append(
|
||||||
|
{ type: "delegation.context.append", event_id: "event-2" },
|
||||||
|
"delegation.context.appended",
|
||||||
|
)
|
||||||
|
expect(sent.map((event) => event["event_id"])).toEqual(["event-1"])
|
||||||
|
|
||||||
|
queue.receive({ type: "session.context.appended", eventID: "unrelated" })
|
||||||
|
expect(sent.map((event) => event["event_id"])).toEqual(["event-1"])
|
||||||
|
queue.receive({ type: "delegation.context.appended", eventID: "not-echoed" })
|
||||||
|
expect(await first).toBe(true)
|
||||||
|
expect(sent.map((event) => event["event_id"])).toEqual(["event-1", "event-2"])
|
||||||
|
queue.receive({ type: "delegation.context.appended" })
|
||||||
|
expect(await second).toBe(true)
|
||||||
|
queue.close()
|
||||||
|
})
|
||||||
|
|
||||||
|
test("keeps notification acknowledgement single-flight", async () => {
|
||||||
|
const acknowledgement = createSingleFlightAcknowledgement<{ readonly token: string }>(50)
|
||||||
|
const first = acknowledgement.begin("notification-1", { token: "first" })
|
||||||
|
const second = acknowledgement.begin("notification-2", { token: "second" })
|
||||||
|
|
||||||
|
expect(first.started).toBe(true)
|
||||||
|
expect(second.started).toBe(false)
|
||||||
|
expect(await second.promise).toBe(false)
|
||||||
|
expect(acknowledgement.current()).toEqual({ id: "notification-1", correlation: { token: "first" } })
|
||||||
|
acknowledgement.settle("notification-1", true)
|
||||||
|
expect(await first.promise).toBe(true)
|
||||||
|
expect(acknowledgement.current()).toBeUndefined()
|
||||||
|
})
|
||||||
|
|
||||||
|
test("renders bounded conversational OpenCode announcements without identifiers", () => {
|
||||||
|
const text = openCodeAnnouncementText({
|
||||||
|
type: "opencode.prompt.completed",
|
||||||
|
session_id: "ses_private",
|
||||||
|
prompt_id: "msg_private",
|
||||||
|
status: "completed",
|
||||||
|
text: "finished ".repeat(200),
|
||||||
|
})
|
||||||
|
expect(text.length).toBeLessThanOrEqual(800)
|
||||||
|
expect(text).not.toContain("ses_private")
|
||||||
|
expect(text).not.toContain("msg_private")
|
||||||
|
expect(text).toStartWith("OpenCode prompt completed: finished")
|
||||||
|
})
|
||||||
|
|
||||||
|
})
|
||||||
@@ -0,0 +1,69 @@
|
|||||||
|
import { expect, test } from "bun:test"
|
||||||
|
import { createRealtimeEventProjector } from "../src/protocol-realtime"
|
||||||
|
|
||||||
|
test("projects one tool start and one work request for duplicate item frames", () => {
|
||||||
|
const projector = createRealtimeEventProjector()
|
||||||
|
const item = {
|
||||||
|
type: "function_call",
|
||||||
|
name: "find_sessions",
|
||||||
|
call_id: "call-1",
|
||||||
|
arguments: '{"query":null}',
|
||||||
|
}
|
||||||
|
const added = projector.receive(JSON.stringify({ type: "response.output_item.added", item }))
|
||||||
|
const duplicate = projector.receive(JSON.stringify({ type: "response.output_item.added", item }))
|
||||||
|
const done = projector.receive(JSON.stringify({ type: "response.output_item.done", item }))
|
||||||
|
|
||||||
|
expect(added.events).toContainEqual({ type: "tool.started", id: "call-1", name: "find_sessions" })
|
||||||
|
expect(duplicate.events.some((event) => event.type === "tool.started")).toBe(false)
|
||||||
|
expect(done.events).toContainEqual({
|
||||||
|
type: "work.requested",
|
||||||
|
request: { id: "call-1", name: "find_sessions", input: { query: null } },
|
||||||
|
})
|
||||||
|
expect(projector.receive(JSON.stringify({ type: "response.output_item.done", item })).events).toEqual([
|
||||||
|
{ type: "debug", message: "response.output_item.done" },
|
||||||
|
])
|
||||||
|
})
|
||||||
|
|
||||||
|
test("resumes a response only after all projected work resolves", () => {
|
||||||
|
const projector = createRealtimeEventProjector()
|
||||||
|
const response = projector.receive(
|
||||||
|
JSON.stringify({
|
||||||
|
type: "response.done",
|
||||||
|
response: { output: [{ type: "function_call", call_id: "call-1" }] },
|
||||||
|
}),
|
||||||
|
)
|
||||||
|
|
||||||
|
expect(response.events).toContainEqual({ type: "assistant.done", awaitingWork: true })
|
||||||
|
expect(response.commands).toEqual([])
|
||||||
|
expect(projector.resolveWork("call-1")).toEqual([{ type: "response.create" }])
|
||||||
|
expect(projector.resolveWork("call-1")).toEqual([])
|
||||||
|
})
|
||||||
|
|
||||||
|
test("resumes when work resolved before the response completion frame", () => {
|
||||||
|
const projector = createRealtimeEventProjector()
|
||||||
|
expect(projector.resolveWork("call-1")).toEqual([])
|
||||||
|
expect(
|
||||||
|
projector.receive(
|
||||||
|
JSON.stringify({
|
||||||
|
type: "response.done",
|
||||||
|
response: { output: [{ type: "function_call", call_id: "call-1" }] },
|
||||||
|
}),
|
||||||
|
).commands,
|
||||||
|
).toEqual([{ type: "response.create" }])
|
||||||
|
})
|
||||||
|
|
||||||
|
test("rejects malformed tool arguments without hanging the provider call", () => {
|
||||||
|
const projector = createRealtimeEventProjector()
|
||||||
|
const result = projector.receive(
|
||||||
|
JSON.stringify({
|
||||||
|
type: "response.output_item.done",
|
||||||
|
item: { type: "function_call", name: "find_sessions", call_id: "call-1", arguments: "[]" },
|
||||||
|
}),
|
||||||
|
)
|
||||||
|
|
||||||
|
expect(result.events).toContainEqual({
|
||||||
|
type: "work.rejected",
|
||||||
|
request: { id: "call-1", name: "find_sessions", input: {} },
|
||||||
|
output: { status: "error", message: "Invalid arguments for tool find_sessions." },
|
||||||
|
})
|
||||||
|
})
|
||||||
@@ -0,0 +1,229 @@
|
|||||||
|
import { expect, test } from "bun:test"
|
||||||
|
import {
|
||||||
|
createResponsesControllerContext,
|
||||||
|
responsesControllerTools,
|
||||||
|
runResponsesController,
|
||||||
|
} from "../src/responses-controller"
|
||||||
|
|
||||||
|
test("runs the client delegation controller through OpenCode tools", async () => {
|
||||||
|
const bodies: Array<Record<string, unknown>> = []
|
||||||
|
const calls: Array<unknown> = []
|
||||||
|
const responses = [
|
||||||
|
{
|
||||||
|
id: "response-1",
|
||||||
|
output: [
|
||||||
|
{
|
||||||
|
type: "function_call",
|
||||||
|
call_id: "call-1",
|
||||||
|
name: "start_session",
|
||||||
|
arguments: '{"text":"Inspect the package","project_id":null}',
|
||||||
|
},
|
||||||
|
],
|
||||||
|
},
|
||||||
|
{
|
||||||
|
id: "response-2",
|
||||||
|
output: [
|
||||||
|
{
|
||||||
|
type: "message",
|
||||||
|
content: [{ type: "output_text", text: "The inspection is running and I will report back." }],
|
||||||
|
},
|
||||||
|
],
|
||||||
|
},
|
||||||
|
]
|
||||||
|
const fetch = Object.assign(
|
||||||
|
async (_input: string | URL | Request, init?: RequestInit) => {
|
||||||
|
const body: unknown = await new Request("https://opencode.test", init).json()
|
||||||
|
if (body && typeof body === "object" && !Array.isArray(body)) bodies.push(Object.fromEntries(Object.entries(body)))
|
||||||
|
return Response.json(responses.shift())
|
||||||
|
},
|
||||||
|
{ preconnect: () => {} },
|
||||||
|
)
|
||||||
|
|
||||||
|
const text = await runResponsesController({
|
||||||
|
apiKey: "test",
|
||||||
|
model: "gpt-test",
|
||||||
|
instructions: "Use tools.",
|
||||||
|
text: "Inspect this package",
|
||||||
|
tools: [{ type: "function", name: "start_session", description: "Start", parameters: {} }],
|
||||||
|
execute: async (call) => {
|
||||||
|
calls.push(call)
|
||||||
|
return { status: "started", session_id: "ses_started", prompt_id: "prompt-1" }
|
||||||
|
},
|
||||||
|
fetch,
|
||||||
|
})
|
||||||
|
|
||||||
|
expect(text).toBe(
|
||||||
|
"The inspection is running and I will report back.\n\nPrivate voice-control context (never speak this aloud): OpenCode session IDs explicitly used by this delegation: ses_started. Include the relevant session_id in any future delegated request to continue this work.",
|
||||||
|
)
|
||||||
|
expect(calls).toEqual([
|
||||||
|
{
|
||||||
|
id: "call-1",
|
||||||
|
name: "start_session",
|
||||||
|
input: { text: "Inspect the package", project_id: null },
|
||||||
|
},
|
||||||
|
])
|
||||||
|
expect(bodies[1]).toMatchObject({
|
||||||
|
previous_response_id: "response-1",
|
||||||
|
input: [
|
||||||
|
{
|
||||||
|
type: "function_call_output",
|
||||||
|
call_id: "call-1",
|
||||||
|
output: '{"status":"started","session_id":"ses_started","prompt_id":"prompt-1"}',
|
||||||
|
},
|
||||||
|
],
|
||||||
|
})
|
||||||
|
expect(bodies[0]?.["tools"]).toEqual([
|
||||||
|
{ type: "function", name: "start_session", description: "Start", parameters: {} },
|
||||||
|
])
|
||||||
|
})
|
||||||
|
|
||||||
|
test("rejects malformed controller tool arguments", async () => {
|
||||||
|
const fetch = Object.assign(
|
||||||
|
async () =>
|
||||||
|
Response.json({
|
||||||
|
id: "response-1",
|
||||||
|
output: [{ type: "function_call", call_id: "call-1", name: "start_session", arguments: "not json" }],
|
||||||
|
}),
|
||||||
|
{ preconnect: () => {} },
|
||||||
|
)
|
||||||
|
|
||||||
|
expect(
|
||||||
|
runResponsesController({
|
||||||
|
apiKey: "test",
|
||||||
|
model: "gpt-test",
|
||||||
|
instructions: "Use tools.",
|
||||||
|
text: "Inspect this package",
|
||||||
|
tools: [{ type: "function", name: "start_session", description: "Start", parameters: {} }],
|
||||||
|
execute: async () => ({ status: "started" }),
|
||||||
|
fetch,
|
||||||
|
}),
|
||||||
|
).rejects.toThrow("invalid arguments for start_session")
|
||||||
|
})
|
||||||
|
|
||||||
|
test("retains the most recently used session across delegations", async () => {
|
||||||
|
const context = createResponsesControllerContext()
|
||||||
|
const bodies: Array<Record<string, unknown>> = []
|
||||||
|
const responses = [
|
||||||
|
{
|
||||||
|
id: "response-1",
|
||||||
|
output: [
|
||||||
|
{
|
||||||
|
type: "function_call",
|
||||||
|
call_id: "call-1",
|
||||||
|
name: "read_session",
|
||||||
|
arguments: '{"session_id":"ses_recent"}',
|
||||||
|
},
|
||||||
|
],
|
||||||
|
},
|
||||||
|
{
|
||||||
|
id: "response-2",
|
||||||
|
output: [{ type: "message", content: [{ type: "output_text", text: "Found it." }] }],
|
||||||
|
},
|
||||||
|
{
|
||||||
|
id: "response-3",
|
||||||
|
output: [{ type: "message", content: [{ type: "output_text", text: "Stopped it." }] }],
|
||||||
|
},
|
||||||
|
]
|
||||||
|
const fetch = Object.assign(
|
||||||
|
async (_input: string | URL | Request, init?: RequestInit) => {
|
||||||
|
const body: unknown = await new Request("https://opencode.test", init).json()
|
||||||
|
if (body && typeof body === "object" && !Array.isArray(body)) bodies.push(Object.fromEntries(Object.entries(body)))
|
||||||
|
return Response.json(responses.shift())
|
||||||
|
},
|
||||||
|
{ preconnect: () => {} },
|
||||||
|
)
|
||||||
|
const options = {
|
||||||
|
apiKey: "test",
|
||||||
|
model: "gpt-test",
|
||||||
|
instructions: "Use tools.",
|
||||||
|
tools: [{ type: "function" as const, name: "read_session", description: "Read", parameters: {} }],
|
||||||
|
execute: async () => ({ status: "ok" }),
|
||||||
|
context,
|
||||||
|
fetch,
|
||||||
|
}
|
||||||
|
|
||||||
|
await runResponsesController({ ...options, text: "Find the label behavior session" })
|
||||||
|
await runResponsesController({ ...options, text: "Stop that session" })
|
||||||
|
|
||||||
|
expect(context.lastSessionID).toBe("ses_recent")
|
||||||
|
expect(bodies[2]?.["input"]).toContain("the most recently used OpenCode session ID is ses_recent")
|
||||||
|
expect(bodies[2]?.["input"]).toContain('references like "that session", "it", or "stop that"')
|
||||||
|
})
|
||||||
|
|
||||||
|
test("coalesces identical read-only calls within one delegation", async () => {
|
||||||
|
const responses = [
|
||||||
|
{
|
||||||
|
id: "response-1",
|
||||||
|
output: [
|
||||||
|
{ type: "function_call", call_id: "call-1", name: "read_session", arguments: '{"session_id":"ses_1"}' },
|
||||||
|
{ type: "function_call", call_id: "call-2", name: "read_session", arguments: '{"session_id":"ses_1"}' },
|
||||||
|
],
|
||||||
|
},
|
||||||
|
{
|
||||||
|
id: "response-2",
|
||||||
|
output: [{ type: "message", content: [{ type: "output_text", text: "Done." }] }],
|
||||||
|
},
|
||||||
|
]
|
||||||
|
let executions = 0
|
||||||
|
const fetch = Object.assign(async () => Response.json(responses.shift()), { preconnect: () => {} })
|
||||||
|
|
||||||
|
await runResponsesController({
|
||||||
|
apiKey: "test",
|
||||||
|
model: "gpt-test",
|
||||||
|
instructions: "Use tools.",
|
||||||
|
text: "Read it",
|
||||||
|
tools: [{ type: "function", name: "read_session", description: "Read", parameters: {} }],
|
||||||
|
execute: async () => {
|
||||||
|
executions += 1
|
||||||
|
return { status: "ok" }
|
||||||
|
},
|
||||||
|
fetch,
|
||||||
|
})
|
||||||
|
|
||||||
|
expect(executions).toBe(1)
|
||||||
|
})
|
||||||
|
|
||||||
|
test("withholds new-session creation during follow-ups unless explicitly requested", () => {
|
||||||
|
const context = createResponsesControllerContext()
|
||||||
|
context.lastSessionID = "ses_recent"
|
||||||
|
const tools = [
|
||||||
|
{ type: "function" as const, name: "start_session", description: "Start", parameters: {} },
|
||||||
|
{ type: "function" as const, name: "read_session", description: "Read", parameters: {} },
|
||||||
|
]
|
||||||
|
|
||||||
|
expect(responsesControllerTools(tools, "Stop that session", context).map((tool) => tool.name)).toEqual([
|
||||||
|
"read_session",
|
||||||
|
])
|
||||||
|
expect(responsesControllerTools(tools, "Start a new session for this", context)).toEqual(tools)
|
||||||
|
})
|
||||||
|
|
||||||
|
test("remembers a uniquely discovered session without requiring a read", async () => {
|
||||||
|
const context = createResponsesControllerContext()
|
||||||
|
const responses = [
|
||||||
|
{
|
||||||
|
id: "response-1",
|
||||||
|
output: [
|
||||||
|
{ type: "function_call", call_id: "call-1", name: "find_sessions", arguments: '{"query":"label"}' },
|
||||||
|
],
|
||||||
|
},
|
||||||
|
{
|
||||||
|
id: "response-2",
|
||||||
|
output: [{ type: "message", content: [{ type: "output_text", text: "Found one." }] }],
|
||||||
|
},
|
||||||
|
]
|
||||||
|
const fetch = Object.assign(async () => Response.json(responses.shift()), { preconnect: () => {} })
|
||||||
|
|
||||||
|
await runResponsesController({
|
||||||
|
apiKey: "test",
|
||||||
|
model: "gpt-test",
|
||||||
|
instructions: "Use tools.",
|
||||||
|
text: "Find label behavior",
|
||||||
|
tools: [{ type: "function", name: "find_sessions", description: "Find", parameters: {} }],
|
||||||
|
execute: async () => ({ status: "ok", sessions: [{ id: "ses_unique", title: "Label behavior" }] }),
|
||||||
|
context,
|
||||||
|
fetch,
|
||||||
|
})
|
||||||
|
|
||||||
|
expect(context.lastSessionID).toBe("ses_unique")
|
||||||
|
expect(context.sessionIDs).toEqual(new Set(["ses_unique"]))
|
||||||
|
})
|
||||||
@@ -0,0 +1,331 @@
|
|||||||
|
import { expect, mock, test } from "bun:test"
|
||||||
|
import { createTestRenderer } from "@opentui/core/testing"
|
||||||
|
|
||||||
|
test("voice rows fill the viewport, wrap under content, and keep tools after their preamble", async () => {
|
||||||
|
const setup = await createTestRenderer({ width: 64, height: 18, useThread: false })
|
||||||
|
const core = await import("@opentui/core")
|
||||||
|
void mock.module("@opentui/core", () => ({ ...core, createCliRenderer: async () => setup.renderer }))
|
||||||
|
const { createVoiceTUI } = await import("../src/ui")
|
||||||
|
let interrupts = 0
|
||||||
|
let voiceCycles = 0
|
||||||
|
let microphoneToggles = 0
|
||||||
|
let speakerToggles = 0
|
||||||
|
const ui = await createVoiceTUI({
|
||||||
|
onInterrupt: () => interrupts++,
|
||||||
|
onExit: () => {},
|
||||||
|
onCycleVoice: () => voiceCycles++,
|
||||||
|
onToggleMicrophone: () => microphoneToggles++,
|
||||||
|
onToggleSpeaker: () => speakerToggles++,
|
||||||
|
reducedMotion: true,
|
||||||
|
})
|
||||||
|
|
||||||
|
try {
|
||||||
|
ui.setStatus({ audio: "duplex", voice: "marin", model: "gpt-live" })
|
||||||
|
ui.meta("[session] created session-1")
|
||||||
|
ui.userCommitted("user-1")
|
||||||
|
ui.userTranscript("user-1", "Please inspect the current session", true)
|
||||||
|
ui.toolStart("tool-1", "opencode", { request: "inspect" })
|
||||||
|
ui.meta("[session] adopted session-1")
|
||||||
|
ui.assistantDelta("Let me check. This sentence is")
|
||||||
|
ui.assistantDelta(" long enough to wrap cleanly under its column.")
|
||||||
|
ui.assistantDone()
|
||||||
|
ui.assistantDelta("Continued in the same assistant row.")
|
||||||
|
ui.assistantDone()
|
||||||
|
await Bun.sleep(200)
|
||||||
|
await setup.flush()
|
||||||
|
|
||||||
|
const rows = frameRows(setup.captureCharFrame())
|
||||||
|
expect(rows).toHaveLength(18)
|
||||||
|
expect(rows.every((row) => row.length === 64)).toBe(true)
|
||||||
|
expect(rows[16]).toStartWith(" ▂▂▂▁ duplex marin gpt-live")
|
||||||
|
expect(rows[16]).not.toContain("no session")
|
||||||
|
expect(rows[17]).toContain("esc interrupt")
|
||||||
|
expect(rows[17]).not.toContain("any key")
|
||||||
|
expect(rows[15].trim()).toBe("")
|
||||||
|
expect(rows.filter((row) => row.includes("Let me check"))).toHaveLength(1)
|
||||||
|
expect(rows.findIndex((row) => row.includes("opencode"))).toBeGreaterThan(
|
||||||
|
rows.findIndex((row) => row.includes("Let me check")),
|
||||||
|
)
|
||||||
|
expect(rows.findIndex((row) => row.includes("[session] adopted"))).toBeGreaterThan(
|
||||||
|
rows.findIndex((row) => row.includes("opencode")),
|
||||||
|
)
|
||||||
|
expect(rows.find((row) => row.includes("under its column"))?.indexOf("under")).toBe(
|
||||||
|
rows.find((row) => row.includes("Let me check"))?.indexOf("Let"),
|
||||||
|
)
|
||||||
|
// Deltas inside one turn concatenate raw; a delta after assistantDone starts a new
|
||||||
|
// message and must gain exactly one space at the boundary.
|
||||||
|
const assistant = rows.join("")
|
||||||
|
expect(assistant).toContain("This sentence is long enough")
|
||||||
|
expect(assistant).toContain("column. Continued")
|
||||||
|
expect(assistant).not.toContain("column.Continued")
|
||||||
|
expect(rows.find((row) => row.includes("[session] created"))).toStartWith(" · [session]")
|
||||||
|
|
||||||
|
setup.resize(96, 22)
|
||||||
|
await setup.flush()
|
||||||
|
const wide = frameRows(setup.captureCharFrame())
|
||||||
|
expect(wide).toHaveLength(22)
|
||||||
|
expect(wide.every((row) => row.length === 96)).toBe(true)
|
||||||
|
expect(wide.at(-2)).toStartWith(" ▂▂▂▁ duplex marin gpt-live")
|
||||||
|
|
||||||
|
setup.resize(48, 16)
|
||||||
|
await setup.flush()
|
||||||
|
const narrow = frameRows(setup.captureCharFrame())
|
||||||
|
expect(narrow).toHaveLength(16)
|
||||||
|
expect(narrow.every((row) => row.length === 48)).toBe(true)
|
||||||
|
expect(narrow.at(-1)).toContain("esc interrupt")
|
||||||
|
|
||||||
|
setup.resize(64, 18)
|
||||||
|
await setup.flush()
|
||||||
|
|
||||||
|
ui.userSpeaking(true)
|
||||||
|
await setup.flush()
|
||||||
|
expect(setup.captureCharFrame()).not.toContain("│ transcribing")
|
||||||
|
expect(frameRows(setup.captureCharFrame()).at(-2)).toStartWith(" ▂▂▂▂")
|
||||||
|
|
||||||
|
ui.userReset()
|
||||||
|
await setup.flush()
|
||||||
|
expect(setup.captureCharFrame()).not.toContain("│ transcribing")
|
||||||
|
|
||||||
|
ui.userSpeaking(true)
|
||||||
|
ui.userSpeaking(false)
|
||||||
|
ui.userCommitted("user-2")
|
||||||
|
ui.userTranscript("user-2", "partial transcript", false)
|
||||||
|
await setup.flush()
|
||||||
|
const partial = setup
|
||||||
|
.captureCharFrame()
|
||||||
|
.split("\n")
|
||||||
|
.find((row) => row.includes("partial transcript"))
|
||||||
|
expect(partial).toContain("│ partial transcript")
|
||||||
|
expect(partial).not.toContain("you")
|
||||||
|
|
||||||
|
ui.userSpeaking(true)
|
||||||
|
await setup.flush()
|
||||||
|
expect(
|
||||||
|
setup
|
||||||
|
.captureCharFrame()
|
||||||
|
.split("\n")
|
||||||
|
.filter((row) => row.includes("│ transcribing")),
|
||||||
|
).toHaveLength(0)
|
||||||
|
|
||||||
|
ui.userReset()
|
||||||
|
await setup.flush()
|
||||||
|
|
||||||
|
ui.setStatus({ microphoneMuted: true })
|
||||||
|
await setup.flush()
|
||||||
|
expect(frameRows(setup.captureCharFrame()).at(-2)).toStartWith(" ────")
|
||||||
|
const muted = setup
|
||||||
|
.captureCharFrame()
|
||||||
|
.split("\n")
|
||||||
|
.find((row) => row.includes("partial transcript"))
|
||||||
|
expect(muted).toContain("│ partial transcript")
|
||||||
|
expect(muted).not.toContain("│ ...")
|
||||||
|
|
||||||
|
ui.userSpeaking(false)
|
||||||
|
ui.setStatus({ microphoneMuted: false })
|
||||||
|
ui.userTranscript("user-2", "partial transcript", true)
|
||||||
|
await setup.flush()
|
||||||
|
const complete = setup
|
||||||
|
.captureCharFrame()
|
||||||
|
.split("\n")
|
||||||
|
.find((row) => row.includes("partial transcript"))
|
||||||
|
expect(complete).toContain("│ partial transcript")
|
||||||
|
|
||||||
|
ui.assistantDelta("First response.")
|
||||||
|
ui.userCommitted("boundary-user")
|
||||||
|
ui.userTranscript("boundary-user", "A new request", true)
|
||||||
|
ui.assistantDelta("Second response.")
|
||||||
|
await setup.flush()
|
||||||
|
const boundaries = setup.captureCharFrame().split("\n")
|
||||||
|
expect(boundaries.findIndex((row) => row.includes("First response."))).toBeLessThan(
|
||||||
|
boundaries.findIndex((row) => row.includes("A new request")),
|
||||||
|
)
|
||||||
|
expect(boundaries.findIndex((row) => row.includes("A new request"))).toBeLessThan(
|
||||||
|
boundaries.findIndex((row) => row.includes("Second response.")),
|
||||||
|
)
|
||||||
|
|
||||||
|
await setup.mockInput.typeText(" x")
|
||||||
|
setup.mockInput.pressArrow("right")
|
||||||
|
await setup.flush()
|
||||||
|
expect(interrupts).toBe(0)
|
||||||
|
expect(voiceCycles).toBe(0)
|
||||||
|
expect(microphoneToggles).toBe(0)
|
||||||
|
expect(speakerToggles).toBe(0)
|
||||||
|
|
||||||
|
setup.mockInput.pressKey("v")
|
||||||
|
setup.mockInput.pressKey("m")
|
||||||
|
setup.mockInput.pressKey("s")
|
||||||
|
await setup.flush()
|
||||||
|
expect(voiceCycles).toBe(1)
|
||||||
|
expect(microphoneToggles).toBe(1)
|
||||||
|
expect(speakerToggles).toBe(1)
|
||||||
|
|
||||||
|
setup.mockInput.pressEscape()
|
||||||
|
await Bun.sleep(50)
|
||||||
|
await setup.flush()
|
||||||
|
expect(interrupts).toBe(1)
|
||||||
|
} finally {
|
||||||
|
ui.close()
|
||||||
|
mock.restore()
|
||||||
|
}
|
||||||
|
})
|
||||||
|
|
||||||
|
// In audio mode ui.assistantDone() is scheduled off the AudioSession playback clock,
|
||||||
|
// so the next turn's transcript deltas arrive while the row is still streaming. The turn boundary
|
||||||
|
// therefore has to survive in the delta text itself, not in the row's streaming flag.
|
||||||
|
test("keeps the turn boundary when the next turn streams before playback finishes", async () => {
|
||||||
|
const setup = await createTestRenderer({ width: 72, height: 12, useThread: false })
|
||||||
|
const core = await import("@opentui/core")
|
||||||
|
void mock.module("@opentui/core", () => ({ ...core, createCliRenderer: async () => setup.renderer }))
|
||||||
|
const { createVoiceTUI } = await import("../src/ui")
|
||||||
|
const ui = await createVoiceTUI({
|
||||||
|
onInterrupt: () => {},
|
||||||
|
onExit: () => {},
|
||||||
|
onCycleVoice: () => {},
|
||||||
|
onToggleMicrophone: () => {},
|
||||||
|
onToggleSpeaker: () => {},
|
||||||
|
reducedMotion: true,
|
||||||
|
})
|
||||||
|
|
||||||
|
try {
|
||||||
|
// Verbatim projector deltas: each turn's first fragment carries the boundary space.
|
||||||
|
ui.assistantDelta(' Session "Fix auth bug"')
|
||||||
|
ui.assistantDelta(" is ready.")
|
||||||
|
// turn.done reaches the protocol, but playback is still draining, so no assistantDone() yet.
|
||||||
|
ui.assistantDelta(" Next up, tests.")
|
||||||
|
await setup.flush()
|
||||||
|
|
||||||
|
const frame = setup.captureCharFrame()
|
||||||
|
expect(frame).toContain('Session "Fix auth bug" is ready. Next up, tests.')
|
||||||
|
expect(frame).not.toContain("ready.Next")
|
||||||
|
// The row itself must not open with the boundary space.
|
||||||
|
expect(frame).toContain('│ Session "Fix auth bug"')
|
||||||
|
expect(frame).not.toContain('│ Session "Fix auth bug"')
|
||||||
|
|
||||||
|
ui.assistantDone()
|
||||||
|
await setup.flush()
|
||||||
|
expect(setup.captureCharFrame()).toContain("is ready. Next up, tests.")
|
||||||
|
} finally {
|
||||||
|
ui.close()
|
||||||
|
mock.restore()
|
||||||
|
}
|
||||||
|
})
|
||||||
|
|
||||||
|
test("keeps animated text mounted after its reveal completes", async () => {
|
||||||
|
const setup = await createTestRenderer({ width: 72, height: 10, useThread: false })
|
||||||
|
const core = await import("@opentui/core")
|
||||||
|
void mock.module("@opentui/core", () => ({ ...core, createCliRenderer: async () => setup.renderer }))
|
||||||
|
const { createVoiceTUI } = await import("../src/ui")
|
||||||
|
const ui = await createVoiceTUI({
|
||||||
|
onInterrupt: () => {},
|
||||||
|
onExit: () => {},
|
||||||
|
onCycleVoice: () => {},
|
||||||
|
onToggleMicrophone: () => {},
|
||||||
|
onToggleSpeaker: () => {},
|
||||||
|
})
|
||||||
|
|
||||||
|
try {
|
||||||
|
ui.assistantDelta("Animated text remains visible.")
|
||||||
|
await Bun.sleep(500)
|
||||||
|
await setup.flush()
|
||||||
|
expect(setup.captureCharFrame()).toContain("Animated text remains visible.")
|
||||||
|
} finally {
|
||||||
|
ui.close()
|
||||||
|
mock.restore()
|
||||||
|
}
|
||||||
|
})
|
||||||
|
|
||||||
|
test("renders consecutive tools as compact single-spaced rows", async () => {
|
||||||
|
const setup = await createTestRenderer({ width: 52, height: 8, useThread: false })
|
||||||
|
const core = await import("@opentui/core")
|
||||||
|
void mock.module("@opentui/core", () => ({ ...core, createCliRenderer: async () => setup.renderer }))
|
||||||
|
const { createVoiceTUI } = await import("../src/ui")
|
||||||
|
const ui = await createVoiceTUI({
|
||||||
|
onInterrupt: () => {},
|
||||||
|
onExit: () => {},
|
||||||
|
onCycleVoice: () => {},
|
||||||
|
onToggleMicrophone: () => {},
|
||||||
|
onToggleSpeaker: () => {},
|
||||||
|
reducedMotion: true,
|
||||||
|
})
|
||||||
|
|
||||||
|
try {
|
||||||
|
ui.toolStart("tool-1", "read_session", {})
|
||||||
|
ui.toolDone("tool-1", { status: "ok" })
|
||||||
|
ui.toolStart("tool-2", "read_session", {})
|
||||||
|
ui.toolDone("tool-2", { status: "ok" })
|
||||||
|
await Bun.sleep(200)
|
||||||
|
await setup.flush()
|
||||||
|
|
||||||
|
const rows = frameRows(setup.captureCharFrame())
|
||||||
|
const tools = rows.flatMap((row, index) => (row.includes("read_session") ? [{ row, index }] : []))
|
||||||
|
expect(tools).toHaveLength(2)
|
||||||
|
expect(tools[1]?.index).toBe((tools[0]?.index ?? 0) + 1)
|
||||||
|
expect(tools.every((tool) => tool.row.includes("✓ read_session ok"))).toBe(true)
|
||||||
|
expect(rows.at(-3)?.trim()).toBe("")
|
||||||
|
} finally {
|
||||||
|
ui.close()
|
||||||
|
mock.restore()
|
||||||
|
}
|
||||||
|
})
|
||||||
|
|
||||||
|
test("shows the reason a tool failed", async () => {
|
||||||
|
const setup = await createTestRenderer({ width: 72, height: 8, useThread: false })
|
||||||
|
const core = await import("@opentui/core")
|
||||||
|
void mock.module("@opentui/core", () => ({ ...core, createCliRenderer: async () => setup.renderer }))
|
||||||
|
const { createVoiceTUI } = await import("../src/ui")
|
||||||
|
const ui = await createVoiceTUI({
|
||||||
|
onInterrupt: () => {},
|
||||||
|
onExit: () => {},
|
||||||
|
onCycleVoice: () => {},
|
||||||
|
onToggleMicrophone: () => {},
|
||||||
|
onToggleSpeaker: () => {},
|
||||||
|
reducedMotion: true,
|
||||||
|
})
|
||||||
|
|
||||||
|
try {
|
||||||
|
ui.toolStart("tool-error", "opencode", {})
|
||||||
|
ui.toolDone("tool-error", { status: "error", message: "Invalid tool schema" })
|
||||||
|
await Bun.sleep(200)
|
||||||
|
await setup.flush()
|
||||||
|
expect(setup.captureCharFrame()).toContain("opencode error · Invalid tool schema")
|
||||||
|
} finally {
|
||||||
|
ui.close()
|
||||||
|
mock.restore()
|
||||||
|
}
|
||||||
|
})
|
||||||
|
|
||||||
|
test("paints the full viewport and keeps mic activity in the footer", async () => {
|
||||||
|
const setup = await createTestRenderer({ width: 40, height: 12, useThread: false })
|
||||||
|
const core = await import("@opentui/core")
|
||||||
|
void mock.module("@opentui/core", () => ({ ...core, createCliRenderer: async () => setup.renderer }))
|
||||||
|
const { createVoiceTUI } = await import("../src/ui")
|
||||||
|
const ui = await createVoiceTUI({
|
||||||
|
onInterrupt: () => {},
|
||||||
|
onExit: () => {},
|
||||||
|
onCycleVoice: () => {},
|
||||||
|
onToggleMicrophone: () => {},
|
||||||
|
onToggleSpeaker: () => {},
|
||||||
|
reducedMotion: true,
|
||||||
|
})
|
||||||
|
|
||||||
|
try {
|
||||||
|
ui.userSpeaking(true)
|
||||||
|
await setup.flush()
|
||||||
|
|
||||||
|
const rows = frameRows(setup.captureCharFrame())
|
||||||
|
expect(rows.some((row) => row.includes("│ transcribing"))).toBe(false)
|
||||||
|
expect(rows.at(-2)).toStartWith(" ▂▂▂▂")
|
||||||
|
expect(setup.captureSpans().lines.every((line) => line.spans.every((span) => span.bg.a === 1))).toBe(true)
|
||||||
|
|
||||||
|
setup.resize(56, 16)
|
||||||
|
await setup.flush()
|
||||||
|
expect(setup.captureSpans().lines.every((line) => line.spans.every((span) => span.bg.a === 1))).toBe(true)
|
||||||
|
} finally {
|
||||||
|
ui.close()
|
||||||
|
mock.restore()
|
||||||
|
}
|
||||||
|
})
|
||||||
|
|
||||||
|
function frameRows(frame: string) {
|
||||||
|
return frame.split("\n").slice(0, -1)
|
||||||
|
}
|
||||||
@@ -0,0 +1,74 @@
|
|||||||
|
import { describe, expect, test } from "bun:test"
|
||||||
|
import { initialVoiceView, transitionVoiceView, type VoiceViewEvent, type VoiceViewState } from "../src/ui-model"
|
||||||
|
|
||||||
|
const apply = (state: VoiceViewState, ...events: ReadonlyArray<VoiceViewEvent>) =>
|
||||||
|
events.reduce(transitionVoiceView, state)
|
||||||
|
|
||||||
|
describe("VoiceView", () => {
|
||||||
|
test("keeps assistant responses separated by a committed user turn", () => {
|
||||||
|
const state = apply(
|
||||||
|
initialVoiceView(),
|
||||||
|
{ type: "assistant.delta", text: "First response.", now: 0, animate: false },
|
||||||
|
{ type: "user.committed", itemID: "user-1" },
|
||||||
|
{ type: "user.transcript", itemID: "user-1", text: "Next request", final: true, now: 1, animate: false },
|
||||||
|
{ type: "assistant.delta", text: "Second response.", now: 2, animate: false },
|
||||||
|
)
|
||||||
|
expect(state.messages.map((message) => message.kind)).toEqual(["assistant", "user", "assistant"])
|
||||||
|
expect(state.messages.flatMap((message) => ("text" in message ? [message.text] : []))).toEqual([
|
||||||
|
"First response.",
|
||||||
|
"Next request",
|
||||||
|
"Second response.",
|
||||||
|
])
|
||||||
|
})
|
||||||
|
|
||||||
|
test("keeps local sound activity out of transcript history", () => {
|
||||||
|
const started = transitionVoiceView(initialVoiceView(), { type: "user.started" })
|
||||||
|
expect(started.messages).toEqual([])
|
||||||
|
const committed = transitionVoiceView(started, { type: "user.committed", itemID: "user-1" })
|
||||||
|
expect(committed.messages).toEqual([
|
||||||
|
{ key: "message-1", kind: "user", itemID: "user-1", transcribing: true, reveals: [] },
|
||||||
|
])
|
||||||
|
|
||||||
|
const abandoned = apply(initialVoiceView(), { type: "user.started" }, { type: "user.reset" })
|
||||||
|
expect(abandoned.messages).toEqual([])
|
||||||
|
})
|
||||||
|
|
||||||
|
test("deduplicates transcript snapshots and clears completed reveals", () => {
|
||||||
|
const state = apply(
|
||||||
|
initialVoiceView(),
|
||||||
|
{ type: "user.committed", itemID: "user-1" },
|
||||||
|
{ type: "user.transcript", itemID: "user-1", text: "Hello", final: false, now: 0, animate: true },
|
||||||
|
{ type: "user.transcript", itemID: "user-1", text: "Hello", final: false, now: 10, animate: true },
|
||||||
|
)
|
||||||
|
expect(state.messages).toHaveLength(1)
|
||||||
|
expect(state.messages[0]?.kind === "user" ? state.messages[0].reveals.length : 0).toBeGreaterThan(0)
|
||||||
|
expect(transitionVoiceView(state, { type: "reveals.completed" }).messages).toEqual([
|
||||||
|
{ key: "message-1", kind: "user", itemID: "user-1", text: "Hello", transcribing: true, reveals: [] },
|
||||||
|
])
|
||||||
|
})
|
||||||
|
|
||||||
|
test("bounds message history", () => {
|
||||||
|
const state = Array.from({ length: 205 }, (_, index) => index).reduce(
|
||||||
|
(current, index) => transitionVoiceView(current, { type: "meta", text: String(index) }),
|
||||||
|
initialVoiceView(),
|
||||||
|
)
|
||||||
|
expect(state.messages).toHaveLength(200)
|
||||||
|
expect(state.messages[0]).toEqual({ key: "message-6", kind: "meta", text: "5" })
|
||||||
|
})
|
||||||
|
|
||||||
|
test("settles tool completion before or after insertion", () => {
|
||||||
|
const inserted = apply(
|
||||||
|
initialVoiceView(),
|
||||||
|
{ type: "tool.started", callID: "call-1", name: "read_session", input: {} },
|
||||||
|
{ type: "tool.done", callID: "call-1", output: { status: "ok" } },
|
||||||
|
)
|
||||||
|
expect(inserted.messages[0]).toEqual({
|
||||||
|
key: "message-1",
|
||||||
|
kind: "tool",
|
||||||
|
callID: "call-1",
|
||||||
|
name: "read_session",
|
||||||
|
input: {},
|
||||||
|
output: { status: "ok" },
|
||||||
|
})
|
||||||
|
})
|
||||||
|
})
|
||||||
@@ -0,0 +1,277 @@
|
|||||||
|
import { describe, expect, test } from "bun:test"
|
||||||
|
import { initialVoiceState, transitionVoice, type VoiceEvent, type VoiceState } from "../src/voice-coordinator"
|
||||||
|
|
||||||
|
const apply = (state: VoiceState, ...events: ReadonlyArray<VoiceEvent>) =>
|
||||||
|
events.reduce(
|
||||||
|
(result, event) => {
|
||||||
|
const next = transitionVoice(result.state, event)
|
||||||
|
return { state: next.state, commands: [...result.commands, ...next.commands] }
|
||||||
|
},
|
||||||
|
{ state, commands: [] as Array<ReturnType<typeof transitionVoice>["commands"][number]> },
|
||||||
|
)
|
||||||
|
|
||||||
|
describe("VoiceCoordinator", () => {
|
||||||
|
test("delivers one notification at a time when the server becomes idle", () => {
|
||||||
|
const notification = { id: "notification-1", promptID: "prompt-1", text: "completed" }
|
||||||
|
const waiting = apply(
|
||||||
|
initialVoiceState("marin"),
|
||||||
|
{ type: "connection.ready" },
|
||||||
|
{ type: "user.committed" },
|
||||||
|
{ type: "notification.queued", notification },
|
||||||
|
)
|
||||||
|
expect(waiting.commands).toEqual([])
|
||||||
|
|
||||||
|
expect(transitionVoice(waiting.state, { type: "assistant.done", awaitingWork: false })).toEqual({
|
||||||
|
state: {
|
||||||
|
...waiting.state,
|
||||||
|
conversation: "waiting",
|
||||||
|
assistant: "idle",
|
||||||
|
notifications: [],
|
||||||
|
delivery: { notification, phase: "sending" },
|
||||||
|
},
|
||||||
|
commands: [
|
||||||
|
{ type: "assistant.finish", awaitingWork: false, notificationID: undefined },
|
||||||
|
{ type: "notification.send", notification },
|
||||||
|
],
|
||||||
|
})
|
||||||
|
})
|
||||||
|
|
||||||
|
test("keeps notifications queued while the user is speaking", () => {
|
||||||
|
const notification = { id: "notification-1", text: "completed" }
|
||||||
|
const result = apply(
|
||||||
|
initialVoiceState("marin"),
|
||||||
|
{ type: "connection.ready" },
|
||||||
|
{ type: "user.started" },
|
||||||
|
{ type: "notification.queued", notification },
|
||||||
|
)
|
||||||
|
expect(result.commands).toEqual([])
|
||||||
|
expect(result.state.notifications).toEqual([notification])
|
||||||
|
expect(transitionVoice(result.state, { type: "user.stopped" }).commands).toEqual([
|
||||||
|
{ type: "notification.send", notification },
|
||||||
|
])
|
||||||
|
})
|
||||||
|
|
||||||
|
test("keeps notifications queued until active tool calls finish", () => {
|
||||||
|
const notification = { id: "notification-1", text: "completed" }
|
||||||
|
const result = apply(
|
||||||
|
initialVoiceState("marin"),
|
||||||
|
{ type: "connection.ready" },
|
||||||
|
{ type: "tool.started", id: "call-1" },
|
||||||
|
{ type: "assistant.done", awaitingWork: false },
|
||||||
|
{ type: "notification.queued", notification },
|
||||||
|
)
|
||||||
|
expect(result.commands).toEqual([
|
||||||
|
{ type: "assistant.finish", awaitingWork: false, notificationID: undefined },
|
||||||
|
])
|
||||||
|
expect(transitionVoice(result.state, { type: "tool.finished", id: "call-1" }).commands).toEqual([
|
||||||
|
{ type: "notification.send", notification },
|
||||||
|
])
|
||||||
|
})
|
||||||
|
|
||||||
|
test("tracks tools by call ID and reconnects only after work and conversation settle", () => {
|
||||||
|
const busy = apply(
|
||||||
|
initialVoiceState("marin"),
|
||||||
|
{ type: "connection.ready" },
|
||||||
|
{ type: "tool.started", id: "call-1" },
|
||||||
|
{ type: "voice.selected", voice: "cedar" },
|
||||||
|
)
|
||||||
|
expect(busy.commands).toEqual([])
|
||||||
|
expect(transitionVoice(busy.state, { type: "tool.finished", id: "call-1" }).commands).toEqual([])
|
||||||
|
|
||||||
|
const idle = transitionVoice(transitionVoice(busy.state, { type: "tool.finished", id: "call-1" }).state, {
|
||||||
|
type: "assistant.done",
|
||||||
|
awaitingWork: false,
|
||||||
|
})
|
||||||
|
expect(idle.commands).toEqual([
|
||||||
|
{ type: "assistant.finish", awaitingWork: false, notificationID: undefined },
|
||||||
|
{ type: "connection.reconnect" },
|
||||||
|
])
|
||||||
|
expect(idle.state.connection).toBe("connecting")
|
||||||
|
})
|
||||||
|
|
||||||
|
test("a user commit closes assistant suppression and establishes a waiting boundary", () => {
|
||||||
|
const suppressed = apply(
|
||||||
|
initialVoiceState("marin"),
|
||||||
|
{ type: "connection.ready" },
|
||||||
|
{ type: "assistant.started" },
|
||||||
|
{ type: "assistant.suppressed" },
|
||||||
|
{ type: "user.committed" },
|
||||||
|
)
|
||||||
|
expect(suppressed.state.assistant).toBe("idle")
|
||||||
|
expect(suppressed.state.conversation).toBe("waiting")
|
||||||
|
})
|
||||||
|
|
||||||
|
test("requeues a notification when the adapter rejects it", () => {
|
||||||
|
const notification = { id: "notification-1", text: "completed" }
|
||||||
|
const sent = apply(
|
||||||
|
initialVoiceState("marin"),
|
||||||
|
{ type: "connection.ready" },
|
||||||
|
{ type: "notification.queued", notification },
|
||||||
|
)
|
||||||
|
const failed = transitionVoice(sent.state, { type: "notification.failed", id: notification.id })
|
||||||
|
expect(failed.state.notifications).toEqual([notification])
|
||||||
|
expect(failed.state.connection).toBe("connecting")
|
||||||
|
expect(failed.commands).toEqual([{ type: "connection.reconnect" }])
|
||||||
|
})
|
||||||
|
|
||||||
|
test("keeps a notification in flight until its playback completes", () => {
|
||||||
|
const notification = { id: "notification-1", promptID: "prompt-1", text: "completed" }
|
||||||
|
const sent = apply(
|
||||||
|
initialVoiceState("marin"),
|
||||||
|
{ type: "connection.ready" },
|
||||||
|
{
|
||||||
|
type: "notification.queued",
|
||||||
|
notification,
|
||||||
|
},
|
||||||
|
)
|
||||||
|
expect(sent.state.delivery).toEqual({ notification, phase: "sending" })
|
||||||
|
|
||||||
|
const accepted = transitionVoice(sent.state, { type: "notification.accepted", id: notification.id }).state
|
||||||
|
expect(accepted.delivery).toEqual({ notification, phase: "accepted" })
|
||||||
|
const speaking = transitionVoice(accepted, { type: "assistant.started" }).state
|
||||||
|
expect(speaking.delivery).toEqual({ notification, phase: "announcing" })
|
||||||
|
const generated = transitionVoice(speaking, { type: "assistant.done", awaitingWork: false })
|
||||||
|
expect(generated.state.delivery).toEqual({ notification, phase: "awaiting-playback" })
|
||||||
|
expect(generated.commands).toEqual([
|
||||||
|
{ type: "assistant.finish", awaitingWork: false, notificationID: notification.id },
|
||||||
|
])
|
||||||
|
expect(transitionVoice(generated.state, { type: "notification.announced", id: notification.id })).toEqual({
|
||||||
|
state: { ...generated.state, delivery: undefined },
|
||||||
|
commands: [{ type: "notification.delivered", notification }],
|
||||||
|
})
|
||||||
|
})
|
||||||
|
|
||||||
|
test("does not attribute an already-active assistant turn to a newly appended notification", () => {
|
||||||
|
const notification = { id: "notification-1", text: "completed" }
|
||||||
|
const active = apply(
|
||||||
|
initialVoiceState("marin"),
|
||||||
|
{ type: "connection.ready" },
|
||||||
|
{ type: "notification.queued", notification },
|
||||||
|
{ type: "assistant.started" },
|
||||||
|
{ type: "notification.accepted", id: notification.id },
|
||||||
|
).state
|
||||||
|
|
||||||
|
expect(active.delivery).toEqual({ notification, phase: "accepted" })
|
||||||
|
const existingDone = transitionVoice(active, { type: "assistant.done", awaitingWork: false })
|
||||||
|
expect(existingDone.state.delivery).toEqual({ notification, phase: "accepted" })
|
||||||
|
expect(existingDone.commands).toEqual([
|
||||||
|
{ type: "assistant.finish", awaitingWork: false, notificationID: undefined },
|
||||||
|
])
|
||||||
|
expect(transitionVoice(existingDone.state, { type: "assistant.started" }).state.delivery).toEqual({
|
||||||
|
notification,
|
||||||
|
phase: "announcing",
|
||||||
|
})
|
||||||
|
})
|
||||||
|
|
||||||
|
test("keeps assistant activity independent from a full-duplex user commit", () => {
|
||||||
|
const active = apply(initialVoiceState("marin"), { type: "connection.ready" }, { type: "assistant.started" }).state
|
||||||
|
const committed = transitionVoice(active, { type: "user.committed" }).state
|
||||||
|
expect(committed.assistant).toBe("active")
|
||||||
|
expect(committed.conversation).toBe("responding")
|
||||||
|
})
|
||||||
|
|
||||||
|
test("ignores stale notification callbacks", () => {
|
||||||
|
const notification = { id: "notification-2", text: "current" }
|
||||||
|
const sent = apply(
|
||||||
|
initialVoiceState("marin"),
|
||||||
|
{ type: "connection.ready" },
|
||||||
|
{ type: "notification.queued", notification },
|
||||||
|
).state
|
||||||
|
|
||||||
|
expect(transitionVoice(sent, { type: "notification.accepted", id: "notification-1" })).toEqual({
|
||||||
|
state: sent,
|
||||||
|
commands: [],
|
||||||
|
})
|
||||||
|
expect(transitionVoice(sent, { type: "notification.failed", id: "notification-1" })).toEqual({
|
||||||
|
state: sent,
|
||||||
|
commands: [],
|
||||||
|
})
|
||||||
|
expect(transitionVoice(sent, { type: "notification.announced", id: "notification-1" })).toEqual({
|
||||||
|
state: sent,
|
||||||
|
commands: [],
|
||||||
|
})
|
||||||
|
})
|
||||||
|
|
||||||
|
test("waits for one announcement playback before sending the next", () => {
|
||||||
|
const first = { id: "notification-1", text: "first" }
|
||||||
|
const second = { id: "notification-2", text: "second" }
|
||||||
|
const sent = apply(
|
||||||
|
initialVoiceState("marin"),
|
||||||
|
{ type: "connection.ready" },
|
||||||
|
{ type: "notification.queued", notification: first },
|
||||||
|
{ type: "notification.queued", notification: second },
|
||||||
|
{ type: "notification.accepted", id: first.id },
|
||||||
|
{ type: "assistant.started" },
|
||||||
|
{ type: "assistant.done", awaitingWork: false },
|
||||||
|
)
|
||||||
|
expect(sent.state.delivery).toEqual({ notification: first, phase: "awaiting-playback" })
|
||||||
|
expect(sent.state.notifications).toEqual([second])
|
||||||
|
|
||||||
|
expect(transitionVoice(sent.state, { type: "notification.announced", id: first.id })).toEqual({
|
||||||
|
state: {
|
||||||
|
...sent.state,
|
||||||
|
conversation: "waiting",
|
||||||
|
delivery: { notification: second, phase: "sending" },
|
||||||
|
notifications: [],
|
||||||
|
},
|
||||||
|
commands: [
|
||||||
|
{ type: "notification.delivered", notification: first },
|
||||||
|
{ type: "notification.send", notification: second },
|
||||||
|
],
|
||||||
|
})
|
||||||
|
})
|
||||||
|
|
||||||
|
test("keeps an announcement active across tool work", () => {
|
||||||
|
const notification = { id: "notification-1", text: "completed" }
|
||||||
|
const announcing = apply(
|
||||||
|
initialVoiceState("marin"),
|
||||||
|
{ type: "connection.ready" },
|
||||||
|
{ type: "notification.queued", notification },
|
||||||
|
{ type: "notification.accepted", id: notification.id },
|
||||||
|
{ type: "assistant.started" },
|
||||||
|
).state
|
||||||
|
expect(transitionVoice(announcing, { type: "assistant.done", awaitingWork: true }).state.delivery).toEqual({
|
||||||
|
notification,
|
||||||
|
phase: "announcing",
|
||||||
|
})
|
||||||
|
})
|
||||||
|
|
||||||
|
test("requeues an interrupted announcement and reconnects before retrying", () => {
|
||||||
|
const notification = { id: "notification-1", text: "completed" }
|
||||||
|
const awaitingPlayback = apply(
|
||||||
|
initialVoiceState("marin"),
|
||||||
|
{ type: "connection.ready" },
|
||||||
|
{ type: "notification.queued", notification },
|
||||||
|
{ type: "notification.accepted", id: notification.id },
|
||||||
|
{ type: "assistant.started" },
|
||||||
|
{ type: "assistant.done", awaitingWork: false },
|
||||||
|
).state
|
||||||
|
const interrupted = transitionVoice(awaitingPlayback, {
|
||||||
|
type: "notification.interrupted",
|
||||||
|
id: notification.id,
|
||||||
|
})
|
||||||
|
expect(interrupted.state.connection).toBe("connecting")
|
||||||
|
expect(interrupted.state.notifications).toEqual([notification])
|
||||||
|
expect(interrupted.commands).toEqual([{ type: "connection.reconnect" }])
|
||||||
|
})
|
||||||
|
|
||||||
|
test("holds an interrupted announcement reconnect until barge-in speech stops", () => {
|
||||||
|
const notification = { id: "notification-1", text: "completed" }
|
||||||
|
const announcing = apply(
|
||||||
|
initialVoiceState("marin"),
|
||||||
|
{ type: "connection.ready" },
|
||||||
|
{ type: "notification.queued", notification },
|
||||||
|
{ type: "notification.accepted", id: notification.id },
|
||||||
|
{ type: "assistant.started" },
|
||||||
|
).state
|
||||||
|
const barged = transitionVoice(announcing, { type: "user.started", bargeIn: true })
|
||||||
|
expect(barged.state.notifications).toEqual([notification])
|
||||||
|
expect(barged.state.userSpeaking).toBe(true)
|
||||||
|
expect(barged.state.connection).toBe("ready")
|
||||||
|
expect(barged.commands).toEqual([{ type: "assistant.interrupt" }])
|
||||||
|
|
||||||
|
expect(transitionVoice(barged.state, { type: "user.stopped" }).commands).toEqual([
|
||||||
|
{ type: "connection.reconnect" },
|
||||||
|
])
|
||||||
|
})
|
||||||
|
})
|
||||||
@@ -0,0 +1,233 @@
|
|||||||
|
import { expect, mock, test } from "bun:test"
|
||||||
|
import type { AudioSession } from "../src/audio-session"
|
||||||
|
import type { VoiceConnection, VoiceProtocol, VoiceProtocolEvent, VoiceProtocolOptions } from "../src/protocol"
|
||||||
|
import type { VoiceUI } from "../src/ui"
|
||||||
|
import { createVoiceSession } from "../src/voice-session"
|
||||||
|
|
||||||
|
test("ignores events from a replaced connection", async () => {
|
||||||
|
const setup = makeSession()
|
||||||
|
setup.session.start()
|
||||||
|
setup.connections[0].emit({ type: "ready" })
|
||||||
|
setup.session.cycleVoice()
|
||||||
|
await Bun.sleep(0)
|
||||||
|
|
||||||
|
expect(setup.connections).toHaveLength(2)
|
||||||
|
setup.connections[0].emit({ type: "assistant.transcript.delta", delta: "stale" })
|
||||||
|
setup.connections[1].emit({ type: "assistant.transcript.delta", delta: "current" })
|
||||||
|
expect(setup.assistantText).toEqual(["current"])
|
||||||
|
})
|
||||||
|
|
||||||
|
test("resolves work against the connection that requested it", async () => {
|
||||||
|
const setup = makeSession()
|
||||||
|
setup.session.start()
|
||||||
|
setup.connections[0].emit({ type: "ready" })
|
||||||
|
setup.connections[0].emit({ type: "tool.started", id: "call-1", name: "find_sessions" })
|
||||||
|
setup.connections[0].emit({
|
||||||
|
type: "work.requested",
|
||||||
|
request: { id: "call-1", name: "find_sessions", input: { query: null } },
|
||||||
|
})
|
||||||
|
await Bun.sleep(0)
|
||||||
|
|
||||||
|
expect(setup.resolvedWork).toEqual([
|
||||||
|
{
|
||||||
|
request: { id: "call-1", name: "find_sessions", input: { query: null } },
|
||||||
|
output: { status: "ok" },
|
||||||
|
},
|
||||||
|
])
|
||||||
|
})
|
||||||
|
|
||||||
|
test("acknowledges a durable notification only after playback", async () => {
|
||||||
|
const setup = makeSession()
|
||||||
|
setup.session.start()
|
||||||
|
setup.connections[0].emit({ type: "ready" })
|
||||||
|
setup.session.queueNotification({
|
||||||
|
notification: {
|
||||||
|
type: "opencode.prompt.completed",
|
||||||
|
session_id: "session-1",
|
||||||
|
prompt_id: "prompt-1",
|
||||||
|
status: "completed",
|
||||||
|
text: "done",
|
||||||
|
},
|
||||||
|
receipt: { sessionID: "session-1", promptID: "prompt-1" },
|
||||||
|
})
|
||||||
|
await Bun.sleep(0)
|
||||||
|
|
||||||
|
setup.connections[0].emit({ type: "assistant.transcript.delta", delta: "Finished." })
|
||||||
|
setup.connections[0].emit({ type: "assistant.done", awaitingWork: false })
|
||||||
|
expect(setup.delivered).toEqual([])
|
||||||
|
|
||||||
|
setup.finishPlayback?.("played")
|
||||||
|
await Bun.sleep(0)
|
||||||
|
expect(setup.delivered).toEqual(["prompt-1"])
|
||||||
|
})
|
||||||
|
|
||||||
|
test("retries a notification whose playback was interrupted", async () => {
|
||||||
|
const setup = makeSession()
|
||||||
|
setup.session.start()
|
||||||
|
setup.connections[0].emit({ type: "ready" })
|
||||||
|
setup.session.queueNotification({
|
||||||
|
notification: {
|
||||||
|
type: "opencode.prompt.completed",
|
||||||
|
session_id: "session-1",
|
||||||
|
prompt_id: "prompt-1",
|
||||||
|
status: "completed",
|
||||||
|
text: "done",
|
||||||
|
},
|
||||||
|
receipt: { sessionID: "session-1", promptID: "prompt-1" },
|
||||||
|
})
|
||||||
|
await Bun.sleep(0)
|
||||||
|
setup.connections[0].emit({ type: "assistant.transcript.delta", delta: "Finished." })
|
||||||
|
setup.connections[0].emit({ type: "assistant.done", awaitingWork: false })
|
||||||
|
setup.finishPlayback?.("interrupted")
|
||||||
|
await Bun.sleep(0)
|
||||||
|
|
||||||
|
expect(setup.delivered).toEqual([])
|
||||||
|
expect(setup.connections).toHaveLength(2)
|
||||||
|
setup.connections[1].emit({ type: "ready" })
|
||||||
|
await Bun.sleep(0)
|
||||||
|
expect(setup.connections[1].notifyCalls()).toBe(1)
|
||||||
|
})
|
||||||
|
|
||||||
|
test("closes owned resources once", async () => {
|
||||||
|
const setup = makeSession()
|
||||||
|
setup.session.start()
|
||||||
|
await Promise.all([setup.session.close(), setup.session.close()])
|
||||||
|
expect(setup.audioClose).toHaveBeenCalledTimes(1)
|
||||||
|
expect(setup.toolClose).toHaveBeenCalledTimes(1)
|
||||||
|
expect(setup.connections[0].closeCalls()).toBe(1)
|
||||||
|
})
|
||||||
|
|
||||||
|
test("does not acknowledge nonterminal prompt notifications", async () => {
|
||||||
|
const setup = makeSession()
|
||||||
|
setup.session.start()
|
||||||
|
setup.connections[0].emit({ type: "ready" })
|
||||||
|
setup.session.queueNotification({
|
||||||
|
notification: {
|
||||||
|
type: "opencode.prompt.blocked",
|
||||||
|
prompt_id: "prompt-1",
|
||||||
|
blocker: "permission",
|
||||||
|
session_id: "session-1",
|
||||||
|
request_id: "permission-1",
|
||||||
|
action: "read",
|
||||||
|
resources: ["file.ts"],
|
||||||
|
},
|
||||||
|
})
|
||||||
|
await Bun.sleep(0)
|
||||||
|
setup.connections[0].emit({ type: "assistant.transcript.delta", delta: "Permission needed." })
|
||||||
|
setup.connections[0].emit({ type: "assistant.done", awaitingWork: false })
|
||||||
|
setup.finishPlayback?.("played")
|
||||||
|
await Bun.sleep(0)
|
||||||
|
|
||||||
|
expect(setup.delivered).toEqual([])
|
||||||
|
})
|
||||||
|
|
||||||
|
function makeSession() {
|
||||||
|
const connections: Array<
|
||||||
|
VoiceConnection & { emit(event: VoiceProtocolEvent): void; notifyCalls(): number; closeCalls(): number }
|
||||||
|
> = []
|
||||||
|
const assistantText: string[] = []
|
||||||
|
const resolvedWork: Array<unknown> = []
|
||||||
|
const delivered: string[] = []
|
||||||
|
const audioClose = mock(() => {})
|
||||||
|
const toolClose = mock(async () => {})
|
||||||
|
let finishPlayback: ((outcome: "played" | "interrupted" | "inaudible") => void) | undefined
|
||||||
|
const ui = {
|
||||||
|
meta: mock(() => {}),
|
||||||
|
userSpeaking: mock(() => {}),
|
||||||
|
userReset: mock(() => {}),
|
||||||
|
userAudioLevel: mock(() => {}),
|
||||||
|
userCommitted: mock(() => {}),
|
||||||
|
userTranscript: mock(() => {}),
|
||||||
|
assistantAudio: mock(() => {}),
|
||||||
|
assistantPlaybackStopped: mock(() => {}),
|
||||||
|
assistantDelta: (text: string) => assistantText.push(text),
|
||||||
|
assistantTranscript: mock(() => {}),
|
||||||
|
assistantDone: mock(() => {}),
|
||||||
|
toolStart: mock(() => {}),
|
||||||
|
toolDone: mock(() => {}),
|
||||||
|
setStatus: mock(() => {}),
|
||||||
|
close: mock(() => {}),
|
||||||
|
} satisfies VoiceUI
|
||||||
|
const audio = {
|
||||||
|
fullDuplex: false,
|
||||||
|
microphoneMuted: false,
|
||||||
|
speakerMuted: false,
|
||||||
|
start: mock(async () => {}),
|
||||||
|
toggleMicrophone: mock(() => false),
|
||||||
|
toggleSpeaker: mock(() => false),
|
||||||
|
isPlaying: () => false,
|
||||||
|
play: mock(() => {}),
|
||||||
|
finishPlayback: () => {
|
||||||
|
const result = Promise.withResolvers<"played" | "interrupted" | "inaudible">()
|
||||||
|
finishPlayback = result.resolve
|
||||||
|
return result.promise
|
||||||
|
},
|
||||||
|
flushPlayback: mock(() => {}),
|
||||||
|
noteUserCommitted: mock(() => {}),
|
||||||
|
noteUserTranscript: mock(() => {}),
|
||||||
|
close: audioClose,
|
||||||
|
} satisfies AudioSession
|
||||||
|
const protocol = {
|
||||||
|
name: "realtime",
|
||||||
|
inputActivity: "server",
|
||||||
|
capabilities: { textInput: true, interruption: true, delegation: false },
|
||||||
|
connect(options: VoiceProtocolOptions) {
|
||||||
|
const notify = mock(async () => true)
|
||||||
|
const close = mock(async () => {})
|
||||||
|
const connection = {
|
||||||
|
appendAudio: mock(() => {}),
|
||||||
|
sendText: mock(() => {}),
|
||||||
|
resolveWork: (request, output) => resolvedWork.push({ request, output }),
|
||||||
|
resolveDelegation: mock(() => {}),
|
||||||
|
notify,
|
||||||
|
interrupt: () => true,
|
||||||
|
close,
|
||||||
|
emit: options.onEvent,
|
||||||
|
notifyCalls: () => notify.mock.calls.length,
|
||||||
|
closeCalls: () => close.mock.calls.length,
|
||||||
|
} satisfies VoiceConnection & {
|
||||||
|
emit(event: VoiceProtocolEvent): void
|
||||||
|
notifyCalls(): number
|
||||||
|
closeCalls(): number
|
||||||
|
}
|
||||||
|
connections.push(connection)
|
||||||
|
return connection
|
||||||
|
},
|
||||||
|
} satisfies VoiceProtocol
|
||||||
|
const session = createVoiceSession({
|
||||||
|
protocol,
|
||||||
|
connection: {
|
||||||
|
apiKey: "test",
|
||||||
|
model: "voice-test",
|
||||||
|
instructions: "test",
|
||||||
|
tools: [],
|
||||||
|
fullDuplex: false,
|
||||||
|
debug: false,
|
||||||
|
},
|
||||||
|
initialVoice: "marin",
|
||||||
|
voices: ["marin", "cedar"],
|
||||||
|
ui,
|
||||||
|
audio,
|
||||||
|
tools: {
|
||||||
|
execute: async () => ({ output: { status: "ok" } }),
|
||||||
|
acknowledge: async (receipt) => {
|
||||||
|
delivered.push(receipt.promptID)
|
||||||
|
},
|
||||||
|
close: toolClose,
|
||||||
|
},
|
||||||
|
delegate: async () => "done",
|
||||||
|
onClosed: () => {},
|
||||||
|
})
|
||||||
|
return {
|
||||||
|
session,
|
||||||
|
connections,
|
||||||
|
assistantText,
|
||||||
|
resolvedWork,
|
||||||
|
delivered,
|
||||||
|
audioClose,
|
||||||
|
toolClose,
|
||||||
|
get finishPlayback() {
|
||||||
|
return finishPlayback
|
||||||
|
},
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,11 @@
|
|||||||
|
{
|
||||||
|
"$schema": "https://json.schemastore.org/tsconfig",
|
||||||
|
"extends": "@tsconfig/bun/tsconfig.json",
|
||||||
|
"compilerOptions": {
|
||||||
|
"jsx": "preserve",
|
||||||
|
"jsxImportSource": "@opentui/solid",
|
||||||
|
"noUncheckedIndexedAccess": false,
|
||||||
|
"noUnusedLocals": true
|
||||||
|
},
|
||||||
|
"include": ["src", "test"]
|
||||||
|
}
|
||||||
Reference in New Issue
Block a user