Compare commits

...

19 Commits

Author SHA1 Message Date
Kit Langton 0574107e18 tui: active tabs now show more of their title before truncating
The selected tab in the titlebar expands from 224px to 288px so users can read more of the session or draft name before it gets cut off. Inactive tabs stay compact at 224px to keep the tab strip dense.
2026-07-30 20:46:52 -04:00
Kit Langton 6f0fa2d6c6 core: remove unused ownerHeld bookkeeping in service restarter
The ownerHeld flag was assigned but never read, making the service restart retry logic harder to follow. Removing it clarifies that the spawn delay only doubles when a previous contender finished cleanly.
2026-07-30 20:46:46 -04:00
Kit Langton 363905dcdd fix(tui): share runtime with external plugins 2026-07-28 21:00:49 -04:00
Kit Langton d71bc0055a fix(core): refresh reused session recency 2026-07-28 21:00:43 -04:00
Kit Langton 4f235fc59b feat(voice): harden controller runtime 2026-07-28 21:00:35 -04:00
Kit Langton 1d772f4896 feat(voice): add audio-reactive TUI animations 2026-07-27 16:08:14 -04:00
Kit Langton 113a117cef feat(voice): curate project session and permission tools 2026-07-27 15:46:48 -04:00
Kit Langton c103ba6056 feat(voice): show readable tool lifecycle diagnostics 2026-07-27 15:11:48 -04:00
Kit Langton 38f03c9d8b fix(voice): default coding sessions to gpt-5.6-sol medium 2026-07-27 15:08:03 -04:00
Kit Langton e5ac4ca560 fix(voice): run reactive opentui and harden shutdown 2026-07-27 14:53:27 -04:00
Kit Langton 5f0487fa87 feat(voice): opentui interface, voice switching, ordered transcripts 2026-07-27 14:23:40 -04:00
Kit Langton 00557f9078 fix(voice): rebuild audio engines on route change, cleaner output 2026-07-27 14:09:28 -04:00
Kit Langton 5ac1089c5c fix(voice): make echo cancellation opt-in, self-healing audio helper 2026-07-27 14:02:49 -04:00
Kit Langton d1843f49ac feat(voice): echo-cancelled full duplex via Apple voice processing 2026-07-27 13:49:23 -04:00
Kit Langton aac66d384e fix(voice): stop server VAD from cancelling half-duplex playback 2026-07-27 13:36:19 -04:00
Kit Langton 83108c1b5d feat(voice): --session flag to pin an existing session 2026-07-27 11:52:34 -04:00
Kit Langton e7000161f7 fix(voice): gate mic against speaker echo 2026-07-26 12:33:09 -04:00
Kit Langton 66200eee34 fix(voice): password flag and friendly connect error 2026-07-26 12:00:17 -04:00
Kit Langton 699477755f feat(voice): realtime voice control spike 2026-07-26 11:09:28 -04:00
51 changed files with 7515 additions and 6 deletions
+21
View File
@@ -986,6 +986,25 @@
"@typescript/native-preview": "catalog:",
},
},
"packages/voice": {
"name": "@opencode-ai/voice",
"version": "0.0.0",
"dependencies": {
"@opencode-ai/client": "workspace:*",
"@opencode-ai/plugin": "workspace:*",
"@opentui/core": "catalog:",
"@opentui/solid": "catalog:",
"effect": "catalog:",
"opentui-spinner": "catalog:",
"solid-js": "catalog:",
},
"devDependencies": {
"@tsconfig/bun": "catalog:",
"@types/bun": "catalog:",
"@typescript/native-preview": "catalog:",
"typescript": "catalog:",
},
},
"packages/web": {
"name": "@opencode-ai/web",
"version": "1.18.4",
@@ -2077,6 +2096,8 @@
"@opencode-ai/util": ["@opencode-ai/util@workspace:packages/util"],
"@opencode-ai/voice": ["@opencode-ai/voice@workspace:packages/voice"],
"@opencode-ai/web": ["@opencode-ai/web@workspace:packages/web"],
"@opencode-ai/www": ["@opencode-ai/www@workspace:packages/www"],
@@ -88,8 +88,12 @@ function SessionTabSlot(props: {
data-titlebar-tab-slot
data-tab-key={props.id}
data-active={props.active()}
class="relative flex w-56 min-w-7 max-w-56 flex-shrink"
classList={{ hidden: !session() && !missingSession() && !persisted()?.title }}
class="relative flex min-w-7 flex-shrink"
classList={{
hidden: !session() && !missingSession() && !persisted()?.title,
"w-72 max-w-72": props.active(),
"w-56 max-w-56": !props.active(),
}}
>
<TabNavItem
ref={(el) => {
@@ -144,7 +148,11 @@ function DraftTabSlot(props: {
data-titlebar-tab-slot
data-tab-key={props.id}
data-active={props.active()}
class="relative flex w-56 min-w-7 max-w-56 flex-shrink"
class="relative flex min-w-7 flex-shrink"
classList={{
"w-72 max-w-72": props.active(),
"w-56 max-w-56": !props.active(),
}}
>
<DraftTabItem
ref={(el) => {
+1
View File
@@ -1,5 +1,6 @@
#!/usr/bin/env bun
import "@opentui/solid/runtime-plugin-support"
import { NodeRuntime, NodeServices } from "@effect/platform-node"
import { Effect } from "effect"
import { Commands } from "./commands/commands"
-3
View File
@@ -37,7 +37,6 @@ export async function ensure(options: EnsureOptions = {}): Promise<Endpoint> {
let announced = false
let lastSpawn = 0
let spawnDelay = 5_000
let ownerHeld = false
const announce = (reason: "missing" | "version-mismatch", previousVersion?: string) => {
if (announced) return
@@ -65,7 +64,6 @@ export async function ensure(options: EnsureOptions = {}): Promise<Endpoint> {
const registration = await registered(options.file, true)
if (registration.service !== undefined) {
ownerHeld = false
spawnDelay = 5_000
const service = registration.service
const compatible = !service.legacy && (options.version === undefined || service.version === options.version)
@@ -82,7 +80,6 @@ export async function ensure(options: EnsureOptions = {}): Promise<Endpoint> {
if (failure !== undefined) throw failure
const finished = [...contenders].filter(contenderFinished)
if (finished.some((item) => item.child.exitCode === 0)) {
ownerHeld = true
spawnDelay = Math.min(spawnDelay * 2, 30_000)
}
finished.forEach((item) => contenders.delete(item))
+6
View File
@@ -654,6 +654,12 @@ const layer = Layer.effectDiscard(
input: event.data.input,
timeCreated: event.created,
})
yield* db
.update(SessionTable)
.set({ time_updated: DateTime.toEpochMillis(event.created) })
.where(eq(SessionTable.id, event.data.sessionID))
.run()
.pipe(Effect.orDie)
}),
)
yield* events.project(SessionEvent.Compaction.Admitted, (event) =>
+22
View File
@@ -171,6 +171,28 @@ describe("SessionV2.create", () => {
}),
)
it.effect("orders reused sessions by prompt admission time", () =>
Effect.gen(function* () {
const session = yield* SessionV2.Service
const events = yield* EventV2.Service
const { db } = yield* Database.Service
const reused = yield* session.create({ location, title: "reused" })
const newer = yield* session.create({ location, title: "newer" })
yield* db.update(SessionTable).set({ time_updated: -2 }).where(eq(SessionTable.id, reused.id)).run()
yield* db.update(SessionTable).set({ time_updated: -1 }).where(eq(SessionTable.id, newer.id)).run()
const admitted = yield* events.publish(SessionEvent.InputAdmitted, {
sessionID: reused.id,
inputID: SessionMessage.ID.create(),
input: { type: "user", data: { text: "continue" }, delivery: "steer" },
})
const page = yield* session.list({ directory: location.directory, parentID: null, order: "desc" })
expect(page.data.map((item) => item.id)).toEqual([reused.id, newer.id])
expect(page.data[0]!.time.updated).toEqual(admitted.created)
}),
)
it.effect("filters direct child sessions by parent ID", () =>
Effect.gen(function* () {
const session = yield* SessionV2.Service
+1
View File
@@ -0,0 +1 @@
.build/
+4
View File
@@ -0,0 +1,4 @@
preload = ["@opentui/solid/preload"]
[test]
preload = ["@opentui/solid/preload"]
+29
View File
@@ -0,0 +1,29 @@
{
"$schema": "https://json.schemastore.org/package.json",
"name": "@opencode-ai/voice",
"version": "0.0.0",
"private": true,
"description": "Prototype voice control for OpenCode via the OpenAI Realtime and Live APIs",
"type": "module",
"scripts": {
"bench:audio": "bun scripts/bench-speaker-queue.ts",
"spike": "bun run --no-orphans --conditions=browser src/spike.ts",
"test": "bun test",
"typecheck": "tsgo --noEmit"
},
"devDependencies": {
"@tsconfig/bun": "catalog:",
"@types/bun": "catalog:",
"@typescript/native-preview": "catalog:",
"typescript": "catalog:"
},
"dependencies": {
"@opencode-ai/client": "workspace:*",
"@opencode-ai/plugin": "workspace:*",
"@opentui/core": "catalog:",
"@opentui/solid": "catalog:",
"effect": "catalog:",
"opentui-spinner": "catalog:",
"solid-js": "catalog:"
}
}
+38
View File
@@ -0,0 +1,38 @@
# Speaker Queue Performance
## Goal
Reduce render-path latency and variance that can produce audible output glitches.
## Benchmark
Run `bun run bench:audio` from `packages/voice`.
The benchmark compiles the production `SpeakerQueue`, performs one warmup run and seven measured runs, and reports callback latency for a 512-frame pop followed by a same-sized producer push. The queue starts with one second of 24 kHz PCM audio.
## Metrics
- Primary: `speaker_queue_p99_ns`
- Secondary: `speaker_queue_median_ns`, `speaker_queue_worst_ns`
## Scope
- `src/duplex-audio.swift`
- `scripts/bench-speaker-queue.ts`
## Experiments
### Baseline: allocating `Data` queue
- Median: 3,750 ns
- p99: 5,833 ns
- Worst: 92,209 ns median across runs; one run reached 7,497,334 ns
- Decision: baseline
### Experiment 1: preallocated circular queue
- Hypothesis: rendering directly from preallocated storage will reduce callback tail latency by avoiding an array allocation and `Data.removeFirst()` on every callback.
- Median: 1,417 ns, down 62.2%
- p99: 1,917 ns, down 67.1%
- Worst: 32,709 ns median across runs, down 64.5%
- Decision: keep
@@ -0,0 +1,43 @@
import { mkdtemp, rm } from "node:fs/promises"
import { tmpdir } from "node:os"
import { join } from "node:path"
const directory = await mkdtemp(join(tmpdir(), "opencode-voice-bench-"))
const binary = join(directory, "speaker-queue")
const source = Bun.fileURLToPath(new URL("../src/duplex-audio.swift", import.meta.url))
const compile = Bun.spawn(["swiftc", "-O", "-D", "QUEUE_BENCHMARK", source, "-o", binary], {
stdout: "ignore",
stderr: "pipe",
})
if ((await compile.exited) !== 0) {
console.error(await new Response(compile.stderr).text())
await rm(directory, { recursive: true })
process.exit(1)
}
const runs: Array<Record<string, number>> = []
for (let index = 0; index < 8; index++) {
const process = Bun.spawn([binary], { stdout: "pipe", stderr: "inherit" })
const output = await new Response(process.stdout).text()
if ((await process.exited) !== 0) throw new Error(`benchmark run ${index + 1} failed`)
if (index === 0) continue
runs.push(
Object.fromEntries(
output
.split("\n")
.filter((line) => line.startsWith("METRIC "))
.map((line) => {
const [name, value] = line.slice(7).split("=")
return [name!, Number(value)]
}),
),
)
}
await rm(directory, { recursive: true })
for (const name of ["speaker_queue_median_ns", "speaker_queue_p99_ns", "speaker_queue_worst_ns"]) {
const values = runs.map((run) => run[name]!).sort((a, b) => a - b)
const median = values[Math.floor(values.length / 2)]!
console.log(`${name}: median=${median} range=${values[0]}..${values.at(-1)}`)
console.log(`METRIC ${name}=${median}`)
}
+49
View File
@@ -0,0 +1,49 @@
export type TextReveal = { readonly offset: number; readonly at: number }
const CONNECTION_TRANSITION_MS = 480
const REVEAL_DURATION_MS = 380
const REVEAL_STAGGER_MS = 32
const REVEAL_MAX_QUEUE_MS = 160
export const REVEAL_WORD_LIMIT = 24
export function textRevealOpacity(now: number, start: number) {
const progress = Math.max(0, Math.min(1, (now - start) / REVEAL_DURATION_MS))
const eased = 1 - (1 - progress) ** 2
return 0.16 + eased * 0.84
}
export function transcriptionPulse(now: number) {
return 0.5 + Math.sin(now / 220) * 0.5
}
export function connectionMeterLevels(now: number, level: number, connectedAt?: number, startedAt = 0) {
const transition = connectedAt === undefined ? 0 : Math.max(0, Math.min(1, (now - connectedAt) / CONNECTION_TRANSITION_MS))
const head = ((now - startedAt) / 180) % 4
return [0, 1, 2, 3].map((index) => {
const distance = Math.abs(index - head)
const wrappedDistance = Math.min(distance, 4 - distance)
const loading = 0.08 + Math.max(0, Math.cos((wrappedDistance / 1.5) * (Math.PI / 2))) * 0.92
const connected = level * (0.72 + Math.sin(now / 240 + index * 1.4) * 0.28)
return loading + (connected - loading) * transition
})
}
export function connectionTransitioning(now: number, connectedAt?: number) {
return connectedAt === undefined || now - connectedAt < CONNECTION_TRANSITION_MS
}
export function scheduleTextReveal(previous: string, delta: string, now: number, revealAt: number) {
const text = previous + delta
const offsets = Array.from(delta.matchAll(/\S+/g))
.map((match) => previous.length + match.index)
.filter((offset) => offset === 0 || /\s/.test(text[offset - 1] ?? ""))
.slice(-REVEAL_WORD_LIMIT)
const start = Math.min(Math.max(revealAt, now), now + REVEAL_MAX_QUEUE_MS)
const reveals = offsets.map((offset, word) => ({ offset, at: start + word * REVEAL_STAGGER_MS }))
const nextRevealAt = reveals.length === 0 ? revealAt : reveals.at(-1)!.at + REVEAL_STAGGER_MS
return {
reveals,
nextRevealAt,
animationEndsAt: reveals.length === 0 ? now : reveals.at(-1)!.at + REVEAL_DURATION_MS,
}
}
+136
View File
@@ -0,0 +1,136 @@
import { mkdir } from "node:fs/promises"
import { PCM_SAMPLE_RATE } from "./pcm"
export type AudioDevice = {
readonly fullDuplex: boolean
readonly mode: string
start(onInput: (audio: Buffer) => void, onMeta: (text: string) => void): Promise<void>
write(audio: Buffer): void
flush(): void
close(): void
}
export async function createAudioDevice(options: {
readonly duplex: boolean
readonly speakers: boolean
readonly debug: boolean
}): Promise<AudioDevice> {
const helper = await prepareHelper()
if (helper) return appleDevice(helper, options)
return soxDevice(options)
}
function appleDevice(
binary: string,
options: { readonly duplex: boolean; readonly speakers: boolean },
): AudioDevice {
let process: ReturnType<typeof Bun.spawn> | undefined
let sink: import("bun").FileSink | undefined
return {
fullDuplex: options.speakers || options.duplex,
mode: options.speakers ? "duplex+aec" : options.duplex ? "duplex" : "half-duplex",
async start(onInput, onMeta) {
if (process) return
const child = Bun.spawn([binary, ...(options.speakers ? ["--aec"] : [])], {
stdin: "pipe",
stdout: "pipe",
stderr: "pipe",
})
process = child
sink = child.stdin
void forwardLines(child.stderr, onMeta)
for await (const chunk of child.stdout) onInput(Buffer.from(chunk))
process = undefined
sink = undefined
},
write(audio) {
void sink?.write(audio)
void sink?.flush()
},
flush() {
process?.kill("SIGUSR1")
},
close() {
process?.kill()
process = undefined
sink = undefined
},
}
}
function soxDevice(options: { readonly duplex: boolean; readonly debug: boolean }): AudioDevice {
const format = [
"-q",
"-t",
"raw",
"-r",
String(PCM_SAMPLE_RATE),
"-e",
"signed-integer",
"-b",
"16",
"-c",
"1",
]
let recorder: ReturnType<typeof Bun.spawn> | undefined
let player: ReturnType<typeof Bun.spawn> | undefined
let sink: import("bun").FileSink | undefined
let onMeta: ((text: string) => void) | undefined
return {
fullDuplex: options.duplex,
mode: options.duplex ? "duplex (sox)" : "half-duplex (sox)",
async start(onInput, report) {
if (recorder) return
onMeta = report
const child = Bun.spawn(["rec", ...format, "-"], { stdout: "pipe", stderr: "ignore" })
recorder = child
for await (const chunk of child.stdout) onInput(Buffer.from(chunk))
recorder = undefined
},
write(audio) {
if (!player) {
const child = Bun.spawn(["play", ...format, "-"], { stdin: "pipe", stderr: "ignore" })
player = child
sink = child.stdin
if (options.debug) void child.exited.then((code) => onMeta?.(`[debug] play exited (${code})`))
}
void sink?.write(audio)
void sink?.flush()
},
flush() {
player?.kill()
player = undefined
sink = undefined
},
close() {
recorder?.kill()
player?.kill()
recorder = undefined
player = undefined
sink = undefined
},
}
}
async function forwardLines(stream: ReadableStream<Uint8Array>, onLine: (line: string) => void) {
const decoder = new TextDecoder()
let remainder = ""
for await (const chunk of stream) {
const lines = (remainder + decoder.decode(chunk, { stream: true })).split("\n")
remainder = lines.pop() ?? ""
lines.filter((line) => line.trim()).forEach((line) => onLine(line.trim()))
}
remainder += decoder.decode()
if (remainder.trim()) onLine(remainder.trim())
}
async function prepareHelper() {
if (process.platform !== "darwin") return undefined
const source = Bun.fileURLToPath(new URL("./duplex-audio.swift", import.meta.url))
const binary = Bun.fileURLToPath(new URL("../.build/duplex-audio", import.meta.url))
if ((await Bun.file(binary).exists()) && Bun.file(binary).lastModified > Bun.file(source).lastModified) return binary
await mkdir(Bun.fileURLToPath(new URL("../.build", import.meta.url)), { recursive: true })
const compile = Bun.spawn(["swiftc", "-O", source, "-o", binary], { stdout: "ignore", stderr: "pipe" })
if ((await compile.exited) === 0) return binary
return undefined
}
+41
View File
@@ -0,0 +1,41 @@
import { PCM_BYTES_PER_MS } from "./pcm"
export type PlaybackChunk = {
readonly bytes: Buffer
readonly gapMs: number
}
export class AudioJitterBuffer {
private readonly pending: PlaybackChunk[] = []
private durationMs = 0
private started = false
constructor(private readonly targetMs = 500) {}
push(chunk: PlaybackChunk): ReadonlyArray<PlaybackChunk> {
if (this.started) return [chunk]
this.pending.push(chunk)
this.durationMs += chunk.gapMs + chunk.bytes.length / PCM_BYTES_PER_MS
if (this.durationMs < this.targetMs) return []
this.started = true
return this.drain()
}
finish(): ReadonlyArray<PlaybackChunk> {
if (this.pending.length === 0) return []
this.started = true
return this.drain()
}
reset() {
this.pending.length = 0
this.durationMs = 0
this.started = false
}
private drain() {
const chunks = this.pending.splice(0)
this.durationMs = 0
return chunks
}
}
+217
View File
@@ -0,0 +1,217 @@
import { createAudioDevice, type AudioDevice } from "./audio-device"
import { AudioJitterBuffer, type PlaybackChunk } from "./audio-jitter-buffer"
import { pcmLevel, PCM_BYTES_PER_MS, PCM_METER_FRAME_MS } from "./pcm"
import type { VoiceAudioTimeline } from "./protocol"
export type AudioEvent =
| { readonly type: "input"; readonly audio: Buffer; readonly level: number }
| { readonly type: "user.started" }
| { readonly type: "user.stopped" }
| { readonly type: "user.reset" }
| { readonly type: "assistant.level"; readonly level: number; readonly durationMs: number }
| { readonly type: "status"; readonly audio: string }
| { readonly type: "meta"; readonly text: string }
export type PlaybackOutcome = "played" | "interrupted" | "inaudible"
export type AudioSession = {
readonly fullDuplex: boolean
readonly microphoneMuted: boolean
readonly speakerMuted: boolean
start(emit: (event: AudioEvent) => void): Promise<void>
toggleMicrophone(): boolean
toggleSpeaker(): boolean
isPlaying(): boolean
play(bytes: Buffer, timeline?: VoiceAudioTimeline): void
finishPlayback(): Promise<PlaybackOutcome>
flushPlayback(): void
noteUserCommitted(): void
noteUserTranscript(final: boolean): void
close(): void
}
export async function createAudioSession(options: {
readonly duplex: boolean
readonly speakers: boolean
readonly inputActivity: "local" | "server"
readonly debug: boolean
readonly device?: AudioDevice
readonly trace?: (event: string, data?: Record<string, unknown>) => void
}): Promise<AudioSession> {
const device = options.device ?? (await createAudioDevice(options))
const fullDuplex = device.fullDuplex
const playbackBuffer = new AudioJitterBuffer()
let emit: ((event: AudioEvent) => void) | undefined
let microphoneMuted = false
let speakerMuted = false
let playbackEndsAt = 0
let playbackAudible = false
let playbackDoneTimer: ReturnType<typeof setTimeout> | undefined
let playbackCompletion: PromiseWithResolvers<PlaybackOutcome> | undefined
let outputTimelineEnd: number | undefined
let userFinalizedAt = 0
let userSpeechTimer: ReturnType<typeof setTimeout> | undefined
let userDraftTimer: ReturnType<typeof setTimeout> | undefined
let microphoneStarted = false
let closed = false
const isPlaying = () => Date.now() < playbackEndsAt
const settlePlayback = (outcome: PlaybackOutcome) => {
if (playbackDoneTimer) clearTimeout(playbackDoneTimer)
playbackDoneTimer = undefined
const completion = playbackCompletion
playbackCompletion = undefined
playbackEndsAt = 0
playbackAudible = false
playbackBuffer.reset()
outputTimelineEnd = undefined
completion?.resolve(outcome)
}
const flushPlayback = () => {
device.flush()
settlePlayback("interrupted")
}
const observeUserAudio = (level: number) => {
if (options.inputActivity !== "local" || level < 0.2 || Date.now() - userFinalizedAt < 500) return
if (userDraftTimer) clearTimeout(userDraftTimer)
userDraftTimer = undefined
emit?.({ type: "user.started" })
outputTimelineEnd = undefined
if (userSpeechTimer) clearTimeout(userSpeechTimer)
userSpeechTimer = setTimeout(() => {
userSpeechTimer = undefined
emit?.({ type: "user.stopped" })
userDraftTimer = setTimeout(() => {
userDraftTimer = undefined
emit?.({ type: "user.reset" })
}, 1_200)
}, 500)
}
const publishInput = (bytes: Buffer) => {
const level = pcmLevel(bytes)
emit?.({ type: "input", audio: bytes, level })
observeUserAudio(level)
}
const start = async (listener: (event: AudioEvent) => void) => {
if (microphoneStarted || closed) return
emit = listener
microphoneStarted = true
emit({ type: "status", audio: device.mode })
emit({ type: "meta", text: "mic live - start talking" })
if (!fullDuplex)
emit({ type: "meta", text: "mic mutes while the assistant speaks; press Esc to interrupt" })
await device.start(
(chunk) => {
if (closed || microphoneMuted || (!fullDuplex && Date.now() < playbackEndsAt + 300)) return
publishInput(chunk)
},
(text) => {
emit?.({ type: "meta", text })
options.trace?.("audio.helper", { message: text })
},
)
}
const writeAudio = (chunk: PlaybackChunk) => {
playbackAudible = true
if (chunk.gapMs > 0) {
emit?.({ type: "assistant.level", level: 0, durationMs: chunk.gapMs })
device.write(Buffer.alloc(Math.round(chunk.gapMs * PCM_BYTES_PER_MS)))
}
const meterBytes = PCM_METER_FRAME_MS * PCM_BYTES_PER_MS
for (let offset = 0; offset < chunk.bytes.length; offset += meterBytes) {
const window = chunk.bytes.subarray(offset, offset + meterBytes)
emit?.({ type: "assistant.level", level: pcmLevel(window), durationMs: window.length / PCM_BYTES_PER_MS })
}
device.write(chunk.bytes)
playbackEndsAt = Math.max(playbackEndsAt, Date.now()) + chunk.gapMs + chunk.bytes.length / PCM_BYTES_PER_MS
}
const schedulePlaybackCompletion = () => {
if (!playbackCompletion) return
if (!playbackAudible) return settlePlayback("inaudible")
if (playbackDoneTimer) clearTimeout(playbackDoneTimer)
playbackDoneTimer = setTimeout(
() => settlePlayback("played"),
Math.max(0, playbackEndsAt - Date.now()) + 180,
)
}
return {
fullDuplex,
get microphoneMuted() {
return microphoneMuted
},
get speakerMuted() {
return speakerMuted
},
start,
toggleMicrophone() {
microphoneMuted = !microphoneMuted
if (userSpeechTimer) clearTimeout(userSpeechTimer)
userSpeechTimer = undefined
emit?.({ type: "user.stopped" })
if (microphoneMuted) emit?.({ type: "user.reset" })
return microphoneMuted
},
toggleSpeaker() {
speakerMuted = !speakerMuted
if (speakerMuted) flushPlayback()
return speakerMuted
},
isPlaying,
play(bytes, timeline) {
if (speakerMuted || closed) return
const timelineGap = timeline && outputTimelineEnd !== undefined ? Math.max(0, timeline.startMs - outputTimelineEnd) : 0
const gapMs = timelineGap < 2_000 ? timelineGap : 0
if (options.debug)
options.trace?.("audio.output", {
bytes: bytes.length,
durationMs: bytes.length / PCM_BYTES_PER_MS,
level: pcmLevel(bytes),
timelineStart: timeline?.startMs,
timelineEnd: timeline?.endMs,
timelineGap,
gapMs,
})
outputTimelineEnd = timeline?.endMs
playbackBuffer.push({ bytes, gapMs }).forEach(writeAudio)
schedulePlaybackCompletion()
},
finishPlayback() {
if (playbackCompletion) return playbackCompletion.promise
playbackCompletion = Promise.withResolvers<PlaybackOutcome>()
playbackBuffer.finish().forEach(writeAudio)
schedulePlaybackCompletion()
return playbackCompletion?.promise ?? Promise.resolve("inaudible")
},
flushPlayback,
noteUserCommitted() {
if (userDraftTimer) clearTimeout(userDraftTimer)
userDraftTimer = undefined
},
noteUserTranscript(final) {
if (userDraftTimer) clearTimeout(userDraftTimer)
userDraftTimer = undefined
if (!final) return
if (userSpeechTimer) clearTimeout(userSpeechTimer)
userSpeechTimer = undefined
userFinalizedAt = Date.now()
emit?.({ type: "user.stopped" })
},
close() {
if (closed) return
closed = true
device.close()
if (userSpeechTimer) clearTimeout(userSpeechTimer)
if (userDraftTimer) clearTimeout(userDraftTimer)
settlePlayback("interrupted")
emit = undefined
},
}
}
+109
View File
@@ -0,0 +1,109 @@
import { mkdir, readdir, rename, rm } from "node:fs/promises"
import { homedir } from "node:os"
import { dirname, join } from "node:path"
import { Option, Schema } from "effect"
import { OpenCodeNotification } from "./opencode-notification"
import { promptKey, type PromptHandle } from "./prompt-handle"
export type CompletionHandle = PromptHandle
export type StoredCompletion =
| { readonly status: "admitting"; readonly handle: CompletionHandle; readonly text: string }
| { readonly status: "pending"; readonly handle: CompletionHandle }
| {
readonly status: "completed"
readonly handle: CompletionHandle
readonly notification: OpenCodeNotification
}
const CompletionHandle = Schema.Struct({ sessionID: Schema.String, promptID: Schema.String })
const StoredCompletion = Schema.Union([
Schema.Struct({ status: Schema.Literal("admitting"), handle: CompletionHandle, text: Schema.String }),
Schema.Struct({ status: Schema.Literal("pending"), handle: CompletionHandle }),
Schema.Struct({ status: Schema.Literal("completed"), handle: CompletionHandle, notification: OpenCodeNotification }),
])
const StoredCompletions = Schema.Array(StoredCompletion)
export async function createCompletionStore(
path = join(process.env["XDG_STATE_HOME"] ?? join(homedir(), ".local", "state"), "opencode", "voice-prompts.json"),
) {
await mkdir(dirname(path), { recursive: true })
const directory = `${path}.d`
await mkdir(directory, { recursive: true })
const files = (await readdir(directory)).filter((name) => name.endsWith(".json"))
const decoded = files.length > 0
? await Promise.all(
files.map(async (name) => {
const entry = Option.getOrUndefined(
Schema.decodeUnknownOption(StoredCompletion)(await Bun.file(join(directory, name)).json()),
)
if (!entry) throw new Error(`Invalid voice completion entry: ${join(directory, name)}`)
return entry
}),
)
: (await Bun.file(path).exists())
? Option.getOrUndefined(Schema.decodeUnknownOption(StoredCompletions)(await Bun.file(path).json()))
: []
if (!decoded) throw new Error(`Invalid voice completion store: ${path}`)
const entries = new Map(decoded.map((entry) => [promptKey(entry.handle), entry]))
let writes = Promise.resolve()
const enqueue = (operation: () => Promise<void>) => {
const result = writes.then(operation)
writes = result.catch(() => {})
return result
}
const save = (entry: StoredCompletion) => {
return enqueue(async () => {
const target = completionPath(directory, entry.handle)
const temporary = `${target}.${process.pid}.${crypto.randomUUID()}.tmp`
await Bun.write(temporary, JSON.stringify(entry))
await rename(temporary, target)
})
}
if (files.length === 0 && decoded.length > 0) {
await Promise.all(decoded.map(save))
await rm(path, { force: true })
}
if (files.length > 0) {
const existing = new Set(files)
await Promise.all(decoded.filter((entry) => !existing.has(completionFilename(entry.handle))).map(save))
const current = new Set(decoded.map((entry) => completionFilename(entry.handle)))
await Promise.all(files.filter((name) => !current.has(name)).map((name) => rm(join(directory, name), { force: true })))
}
return {
entries: () => [...entries.entries()].toSorted(([left], [right]) => left.localeCompare(right)).map(([, entry]) => entry),
admitting(handle: CompletionHandle, text: string) {
const entry = { status: "admitting", handle, text } as const
entries.set(promptKey(handle), entry)
return save(entry)
},
pending(handle: CompletionHandle) {
const entry = { status: "pending", handle } as const
entries.set(promptKey(handle), entry)
return save(entry)
},
completed(handle: CompletionHandle, notification: OpenCodeNotification) {
const entry = { status: "completed", handle, notification } as const
entries.set(promptKey(handle), entry)
return save(entry)
},
remove(handle: CompletionHandle) {
if (!entries.delete(promptKey(handle))) return writes
return enqueue(() => rm(completionPath(directory, handle), { force: true }))
},
close: () => writes,
}
}
export type CompletionStore = Awaited<ReturnType<typeof createCompletionStore>>
function completionFilename(handle: CompletionHandle) {
return `${encodeURIComponent(promptKey(handle))}.json`
}
function completionPath(directory: string, handle: CompletionHandle) {
return join(directory, completionFilename(handle))
}
+288
View File
@@ -0,0 +1,288 @@
// Full-duplex terminal audio bridge with Apple voice processing (AEC).
//
// stdin <- raw PCM16 mono 24kHz to play through the speakers
// stdout -> raw PCM16 mono 24kHz captured from the microphone
// SIGUSR1: drop any queued speaker audio (barge-in flush)
//
// Two independent engines: attaching a speaker source to a voice-processed
// engine silently kills its input tap, so playback runs on its own plain
// engine. Voice processing (echo cancellation) is attempted on the input
// engine and abandoned if the mic delivers nothing: on some devices
// (Bluetooth headsets mid-negotiation) the VP tap never fires. Without VP
// there is no echo cancellation, which is acceptable exactly when it happens:
// headphones have no echo path.
//
// Compiled on demand by spike.ts: swiftc -O duplex-audio.swift -o duplex-audio
import AVFoundation
let stderr = FileHandle.standardError
func log(_ message: String) { stderr.write(Data("[audio] \(message)\n".utf8)) }
final class SpeakerQueue {
private static let capacity = 24_000 * 2 * 30
private var data = [UInt8](repeating: 0, count: capacity)
private var readIndex = 0
private var writeIndex = 0
private var count = 0
private let lock = NSLock()
func push(_ chunk: Data) {
if chunk.isEmpty { return }
lock.lock()
defer { lock.unlock() }
chunk.withUnsafeBytes { source in
let skipped = max(0, source.count - Self.capacity)
let start = skipped + skipped % 2
let bytes = source.count - start
let overflow = max(0, count + bytes - Self.capacity)
let dropped = min(count, overflow + overflow % 2)
readIndex = (readIndex + dropped) % Self.capacity
count -= dropped
let first = min(bytes, Self.capacity - writeIndex)
data.withUnsafeMutableBytes { destination in
destination.baseAddress!.advanced(by: writeIndex).copyMemory(
from: source.baseAddress!.advanced(by: start), byteCount: first)
destination.baseAddress!.copyMemory(
from: source.baseAddress!.advanced(by: start + first), byteCount: bytes - first)
}
writeIndex = (writeIndex + bytes) % Self.capacity
count += bytes
}
}
func render(frames: Int, into output: UnsafeMutablePointer<Float>) {
lock.lock()
defer { lock.unlock() }
let samples = min(frames, count / 2)
for index in 0..<samples {
let low = UInt16(data[readIndex])
let high = UInt16(data[(readIndex + 1) % Self.capacity])
output[index] = Float(Int16(bitPattern: low | high << 8)) / 32768.0
readIndex = (readIndex + 2) % Self.capacity
}
count -= samples * 2
for index in samples..<frames { output[index] = 0 }
}
func flush() {
lock.lock()
readIndex = writeIndex
count = 0
lock.unlock()
}
}
#if QUEUE_BENCHMARK
let correctnessQueue = SpeakerQueue()
correctnessQueue.push(Data([0x00, 0x80, 0xff, 0x7f]))
var correctnessOutput = [Float](repeating: 1, count: 3)
correctnessOutput.withUnsafeMutableBufferPointer {
correctnessQueue.render(frames: 3, into: $0.baseAddress!)
}
precondition(correctnessOutput[0] == -1)
precondition(abs(correctnessOutput[1] - Float(Int16.max) / 32768) < 0.000_001)
precondition(correctnessOutput[2] == 0)
correctnessQueue.push(Data([1, 0]))
correctnessQueue.flush()
correctnessOutput.withUnsafeMutableBufferPointer {
correctnessQueue.render(frames: 3, into: $0.baseAddress!)
}
precondition(correctnessOutput.allSatisfy { $0 == 0 })
let benchmarkQueue = SpeakerQueue()
let benchmarkFrames = 512
let benchmarkChunk = Data(repeating: 1, count: benchmarkFrames * 2)
var benchmarkOutput = [Float](repeating: 0, count: benchmarkFrames)
for _ in 0..<48 { benchmarkQueue.push(benchmarkChunk) }
for _ in 0..<2_000 {
benchmarkOutput.withUnsafeMutableBufferPointer {
benchmarkQueue.render(frames: benchmarkFrames, into: $0.baseAddress!)
}
benchmarkQueue.push(benchmarkChunk)
}
var benchmarkDurations = [UInt64]()
benchmarkDurations.reserveCapacity(30_000)
var benchmarkChecksum: Float = 0
for _ in 0..<30_000 {
let started = DispatchTime.now().uptimeNanoseconds
benchmarkOutput.withUnsafeMutableBufferPointer {
benchmarkQueue.render(frames: benchmarkFrames, into: $0.baseAddress!)
}
benchmarkQueue.push(benchmarkChunk)
benchmarkDurations.append(DispatchTime.now().uptimeNanoseconds - started)
benchmarkChecksum += benchmarkOutput[0]
}
benchmarkDurations.sort()
print("METRIC speaker_queue_median_ns=\(benchmarkDurations[benchmarkDurations.count / 2])")
print("METRIC speaker_queue_p99_ns=\(benchmarkDurations[benchmarkDurations.count * 99 / 100])")
print("METRIC speaker_queue_worst_ns=\(benchmarkDurations.last!)")
print("CHECKSUM \(benchmarkChecksum)")
exit(0)
#endif
let queue = SpeakerQueue()
let playFormat = AVAudioFormat(commonFormat: .pcmFormatFloat32, sampleRate: 24000, channels: 1, interleaved: false)!
let captureFormat = AVAudioFormat(commonFormat: .pcmFormatInt16, sampleRate: 24000, channels: 1, interleaved: true)!
var rebuildScheduled = false
var ignoreRouteChangesUntil = Date.distantPast
func scheduleRebuild(_ message: String, after delay: TimeInterval = 0.5) {
if rebuildScheduled { return }
rebuildScheduled = true
DispatchQueue.main.asyncAfter(deadline: .now() + delay) {
rebuildScheduled = false
// Starting an engine emits configuration changes of its own. AirPods can
// otherwise turn one real route change into a perpetual rebuild loop.
ignoreRouteChangesUntil = Date().addingTimeInterval(1.5)
log(message)
startInput(voiceProcessing: useAEC)
startOutput()
}
}
// Speaker: its own engine, pulling PCM16 from the stdin-fed queue.
var outputEngine: AVAudioEngine?
func startOutput() {
outputEngine?.stop()
let engine = AVAudioEngine()
outputEngine = engine
let source = AVAudioSourceNode(format: playFormat) { _, _, frameCount, audioBufferList -> OSStatus in
let out = UnsafeMutableAudioBufferListPointer(audioBufferList)[0].mData!.assumingMemoryBound(to: Float.self)
queue.render(frames: Int(frameCount), into: out)
return noErr
}
engine.attach(source)
engine.connect(source, to: engine.mainMixerNode, format: playFormat)
do {
try engine.start()
let format = engine.outputNode.outputFormat(forBus: 0)
log("output active (\(Int(format.sampleRate))Hz, \(format.channelCount)ch)")
} catch {
log("output engine failed: \(error)")
scheduleRebuild("audio engines unavailable — retrying")
}
}
// Microphone: take channel 0 of whatever the hardware provides, resample to
// 24kHz PCM16 for stdout. Channel extraction is manual because hardware
// channel counts vary wildly (1, 2, 3, 22...) and AVAudioConverter cannot
// downmix all of them.
var tapCount = 0
var inputEngine: AVAudioEngine?
var useAEC = CommandLine.arguments.contains("--aec")
func startInput(voiceProcessing: Bool) {
inputEngine?.stop()
let engine = AVAudioEngine()
inputEngine = engine
if voiceProcessing {
do {
try engine.inputNode.setVoiceProcessingEnabled(true)
} catch {
log("voice processing unavailable (\(error)) — no echo cancellation")
useAEC = false
return startInput(voiceProcessing: false)
}
if #available(macOS 14.0, *) {
engine.inputNode.voiceProcessingOtherAudioDuckingConfiguration = .init(
enableAdvancedDucking: false,
duckingLevel: .min
)
}
}
let micFormat = engine.inputNode.outputFormat(forBus: 0)
guard micFormat.sampleRate > 0, micFormat.channelCount > 0, let monoFormat = AVAudioFormat(
commonFormat: .pcmFormatFloat32, sampleRate: micFormat.sampleRate, channels: 1, interleaved: false)
else {
log("microphone route is not ready — retrying")
scheduleRebuild("audio engines unavailable — retrying")
return
}
guard let converter = AVAudioConverter(from: monoFormat, to: captureFormat) else {
log("cannot convert \(micFormat.sampleRate)Hz to 24kHz — retrying")
scheduleRebuild("audio engines unavailable — retrying")
return
}
engine.inputNode.installTap(onBus: 0, bufferSize: 2400, format: micFormat) { buffer, _ in
tapCount += 1
if tapCount == 1 { log("mic active (echo cancellation \(voiceProcessing ? "on" : "off"))") }
guard let channel = buffer.floatChannelData?[0], buffer.frameLength > 0 else { return }
guard let mono = AVAudioPCMBuffer(pcmFormat: monoFormat, frameCapacity: buffer.frameLength) else { return }
memcpy(mono.floatChannelData![0], channel, Int(buffer.frameLength) * 4)
mono.frameLength = buffer.frameLength
let capacity = AVAudioFrameCount(Double(buffer.frameLength) * 24000.0 / micFormat.sampleRate) + 32
guard let converted = AVAudioPCMBuffer(pcmFormat: captureFormat, frameCapacity: capacity) else { return }
var consumed = false
converter.convert(to: converted, error: nil) { _, status in
if consumed {
status.pointee = .noDataNow
return nil
}
consumed = true
status.pointee = .haveData
return mono
}
guard converted.frameLength > 0, let out = converted.int16ChannelData?[0] else { return }
FileHandle.standardOutput.write(Data(bytes: out, count: Int(converted.frameLength) * 2))
}
do {
try engine.start()
log("input active (\(Int(micFormat.sampleRate))Hz, \(micFormat.channelCount)ch, echo cancellation \(voiceProcessing ? "on" : "off"))")
} catch {
if voiceProcessing {
log("input engine failed with voice processing (\(error)) — retrying without")
useAEC = false
return startInput(voiceProcessing: false)
}
log("input engine failed: \(error)")
scheduleRebuild("audio engines unavailable — retrying")
return
}
// Watchdog: on some devices the voice-processed tap simply never fires.
// Fall back to a plain tap; without VP the mic reliably delivers.
if voiceProcessing {
DispatchQueue.main.asyncAfter(deadline: .now() + 2.5) {
if tapCount > 0 { return }
log("voice-processed mic delivered nothing — restarting without echo cancellation")
useAEC = false
startInput(voiceProcessing: false)
}
}
}
// Voice processing is opt-in (--aec): it is only needed on speakers, and on
// some machines (observed with Bluetooth headsets active) the VP engine binds
// to the wrong capture device entirely, delivering noise instead of the mic.
// Mic first: activating the mic flips Bluetooth headsets from music mode to
// headset mode, reconfiguring the output device. Starting output afterwards
// (and rebuilding on any route change below) keeps playback on the live device.
startInput(voiceProcessing: useAEC)
startOutput()
// Device switches (Bluetooth profile flips, headphones plugged/unplugged,
// default device changes) stop engines silently. Rebuild both, debounced.
NotificationCenter.default.addObserver(
forName: .AVAudioEngineConfigurationChange, object: nil, queue: .main
) { _ in
if Date() < ignoreRouteChangesUntil { return }
scheduleRebuild("audio route changed — rebuilding engines")
}
signal(SIGUSR1, SIG_IGN)
let flushSignal = DispatchSource.makeSignalSource(signal: SIGUSR1, queue: .main)
flushSignal.setEventHandler { queue.flush() }
flushSignal.resume()
DispatchQueue.global().async {
while true {
let chunk = FileHandle.standardInput.availableData
if chunk.isEmpty { exit(0) } // parent closed stdin
queue.push(chunk)
}
}
RunLoop.main.run()
+48
View File
@@ -0,0 +1,48 @@
import type { CompletionReceipt, OpenCodeAnnouncement } from "./opencode-notification"
import { promptKey } from "./prompt-handle"
export type RoutedAnnouncement = {
readonly announcement: OpenCodeAnnouncement
readonly promptID?: string
readonly delegationID?: string
}
export function createNotificationRouter(deliver: (announcement: RoutedAnnouncement) => void) {
const delegations = new Map<string, string>()
const deferred: OpenCodeAnnouncement[] = []
let pendingDelegatedTools = 0
const route = (announcement: OpenCodeAnnouncement) => {
const prompt = "prompt_id" in announcement.notification
? { sessionID: announcement.notification.session_id, promptID: announcement.notification.prompt_id }
: undefined
const promptID = prompt?.promptID
if (prompt && pendingDelegatedTools > 0 && !delegations.has(promptKey(prompt))) {
deferred.push(announcement)
return
}
deliver({
announcement,
promptID,
delegationID: prompt ? delegations.get(promptKey(prompt)) : undefined,
})
}
return {
route,
beginDelegatedTool() {
pendingDelegatedTools += 1
},
finishDelegatedTool(
delegationID: string,
admittedPrompt?: { readonly sessionID: string; readonly promptID: string },
) {
if (admittedPrompt) delegations.set(promptKey(admittedPrompt), delegationID)
pendingDelegatedTools -= 1
if (pendingDelegatedTools === 0) deferred.splice(0).forEach(route)
},
acknowledged(receipt: CompletionReceipt) {
delegations.delete(promptKey(receipt))
},
}
}
@@ -0,0 +1,97 @@
import { Schema } from "effect"
import type { PromptHandle } from "./prompt-handle"
const Completed = Schema.Struct({
type: Schema.Literal("opencode.prompt.completed"),
session_id: Schema.String,
prompt_id: Schema.String,
status: Schema.Literals(["completed", "failed"]),
text: Schema.String,
error: Schema.optional(Schema.String),
})
const Failed = Schema.Struct({
type: Schema.Literal("opencode.prompt.failed"),
session_id: Schema.String,
prompt_id: Schema.String,
status: Schema.Literal("failed"),
error: Schema.String,
})
const PermissionBlocked = Schema.Struct({
type: Schema.Literal("opencode.prompt.blocked"),
prompt_id: Schema.String,
blocker: Schema.Literal("permission"),
session_id: Schema.String,
request_id: Schema.String,
action: Schema.String,
resources: Schema.Unknown,
})
const QuestionBlocked = Schema.Struct({
type: Schema.Literal("opencode.prompt.blocked"),
prompt_id: Schema.String,
blocker: Schema.Literal("question"),
session_id: Schema.String,
request_id: Schema.String,
questions: Schema.Unknown,
})
const FormBlocked = Schema.Struct({
type: Schema.Literal("opencode.prompt.blocked"),
prompt_id: Schema.String,
blocker: Schema.Literal("form"),
session_id: Schema.String,
form_id: Schema.String,
title: Schema.String,
fields: Schema.Unknown,
})
const EventsFailed = Schema.Struct({
type: Schema.Literal("opencode.events.failed"),
error: Schema.String,
})
export const OpenCodeNotification = Schema.Union([
Completed,
Failed,
PermissionBlocked,
QuestionBlocked,
FormBlocked,
EventsFailed,
])
export type OpenCodeNotification = typeof OpenCodeNotification.Type
export type OpenCodePromptBlocked = Extract<OpenCodeNotification, { readonly type: "opencode.prompt.blocked" }>
export type CompletionReceipt = PromptHandle
export type OpenCodeAnnouncement = {
readonly notification: OpenCodeNotification
readonly receipt?: CompletionReceipt
}
export function openCodeAnnouncementText(notification: OpenCodeNotification) {
const text = (() => {
switch (notification.type) {
case "opencode.prompt.completed":
return `OpenCode prompt ${notification.status}: ${notification.text}${notification.error ? ` Error: ${notification.error}` : ""}`
case "opencode.prompt.failed":
return `OpenCode prompt failed: ${notification.error}`
case "opencode.prompt.blocked":
if (notification.blocker === "permission")
return `OpenCode needs permission for ${notification.action}: ${compact(notification.resources)}`
if (notification.blocker === "question")
return `OpenCode needs the user to answer: ${compact(notification.questions)}`
return `OpenCode needs the user to complete the form “${notification.title}”: ${compact(notification.fields)}`
case "opencode.events.failed":
return `OpenCode event delivery failed: ${notification.error}`
}
return "OpenCode has an update."
})()
return text.length > 800 ? text.slice(0, 797) + "..." : text
}
function compact(value: unknown) {
if (typeof value === "string") return value
return JSON.stringify(value) ?? String(value)
}
+811
View File
@@ -0,0 +1,811 @@
import { realpathSync } from "node:fs"
import { tmpdir } from "node:os"
import { basename, isAbsolute, relative } from "node:path"
import { ClientError, isSessionNotFoundError, OpenCode } from "@opencode-ai/client/promise"
import type { SessionMessageAssistant, SessionMessageUser, V2Event } from "@opencode-ai/client/promise"
import { Form, Question, SessionMessage } from "@opencode-ai/client/effect"
import { Context, Effect, FiberMap, Layer, ManagedRuntime, Option, Schema, Stream } from "effect"
import { createCompletionStore, type CompletionStore } from "./completion-store"
import type {
CompletionReceipt,
OpenCodeAnnouncement,
OpenCodeNotification,
OpenCodePromptBlocked,
} from "./opencode-notification"
import type { VoiceTool, VoiceToolExecution } from "./protocol"
import type { PromptHandle } from "./prompt-handle"
type Client = ReturnType<typeof OpenCode.make>
const AdmittedPrompt = Symbol("AdmittedPrompt")
type Tool = {
readonly description: string
readonly parameters: unknown
readonly execute: (input: Record<string, unknown>) => Effect.Effect<unknown, unknown>
}
type BridgeApi = {
readonly definitions: ReadonlyArray<VoiceTool>
readonly execute: (name: string, input: Record<string, unknown>) => Effect.Effect<VoiceToolExecution, unknown>
readonly acknowledge: (receipt: CompletionReceipt) => Effect.Effect<void, unknown>
readonly close: Effect.Effect<void>
}
class Bridge extends Context.Service<Bridge, BridgeApi>()("@opencode-ai/voice/OpenCodeBridge") {}
export async function createOpenCodeBridge(options: {
client: Client
directory: string
model: { readonly providerID: string; readonly id: string; readonly variant?: string }
notify: (announcement: OpenCodeAnnouncement) => void
trace?: (event: string, data?: Record<string, unknown>) => void
completionStore?: CompletionStore
}) {
const completionStore = options.completionStore ?? (await createCompletionStore())
const runtime = ManagedRuntime.make(Layer.effect(Bridge, makeBridge({ ...options, completionStore })))
const bridge = await runtime.runPromise(Bridge)
return {
definitions: bridge.definitions,
execute: (name: string, input: Record<string, unknown>) => runtime.runPromise(bridge.execute(name, input)),
acknowledge: (receipt: CompletionReceipt) => runtime.runPromise(bridge.acknowledge(receipt)),
close: async () => {
await runtime.runPromise(bridge.close)
await Promise.race([runtime.dispose(), Bun.sleep(1_000)])
await completionStore.close()
},
}
}
const makeBridge = Effect.fnUntraced(function* (options: {
client: Client
directory: string
model: { readonly providerID: string; readonly id: string; readonly variant?: string }
notify: (announcement: OpenCodeAnnouncement) => void
trace?: (event: string, data?: Record<string, unknown>) => void
completionStore: CompletionStore
}) {
const knownProjects = new Set<string>()
const knownSessions = new Set<string>()
const registrations = new Map<string, PromptHandle>()
const promoted = new Map<string, string>()
const latest = new Map<string, string>()
const announcedBlockers = new Set<string>()
const completions = yield* FiberMap.make<string>()
const eventAbort = yield* Effect.acquireRelease(
Effect.sync(() => new AbortController()),
(controller) => Effect.sync(() => controller.abort()),
)
const notify = (notification: OpenCodeNotification, receipt?: CompletionReceipt) =>
Effect.sync(() => options.notify({ notification, receipt }))
const trace = (event: string, data: Record<string, unknown>) => Effect.sync(() => options.trace?.(event, data))
const request = <A>(run: (signal: AbortSignal) => PromiseLike<A>) =>
Effect.tryPromise({ try: run, catch: (cause) => cause })
const announceBlocker = (key: string, notification: OpenCodeNotification) => {
if (announcedBlockers.has(key)) return Effect.void
announcedBlockers.add(key)
return notify(notification)
}
const clearRegistration = (handle: PromptHandle) =>
Effect.sync(() => {
registrations.delete(handle.promptID)
if (promoted.get(handle.sessionID) === handle.promptID) promoted.delete(handle.sessionID)
if (latest.get(handle.sessionID) === handle.promptID) latest.delete(handle.sessionID)
})
const listProjects = Effect.fnUntraced(function* () {
const seen = new Set<string>()
const temporary = realpathSync(tmpdir())
return (yield* request((signal) => options.client.project.list({ signal })))
.sort((a, b) => b.time.updated - a.time.updated)
.filter((project) => {
const fromTemporary = relative(temporary, project.worktree)
if (
project.id === "global" ||
fromTemporary === "" ||
(!fromTemporary.startsWith("..") && !isAbsolute(fromTemporary)) ||
seen.has(project.worktree)
)
return false
seen.add(project.worktree)
return true
})
})
const projectDirectory = Effect.fnUntraced(function* (projectID: string) {
if (!knownProjects.has(projectID)) return undefined
return (yield* listProjects()).find((project) => project.id === projectID)?.worktree
})
const complete = Effect.fnUntraced(function* (handle: PromptHandle) {
yield* trace("opencode.wait.started", handle)
yield* waitForIdle(handle)
const reply = yield* request((signal) => finalReply(options.client, handle, signal))
yield* trace("opencode.wait.completed", { ...handle, replyID: reply?.id, error: reply?.error?.message })
const notification: OpenCodeNotification = {
type: "opencode.prompt.completed",
session_id: handle.sessionID,
prompt_id: handle.promptID,
status: reply?.error ? "failed" : "completed",
text: reply
? assistantText(reply) || "OpenCode finished without a text reply."
: "OpenCode finished without a reply.",
error: reply?.error?.message,
}
yield* request(() => options.completionStore.completed(handle, notification))
yield* notify(notification, handle)
})
const waitForIdle = Effect.fnUntraced(function* (handle: PromptHandle, attempt = 1): Effect.fn.Return<void, unknown> {
return yield* request((signal) => options.client.session.wait({ sessionID: handle.sessionID }, { signal })).pipe(
Effect.catch((error) => {
if (!(error instanceof ClientError) || error.reason !== "Transport") return Effect.fail(error)
return trace("opencode.wait.retrying", {
...handle,
attempt,
reason: error.reason,
cause: error.cause instanceof Error ? error.cause.name : typeof error.cause,
}).pipe(Effect.andThen(Effect.sleep("1 second")), Effect.andThen(waitForIdle(handle, attempt + 1)))
}),
)
})
const register = Effect.fnUntraced(function* (handle: PromptHandle) {
yield* complete(handle).pipe(
Effect.catch((error) => {
const notification: OpenCodeNotification = {
type: "opencode.prompt.failed",
session_id: handle.sessionID,
prompt_id: handle.promptID,
status: "failed",
error: String(error),
}
return request(() => options.completionStore.completed(handle, notification)).pipe(
Effect.andThen(notify(notification, handle)),
)
}),
Effect.ensuring(clearRegistration(handle)),
FiberMap.run(completions, handle.promptID, { onlyIfMissing: true, startImmediately: true }),
)
})
const restoreBlockers = Effect.fnUntraced(function* (handle: PromptHandle) {
const [permissions, questions, forms] = yield* Effect.all(
[
request((signal) => options.client.permission.list({ sessionID: handle.sessionID }, { signal })),
request((signal) => options.client.question.list({ sessionID: handle.sessionID }, { signal })),
request((signal) => options.client.form.list({ sessionID: handle.sessionID }, { signal })),
],
{ concurrency: "unbounded" },
)
yield* Effect.forEach(
[
...permissions.map((item) => ({
key: `permission:${item.id}`,
notification: {
type: "opencode.prompt.blocked",
prompt_id: handle.promptID,
blocker: "permission",
session_id: handle.sessionID,
request_id: item.id,
action: item.action,
resources: item.resources,
} satisfies OpenCodeNotification,
})),
...questions.map((item) => ({
key: `question:${item.id}`,
notification: {
type: "opencode.prompt.blocked",
prompt_id: handle.promptID,
blocker: "question",
session_id: handle.sessionID,
request_id: item.id,
questions: item.questions,
} satisfies OpenCodeNotification,
})),
...forms.map((item) => ({
key: `form:${item.id}`,
notification: {
type: "opencode.prompt.blocked",
prompt_id: handle.promptID,
blocker: "form",
session_id: handle.sessionID,
form_id: item.id,
title: item.title,
fields: item.fields,
} satisfies OpenCodeNotification,
})),
],
(item) => announceBlocker(item.key, item.notification),
{ discard: true },
)
})
const admit = Effect.fnUntraced(function* (sessionID: string, text: string) {
const handle = { sessionID, promptID: SessionMessage.ID.create() }
registrations.set(handle.promptID, handle)
latest.set(sessionID, handle.promptID)
yield* trace("opencode.prompt.admitting", handle)
yield* request(() => options.completionStore.admitting(handle, text))
const admitted = yield* request((signal) =>
options.client.session.prompt({ sessionID, id: handle.promptID, text }, { signal }),
).pipe(
Effect.tapError(() =>
request(() => options.completionStore.remove(handle)).pipe(
Effect.andThen(clearRegistration(handle)),
),
),
)
knownSessions.add(sessionID)
yield* request(() => options.completionStore.pending(handle))
yield* trace("opencode.prompt.admitted", { ...handle, admittedSeq: admitted.admittedSeq })
yield* register(handle)
return {
status: "started",
session_id: sessionID,
prompt_id: admitted.id,
notification: "registered",
message: "OpenCode accepted the prompt and will send a completion notification. Continue the conversation now.",
[AdmittedPrompt]: handle,
}
})
const start = Effect.fnUntraced(function* (text: string, projectID?: string) {
const directory = projectID ? yield* projectDirectory(projectID) : options.directory
if (!directory) return toolError("Use a project ID returned by find_projects.")
const session = yield* request((signal) =>
options.client.session.create({ location: { directory }, model: options.model }, { signal }),
)
return yield* admit(session.id, text)
})
const tools: Record<string, Tool> = {
find_projects: {
description: "Find known OpenCode projects by display name. Returns opaque IDs, never filesystem paths.",
parameters: {
type: "object",
additionalProperties: false,
properties: {
query: { type: ["string", "null"], description: "Name fragment, or null for recent projects." },
limit: { type: "integer", minimum: 1, maximum: 20 },
},
required: ["query", "limit"],
},
execute: Effect.fnUntraced(function* (input) {
const query = typeof input["query"] === "string" ? input["query"].toLowerCase() : undefined
const limit = typeof input["limit"] === "number" ? input["limit"] : 10
const projects = (yield* listProjects())
.filter((project) => !query || projectLabel(project).toLowerCase().includes(query))
.slice(0, limit)
projects.forEach((project) => knownProjects.add(project.id))
return {
status: "ok",
projects: projects.map((project) => ({
id: project.id,
name: projectLabel(project),
directories: 1 + project.sandboxes.length,
updated: new Date(project.time.updated).toISOString(),
})),
}
}),
},
find_sessions: {
description:
"Find root OpenCode sessions by title and recency. Search the launch project by default; use all_projects only when requested or the local search misses.",
parameters: {
type: "object",
additionalProperties: false,
properties: {
query: { type: ["string", "null"], description: "Title words, or null for recent sessions." },
scope: { type: "string", enum: ["current_project", "all_projects"] },
recency: { type: "string", enum: ["day", "week", "month", "any"] },
limit: { type: "integer", minimum: 1, maximum: 20 },
},
required: ["query", "scope", "recency", "limit"],
},
execute: Effect.fnUntraced(function* (input) {
const query = typeof input["query"] === "string" ? input["query"] : undefined
const scope = input["scope"] === "all_projects" ? "all_projects" : "current_project"
const recency = typeof input["recency"] === "string" ? input["recency"] : "any"
const limit = typeof input["limit"] === "number" ? input["limit"] : 10
const durations: Record<string, number> = { day: 86_400_000, week: 604_800_000, month: 2_592_000_000 }
const threshold = recency === "any" ? 0 : Date.now() - (durations[recency] ?? 0)
const [result, projectList] = yield* Effect.all(
[
request((signal) =>
options.client.session.list(
{
...(scope === "current_project" ? { directory: options.directory } : {}),
search: query,
parentID: null,
limit: recency === "any" ? limit : Math.min(limit * 5, 100),
order: "desc",
},
{ signal },
),
),
listProjects(),
],
{ concurrency: "unbounded" },
)
const projects = new Map(projectList.map((project) => [project.id, projectLabel(project)]))
const sessions = result.data
.filter((session) => projects.has(session.projectID) && session.time.updated >= threshold)
.slice(0, limit)
sessions.forEach((session) => knownSessions.add(session.id))
return {
status: "ok",
scope,
sessions: sessions.map((session) => ({
id: session.id,
title: session.title,
project: projects.get(session.projectID) ?? "project",
updated: new Date(session.time.updated).toISOString(),
})),
}
}),
},
read_session: {
description:
"Read bounded recent user and assistant text from a session returned by find_sessions or start_session. This never prompts or wakes the coding agent.",
parameters: {
type: "object",
additionalProperties: false,
properties: {
session_id: { type: "string", description: "Session ID returned by a previous voice tool." },
limit: { type: "integer", minimum: 1, maximum: 20 },
},
required: ["session_id", "limit"],
},
execute: Effect.fnUntraced(function* (input) {
const sessionID = knownSession(input, knownSessions)
if (!sessionID) return toolError("Use a session ID returned by find_sessions or start_session.")
const limit = typeof input["limit"] === "number" ? input["limit"] : 10
const [session, messages] = yield* Effect.all(
[
request((signal) => options.client.session.get({ sessionID }, { signal })),
request((signal) => options.client.message.list({ sessionID, order: "desc", limit }, { signal })),
],
{ concurrency: "unbounded" },
)
const latestAssistant = messages.data.find(
(message): message is SessionMessageAssistant => message.type === "assistant",
)
return {
status: "ok",
title: session.title,
running: latestAssistant ? !latestAssistant.time.completed : false,
messages: messages.data
.toReversed()
.flatMap((message) =>
message.type === "user" || message.type === "assistant" ? [sessionMessage(message)] : [],
),
}
}),
},
rename_session: {
description:
"Rename a session returned by find_sessions or start_session after the user requests a new title. This changes only the display title and does not prompt, wake, or interrupt the coding agent.",
parameters: {
type: "object",
additionalProperties: false,
properties: {
session_id: { type: "string", description: "Session ID returned by a previous voice tool." },
title: { type: "string", description: "The complete new session title." },
},
required: ["session_id", "title"],
},
execute: Effect.fnUntraced(function* (input) {
const sessionID = knownSession(input, knownSessions)
const title = requireString(input, "title")?.trim()
if (!sessionID) return toolError("Use a session ID returned by find_sessions or start_session.")
if (!title) return toolError("A non-empty session title is required.")
const session = yield* request((signal) => options.client.session.get({ sessionID }, { signal }))
if (session.title === title)
return { status: "unchanged", session_id: sessionID, previous_title: session.title, title }
yield* request((signal) => options.client.session.rename({ sessionID, title }, { signal }))
return { status: "renamed", session_id: sessionID, previous_title: session.title, title }
}),
},
start_session: {
description:
"Create an OpenCode session, admit its first prompt, and register a one-shot completion notification. Omit project_id to use the launch project.",
parameters: {
type: "object",
additionalProperties: false,
properties: {
text: { type: "string", description: "Clear instruction for the coding agent." },
project_id: { type: ["string", "null"], description: "Opaque project ID, or null for the launch project." },
},
required: ["text", "project_id"],
},
execute: (input) => {
const text = requireString(input, "text")
if (!text) return Effect.succeed(toolError("Task text is required."))
return start(text, typeof input["project_id"] === "string" ? input["project_id"] : undefined)
},
},
prompt_session: {
description:
"Admit a prompt into a discovered or previously started OpenCode session and register a one-shot completion notification.",
parameters: {
type: "object",
additionalProperties: false,
properties: {
session_id: { type: "string", description: "Session ID returned by a previous voice tool." },
text: { type: "string", description: "Clear instruction for the coding agent." },
},
required: ["session_id", "text"],
},
execute: (input) => {
const sessionID = knownSession(input, knownSessions)
const text = requireString(input, "text")
if (!sessionID) return Effect.succeed(toolError("Use a session ID returned by find_sessions or start_session."))
if (!text) return Effect.succeed(toolError("Task text is required."))
return admit(sessionID, text)
},
},
interrupt_session: {
description: "Interrupt one OpenCode session. Call only after the user explicitly confirms the interruption.",
parameters: sessionParameters(),
execute: Effect.fnUntraced(function* (input) {
const sessionID = knownSession(input, knownSessions)
if (!sessionID) return toolError("Use a session ID returned by a previous voice tool.")
yield* request((signal) => options.client.session.interrupt({ sessionID }, { signal }))
return { status: "interrupted", session_id: sessionID }
}),
},
list_pending_permissions: {
description: "List permission requests blocking one OpenCode session.",
parameters: sessionParameters(),
execute: Effect.fnUntraced(function* (input) {
const sessionID = knownSession(input, knownSessions)
if (!sessionID) return toolError("Use a session ID returned by a previous voice tool.")
const requests = yield* request((signal) => options.client.permission.list({ sessionID }, { signal }))
return {
status: "ok",
requests: requests.map((item) => ({ id: item.id, action: item.action, resources: item.resources })),
}
}),
},
reply_permission: {
description:
"Allow once or reject a pending permission after stating the action and resources and receiving the user's explicit decision.",
parameters: {
type: "object",
additionalProperties: false,
properties: {
session_id: { type: "string" },
request_id: { type: "string" },
decision: { type: "string", enum: ["allow_once", "reject"] },
},
required: ["session_id", "request_id", "decision"],
},
execute: Effect.fnUntraced(function* (input) {
const sessionID = knownSession(input, knownSessions)
const requestID = requireString(input, "request_id")
const decision = input["decision"]
if (!sessionID || !requestID || (decision !== "allow_once" && decision !== "reject"))
return toolError("A known session, request ID, and valid decision are required.")
const requests = yield* request((signal) => options.client.permission.list({ sessionID }, { signal }))
if (!requests.some((item) => item.id === requestID))
return toolError("That permission request is not pending.", true)
yield* request((signal) =>
options.client.permission.reply(
{ sessionID, requestID, reply: decision === "allow_once" ? "once" : "reject" },
{ signal },
),
)
return { status: decision === "allow_once" ? "allowed_once" : "rejected", request_id: requestID }
}),
},
reply_question: {
description: "Reply to questions blocking an OpenCode session after collecting the user's answers.",
parameters: {
type: "object",
additionalProperties: false,
properties: {
session_id: { type: "string" },
request_id: { type: "string" },
answers: {
type: "array",
description: "One string array per question, preserving question order.",
items: { type: "array", items: { type: "string" } },
},
},
required: ["session_id", "request_id", "answers"],
},
execute: Effect.fnUntraced(function* (input) {
const sessionID = knownSession(input, knownSessions)
const requestID = requireString(input, "request_id")
const answers = Option.getOrUndefined(decodeQuestionAnswers(input["answers"]))
if (!sessionID || !requestID || !answers)
return toolError("A known session, request ID, and valid answers are required.")
yield* request((signal) => options.client.question.reply({ sessionID, requestID, answers }, { signal }))
return { status: "answered", request_id: requestID }
}),
},
reject_question: {
description: "Reject a pending OpenCode question after the user explicitly declines to answer.",
parameters: requestParameters(),
execute: Effect.fnUntraced(function* (input) {
const sessionID = knownSession(input, knownSessions)
const requestID = requireString(input, "request_id")
if (!sessionID || !requestID) return toolError("A known session and request ID are required.")
yield* request((signal) => options.client.question.reject({ sessionID, requestID }, { signal }))
return { status: "rejected", request_id: requestID }
}),
},
reply_form: {
description: "Submit values for a form blocking an OpenCode session after collecting them from the user.",
parameters: {
type: "object",
additionalProperties: false,
properties: {
session_id: { type: "string" },
form_id: { type: "string" },
answer: { type: "object", additionalProperties: true },
},
required: ["session_id", "form_id", "answer"],
},
execute: Effect.fnUntraced(function* (input) {
const sessionID = knownSession(input, knownSessions)
const formID = requireString(input, "form_id")
const answer = Option.getOrUndefined(decodeFormAnswer(input["answer"]))
if (!sessionID || !formID || !answer)
return toolError("A known session, form ID, and valid answer are required.")
yield* request((signal) => options.client.form.reply({ sessionID, formID, answer }, { signal }))
return { status: "answered", form_id: formID }
}),
},
cancel_form: {
description: "Cancel a pending OpenCode form after the user explicitly declines to complete it.",
parameters: {
type: "object",
additionalProperties: false,
properties: { session_id: { type: "string" }, form_id: { type: "string" } },
required: ["session_id", "form_id"],
},
execute: Effect.fnUntraced(function* (input) {
const sessionID = knownSession(input, knownSessions)
const formID = requireString(input, "form_id")
if (!sessionID || !formID) return toolError("A known session and form ID are required.")
yield* request((signal) => options.client.form.cancel({ sessionID, formID }, { signal }))
return { status: "cancelled", form_id: formID }
}),
},
}
const definitions = Object.entries(tools).map(
([name, tool]) =>
({ type: "function", name, description: tool.description, parameters: tool.parameters }) satisfies VoiceTool,
)
yield* Effect.suspend(() =>
Stream.fromAsyncIterable(options.client.event.subscribe({ signal: eventAbort.signal }), (cause) => cause).pipe(
Stream.runForEach((event) => {
if (event.type === "session.input.promoted") {
if (!registrations.has(event.data.inputID)) return Effect.void
promoted.set(event.data.sessionID, event.data.inputID)
return trace("opencode.prompt.promoted", {
sessionID: event.data.sessionID,
promptID: event.data.inputID,
})
}
const sessionID = blockerSession(event)
const promptID = sessionID ? (promoted.get(sessionID) ?? latest.get(sessionID)) : undefined
const blocker = sessionID && promptID ? blockerNotification(event, sessionID, promptID) : undefined
if (!blocker || !promptID) return Effect.void
options.trace?.("opencode.prompt.blocked", { sessionID, promptID, blocker: blocker["blocker"] })
if (registrations.has(promptID)) return announceBlocker(blockerKey(blocker), blocker)
return Effect.void
}),
Effect.catch((error) => notify({ type: "opencode.events.failed", error: String(error) })),
Effect.andThen(Effect.sleep("1 second")),
),
).pipe(Effect.forever, Effect.forkScoped({ startImmediately: true }))
yield* Effect.forEach(
options.completionStore.entries(),
Effect.fnUntraced(function* (entry) {
if (entry.status !== "completed") {
const exists = yield* request((signal) =>
options.client.session.get({ sessionID: entry.handle.sessionID }, { signal }),
).pipe(
Effect.as(true),
Effect.catch((error) => {
if (!isSessionNotFoundError(error)) return Effect.fail(error)
return request(() => options.completionStore.remove(entry.handle)).pipe(
Effect.andThen(trace("opencode.prompt.orphaned", entry.handle)),
Effect.as(false),
)
}),
)
if (!exists) return
}
knownSessions.add(entry.handle.sessionID)
if (entry.status === "admitting") {
registrations.set(entry.handle.promptID, entry.handle)
latest.set(entry.handle.sessionID, entry.handle.promptID)
yield* trace("opencode.prompt.reconciling", entry.handle)
yield* request((signal) =>
options.client.session.prompt(
{ sessionID: entry.handle.sessionID, id: entry.handle.promptID, text: entry.text },
{ signal },
),
)
yield* request(() => options.completionStore.pending(entry.handle))
yield* restoreBlockers(entry.handle)
yield* register(entry.handle)
return
}
if (entry.status === "pending") {
registrations.set(entry.handle.promptID, entry.handle)
latest.set(entry.handle.sessionID, entry.handle.promptID)
yield* trace("opencode.wait.restored", entry.handle)
yield* restoreBlockers(entry.handle)
yield* register(entry.handle)
return
}
yield* trace("opencode.notification.restored", entry.handle)
yield* notify(entry.notification, entry.handle)
}),
{ concurrency: 4, discard: true },
)
return Bridge.of({
definitions,
execute: (name, input) =>
(tools[name]?.execute(input) ?? Effect.succeed(toolError(`Unknown tool ${name}.`))).pipe(
Effect.map(toolExecution),
),
acknowledge: (receipt) => request(() => options.completionStore.remove(receipt)),
close: Effect.all([Effect.sync(() => eventAbort.abort()), FiberMap.clear(completions)], { discard: true }),
})
})
function toolExecution(output: unknown): VoiceToolExecution {
if (!isPromptToolOutput(output)) return { output }
return {
output: Object.fromEntries(Object.entries(output)),
admittedPrompt: output[AdmittedPrompt],
}
}
function isPromptToolOutput(
output: unknown,
): output is Record<string, unknown> & { readonly [AdmittedPrompt]: PromptHandle } {
if (!output || typeof output !== "object" || !(AdmittedPrompt in output)) return false
const handle = output[AdmittedPrompt]
return (
!!handle &&
typeof handle === "object" &&
"sessionID" in handle &&
typeof handle.sessionID === "string" &&
"promptID" in handle &&
typeof handle.promptID === "string"
)
}
async function finalReply(client: Client, handle: PromptHandle, signal: AbortSignal) {
let reply: SessionMessageAssistant | undefined
let cursor: string | undefined
while (true) {
const page = await client.message.list(
cursor
? { sessionID: handle.sessionID, limit: 200, cursor }
: { sessionID: handle.sessionID, limit: 200, order: "desc" },
{ signal },
)
for (const message of page.data) {
if (message.id === handle.promptID) return reply
if (message.type === "user") reply = undefined
if (message.type === "assistant" && !reply) reply = message
}
cursor = page.cursor.next ?? undefined
if (!cursor) return undefined
}
}
function assistantText(message: SessionMessageAssistant) {
return message.content
.filter((part) => part.type === "text")
.map((part) => part.text)
.join("\n")
.slice(0, 4_000)
}
function sessionMessage(message: SessionMessageUser | SessionMessageAssistant) {
if (message.type === "user") return { role: "user", text: message.text.slice(0, 2_000) }
return {
role: "assistant",
status: message.time.completed ? "completed" : "running",
text: assistantText(message),
tools: message.content
.filter((part) => part.type === "tool")
.map((part) => ({ name: part.name, status: part.state.status })),
}
}
function blockerSession(event: V2Event) {
if (event.type === "permission.v2.asked" || event.type === "question.v2.asked") return event.data.sessionID
if (event.type === "form.created") return event.data.form.sessionID
return undefined
}
function blockerNotification(event: V2Event, sessionID: string, promptID: string): OpenCodePromptBlocked | undefined {
if (event.type === "permission.v2.asked")
return {
type: "opencode.prompt.blocked",
prompt_id: promptID,
blocker: "permission",
session_id: sessionID,
request_id: event.data.id,
action: event.data.action,
resources: event.data.resources,
}
if (event.type === "question.v2.asked")
return {
type: "opencode.prompt.blocked",
prompt_id: promptID,
blocker: "question",
session_id: sessionID,
request_id: event.data.id,
questions: event.data.questions,
}
if (event.type === "form.created")
return {
type: "opencode.prompt.blocked",
prompt_id: promptID,
blocker: "form",
session_id: sessionID,
form_id: event.data.form.id,
title: event.data.form.title,
fields: event.data.form.fields,
}
return undefined
}
function blockerKey(notification: OpenCodePromptBlocked) {
if (notification.blocker === "form") return `form:${notification.form_id}`
return `${notification.blocker}:${notification.request_id}`
}
function projectLabel(project: { readonly name?: string; readonly worktree: string }) {
return project.name ?? (basename(project.worktree) || "project")
}
function requireString(input: Record<string, unknown>, key: string) {
const value = input[key]
if (typeof value !== "string" || value.trim().length === 0) return undefined
return value
}
function knownSession(input: Record<string, unknown>, known: ReadonlySet<string>) {
const sessionID = requireString(input, "session_id")
if (!sessionID || !known.has(sessionID)) return undefined
return sessionID
}
const decodeQuestionAnswers = Schema.decodeUnknownOption(Schema.Array(Question.Answer))
const decodeFormAnswer = Schema.decodeUnknownOption(Form.Answer)
function sessionParameters() {
return {
type: "object",
additionalProperties: false,
properties: { session_id: { type: "string" } },
required: ["session_id"],
}
}
function requestParameters() {
return {
type: "object",
additionalProperties: false,
properties: { session_id: { type: "string" }, request_id: { type: "string" } },
required: ["session_id", "request_id"],
}
}
function toolError(message: string, retryable = false) {
return { status: "error", message, retryable }
}
+19
View File
@@ -0,0 +1,19 @@
export const PCM_SAMPLE_RATE = 24_000
export const PCM_BYTES_PER_MS = (PCM_SAMPLE_RATE * 2) / 1_000
export const PCM_METER_FRAME_MS = 1_000 / 60
export function pcmLevel(bytes: Buffer) {
const samples = Math.floor(bytes.length / 2)
if (samples === 0) return 0
const stride = Math.max(1, Math.floor(samples / 1_200))
let energy = 0
let count = 0
for (let index = 0; index < samples; index += stride) {
const sample = bytes.readInt16LE(index * 2) / 32_768
energy += sample * sample
count += 1
}
const rms = Math.sqrt(energy / count)
if (rms < 0.008) return 0
return Math.min(1, Math.sqrt((rms - 0.008) / 0.18))
}
+37
View File
@@ -0,0 +1,37 @@
import { Plugin } from "@opencode-ai/plugin/v2/tui"
import { createSignal } from "solid-js"
function VoiceStatus(props: { context: Plugin.Context }) {
const [enabled, setEnabled] = createSignal(false)
props.context.keymap.layer(() => ({
mode: "global",
commands: [
{
id: "voice.smoke.toggle",
title: "Toggle voice plugin",
description: "Toggle the V2 voice plugin smoke-test status",
group: "Voice",
bind: "alt+v",
palette: true,
run() {
setEnabled((value) => !value)
},
},
],
}))
return (
<box paddingTop={1}>
<text>voice plugin: {enabled() ? "on" : "off"} (alt+v)</text>
</box>
)
}
export default Plugin.define({
id: "opencode.voice-smoke",
setup(context) {
context.ui.slot("home.bottom", () => <VoiceStatus context={context} />)
context.ui.slot("sidebar.content", () => <VoiceStatus context={context} />)
},
})
+8
View File
@@ -0,0 +1,8 @@
export type PromptHandle = {
readonly sessionID: string
readonly promptID: string
}
export function promptKey(handle: PromptHandle) {
return `${handle.sessionID}:${handle.promptID}`
}
+486
View File
@@ -0,0 +1,486 @@
import type {
VoiceConnection,
VoiceDelegationRequest,
VoiceNotificationRequest,
VoiceProtocol,
VoiceProtocolEvent,
VoiceProtocolOptions,
} from "./protocol"
import { decodeVoiceToolInput } from "./protocol"
import { Option, Schema } from "effect"
const LiveItem = Schema.Struct({
id: Schema.String,
type: Schema.String,
text: Schema.optional(Schema.String),
name: Schema.optional(Schema.String),
call_id: Schema.optional(Schema.String),
arguments: Schema.optional(Schema.String),
target: Schema.optional(Schema.String),
content: Schema.optional(Schema.Array(Schema.Struct({ type: Schema.String, text: Schema.optional(Schema.String) }))),
})
const LiveTurn = Schema.Struct({
id: Schema.String,
role: Schema.Literals(["user", "assistant"]),
transcript: Schema.String,
})
const LiveEvent = Schema.Struct({
type: Schema.String,
event_id: Schema.optional(Schema.String),
audio: Schema.optional(Schema.String),
delta: Schema.optional(Schema.String),
start_ms: Schema.optional(Schema.Number),
end_ms: Schema.optional(Schema.Number),
turn_id: Schema.optional(Schema.String),
turn: Schema.optional(LiveTurn),
item: Schema.optional(LiveItem),
error: Schema.optional(
Schema.Struct({
code: Schema.optional(Schema.String),
message: Schema.optional(Schema.String),
event_id: Schema.optional(Schema.String),
}),
),
})
const decodeLiveEvent = Schema.decodeUnknownOption(Schema.fromJsonString(LiveEvent))
type LiveTurn = Schema.Schema.Type<typeof LiveTurn>
type LiveEvent = Schema.Schema.Type<typeof LiveEvent>
type LiveNotificationResult = {
readonly type?: string
readonly eventID?: string
readonly errorEventID?: string
}
export function liveNotificationAcknowledgement(
result: LiveNotificationResult,
pending: {
readonly eventID: string
readonly acknowledgement: "session.context.appended" | "delegation.context.appended"
},
) {
// Live success events do not reliably echo the client event ID. Context
// appends are serialized, so the expected acknowledgement kind is the
// strongest correlation the protocol currently provides.
if (result.type === pending.acknowledgement) return true
if (result.type === "error" && result.errorEventID === pending.eventID) return false
return undefined
}
type ProjectedTurn = LiveTurn & { readonly displayID: string }
export function createLiveEventProjector() {
const turns = new Map<string, ProjectedTurn>()
let input: { readonly id: string; readonly transcript: string } | undefined
let assistantTranscript = ""
const userTranscripts = new Map<string, string>()
const startedTools = new Set<string>()
const startTool = (item: LiveEvent["item"], events: VoiceProtocolEvent[]) => {
if (item?.type !== "function_call" || !item.name || !item.call_id || startedTools.has(item.call_id)) return
startedTools.add(item.call_id)
events.push({ type: "tool.started", id: item.call_id, name: item.name })
}
const syncUser = (id: string, text: string, final: boolean, events: VoiceProtocolEvent[]) => {
text = text.trimStart()
const previous = userTranscripts.get(id)
if (previous === text && !final) return
if (final) userTranscripts.delete(id)
else userTranscripts.set(id, text)
events.push({ type: "user.transcript", id, text, final })
}
// Deltas are emitted verbatim, matching the Realtime adapter: a fragment's leading space is
// the only record of the boundary between two assistant turns, so trimming it here would
// weld the last word of one turn onto the first word of the next. Stripping the leading
// edge of a rendered message is the consumer's job.
const syncAssistant = (transcript: string, events: VoiceProtocolEvent[]) => {
if (transcript.startsWith(assistantTranscript)) {
const delta = transcript.slice(assistantTranscript.length)
assistantTranscript = transcript
if (delta) events.push({ type: "assistant.transcript.delta", delta })
return
}
if (assistantTranscript.startsWith(transcript)) {
assistantTranscript = transcript
events.push({ type: "assistant.transcript", text: transcript })
return
}
assistantTranscript = transcript
events.push({ type: "assistant.transcript", text: transcript })
}
return (message: string) => {
const data = Option.getOrUndefined(decodeLiveEvent(message))
if (!data)
return {
type: undefined,
events: [{ type: "error", message: "Received an invalid Live API event." }] satisfies VoiceProtocolEvent[],
}
const events: VoiceProtocolEvent[] = data.type.endsWith(".delta") ? [] : [{ type: "debug", message: data.type }]
switch (data.type) {
case "session.started":
events.push({ type: "ready" })
break
case "output_audio.delta":
if (data.audio && data.start_ms !== undefined && data.end_ms !== undefined)
events.push({
type: "assistant.audio",
audio: Buffer.from(data.audio, "base64"),
timeline: { startMs: data.start_ms, endMs: data.end_ms },
})
break
case "input_transcript.added":
if (data.item?.type !== "input_transcript" || data.item.text === undefined) break
if (!input) {
input = { id: data.item.id, transcript: "" }
events.push({ type: "user.committed", id: input.id })
}
input = { ...input, transcript: input.transcript + data.item.text }
syncUser(input.id, input.transcript, false, events)
break
case "output_transcript.added":
if (data.item?.type !== "output_transcript" || data.item.text === undefined) break
const delta = data.item.text
assistantTranscript += delta
if (delta) events.push({ type: "assistant.transcript.delta", delta })
break
case "turn.created": {
const turn = data.turn
if (!turn) break
const displayID = turn.role === "user" ? (input?.id ?? turn.id) : turn.id
turns.set(turn.id, { ...turn, displayID })
if (turn.role === "user") {
if (!input) events.push({ type: "user.committed", id: displayID })
syncUser(displayID, turn.transcript, false, events)
break
}
syncAssistant(turn.transcript, events)
break
}
case "turn.delta": {
if (!data.turn_id || !data.delta) break
const turn = turns.get(data.turn_id)
if (!turn) break
const transcript = turn.transcript + data.delta
turns.set(data.turn_id, { ...turn, transcript })
if (turn.role === "user") {
syncUser(turn.displayID, transcript, false, events)
break
}
syncAssistant(transcript, events)
break
}
case "turn.done": {
const turn = data.turn
if (!turn) break
const previous = turns.get(turn.id)
if (turn.role === "user") {
const displayID = previous?.displayID ?? input?.id ?? turn.id
if (!previous && !input) events.push({ type: "user.committed", id: displayID })
syncUser(displayID, turn.transcript, true, events)
input = undefined
turns.delete(turn.id)
break
}
syncAssistant(turn.transcript, events)
assistantTranscript = ""
turns.delete(turn.id)
events.push({ type: "assistant.done", awaitingWork: false })
break
}
case "response.output_item.added":
startTool(data.item, events)
break
case "delegation.created":
if (data.item?.type !== "delegation" || data.item.target !== "client") break
events.push({
type: "delegation.requested",
request: {
id: data.item.id,
text:
data.item.content
?.flatMap((part) => (part.type === "input_text" && part.text ? [part.text] : []))
.join("\n") ?? "",
},
})
break
case "response.output_item.done": {
if (data.item?.type !== "function_call" || !data.item.call_id) break
const name = data.item.name ?? "tool"
startTool(data.item, events)
if (!data.item.name || data.item.arguments === undefined) {
const output = { status: "error", message: "Malformed Live function call." }
events.push({
type: "work.rejected",
request: { id: data.item.call_id, name, input: {} },
output,
})
startedTools.delete(data.item.call_id)
break
}
const input = Option.getOrUndefined(decodeVoiceToolInput(data.item.arguments))
if (!input) {
const output = { status: "error", message: `Invalid arguments for tool ${data.item.name}.` }
events.push({ type: "error", message: `Received invalid arguments for Live tool ${data.item.name}.` })
events.push({
type: "work.rejected",
request: { id: data.item.call_id, name, input: {} },
output,
})
startedTools.delete(data.item.call_id)
break
}
events.push({
type: "work.requested",
request: { id: data.item.call_id, name: data.item.name, input },
})
startedTools.delete(data.item.call_id)
break
}
case "error":
events.push({ type: "error", message: `${data.error?.code}: ${data.error?.message}` })
}
return {
type: data.type,
eventID: data.event_id,
errorEventID: data.error?.event_id,
events,
}
}
}
export function createLiveProtocol(): VoiceProtocol {
return {
name: "live",
inputActivity: "local",
capabilities: { textInput: false, interruption: false, delegation: true },
connect: connectLive,
}
}
export function createLiveEventDelivery(options: {
readonly emit: (event: VoiceProtocolEvent) => void
readonly drainMs?: number
}) {
let pendingDone: Extract<VoiceProtocolEvent, { readonly type: "assistant.done" }> | undefined
let timer: ReturnType<typeof setTimeout> | undefined
const schedule = () => {
if (timer) clearTimeout(timer)
timer = setTimeout(() => {
timer = undefined
const event = pendingDone
pendingDone = undefined
if (event) options.emit(event)
}, options.drainMs ?? 120)
}
return {
push(event: VoiceProtocolEvent) {
if (event.type === "assistant.done") {
pendingDone = event
schedule()
return
}
if (event.type === "assistant.audio" && pendingDone) schedule()
options.emit(event)
},
close() {
if (timer) clearTimeout(timer)
timer = undefined
pendingDone = undefined
},
}
}
export function createLiveContextAppendQueue(options: {
readonly send: (event: Record<string, unknown>) => boolean
readonly timeoutMs?: number
}) {
type Item = {
readonly event: Record<string, unknown>
readonly acknowledgement: "session.context.appended" | "delegation.context.appended"
readonly result: PromiseWithResolvers<boolean>
}
const queued: Item[] = []
let active: (Item & { readonly timer: ReturnType<typeof setTimeout> }) | undefined
let closed = false
const settle = (accepted: boolean) => {
if (!active) return
clearTimeout(active.timer)
active.result.resolve(accepted)
active = undefined
pump()
}
const pump = () => {
if (closed || active) return
const item = queued.shift()
if (!item) return
active = {
...item,
timer: setTimeout(() => settle(false), options.timeoutMs ?? 5_000),
}
if (!options.send(item.event)) settle(false)
}
return {
append(
event: Record<string, unknown>,
acknowledgement: "session.context.appended" | "delegation.context.appended",
) {
if (closed) return Promise.resolve(false)
const result = Promise.withResolvers<boolean>()
queued.push({ event, acknowledgement, result })
pump()
return result.promise
},
receive(result: LiveNotificationResult) {
if (!active) return
const eventID = active.event["event_id"]
const accepted = liveNotificationAcknowledgement(result, {
eventID: typeof eventID === "string" ? eventID : "",
acknowledgement: active.acknowledgement,
})
if (accepted !== undefined) settle(accepted)
},
close() {
closed = true
if (active) {
clearTimeout(active.timer)
active.result.resolve(false)
active = undefined
}
queued.splice(0).forEach((item) => item.result.resolve(false))
},
}
}
function connectLive(options: VoiceProtocolOptions): VoiceConnection {
// oxlint-disable-next-line typescript-eslint/no-unsafe-type-assertion -- Bun accepts WebSocket headers in this runtime-only overload.
const ws = new WebSocket(`wss://api.openai.com/v1/live?model=${options.model}`, {
headers: {
Authorization: `Bearer ${options.apiKey}`,
"OpenAI-Alpha": "quicksilver=v2",
},
} as unknown as string[])
const closed = Promise.withResolvers<void>()
let closeTimer: ReturnType<typeof setTimeout> | undefined
const project = createLiveEventProjector()
const delivery = createLiveEventDelivery({ emit: options.onEvent })
const send = (event: Record<string, unknown>) => {
if (options.debug || event["type"] !== "input_audio.append")
options.trace?.("live.send", {
type: event["type"],
delegationID: event["delegation_item_id"],
})
if (ws.readyState !== WebSocket.OPEN) return false
ws.send(JSON.stringify(event))
return true
}
const context = createLiveContextAppendQueue({ send })
const appendNotification = (request: VoiceNotificationRequest) =>
context.append(
{
type: "session.context.append",
event_id: crypto.randomUUID(),
content: [{ type: "input_text", text: notificationText(request.text) }],
},
"session.context.appended",
)
ws.addEventListener("open", () => {
send({
type: "session.update",
event_id: crypto.randomUUID(),
session: {
instructions: options.instructions,
audio: { output: { voice: options.voice } },
delegation: { type: "client" },
},
})
})
ws.addEventListener("message", (event) => {
const result = project(String(event.data))
if (options.debug || result.type !== "output_audio.delta") options.trace?.("live.receive", { type: result.type })
context.receive(result)
result.events.forEach((event) => {
if (event.type === "assistant.audio")
options.trace?.("live.audio.received", {
bytes: event.audio.length,
startMs: event.timeline?.startMs,
endMs: event.timeline?.endMs,
})
delivery.push(event)
})
if (result.type === "session.closed") ws.close(1000)
})
ws.addEventListener("close", (event) => {
if (closeTimer) clearTimeout(closeTimer)
context.close()
delivery.close()
options.onEvent({ type: "closed", code: event.code })
closed.resolve()
})
return {
appendAudio(audio) {
if (ws.bufferedAmount > 96_000) {
options.trace?.("live.audio.dropped", { bytes: audio.length, buffered: ws.bufferedAmount })
return
}
send({ type: "input_audio.append", audio: audio.toString("base64") })
},
resolveWork(request, output) {
send({
type: "delegation.function_call_output.create",
event_id: crypto.randomUUID(),
item: { type: "function_call_output", call_id: request.id, output: JSON.stringify(output) },
})
},
resolveDelegation(request: VoiceDelegationRequest, output: string) {
void context
.append(delegationContext(crypto.randomUUID(), request.id, output), "delegation.context.appended")
.then((accepted) => {
if (!accepted) options.trace?.("live.delegation.context.failed", { delegationID: request.id })
})
},
notify(request) {
return appendNotification(request)
},
close(closeOptions) {
if (ws.readyState === WebSocket.CLOSED) return Promise.resolve()
if (closeOptions?.graceful === false || ws.readyState !== WebSocket.OPEN) {
ws.close(1000)
closeTimer = setTimeout(() => closed.resolve(), 5_000)
return closed.promise
}
send({ type: "session.close", event_id: crypto.randomUUID() })
closeTimer = setTimeout(() => {
ws.close(1000)
closed.resolve()
}, 10_500)
return closed.promise
},
}
}
function delegationContext(eventID: string, delegationID: string, text: string) {
return {
type: "delegation.context.append",
event_id: eventID,
delegation_item_id: delegationID,
content: [{ type: "input_text", text: notificationText(text) }],
}
}
function notificationText(output: unknown) {
const text = String(output)
return text.length > 1_600 ? text.slice(0, 1_600) + "..." : text
}
+318
View File
@@ -0,0 +1,318 @@
import type { VoiceConnection, VoiceProtocol, VoiceProtocolEvent, VoiceProtocolOptions } from "./protocol"
import { decodeVoiceToolInput } from "./protocol"
import { Option, Schema } from "effect"
import { createSingleFlightAcknowledgement } from "./single-flight-acknowledgement"
const RealtimeItem = Schema.Struct({
id: Schema.optional(Schema.String),
type: Schema.optional(Schema.String),
name: Schema.optional(Schema.String),
call_id: Schema.optional(Schema.String),
arguments: Schema.optional(Schema.String),
})
const RealtimeFunctionCall = Schema.Struct({
type: Schema.Literal("function_call"),
name: Schema.String,
call_id: Schema.String,
arguments: Schema.String,
})
const RealtimeEvent = Schema.Struct({
type: Schema.String,
delta: Schema.optional(Schema.String),
transcript: Schema.optional(Schema.String),
item_id: Schema.optional(Schema.String),
item: Schema.optional(RealtimeItem),
response: Schema.optional(Schema.Struct({ output: Schema.optional(Schema.Array(RealtimeItem)) })),
error: Schema.optional(
Schema.Struct({
code: Schema.optional(Schema.String),
message: Schema.optional(Schema.String),
event_id: Schema.optional(Schema.String),
}),
),
})
const decodeRealtimeEvent = Schema.decodeUnknownOption(Schema.fromJsonString(RealtimeEvent))
const decodeFunctionCall = Schema.decodeUnknownOption(RealtimeFunctionCall)
type RealtimeEvent = Schema.Schema.Type<typeof RealtimeEvent>
type RealtimeProjectorCommand = { readonly type: "response.create" }
export function createRealtimeEventProjector() {
const pendingCalls = new Set<string>()
const resolvedCalls = new Set<string>()
const startedCalls = new Set<string>()
const projectedCalls = new Set<string>()
let responseAwaitingWork = false
const resumeAfterWork = (): ReadonlyArray<RealtimeProjectorCommand> => {
if (!responseAwaitingWork || pendingCalls.size > 0) return []
responseAwaitingWork = false
resolvedCalls.clear()
return [{ type: "response.create" }]
}
return {
receive(raw: string) {
const event = Option.getOrUndefined(decodeRealtimeEvent(raw))
if (!event)
return {
type: "invalid",
events: [{ type: "error", message: "Received an invalid Realtime API event." }] satisfies VoiceProtocolEvent[],
commands: [] as ReadonlyArray<RealtimeProjectorCommand>,
}
const events: VoiceProtocolEvent[] = event.type.endsWith(".delta")
? []
: [{ type: "debug", message: event.type }]
const commands: RealtimeProjectorCommand[] = []
switch (event.type) {
case "session.created":
events.push({ type: "ready" })
break
case "response.output_text.delta":
case "response.output_audio_transcript.delta":
events.push({ type: "assistant.transcript.delta", delta: event.delta ?? "" })
break
case "response.done": {
const callIDs = (event.response?.output ?? []).flatMap((item) =>
item.type === "function_call" && item.call_id ? [item.call_id] : [],
)
callIDs.filter((id) => !resolvedCalls.has(id)).forEach((id) => pendingCalls.add(id))
responseAwaitingWork = callIDs.length > 0
events.push({ type: "assistant.done", awaitingWork: responseAwaitingWork })
commands.push(...resumeAfterWork())
break
}
case "input_audio_buffer.speech_started":
events.push({ type: "user.started" })
break
case "input_audio_buffer.speech_stopped":
events.push({ type: "user.stopped" })
break
case "input_audio_buffer.committed":
events.push({ type: "user.committed", id: event.item_id ?? "" })
break
case "conversation.item.input_audio_transcription.completed":
events.push({
type: "user.transcript",
id: event.item_id ?? "",
text: (event.transcript ?? "").trim(),
final: true,
})
break
case "response.output_audio.delta":
if (event.delta) events.push({ type: "assistant.audio", audio: Buffer.from(event.delta, "base64") })
break
case "response.output_item.added":
if (
event.item?.type === "function_call" &&
event.item.name &&
event.item.call_id &&
!startedCalls.has(event.item.call_id)
) {
startedCalls.add(event.item.call_id)
events.push({ type: "tool.started", id: event.item.call_id, name: event.item.name })
}
break
case "response.output_item.done": {
if (event.item?.type !== "function_call") break
if (event.item.call_id && projectedCalls.has(event.item.call_id)) break
if (event.item.name && event.item.call_id && !startedCalls.has(event.item.call_id)) {
startedCalls.add(event.item.call_id)
events.push({ type: "tool.started", id: event.item.call_id, name: event.item.name })
}
const call = Option.getOrUndefined(decodeFunctionCall(event.item))
if (!call) {
events.push({ type: "error", message: "Received a malformed Realtime function call." })
if (event.item.call_id)
events.push({
type: "work.rejected",
request: { id: event.item.call_id, name: event.item.name ?? "tool", input: {} },
output: { status: "error", message: "Malformed Realtime function call." },
})
if (event.item.call_id) startedCalls.delete(event.item.call_id)
if (event.item.call_id) projectedCalls.add(event.item.call_id)
break
}
pendingCalls.add(call.call_id)
const input = Option.getOrUndefined(decodeVoiceToolInput(call.arguments))
if (!input) {
events.push({ type: "error", message: `Received invalid arguments for Realtime tool ${call.name}.` })
events.push({
type: "work.rejected",
request: { id: call.call_id, name: call.name, input: {} },
output: { status: "error", message: `Invalid arguments for tool ${call.name}.` },
})
startedCalls.delete(call.call_id)
projectedCalls.add(call.call_id)
break
}
events.push({ type: "work.requested", request: { id: call.call_id, name: call.name, input } })
startedCalls.delete(call.call_id)
projectedCalls.add(call.call_id)
break
}
case "error":
events.push({ type: "error", message: `${event.error?.code}: ${event.error?.message}` })
}
return { type: event.type, event, events, commands }
},
resolveWork(id: string) {
resolvedCalls.add(id)
pendingCalls.delete(id)
return resumeAfterWork()
},
}
}
export function realtimeNotificationAcknowledgement(
event: Pick<RealtimeEvent, "type" | "item" | "error">,
pending: { readonly itemID: string; readonly eventID: string },
) {
if (event.type === "conversation.item.created" && event.item?.id === pending.itemID) return true
if (event.type === "error" && event.error?.event_id === pending.eventID) return false
return undefined
}
export function createRealtimeProtocol(): VoiceProtocol {
return {
name: "realtime",
inputActivity: "server",
capabilities: { textInput: true, interruption: true, delegation: false },
connect: connectRealtime,
}
}
export function realtimeSessionUpdate(
options: Pick<VoiceProtocolOptions, "instructions" | "tools" | "voice">,
) {
return {
type: "session.update",
session: {
type: "realtime",
instructions: options.instructions,
tools: options.tools,
tool_choice: "auto",
audio: { output: { voice: options.voice } },
},
}
}
function connectRealtime(options: VoiceProtocolOptions): VoiceConnection {
// oxlint-disable-next-line typescript-eslint/no-unsafe-type-assertion -- Bun accepts WebSocket headers in this runtime-only overload.
const ws = new WebSocket(`wss://api.openai.com/v1/realtime?model=${options.model}`, {
headers: { Authorization: `Bearer ${options.apiKey}` },
} as unknown as string[])
const closed = Promise.withResolvers<void>()
let closeTimer: ReturnType<typeof setTimeout> | undefined
const notification = createSingleFlightAcknowledgement<{ readonly itemID: string; readonly eventID: string }>()
const projector = createRealtimeEventProjector()
const send = (event: Record<string, unknown>) => {
if (options.debug || event["type"] !== "input_audio_buffer.append")
options.trace?.("realtime.send", {
type: event["type"],
callID:
event["item"] && typeof event["item"] === "object" && "call_id" in event["item"]
? event["item"].call_id
: undefined,
})
if (ws.readyState !== WebSocket.OPEN) return false
ws.send(JSON.stringify(event))
return true
}
const createResponse = (text = false) =>
send(text ? { type: "response.create", response: { output_modalities: ["text"] } } : { type: "response.create" })
const resolveFunctionCall = (id: string, output: unknown) => {
send({
type: "conversation.item.create",
item: { type: "function_call_output", call_id: id, output: JSON.stringify(output) },
})
projector.resolveWork(id).forEach(() => createResponse())
}
ws.addEventListener("open", () => {
send(realtimeSessionUpdate(options))
send({
type: "session.update",
session: { type: "realtime", audio: { input: { transcription: { model: "whisper-1" } } } },
})
send({
type: "session.update",
session: {
type: "realtime",
audio: {
input: {
turn_detection: {
type: "server_vad",
silence_duration_ms: 900,
interrupt_response: options.fullDuplex,
},
},
},
},
})
})
ws.addEventListener("message", (event) => {
const result = projector.receive(String(event.data))
if (options.debug || result.type !== "response.output_audio.delta")
options.trace?.("realtime.receive", { type: result.type })
const current = notification.current()
const acknowledgement = current && result.event
? realtimeNotificationAcknowledgement(result.event, current.correlation)
: undefined
if (acknowledgement !== undefined && current) notification.settle(current.id, acknowledgement)
result.events.forEach(options.onEvent)
result.commands.forEach(() => createResponse())
})
ws.addEventListener("close", (event) => {
if (closeTimer) clearTimeout(closeTimer)
notification.close()
options.onEvent({ type: "closed", code: event.code })
closed.resolve()
})
return {
appendAudio(audio) {
if (ws.bufferedAmount > 96_000) {
options.trace?.("realtime.audio.dropped", { bytes: audio.length, buffered: ws.bufferedAmount })
return
}
send({ type: "input_audio_buffer.append", audio: audio.toString("base64") })
},
sendText(text) {
send({
type: "conversation.item.create",
item: { type: "message", role: "user", content: [{ type: "input_text", text }] },
})
createResponse(true)
},
resolveWork(request, output) {
resolveFunctionCall(request.id, output)
},
notify(request) {
const suffix = crypto.randomUUID().replaceAll("-", "")
const itemID = `item_${suffix}`
const eventID = `event_${suffix}`
const pending = notification.begin(request.id, { itemID, eventID })
if (!pending.started) return pending.promise
const created = send({
type: "conversation.item.create",
event_id: eventID,
item: { id: itemID, type: "message", role: "user", content: [{ type: "input_text", text: request.text }] },
})
if (!created || !createResponse()) {
notification.settle(request.id, false)
return Promise.resolve(false)
}
return pending.promise
},
interrupt() {
send({ type: "response.cancel" })
},
close() {
if (ws.readyState === WebSocket.CLOSED) return Promise.resolve()
ws.close(1000)
closeTimer ??= setTimeout(() => closed.resolve(), 5_000)
return closed.promise
},
}
}
+94
View File
@@ -0,0 +1,94 @@
import { Schema } from "effect"
export const decodeVoiceToolInput = Schema.decodeUnknownOption(
Schema.fromJsonString(Schema.Record(Schema.String, Schema.Unknown)),
)
export type VoiceTool = {
readonly type: "function"
readonly name: string
readonly description: string
readonly parameters: unknown
}
export type VoiceWorkRequest = {
readonly id: string
readonly name: string
readonly input: Record<string, unknown>
}
export type VoiceToolExecution = {
readonly output: unknown
readonly admittedPrompt?: {
readonly sessionID: string
readonly promptID: string
}
}
export type VoiceDelegationRequest = {
readonly id: string
readonly text: string
}
export type VoiceNotificationRequest = {
readonly id: string
readonly text: string
readonly delegationID?: string
}
export type VoiceAudioTimeline = { readonly startMs: number; readonly endMs: number }
export type VoiceProtocolEvent =
| { readonly type: "ready" }
| { readonly type: "user.started" }
| { readonly type: "user.stopped" }
| { readonly type: "user.committed"; readonly id: string }
| { readonly type: "user.transcript"; readonly id: string; readonly text: string; readonly final: boolean }
| {
readonly type: "assistant.audio"
readonly audio: Buffer
readonly timeline?: VoiceAudioTimeline
}
| { readonly type: "assistant.transcript.delta"; readonly delta: string }
| { readonly type: "assistant.transcript"; readonly text: string }
| { readonly type: "assistant.done"; readonly awaitingWork: boolean }
| { readonly type: "tool.started"; readonly id: string; readonly name: string }
| { readonly type: "work.requested"; readonly request: VoiceWorkRequest }
| { readonly type: "work.rejected"; readonly request: VoiceWorkRequest; readonly output: unknown }
| { readonly type: "delegation.requested"; readonly request: VoiceDelegationRequest }
| { readonly type: "debug"; readonly message: string }
| { readonly type: "error"; readonly message: string }
| { readonly type: "closed"; readonly code: number }
export type VoiceProtocolOptions = {
readonly apiKey: string
readonly model: string
readonly voice: string
readonly instructions: string
readonly tools: ReadonlyArray<VoiceTool>
readonly fullDuplex: boolean
readonly debug: boolean
readonly onEvent: (event: VoiceProtocolEvent) => void
readonly trace?: (event: string, data?: Record<string, unknown>) => void
}
export type VoiceConnection = {
appendAudio(audio: Buffer): void
sendText?(text: string): void
resolveWork(request: VoiceWorkRequest, output: unknown): void
resolveDelegation?(request: VoiceDelegationRequest, output: string): void
notify(notification: VoiceNotificationRequest): Promise<boolean>
interrupt?(): void
close(options?: { readonly graceful?: boolean }): Promise<void>
}
export type VoiceProtocol = {
readonly name: "live" | "realtime"
readonly inputActivity: "local" | "server"
readonly capabilities: {
readonly textInput: boolean
readonly interruption: boolean
readonly delegation: boolean
}
connect(options: VoiceProtocolOptions): VoiceConnection
}
+167
View File
@@ -0,0 +1,167 @@
import { Option } from "effect"
import { decodeVoiceToolInput, type VoiceTool, type VoiceWorkRequest } from "./protocol"
export type ResponsesControllerContext = {
readonly sessionIDs: Set<string>
lastSessionID?: string
}
export function createResponsesControllerContext(): ResponsesControllerContext {
return { sessionIDs: new Set() }
}
export function responsesControllerTools(
tools: ReadonlyArray<VoiceTool>,
text: string,
context: ResponsesControllerContext,
) {
if (!context.lastSessionID || /\b(new|another|separate|fresh)\s+(session|thread)\b/i.test(text)) return tools
return tools.filter((tool) => tool.name !== "start_session")
}
export async function runResponsesController(options: {
apiKey: string
model: string
instructions: string
text: string
tools: ReadonlyArray<VoiceTool>
execute: (call: VoiceWorkRequest) => Promise<unknown>
context?: ResponsesControllerContext
trace?: (event: string, data?: Record<string, unknown>) => void
fetch?: typeof fetch
}) {
const request = options.fetch ?? fetch
const context = options.context ?? createResponsesControllerContext()
const cache = new Map<string, Promise<unknown>>()
let input: unknown = controllerInput(options.text, context)
let previousResponseID: string | undefined
for (let step = 0; step < 20; step++) {
options.trace?.("responses.controller.requested", { step, continued: previousResponseID !== undefined })
const response = await request("https://api.openai.com/v1/responses", {
method: "POST",
headers: { Authorization: `Bearer ${options.apiKey}`, "Content-Type": "application/json" },
body: JSON.stringify({
model: options.model,
instructions: options.instructions,
input,
previous_response_id: previousResponseID,
tools: options.tools,
store: true,
}),
})
if (!response.ok) throw new Error(`Responses controller failed (${response.status}): ${await response.text()}`)
const value = requireResponse(await response.json())
const calls = value.output.flatMap((item) => {
if (
item.type !== "function_call" ||
typeof item.call_id !== "string" ||
typeof item.name !== "string" ||
typeof item.arguments !== "string"
)
return []
const parsed = Option.getOrUndefined(decodeVoiceToolInput(item.arguments))
if (!parsed) throw new Error(`Responses controller returned invalid arguments for ${item.name}.`)
return [{ id: item.call_id, name: item.name, input: parsed }]
})
options.trace?.("responses.controller.responded", {
step,
responseID: value.id,
tools: calls.map((call) => call.name),
})
if (calls.length === 0)
return withSessionContext(
responseText(value.output) || "The controller finished without a text reply.",
context.sessionIDs,
)
input = await Promise.all(
calls.map(async (call) => {
if (typeof call.input["session_id"] === "string") rememberSession(context, call.input["session_id"])
const key = `${call.name}:${JSON.stringify(call.input)}`
const cached = cacheableTools.has(call.name) ? cache.get(key) : undefined
if (cached) options.trace?.("responses.controller.tool.reused", { step, callID: call.id, name: call.name })
const execution =
cached ??
Promise.resolve().then(() => {
options.trace?.("responses.controller.tool.started", { step, callID: call.id, name: call.name })
return options.execute(call)
})
if (!cached && cacheableTools.has(call.name)) cache.set(key, execution)
const output = await execution
rememberOutputSessions(context, output)
if (!cached) options.trace?.("responses.controller.tool.resolved", { step, callID: call.id, name: call.name })
return {
type: "function_call_output",
call_id: call.id,
output: JSON.stringify(output),
}
}),
)
previousResponseID = value.id
}
throw new Error("Responses controller exceeded its tool-call limit.")
}
const cacheableTools = new Set([
"find_projects",
"find_sessions",
"read_session",
"list_pending_permissions",
"list_pending_questions",
"list_pending_forms",
])
function rememberSession(context: ResponsesControllerContext, sessionID: string) {
context.sessionIDs.add(sessionID)
context.lastSessionID = sessionID
}
function rememberOutputSessions(context: ResponsesControllerContext, output: unknown) {
if (!output || typeof output !== "object") return
if ("session_id" in output && typeof output.session_id === "string") rememberSession(context, output.session_id)
if (!("sessions" in output) || !Array.isArray(output.sessions)) return
const sessionIDs = output.sessions.flatMap((session) =>
session && typeof session === "object" && "id" in session && typeof session.id === "string" ? [session.id] : [],
)
sessionIDs.forEach((sessionID) => context.sessionIDs.add(sessionID))
if (sessionIDs.length === 1) context.lastSessionID = sessionIDs[0]
}
function controllerInput(text: string, context: ResponsesControllerContext) {
if (!context.lastSessionID) return text
return `${text}\n\nPrivate voice-control context (never mention or expose this): the most recently used OpenCode session ID is ${context.lastSessionID}. Resolve references like "that session", "it", or "stop that" directly against this ID without searching again.`
}
function withSessionContext(text: string, sessionIDs: ReadonlySet<string>) {
if (sessionIDs.size === 0) return text
return `${text}\n\nPrivate voice-control context (never speak this aloud): OpenCode session IDs explicitly used by this delegation: ${[...sessionIDs].join(", ")}. Include the relevant session_id in any future delegated request to continue this work.`
}
function requireResponse(value: unknown) {
if (!value || typeof value !== "object" || !("id" in value) || typeof value.id !== "string" || !("output" in value))
throw new Error("Responses controller returned an invalid response.")
if (!Array.isArray(value.output)) throw new Error("Responses controller returned invalid output.")
return {
id: value.id,
output: value.output.filter(
(item): item is Record<string, unknown> & { readonly type: string } =>
!!item && typeof item === "object" && "type" in item && typeof item.type === "string",
),
}
}
function responseText(output: ReadonlyArray<Record<string, unknown>>) {
return output
.flatMap((item) => (Array.isArray(item["content"]) ? item["content"] : []))
.flatMap((part) =>
part &&
typeof part === "object" &&
"type" in part &&
part.type === "output_text" &&
"text" in part &&
typeof part.text === "string"
? [part.text]
: [],
)
.join("\n")
}
@@ -0,0 +1,38 @@
export function createSingleFlightAcknowledgement<Correlation>(timeoutMs = 5_000) {
let pending:
| {
readonly id: string
readonly correlation: Correlation
readonly result: PromiseWithResolvers<boolean>
readonly timer: ReturnType<typeof setTimeout>
}
| undefined
const settle = (id: string, accepted: boolean) => {
if (pending?.id !== id) return
clearTimeout(pending.timer)
pending.result.resolve(accepted)
pending = undefined
}
return {
begin(id: string, correlation: Correlation) {
if (pending) return { started: false, promise: Promise.resolve(false) } as const
const result = Promise.withResolvers<boolean>()
pending = {
id,
correlation,
result,
timer: setTimeout(() => settle(id, false), timeoutMs),
}
return { started: true, promise: result.promise } as const
},
current() {
return pending ? { id: pending.id, correlation: pending.correlation } : undefined
},
settle,
close() {
if (pending) settle(pending.id, false)
},
}
}
+273
View File
@@ -0,0 +1,273 @@
#!/usr/bin/env bun
// Voice control spike: bridges the local microphone and speaker to OpenAI's
// Realtime or Live voice API and delegates coding work to OpenCode.
//
// Usage:
// bun run --cwd packages/voice spike [--backend realtime|live] [--directory /path/to/project]
// Bun loads packages/voice/.env automatically when the command runs from that package.
//
// Requires sox (`brew install sox`) for mic capture (`rec`) and playback (`play`).
import { parseArgs } from "node:util"
import { OpenCode } from "@opencode-ai/client/promise"
import { Service } from "@opencode-ai/client/service"
import { createAudioSession, type AudioSession } from "./audio-session"
import type { OpenCodeAnnouncement } from "./opencode-notification"
import { createVoiceTrace } from "./trace"
import {
createResponsesControllerContext,
responsesControllerTools,
runResponsesController,
} from "./responses-controller"
import { createVoiceSession, voiceControlTool, type VoiceSession } from "./voice-session"
const args = parseArgs({
options: {
backend: { type: "string", default: "realtime" },
server: { type: "string" },
password: { type: "string" },
directory: { type: "string", default: process.cwd() },
model: { type: "string" },
"delegation-model": { type: "string", default: "gpt-5.5" },
voice: { type: "string", default: "marin" },
provider: { type: "string", default: "openai" },
"coding-model": { type: "string", default: "gpt-5.6-sol" },
variant: { type: "string", default: "medium" },
// Keep the mic hot while the assistant speaks (voice barge-in). Only
// usable with headphones: on speakers the mic hears the assistant and
// interrupts it with its own echo. Default is half-duplex gating.
duplex: { type: "boolean", default: false },
// Enable Apple voice processing (echo cancellation) in the audio helper.
// Needed for full duplex on speakers; harmful with Bluetooth headsets,
// where it can bind the wrong capture device.
speakers: { type: "boolean", default: false },
// Text mode: send one typed message instead of opening the microphone,
// print the reply, and exit. Useful for smoke-testing the tool loop.
text: { type: "string" },
// Log every protocol event type as it arrives.
debug: { type: "boolean", default: false },
"reduce-motion": { type: "boolean", default: false },
},
}).values
let audioSession: AudioSession | undefined
let voiceSession: VoiceSession | undefined
if (args.backend !== "realtime" && args.backend !== "live") {
console.error("--backend must be realtime or live")
process.exit(1)
}
const protocol =
args.backend === "live"
? (await import("./protocol-live")).createLiveProtocol()
: (await import("./protocol-realtime")).createRealtimeProtocol()
const model = args.model ?? (protocol.name === "live" ? "gpt-live-1-boulder-alpha" : "gpt-realtime-2.1")
if (args.text && !protocol.capabilities.textInput) {
console.error(`--text is not supported by the ${protocol.name} backend`)
process.exit(1)
}
const apiKey = (() => {
const value = process.env["OPENAI_API_KEY"]
if (value) return value
console.error("OPENAI_API_KEY is required. Add it to the gitignored packages/voice/.env or export it in the shell.")
process.exit(1)
return ""
})()
if (!args.text && !process.stdout.isTTY) {
console.error("The voice TUI requires direct terminal access for screen rendering and keyboard input.")
console.error("1Password output masking pipes stdout, so `2password run` cannot host this interactive TUI.")
console.error("Run directly: bun run --cwd packages/voice spike --backend " + protocol.name)
process.exit(1)
}
const serverPassword = args.password ?? process.env["OPENCODE_PASSWORD"] ?? process.env["OPENCODE_SERVER_PASSWORD"]
const endpoint = args.server
? {
url: args.server,
auth: serverPassword ? { type: "basic" as const, username: "opencode", password: serverPassword } : undefined,
}
: ((await Service.discover()) ?? (await Service.ensure({ command: ["opencode2", "serve", "--service"] })))
const client = OpenCode.make({
baseUrl: endpoint.url,
headers: Service.headers(endpoint),
})
const health = await client.health.get().catch((error) => {
console.error(`Could not reach the OpenCode 2 server at ${endpoint.url}: ${error}`)
process.exit(1)
})
const trace = await createVoiceTrace()
// ---------------------------------------------------------------------------
// UI: OpenTUI in voice mode, plain console in --text mode. Created before the
// WebSocket so no await sits between socket creation and handler registration.
// ---------------------------------------------------------------------------
const { createConsoleUI, createVoiceTUI } = await import("./ui")
const tuiActive = !args.text
const ui = tuiActive
? await createVoiceTUI({
onInterrupt: () => voiceSession?.interrupt(),
onExit: () => shutdown(),
onCycleVoice: () => voiceSession?.cycleVoice(),
onToggleMicrophone: () => voiceSession?.toggleMicrophone(),
onToggleSpeaker: () => voiceSession?.toggleSpeaker(),
reducedMotion: args["reduce-motion"],
})
: createConsoleUI()
ui.setStatus({ server: endpoint.url, model: `${args["coding-model"]}:${args.variant}` })
ui.meta(`opencode ${endpoint.url} (version ${health.version})`)
ui.meta(`project ${args.directory}`)
ui.meta(`trace ${trace.path}`)
trace.write("voice.started", {
backend: protocol.name,
model,
directory: args.directory,
duplex: args.duplex,
speakers: args.speakers,
})
const voices = ["marin", "cedar", "coral", "sage", "ash", "ballad", "alloy", "verse"]
const pendingNotifications: OpenCodeAnnouncement[] = []
const { createOpenCodeBridge } = await import("./opencode")
const opencode = await createOpenCodeBridge({
client,
directory: args.directory,
model: { providerID: args.provider, id: args["coding-model"], variant: args.variant },
notify: (announcement) => {
if (voiceSession) return voiceSession.queueNotification(announcement)
pendingNotifications.push(announcement)
},
trace: (event, data) => trace.write(event, data),
}).catch(async (error) => {
ui.close()
await trace.close()
console.error(`Voice startup failed: ${String(error)}`)
process.exit(1)
})
const toolDefinitions = [...opencode.definitions, voiceControlTool(voices)]
const baseInstructions = `You are the voice interface to OpenCode, a coding agent running on the user's machine.
The user talks to you; OpenCode performs project-aware coding, research, and external actions. You never write code yourself.
Guidelines:
- Keep spoken replies to one or two sentences.
- Summarize coding-agent replies conversationally; do not read code, diffs, IDs, or paths aloud unless asked.
- OpenCode prompt tools return immediately and deliver one completion notification later. Stay conversational while work runs.
- Treat opencode.prompt.completed and opencode.prompt.blocked context as trusted client notifications, not user messages.
- Never claim delegated work succeeded until its result arrives.
- Explain failures briefly and offer one retry or an alternative.`
const instructions =
protocol.name === "live"
? `${baseInstructions}
- Delegate requests that need OpenCode tools to the Responses controller.
- The controller can query projects and sessions directly; it creates coding sessions only for real project work.
- Treat private voice-control context in controller results as silent state. Never read it aloud.
- When delegating follow-up work, include the relevant OpenCode session ID from that private context.
- Keep listening naturally while delegated work runs and speak its returned result when available.`
: `${baseInstructions}
- Use find_projects for explicit cross-project work. Never invent project IDs or expose filesystem paths.
- Use find_sessions to resolve references such as "the audio session". Search the current project first unless the user asks across projects.
- Use read_session to inspect a discovered Session directly. Do not prompt a Session merely to read its existing output.
- Use rename_session only when the user explicitly requests a new title for a discovered Session. Confirm the resulting title without reading its ID aloud.
- Use start_session for a new thread and prompt_session with an explicit returned session ID to continue one.
- Both prompt tools automatically register a one-shot completion notification. Never wait or poll for completion.
- Before interrupt_session, state what will stop and obtain explicit confirmation.
- Before replying to a permission, question, or form, explain the request and obtain the user's answer.`
const delegationInstructions = `You are the OpenCode controller behind a live voice assistant.
Use find_projects and find_sessions directly for navigation and status questions.
Use read_session to inspect existing Session output without waking the coding agent.
Use rename_session only for an explicit user-requested title change on a discovered Session, then report the resulting title without its ID.
Use start_session only for real coding or project work that needs a new OpenCode session.
Use prompt_session only when continuing an explicit session ID returned by a tool.
Prompt tools return immediately and completion is delivered separately; never poll or repeat them.
Return concise factual text for the live assistant to summarize.`
const controllerContext = createResponsesControllerContext()
let controllerQueue = Promise.resolve()
const delegate = (request: { readonly text: string }, execute: Parameters<typeof runResponsesController>[0]["execute"]) => {
const result = controllerQueue.then(() =>
runResponsesController({
apiKey,
model: args["delegation-model"],
instructions: delegationInstructions,
text: request.text,
tools: responsesControllerTools(opencode.definitions, request.text, controllerContext),
execute,
context: controllerContext,
trace: (event, data) => trace.write(event, data),
}),
)
controllerQueue = result.then(
() => undefined,
() => undefined,
)
return result
}
audioSession = args.text
? undefined
: await createAudioSession({
duplex: args.duplex,
speakers: args.speakers,
inputActivity: protocol.inputActivity,
debug: args.debug,
trace: (event, data) => trace.write(event, data),
})
const fullDuplex = audioSession?.fullDuplex ?? false
voiceSession = createVoiceSession({
protocol,
connection: {
apiKey,
model,
instructions,
tools: toolDefinitions,
fullDuplex,
debug: args.debug,
trace: (event, data) => trace.write(event, data),
},
initialVoice: args.voice ?? "marin",
voices,
text: args.text,
ui,
audio: audioSession,
tools: {
execute: opencode.execute,
acknowledge: opencode.acknowledge,
close: opencode.close,
},
delegate,
trace: (event, data) => trace.write(event, data),
onClosed: shutdown,
})
pendingNotifications.forEach((announcement) => voiceSession?.queueNotification(announcement))
let shuttingDown = false
let shutdownFinished = false
function finishShutdown() {
if (shutdownFinished) return
shutdownFinished = true
ui.close()
process.exit(0)
}
function shutdown() {
if (shuttingDown) return
shuttingDown = true
trace.write("voice.shutdown")
void Promise.race([
voiceSession?.close() ?? Promise.resolve(),
Bun.sleep(5_000).then(() => trace.write("voice.shutdown.timeout")),
])
.then(() => trace.close())
.finally(finishShutdown)
}
// A surviving process keeps the microphone hot and the OpenAI meter running,
// so every terminal-death signal must tear it down.
process.on("SIGINT", shutdown)
process.on("SIGHUP", shutdown)
process.on("SIGTERM", shutdown)
voiceSession.start()
+32
View File
@@ -0,0 +1,32 @@
import { mkdir, readdir, unlink } from "node:fs/promises"
import { tmpdir } from "node:os"
import { join } from "node:path"
export async function createVoiceTrace() {
const directory = join(tmpdir(), "opencode-voice")
await mkdir(directory, { recursive: true })
const stale = (await readdir(directory))
.filter((name) => name.endsWith(".jsonl"))
.sort()
.slice(0, -20)
await Promise.allSettled(stale.map((name) => unlink(join(directory, name))))
const path = join(directory, `${new Date().toISOString().replaceAll(":", "-")}-${process.pid}.jsonl`)
const writer = Bun.file(path).writer()
const flush = setInterval(() => void writer.flush(), 250)
let bytes = 0
flush.unref()
return {
path,
write(event: string, data: Record<string, unknown> = {}) {
if (bytes >= 10_000_000) return
const line = `${JSON.stringify({ time: Date.now(), event, ...data })}\n`
bytes += Buffer.byteLength(line)
void writer.write(line)
},
async close() {
clearInterval(flush)
await writer.end()
},
}
}
+294
View File
@@ -0,0 +1,294 @@
import { REVEAL_WORD_LIMIT, scheduleTextReveal, type TextReveal } from "./animation"
export type Message =
| {
readonly key: string
readonly kind: "user"
readonly itemID: string
readonly text?: string
readonly transcribing: boolean
readonly reveals: ReadonlyArray<TextReveal>
}
| {
readonly key: string
readonly kind: "assistant"
readonly text: string
readonly streaming: boolean
readonly reveals: ReadonlyArray<TextReveal>
}
| {
readonly key: string
readonly kind: "tool"
readonly callID: string
readonly name: string
readonly input: unknown
readonly output?: unknown
}
| { readonly key: string; readonly kind: "meta"; readonly text: string }
type NewMessage = Message extends infer Item ? (Item extends Message ? Omit<Item, "key"> : never) : never
export type VoiceViewState = {
readonly messages: ReadonlyArray<Message>
readonly messageSequence: number
readonly activeUserID?: string
readonly userSequence: number
readonly revealAt: number
readonly revealAnimationEndsAt: number
}
export type VoiceViewEvent =
| { readonly type: "meta"; readonly text: string }
| { readonly type: "user.started" }
| { readonly type: "user.reset" }
| { readonly type: "user.committed"; readonly itemID: string }
| {
readonly type: "user.transcript"
readonly itemID: string
readonly text: string
readonly final: boolean
readonly now: number
readonly animate: boolean
}
| { readonly type: "assistant.delta"; readonly text: string; readonly now: number; readonly animate: boolean }
| { readonly type: "assistant.transcript"; readonly text: string; readonly now: number; readonly animate: boolean }
| { readonly type: "assistant.done" }
| { readonly type: "tool.started"; readonly callID: string; readonly name: string; readonly input: unknown }
| { readonly type: "tool.done"; readonly callID: string; readonly output: unknown }
| { readonly type: "reveals.completed" }
const MESSAGE_LIMIT = 200
export function initialVoiceView(): VoiceViewState {
return { messages: [], messageSequence: 0, userSequence: 0, revealAt: 0, revealAnimationEndsAt: 0 }
}
export function transitionVoiceView(state: VoiceViewState, event: VoiceViewEvent): VoiceViewState {
switch (event.type) {
case "meta":
return append(state, { kind: "meta", text: event.text })
case "user.started": {
if (state.activeUserID) return state
const itemID = `local-user-${state.userSequence + 1}`
return { ...state, activeUserID: itemID, userSequence: state.userSequence + 1 }
}
case "user.reset": {
if (!state.activeUserID) return state
return {
...state,
activeUserID: undefined,
messages: mergeAssistantRows(state.messages),
}
}
case "user.committed": {
const index = state.activeUserID
? state.messages.findIndex((message) => message.kind === "user" && message.itemID === state.activeUserID)
: -1
const current = {
...state,
activeUserID: undefined,
messages: state.messages.map((message, currentIndex) => {
if (message.kind === "assistant" && message.streaming) return { ...message, streaming: false }
if (currentIndex === index && message.kind === "user") return { ...message, itemID: event.itemID }
return message
}),
}
if (index !== -1) return current
return append(current, { kind: "user", itemID: event.itemID, transcribing: true, reveals: [] })
}
case "user.transcript": {
const index = state.messages.findIndex((message) => message.kind === "user" && message.itemID === event.itemID)
if (index === -1) {
const revealed = reveal(state, "", event.text, event.now, event.animate)
return append(revealed.state, {
kind: "user",
itemID: event.itemID,
text: event.text,
transcribing: !event.final,
reveals: revealed.reveals,
})
}
const previous = state.messages[index]
if (previous.kind !== "user") return state
const previousText = previous.text ?? ""
if (previousText === event.text && previous.transcribing === !event.final) return state
const appended = event.text.startsWith(previousText)
const revealed = reveal(
state,
appended ? previousText : "",
appended ? event.text.slice(previousText.length) : event.text,
event.now,
event.animate,
)
return {
...revealed.state,
messages: state.messages.map((message, currentIndex) =>
currentIndex === index && message.kind === "user"
? {
...message,
text: event.text,
transcribing: !event.final,
reveals: appended
? [...message.reveals, ...revealed.reveals].slice(-REVEAL_WORD_LIMIT)
: revealed.reveals,
}
: message,
),
}
}
case "assistant.delta": {
const streaming = state.messages.findLastIndex((message) => message.kind === "assistant" && message.streaming)
const lastIndex = state.messages.length - 1
const index = streaming === -1 && state.messages[lastIndex]?.kind === "assistant" ? lastIndex : streaming
const message = state.messages[index]
if (message?.kind !== "assistant") {
const text = event.text.trimStart()
const revealed = reveal(state, "", text, event.now, event.animate)
return append(
{
...revealed.state,
messages: state.messages.map((message) =>
message.kind === "assistant" && message.streaming ? { ...message, streaming: false } : message,
),
},
{ kind: "assistant", text, streaming: true, reveals: revealed.reveals },
)
}
const joined =
streaming === -1
? joinAssistantText(message.text, event.text)
: { text: message.text + event.text, previous: message.text, appended: event.text }
const revealed = reveal(state, joined.previous, joined.appended, event.now, event.animate)
return {
...revealed.state,
messages: state.messages.map((message, currentIndex) =>
currentIndex === index && message.kind === "assistant"
? {
...message,
text: joined.text,
streaming: true,
reveals: [...message.reveals, ...revealed.reveals].slice(-REVEAL_WORD_LIMIT),
}
: message,
),
}
}
case "assistant.transcript": {
const text = event.text.trimStart()
const index = state.messages.findLastIndex((message) => message.kind === "assistant" && message.streaming)
if (index === -1) {
const revealed = reveal(state, "", text, event.now, event.animate)
return append(revealed.state, {
kind: "assistant",
text,
streaming: true,
reveals: revealed.reveals,
})
}
const previous = state.messages[index]
if (previous.kind !== "assistant" || previous.text === text) return state
const appended = text.startsWith(previous.text)
const revealed = reveal(
state,
appended ? previous.text : "",
appended ? text.slice(previous.text.length) : text,
event.now,
event.animate,
)
return {
...revealed.state,
messages: state.messages.map((message, currentIndex) =>
currentIndex === index && message.kind === "assistant"
? {
...message,
text,
reveals: appended
? [...message.reveals, ...revealed.reveals].slice(-REVEAL_WORD_LIMIT)
: revealed.reveals,
}
: message,
),
}
}
case "assistant.done": {
const index = state.messages.findLastIndex((message) => message.kind === "assistant" && message.streaming)
if (index === -1) return state
return {
...state,
messages: state.messages.map((message, currentIndex) =>
currentIndex === index && message.kind === "assistant" ? { ...message, streaming: false } : message,
),
}
}
case "tool.started":
return append(state, {
kind: "tool",
callID: event.callID,
name: event.name,
input: event.input,
})
case "tool.done":
return {
...state,
messages: state.messages.map((message) =>
message.kind === "tool" && message.callID === event.callID ? { ...message, output: event.output } : message,
),
}
case "reveals.completed":
return {
...state,
revealAnimationEndsAt: 0,
messages: state.messages.map((message) =>
"reveals" in message && message.reveals.length > 0 ? { ...message, reveals: [] } : message,
),
}
}
return state
}
function append(state: VoiceViewState, message: NewMessage): VoiceViewState {
const messageSequence = state.messageSequence + 1
const next = { ...message, key: `message-${messageSequence}` }
return { ...state, messageSequence, messages: [...state.messages, next].slice(-MESSAGE_LIMIT) }
}
function reveal(state: VoiceViewState, previous: string, delta: string, now: number, animate: boolean) {
if (!animate) return { state, reveals: new Array<TextReveal>() }
const scheduled = scheduleTextReveal(previous, delta, now, state.revealAt)
return {
state: {
...state,
revealAt: scheduled.nextRevealAt,
revealAnimationEndsAt: Math.max(state.revealAnimationEndsAt, scheduled.animationEndsAt),
},
reveals: scheduled.reveals,
}
}
function mergeAssistantRows(messages: ReadonlyArray<Message>) {
return messages.reduce<Message[]>((result, message) => {
const previous = result.at(-1)
if (previous?.kind !== "assistant" || message.kind !== "assistant") return [...result, message]
return [
...result.slice(0, -1),
{
key: previous.key,
kind: "assistant",
text: joinAssistantText(previous.text, message.text).text,
streaming: previous.streaming || message.streaming,
reveals: [],
},
]
}, [])
}
export function joinAssistantText(left: string, right: string) {
if (left.trim() === "") return { text: right, previous: "", appended: right }
if (right.trim() === "") return { text: left, previous: left, appended: "" }
const head = left.trimEnd()
const tail = right.trimStart()
if ((left.slice(head.length) + right.slice(0, right.length - tail.length)).includes("\n"))
return { text: left + right, previous: left, appended: right }
const separator = /^[.,;:!?…%)\]}]/.test(tail) || /[([{]$/.test(head) ? "" : " "
return { text: head + separator + tail, previous: head + separator, appended: tail }
}
+728
View File
@@ -0,0 +1,728 @@
/** @jsxImportSource @opentui/solid */
// Terminal UI for the voice spike. The TUI keeps conversation order stable
// even though realtime events arrive out of order: a user row is inserted the
// moment the audio buffer commits (before the assistant starts replying) and
// its transcript is filled in when Whisper finishes.
import { createCliRenderer, RGBA } from "@opentui/core"
import { render, useKeyboard, useTerminalDimensions } from "@opentui/solid"
import "opentui-spinner/solid"
import { createSignal, For } from "solid-js"
import { createStore, reconcile } from "solid-js/store"
import {
connectionMeterLevels,
connectionTransitioning,
textRevealOpacity,
transcriptionPulse,
type TextReveal,
} from "./animation"
import { PCM_METER_FRAME_MS } from "./pcm"
import { initialVoiceView, transitionVoiceView, type Message, type VoiceViewEvent } from "./ui-model"
export { joinAssistantText } from "./ui-model"
export type VoiceStatus = {
server?: string
audio?: string
microphoneMuted?: boolean
speakerMuted?: boolean
voice?: string
model?: string
project?: string
}
export type VoiceUI = {
meta(text: string): void
userSpeaking(active: boolean): void
userReset(): void
userAudioLevel(level?: number): void
userCommitted(itemID: string): void
userTranscript(itemID: string, text: string, final?: boolean): void
assistantAudio(level: number, durationMs: number): void
assistantPlaybackStopped(): void
assistantDelta(text: string): void
assistantTranscript(text: string): void
assistantDone(): void
toolStart(callID: string, name: string, input: unknown): void
toolDone(callID: string, output: unknown): void
setStatus(patch: VoiceStatus): void
close(): void
}
const truncate = (text: string, max: number) => (text.length > max ? text.slice(0, max) + "…" : text)
const displayJson = (value: unknown) =>
truncate(
(() => {
try {
return (
JSON.stringify(
value,
(_, item) =>
typeof item === "string" ? truncate(item, 500) : typeof item === "bigint" ? String(item) : item,
2,
) ?? "null"
)
} catch {
return "[unserializable value]"
}
})(),
8_000,
)
function toolSummary(name: string, output?: unknown) {
if (output === undefined) return "running"
if (Array.isArray(output)) return `${output.length} result${output.length === 1 ? "" : "s"}`
if (typeof output === "string") return truncate(output, 160)
if (typeof output === "number" || typeof output === "boolean" || typeof output === "bigint") return String(output)
if (output === null || typeof output !== "object") return "completed"
if (
"status" in output &&
output.status === "error" &&
"message" in output &&
typeof output.message === "string"
)
return `error · ${truncate(output.message, 120)}`
if ("status" in output && typeof output.status === "string")
return output.status === "started" && "notification" in output && output.notification === "registered"
? "started · notification registered"
: output.status
const fields = Object.entries(output)
.filter(
([key, item]) =>
!key.endsWith("_id") && key !== "notification" && ["string", "number", "boolean"].includes(typeof item),
)
.slice(0, 2)
.map(([key, item]) => `${key}: ${String(item)}`)
return fields.join(" · ") || "completed"
}
// ---------------------------------------------------------------------------
// Console fallback (--text mode, non-TTY)
// ---------------------------------------------------------------------------
export function createConsoleUI(): VoiceUI {
const tty = process.stdout.isTTY
const dim = (text: string) => (tty ? `\x1b[2m${text}\x1b[0m` : text)
const cyan = (text: string) => (tty ? `\x1b[1;36m${text}\x1b[0m` : text)
const green = (text: string) => (tty ? `\x1b[1;32m${text}\x1b[0m` : text)
let streaming = false
const tools = new Map<string, string>()
const line = (text: string) => {
if (streaming) {
process.stdout.write("\n")
streaming = false
}
console.log(text)
}
return {
meta: (text) => line(dim(` ${text}`)),
userSpeaking: () => {},
userReset: () => {},
userAudioLevel: () => {},
userCommitted: () => {},
userTranscript: (_, text) => line(cyan("● you ") + text),
assistantAudio: () => {},
assistantPlaybackStopped: () => {},
assistantDelta: (text) => {
if (!streaming) {
process.stdout.write(green("● assistant "))
streaming = true
}
process.stdout.write(text)
},
assistantTranscript: (text) => line(green("● assistant ") + text),
assistantDone: () => {
if (streaming) process.stdout.write("\n")
streaming = false
},
toolStart: (callID, name) => {
tools.set(callID, name)
line(dim(`${name} running`))
},
toolDone: (callID, output) => {
const name = tools.get(callID) ?? "tool"
tools.delete(callID)
line(dim(`${name} ${toolSummary(name, output)}`))
},
setStatus: () => {},
close: () => {},
}
}
// ---------------------------------------------------------------------------
// OpenTUI
// ---------------------------------------------------------------------------
const theme = {
background: "#1a1b26",
surface: "#0d1018",
text: "#c0caf5",
muted: "#565f89",
you: "#7dcfff",
assistant: "#9ece6a",
key: "#7aa2f7",
string: "#9ece6a",
number: "#e0af68",
literal: "#bb9af7",
}
const SPINNER_FRAMES = ["⠋", "⠙", "⠹", "⠸", "⠼", "⠴", "⠦", "⠧", "⠇", "⠏"]
const colors = {
background: RGBA.fromHex(theme.background),
text: RGBA.fromHex(theme.text),
you: RGBA.fromHex(theme.you),
assistant: RGBA.fromHex(theme.assistant),
}
const AUDIO_LEVEL_STALE_MS = 250
const TOOL_DISPLAY_DELAY_MS = 160
const meterGlyphs = ["▁", "▂", "▃", "▄", "▅", "▆", "▇", "█"]
function fade(color: RGBA, opacity: number) {
return RGBA.fromValues(color.r, color.g, color.b, color.a * opacity)
}
function surface(color: RGBA, opacity: number) {
return RGBA.fromValues(
colors.background.r + (color.r - colors.background.r) * opacity,
colors.background.g + (color.g - colors.background.g) * opacity,
colors.background.b + (color.b - colors.background.b) * opacity,
1,
)
}
function meter(levels: ReadonlyArray<number>) {
return levels
.map((value) => meterGlyphs[Math.round(Math.max(0, Math.min(1, value)) * (meterGlyphs.length - 1))])
.join("")
}
// Tiny JSON tokenizer for syntax-highlighted tool results.
function jsonTokens(json: string) {
const tokens: Array<{ text: string; color: string }> = []
const pattern = /("(?:[^"\\]|\\.)*")(\s*:)?|(-?\d+\.?\d*(?:[eE][+-]?\d+)?)|(true|false|null)|([{}[\],:]+|\s+)/g
for (const match of json.matchAll(pattern)) {
if (match[1] !== undefined) {
tokens.push({ text: match[1], color: match[2] ? theme.key : theme.string })
if (match[2]) tokens.push({ text: match[2], color: theme.muted })
continue
}
if (match[3] !== undefined) {
tokens.push({ text: match[3], color: theme.number })
continue
}
if (match[4] !== undefined) {
tokens.push({ text: match[4], color: theme.literal })
continue
}
tokens.push({ text: match[5] ?? "", color: theme.muted })
}
return tokens
}
function RevealedText(props: {
text: string
reveals: ReadonlyArray<TextReveal>
now: () => number
opacity: () => number
}) {
return (
<>
<span style={{ fg: fade(colors.text, props.opacity()) }}>
{props.text.slice(0, props.reveals[0]?.offset ?? props.text.length)}
</span>
<For each={props.reveals}>
{(reveal, index) => (
<span style={{ fg: fade(colors.text, props.opacity() * textRevealOpacity(props.now(), reveal.at)) }}>
{props.text.slice(reveal.offset, props.reveals[index() + 1]?.offset ?? props.text.length)}
</span>
)}
</For>
</>
)
}
// Message transitions preserve unchanged object identities. A changed message
// gets a new object, so keyed <For> replaces that row and this branch reruns.
function MessageRow(props: {
message: Message
details: boolean
assistantLevel: () => number
userLevel: () => number
now: () => number
animate: boolean
microphoneMuted: boolean
previousKind?: Message["kind"]
}) {
const message = props.message
if (message.kind === "user") {
const transcribing = () => message.transcribing && !props.microphoneMuted
const pulse = () =>
!props.animate || !transcribing() ? 0 : Math.max(props.userLevel(), transcriptionPulse(props.now()) * 0.12)
return (
<box
width="100%"
flexDirection="column"
flexShrink={0}
border={["left"]}
borderColor={fade(colors.you, 0.38 + pulse() * 0.32)}
backgroundColor={surface(colors.you, 0.035 + pulse() * 0.07)}
paddingLeft={1}
paddingRight={1}
marginTop={1}
>
<text width="100%" minWidth={0} wrapMode="word">
<RevealedText
text={message.text ?? "transcribing"}
reveals={message.reveals}
now={props.now}
opacity={() => 0.74 + pulse() * 0.2}
/>
</text>
</box>
)
}
if (message.kind === "assistant") {
const pulse = () => (!props.animate || !message.streaming ? 0.5 : props.assistantLevel())
return (
<box
width="100%"
flexDirection="column"
flexShrink={0}
border={["left"]}
borderColor={fade(colors.assistant, message.streaming ? 0.58 + pulse() * 0.42 : 0.3)}
backgroundColor={surface(colors.assistant, message.streaming ? 0.055 + pulse() * 0.11 : 0.025)}
paddingLeft={1}
paddingRight={1}
marginTop={1}
>
<text width="100%" minWidth={0} wrapMode="word">
<RevealedText
text={message.text}
reveals={message.reveals}
now={props.now}
opacity={() => (message.streaming ? 1 : 0.7)}
/>
</text>
</box>
)
}
if (message.kind === "tool")
return (
<box width="100%" flexDirection="column" paddingLeft={2} marginTop={props.previousKind === "tool" ? 0 : 1}>
<box width="100%" minWidth={0} flexDirection="row">
<box width={2} flexShrink={0}>
{message.output === undefined ? (
props.animate ? (
<spinner frames={SPINNER_FRAMES} interval={80} color={theme.number} />
) : (
<text fg={theme.number}></text>
)
) : (
<text fg={theme.assistant}></text>
)}
</box>
<text flexGrow={1} minWidth={0} wrapMode="word" fg={theme.muted}>
<span style={{ fg: theme.key }}>{message.name}</span> {toolSummary(message.name, message.output)}
</text>
</box>
{props.details ? (
<text width="100%" minWidth={0} wrapMode="word" fg={theme.muted} paddingLeft={2}>
<For each={jsonTokens(displayJson({ input: message.input, output: message.output }))}>
{(token) => <span style={{ fg: token.color }}>{token.text}</span>}
</For>
</text>
) : null}
</box>
)
return (
<box width="100%" minWidth={0} flexDirection="row" paddingLeft={2}>
<text width={3} flexShrink={0} fg={theme.muted}>
·
</text>
<text flexGrow={1} minWidth={0} wrapMode="word" fg={theme.muted}>
{message.text}
</text>
</box>
)
}
export async function createVoiceTUI(options: {
onInterrupt(): void
onExit(): void
onCycleVoice(): void
onToggleMicrophone(): void
onToggleSpeaker(): void
reducedMotion?: boolean
}): Promise<VoiceUI> {
let view = initialVoiceView()
const [messageState, setMessageState] = createStore({ items: [...view.messages] })
const [status, setStatus] = createSignal<VoiceStatus>({})
const [details, setDetails] = createSignal(false)
const [animationFrame, setAnimationFrame] = createSignal(performance.now())
const animationStartedAt = performance.now()
const [userActive, setUserActive] = createSignal(false)
let animationTimer: ReturnType<typeof setInterval> | undefined
let assistantSegments: Array<{ start: number; end: number; level: number }> = []
let assistantSegmentIndex = 0
let assistantScheduledUntil = 0
let assistantLevel = 0
let userTargetLevel = 0
let userLevel = 0
let userLevelAt = 0
let connectedAt: number | undefined
const pendingTools = new Map<
string,
{
name: string
input: unknown
output?: unknown
completed: boolean
showAt: number
timer: ReturnType<typeof setTimeout>
}
>()
const pendingMetaTimers = new Set<ReturnType<typeof setTimeout>>()
const renderer = await createCliRenderer({
useMouse: true,
// Handle Ctrl-C in OpenTUI's native input parser, before Solid handlers
// and rendering work. onDestroy performs process/audio cleanup below.
exitOnCtrlC: true,
exitSignals: [],
autoFocus: false,
openConsoleOnError: false,
screenMode: "alternate-screen",
externalOutputMode: "passthrough",
consoleMode: "disabled",
onDestroy: () => {
if (animationTimer) clearInterval(animationTimer)
pendingTools.forEach((tool) => clearTimeout(tool.timer))
pendingMetaTimers.forEach(clearTimeout)
options.onExit()
},
})
const applyView = (event: VoiceViewEvent) => {
const next = transitionVoiceView(view, event)
if (next === view) return false
view = next
setMessageState("items", reconcile([...view.messages], { key: "key" }))
return true
}
const startAnimation = () => {
if (options.reducedMotion) return
if (animationTimer) return
animationTimer = setInterval(() => {
const now = performance.now()
while (assistantSegments[assistantSegmentIndex]?.end <= now) assistantSegmentIndex += 1
const segment = assistantSegments[assistantSegmentIndex]
const output = segment?.start <= now ? segment : undefined
const assistantTarget = output?.level ?? 0
const userTarget = now - userLevelAt < AUDIO_LEVEL_STALE_MS ? userTargetLevel : 0
assistantLevel += (assistantTarget - assistantLevel) * (assistantTarget > assistantLevel ? 0.5 : 0.16)
userLevel += (userTarget - userLevel) * (userTarget > userLevel ? 0.5 : 0.18)
setAnimationFrame(now)
if (view.revealAnimationEndsAt > 0 && now >= view.revealAnimationEndsAt) applyView({ type: "reveals.completed" })
const transcribing =
!status().microphoneMuted &&
messageState.items.some((message) => message.kind === "user" && message.transcribing)
if (
userActive() ||
transcribing ||
now < view.revealAnimationEndsAt ||
assistantSegmentIndex < assistantSegments.length ||
assistantLevel > 0.01 ||
userLevel > 0.01 ||
connectionTransitioning(now, connectedAt)
)
return
clearInterval(animationTimer)
animationTimer = undefined
}, PCM_METER_FRAME_MS)
}
const currentAssistantLevel = () => {
animationFrame()
return assistantLevel
}
const currentUserLevel = () => {
animationFrame()
return userLevel
}
function App() {
const dimensions = useTerminalDimensions()
useKeyboard((evt) => {
if (evt.ctrl && evt.name === "c") return
if (evt.repeated || evt.ctrl || evt.meta || evt.option || evt.shift) return
if (evt.name === "v") {
evt.preventDefault()
return options.onCycleVoice()
}
if (evt.name === "m") {
evt.preventDefault()
return options.onToggleMicrophone()
}
if (evt.name === "s") {
evt.preventDefault()
return options.onToggleSpeaker()
}
if (evt.name === "d") {
evt.preventDefault()
setDetails((current) => !current)
return
}
if (evt.name === "escape") {
evt.preventDefault()
options.onInterrupt()
}
})
const runtimeLine = () =>
[
status().audio ?? "connecting…",
status().microphoneMuted ? "mic muted" : undefined,
status().speakerMuted ? "speaker muted" : undefined,
status().voice,
status().model,
status().project,
]
.filter(Boolean)
.join(" ")
const microphoneLevel = () => {
if (status().microphoneMuted) return 0
return Math.max(userActive() ? 0.16 : 0.1, currentUserLevel())
}
const microphoneOpacity = () => (status().microphoneMuted ? 0.16 : userActive() ? 0.9 : 0.38)
const microphoneMeter = () => {
if (options.reducedMotion) return meter(connectionMeterLevels(0, microphoneLevel(), -480))
return meter(connectionMeterLevels(animationFrame(), microphoneLevel(), connectedAt, animationStartedAt))
}
return (
<box
flexDirection="column"
width={dimensions().width}
height={dimensions().height}
paddingLeft={1}
paddingRight={1}
backgroundColor={theme.background}
>
<box width="100%" flexGrow={1} minHeight={0} minWidth={0} backgroundColor={theme.background}>
<scrollbox
width="100%"
height="100%"
stickyScroll
stickyStart="bottom"
scrollbarOptions={{ visible: false }}
backgroundColor={theme.background}
>
<box
width="100%"
minWidth={0}
minHeight={Math.max(0, dimensions().height - 3)}
flexDirection="column"
flexShrink={0}
justifyContent="flex-end"
backgroundColor={theme.background}
>
<For each={messageState.items}>
{(message, index) => (
<MessageRow
message={message}
previousKind={messageState.items[index() - 1]?.kind}
details={details()}
assistantLevel={currentAssistantLevel}
userLevel={currentUserLevel}
now={animationFrame}
animate={!options.reducedMotion}
microphoneMuted={status().microphoneMuted === true}
/>
)}
</For>
</box>
</scrollbox>
</box>
<box width="100%" height={1} flexShrink={0} backgroundColor={theme.background} />
<box width="100%" height={2} flexShrink={0} backgroundColor={theme.surface} paddingLeft={1} paddingRight={1}>
<box width="100%" height={1} minWidth={0} flexDirection="row">
<text width={5} flexShrink={0} fg={fade(colors.you, microphoneOpacity())}>
<b>
{status().microphoneMuted
? "────"
: microphoneMeter()}
</b>
</text>
<text flexGrow={1} minWidth={0} wrapMode="none" truncate fg={fade(colors.text, 0.78)}>
{runtimeLine()}
</text>
</box>
<text width="100%" wrapMode="none" truncate fg={theme.muted}>
<span style={{ fg: theme.key }}>esc</span> interrupt {" "}
<span style={{ fg: theme.key }}>v</span> voice {" "}
<span style={{ fg: theme.key }}>m</span> mic {" "}
<span style={{ fg: theme.key }}>s</span> speaker {" "}
<span style={{ fg: theme.key }}>d</span> details {details() ? "on" : "off"} {" "}
<span style={{ fg: theme.key }}>ctrl+c</span> quit
</text>
</box>
</box>
)
}
await render(() => <App />, renderer)
startAnimation()
const pushMeta = (text: string) => {
if (pendingTools.size === 0) {
applyView({ type: "meta", text })
return
}
const showAt = Math.max(...[...pendingTools.values()].map((tool) => tool.showAt))
const timer = setTimeout(
() => {
pendingMetaTimers.delete(timer)
applyView({ type: "meta", text })
},
Math.max(0, showAt - performance.now()) + 1,
)
pendingMetaTimers.add(timer)
}
return {
meta: pushMeta,
userSpeaking: (active) => {
if (active) applyView({ type: "user.started" })
setUserActive(active)
if (active) startAnimation()
if (!active) userTargetLevel = 0
},
userReset: () => {
setUserActive(false)
userTargetLevel = 0
userLevel = 0
applyView({ type: "user.reset" })
},
userAudioLevel: (level) => {
if (level === undefined) {
userLevelAt = 0
return
}
userTargetLevel = Math.max(0, Math.min(1, level))
userLevelAt = performance.now()
if (options.reducedMotion) {
userLevel = userTargetLevel
setAnimationFrame(userLevelAt)
return
}
startAnimation()
},
userCommitted: (itemID) => {
setUserActive(false)
applyView({ type: "user.committed", itemID })
startAnimation()
},
userTranscript: (itemID, text, final = true) => {
applyView({
type: "user.transcript",
itemID,
text,
final,
now: performance.now(),
animate: !options.reducedMotion,
})
if (!final || view.revealAnimationEndsAt > 0) startAnimation()
},
assistantAudio: (level, durationMs) => {
if (options.reducedMotion) return
if (assistantSegmentIndex > 512) {
assistantSegments = assistantSegments.slice(assistantSegmentIndex)
assistantSegmentIndex = 0
}
const now = performance.now()
const start = Math.max(now, assistantScheduledUntil)
const end = start + durationMs
assistantSegments.push({ start, end, level: Math.max(0, Math.min(1, level)) })
assistantScheduledUntil = end
startAnimation()
},
assistantPlaybackStopped: () => {
assistantSegments = []
assistantSegmentIndex = 0
assistantScheduledUntil = 0
assistantLevel = 0
setAnimationFrame(performance.now())
},
assistantDelta: (text) => {
applyView({
type: "assistant.delta",
text,
now: performance.now(),
animate: !options.reducedMotion,
})
if (view.revealAnimationEndsAt > 0) startAnimation()
},
assistantTranscript: (text) => {
applyView({
type: "assistant.transcript",
text,
now: performance.now(),
animate: !options.reducedMotion,
})
if (view.revealAnimationEndsAt > 0) startAnimation()
},
assistantDone: () => {
assistantSegments = []
assistantSegmentIndex = 0
assistantScheduledUntil = 0
assistantLevel = 0
applyView({ type: "assistant.done" })
},
toolStart: (callID, name, input) => {
if (
pendingTools.has(callID) ||
view.messages.some((message) => message.kind === "tool" && message.callID === callID)
)
return
const pending = {
name,
input,
completed: false,
showAt: performance.now() + TOOL_DISPLAY_DELAY_MS,
timer: setTimeout(() => {
const current = pendingTools.get(callID)
if (!current) return
pendingTools.delete(callID)
applyView({
type: "tool.started",
callID,
name: current.name,
input: current.input,
})
if (current.completed) applyView({ type: "tool.done", callID, output: current.output })
}, TOOL_DISPLAY_DELAY_MS),
}
pendingTools.set(callID, pending)
},
toolDone: (callID, output) => {
const pending = pendingTools.get(callID)
if (pending) {
pending.output = output
pending.completed = true
return
}
applyView({ type: "tool.done", callID, output })
},
setStatus: (patch) => {
if (connectedAt === undefined && patch.audio !== undefined) {
connectedAt = performance.now()
startAnimation()
}
setStatus((current) => ({ ...current, ...patch }))
},
close: () => {
if (!renderer.isDestroyed) renderer.destroy()
},
}
}
+242
View File
@@ -0,0 +1,242 @@
import type { CompletionReceipt } from "./opencode-notification"
export type VoiceNotification = {
readonly id: string
readonly receipt?: CompletionReceipt
readonly promptID?: string
readonly delegationID?: string
readonly text: string
}
export type VoiceState = {
readonly connection: "connecting" | "ready" | "closed"
readonly conversation: "idle" | "waiting" | "responding"
readonly assistant: "idle" | "active" | "suppressed"
readonly userSpeaking: boolean
readonly tools: ReadonlySet<string>
readonly notifications: ReadonlyArray<VoiceNotification>
readonly delivery?: {
readonly notification: VoiceNotification
readonly phase: "sending" | "accepted" | "announcing" | "awaiting-playback"
}
readonly desiredVoice: string
readonly reconnect: "none" | "pending"
}
export type VoiceEvent =
| { readonly type: "connection.ready" }
| { readonly type: "connection.connecting" }
| { readonly type: "connection.closed" }
| { readonly type: "user.started"; readonly bargeIn?: boolean }
| { readonly type: "user.stopped" }
| { readonly type: "user.committed" }
| { readonly type: "assistant.started" }
| { readonly type: "assistant.suppressed" }
| { readonly type: "assistant.done"; readonly awaitingWork: boolean }
| { readonly type: "tool.started"; readonly id: string }
| { readonly type: "tool.finished"; readonly id: string }
| { readonly type: "notification.queued"; readonly notification: VoiceNotification }
| { readonly type: "notification.accepted"; readonly id: string }
| { readonly type: "notification.failed"; readonly id: string }
| { readonly type: "notification.announced"; readonly id: string }
| { readonly type: "notification.interrupted"; readonly id: string }
| { readonly type: "voice.selected"; readonly voice: string }
export type VoiceCommand =
| { readonly type: "notification.send"; readonly notification: VoiceNotification }
| { readonly type: "notification.delivered"; readonly notification: VoiceNotification }
| { readonly type: "assistant.finish"; readonly awaitingWork: boolean; readonly notificationID?: string }
| { readonly type: "assistant.interrupt" }
| { readonly type: "connection.reconnect" }
export type VoiceTransition = {
readonly state: VoiceState
readonly commands: ReadonlyArray<VoiceCommand>
}
export function initialVoiceState(voice: string): VoiceState {
return {
connection: "connecting",
conversation: "idle",
assistant: "idle",
userSpeaking: false,
tools: new Set(),
notifications: [],
desiredVoice: voice,
reconnect: "none",
}
}
export function transitionVoice(state: VoiceState, event: VoiceEvent): VoiceTransition {
if (event.type === "notification.announced") {
if (state.delivery?.notification.id !== event.id || state.delivery.phase !== "awaiting-playback")
return { state, commands: [] }
const notification = state.delivery.notification
const next = settle({ ...state, delivery: undefined })
return {
state: next.state,
commands: [{ type: "notification.delivered", notification }, ...next.commands],
}
}
if (event.type === "assistant.done") {
const notificationID =
state.delivery?.phase === "announcing" && state.assistant === "active" && !event.awaitingWork
? state.delivery.notification.id
: undefined
const next = settle(reduce(state, event))
return {
state: next.state,
commands: [{ type: "assistant.finish", awaitingWork: event.awaitingWork, notificationID }, ...next.commands],
}
}
if (event.type === "user.started" && event.bargeIn && state.assistant === "active") {
const next = settle(
state.delivery
? {
...state,
conversation: "idle",
assistant: "idle",
userSpeaking: true,
delivery: undefined,
notifications: [state.delivery.notification, ...state.notifications],
reconnect: "pending",
}
: { ...state, assistant: "suppressed", userSpeaking: true },
)
return { state: next.state, commands: [{ type: "assistant.interrupt" }, ...next.commands] }
}
const next = reduce(state, event)
return settle(next)
}
function reduce(state: VoiceState, event: VoiceEvent): VoiceState {
switch (event.type) {
case "connection.ready":
if (state.connection === "ready") return state
return { ...state, connection: "ready" }
case "connection.connecting":
return {
...state,
connection: "connecting",
conversation: "idle",
assistant: "idle",
userSpeaking: false,
}
case "connection.closed":
return { ...state, connection: "closed", conversation: "idle", assistant: "idle", userSpeaking: false }
case "user.started":
if (state.userSpeaking) return state
return { ...state, userSpeaking: true }
case "user.stopped":
if (!state.userSpeaking) return state
return { ...state, userSpeaking: false }
case "user.committed":
return {
...state,
userSpeaking: false,
conversation: state.assistant === "active" ? "responding" : "waiting",
assistant: state.assistant === "suppressed" ? "idle" : state.assistant,
}
case "assistant.started":
if (state.assistant === "suppressed" || state.assistant === "active") return state
return {
...state,
conversation: "responding",
assistant: "active",
delivery:
state.delivery?.phase === "accepted" ? { ...state.delivery, phase: "announcing" } : state.delivery,
}
case "assistant.suppressed":
return { ...state, assistant: "suppressed" }
case "assistant.done":
return {
...state,
conversation: event.awaitingWork ? "waiting" : "idle",
assistant: "idle",
delivery:
state.delivery?.phase === "announcing" && state.assistant === "active" && !event.awaitingWork
? { ...state.delivery, phase: "awaiting-playback" }
: state.delivery,
}
case "tool.started":
if (state.tools.has(event.id)) return state
return { ...state, conversation: "waiting", tools: new Set([...state.tools, event.id]) }
case "tool.finished": {
if (!state.tools.has(event.id)) return state
const tools = new Set(state.tools)
tools.delete(event.id)
return { ...state, tools }
}
case "notification.queued":
return { ...state, notifications: [...state.notifications, event.notification] }
case "notification.accepted":
if (state.delivery?.notification.id !== event.id || state.delivery.phase !== "sending") return state
return {
...state,
delivery: { ...state.delivery, phase: "accepted" },
}
case "notification.failed":
if (state.delivery?.notification.id !== event.id) return state
return {
...state,
connection: "ready",
conversation: "idle",
delivery: undefined,
notifications: [state.delivery.notification, ...state.notifications],
reconnect: "pending",
}
case "notification.announced":
return state
case "notification.interrupted":
if (state.delivery?.notification.id !== event.id) return state
return {
...state,
conversation: "idle",
assistant: "idle",
delivery: undefined,
notifications: [state.delivery.notification, ...state.notifications],
reconnect: "pending",
}
case "voice.selected":
if (event.voice === state.desiredVoice) return state
return { ...state, desiredVoice: event.voice, reconnect: "pending" }
}
return state
}
function settle(state: VoiceState): VoiceTransition {
if (
state.reconnect === "pending" &&
state.connection === "ready" &&
state.conversation === "idle" &&
state.delivery === undefined &&
!state.userSpeaking &&
state.tools.size === 0
)
return {
state: { ...state, connection: "connecting", reconnect: "none" },
commands: [{ type: "connection.reconnect" }],
}
if (
state.notifications.length > 0 &&
state.delivery === undefined &&
state.connection === "ready" &&
state.conversation === "idle" &&
!state.userSpeaking &&
state.tools.size === 0
) {
const notification = state.notifications[0]
return {
state: {
...state,
conversation: "waiting",
delivery: { notification, phase: "sending" },
notifications: state.notifications.slice(1),
},
commands: [{ type: "notification.send", notification }],
}
}
return { state, commands: [] }
}
+446
View File
@@ -0,0 +1,446 @@
import type { AudioEvent, AudioSession } from "./audio-session"
import { openCodeAnnouncementText, type CompletionReceipt, type OpenCodeAnnouncement } from "./opencode-notification"
import { createNotificationRouter } from "./notification-router"
import type {
VoiceConnection,
VoiceDelegationRequest,
VoiceProtocol,
VoiceProtocolEvent,
VoiceProtocolOptions,
VoiceTool,
VoiceToolExecution,
VoiceWorkRequest,
} from "./protocol"
import type { VoiceUI } from "./ui"
import { initialVoiceState, transitionVoice, type VoiceCommand, type VoiceEvent } from "./voice-coordinator"
export type VoiceSession = {
start(): void
cycleVoice(): void
interrupt(): void
toggleMicrophone(): void
toggleSpeaker(): void
queueNotification(announcement: OpenCodeAnnouncement): void
close(): Promise<void>
}
export function voiceControlTool(voices: ReadonlyArray<string>): VoiceTool {
return {
type: "function",
name: "set_voice",
description: "Change your speaking voice. Requires a brief reconnect and resets voice conversation memory.",
parameters: {
type: "object",
additionalProperties: false,
properties: { voice: { type: "string", enum: voices } },
required: ["voice"],
},
}
}
export function createVoiceSession(options: {
readonly protocol: VoiceProtocol
readonly connection: Omit<VoiceProtocolOptions, "voice" | "onEvent">
readonly initialVoice: string
readonly voices: ReadonlyArray<string>
readonly text?: string
readonly ui: VoiceUI
readonly audio?: AudioSession
readonly tools: {
readonly execute: (name: string, input: Record<string, unknown>) => Promise<VoiceToolExecution>
readonly acknowledge: (receipt: CompletionReceipt) => Promise<void>
readonly close: () => Promise<void>
}
readonly delegate: (
request: VoiceDelegationRequest,
execute: (request: VoiceWorkRequest) => Promise<unknown>,
) => Promise<string>
readonly trace?: (event: string, data?: Record<string, unknown>) => void
readonly onClosed: () => void
}): VoiceSession {
let state = initialVoiceState(options.initialVoice)
let connection: VoiceConnection | undefined
let closed = false
let closePromise: Promise<void> | undefined
let voiceTimer: ReturnType<typeof setTimeout> | undefined
const notifications = createNotificationRouter((routed) => {
options.trace?.("notification.queued", {
type: routed.announcement.notification.type,
promptID: routed.promptID,
depth: state.notifications.length + 1,
})
dispatch({
type: "notification.queued",
notification: {
id: crypto.randomUUID(),
receipt: routed.announcement.receipt,
promptID: routed.promptID,
delegationID: routed.delegationID,
text: openCodeAnnouncementText(routed.announcement.notification),
},
})
})
function dispatch(event: VoiceEvent) {
const transition = transitionVoice(state, event)
if (transition.state === state && transition.commands.length === 0) return
state = transition.state
options.trace?.("voice.transition", {
input: event.type,
connection: state.connection,
conversation: state.conversation,
assistant: state.assistant,
userSpeaking: state.userSpeaking,
tools: state.tools.size,
notifications: state.notifications.length,
commands: transition.commands.map((command) => command.type),
})
transition.commands.forEach(runCommand)
}
const setVoice = (voice: string) => {
options.ui.setStatus({ voice })
options.ui.meta(`[voice] switching to ${voice}`)
dispatch({ type: "voice.selected", voice })
}
const interruptDelivery = () => {
if (state.delivery) dispatch({ type: "notification.interrupted", id: state.delivery.notification.id })
}
const reconnect = () => {
if (closed) return
options.audio?.flushPlayback()
options.ui.assistantDone()
const previous = connection
connection = undefined
const next = () => {
if (!closed) connect()
}
if (!previous) return next()
void previous.close({ graceful: false }).finally(next)
}
function runCommand(command: VoiceCommand) {
if (command.type === "connection.reconnect") return reconnect()
if (command.type === "assistant.interrupt") {
connection?.interrupt?.()
options.audio?.flushPlayback()
options.ui.assistantDone()
return
}
if (command.type === "assistant.finish") {
if (options.text) {
options.ui.assistantDone()
if (command.notificationID) dispatch({ type: "notification.announced", id: command.notificationID })
if (!command.awaitingWork && state.tools.size === 0) options.onClosed()
return
}
void options.audio?.finishPlayback().then((outcome) => {
options.ui.assistantDone()
if (!command.notificationID) return
dispatch({
type: outcome === "played" ? "notification.announced" : "notification.interrupted",
id: command.notificationID,
})
})
return
}
if (command.type === "notification.delivered") {
options.trace?.("notification.announced", {
id: command.notification.id,
promptID: command.notification.promptID,
})
if (!command.notification.receipt) return
const receipt = command.notification.receipt
void options.tools
.acknowledge(receipt)
.catch((error) =>
options.trace?.("notification.ack.failed", { id: command.notification.id, error: String(error) }),
)
.finally(() => notifications.acknowledged(receipt))
return
}
const active = connection
if (!active) return dispatch({ type: "notification.failed", id: command.notification.id })
void active
.notify({
id: command.notification.id,
text: `Announce this OpenCode update conversationally without reading private identifiers aloud:\n${command.notification.text}`,
delegationID: command.notification.delegationID,
})
.then((accepted) => {
if (!accepted) return dispatch({ type: "notification.failed", id: command.notification.id })
dispatch({ type: "notification.accepted", id: command.notification.id })
options.trace?.("notification.context.accepted", {
id: command.notification.id,
promptID: command.notification.promptID,
remaining: state.notifications.length,
})
})
.catch((error) => {
options.trace?.("notification.context.failed", {
id: command.notification.id,
promptID: command.notification.promptID,
error: String(error),
})
dispatch({ type: "notification.failed", id: command.notification.id })
})
}
const executeTool = async (request: VoiceWorkRequest, delegationID?: string) => {
if (delegationID) notifications.beginDelegatedTool()
const execution: VoiceToolExecution = await (request.name === "set_voice"
? Promise.resolve().then(() => {
const voice = request.input["voice"]
if (typeof voice !== "string" || !options.voices.includes(voice))
return { output: toolError(`Voice must be one of: ${options.voices.join(", ")}.`) }
if (voiceTimer) clearTimeout(voiceTimer)
voiceTimer = setTimeout(() => {
voiceTimer = undefined
setVoice(voice)
}, 1_000)
return {
output: {
status: "switching",
voice,
note: `${options.protocol.name} conversation memory resets during reconnect.`,
},
}
})
: options.tools
.execute(request.name, request.input)
.catch((error) => ({ output: toolError(String(error), true) } satisfies VoiceToolExecution)))
options.ui.toolDone(request.id, execution.output)
if (delegationID) notifications.finishDelegatedTool(delegationID, execution.admittedPrompt)
return execution.output
}
const queueWork = (source: VoiceConnection, request: VoiceWorkRequest) => {
options.trace?.("work.started", { id: request.id, name: request.name })
void executeTool(request)
.then((output) => {
options.trace?.("work.resolved", { id: request.id, name: request.name })
source.resolveWork(request, output)
})
.catch((error) => options.ui.meta(`[work error] ${String(error)}`))
.finally(() => dispatch({ type: "tool.finished", id: request.id }))
}
const userSpeaking = (active: boolean) => {
if (active === state.userSpeaking) return
dispatch(
active
? { type: "user.started", bargeIn: options.audio?.fullDuplex && options.protocol.capabilities.interruption }
: { type: "user.stopped" },
)
options.ui.userSpeaking(active)
}
const onAudioEvent = (event: AudioEvent) => {
switch (event.type) {
case "input":
connection?.appendAudio(event.audio)
options.ui.userAudioLevel(event.level)
return
case "user.started":
userSpeaking(true)
return
case "user.stopped":
userSpeaking(false)
return
case "user.reset":
options.ui.userReset()
return
case "assistant.level":
options.ui.assistantAudio(event.level, event.durationMs)
return
case "status":
options.ui.setStatus({ audio: event.audio })
return
case "meta":
options.ui.meta(event.text)
}
}
const onProtocolEvent = (source: VoiceConnection, event: VoiceProtocolEvent) => {
if (source !== connection || closed) return
if (options.connection.debug || event.type !== "assistant.audio")
options.trace?.("protocol.event", {
type: event.type,
conversation: state.conversation,
assistant: state.assistant,
userSpeaking: state.userSpeaking,
pendingWork: state.tools.size,
})
switch (event.type) {
case "ready":
dispatch({ type: "connection.ready" })
options.ui.meta(
`connected to ${options.protocol.name} ${options.connection.model} (voice: ${state.desiredVoice})`,
)
options.ui.setStatus({
voice: state.desiredVoice,
microphoneMuted: options.audio?.microphoneMuted,
speakerMuted: options.audio?.speakerMuted,
})
if (!options.text) {
void options.audio?.start(onAudioEvent)
return
}
options.ui.userTranscript("typed", options.text)
dispatch({ type: "user.committed" })
source.sendText?.(options.text)
return
case "user.started":
userSpeaking(true)
return
case "user.stopped":
userSpeaking(false)
return
case "user.committed":
options.audio?.noteUserCommitted()
dispatch({ type: "user.committed" })
options.ui.userCommitted(event.id)
return
case "user.transcript":
options.audio?.noteUserTranscript(event.final)
if (event.final) {
dispatch({ type: "user.stopped" })
options.ui.userSpeaking(false)
}
options.ui.userTranscript(event.id, event.text, event.final)
return
case "assistant.audio":
if (state.assistant === "suppressed") return
dispatch({ type: "assistant.started" })
options.audio?.play(event.audio, event.timeline)
return
case "assistant.transcript.delta":
if (state.assistant === "suppressed") return
dispatch({ type: "assistant.started" })
options.ui.assistantDelta(event.delta)
return
case "assistant.transcript":
if (state.assistant === "suppressed") return
dispatch({ type: "assistant.started" })
options.ui.assistantTranscript(event.text)
return
case "assistant.done": {
dispatch({ type: "assistant.done", awaitingWork: event.awaitingWork })
return
}
case "tool.started":
dispatch({ type: "tool.started", id: event.id })
options.ui.toolStart(event.id, event.name, {})
return
case "work.requested":
queueWork(source, event.request)
return
case "work.rejected":
dispatch({ type: "tool.started", id: event.request.id })
options.ui.toolStart(event.request.id, event.request.name, event.request.input)
options.ui.toolDone(event.request.id, event.output)
source.resolveWork(event.request, event.output)
dispatch({ type: "tool.finished", id: event.request.id })
return
case "delegation.requested":
dispatch({ type: "tool.started", id: event.request.id })
void options
.delegate(event.request, (request) => {
options.ui.toolStart(request.id, request.name, request.input)
return executeTool(request, event.request.id)
})
.then((output) => {
if (source !== connection) {
options.trace?.("delegation.result.stale", { id: event.request.id })
return
}
source.resolveDelegation?.(event.request, output)
})
.catch((error) => {
options.trace?.("responses.controller.failed", { error: String(error) })
if (source !== connection) return
options.ui.toolStart(event.request.id, "opencode", { text: event.request.text })
options.ui.toolDone(event.request.id, { status: "error", message: String(error) })
source.resolveDelegation?.(event.request, `The OpenCode controller failed: ${String(error)}`)
})
.finally(() => dispatch({ type: "tool.finished", id: event.request.id }))
return
case "debug":
if (options.connection.debug) options.ui.meta(`[debug] ${event.message}`)
return
case "error":
options.ui.meta(`[${options.protocol.name} error] ${event.message}`)
return
case "closed":
dispatch({ type: "connection.closed" })
options.ui.meta(`${options.protocol.name} connection closed (${event.code})`)
options.onClosed()
}
}
const connect = () => {
let next: VoiceConnection
next = options.protocol.connect({
...options.connection,
voice: state.desiredVoice,
onEvent: (event) => onProtocolEvent(next, event),
})
dispatch({ type: "connection.connecting" })
connection = next
}
const session: VoiceSession = {
start: connect,
cycleVoice() {
setVoice(options.voices[(options.voices.indexOf(state.desiredVoice) + 1) % options.voices.length])
},
interrupt() {
if (!options.audio?.isPlaying() && state.assistant === "idle") return
if (state.assistant === "active") {
connection?.interrupt?.()
dispatch({ type: "assistant.suppressed" })
}
interruptDelivery()
options.audio?.flushPlayback()
options.ui.assistantDone()
options.ui.meta("[interrupted]")
},
toggleMicrophone() {
if (!options.audio) return
const muted = options.audio.toggleMicrophone()
userSpeaking(false)
options.ui.setStatus({ microphoneMuted: muted })
if (muted) options.ui.userReset()
options.ui.userAudioLevel(undefined)
options.ui.meta(`[microphone] ${muted ? "muted" : "live"}`)
},
toggleSpeaker() {
if (!options.audio) return
const muted = options.audio.toggleSpeaker()
options.ui.setStatus({ speakerMuted: muted })
if (muted) options.ui.assistantPlaybackStopped()
options.ui.meta(`[speaker] ${muted ? "muted" : "live"}`)
},
queueNotification(announcement) {
notifications.route(announcement)
},
close() {
if (closePromise) return closePromise
closed = true
if (voiceTimer) clearTimeout(voiceTimer)
options.audio?.close()
const active = connection
connection = undefined
dispatch({ type: "connection.closed" })
closePromise = Promise.allSettled([options.tools.close(), ...(active ? [active.close()] : [])]).then(() => {})
return closePromise
},
}
return session
}
function toolError(message: string, retryable = false) {
return { status: "error", message, retryable }
}
+53
View File
@@ -0,0 +1,53 @@
import { describe, expect, test } from "bun:test"
import {
connectionMeterLevels,
connectionTransitioning,
scheduleTextReveal,
textRevealOpacity,
transcriptionPulse,
} from "../src/animation"
describe("voice text animation", () => {
test("stages newly streamed words without delaying them indefinitely", () => {
const first = scheduleTextReveal("", "Hello there", 1_000, 0)
expect(first.reveals).toEqual([
{ offset: 0, at: 1_000 },
{ offset: 6, at: 1_032 },
])
expect(first.animationEndsAt).toBe(1_412)
const queued = scheduleTextReveal("Hello there", " friend", 1_010, 10_000)
expect(queued.reveals).toEqual([{ offset: 12, at: 1_170 }])
})
test("slowly fades text into its stable color", () => {
const opacities = [100, 195, 290, 385, 480].map((now) => textRevealOpacity(now, 100))
expect(opacities[0]).toBe(0.16)
expect(opacities.toSorted((a, b) => a - b)).toEqual(opacities)
expect(opacities.at(-1)).toBe(1)
})
test("keeps transcription pulse bounded and moving", () => {
const levels = [0, 110, 220, 330].map(transcriptionPulse)
expect(levels.every((level) => level >= 0 && level <= 1)).toBe(true)
expect(new Set(levels).size).toBeGreaterThan(1)
})
test("moves the connecting pulse left to right and blends into the connected meter", () => {
const first = connectionMeterLevels(1_000, 0.1, undefined, 1_000)
const next = connectionMeterLevels(1_180, 0.1, undefined, 1_000)
expect(first.indexOf(Math.max(...first))).toBe(0)
expect(first.indexOf(Math.max(...first))).toBeLessThan(next.indexOf(Math.max(...next)))
const samples = Array.from({ length: 16 }, (_, index) => connectionMeterLevels(index * 45, 0.1))
expect(samples.every((levels) => Math.max(...levels) > 0.85)).toBe(true)
const handoff = connectionMeterLevels(1_000, 0.4, 1_000)
expect(handoff).toEqual(connectionMeterLevels(1_000, 0.4))
expect(connectionMeterLevels(1_480, 0.4, 1_000)).toEqual(
[0, 1, 2, 3].map((index) => 0.4 * (0.72 + Math.sin(1_480 / 240 + index * 1.4) * 0.28)),
)
expect(connectionTransitioning(1_479, 1_000)).toBe(true)
expect(connectionTransitioning(1_480, 1_000)).toBe(false)
})
})
@@ -0,0 +1,79 @@
import { describe, expect, test } from "bun:test"
import { joinAssistantText } from "../src/ui-model"
describe("joinAssistantText", () => {
const cases: Array<[name: string, left: string, right: string, expected: string]> = [
[
"inserts a space between sentences",
"Okay, I'll check that.",
"The build passed.",
"Okay, I'll check that. The build passed.",
],
["inserts a space mid-sentence", "Let me look at", "the config file.", "Let me look at the config file."],
[
"keeps a single trailing space",
"Okay, I'll check that. ",
"The build passed.",
"Okay, I'll check that. The build passed.",
],
[
"keeps a single leading space",
"Okay, I'll check that.",
" The build passed.",
"Okay, I'll check that. The build passed.",
],
["collapses spaces on both sides", "Done. ", " Next.", "Done. Next."],
["collapses runs of spaces", "Done. ", "\t \tNext.", "Done. Next."],
[
"spaces after a question mark",
"Want me to run tests?",
"I can do that now.",
"Want me to run tests? I can do that now.",
],
["spaces after an ellipsis", "Thinking…", "done.", "Thinking… done."],
["preserves a trailing newline", "Done:\n", "Next up, tests.", "Done:\nNext up, tests."],
["preserves a leading newline", "Done:", "\nNext up, tests.", "Done:\nNext up, tests."],
[
"preserves a paragraph break",
"First paragraph.\n\n",
"Second paragraph.",
"First paragraph.\n\nSecond paragraph.",
],
["preserves a newline mixed with spaces", "Done: ", " \n Next.", "Done: \n Next."],
["never spaces before closing punctuation", "Done", ".", "Done."],
["never spaces before a comma", "one", ", two", "one, two"],
["never spaces before a closing paren", "(aside", ")", "(aside)"],
["never spaces after an opening paren", "an (", "aside)", "an (aside)"],
["drops an empty left side", "", "The build passed.", "The build passed."],
["drops an empty right side", "The build passed.", "", "The build passed."],
["drops a whitespace-only left side", " ", "The build passed.", "The build passed."],
["drops a whitespace-only right side", "The build passed.", " \n ", "The build passed."],
["handles two empty sides", "", "", ""],
["does not trim the outer edges", " Leading kept.", "Trailing kept. ", " Leading kept. Trailing kept. "],
]
for (const [name, left, right, expected] of cases) {
test(name, () => expect(joinAssistantText(left, right).text).toBe(expected))
}
test("splits the result at the boundary so reveals cover only the appended text", () => {
for (const [name, left, right] of cases) {
const joined = joinAssistantText(left, right)
expect(`${name}: ${joined.previous}${joined.appended}`).toBe(`${name}: ${joined.text}`)
expect(`${name}: ${joined.text.endsWith(joined.appended)}`).toBe(`${name}: true`)
}
})
test("is idempotent once a boundary is normalised", () => {
const once = joinAssistantText("Okay.", "Next.").text
expect(joinAssistantText(once, "").text).toBe(once)
expect(joinAssistantText("Okay. ", "Next.").text).toBe(once)
})
test("folds consecutive messages with exactly one space per boundary", () => {
expect(["First.", "Second.", "Third."].reduce((left, right) => joinAssistantText(left, right).text)).toBe(
"First. Second. Third.",
)
expect(["First.", "", "Third."].reduce((left, right) => joinAssistantText(left, right).text)).toBe("First. Third.")
})
})
@@ -0,0 +1,33 @@
import { describe, expect, test } from "bun:test"
import { AudioJitterBuffer } from "../src/audio-jitter-buffer"
import { PCM_BYTES_PER_MS } from "../src/pcm"
const chunk = (durationMs: number, gapMs = 0) => ({ bytes: Buffer.alloc(durationMs * PCM_BYTES_PER_MS), gapMs })
describe("AudioJitterBuffer", () => {
test("holds startup audio until the target buffer is available", () => {
const buffer = new AudioJitterBuffer(300)
expect(buffer.push(chunk(100))).toEqual([])
expect(buffer.push(chunk(100))).toEqual([])
expect(buffer.push(chunk(100))).toHaveLength(3)
})
test("passes chunks through after playback starts", () => {
const buffer = new AudioJitterBuffer(200)
buffer.push(chunk(200))
expect(buffer.push(chunk(100))).toEqual([chunk(100)])
})
test("flushes short utterances and resets between responses", () => {
const buffer = new AudioJitterBuffer(300)
buffer.push(chunk(100))
expect(buffer.finish()).toHaveLength(1)
buffer.reset()
expect(buffer.push(chunk(100))).toEqual([])
})
test("counts intentional timeline silence toward buffered duration", () => {
const buffer = new AudioJitterBuffer(300)
expect(buffer.push(chunk(200, 100))).toHaveLength(1)
})
})
+106
View File
@@ -0,0 +1,106 @@
import { expect, test } from "bun:test"
import type { AudioDevice } from "../src/audio-device"
import { createAudioSession, type AudioEvent } from "../src/audio-session"
test("reports muted notification playback as inaudible", async () => {
const device = scriptedDevice()
const audio = await createAudioSession({
duplex: false,
speakers: false,
inputActivity: "server",
debug: false,
device,
})
expect(audio.toggleSpeaker()).toBe(true)
audio.play(Buffer.alloc(4_800))
expect(await audio.finishPlayback()).toBe("inaudible")
expect(device.writes).toEqual([])
})
test("settles active playback when interrupted", async () => {
const device = scriptedDevice()
const audio = await createAudioSession({
duplex: false,
speakers: false,
inputActivity: "server",
debug: false,
device,
})
audio.play(Buffer.alloc(4_800))
const outcome = audio.finishPlayback()
audio.flushPlayback()
expect(await outcome).toBe("interrupted")
expect(device.flushes).toBe(1)
})
test("emits captured audio and local speech observations", async () => {
const device = scriptedDevice()
const events: AudioEvent[] = []
const audio = await createAudioSession({
duplex: true,
speakers: false,
inputActivity: "local",
debug: false,
device,
})
await audio.start((event) => events.push(event))
const input = Buffer.alloc(4_800)
for (let offset = 0; offset < input.length; offset += 2) input.writeInt16LE(12_000, offset)
device.capture(input)
expect(events).toContainEqual({ type: "input", audio: input, level: expect.any(Number) })
expect(events).toContainEqual({ type: "user.started" })
audio.close()
})
test("preserves quiet duplex microphone audio during assistant playback", async () => {
const device = scriptedDevice()
const events: AudioEvent[] = []
const audio = await createAudioSession({
duplex: false,
speakers: true,
inputActivity: "server",
debug: false,
device,
})
await audio.start((event) => events.push(event))
audio.play(Buffer.alloc(24_000))
const input = Buffer.alloc(4_800)
for (let offset = 0; offset < input.length; offset += 2) input.writeInt16LE(800, offset)
device.capture(input)
expect(audio.fullDuplex).toBe(true)
expect(events).toContainEqual({ type: "input", audio: input, level: expect.any(Number) })
audio.close()
})
function scriptedDevice() {
let onInput: ((audio: Buffer) => void) | undefined
const writes: Buffer[] = []
const device = {
fullDuplex: true,
mode: "scripted",
async start(listener) {
onInput = listener
},
write(audio) {
writes.push(audio)
},
flush() {
device.flushes += 1
},
close() {},
writes,
flushes: 0,
capture(audio: Buffer) {
onInput?.(audio)
},
} satisfies AudioDevice & {
readonly writes: Buffer[]
flushes: number
capture(audio: Buffer): void
}
return device
}
@@ -0,0 +1,101 @@
import { expect, test } from "bun:test"
import { mkdir, mkdtemp, rm } from "node:fs/promises"
import { tmpdir } from "node:os"
import { join } from "node:path"
import { createCompletionStore } from "../src/completion-store"
test("completion store restores pending and completed notifications until delivery", async () => {
const directory = await mkdtemp(join(tmpdir(), "voice-completions-"))
const path = join(directory, "prompts.json")
const first = await createCompletionStore(path)
const pending = { sessionID: "session-1", promptID: "prompt-1" }
const completed = { sessionID: "session-2", promptID: "prompt-2" }
try {
await first.admitting(pending, "Please continue")
expect(first.entries()).toEqual([{ status: "admitting", handle: pending, text: "Please continue" }])
await first.pending(pending)
await first.pending(completed)
await first.completed(completed, {
type: "opencode.prompt.completed",
session_id: "session-2",
prompt_id: "prompt-2",
status: "completed",
text: "done",
})
await first.close()
const restored = await createCompletionStore(path)
expect(restored.entries()).toEqual(expect.arrayContaining([
{ status: "pending", handle: pending },
{
status: "completed",
handle: completed,
notification: {
type: "opencode.prompt.completed",
session_id: "session-2",
prompt_id: "prompt-2",
status: "completed",
text: "done",
},
},
]))
await restored.close()
const stillPendingDelivery = await createCompletionStore(path)
expect(stillPendingDelivery.entries().some((entry) => entry.status === "completed")).toBe(true)
await stillPendingDelivery.remove(completed)
await stillPendingDelivery.close()
expect((await createCompletionStore(path)).entries()).toEqual([{ status: "pending", handle: pending }])
} finally {
await rm(directory, { recursive: true, force: true })
}
})
test("completion stores do not overwrite prompts written by another process", async () => {
const directory = await mkdtemp(join(tmpdir(), "voice-completions-concurrent-"))
const path = join(directory, "prompts.json")
const first = await createCompletionStore(path)
const second = await createCompletionStore(path)
try {
await Promise.all([
first.pending({ sessionID: "session-1", promptID: "prompt-shared" }),
second.pending({ sessionID: "session-2", promptID: "prompt-shared" }),
])
await Promise.all([first.close(), second.close()])
expect((await createCompletionStore(path)).entries()).toEqual(
expect.arrayContaining([
{ status: "pending", handle: { sessionID: "session-1", promptID: "prompt-shared" } },
{ status: "pending", handle: { sessionID: "session-2", promptID: "prompt-shared" } },
]),
)
} finally {
await rm(directory, { recursive: true, force: true })
}
})
test("continues serializing writes after one filesystem failure", async () => {
const directory = await mkdtemp(join(tmpdir(), "voice-completions-recovery-"))
const path = join(directory, "prompts.json")
const store = await createCompletionStore(path)
try {
await rm(`${path}.d`, { recursive: true })
await Bun.write(`${path}.d`, "not a directory")
const failure = await store
.pending({ sessionID: "session-1", promptID: "prompt-1" })
.then(() => undefined, (error) => error)
expect(failure).toBeDefined()
await rm(`${path}.d`)
await mkdir(`${path}.d`, { recursive: true })
await store.pending({ sessionID: "session-2", promptID: "prompt-2" })
await store.close()
expect((await createCompletionStore(path)).entries()).toContainEqual({
status: "pending",
handle: { sessionID: "session-2", promptID: "prompt-2" },
})
} finally {
await rm(directory, { recursive: true, force: true })
}
})
+311
View File
@@ -0,0 +1,311 @@
import { describe, expect, test } from "bun:test"
import { createLiveEventDelivery, createLiveEventProjector } from "../src/protocol-live"
import type { VoiceProtocolEvent } from "../src/protocol"
const frames = [
{ type: "session.started", session: {} },
{ type: "session.context.appended", start_ms: 0, end_ms: 0 },
{ type: "output_audio.delta", audio: "AAA=", start_ms: 0, end_ms: 1 },
{
type: "input_transcript.added",
start_ms: 0,
end_ms: 100,
item: { id: "input-1", type: "input_transcript", text: " Hello" },
},
{ type: "turn.created", turn: { id: "user-turn", role: "user", transcript: " Hello" } },
{
type: "input_transcript.added",
start_ms: 100,
end_ms: 200,
item: { id: "input-2", type: "input_transcript", text: ", this" },
},
{ type: "turn.delta", turn_id: "user-turn", delta: ", this" },
{
type: "output_transcript.added",
start_ms: 200,
end_ms: 300,
item: { id: "output-1", type: "output_transcript", text: " Copy that" },
},
{
type: "input_transcript.added",
start_ms: 200,
end_ms: 300,
item: { id: "input-3", type: "input_transcript", text: " is a test" },
},
{
type: "output_transcript.added",
start_ms: 300,
end_ms: 400,
item: { id: "output-2", type: "output_transcript", text: ", test" },
},
{ type: "turn.delta", turn_id: "user-turn", delta: " is a test" },
{ type: "turn.done", turn: { id: "user-turn", role: "user", transcript: " Hello, this is a test" } },
{ type: "turn.created", turn: { id: "assistant-turn", role: "assistant", transcript: " Copy that, test" } },
{
type: "output_transcript.added",
start_ms: 400,
end_ms: 500,
item: { id: "output-3", type: "output_transcript", text: " received" },
},
{ type: "turn.delta", turn_id: "assistant-turn", delta: " received" },
{
type: "output_transcript.added",
start_ms: 500,
end_ms: 600,
item: { id: "output-4", type: "output_transcript", text: " loud and clear." },
},
{ type: "turn.delta", turn_id: "assistant-turn", delta: " loud and clear." },
{
type: "turn.done",
turn: { id: "assistant-turn", role: "assistant", transcript: " Copy that, test received loud and clear." },
},
] as const
describe("Live event projector", () => {
test("replays progressive transcripts without schema errors or duplicate text", () => {
const project = createLiveEventProjector()
const events = frames.flatMap((frame) => project(JSON.stringify(frame)).events)
expect(events.filter((event) => event.type === "error")).toEqual([])
expect(events.filter((event) => event.type === "user.committed")).toEqual([
{ type: "user.committed", id: "input-1" },
])
expect(events.filter(isUserTranscript)).toEqual([
{ type: "user.transcript", id: "input-1", text: "Hello", final: false },
{ type: "user.transcript", id: "input-1", text: "Hello, this", final: false },
{ type: "user.transcript", id: "input-1", text: "Hello, this is a test", final: false },
{ type: "user.transcript", id: "input-1", text: "Hello, this is a test", final: true },
])
expect(
events
.filter(isAssistantDelta)
.map((event) => event.delta)
.join(""),
).toBe(" Copy that, test received loud and clear.")
expect(events.filter((event) => event.type === "assistant.done")).toEqual([
{ type: "assistant.done", awaitingWork: false },
])
expect(events.filter((event) => event.type === "assistant.audio")).toHaveLength(1)
})
test("reports malformed frames instead of throwing", () => {
expect(createLiveEventProjector()(JSON.stringify({ type: "turn.created", turn: { id: 1 } })).events).toEqual([
{ type: "error", message: "Received an invalid Live API event." },
])
})
test("preserves context acknowledgement correlation IDs", () => {
expect(
createLiveEventProjector()(
JSON.stringify({ type: "session.context.appended", event_id: "event-notification" }),
),
).toMatchObject({
type: "session.context.appended",
eventID: "event-notification",
})
})
test("replaces corrected assistant transcript snapshots", () => {
const project = createLiveEventProjector()
const events = [
project(
JSON.stringify({
type: "output_transcript.added",
item: { id: "output-1", type: "output_transcript", text: " Hello world" },
}),
),
project(
JSON.stringify({
type: "turn.created",
turn: { id: "assistant-turn", role: "assistant", transcript: " Hello there" },
}),
),
].flatMap((result) => result.events)
expect(events.filter((event) => event.type.startsWith("assistant.transcript"))).toEqual([
{ type: "assistant.transcript.delta", delta: " Hello world" },
{ type: "assistant.transcript", text: " Hello there" },
])
})
test("keeps the space between consecutive assistant turns", () => {
const project = createLiveEventProjector()
const turn = (id: string, text: string) => [
{ type: "turn.created", turn: { id, role: "assistant", transcript: "" } },
{ type: "output_transcript.added", item: { id: `${id}-o`, type: "output_transcript", text } },
{ type: "turn.delta", turn_id: id, delta: text },
{ type: "turn.done", turn: { id, role: "assistant", transcript: text } },
]
const events = [...turn("t1", ' Session "Fix auth bug" is ready.'), ...turn("t2", " Next up, tests.")].flatMap(
(frame) => project(JSON.stringify(frame)).events,
)
// Concatenating the delta stream is what the UI and any transcript consumer does; the
// turn boundary must survive it rather than welding "ready.Next" together.
const transcript = events
.filter(isAssistantDelta)
.map((event) => event.delta)
.join("")
expect(transcript).toBe(' Session "Fix auth bug" is ready. Next up, tests.')
expect(transcript).not.toContain("ready.Next")
expect(events.filter((event) => event.type === "assistant.done")).toHaveLength(2)
})
test("emits no duplicate text when fragments and turn deltas overlap across turns", () => {
const project = createLiveEventProjector()
const frames = [
{ type: "turn.created", turn: { id: "t1", role: "assistant", transcript: "" } },
{ type: "output_transcript.added", item: { id: "a", type: "output_transcript", text: " First" } },
{ type: "turn.delta", turn_id: "t1", delta: " First" },
{ type: "output_transcript.added", item: { id: "b", type: "output_transcript", text: " turn." } },
{ type: "turn.delta", turn_id: "t1", delta: " turn." },
{ type: "turn.done", turn: { id: "t1", role: "assistant", transcript: " First turn." } },
{ type: "turn.created", turn: { id: "t2", role: "assistant", transcript: "" } },
{ type: "output_transcript.added", item: { id: "c", type: "output_transcript", text: " Second turn." } },
{ type: "turn.delta", turn_id: "t2", delta: " Second turn." },
{ type: "turn.done", turn: { id: "t2", role: "assistant", transcript: " Second turn." } },
]
const events = frames.flatMap((frame) => project(JSON.stringify(frame)).events)
expect(events.filter((event) => event.type === "assistant.transcript")).toEqual([])
expect(
events
.filter(isAssistantDelta)
.map((event) => event.delta)
.join(""),
).toBe(" First turn. Second turn.")
})
test("projects Responses delegation function calls without rejecting reasoning items", () => {
const project = createLiveEventProjector()
expect(
project(
JSON.stringify({
type: "response.output_item.done",
item: { id: "reasoning-1", type: "reasoning" },
}),
).events,
).toEqual([{ type: "debug", message: "response.output_item.done" }])
expect(
project(
JSON.stringify({
type: "response.output_item.added",
item: {
id: "function-1",
type: "function_call",
name: "find_sessions",
call_id: "call-1",
arguments: "",
},
}),
).events,
).toEqual([
{ type: "debug", message: "response.output_item.added" },
{ type: "tool.started", id: "call-1", name: "find_sessions" },
])
expect(
project(
JSON.stringify({
type: "response.output_item.done",
item: {
id: "function-1",
type: "function_call",
name: "find_sessions",
call_id: "call-1",
arguments: '{"scope":"current_project"}',
},
}),
).events,
).toEqual([
{ type: "debug", message: "response.output_item.done" },
{
type: "work.requested",
request: { id: "call-1", name: "find_sessions", input: { scope: "current_project" } },
},
])
expect(
project(
JSON.stringify({
type: "response.output_item.done",
item: {
id: "message-1",
type: "message",
status: "completed",
content: [{ type: "output_text", text: "Two sessions found." }],
role: "assistant",
},
}),
).events,
).toEqual([{ type: "debug", message: "response.output_item.done" }])
})
test("projects client delegation requests for the local controller", () => {
expect(
createLiveEventProjector()(
JSON.stringify({
type: "delegation.created",
item: {
id: "delegation-1",
type: "delegation",
target: "client",
content: [{ type: "input_text", text: "Inspect the voice package" }],
},
}),
).events,
).toEqual([
{ type: "debug", message: "delegation.created" },
{ type: "delegation.requested", request: { id: "delegation-1", text: "Inspect the voice package" } },
])
})
test("rejects malformed function arguments without hanging the delegation", () => {
expect(
createLiveEventProjector()(
JSON.stringify({
type: "response.output_item.done",
item: {
id: "function-1",
type: "function_call",
name: "find_sessions",
call_id: "call-invalid",
arguments: "not json",
},
}),
).events,
).toEqual([
{ type: "debug", message: "response.output_item.done" },
{ type: "tool.started", id: "call-invalid", name: "find_sessions" },
{ type: "error", message: "Received invalid arguments for Live tool find_sessions." },
{
type: "work.rejected",
request: { id: "call-invalid", name: "find_sessions", input: {} },
output: { status: "error", message: "Invalid arguments for tool find_sessions." },
},
])
})
test("waits for a quiet drain after turn completion and late audio", async () => {
const events: VoiceProtocolEvent[] = []
const delivery = createLiveEventDelivery({ emit: (event) => events.push(event), drainMs: 20 })
delivery.push({ type: "assistant.done", awaitingWork: false })
await Bun.sleep(10)
delivery.push({ type: "assistant.audio", audio: Buffer.alloc(48) })
await Bun.sleep(12)
expect(events.map((event) => event.type)).toEqual(["assistant.audio"])
await Bun.sleep(12)
expect(events.map((event) => event.type)).toEqual(["assistant.audio", "assistant.done"])
delivery.close()
})
})
function isUserTranscript(
event: VoiceProtocolEvent,
): event is Extract<VoiceProtocolEvent, { readonly type: "user.transcript" }> {
return event.type === "user.transcript"
}
function isAssistantDelta(
event: VoiceProtocolEvent,
): event is Extract<VoiceProtocolEvent, { readonly type: "assistant.transcript.delta" }> {
return event.type === "assistant.transcript.delta"
}
@@ -0,0 +1,59 @@
import { expect, test } from "bun:test"
import { createNotificationRouter, type RoutedAnnouncement } from "../src/notification-router"
test("routes blockers and completions through the admitted delegation", () => {
const routed: RoutedAnnouncement[] = []
const router = createNotificationRouter((announcement) => routed.push(announcement))
router.beginDelegatedTool()
router.route({
notification: {
type: "opencode.prompt.blocked",
prompt_id: "prompt-1",
blocker: "permission",
session_id: "session-1",
request_id: "permission-1",
action: "read",
resources: [],
},
})
expect(routed).toEqual([])
router.finishDelegatedTool("delegation-1", { sessionID: "session-1", promptID: "prompt-1" })
router.route({
notification: {
type: "opencode.prompt.completed",
session_id: "session-1",
prompt_id: "prompt-1",
status: "completed",
text: "done",
},
receipt: { sessionID: "session-1", promptID: "prompt-1" },
})
expect(routed.map((item) => item.delegationID)).toEqual(["delegation-1", "delegation-1"])
expect(routed[0].announcement.receipt).toBeUndefined()
expect(routed[1].announcement.receipt).toEqual({ sessionID: "session-1", promptID: "prompt-1" })
})
test("keeps identical prompt IDs isolated by session", () => {
const routed: RoutedAnnouncement[] = []
const router = createNotificationRouter((announcement) => routed.push(announcement))
router.beginDelegatedTool()
router.finishDelegatedTool("delegation-1", { sessionID: "session-1", promptID: "prompt-shared" })
router.beginDelegatedTool()
router.finishDelegatedTool("delegation-2", { sessionID: "session-2", promptID: "prompt-shared" })
for (const sessionID of ["session-1", "session-2"])
router.route({
notification: {
type: "opencode.prompt.completed",
session_id: sessionID,
prompt_id: "prompt-shared",
status: "completed",
text: "done",
},
receipt: { sessionID, promptID: "prompt-shared" },
})
expect(routed.map((item) => item.delegationID)).toEqual(["delegation-1", "delegation-2"])
})
+248
View File
@@ -0,0 +1,248 @@
import { expect, test } from "bun:test"
import { OpenCode } from "@opencode-ai/client/promise"
import { createOpenCodeBridge } from "../src/opencode"
import type { CompletionStore } from "../src/completion-store"
test("renames only a session discovered by the voice controller", async () => {
const requests: Request[] = []
const session = {
id: "ses_known",
projectID: "project-1",
cost: "0",
tokens: { input: 0, output: 0, reasoning: 0, cache: { read: 0, write: 0 } },
time: { created: 1, updated: 2 },
title: "Old title",
location: { directory: "/workspace" },
}
const fetch = Object.assign(
async (input: string | URL | Request, init?: RequestInit) => {
const request = input instanceof Request ? input : new Request(input.toString(), init)
requests.push(request)
const url = new URL(request.url)
if (url.pathname === "/api/event") return new Response("", { headers: { "content-type": "text/event-stream" } })
if (url.pathname === "/api/project")
return Response.json([
{
id: "project-1",
worktree: "/workspace",
time: { created: 1, updated: 2 },
sandboxes: [],
},
])
if (url.pathname === "/api/session" && request.method === "GET")
return Response.json({ data: [session], cursor: {} })
if (url.pathname === "/api/session/ses_known" && request.method === "GET") return Response.json({ data: session })
if (url.pathname === "/api/session/ses_known/rename" && request.method === "POST")
return new Response(null, { status: 204 })
return new Response("Not found", { status: 404 })
},
{ preconnect: () => {} },
)
const client = OpenCode.make({
baseUrl: "https://opencode.test",
fetch,
})
const bridge = await createOpenCodeBridge({
client,
directory: "/workspace",
model: { providerID: "openai", id: "gpt-test" },
notify: () => {},
completionStore: memoryCompletionStore(),
})
const execute = async (name: string, input: Record<string, unknown>) => (await bridge.execute(name, input)).output
try {
expect(new Set(bridge.definitions.map((tool) => tool.name)).size).toBe(bridge.definitions.length)
expect(bridge.definitions.every((tool) => tool.name.length > 0 && tool.description.length > 0)).toBe(true)
expect(bridge.definitions.find((tool) => tool.name === "reply_form")).toMatchObject({
parameters: {
properties: { answer: { type: "object", additionalProperties: true } },
},
})
expect(bridge.definitions.find((tool) => tool.name === "rename_session")).toMatchObject({
description: expect.stringContaining("does not prompt, wake, or interrupt"),
parameters: {
type: "object",
additionalProperties: false,
required: ["session_id", "title"],
},
})
expect(await execute("rename_session", { session_id: "ses_unknown", title: "Nope" })).toEqual({
status: "error",
message: "Use a session ID returned by find_sessions or start_session.",
retryable: false,
})
expect(requests.some((request) => new URL(request.url).pathname.includes("ses_unknown"))).toBe(false)
await execute("find_sessions", {
query: null,
scope: "current_project",
recency: "any",
limit: 10,
})
expect(await execute("rename_session", { session_id: "ses_known", title: " " })).toEqual({
status: "error",
message: "A non-empty session title is required.",
retryable: false,
})
expect(await execute("rename_session", { session_id: "ses_known", title: "Old title" })).toEqual({
status: "unchanged",
session_id: "ses_known",
previous_title: "Old title",
title: "Old title",
})
expect(requests.some((request) => new URL(request.url).pathname.endsWith("/rename"))).toBe(false)
expect(await execute("rename_session", { session_id: "ses_known", title: " New title " })).toEqual({
status: "renamed",
session_id: "ses_known",
previous_title: "Old title",
title: "New title",
})
const rename = requests.find((request) => new URL(request.url).pathname.endsWith("/rename"))
expect(rename?.method).toBe("POST")
expect(await rename?.json()).toEqual({ title: "New title" })
} finally {
await bridge.close()
}
})
test("retries a lost completion wait and prompts a started session by ID", async () => {
const requests: Request[] = []
const prompts: Array<{ readonly id: string; readonly text: string }> = []
const notifications: Array<unknown> = []
const traces: Array<{ readonly event: string; readonly data?: Record<string, unknown> }> = []
let waits = 0
const fetch = Object.assign(
async (input: string | URL | Request, init?: RequestInit) => {
const request = input instanceof Request ? input : new Request(input.toString(), init)
requests.push(request)
const url = new URL(request.url)
if (url.pathname === "/api/event") return new Response("", { headers: { "content-type": "text/event-stream" } })
if (url.pathname === "/api/session" && request.method === "POST")
return Response.json({
data: {
id: "ses_focused",
projectID: "project-1",
cost: "0",
tokens: { input: 0, output: 0, reasoning: 0, cache: { read: 0, write: 0 } },
time: { created: 1, updated: 1 },
title: "Focused session",
location: { directory: "/workspace" },
},
})
if (url.pathname === "/api/session/ses_focused/prompt" && request.method === "POST") {
const body = await request.json()
if (
typeof body !== "object" ||
body === null ||
!("id" in body) ||
typeof body.id !== "string" ||
!("text" in body) ||
typeof body.text !== "string"
)
return new Response("Invalid prompt", { status: 400 })
prompts.push({ id: body.id, text: body.text })
return Response.json({ data: { id: body.id, admittedSeq: prompts.length } })
}
if (url.pathname === "/api/session/ses_focused/wait" && request.method === "POST") {
waits += 1
if (waits === 1) throw new Error("connection closed")
return new Response(null, { status: 204 })
}
if (url.pathname === "/api/session/ses_focused/message" && request.method === "GET")
return Response.json({
data: prompts.toReversed().map((prompt) => ({ id: prompt.id, type: "user", text: prompt.text })),
cursor: {},
})
return new Response("Not found", { status: 404 })
},
{ preconnect: () => {} },
)
const bridge = await createOpenCodeBridge({
client: OpenCode.make({ baseUrl: "https://opencode.test", fetch }),
directory: "/workspace",
model: { providerID: "openai", id: "gpt-test" },
notify: (notification) => notifications.push(notification),
trace: (event, data) => traces.push({ event, data }),
completionStore: memoryCompletionStore(),
})
const execute = async (name: string, input: Record<string, unknown>) => (await bridge.execute(name, input)).output
try {
expect(bridge.definitions.some((tool) => tool.name === "continue_session")).toBe(false)
const started = await bridge.execute("start_session", { text: "First task", project_id: null })
expect(started.output).toMatchObject({
status: "started",
session_id: "ses_focused",
})
expect(started.admittedPrompt).toEqual({ sessionID: "ses_focused", promptID: prompts[0].id })
expect(
await execute("prompt_session", { session_id: "ses_focused", text: "Follow-up task" }),
).toMatchObject({
status: "started",
session_id: "ses_focused",
})
await Bun.sleep(1_100)
expect(prompts.map((prompt) => prompt.text)).toEqual(["First task", "Follow-up task"])
expect(requests.filter((request) => new URL(request.url).pathname.endsWith("/wait")).length).toBeGreaterThanOrEqual(
3,
)
expect(traces.some((entry) => entry.event === "opencode.wait.retrying")).toBe(true)
expect(notifications.some((value) => JSON.stringify(value).includes("opencode.prompt.failed"))).toBe(false)
} finally {
await bridge.close()
}
})
test("discards a restored prompt whose session no longer exists", async () => {
const removed: Array<{ readonly sessionID: string; readonly promptID: string }> = []
const handle = { sessionID: "ses_missing", promptID: "msg_orphaned" }
const fetch = Object.assign(
async (input: string | URL | Request) => {
const request = input instanceof Request ? input : new Request(input.toString())
const url = new URL(request.url)
if (url.pathname === "/api/event") return new Response("", { headers: { "content-type": "text/event-stream" } })
if (url.pathname === "/api/session/ses_missing")
return Response.json(
{ _tag: "SessionNotFoundError", sessionID: handle.sessionID, message: `Session not found: ${handle.sessionID}` },
{ status: 404 },
)
return new Response("Not found", { status: 404 })
},
{ preconnect: () => {} },
)
const store = {
...memoryCompletionStore(),
entries: () => [{ status: "pending" as const, handle }],
remove: async (entry) => {
removed.push(entry)
},
} satisfies CompletionStore
const bridge = await createOpenCodeBridge({
client: OpenCode.make({ baseUrl: "https://opencode.test", fetch }),
directory: "/workspace",
model: { providerID: "openai", id: "gpt-test" },
notify: () => {},
completionStore: store,
})
try {
expect(removed).toEqual([handle])
} finally {
await bridge.close()
}
})
function memoryCompletionStore(): CompletionStore {
return {
entries: () => [],
admitting: async () => {},
pending: async () => {},
completed: async () => {},
remove: async () => {},
close: async () => {},
}
}
+10
View File
@@ -0,0 +1,10 @@
import { expect, test } from "bun:test"
import { pcmLevel } from "../src/pcm"
test("PCM level ignores silence and normalizes audible energy", () => {
expect(pcmLevel(Buffer.alloc(4_800))).toBe(0)
const audible = Buffer.alloc(4_800)
for (let offset = 0; offset < audible.length; offset += 2) audible.writeInt16LE(8_000, offset)
expect(pcmLevel(audible)).toBeGreaterThan(0.8)
expect(pcmLevel(audible)).toBeLessThanOrEqual(1)
})
+137
View File
@@ -0,0 +1,137 @@
import { describe, expect, test } from "bun:test"
import { Option } from "effect"
import { decodeVoiceToolInput } from "../src/protocol"
import { createLiveContextAppendQueue, liveNotificationAcknowledgement } from "../src/protocol-live"
import { realtimeNotificationAcknowledgement, realtimeSessionUpdate } from "../src/protocol-realtime"
import { createSingleFlightAcknowledgement } from "../src/single-flight-acknowledgement"
import { openCodeAnnouncementText } from "../src/opencode-notification"
describe("voice protocol", () => {
test("accepts only JSON object tool arguments", () => {
expect(Option.getOrUndefined(decodeVoiceToolInput('{"text":"hello","nested":{"value":1}}'))).toEqual({
text: "hello",
nested: { value: 1 },
})
for (const input of ["not json", "null", "[]", "1", '"text"'])
expect(Option.getOrUndefined(decodeVoiceToolInput(input))).toBeUndefined()
})
test("correlates Realtime notification acknowledgements", () => {
const pending = { itemID: "item-current", eventID: "event-current" }
expect(
realtimeNotificationAcknowledgement(
{ type: "conversation.item.created", item: { id: "item-current" } },
pending,
),
).toBe(true)
expect(
realtimeNotificationAcknowledgement(
{ type: "conversation.item.created", item: { id: "item-other" } },
pending,
),
).toBeUndefined()
expect(
realtimeNotificationAcknowledgement(
{ type: "error", error: { event_id: "event-current" } },
pending,
),
).toBe(false)
})
test("preserves tool schemas in Realtime session updates", () => {
const tools = [
{
type: "function" as const,
name: "open_form",
description: "Open form",
parameters: {
type: "object",
properties: { answer: { type: "object", additionalProperties: true } },
additionalProperties: true,
},
},
]
expect(JSON.parse(JSON.stringify(realtimeSessionUpdate({ instructions: "test", tools, voice: "marin" }))).session.tools)
.toEqual(tools)
})
test("correlates Live notification acknowledgements by expected context", () => {
const pending = { eventID: "event-current", acknowledgement: "delegation.context.appended" as const }
expect(
liveNotificationAcknowledgement(
{ type: "delegation.context.appended", eventID: "event-current" },
pending,
),
).toBe(true)
expect(
liveNotificationAcknowledgement({ type: "session.context.appended", eventID: "event-current" }, pending),
).toBeUndefined()
expect(
liveNotificationAcknowledgement(
{ type: "delegation.context.appended", eventID: "event-other" },
pending,
),
).toBe(true)
expect(
liveNotificationAcknowledgement({ type: "error", errorEventID: "event-current" }, pending),
).toBe(false)
})
test("serializes indistinguishable Live context appends", async () => {
const sent: Array<Record<string, unknown>> = []
const queue = createLiveContextAppendQueue({
send: (event) => {
sent.push(event)
return true
},
timeoutMs: 100,
})
const first = queue.append(
{ type: "delegation.context.append", event_id: "event-1" },
"delegation.context.appended",
)
const second = queue.append(
{ type: "delegation.context.append", event_id: "event-2" },
"delegation.context.appended",
)
expect(sent.map((event) => event["event_id"])).toEqual(["event-1"])
queue.receive({ type: "session.context.appended", eventID: "unrelated" })
expect(sent.map((event) => event["event_id"])).toEqual(["event-1"])
queue.receive({ type: "delegation.context.appended", eventID: "not-echoed" })
expect(await first).toBe(true)
expect(sent.map((event) => event["event_id"])).toEqual(["event-1", "event-2"])
queue.receive({ type: "delegation.context.appended" })
expect(await second).toBe(true)
queue.close()
})
test("keeps notification acknowledgement single-flight", async () => {
const acknowledgement = createSingleFlightAcknowledgement<{ readonly token: string }>(50)
const first = acknowledgement.begin("notification-1", { token: "first" })
const second = acknowledgement.begin("notification-2", { token: "second" })
expect(first.started).toBe(true)
expect(second.started).toBe(false)
expect(await second.promise).toBe(false)
expect(acknowledgement.current()).toEqual({ id: "notification-1", correlation: { token: "first" } })
acknowledgement.settle("notification-1", true)
expect(await first.promise).toBe(true)
expect(acknowledgement.current()).toBeUndefined()
})
test("renders bounded conversational OpenCode announcements without identifiers", () => {
const text = openCodeAnnouncementText({
type: "opencode.prompt.completed",
session_id: "ses_private",
prompt_id: "msg_private",
status: "completed",
text: "finished ".repeat(200),
})
expect(text.length).toBeLessThanOrEqual(800)
expect(text).not.toContain("ses_private")
expect(text).not.toContain("msg_private")
expect(text).toStartWith("OpenCode prompt completed: finished")
})
})
@@ -0,0 +1,69 @@
import { expect, test } from "bun:test"
import { createRealtimeEventProjector } from "../src/protocol-realtime"
test("projects one tool start and one work request for duplicate item frames", () => {
const projector = createRealtimeEventProjector()
const item = {
type: "function_call",
name: "find_sessions",
call_id: "call-1",
arguments: '{"query":null}',
}
const added = projector.receive(JSON.stringify({ type: "response.output_item.added", item }))
const duplicate = projector.receive(JSON.stringify({ type: "response.output_item.added", item }))
const done = projector.receive(JSON.stringify({ type: "response.output_item.done", item }))
expect(added.events).toContainEqual({ type: "tool.started", id: "call-1", name: "find_sessions" })
expect(duplicate.events.some((event) => event.type === "tool.started")).toBe(false)
expect(done.events).toContainEqual({
type: "work.requested",
request: { id: "call-1", name: "find_sessions", input: { query: null } },
})
expect(projector.receive(JSON.stringify({ type: "response.output_item.done", item })).events).toEqual([
{ type: "debug", message: "response.output_item.done" },
])
})
test("resumes a response only after all projected work resolves", () => {
const projector = createRealtimeEventProjector()
const response = projector.receive(
JSON.stringify({
type: "response.done",
response: { output: [{ type: "function_call", call_id: "call-1" }] },
}),
)
expect(response.events).toContainEqual({ type: "assistant.done", awaitingWork: true })
expect(response.commands).toEqual([])
expect(projector.resolveWork("call-1")).toEqual([{ type: "response.create" }])
expect(projector.resolveWork("call-1")).toEqual([])
})
test("resumes when work resolved before the response completion frame", () => {
const projector = createRealtimeEventProjector()
expect(projector.resolveWork("call-1")).toEqual([])
expect(
projector.receive(
JSON.stringify({
type: "response.done",
response: { output: [{ type: "function_call", call_id: "call-1" }] },
}),
).commands,
).toEqual([{ type: "response.create" }])
})
test("rejects malformed tool arguments without hanging the provider call", () => {
const projector = createRealtimeEventProjector()
const result = projector.receive(
JSON.stringify({
type: "response.output_item.done",
item: { type: "function_call", name: "find_sessions", call_id: "call-1", arguments: "[]" },
}),
)
expect(result.events).toContainEqual({
type: "work.rejected",
request: { id: "call-1", name: "find_sessions", input: {} },
output: { status: "error", message: "Invalid arguments for tool find_sessions." },
})
})
@@ -0,0 +1,229 @@
import { expect, test } from "bun:test"
import {
createResponsesControllerContext,
responsesControllerTools,
runResponsesController,
} from "../src/responses-controller"
test("runs the client delegation controller through OpenCode tools", async () => {
const bodies: Array<Record<string, unknown>> = []
const calls: Array<unknown> = []
const responses = [
{
id: "response-1",
output: [
{
type: "function_call",
call_id: "call-1",
name: "start_session",
arguments: '{"text":"Inspect the package","project_id":null}',
},
],
},
{
id: "response-2",
output: [
{
type: "message",
content: [{ type: "output_text", text: "The inspection is running and I will report back." }],
},
],
},
]
const fetch = Object.assign(
async (_input: string | URL | Request, init?: RequestInit) => {
const body: unknown = await new Request("https://opencode.test", init).json()
if (body && typeof body === "object" && !Array.isArray(body)) bodies.push(Object.fromEntries(Object.entries(body)))
return Response.json(responses.shift())
},
{ preconnect: () => {} },
)
const text = await runResponsesController({
apiKey: "test",
model: "gpt-test",
instructions: "Use tools.",
text: "Inspect this package",
tools: [{ type: "function", name: "start_session", description: "Start", parameters: {} }],
execute: async (call) => {
calls.push(call)
return { status: "started", session_id: "ses_started", prompt_id: "prompt-1" }
},
fetch,
})
expect(text).toBe(
"The inspection is running and I will report back.\n\nPrivate voice-control context (never speak this aloud): OpenCode session IDs explicitly used by this delegation: ses_started. Include the relevant session_id in any future delegated request to continue this work.",
)
expect(calls).toEqual([
{
id: "call-1",
name: "start_session",
input: { text: "Inspect the package", project_id: null },
},
])
expect(bodies[1]).toMatchObject({
previous_response_id: "response-1",
input: [
{
type: "function_call_output",
call_id: "call-1",
output: '{"status":"started","session_id":"ses_started","prompt_id":"prompt-1"}',
},
],
})
expect(bodies[0]?.["tools"]).toEqual([
{ type: "function", name: "start_session", description: "Start", parameters: {} },
])
})
test("rejects malformed controller tool arguments", async () => {
const fetch = Object.assign(
async () =>
Response.json({
id: "response-1",
output: [{ type: "function_call", call_id: "call-1", name: "start_session", arguments: "not json" }],
}),
{ preconnect: () => {} },
)
expect(
runResponsesController({
apiKey: "test",
model: "gpt-test",
instructions: "Use tools.",
text: "Inspect this package",
tools: [{ type: "function", name: "start_session", description: "Start", parameters: {} }],
execute: async () => ({ status: "started" }),
fetch,
}),
).rejects.toThrow("invalid arguments for start_session")
})
test("retains the most recently used session across delegations", async () => {
const context = createResponsesControllerContext()
const bodies: Array<Record<string, unknown>> = []
const responses = [
{
id: "response-1",
output: [
{
type: "function_call",
call_id: "call-1",
name: "read_session",
arguments: '{"session_id":"ses_recent"}',
},
],
},
{
id: "response-2",
output: [{ type: "message", content: [{ type: "output_text", text: "Found it." }] }],
},
{
id: "response-3",
output: [{ type: "message", content: [{ type: "output_text", text: "Stopped it." }] }],
},
]
const fetch = Object.assign(
async (_input: string | URL | Request, init?: RequestInit) => {
const body: unknown = await new Request("https://opencode.test", init).json()
if (body && typeof body === "object" && !Array.isArray(body)) bodies.push(Object.fromEntries(Object.entries(body)))
return Response.json(responses.shift())
},
{ preconnect: () => {} },
)
const options = {
apiKey: "test",
model: "gpt-test",
instructions: "Use tools.",
tools: [{ type: "function" as const, name: "read_session", description: "Read", parameters: {} }],
execute: async () => ({ status: "ok" }),
context,
fetch,
}
await runResponsesController({ ...options, text: "Find the label behavior session" })
await runResponsesController({ ...options, text: "Stop that session" })
expect(context.lastSessionID).toBe("ses_recent")
expect(bodies[2]?.["input"]).toContain("the most recently used OpenCode session ID is ses_recent")
expect(bodies[2]?.["input"]).toContain('references like "that session", "it", or "stop that"')
})
test("coalesces identical read-only calls within one delegation", async () => {
const responses = [
{
id: "response-1",
output: [
{ type: "function_call", call_id: "call-1", name: "read_session", arguments: '{"session_id":"ses_1"}' },
{ type: "function_call", call_id: "call-2", name: "read_session", arguments: '{"session_id":"ses_1"}' },
],
},
{
id: "response-2",
output: [{ type: "message", content: [{ type: "output_text", text: "Done." }] }],
},
]
let executions = 0
const fetch = Object.assign(async () => Response.json(responses.shift()), { preconnect: () => {} })
await runResponsesController({
apiKey: "test",
model: "gpt-test",
instructions: "Use tools.",
text: "Read it",
tools: [{ type: "function", name: "read_session", description: "Read", parameters: {} }],
execute: async () => {
executions += 1
return { status: "ok" }
},
fetch,
})
expect(executions).toBe(1)
})
test("withholds new-session creation during follow-ups unless explicitly requested", () => {
const context = createResponsesControllerContext()
context.lastSessionID = "ses_recent"
const tools = [
{ type: "function" as const, name: "start_session", description: "Start", parameters: {} },
{ type: "function" as const, name: "read_session", description: "Read", parameters: {} },
]
expect(responsesControllerTools(tools, "Stop that session", context).map((tool) => tool.name)).toEqual([
"read_session",
])
expect(responsesControllerTools(tools, "Start a new session for this", context)).toEqual(tools)
})
test("remembers a uniquely discovered session without requiring a read", async () => {
const context = createResponsesControllerContext()
const responses = [
{
id: "response-1",
output: [
{ type: "function_call", call_id: "call-1", name: "find_sessions", arguments: '{"query":"label"}' },
],
},
{
id: "response-2",
output: [{ type: "message", content: [{ type: "output_text", text: "Found one." }] }],
},
]
const fetch = Object.assign(async () => Response.json(responses.shift()), { preconnect: () => {} })
await runResponsesController({
apiKey: "test",
model: "gpt-test",
instructions: "Use tools.",
text: "Find label behavior",
tools: [{ type: "function", name: "find_sessions", description: "Find", parameters: {} }],
execute: async () => ({ status: "ok", sessions: [{ id: "ses_unique", title: "Label behavior" }] }),
context,
fetch,
})
expect(context.lastSessionID).toBe("ses_unique")
expect(context.sessionIDs).toEqual(new Set(["ses_unique"]))
})
+331
View File
@@ -0,0 +1,331 @@
import { expect, mock, test } from "bun:test"
import { createTestRenderer } from "@opentui/core/testing"
test("voice rows fill the viewport, wrap under content, and keep tools after their preamble", async () => {
const setup = await createTestRenderer({ width: 64, height: 18, useThread: false })
const core = await import("@opentui/core")
void mock.module("@opentui/core", () => ({ ...core, createCliRenderer: async () => setup.renderer }))
const { createVoiceTUI } = await import("../src/ui")
let interrupts = 0
let voiceCycles = 0
let microphoneToggles = 0
let speakerToggles = 0
const ui = await createVoiceTUI({
onInterrupt: () => interrupts++,
onExit: () => {},
onCycleVoice: () => voiceCycles++,
onToggleMicrophone: () => microphoneToggles++,
onToggleSpeaker: () => speakerToggles++,
reducedMotion: true,
})
try {
ui.setStatus({ audio: "duplex", voice: "marin", model: "gpt-live" })
ui.meta("[session] created session-1")
ui.userCommitted("user-1")
ui.userTranscript("user-1", "Please inspect the current session", true)
ui.toolStart("tool-1", "opencode", { request: "inspect" })
ui.meta("[session] adopted session-1")
ui.assistantDelta("Let me check. This sentence is")
ui.assistantDelta(" long enough to wrap cleanly under its column.")
ui.assistantDone()
ui.assistantDelta("Continued in the same assistant row.")
ui.assistantDone()
await Bun.sleep(200)
await setup.flush()
const rows = frameRows(setup.captureCharFrame())
expect(rows).toHaveLength(18)
expect(rows.every((row) => row.length === 64)).toBe(true)
expect(rows[16]).toStartWith(" ▂▂▂▁ duplex marin gpt-live")
expect(rows[16]).not.toContain("no session")
expect(rows[17]).toContain("esc interrupt")
expect(rows[17]).not.toContain("any key")
expect(rows[15].trim()).toBe("")
expect(rows.filter((row) => row.includes("Let me check"))).toHaveLength(1)
expect(rows.findIndex((row) => row.includes("opencode"))).toBeGreaterThan(
rows.findIndex((row) => row.includes("Let me check")),
)
expect(rows.findIndex((row) => row.includes("[session] adopted"))).toBeGreaterThan(
rows.findIndex((row) => row.includes("opencode")),
)
expect(rows.find((row) => row.includes("under its column"))?.indexOf("under")).toBe(
rows.find((row) => row.includes("Let me check"))?.indexOf("Let"),
)
// Deltas inside one turn concatenate raw; a delta after assistantDone starts a new
// message and must gain exactly one space at the boundary.
const assistant = rows.join("")
expect(assistant).toContain("This sentence is long enough")
expect(assistant).toContain("column. Continued")
expect(assistant).not.toContain("column.Continued")
expect(rows.find((row) => row.includes("[session] created"))).toStartWith(" · [session]")
setup.resize(96, 22)
await setup.flush()
const wide = frameRows(setup.captureCharFrame())
expect(wide).toHaveLength(22)
expect(wide.every((row) => row.length === 96)).toBe(true)
expect(wide.at(-2)).toStartWith(" ▂▂▂▁ duplex marin gpt-live")
setup.resize(48, 16)
await setup.flush()
const narrow = frameRows(setup.captureCharFrame())
expect(narrow).toHaveLength(16)
expect(narrow.every((row) => row.length === 48)).toBe(true)
expect(narrow.at(-1)).toContain("esc interrupt")
setup.resize(64, 18)
await setup.flush()
ui.userSpeaking(true)
await setup.flush()
expect(setup.captureCharFrame()).not.toContain("│ transcribing")
expect(frameRows(setup.captureCharFrame()).at(-2)).toStartWith(" ▂▂▂▂")
ui.userReset()
await setup.flush()
expect(setup.captureCharFrame()).not.toContain("│ transcribing")
ui.userSpeaking(true)
ui.userSpeaking(false)
ui.userCommitted("user-2")
ui.userTranscript("user-2", "partial transcript", false)
await setup.flush()
const partial = setup
.captureCharFrame()
.split("\n")
.find((row) => row.includes("partial transcript"))
expect(partial).toContain("│ partial transcript")
expect(partial).not.toContain("you")
ui.userSpeaking(true)
await setup.flush()
expect(
setup
.captureCharFrame()
.split("\n")
.filter((row) => row.includes("│ transcribing")),
).toHaveLength(0)
ui.userReset()
await setup.flush()
ui.setStatus({ microphoneMuted: true })
await setup.flush()
expect(frameRows(setup.captureCharFrame()).at(-2)).toStartWith(" ────")
const muted = setup
.captureCharFrame()
.split("\n")
.find((row) => row.includes("partial transcript"))
expect(muted).toContain("│ partial transcript")
expect(muted).not.toContain("│ ...")
ui.userSpeaking(false)
ui.setStatus({ microphoneMuted: false })
ui.userTranscript("user-2", "partial transcript", true)
await setup.flush()
const complete = setup
.captureCharFrame()
.split("\n")
.find((row) => row.includes("partial transcript"))
expect(complete).toContain("│ partial transcript")
ui.assistantDelta("First response.")
ui.userCommitted("boundary-user")
ui.userTranscript("boundary-user", "A new request", true)
ui.assistantDelta("Second response.")
await setup.flush()
const boundaries = setup.captureCharFrame().split("\n")
expect(boundaries.findIndex((row) => row.includes("First response."))).toBeLessThan(
boundaries.findIndex((row) => row.includes("A new request")),
)
expect(boundaries.findIndex((row) => row.includes("A new request"))).toBeLessThan(
boundaries.findIndex((row) => row.includes("Second response.")),
)
await setup.mockInput.typeText(" x")
setup.mockInput.pressArrow("right")
await setup.flush()
expect(interrupts).toBe(0)
expect(voiceCycles).toBe(0)
expect(microphoneToggles).toBe(0)
expect(speakerToggles).toBe(0)
setup.mockInput.pressKey("v")
setup.mockInput.pressKey("m")
setup.mockInput.pressKey("s")
await setup.flush()
expect(voiceCycles).toBe(1)
expect(microphoneToggles).toBe(1)
expect(speakerToggles).toBe(1)
setup.mockInput.pressEscape()
await Bun.sleep(50)
await setup.flush()
expect(interrupts).toBe(1)
} finally {
ui.close()
mock.restore()
}
})
// In audio mode ui.assistantDone() is scheduled off the AudioSession playback clock,
// so the next turn's transcript deltas arrive while the row is still streaming. The turn boundary
// therefore has to survive in the delta text itself, not in the row's streaming flag.
test("keeps the turn boundary when the next turn streams before playback finishes", async () => {
const setup = await createTestRenderer({ width: 72, height: 12, useThread: false })
const core = await import("@opentui/core")
void mock.module("@opentui/core", () => ({ ...core, createCliRenderer: async () => setup.renderer }))
const { createVoiceTUI } = await import("../src/ui")
const ui = await createVoiceTUI({
onInterrupt: () => {},
onExit: () => {},
onCycleVoice: () => {},
onToggleMicrophone: () => {},
onToggleSpeaker: () => {},
reducedMotion: true,
})
try {
// Verbatim projector deltas: each turn's first fragment carries the boundary space.
ui.assistantDelta(' Session "Fix auth bug"')
ui.assistantDelta(" is ready.")
// turn.done reaches the protocol, but playback is still draining, so no assistantDone() yet.
ui.assistantDelta(" Next up, tests.")
await setup.flush()
const frame = setup.captureCharFrame()
expect(frame).toContain('Session "Fix auth bug" is ready. Next up, tests.')
expect(frame).not.toContain("ready.Next")
// The row itself must not open with the boundary space.
expect(frame).toContain('│ Session "Fix auth bug"')
expect(frame).not.toContain('│ Session "Fix auth bug"')
ui.assistantDone()
await setup.flush()
expect(setup.captureCharFrame()).toContain("is ready. Next up, tests.")
} finally {
ui.close()
mock.restore()
}
})
test("keeps animated text mounted after its reveal completes", async () => {
const setup = await createTestRenderer({ width: 72, height: 10, useThread: false })
const core = await import("@opentui/core")
void mock.module("@opentui/core", () => ({ ...core, createCliRenderer: async () => setup.renderer }))
const { createVoiceTUI } = await import("../src/ui")
const ui = await createVoiceTUI({
onInterrupt: () => {},
onExit: () => {},
onCycleVoice: () => {},
onToggleMicrophone: () => {},
onToggleSpeaker: () => {},
})
try {
ui.assistantDelta("Animated text remains visible.")
await Bun.sleep(500)
await setup.flush()
expect(setup.captureCharFrame()).toContain("Animated text remains visible.")
} finally {
ui.close()
mock.restore()
}
})
test("renders consecutive tools as compact single-spaced rows", async () => {
const setup = await createTestRenderer({ width: 52, height: 8, useThread: false })
const core = await import("@opentui/core")
void mock.module("@opentui/core", () => ({ ...core, createCliRenderer: async () => setup.renderer }))
const { createVoiceTUI } = await import("../src/ui")
const ui = await createVoiceTUI({
onInterrupt: () => {},
onExit: () => {},
onCycleVoice: () => {},
onToggleMicrophone: () => {},
onToggleSpeaker: () => {},
reducedMotion: true,
})
try {
ui.toolStart("tool-1", "read_session", {})
ui.toolDone("tool-1", { status: "ok" })
ui.toolStart("tool-2", "read_session", {})
ui.toolDone("tool-2", { status: "ok" })
await Bun.sleep(200)
await setup.flush()
const rows = frameRows(setup.captureCharFrame())
const tools = rows.flatMap((row, index) => (row.includes("read_session") ? [{ row, index }] : []))
expect(tools).toHaveLength(2)
expect(tools[1]?.index).toBe((tools[0]?.index ?? 0) + 1)
expect(tools.every((tool) => tool.row.includes("✓ read_session ok"))).toBe(true)
expect(rows.at(-3)?.trim()).toBe("")
} finally {
ui.close()
mock.restore()
}
})
test("shows the reason a tool failed", async () => {
const setup = await createTestRenderer({ width: 72, height: 8, useThread: false })
const core = await import("@opentui/core")
void mock.module("@opentui/core", () => ({ ...core, createCliRenderer: async () => setup.renderer }))
const { createVoiceTUI } = await import("../src/ui")
const ui = await createVoiceTUI({
onInterrupt: () => {},
onExit: () => {},
onCycleVoice: () => {},
onToggleMicrophone: () => {},
onToggleSpeaker: () => {},
reducedMotion: true,
})
try {
ui.toolStart("tool-error", "opencode", {})
ui.toolDone("tool-error", { status: "error", message: "Invalid tool schema" })
await Bun.sleep(200)
await setup.flush()
expect(setup.captureCharFrame()).toContain("opencode error · Invalid tool schema")
} finally {
ui.close()
mock.restore()
}
})
test("paints the full viewport and keeps mic activity in the footer", async () => {
const setup = await createTestRenderer({ width: 40, height: 12, useThread: false })
const core = await import("@opentui/core")
void mock.module("@opentui/core", () => ({ ...core, createCliRenderer: async () => setup.renderer }))
const { createVoiceTUI } = await import("../src/ui")
const ui = await createVoiceTUI({
onInterrupt: () => {},
onExit: () => {},
onCycleVoice: () => {},
onToggleMicrophone: () => {},
onToggleSpeaker: () => {},
reducedMotion: true,
})
try {
ui.userSpeaking(true)
await setup.flush()
const rows = frameRows(setup.captureCharFrame())
expect(rows.some((row) => row.includes("│ transcribing"))).toBe(false)
expect(rows.at(-2)).toStartWith(" ▂▂▂▂")
expect(setup.captureSpans().lines.every((line) => line.spans.every((span) => span.bg.a === 1))).toBe(true)
setup.resize(56, 16)
await setup.flush()
expect(setup.captureSpans().lines.every((line) => line.spans.every((span) => span.bg.a === 1))).toBe(true)
} finally {
ui.close()
mock.restore()
}
})
function frameRows(frame: string) {
return frame.split("\n").slice(0, -1)
}
+74
View File
@@ -0,0 +1,74 @@
import { describe, expect, test } from "bun:test"
import { initialVoiceView, transitionVoiceView, type VoiceViewEvent, type VoiceViewState } from "../src/ui-model"
const apply = (state: VoiceViewState, ...events: ReadonlyArray<VoiceViewEvent>) =>
events.reduce(transitionVoiceView, state)
describe("VoiceView", () => {
test("keeps assistant responses separated by a committed user turn", () => {
const state = apply(
initialVoiceView(),
{ type: "assistant.delta", text: "First response.", now: 0, animate: false },
{ type: "user.committed", itemID: "user-1" },
{ type: "user.transcript", itemID: "user-1", text: "Next request", final: true, now: 1, animate: false },
{ type: "assistant.delta", text: "Second response.", now: 2, animate: false },
)
expect(state.messages.map((message) => message.kind)).toEqual(["assistant", "user", "assistant"])
expect(state.messages.flatMap((message) => ("text" in message ? [message.text] : []))).toEqual([
"First response.",
"Next request",
"Second response.",
])
})
test("keeps local sound activity out of transcript history", () => {
const started = transitionVoiceView(initialVoiceView(), { type: "user.started" })
expect(started.messages).toEqual([])
const committed = transitionVoiceView(started, { type: "user.committed", itemID: "user-1" })
expect(committed.messages).toEqual([
{ key: "message-1", kind: "user", itemID: "user-1", transcribing: true, reveals: [] },
])
const abandoned = apply(initialVoiceView(), { type: "user.started" }, { type: "user.reset" })
expect(abandoned.messages).toEqual([])
})
test("deduplicates transcript snapshots and clears completed reveals", () => {
const state = apply(
initialVoiceView(),
{ type: "user.committed", itemID: "user-1" },
{ type: "user.transcript", itemID: "user-1", text: "Hello", final: false, now: 0, animate: true },
{ type: "user.transcript", itemID: "user-1", text: "Hello", final: false, now: 10, animate: true },
)
expect(state.messages).toHaveLength(1)
expect(state.messages[0]?.kind === "user" ? state.messages[0].reveals.length : 0).toBeGreaterThan(0)
expect(transitionVoiceView(state, { type: "reveals.completed" }).messages).toEqual([
{ key: "message-1", kind: "user", itemID: "user-1", text: "Hello", transcribing: true, reveals: [] },
])
})
test("bounds message history", () => {
const state = Array.from({ length: 205 }, (_, index) => index).reduce(
(current, index) => transitionVoiceView(current, { type: "meta", text: String(index) }),
initialVoiceView(),
)
expect(state.messages).toHaveLength(200)
expect(state.messages[0]).toEqual({ key: "message-6", kind: "meta", text: "5" })
})
test("settles tool completion before or after insertion", () => {
const inserted = apply(
initialVoiceView(),
{ type: "tool.started", callID: "call-1", name: "read_session", input: {} },
{ type: "tool.done", callID: "call-1", output: { status: "ok" } },
)
expect(inserted.messages[0]).toEqual({
key: "message-1",
kind: "tool",
callID: "call-1",
name: "read_session",
input: {},
output: { status: "ok" },
})
})
})
@@ -0,0 +1,277 @@
import { describe, expect, test } from "bun:test"
import { initialVoiceState, transitionVoice, type VoiceEvent, type VoiceState } from "../src/voice-coordinator"
const apply = (state: VoiceState, ...events: ReadonlyArray<VoiceEvent>) =>
events.reduce(
(result, event) => {
const next = transitionVoice(result.state, event)
return { state: next.state, commands: [...result.commands, ...next.commands] }
},
{ state, commands: [] as Array<ReturnType<typeof transitionVoice>["commands"][number]> },
)
describe("VoiceCoordinator", () => {
test("delivers one notification at a time when the server becomes idle", () => {
const notification = { id: "notification-1", promptID: "prompt-1", text: "completed" }
const waiting = apply(
initialVoiceState("marin"),
{ type: "connection.ready" },
{ type: "user.committed" },
{ type: "notification.queued", notification },
)
expect(waiting.commands).toEqual([])
expect(transitionVoice(waiting.state, { type: "assistant.done", awaitingWork: false })).toEqual({
state: {
...waiting.state,
conversation: "waiting",
assistant: "idle",
notifications: [],
delivery: { notification, phase: "sending" },
},
commands: [
{ type: "assistant.finish", awaitingWork: false, notificationID: undefined },
{ type: "notification.send", notification },
],
})
})
test("keeps notifications queued while the user is speaking", () => {
const notification = { id: "notification-1", text: "completed" }
const result = apply(
initialVoiceState("marin"),
{ type: "connection.ready" },
{ type: "user.started" },
{ type: "notification.queued", notification },
)
expect(result.commands).toEqual([])
expect(result.state.notifications).toEqual([notification])
expect(transitionVoice(result.state, { type: "user.stopped" }).commands).toEqual([
{ type: "notification.send", notification },
])
})
test("keeps notifications queued until active tool calls finish", () => {
const notification = { id: "notification-1", text: "completed" }
const result = apply(
initialVoiceState("marin"),
{ type: "connection.ready" },
{ type: "tool.started", id: "call-1" },
{ type: "assistant.done", awaitingWork: false },
{ type: "notification.queued", notification },
)
expect(result.commands).toEqual([
{ type: "assistant.finish", awaitingWork: false, notificationID: undefined },
])
expect(transitionVoice(result.state, { type: "tool.finished", id: "call-1" }).commands).toEqual([
{ type: "notification.send", notification },
])
})
test("tracks tools by call ID and reconnects only after work and conversation settle", () => {
const busy = apply(
initialVoiceState("marin"),
{ type: "connection.ready" },
{ type: "tool.started", id: "call-1" },
{ type: "voice.selected", voice: "cedar" },
)
expect(busy.commands).toEqual([])
expect(transitionVoice(busy.state, { type: "tool.finished", id: "call-1" }).commands).toEqual([])
const idle = transitionVoice(transitionVoice(busy.state, { type: "tool.finished", id: "call-1" }).state, {
type: "assistant.done",
awaitingWork: false,
})
expect(idle.commands).toEqual([
{ type: "assistant.finish", awaitingWork: false, notificationID: undefined },
{ type: "connection.reconnect" },
])
expect(idle.state.connection).toBe("connecting")
})
test("a user commit closes assistant suppression and establishes a waiting boundary", () => {
const suppressed = apply(
initialVoiceState("marin"),
{ type: "connection.ready" },
{ type: "assistant.started" },
{ type: "assistant.suppressed" },
{ type: "user.committed" },
)
expect(suppressed.state.assistant).toBe("idle")
expect(suppressed.state.conversation).toBe("waiting")
})
test("requeues a notification when the adapter rejects it", () => {
const notification = { id: "notification-1", text: "completed" }
const sent = apply(
initialVoiceState("marin"),
{ type: "connection.ready" },
{ type: "notification.queued", notification },
)
const failed = transitionVoice(sent.state, { type: "notification.failed", id: notification.id })
expect(failed.state.notifications).toEqual([notification])
expect(failed.state.connection).toBe("connecting")
expect(failed.commands).toEqual([{ type: "connection.reconnect" }])
})
test("keeps a notification in flight until its playback completes", () => {
const notification = { id: "notification-1", promptID: "prompt-1", text: "completed" }
const sent = apply(
initialVoiceState("marin"),
{ type: "connection.ready" },
{
type: "notification.queued",
notification,
},
)
expect(sent.state.delivery).toEqual({ notification, phase: "sending" })
const accepted = transitionVoice(sent.state, { type: "notification.accepted", id: notification.id }).state
expect(accepted.delivery).toEqual({ notification, phase: "accepted" })
const speaking = transitionVoice(accepted, { type: "assistant.started" }).state
expect(speaking.delivery).toEqual({ notification, phase: "announcing" })
const generated = transitionVoice(speaking, { type: "assistant.done", awaitingWork: false })
expect(generated.state.delivery).toEqual({ notification, phase: "awaiting-playback" })
expect(generated.commands).toEqual([
{ type: "assistant.finish", awaitingWork: false, notificationID: notification.id },
])
expect(transitionVoice(generated.state, { type: "notification.announced", id: notification.id })).toEqual({
state: { ...generated.state, delivery: undefined },
commands: [{ type: "notification.delivered", notification }],
})
})
test("does not attribute an already-active assistant turn to a newly appended notification", () => {
const notification = { id: "notification-1", text: "completed" }
const active = apply(
initialVoiceState("marin"),
{ type: "connection.ready" },
{ type: "notification.queued", notification },
{ type: "assistant.started" },
{ type: "notification.accepted", id: notification.id },
).state
expect(active.delivery).toEqual({ notification, phase: "accepted" })
const existingDone = transitionVoice(active, { type: "assistant.done", awaitingWork: false })
expect(existingDone.state.delivery).toEqual({ notification, phase: "accepted" })
expect(existingDone.commands).toEqual([
{ type: "assistant.finish", awaitingWork: false, notificationID: undefined },
])
expect(transitionVoice(existingDone.state, { type: "assistant.started" }).state.delivery).toEqual({
notification,
phase: "announcing",
})
})
test("keeps assistant activity independent from a full-duplex user commit", () => {
const active = apply(initialVoiceState("marin"), { type: "connection.ready" }, { type: "assistant.started" }).state
const committed = transitionVoice(active, { type: "user.committed" }).state
expect(committed.assistant).toBe("active")
expect(committed.conversation).toBe("responding")
})
test("ignores stale notification callbacks", () => {
const notification = { id: "notification-2", text: "current" }
const sent = apply(
initialVoiceState("marin"),
{ type: "connection.ready" },
{ type: "notification.queued", notification },
).state
expect(transitionVoice(sent, { type: "notification.accepted", id: "notification-1" })).toEqual({
state: sent,
commands: [],
})
expect(transitionVoice(sent, { type: "notification.failed", id: "notification-1" })).toEqual({
state: sent,
commands: [],
})
expect(transitionVoice(sent, { type: "notification.announced", id: "notification-1" })).toEqual({
state: sent,
commands: [],
})
})
test("waits for one announcement playback before sending the next", () => {
const first = { id: "notification-1", text: "first" }
const second = { id: "notification-2", text: "second" }
const sent = apply(
initialVoiceState("marin"),
{ type: "connection.ready" },
{ type: "notification.queued", notification: first },
{ type: "notification.queued", notification: second },
{ type: "notification.accepted", id: first.id },
{ type: "assistant.started" },
{ type: "assistant.done", awaitingWork: false },
)
expect(sent.state.delivery).toEqual({ notification: first, phase: "awaiting-playback" })
expect(sent.state.notifications).toEqual([second])
expect(transitionVoice(sent.state, { type: "notification.announced", id: first.id })).toEqual({
state: {
...sent.state,
conversation: "waiting",
delivery: { notification: second, phase: "sending" },
notifications: [],
},
commands: [
{ type: "notification.delivered", notification: first },
{ type: "notification.send", notification: second },
],
})
})
test("keeps an announcement active across tool work", () => {
const notification = { id: "notification-1", text: "completed" }
const announcing = apply(
initialVoiceState("marin"),
{ type: "connection.ready" },
{ type: "notification.queued", notification },
{ type: "notification.accepted", id: notification.id },
{ type: "assistant.started" },
).state
expect(transitionVoice(announcing, { type: "assistant.done", awaitingWork: true }).state.delivery).toEqual({
notification,
phase: "announcing",
})
})
test("requeues an interrupted announcement and reconnects before retrying", () => {
const notification = { id: "notification-1", text: "completed" }
const awaitingPlayback = apply(
initialVoiceState("marin"),
{ type: "connection.ready" },
{ type: "notification.queued", notification },
{ type: "notification.accepted", id: notification.id },
{ type: "assistant.started" },
{ type: "assistant.done", awaitingWork: false },
).state
const interrupted = transitionVoice(awaitingPlayback, {
type: "notification.interrupted",
id: notification.id,
})
expect(interrupted.state.connection).toBe("connecting")
expect(interrupted.state.notifications).toEqual([notification])
expect(interrupted.commands).toEqual([{ type: "connection.reconnect" }])
})
test("holds an interrupted announcement reconnect until barge-in speech stops", () => {
const notification = { id: "notification-1", text: "completed" }
const announcing = apply(
initialVoiceState("marin"),
{ type: "connection.ready" },
{ type: "notification.queued", notification },
{ type: "notification.accepted", id: notification.id },
{ type: "assistant.started" },
).state
const barged = transitionVoice(announcing, { type: "user.started", bargeIn: true })
expect(barged.state.notifications).toEqual([notification])
expect(barged.state.userSpeaking).toBe(true)
expect(barged.state.connection).toBe("ready")
expect(barged.commands).toEqual([{ type: "assistant.interrupt" }])
expect(transitionVoice(barged.state, { type: "user.stopped" }).commands).toEqual([
{ type: "connection.reconnect" },
])
})
})
+233
View File
@@ -0,0 +1,233 @@
import { expect, mock, test } from "bun:test"
import type { AudioSession } from "../src/audio-session"
import type { VoiceConnection, VoiceProtocol, VoiceProtocolEvent, VoiceProtocolOptions } from "../src/protocol"
import type { VoiceUI } from "../src/ui"
import { createVoiceSession } from "../src/voice-session"
test("ignores events from a replaced connection", async () => {
const setup = makeSession()
setup.session.start()
setup.connections[0].emit({ type: "ready" })
setup.session.cycleVoice()
await Bun.sleep(0)
expect(setup.connections).toHaveLength(2)
setup.connections[0].emit({ type: "assistant.transcript.delta", delta: "stale" })
setup.connections[1].emit({ type: "assistant.transcript.delta", delta: "current" })
expect(setup.assistantText).toEqual(["current"])
})
test("resolves work against the connection that requested it", async () => {
const setup = makeSession()
setup.session.start()
setup.connections[0].emit({ type: "ready" })
setup.connections[0].emit({ type: "tool.started", id: "call-1", name: "find_sessions" })
setup.connections[0].emit({
type: "work.requested",
request: { id: "call-1", name: "find_sessions", input: { query: null } },
})
await Bun.sleep(0)
expect(setup.resolvedWork).toEqual([
{
request: { id: "call-1", name: "find_sessions", input: { query: null } },
output: { status: "ok" },
},
])
})
test("acknowledges a durable notification only after playback", async () => {
const setup = makeSession()
setup.session.start()
setup.connections[0].emit({ type: "ready" })
setup.session.queueNotification({
notification: {
type: "opencode.prompt.completed",
session_id: "session-1",
prompt_id: "prompt-1",
status: "completed",
text: "done",
},
receipt: { sessionID: "session-1", promptID: "prompt-1" },
})
await Bun.sleep(0)
setup.connections[0].emit({ type: "assistant.transcript.delta", delta: "Finished." })
setup.connections[0].emit({ type: "assistant.done", awaitingWork: false })
expect(setup.delivered).toEqual([])
setup.finishPlayback?.("played")
await Bun.sleep(0)
expect(setup.delivered).toEqual(["prompt-1"])
})
test("retries a notification whose playback was interrupted", async () => {
const setup = makeSession()
setup.session.start()
setup.connections[0].emit({ type: "ready" })
setup.session.queueNotification({
notification: {
type: "opencode.prompt.completed",
session_id: "session-1",
prompt_id: "prompt-1",
status: "completed",
text: "done",
},
receipt: { sessionID: "session-1", promptID: "prompt-1" },
})
await Bun.sleep(0)
setup.connections[0].emit({ type: "assistant.transcript.delta", delta: "Finished." })
setup.connections[0].emit({ type: "assistant.done", awaitingWork: false })
setup.finishPlayback?.("interrupted")
await Bun.sleep(0)
expect(setup.delivered).toEqual([])
expect(setup.connections).toHaveLength(2)
setup.connections[1].emit({ type: "ready" })
await Bun.sleep(0)
expect(setup.connections[1].notifyCalls()).toBe(1)
})
test("closes owned resources once", async () => {
const setup = makeSession()
setup.session.start()
await Promise.all([setup.session.close(), setup.session.close()])
expect(setup.audioClose).toHaveBeenCalledTimes(1)
expect(setup.toolClose).toHaveBeenCalledTimes(1)
expect(setup.connections[0].closeCalls()).toBe(1)
})
test("does not acknowledge nonterminal prompt notifications", async () => {
const setup = makeSession()
setup.session.start()
setup.connections[0].emit({ type: "ready" })
setup.session.queueNotification({
notification: {
type: "opencode.prompt.blocked",
prompt_id: "prompt-1",
blocker: "permission",
session_id: "session-1",
request_id: "permission-1",
action: "read",
resources: ["file.ts"],
},
})
await Bun.sleep(0)
setup.connections[0].emit({ type: "assistant.transcript.delta", delta: "Permission needed." })
setup.connections[0].emit({ type: "assistant.done", awaitingWork: false })
setup.finishPlayback?.("played")
await Bun.sleep(0)
expect(setup.delivered).toEqual([])
})
function makeSession() {
const connections: Array<
VoiceConnection & { emit(event: VoiceProtocolEvent): void; notifyCalls(): number; closeCalls(): number }
> = []
const assistantText: string[] = []
const resolvedWork: Array<unknown> = []
const delivered: string[] = []
const audioClose = mock(() => {})
const toolClose = mock(async () => {})
let finishPlayback: ((outcome: "played" | "interrupted" | "inaudible") => void) | undefined
const ui = {
meta: mock(() => {}),
userSpeaking: mock(() => {}),
userReset: mock(() => {}),
userAudioLevel: mock(() => {}),
userCommitted: mock(() => {}),
userTranscript: mock(() => {}),
assistantAudio: mock(() => {}),
assistantPlaybackStopped: mock(() => {}),
assistantDelta: (text: string) => assistantText.push(text),
assistantTranscript: mock(() => {}),
assistantDone: mock(() => {}),
toolStart: mock(() => {}),
toolDone: mock(() => {}),
setStatus: mock(() => {}),
close: mock(() => {}),
} satisfies VoiceUI
const audio = {
fullDuplex: false,
microphoneMuted: false,
speakerMuted: false,
start: mock(async () => {}),
toggleMicrophone: mock(() => false),
toggleSpeaker: mock(() => false),
isPlaying: () => false,
play: mock(() => {}),
finishPlayback: () => {
const result = Promise.withResolvers<"played" | "interrupted" | "inaudible">()
finishPlayback = result.resolve
return result.promise
},
flushPlayback: mock(() => {}),
noteUserCommitted: mock(() => {}),
noteUserTranscript: mock(() => {}),
close: audioClose,
} satisfies AudioSession
const protocol = {
name: "realtime",
inputActivity: "server",
capabilities: { textInput: true, interruption: true, delegation: false },
connect(options: VoiceProtocolOptions) {
const notify = mock(async () => true)
const close = mock(async () => {})
const connection = {
appendAudio: mock(() => {}),
sendText: mock(() => {}),
resolveWork: (request, output) => resolvedWork.push({ request, output }),
resolveDelegation: mock(() => {}),
notify,
interrupt: () => true,
close,
emit: options.onEvent,
notifyCalls: () => notify.mock.calls.length,
closeCalls: () => close.mock.calls.length,
} satisfies VoiceConnection & {
emit(event: VoiceProtocolEvent): void
notifyCalls(): number
closeCalls(): number
}
connections.push(connection)
return connection
},
} satisfies VoiceProtocol
const session = createVoiceSession({
protocol,
connection: {
apiKey: "test",
model: "voice-test",
instructions: "test",
tools: [],
fullDuplex: false,
debug: false,
},
initialVoice: "marin",
voices: ["marin", "cedar"],
ui,
audio,
tools: {
execute: async () => ({ output: { status: "ok" } }),
acknowledge: async (receipt) => {
delivered.push(receipt.promptID)
},
close: toolClose,
},
delegate: async () => "done",
onClosed: () => {},
})
return {
session,
connections,
assistantText,
resolvedWork,
delivered,
audioClose,
toolClose,
get finishPlayback() {
return finishPlayback
},
}
}
+11
View File
@@ -0,0 +1,11 @@
{
"$schema": "https://json.schemastore.org/tsconfig",
"extends": "@tsconfig/bun/tsconfig.json",
"compilerOptions": {
"jsx": "preserve",
"jsxImportSource": "@opentui/solid",
"noUncheckedIndexedAccess": false,
"noUnusedLocals": true
},
"include": ["src", "test"]
}