Compare commits

..

1 Commits

Author SHA1 Message Date
Kit Langton b1a61aaf55 fix(core): honor Codex input limits 2026-07-26 17:29:43 +00:00
61 changed files with 118 additions and 7530 deletions
-21
View File
@@ -986,25 +986,6 @@
"@typescript/native-preview": "catalog:",
},
},
"packages/voice": {
"name": "@opencode-ai/voice",
"version": "0.0.0",
"dependencies": {
"@opencode-ai/client": "workspace:*",
"@opencode-ai/plugin": "workspace:*",
"@opentui/core": "catalog:",
"@opentui/solid": "catalog:",
"effect": "catalog:",
"opentui-spinner": "catalog:",
"solid-js": "catalog:",
},
"devDependencies": {
"@tsconfig/bun": "catalog:",
"@types/bun": "catalog:",
"@typescript/native-preview": "catalog:",
"typescript": "catalog:",
},
},
"packages/web": {
"name": "@opencode-ai/web",
"version": "1.18.4",
@@ -2096,8 +2077,6 @@
"@opencode-ai/util": ["@opencode-ai/util@workspace:packages/util"],
"@opencode-ai/voice": ["@opencode-ai/voice@workspace:packages/voice"],
"@opencode-ai/web": ["@opencode-ai/web@workspace:packages/web"],
"@opencode-ai/www": ["@opencode-ai/www@workspace:packages/www"],
+1
View File
@@ -123,6 +123,7 @@ export const mergeGenerationOptions = (...items: ReadonlyArray<GenerationOptions
export class ModelLimits extends Schema.Class<ModelLimits>("LLM.ModelLimits")({
context: Schema.optional(Schema.Number),
input: Schema.optional(Schema.Number),
output: Schema.optional(Schema.Number),
}) {}
@@ -88,12 +88,8 @@ function SessionTabSlot(props: {
data-titlebar-tab-slot
data-tab-key={props.id}
data-active={props.active()}
class="relative flex min-w-7 flex-shrink"
classList={{
hidden: !session() && !missingSession() && !persisted()?.title,
"w-72 max-w-72": props.active(),
"w-56 max-w-56": !props.active(),
}}
class="relative flex w-56 min-w-7 max-w-56 flex-shrink"
classList={{ hidden: !session() && !missingSession() && !persisted()?.title }}
>
<TabNavItem
ref={(el) => {
@@ -148,11 +144,7 @@ function DraftTabSlot(props: {
data-titlebar-tab-slot
data-tab-key={props.id}
data-active={props.active()}
class="relative flex min-w-7 flex-shrink"
classList={{
"w-72 max-w-72": props.active(),
"w-56 max-w-56": !props.active(),
}}
class="relative flex w-56 min-w-7 max-w-56 flex-shrink"
>
<DraftTabItem
ref={(el) => {
-1
View File
@@ -1,6 +1,5 @@
#!/usr/bin/env bun
import "@opentui/solid/runtime-plugin-support"
import { NodeRuntime, NodeServices } from "@effect/platform-node"
import { Effect } from "effect"
import { Commands } from "./commands/commands"
+3
View File
@@ -37,6 +37,7 @@ export async function ensure(options: EnsureOptions = {}): Promise<Endpoint> {
let announced = false
let lastSpawn = 0
let spawnDelay = 5_000
let ownerHeld = false
const announce = (reason: "missing" | "version-mismatch", previousVersion?: string) => {
if (announced) return
@@ -64,6 +65,7 @@ export async function ensure(options: EnsureOptions = {}): Promise<Endpoint> {
const registration = await registered(options.file, true)
if (registration.service !== undefined) {
ownerHeld = false
spawnDelay = 5_000
const service = registration.service
const compatible = !service.legacy && (options.version === undefined || service.version === options.version)
@@ -80,6 +82,7 @@ export async function ensure(options: EnsureOptions = {}): Promise<Endpoint> {
if (failure !== undefined) throw failure
const finished = [...contenders].filter(contenderFinished)
if (finished.some((item) => item.child.exitCode === 0)) {
ownerHeld = true
spawnDelay = Math.min(spawnDelay * 2, 30_000)
}
finished.forEach((item) => contenders.delete(item))
+1 -1
View File
@@ -332,7 +332,7 @@ function modelFromLanguage(info: ModelV2.Info, language: LanguageModelV3) {
body: projected.body === undefined ? undefined : { ...projected.body },
headers: info.headers,
},
limits: { context: info.limit.context, output: info.limit.output },
limits: { context: info.limit.context, input: info.limit.input, output: info.limit.output },
providerOptions,
},
body: {
+2 -2
View File
@@ -81,7 +81,7 @@ const withDefaults = (model: ModelV2.Info, route: AnyRoute) =>
headers: providerHeaders(model),
providerOptions: providerOptions(model),
http: model.body === undefined ? undefined : { body: model.body },
limits: { context: model.limit.context, output: model.limit.output },
limits: { context: model.limit.context, input: model.limit.input, output: model.limit.output },
})
const providerHeaders = (model: ModelV2.Info) => {
@@ -206,7 +206,7 @@ export const fromCatalogModel = (
...nativeCredentialSettings(specifier, credential),
headers: resolved.headers,
body: resolved.body,
limits: { context: resolved.limit.context, output: resolved.limit.output },
limits: { context: resolved.limit.context, input: resolved.limit.input, output: resolved.limit.output },
}
return yield* Effect.try({
try: () => {
@@ -180,6 +180,12 @@ export const OpenAIPlugin = define({
})
yield* load()
yield* ctx.catalog.transform((evt) => {
const codex = evt.provider.get(ProviderV2.ID.make("openai-codex"))
if (codex)
for (const model of codex.models.values())
evt.model.update(codex.provider.id, model.id, (draft) => {
draft.enabled = false
})
for (const item of evt.provider.list()) {
if (!ProviderV2.isAISDK(item.provider.package)) continue
if (ProviderV2.packageName(item.provider.package) !== "@ai-sdk/openai") continue
@@ -205,6 +211,8 @@ export const OpenAIPlugin = define({
draft.enabled = false
return
}
const route = codex?.models.get(draft.id)
if (route) draft.limit = { ...route.limit }
draft.cost = []
})
}
+3 -1
View File
@@ -351,10 +351,12 @@ const make = (dependencies: Dependencies) => {
)
if (!last) return false
const output = Math.min(input.model.route.defaults.limits?.output ?? 0, OUTPUT_TOKEN_MAX)
const inputLimit = input.model.route.defaults.limits?.input
const limit = inputLimit === undefined ? context - (output || config.buffer) : Math.max(0, inputLimit - config.buffer)
const used =
last.tokens.input + last.tokens.output + last.tokens.reasoning + last.tokens.cache.read + last.tokens.cache.write
if (used <= 0) return false
return used >= context - (output || config.buffer)
return used >= limit
}
const compactManual = Effect.fn("SessionCompaction.compactManual")(function* (input: ManualInput) {
const content = planContent(input.messages, config.tokens)
-6
View File
@@ -654,12 +654,6 @@ const layer = Layer.effectDiscard(
input: event.data.input,
timeCreated: event.created,
})
yield* db
.update(SessionTable)
.set({ time_updated: DateTime.toEpochMillis(event.created) })
.where(eq(SessionTable.id, event.data.sessionID))
.run()
.pipe(Effect.orDie)
}),
)
yield* events.project(SessionEvent.Compaction.Admitted, (event) =>
+4 -2
View File
@@ -17,6 +17,7 @@ interface ModelOptions {
readonly headers?: ModelV2.Info["headers"]
readonly body?: ModelV2.Info["body"]
readonly variants?: ModelV2.Info["variants"]
readonly limit?: ModelV2.Info["limit"]
}
const model = (packageName: string | undefined, options: ModelOptions = {}) =>
@@ -36,7 +37,7 @@ const model = (packageName: string | undefined, options: ModelOptions = {}) =>
cost: [],
status: "active",
enabled: true,
limit: { context: 100, output: 20 },
limit: options.limit ?? { context: 100, output: 20 },
})
describe("ModelResolver", () => {
@@ -44,6 +45,7 @@ describe("ModelResolver", () => {
Effect.gen(function* () {
const catalog = model(ProviderV2.aisdk("@ai-sdk/openai"), {
settings: { baseURL: "https://openai.example/v1" },
limit: { context: 100, input: 70, output: 20 },
})
const resolved = yield* ModelResolver.fromCatalogModel(catalog)
@@ -55,7 +57,7 @@ describe("ModelResolver", () => {
endpoint: { baseURL: "https://openai.example/v1" },
defaults: {
headers: { "x-test": "header" },
limits: { context: 100, output: 20 },
limits: { context: 100, input: 70, output: 20 },
http: { body: { custom_extension: { enabled: true } } },
},
})
@@ -164,6 +164,16 @@ describe("OpenAIPlugin", () => {
const catalog = yield* Catalog.Service
const credentials = yield* Credential.Service
yield* catalog.transform((catalog) => {
const codex = ProviderV2.Info.make({
...ProviderV2.Info.empty(ProviderV2.ID.make("openai-codex")),
package: ProviderV2.aisdk("@ai-sdk/openai"),
})
catalog.provider.update(codex.id, (draft) => {
draft.package = codex.package
})
catalog.model.update(codex.id, ModelV2.ID.make("gpt-5.6-sol"), (model) => {
model.limit = { context: 400_000, input: 272_000, output: 128_000 }
})
const item = ProviderV2.Info.make({
...ProviderV2.Info.empty(ProviderV2.ID.openai),
package: ProviderV2.aisdk("@ai-sdk/openai"),
@@ -189,7 +199,9 @@ describe("OpenAIPlugin", () => {
model.body = { reasoning: { mode: "pro" } }
})
catalog.model.update(item.id, ModelV2.ID.make("gpt-5.6"), () => {})
catalog.model.update(item.id, ModelV2.ID.make("gpt-5.6-sol"), () => {})
catalog.model.update(item.id, ModelV2.ID.make("gpt-5.6-sol"), (model) => {
model.limit = { context: 1_050_000, input: 922_000, output: 128_000 }
})
catalog.model.update(item.id, ModelV2.ID.make("gpt-4.1"), () => {})
})
yield* credentials.create({
@@ -215,9 +227,13 @@ describe("OpenAIPlugin", () => {
false,
)
expect(required(yield* catalog.model.get(ProviderV2.ID.openai, ModelV2.ID.make("gpt-5.6"))).enabled).toBe(false)
expect(required(yield* catalog.model.get(ProviderV2.ID.openai, ModelV2.ID.make("gpt-5.6-sol"))).enabled).toBe(
true,
)
const sol = required(yield* catalog.model.get(ProviderV2.ID.openai, ModelV2.ID.make("gpt-5.6-sol")))
expect(sol.enabled).toBe(true)
expect(sol.limit).toEqual({ context: 400_000, input: 272_000, output: 128_000 })
expect(
required(yield* catalog.model.get(ProviderV2.ID.make("openai-codex"), ModelV2.ID.make("gpt-5.6-sol")))
.enabled,
).toBe(false)
expect(required(yield* catalog.model.get(ProviderV2.ID.openai, ModelV2.ID.make("gpt-4.1"))).enabled).toBe(false)
}),
)
@@ -227,6 +243,16 @@ describe("OpenAIPlugin", () => {
const catalog = yield* Catalog.Service
const credentials = yield* Credential.Service
yield* catalog.transform((catalog) => {
const codex = ProviderV2.Info.make({
...ProviderV2.Info.empty(ProviderV2.ID.make("openai-codex")),
package: ProviderV2.aisdk("@ai-sdk/openai"),
})
catalog.provider.update(codex.id, (draft) => {
draft.package = codex.package
})
catalog.model.update(codex.id, ModelV2.ID.make("gpt-5.5"), (model) => {
model.limit = { context: 400_000, input: 272_000, output: 128_000 }
})
const item = ProviderV2.Info.make({
...ProviderV2.Info.empty(ProviderV2.ID.openai),
package: ProviderV2.aisdk("@ai-sdk/openai"),
@@ -234,7 +260,9 @@ describe("OpenAIPlugin", () => {
catalog.provider.update(item.id, (draft) => {
draft.package = item.package
})
catalog.model.update(item.id, ModelV2.ID.make("gpt-5.5"), () => {})
catalog.model.update(item.id, ModelV2.ID.make("gpt-5.5"), (model) => {
model.limit = { context: 1_050_000, input: 922_000, output: 128_000 }
})
catalog.model.update(item.id, ModelV2.ID.make("gpt-4.1"), () => {})
})
yield* credentials.create({
@@ -243,7 +271,9 @@ describe("OpenAIPlugin", () => {
})
yield* addPlugin()
expect(required(yield* catalog.model.get(ProviderV2.ID.openai, ModelV2.ID.make("gpt-5.5"))).enabled).toBe(true)
const model = required(yield* catalog.model.get(ProviderV2.ID.openai, ModelV2.ID.make("gpt-5.5")))
expect(model.enabled).toBe(true)
expect(model.limit).toEqual({ context: 1_050_000, input: 922_000, output: 128_000 })
expect(required(yield* catalog.model.get(ProviderV2.ID.openai, ModelV2.ID.make("gpt-4.1"))).enabled).toBe(true)
}),
)
@@ -16,11 +16,15 @@ import { SessionRunnerModel } from "@opencode-ai/core/session/runner/model"
import { SessionTable } from "@opencode-ai/core/session/sql"
import { SessionStore } from "@opencode-ai/core/session/store"
import { SessionV2 } from "@opencode-ai/core/session"
import { AgentV2 } from "@opencode-ai/core/agent"
import { Location } from "@opencode-ai/core/location"
import { Project } from "@opencode-ai/core/project"
import { ProjectTable } from "@opencode-ai/core/project/sql"
import { App } from "@opencode-ai/core/app"
import { AbsolutePath } from "@opencode-ai/core/schema"
import { Money } from "@opencode-ai/schema/money"
import { ModelV2 } from "@opencode-ai/core/model"
import { ProviderV2 } from "@opencode-ai/core/provider"
import { DateTime, Effect, Fiber, Layer, Stream } from "effect"
import { asc, eq } from "drizzle-orm"
import { testEffect } from "./lib/effect"
@@ -131,6 +135,40 @@ test("compaction prompt requires the checkpoint headings in order", () => {
expect(prompt).toContain("Keep every section, even when empty.")
})
it.effect("auto compaction uses the model input limit", () =>
Effect.gen(function* () {
const compaction = yield* SessionCompaction.Service
const session = SessionV2.Info.make({
id: SessionV2.ID.make("ses_input_limit"),
projectID: Project.ID.global,
title: "Input limit",
cost: Money.USD.zero,
tokens: { input: 0, output: 0, reasoning: 0, cache: { read: 0, write: 0 } },
time: { created: DateTime.makeUnsafe(0), updated: DateTime.makeUnsafe(0) },
location: Location.Ref.make({ directory: AbsolutePath.make("/project") }),
})
const route = Model.make({
id: "codex-model",
provider: "openai",
route: OpenAIChat.route.with({ limits: { context: 400_000, input: 272_000, output: 128_000 } }),
})
const message = (input: number) =>
SessionMessage.Assistant.make({
id: SessionMessage.ID.create(),
type: "assistant",
agent: AgentV2.defaultID,
model: { id: ModelV2.ID.make("codex-model"), providerID: ProviderV2.ID.openai },
content: [],
cost: Money.USD.zero,
tokens: { input, output: 0, reasoning: 0, cache: { read: 0, write: 0 } },
time: { created: DateTime.makeUnsafe(0) },
})
expect(compaction.required({ session, model: route, cost: [], messages: [message(251_999)] })).toBe(false)
expect(compaction.required({ session, model: route, cost: [], messages: [message(252_000)] })).toBe(true)
}),
)
it.effect("manual compaction summarizes short context instead of no-op", () =>
Effect.gen(function* () {
requests = []
-22
View File
@@ -171,28 +171,6 @@ describe("SessionV2.create", () => {
}),
)
it.effect("orders reused sessions by prompt admission time", () =>
Effect.gen(function* () {
const session = yield* SessionV2.Service
const events = yield* EventV2.Service
const { db } = yield* Database.Service
const reused = yield* session.create({ location, title: "reused" })
const newer = yield* session.create({ location, title: "newer" })
yield* db.update(SessionTable).set({ time_updated: -2 }).where(eq(SessionTable.id, reused.id)).run()
yield* db.update(SessionTable).set({ time_updated: -1 }).where(eq(SessionTable.id, newer.id)).run()
const admitted = yield* events.publish(SessionEvent.InputAdmitted, {
sessionID: reused.id,
inputID: SessionMessage.ID.create(),
input: { type: "user", data: { text: "continue" }, delivery: "steer" },
})
const page = yield* session.list({ directory: location.directory, parentID: null, order: "desc" })
expect(page.data.map((item) => item.id)).toEqual([reused.id, newer.id])
expect(page.data[0]!.time.updated).toEqual(admitted.created)
}),
)
it.effect("filters direct child sessions by parent ID", () =>
Effect.gen(function* () {
const session = yield* SessionV2.Service
+2 -1
View File
@@ -29,9 +29,10 @@ export function contextUsage(
last.tokens.input + last.tokens.output + last.tokens.reasoning + last.tokens.cache.read + last.tokens.cache.write
if (tokens <= 0) return
const model = models?.find((model) => model.providerID === last.model.providerID && model.id === last.model.id)
const limit = model?.limit.input ?? model?.limit.context
return {
tokens,
percent: model?.limit.context ? Math.round((tokens / model.limit.context) * 100) : undefined,
percent: limit ? Math.round((tokens / limit) * 100) : undefined,
}
}
+17 -2
View File
@@ -1,6 +1,6 @@
import { describe, expect, test } from "bun:test"
import type { SessionMessageInfo } from "@opencode-ai/client"
import { isDefaultTitle, lastAssistantWithUsage } from "../../src/util/session"
import type { ModelInfo, SessionMessageInfo } from "@opencode-ai/client"
import { contextUsage, isDefaultTitle, lastAssistantWithUsage } from "../../src/util/session"
const assistant = (id: string, input: number): SessionMessageInfo => ({
id,
@@ -45,4 +45,19 @@ describe("util.session", () => {
messages.push(assistant("msg_after", 5))
expect(lastAssistantWithUsage(messages)?.tokens.input).toBe(5)
})
test("reports usage against the effective input window", () => {
const models = [
{
id: "model",
providerID: "provider",
limit: { context: 400_000, input: 272_000, output: 128_000 },
} as unknown as ModelInfo,
]
expect(contextUsage([assistant("msg_input_limit", 252_000)], models)).toEqual({
tokens: 252_000,
percent: 93,
})
})
})
-1
View File
@@ -1 +0,0 @@
.build/
-4
View File
@@ -1,4 +0,0 @@
preload = ["@opentui/solid/preload"]
[test]
preload = ["@opentui/solid/preload"]
-29
View File
@@ -1,29 +0,0 @@
{
"$schema": "https://json.schemastore.org/package.json",
"name": "@opencode-ai/voice",
"version": "0.0.0",
"private": true,
"description": "Prototype voice control for OpenCode via the OpenAI Realtime and Live APIs",
"type": "module",
"scripts": {
"bench:audio": "bun scripts/bench-speaker-queue.ts",
"spike": "bun run --no-orphans --conditions=browser src/spike.ts",
"test": "bun test",
"typecheck": "tsgo --noEmit"
},
"devDependencies": {
"@tsconfig/bun": "catalog:",
"@types/bun": "catalog:",
"@typescript/native-preview": "catalog:",
"typescript": "catalog:"
},
"dependencies": {
"@opencode-ai/client": "workspace:*",
"@opencode-ai/plugin": "workspace:*",
"@opentui/core": "catalog:",
"@opentui/solid": "catalog:",
"effect": "catalog:",
"opentui-spinner": "catalog:",
"solid-js": "catalog:"
}
}
-38
View File
@@ -1,38 +0,0 @@
# Speaker Queue Performance
## Goal
Reduce render-path latency and variance that can produce audible output glitches.
## Benchmark
Run `bun run bench:audio` from `packages/voice`.
The benchmark compiles the production `SpeakerQueue`, performs one warmup run and seven measured runs, and reports callback latency for a 512-frame pop followed by a same-sized producer push. The queue starts with one second of 24 kHz PCM audio.
## Metrics
- Primary: `speaker_queue_p99_ns`
- Secondary: `speaker_queue_median_ns`, `speaker_queue_worst_ns`
## Scope
- `src/duplex-audio.swift`
- `scripts/bench-speaker-queue.ts`
## Experiments
### Baseline: allocating `Data` queue
- Median: 3,750 ns
- p99: 5,833 ns
- Worst: 92,209 ns median across runs; one run reached 7,497,334 ns
- Decision: baseline
### Experiment 1: preallocated circular queue
- Hypothesis: rendering directly from preallocated storage will reduce callback tail latency by avoiding an array allocation and `Data.removeFirst()` on every callback.
- Median: 1,417 ns, down 62.2%
- p99: 1,917 ns, down 67.1%
- Worst: 32,709 ns median across runs, down 64.5%
- Decision: keep
@@ -1,43 +0,0 @@
import { mkdtemp, rm } from "node:fs/promises"
import { tmpdir } from "node:os"
import { join } from "node:path"
const directory = await mkdtemp(join(tmpdir(), "opencode-voice-bench-"))
const binary = join(directory, "speaker-queue")
const source = Bun.fileURLToPath(new URL("../src/duplex-audio.swift", import.meta.url))
const compile = Bun.spawn(["swiftc", "-O", "-D", "QUEUE_BENCHMARK", source, "-o", binary], {
stdout: "ignore",
stderr: "pipe",
})
if ((await compile.exited) !== 0) {
console.error(await new Response(compile.stderr).text())
await rm(directory, { recursive: true })
process.exit(1)
}
const runs: Array<Record<string, number>> = []
for (let index = 0; index < 8; index++) {
const process = Bun.spawn([binary], { stdout: "pipe", stderr: "inherit" })
const output = await new Response(process.stdout).text()
if ((await process.exited) !== 0) throw new Error(`benchmark run ${index + 1} failed`)
if (index === 0) continue
runs.push(
Object.fromEntries(
output
.split("\n")
.filter((line) => line.startsWith("METRIC "))
.map((line) => {
const [name, value] = line.slice(7).split("=")
return [name!, Number(value)]
}),
),
)
}
await rm(directory, { recursive: true })
for (const name of ["speaker_queue_median_ns", "speaker_queue_p99_ns", "speaker_queue_worst_ns"]) {
const values = runs.map((run) => run[name]!).sort((a, b) => a - b)
const median = values[Math.floor(values.length / 2)]!
console.log(`${name}: median=${median} range=${values[0]}..${values.at(-1)}`)
console.log(`METRIC ${name}=${median}`)
}
-49
View File
@@ -1,49 +0,0 @@
export type TextReveal = { readonly offset: number; readonly at: number }
const CONNECTION_TRANSITION_MS = 480
const REVEAL_DURATION_MS = 380
const REVEAL_STAGGER_MS = 32
const REVEAL_MAX_QUEUE_MS = 160
export const REVEAL_WORD_LIMIT = 24
export function textRevealOpacity(now: number, start: number) {
const progress = Math.max(0, Math.min(1, (now - start) / REVEAL_DURATION_MS))
const eased = 1 - (1 - progress) ** 2
return 0.16 + eased * 0.84
}
export function transcriptionPulse(now: number) {
return 0.5 + Math.sin(now / 220) * 0.5
}
export function connectionMeterLevels(now: number, level: number, connectedAt?: number, startedAt = 0) {
const transition = connectedAt === undefined ? 0 : Math.max(0, Math.min(1, (now - connectedAt) / CONNECTION_TRANSITION_MS))
const head = ((now - startedAt) / 180) % 4
return [0, 1, 2, 3].map((index) => {
const distance = Math.abs(index - head)
const wrappedDistance = Math.min(distance, 4 - distance)
const loading = 0.08 + Math.max(0, Math.cos((wrappedDistance / 1.5) * (Math.PI / 2))) * 0.92
const connected = level * (0.72 + Math.sin(now / 240 + index * 1.4) * 0.28)
return loading + (connected - loading) * transition
})
}
export function connectionTransitioning(now: number, connectedAt?: number) {
return connectedAt === undefined || now - connectedAt < CONNECTION_TRANSITION_MS
}
export function scheduleTextReveal(previous: string, delta: string, now: number, revealAt: number) {
const text = previous + delta
const offsets = Array.from(delta.matchAll(/\S+/g))
.map((match) => previous.length + match.index)
.filter((offset) => offset === 0 || /\s/.test(text[offset - 1] ?? ""))
.slice(-REVEAL_WORD_LIMIT)
const start = Math.min(Math.max(revealAt, now), now + REVEAL_MAX_QUEUE_MS)
const reveals = offsets.map((offset, word) => ({ offset, at: start + word * REVEAL_STAGGER_MS }))
const nextRevealAt = reveals.length === 0 ? revealAt : reveals.at(-1)!.at + REVEAL_STAGGER_MS
return {
reveals,
nextRevealAt,
animationEndsAt: reveals.length === 0 ? now : reveals.at(-1)!.at + REVEAL_DURATION_MS,
}
}
-136
View File
@@ -1,136 +0,0 @@
import { mkdir } from "node:fs/promises"
import { PCM_SAMPLE_RATE } from "./pcm"
export type AudioDevice = {
readonly fullDuplex: boolean
readonly mode: string
start(onInput: (audio: Buffer) => void, onMeta: (text: string) => void): Promise<void>
write(audio: Buffer): void
flush(): void
close(): void
}
export async function createAudioDevice(options: {
readonly duplex: boolean
readonly speakers: boolean
readonly debug: boolean
}): Promise<AudioDevice> {
const helper = await prepareHelper()
if (helper) return appleDevice(helper, options)
return soxDevice(options)
}
function appleDevice(
binary: string,
options: { readonly duplex: boolean; readonly speakers: boolean },
): AudioDevice {
let process: ReturnType<typeof Bun.spawn> | undefined
let sink: import("bun").FileSink | undefined
return {
fullDuplex: options.speakers || options.duplex,
mode: options.speakers ? "duplex+aec" : options.duplex ? "duplex" : "half-duplex",
async start(onInput, onMeta) {
if (process) return
const child = Bun.spawn([binary, ...(options.speakers ? ["--aec"] : [])], {
stdin: "pipe",
stdout: "pipe",
stderr: "pipe",
})
process = child
sink = child.stdin
void forwardLines(child.stderr, onMeta)
for await (const chunk of child.stdout) onInput(Buffer.from(chunk))
process = undefined
sink = undefined
},
write(audio) {
void sink?.write(audio)
void sink?.flush()
},
flush() {
process?.kill("SIGUSR1")
},
close() {
process?.kill()
process = undefined
sink = undefined
},
}
}
function soxDevice(options: { readonly duplex: boolean; readonly debug: boolean }): AudioDevice {
const format = [
"-q",
"-t",
"raw",
"-r",
String(PCM_SAMPLE_RATE),
"-e",
"signed-integer",
"-b",
"16",
"-c",
"1",
]
let recorder: ReturnType<typeof Bun.spawn> | undefined
let player: ReturnType<typeof Bun.spawn> | undefined
let sink: import("bun").FileSink | undefined
let onMeta: ((text: string) => void) | undefined
return {
fullDuplex: options.duplex,
mode: options.duplex ? "duplex (sox)" : "half-duplex (sox)",
async start(onInput, report) {
if (recorder) return
onMeta = report
const child = Bun.spawn(["rec", ...format, "-"], { stdout: "pipe", stderr: "ignore" })
recorder = child
for await (const chunk of child.stdout) onInput(Buffer.from(chunk))
recorder = undefined
},
write(audio) {
if (!player) {
const child = Bun.spawn(["play", ...format, "-"], { stdin: "pipe", stderr: "ignore" })
player = child
sink = child.stdin
if (options.debug) void child.exited.then((code) => onMeta?.(`[debug] play exited (${code})`))
}
void sink?.write(audio)
void sink?.flush()
},
flush() {
player?.kill()
player = undefined
sink = undefined
},
close() {
recorder?.kill()
player?.kill()
recorder = undefined
player = undefined
sink = undefined
},
}
}
async function forwardLines(stream: ReadableStream<Uint8Array>, onLine: (line: string) => void) {
const decoder = new TextDecoder()
let remainder = ""
for await (const chunk of stream) {
const lines = (remainder + decoder.decode(chunk, { stream: true })).split("\n")
remainder = lines.pop() ?? ""
lines.filter((line) => line.trim()).forEach((line) => onLine(line.trim()))
}
remainder += decoder.decode()
if (remainder.trim()) onLine(remainder.trim())
}
async function prepareHelper() {
if (process.platform !== "darwin") return undefined
const source = Bun.fileURLToPath(new URL("./duplex-audio.swift", import.meta.url))
const binary = Bun.fileURLToPath(new URL("../.build/duplex-audio", import.meta.url))
if ((await Bun.file(binary).exists()) && Bun.file(binary).lastModified > Bun.file(source).lastModified) return binary
await mkdir(Bun.fileURLToPath(new URL("../.build", import.meta.url)), { recursive: true })
const compile = Bun.spawn(["swiftc", "-O", source, "-o", binary], { stdout: "ignore", stderr: "pipe" })
if ((await compile.exited) === 0) return binary
return undefined
}
-41
View File
@@ -1,41 +0,0 @@
import { PCM_BYTES_PER_MS } from "./pcm"
export type PlaybackChunk = {
readonly bytes: Buffer
readonly gapMs: number
}
export class AudioJitterBuffer {
private readonly pending: PlaybackChunk[] = []
private durationMs = 0
private started = false
constructor(private readonly targetMs = 500) {}
push(chunk: PlaybackChunk): ReadonlyArray<PlaybackChunk> {
if (this.started) return [chunk]
this.pending.push(chunk)
this.durationMs += chunk.gapMs + chunk.bytes.length / PCM_BYTES_PER_MS
if (this.durationMs < this.targetMs) return []
this.started = true
return this.drain()
}
finish(): ReadonlyArray<PlaybackChunk> {
if (this.pending.length === 0) return []
this.started = true
return this.drain()
}
reset() {
this.pending.length = 0
this.durationMs = 0
this.started = false
}
private drain() {
const chunks = this.pending.splice(0)
this.durationMs = 0
return chunks
}
}
-217
View File
@@ -1,217 +0,0 @@
import { createAudioDevice, type AudioDevice } from "./audio-device"
import { AudioJitterBuffer, type PlaybackChunk } from "./audio-jitter-buffer"
import { pcmLevel, PCM_BYTES_PER_MS, PCM_METER_FRAME_MS } from "./pcm"
import type { VoiceAudioTimeline } from "./protocol"
export type AudioEvent =
| { readonly type: "input"; readonly audio: Buffer; readonly level: number }
| { readonly type: "user.started" }
| { readonly type: "user.stopped" }
| { readonly type: "user.reset" }
| { readonly type: "assistant.level"; readonly level: number; readonly durationMs: number }
| { readonly type: "status"; readonly audio: string }
| { readonly type: "meta"; readonly text: string }
export type PlaybackOutcome = "played" | "interrupted" | "inaudible"
export type AudioSession = {
readonly fullDuplex: boolean
readonly microphoneMuted: boolean
readonly speakerMuted: boolean
start(emit: (event: AudioEvent) => void): Promise<void>
toggleMicrophone(): boolean
toggleSpeaker(): boolean
isPlaying(): boolean
play(bytes: Buffer, timeline?: VoiceAudioTimeline): void
finishPlayback(): Promise<PlaybackOutcome>
flushPlayback(): void
noteUserCommitted(): void
noteUserTranscript(final: boolean): void
close(): void
}
export async function createAudioSession(options: {
readonly duplex: boolean
readonly speakers: boolean
readonly inputActivity: "local" | "server"
readonly debug: boolean
readonly device?: AudioDevice
readonly trace?: (event: string, data?: Record<string, unknown>) => void
}): Promise<AudioSession> {
const device = options.device ?? (await createAudioDevice(options))
const fullDuplex = device.fullDuplex
const playbackBuffer = new AudioJitterBuffer()
let emit: ((event: AudioEvent) => void) | undefined
let microphoneMuted = false
let speakerMuted = false
let playbackEndsAt = 0
let playbackAudible = false
let playbackDoneTimer: ReturnType<typeof setTimeout> | undefined
let playbackCompletion: PromiseWithResolvers<PlaybackOutcome> | undefined
let outputTimelineEnd: number | undefined
let userFinalizedAt = 0
let userSpeechTimer: ReturnType<typeof setTimeout> | undefined
let userDraftTimer: ReturnType<typeof setTimeout> | undefined
let microphoneStarted = false
let closed = false
const isPlaying = () => Date.now() < playbackEndsAt
const settlePlayback = (outcome: PlaybackOutcome) => {
if (playbackDoneTimer) clearTimeout(playbackDoneTimer)
playbackDoneTimer = undefined
const completion = playbackCompletion
playbackCompletion = undefined
playbackEndsAt = 0
playbackAudible = false
playbackBuffer.reset()
outputTimelineEnd = undefined
completion?.resolve(outcome)
}
const flushPlayback = () => {
device.flush()
settlePlayback("interrupted")
}
const observeUserAudio = (level: number) => {
if (options.inputActivity !== "local" || level < 0.2 || Date.now() - userFinalizedAt < 500) return
if (userDraftTimer) clearTimeout(userDraftTimer)
userDraftTimer = undefined
emit?.({ type: "user.started" })
outputTimelineEnd = undefined
if (userSpeechTimer) clearTimeout(userSpeechTimer)
userSpeechTimer = setTimeout(() => {
userSpeechTimer = undefined
emit?.({ type: "user.stopped" })
userDraftTimer = setTimeout(() => {
userDraftTimer = undefined
emit?.({ type: "user.reset" })
}, 1_200)
}, 500)
}
const publishInput = (bytes: Buffer) => {
const level = pcmLevel(bytes)
emit?.({ type: "input", audio: bytes, level })
observeUserAudio(level)
}
const start = async (listener: (event: AudioEvent) => void) => {
if (microphoneStarted || closed) return
emit = listener
microphoneStarted = true
emit({ type: "status", audio: device.mode })
emit({ type: "meta", text: "mic live - start talking" })
if (!fullDuplex)
emit({ type: "meta", text: "mic mutes while the assistant speaks; press Esc to interrupt" })
await device.start(
(chunk) => {
if (closed || microphoneMuted || (!fullDuplex && Date.now() < playbackEndsAt + 300)) return
publishInput(chunk)
},
(text) => {
emit?.({ type: "meta", text })
options.trace?.("audio.helper", { message: text })
},
)
}
const writeAudio = (chunk: PlaybackChunk) => {
playbackAudible = true
if (chunk.gapMs > 0) {
emit?.({ type: "assistant.level", level: 0, durationMs: chunk.gapMs })
device.write(Buffer.alloc(Math.round(chunk.gapMs * PCM_BYTES_PER_MS)))
}
const meterBytes = PCM_METER_FRAME_MS * PCM_BYTES_PER_MS
for (let offset = 0; offset < chunk.bytes.length; offset += meterBytes) {
const window = chunk.bytes.subarray(offset, offset + meterBytes)
emit?.({ type: "assistant.level", level: pcmLevel(window), durationMs: window.length / PCM_BYTES_PER_MS })
}
device.write(chunk.bytes)
playbackEndsAt = Math.max(playbackEndsAt, Date.now()) + chunk.gapMs + chunk.bytes.length / PCM_BYTES_PER_MS
}
const schedulePlaybackCompletion = () => {
if (!playbackCompletion) return
if (!playbackAudible) return settlePlayback("inaudible")
if (playbackDoneTimer) clearTimeout(playbackDoneTimer)
playbackDoneTimer = setTimeout(
() => settlePlayback("played"),
Math.max(0, playbackEndsAt - Date.now()) + 180,
)
}
return {
fullDuplex,
get microphoneMuted() {
return microphoneMuted
},
get speakerMuted() {
return speakerMuted
},
start,
toggleMicrophone() {
microphoneMuted = !microphoneMuted
if (userSpeechTimer) clearTimeout(userSpeechTimer)
userSpeechTimer = undefined
emit?.({ type: "user.stopped" })
if (microphoneMuted) emit?.({ type: "user.reset" })
return microphoneMuted
},
toggleSpeaker() {
speakerMuted = !speakerMuted
if (speakerMuted) flushPlayback()
return speakerMuted
},
isPlaying,
play(bytes, timeline) {
if (speakerMuted || closed) return
const timelineGap = timeline && outputTimelineEnd !== undefined ? Math.max(0, timeline.startMs - outputTimelineEnd) : 0
const gapMs = timelineGap < 2_000 ? timelineGap : 0
if (options.debug)
options.trace?.("audio.output", {
bytes: bytes.length,
durationMs: bytes.length / PCM_BYTES_PER_MS,
level: pcmLevel(bytes),
timelineStart: timeline?.startMs,
timelineEnd: timeline?.endMs,
timelineGap,
gapMs,
})
outputTimelineEnd = timeline?.endMs
playbackBuffer.push({ bytes, gapMs }).forEach(writeAudio)
schedulePlaybackCompletion()
},
finishPlayback() {
if (playbackCompletion) return playbackCompletion.promise
playbackCompletion = Promise.withResolvers<PlaybackOutcome>()
playbackBuffer.finish().forEach(writeAudio)
schedulePlaybackCompletion()
return playbackCompletion?.promise ?? Promise.resolve("inaudible")
},
flushPlayback,
noteUserCommitted() {
if (userDraftTimer) clearTimeout(userDraftTimer)
userDraftTimer = undefined
},
noteUserTranscript(final) {
if (userDraftTimer) clearTimeout(userDraftTimer)
userDraftTimer = undefined
if (!final) return
if (userSpeechTimer) clearTimeout(userSpeechTimer)
userSpeechTimer = undefined
userFinalizedAt = Date.now()
emit?.({ type: "user.stopped" })
},
close() {
if (closed) return
closed = true
device.close()
if (userSpeechTimer) clearTimeout(userSpeechTimer)
if (userDraftTimer) clearTimeout(userDraftTimer)
settlePlayback("interrupted")
emit = undefined
},
}
}
-109
View File
@@ -1,109 +0,0 @@
import { mkdir, readdir, rename, rm } from "node:fs/promises"
import { homedir } from "node:os"
import { dirname, join } from "node:path"
import { Option, Schema } from "effect"
import { OpenCodeNotification } from "./opencode-notification"
import { promptKey, type PromptHandle } from "./prompt-handle"
export type CompletionHandle = PromptHandle
export type StoredCompletion =
| { readonly status: "admitting"; readonly handle: CompletionHandle; readonly text: string }
| { readonly status: "pending"; readonly handle: CompletionHandle }
| {
readonly status: "completed"
readonly handle: CompletionHandle
readonly notification: OpenCodeNotification
}
const CompletionHandle = Schema.Struct({ sessionID: Schema.String, promptID: Schema.String })
const StoredCompletion = Schema.Union([
Schema.Struct({ status: Schema.Literal("admitting"), handle: CompletionHandle, text: Schema.String }),
Schema.Struct({ status: Schema.Literal("pending"), handle: CompletionHandle }),
Schema.Struct({ status: Schema.Literal("completed"), handle: CompletionHandle, notification: OpenCodeNotification }),
])
const StoredCompletions = Schema.Array(StoredCompletion)
export async function createCompletionStore(
path = join(process.env["XDG_STATE_HOME"] ?? join(homedir(), ".local", "state"), "opencode", "voice-prompts.json"),
) {
await mkdir(dirname(path), { recursive: true })
const directory = `${path}.d`
await mkdir(directory, { recursive: true })
const files = (await readdir(directory)).filter((name) => name.endsWith(".json"))
const decoded = files.length > 0
? await Promise.all(
files.map(async (name) => {
const entry = Option.getOrUndefined(
Schema.decodeUnknownOption(StoredCompletion)(await Bun.file(join(directory, name)).json()),
)
if (!entry) throw new Error(`Invalid voice completion entry: ${join(directory, name)}`)
return entry
}),
)
: (await Bun.file(path).exists())
? Option.getOrUndefined(Schema.decodeUnknownOption(StoredCompletions)(await Bun.file(path).json()))
: []
if (!decoded) throw new Error(`Invalid voice completion store: ${path}`)
const entries = new Map(decoded.map((entry) => [promptKey(entry.handle), entry]))
let writes = Promise.resolve()
const enqueue = (operation: () => Promise<void>) => {
const result = writes.then(operation)
writes = result.catch(() => {})
return result
}
const save = (entry: StoredCompletion) => {
return enqueue(async () => {
const target = completionPath(directory, entry.handle)
const temporary = `${target}.${process.pid}.${crypto.randomUUID()}.tmp`
await Bun.write(temporary, JSON.stringify(entry))
await rename(temporary, target)
})
}
if (files.length === 0 && decoded.length > 0) {
await Promise.all(decoded.map(save))
await rm(path, { force: true })
}
if (files.length > 0) {
const existing = new Set(files)
await Promise.all(decoded.filter((entry) => !existing.has(completionFilename(entry.handle))).map(save))
const current = new Set(decoded.map((entry) => completionFilename(entry.handle)))
await Promise.all(files.filter((name) => !current.has(name)).map((name) => rm(join(directory, name), { force: true })))
}
return {
entries: () => [...entries.entries()].toSorted(([left], [right]) => left.localeCompare(right)).map(([, entry]) => entry),
admitting(handle: CompletionHandle, text: string) {
const entry = { status: "admitting", handle, text } as const
entries.set(promptKey(handle), entry)
return save(entry)
},
pending(handle: CompletionHandle) {
const entry = { status: "pending", handle } as const
entries.set(promptKey(handle), entry)
return save(entry)
},
completed(handle: CompletionHandle, notification: OpenCodeNotification) {
const entry = { status: "completed", handle, notification } as const
entries.set(promptKey(handle), entry)
return save(entry)
},
remove(handle: CompletionHandle) {
if (!entries.delete(promptKey(handle))) return writes
return enqueue(() => rm(completionPath(directory, handle), { force: true }))
},
close: () => writes,
}
}
export type CompletionStore = Awaited<ReturnType<typeof createCompletionStore>>
function completionFilename(handle: CompletionHandle) {
return `${encodeURIComponent(promptKey(handle))}.json`
}
function completionPath(directory: string, handle: CompletionHandle) {
return join(directory, completionFilename(handle))
}
-288
View File
@@ -1,288 +0,0 @@
// Full-duplex terminal audio bridge with Apple voice processing (AEC).
//
// stdin <- raw PCM16 mono 24kHz to play through the speakers
// stdout -> raw PCM16 mono 24kHz captured from the microphone
// SIGUSR1: drop any queued speaker audio (barge-in flush)
//
// Two independent engines: attaching a speaker source to a voice-processed
// engine silently kills its input tap, so playback runs on its own plain
// engine. Voice processing (echo cancellation) is attempted on the input
// engine and abandoned if the mic delivers nothing: on some devices
// (Bluetooth headsets mid-negotiation) the VP tap never fires. Without VP
// there is no echo cancellation, which is acceptable exactly when it happens:
// headphones have no echo path.
//
// Compiled on demand by spike.ts: swiftc -O duplex-audio.swift -o duplex-audio
import AVFoundation
let stderr = FileHandle.standardError
func log(_ message: String) { stderr.write(Data("[audio] \(message)\n".utf8)) }
final class SpeakerQueue {
private static let capacity = 24_000 * 2 * 30
private var data = [UInt8](repeating: 0, count: capacity)
private var readIndex = 0
private var writeIndex = 0
private var count = 0
private let lock = NSLock()
func push(_ chunk: Data) {
if chunk.isEmpty { return }
lock.lock()
defer { lock.unlock() }
chunk.withUnsafeBytes { source in
let skipped = max(0, source.count - Self.capacity)
let start = skipped + skipped % 2
let bytes = source.count - start
let overflow = max(0, count + bytes - Self.capacity)
let dropped = min(count, overflow + overflow % 2)
readIndex = (readIndex + dropped) % Self.capacity
count -= dropped
let first = min(bytes, Self.capacity - writeIndex)
data.withUnsafeMutableBytes { destination in
destination.baseAddress!.advanced(by: writeIndex).copyMemory(
from: source.baseAddress!.advanced(by: start), byteCount: first)
destination.baseAddress!.copyMemory(
from: source.baseAddress!.advanced(by: start + first), byteCount: bytes - first)
}
writeIndex = (writeIndex + bytes) % Self.capacity
count += bytes
}
}
func render(frames: Int, into output: UnsafeMutablePointer<Float>) {
lock.lock()
defer { lock.unlock() }
let samples = min(frames, count / 2)
for index in 0..<samples {
let low = UInt16(data[readIndex])
let high = UInt16(data[(readIndex + 1) % Self.capacity])
output[index] = Float(Int16(bitPattern: low | high << 8)) / 32768.0
readIndex = (readIndex + 2) % Self.capacity
}
count -= samples * 2
for index in samples..<frames { output[index] = 0 }
}
func flush() {
lock.lock()
readIndex = writeIndex
count = 0
lock.unlock()
}
}
#if QUEUE_BENCHMARK
let correctnessQueue = SpeakerQueue()
correctnessQueue.push(Data([0x00, 0x80, 0xff, 0x7f]))
var correctnessOutput = [Float](repeating: 1, count: 3)
correctnessOutput.withUnsafeMutableBufferPointer {
correctnessQueue.render(frames: 3, into: $0.baseAddress!)
}
precondition(correctnessOutput[0] == -1)
precondition(abs(correctnessOutput[1] - Float(Int16.max) / 32768) < 0.000_001)
precondition(correctnessOutput[2] == 0)
correctnessQueue.push(Data([1, 0]))
correctnessQueue.flush()
correctnessOutput.withUnsafeMutableBufferPointer {
correctnessQueue.render(frames: 3, into: $0.baseAddress!)
}
precondition(correctnessOutput.allSatisfy { $0 == 0 })
let benchmarkQueue = SpeakerQueue()
let benchmarkFrames = 512
let benchmarkChunk = Data(repeating: 1, count: benchmarkFrames * 2)
var benchmarkOutput = [Float](repeating: 0, count: benchmarkFrames)
for _ in 0..<48 { benchmarkQueue.push(benchmarkChunk) }
for _ in 0..<2_000 {
benchmarkOutput.withUnsafeMutableBufferPointer {
benchmarkQueue.render(frames: benchmarkFrames, into: $0.baseAddress!)
}
benchmarkQueue.push(benchmarkChunk)
}
var benchmarkDurations = [UInt64]()
benchmarkDurations.reserveCapacity(30_000)
var benchmarkChecksum: Float = 0
for _ in 0..<30_000 {
let started = DispatchTime.now().uptimeNanoseconds
benchmarkOutput.withUnsafeMutableBufferPointer {
benchmarkQueue.render(frames: benchmarkFrames, into: $0.baseAddress!)
}
benchmarkQueue.push(benchmarkChunk)
benchmarkDurations.append(DispatchTime.now().uptimeNanoseconds - started)
benchmarkChecksum += benchmarkOutput[0]
}
benchmarkDurations.sort()
print("METRIC speaker_queue_median_ns=\(benchmarkDurations[benchmarkDurations.count / 2])")
print("METRIC speaker_queue_p99_ns=\(benchmarkDurations[benchmarkDurations.count * 99 / 100])")
print("METRIC speaker_queue_worst_ns=\(benchmarkDurations.last!)")
print("CHECKSUM \(benchmarkChecksum)")
exit(0)
#endif
let queue = SpeakerQueue()
let playFormat = AVAudioFormat(commonFormat: .pcmFormatFloat32, sampleRate: 24000, channels: 1, interleaved: false)!
let captureFormat = AVAudioFormat(commonFormat: .pcmFormatInt16, sampleRate: 24000, channels: 1, interleaved: true)!
var rebuildScheduled = false
var ignoreRouteChangesUntil = Date.distantPast
func scheduleRebuild(_ message: String, after delay: TimeInterval = 0.5) {
if rebuildScheduled { return }
rebuildScheduled = true
DispatchQueue.main.asyncAfter(deadline: .now() + delay) {
rebuildScheduled = false
// Starting an engine emits configuration changes of its own. AirPods can
// otherwise turn one real route change into a perpetual rebuild loop.
ignoreRouteChangesUntil = Date().addingTimeInterval(1.5)
log(message)
startInput(voiceProcessing: useAEC)
startOutput()
}
}
// Speaker: its own engine, pulling PCM16 from the stdin-fed queue.
var outputEngine: AVAudioEngine?
func startOutput() {
outputEngine?.stop()
let engine = AVAudioEngine()
outputEngine = engine
let source = AVAudioSourceNode(format: playFormat) { _, _, frameCount, audioBufferList -> OSStatus in
let out = UnsafeMutableAudioBufferListPointer(audioBufferList)[0].mData!.assumingMemoryBound(to: Float.self)
queue.render(frames: Int(frameCount), into: out)
return noErr
}
engine.attach(source)
engine.connect(source, to: engine.mainMixerNode, format: playFormat)
do {
try engine.start()
let format = engine.outputNode.outputFormat(forBus: 0)
log("output active (\(Int(format.sampleRate))Hz, \(format.channelCount)ch)")
} catch {
log("output engine failed: \(error)")
scheduleRebuild("audio engines unavailable — retrying")
}
}
// Microphone: take channel 0 of whatever the hardware provides, resample to
// 24kHz PCM16 for stdout. Channel extraction is manual because hardware
// channel counts vary wildly (1, 2, 3, 22...) and AVAudioConverter cannot
// downmix all of them.
var tapCount = 0
var inputEngine: AVAudioEngine?
var useAEC = CommandLine.arguments.contains("--aec")
func startInput(voiceProcessing: Bool) {
inputEngine?.stop()
let engine = AVAudioEngine()
inputEngine = engine
if voiceProcessing {
do {
try engine.inputNode.setVoiceProcessingEnabled(true)
} catch {
log("voice processing unavailable (\(error)) — no echo cancellation")
useAEC = false
return startInput(voiceProcessing: false)
}
if #available(macOS 14.0, *) {
engine.inputNode.voiceProcessingOtherAudioDuckingConfiguration = .init(
enableAdvancedDucking: false,
duckingLevel: .min
)
}
}
let micFormat = engine.inputNode.outputFormat(forBus: 0)
guard micFormat.sampleRate > 0, micFormat.channelCount > 0, let monoFormat = AVAudioFormat(
commonFormat: .pcmFormatFloat32, sampleRate: micFormat.sampleRate, channels: 1, interleaved: false)
else {
log("microphone route is not ready — retrying")
scheduleRebuild("audio engines unavailable — retrying")
return
}
guard let converter = AVAudioConverter(from: monoFormat, to: captureFormat) else {
log("cannot convert \(micFormat.sampleRate)Hz to 24kHz — retrying")
scheduleRebuild("audio engines unavailable — retrying")
return
}
engine.inputNode.installTap(onBus: 0, bufferSize: 2400, format: micFormat) { buffer, _ in
tapCount += 1
if tapCount == 1 { log("mic active (echo cancellation \(voiceProcessing ? "on" : "off"))") }
guard let channel = buffer.floatChannelData?[0], buffer.frameLength > 0 else { return }
guard let mono = AVAudioPCMBuffer(pcmFormat: monoFormat, frameCapacity: buffer.frameLength) else { return }
memcpy(mono.floatChannelData![0], channel, Int(buffer.frameLength) * 4)
mono.frameLength = buffer.frameLength
let capacity = AVAudioFrameCount(Double(buffer.frameLength) * 24000.0 / micFormat.sampleRate) + 32
guard let converted = AVAudioPCMBuffer(pcmFormat: captureFormat, frameCapacity: capacity) else { return }
var consumed = false
converter.convert(to: converted, error: nil) { _, status in
if consumed {
status.pointee = .noDataNow
return nil
}
consumed = true
status.pointee = .haveData
return mono
}
guard converted.frameLength > 0, let out = converted.int16ChannelData?[0] else { return }
FileHandle.standardOutput.write(Data(bytes: out, count: Int(converted.frameLength) * 2))
}
do {
try engine.start()
log("input active (\(Int(micFormat.sampleRate))Hz, \(micFormat.channelCount)ch, echo cancellation \(voiceProcessing ? "on" : "off"))")
} catch {
if voiceProcessing {
log("input engine failed with voice processing (\(error)) — retrying without")
useAEC = false
return startInput(voiceProcessing: false)
}
log("input engine failed: \(error)")
scheduleRebuild("audio engines unavailable — retrying")
return
}
// Watchdog: on some devices the voice-processed tap simply never fires.
// Fall back to a plain tap; without VP the mic reliably delivers.
if voiceProcessing {
DispatchQueue.main.asyncAfter(deadline: .now() + 2.5) {
if tapCount > 0 { return }
log("voice-processed mic delivered nothing — restarting without echo cancellation")
useAEC = false
startInput(voiceProcessing: false)
}
}
}
// Voice processing is opt-in (--aec): it is only needed on speakers, and on
// some machines (observed with Bluetooth headsets active) the VP engine binds
// to the wrong capture device entirely, delivering noise instead of the mic.
// Mic first: activating the mic flips Bluetooth headsets from music mode to
// headset mode, reconfiguring the output device. Starting output afterwards
// (and rebuilding on any route change below) keeps playback on the live device.
startInput(voiceProcessing: useAEC)
startOutput()
// Device switches (Bluetooth profile flips, headphones plugged/unplugged,
// default device changes) stop engines silently. Rebuild both, debounced.
NotificationCenter.default.addObserver(
forName: .AVAudioEngineConfigurationChange, object: nil, queue: .main
) { _ in
if Date() < ignoreRouteChangesUntil { return }
scheduleRebuild("audio route changed — rebuilding engines")
}
signal(SIGUSR1, SIG_IGN)
let flushSignal = DispatchSource.makeSignalSource(signal: SIGUSR1, queue: .main)
flushSignal.setEventHandler { queue.flush() }
flushSignal.resume()
DispatchQueue.global().async {
while true {
let chunk = FileHandle.standardInput.availableData
if chunk.isEmpty { exit(0) } // parent closed stdin
queue.push(chunk)
}
}
RunLoop.main.run()
-48
View File
@@ -1,48 +0,0 @@
import type { CompletionReceipt, OpenCodeAnnouncement } from "./opencode-notification"
import { promptKey } from "./prompt-handle"
export type RoutedAnnouncement = {
readonly announcement: OpenCodeAnnouncement
readonly promptID?: string
readonly delegationID?: string
}
export function createNotificationRouter(deliver: (announcement: RoutedAnnouncement) => void) {
const delegations = new Map<string, string>()
const deferred: OpenCodeAnnouncement[] = []
let pendingDelegatedTools = 0
const route = (announcement: OpenCodeAnnouncement) => {
const prompt = "prompt_id" in announcement.notification
? { sessionID: announcement.notification.session_id, promptID: announcement.notification.prompt_id }
: undefined
const promptID = prompt?.promptID
if (prompt && pendingDelegatedTools > 0 && !delegations.has(promptKey(prompt))) {
deferred.push(announcement)
return
}
deliver({
announcement,
promptID,
delegationID: prompt ? delegations.get(promptKey(prompt)) : undefined,
})
}
return {
route,
beginDelegatedTool() {
pendingDelegatedTools += 1
},
finishDelegatedTool(
delegationID: string,
admittedPrompt?: { readonly sessionID: string; readonly promptID: string },
) {
if (admittedPrompt) delegations.set(promptKey(admittedPrompt), delegationID)
pendingDelegatedTools -= 1
if (pendingDelegatedTools === 0) deferred.splice(0).forEach(route)
},
acknowledged(receipt: CompletionReceipt) {
delegations.delete(promptKey(receipt))
},
}
}
@@ -1,97 +0,0 @@
import { Schema } from "effect"
import type { PromptHandle } from "./prompt-handle"
const Completed = Schema.Struct({
type: Schema.Literal("opencode.prompt.completed"),
session_id: Schema.String,
prompt_id: Schema.String,
status: Schema.Literals(["completed", "failed"]),
text: Schema.String,
error: Schema.optional(Schema.String),
})
const Failed = Schema.Struct({
type: Schema.Literal("opencode.prompt.failed"),
session_id: Schema.String,
prompt_id: Schema.String,
status: Schema.Literal("failed"),
error: Schema.String,
})
const PermissionBlocked = Schema.Struct({
type: Schema.Literal("opencode.prompt.blocked"),
prompt_id: Schema.String,
blocker: Schema.Literal("permission"),
session_id: Schema.String,
request_id: Schema.String,
action: Schema.String,
resources: Schema.Unknown,
})
const QuestionBlocked = Schema.Struct({
type: Schema.Literal("opencode.prompt.blocked"),
prompt_id: Schema.String,
blocker: Schema.Literal("question"),
session_id: Schema.String,
request_id: Schema.String,
questions: Schema.Unknown,
})
const FormBlocked = Schema.Struct({
type: Schema.Literal("opencode.prompt.blocked"),
prompt_id: Schema.String,
blocker: Schema.Literal("form"),
session_id: Schema.String,
form_id: Schema.String,
title: Schema.String,
fields: Schema.Unknown,
})
const EventsFailed = Schema.Struct({
type: Schema.Literal("opencode.events.failed"),
error: Schema.String,
})
export const OpenCodeNotification = Schema.Union([
Completed,
Failed,
PermissionBlocked,
QuestionBlocked,
FormBlocked,
EventsFailed,
])
export type OpenCodeNotification = typeof OpenCodeNotification.Type
export type OpenCodePromptBlocked = Extract<OpenCodeNotification, { readonly type: "opencode.prompt.blocked" }>
export type CompletionReceipt = PromptHandle
export type OpenCodeAnnouncement = {
readonly notification: OpenCodeNotification
readonly receipt?: CompletionReceipt
}
export function openCodeAnnouncementText(notification: OpenCodeNotification) {
const text = (() => {
switch (notification.type) {
case "opencode.prompt.completed":
return `OpenCode prompt ${notification.status}: ${notification.text}${notification.error ? ` Error: ${notification.error}` : ""}`
case "opencode.prompt.failed":
return `OpenCode prompt failed: ${notification.error}`
case "opencode.prompt.blocked":
if (notification.blocker === "permission")
return `OpenCode needs permission for ${notification.action}: ${compact(notification.resources)}`
if (notification.blocker === "question")
return `OpenCode needs the user to answer: ${compact(notification.questions)}`
return `OpenCode needs the user to complete the form “${notification.title}”: ${compact(notification.fields)}`
case "opencode.events.failed":
return `OpenCode event delivery failed: ${notification.error}`
}
return "OpenCode has an update."
})()
return text.length > 800 ? text.slice(0, 797) + "..." : text
}
function compact(value: unknown) {
if (typeof value === "string") return value
return JSON.stringify(value) ?? String(value)
}
-811
View File
@@ -1,811 +0,0 @@
import { realpathSync } from "node:fs"
import { tmpdir } from "node:os"
import { basename, isAbsolute, relative } from "node:path"
import { ClientError, isSessionNotFoundError, OpenCode } from "@opencode-ai/client/promise"
import type { SessionMessageAssistant, SessionMessageUser, V2Event } from "@opencode-ai/client/promise"
import { Form, Question, SessionMessage } from "@opencode-ai/client/effect"
import { Context, Effect, FiberMap, Layer, ManagedRuntime, Option, Schema, Stream } from "effect"
import { createCompletionStore, type CompletionStore } from "./completion-store"
import type {
CompletionReceipt,
OpenCodeAnnouncement,
OpenCodeNotification,
OpenCodePromptBlocked,
} from "./opencode-notification"
import type { VoiceTool, VoiceToolExecution } from "./protocol"
import type { PromptHandle } from "./prompt-handle"
type Client = ReturnType<typeof OpenCode.make>
const AdmittedPrompt = Symbol("AdmittedPrompt")
type Tool = {
readonly description: string
readonly parameters: unknown
readonly execute: (input: Record<string, unknown>) => Effect.Effect<unknown, unknown>
}
type BridgeApi = {
readonly definitions: ReadonlyArray<VoiceTool>
readonly execute: (name: string, input: Record<string, unknown>) => Effect.Effect<VoiceToolExecution, unknown>
readonly acknowledge: (receipt: CompletionReceipt) => Effect.Effect<void, unknown>
readonly close: Effect.Effect<void>
}
class Bridge extends Context.Service<Bridge, BridgeApi>()("@opencode-ai/voice/OpenCodeBridge") {}
export async function createOpenCodeBridge(options: {
client: Client
directory: string
model: { readonly providerID: string; readonly id: string; readonly variant?: string }
notify: (announcement: OpenCodeAnnouncement) => void
trace?: (event: string, data?: Record<string, unknown>) => void
completionStore?: CompletionStore
}) {
const completionStore = options.completionStore ?? (await createCompletionStore())
const runtime = ManagedRuntime.make(Layer.effect(Bridge, makeBridge({ ...options, completionStore })))
const bridge = await runtime.runPromise(Bridge)
return {
definitions: bridge.definitions,
execute: (name: string, input: Record<string, unknown>) => runtime.runPromise(bridge.execute(name, input)),
acknowledge: (receipt: CompletionReceipt) => runtime.runPromise(bridge.acknowledge(receipt)),
close: async () => {
await runtime.runPromise(bridge.close)
await Promise.race([runtime.dispose(), Bun.sleep(1_000)])
await completionStore.close()
},
}
}
const makeBridge = Effect.fnUntraced(function* (options: {
client: Client
directory: string
model: { readonly providerID: string; readonly id: string; readonly variant?: string }
notify: (announcement: OpenCodeAnnouncement) => void
trace?: (event: string, data?: Record<string, unknown>) => void
completionStore: CompletionStore
}) {
const knownProjects = new Set<string>()
const knownSessions = new Set<string>()
const registrations = new Map<string, PromptHandle>()
const promoted = new Map<string, string>()
const latest = new Map<string, string>()
const announcedBlockers = new Set<string>()
const completions = yield* FiberMap.make<string>()
const eventAbort = yield* Effect.acquireRelease(
Effect.sync(() => new AbortController()),
(controller) => Effect.sync(() => controller.abort()),
)
const notify = (notification: OpenCodeNotification, receipt?: CompletionReceipt) =>
Effect.sync(() => options.notify({ notification, receipt }))
const trace = (event: string, data: Record<string, unknown>) => Effect.sync(() => options.trace?.(event, data))
const request = <A>(run: (signal: AbortSignal) => PromiseLike<A>) =>
Effect.tryPromise({ try: run, catch: (cause) => cause })
const announceBlocker = (key: string, notification: OpenCodeNotification) => {
if (announcedBlockers.has(key)) return Effect.void
announcedBlockers.add(key)
return notify(notification)
}
const clearRegistration = (handle: PromptHandle) =>
Effect.sync(() => {
registrations.delete(handle.promptID)
if (promoted.get(handle.sessionID) === handle.promptID) promoted.delete(handle.sessionID)
if (latest.get(handle.sessionID) === handle.promptID) latest.delete(handle.sessionID)
})
const listProjects = Effect.fnUntraced(function* () {
const seen = new Set<string>()
const temporary = realpathSync(tmpdir())
return (yield* request((signal) => options.client.project.list({ signal })))
.sort((a, b) => b.time.updated - a.time.updated)
.filter((project) => {
const fromTemporary = relative(temporary, project.worktree)
if (
project.id === "global" ||
fromTemporary === "" ||
(!fromTemporary.startsWith("..") && !isAbsolute(fromTemporary)) ||
seen.has(project.worktree)
)
return false
seen.add(project.worktree)
return true
})
})
const projectDirectory = Effect.fnUntraced(function* (projectID: string) {
if (!knownProjects.has(projectID)) return undefined
return (yield* listProjects()).find((project) => project.id === projectID)?.worktree
})
const complete = Effect.fnUntraced(function* (handle: PromptHandle) {
yield* trace("opencode.wait.started", handle)
yield* waitForIdle(handle)
const reply = yield* request((signal) => finalReply(options.client, handle, signal))
yield* trace("opencode.wait.completed", { ...handle, replyID: reply?.id, error: reply?.error?.message })
const notification: OpenCodeNotification = {
type: "opencode.prompt.completed",
session_id: handle.sessionID,
prompt_id: handle.promptID,
status: reply?.error ? "failed" : "completed",
text: reply
? assistantText(reply) || "OpenCode finished without a text reply."
: "OpenCode finished without a reply.",
error: reply?.error?.message,
}
yield* request(() => options.completionStore.completed(handle, notification))
yield* notify(notification, handle)
})
const waitForIdle = Effect.fnUntraced(function* (handle: PromptHandle, attempt = 1): Effect.fn.Return<void, unknown> {
return yield* request((signal) => options.client.session.wait({ sessionID: handle.sessionID }, { signal })).pipe(
Effect.catch((error) => {
if (!(error instanceof ClientError) || error.reason !== "Transport") return Effect.fail(error)
return trace("opencode.wait.retrying", {
...handle,
attempt,
reason: error.reason,
cause: error.cause instanceof Error ? error.cause.name : typeof error.cause,
}).pipe(Effect.andThen(Effect.sleep("1 second")), Effect.andThen(waitForIdle(handle, attempt + 1)))
}),
)
})
const register = Effect.fnUntraced(function* (handle: PromptHandle) {
yield* complete(handle).pipe(
Effect.catch((error) => {
const notification: OpenCodeNotification = {
type: "opencode.prompt.failed",
session_id: handle.sessionID,
prompt_id: handle.promptID,
status: "failed",
error: String(error),
}
return request(() => options.completionStore.completed(handle, notification)).pipe(
Effect.andThen(notify(notification, handle)),
)
}),
Effect.ensuring(clearRegistration(handle)),
FiberMap.run(completions, handle.promptID, { onlyIfMissing: true, startImmediately: true }),
)
})
const restoreBlockers = Effect.fnUntraced(function* (handle: PromptHandle) {
const [permissions, questions, forms] = yield* Effect.all(
[
request((signal) => options.client.permission.list({ sessionID: handle.sessionID }, { signal })),
request((signal) => options.client.question.list({ sessionID: handle.sessionID }, { signal })),
request((signal) => options.client.form.list({ sessionID: handle.sessionID }, { signal })),
],
{ concurrency: "unbounded" },
)
yield* Effect.forEach(
[
...permissions.map((item) => ({
key: `permission:${item.id}`,
notification: {
type: "opencode.prompt.blocked",
prompt_id: handle.promptID,
blocker: "permission",
session_id: handle.sessionID,
request_id: item.id,
action: item.action,
resources: item.resources,
} satisfies OpenCodeNotification,
})),
...questions.map((item) => ({
key: `question:${item.id}`,
notification: {
type: "opencode.prompt.blocked",
prompt_id: handle.promptID,
blocker: "question",
session_id: handle.sessionID,
request_id: item.id,
questions: item.questions,
} satisfies OpenCodeNotification,
})),
...forms.map((item) => ({
key: `form:${item.id}`,
notification: {
type: "opencode.prompt.blocked",
prompt_id: handle.promptID,
blocker: "form",
session_id: handle.sessionID,
form_id: item.id,
title: item.title,
fields: item.fields,
} satisfies OpenCodeNotification,
})),
],
(item) => announceBlocker(item.key, item.notification),
{ discard: true },
)
})
const admit = Effect.fnUntraced(function* (sessionID: string, text: string) {
const handle = { sessionID, promptID: SessionMessage.ID.create() }
registrations.set(handle.promptID, handle)
latest.set(sessionID, handle.promptID)
yield* trace("opencode.prompt.admitting", handle)
yield* request(() => options.completionStore.admitting(handle, text))
const admitted = yield* request((signal) =>
options.client.session.prompt({ sessionID, id: handle.promptID, text }, { signal }),
).pipe(
Effect.tapError(() =>
request(() => options.completionStore.remove(handle)).pipe(
Effect.andThen(clearRegistration(handle)),
),
),
)
knownSessions.add(sessionID)
yield* request(() => options.completionStore.pending(handle))
yield* trace("opencode.prompt.admitted", { ...handle, admittedSeq: admitted.admittedSeq })
yield* register(handle)
return {
status: "started",
session_id: sessionID,
prompt_id: admitted.id,
notification: "registered",
message: "OpenCode accepted the prompt and will send a completion notification. Continue the conversation now.",
[AdmittedPrompt]: handle,
}
})
const start = Effect.fnUntraced(function* (text: string, projectID?: string) {
const directory = projectID ? yield* projectDirectory(projectID) : options.directory
if (!directory) return toolError("Use a project ID returned by find_projects.")
const session = yield* request((signal) =>
options.client.session.create({ location: { directory }, model: options.model }, { signal }),
)
return yield* admit(session.id, text)
})
const tools: Record<string, Tool> = {
find_projects: {
description: "Find known OpenCode projects by display name. Returns opaque IDs, never filesystem paths.",
parameters: {
type: "object",
additionalProperties: false,
properties: {
query: { type: ["string", "null"], description: "Name fragment, or null for recent projects." },
limit: { type: "integer", minimum: 1, maximum: 20 },
},
required: ["query", "limit"],
},
execute: Effect.fnUntraced(function* (input) {
const query = typeof input["query"] === "string" ? input["query"].toLowerCase() : undefined
const limit = typeof input["limit"] === "number" ? input["limit"] : 10
const projects = (yield* listProjects())
.filter((project) => !query || projectLabel(project).toLowerCase().includes(query))
.slice(0, limit)
projects.forEach((project) => knownProjects.add(project.id))
return {
status: "ok",
projects: projects.map((project) => ({
id: project.id,
name: projectLabel(project),
directories: 1 + project.sandboxes.length,
updated: new Date(project.time.updated).toISOString(),
})),
}
}),
},
find_sessions: {
description:
"Find root OpenCode sessions by title and recency. Search the launch project by default; use all_projects only when requested or the local search misses.",
parameters: {
type: "object",
additionalProperties: false,
properties: {
query: { type: ["string", "null"], description: "Title words, or null for recent sessions." },
scope: { type: "string", enum: ["current_project", "all_projects"] },
recency: { type: "string", enum: ["day", "week", "month", "any"] },
limit: { type: "integer", minimum: 1, maximum: 20 },
},
required: ["query", "scope", "recency", "limit"],
},
execute: Effect.fnUntraced(function* (input) {
const query = typeof input["query"] === "string" ? input["query"] : undefined
const scope = input["scope"] === "all_projects" ? "all_projects" : "current_project"
const recency = typeof input["recency"] === "string" ? input["recency"] : "any"
const limit = typeof input["limit"] === "number" ? input["limit"] : 10
const durations: Record<string, number> = { day: 86_400_000, week: 604_800_000, month: 2_592_000_000 }
const threshold = recency === "any" ? 0 : Date.now() - (durations[recency] ?? 0)
const [result, projectList] = yield* Effect.all(
[
request((signal) =>
options.client.session.list(
{
...(scope === "current_project" ? { directory: options.directory } : {}),
search: query,
parentID: null,
limit: recency === "any" ? limit : Math.min(limit * 5, 100),
order: "desc",
},
{ signal },
),
),
listProjects(),
],
{ concurrency: "unbounded" },
)
const projects = new Map(projectList.map((project) => [project.id, projectLabel(project)]))
const sessions = result.data
.filter((session) => projects.has(session.projectID) && session.time.updated >= threshold)
.slice(0, limit)
sessions.forEach((session) => knownSessions.add(session.id))
return {
status: "ok",
scope,
sessions: sessions.map((session) => ({
id: session.id,
title: session.title,
project: projects.get(session.projectID) ?? "project",
updated: new Date(session.time.updated).toISOString(),
})),
}
}),
},
read_session: {
description:
"Read bounded recent user and assistant text from a session returned by find_sessions or start_session. This never prompts or wakes the coding agent.",
parameters: {
type: "object",
additionalProperties: false,
properties: {
session_id: { type: "string", description: "Session ID returned by a previous voice tool." },
limit: { type: "integer", minimum: 1, maximum: 20 },
},
required: ["session_id", "limit"],
},
execute: Effect.fnUntraced(function* (input) {
const sessionID = knownSession(input, knownSessions)
if (!sessionID) return toolError("Use a session ID returned by find_sessions or start_session.")
const limit = typeof input["limit"] === "number" ? input["limit"] : 10
const [session, messages] = yield* Effect.all(
[
request((signal) => options.client.session.get({ sessionID }, { signal })),
request((signal) => options.client.message.list({ sessionID, order: "desc", limit }, { signal })),
],
{ concurrency: "unbounded" },
)
const latestAssistant = messages.data.find(
(message): message is SessionMessageAssistant => message.type === "assistant",
)
return {
status: "ok",
title: session.title,
running: latestAssistant ? !latestAssistant.time.completed : false,
messages: messages.data
.toReversed()
.flatMap((message) =>
message.type === "user" || message.type === "assistant" ? [sessionMessage(message)] : [],
),
}
}),
},
rename_session: {
description:
"Rename a session returned by find_sessions or start_session after the user requests a new title. This changes only the display title and does not prompt, wake, or interrupt the coding agent.",
parameters: {
type: "object",
additionalProperties: false,
properties: {
session_id: { type: "string", description: "Session ID returned by a previous voice tool." },
title: { type: "string", description: "The complete new session title." },
},
required: ["session_id", "title"],
},
execute: Effect.fnUntraced(function* (input) {
const sessionID = knownSession(input, knownSessions)
const title = requireString(input, "title")?.trim()
if (!sessionID) return toolError("Use a session ID returned by find_sessions or start_session.")
if (!title) return toolError("A non-empty session title is required.")
const session = yield* request((signal) => options.client.session.get({ sessionID }, { signal }))
if (session.title === title)
return { status: "unchanged", session_id: sessionID, previous_title: session.title, title }
yield* request((signal) => options.client.session.rename({ sessionID, title }, { signal }))
return { status: "renamed", session_id: sessionID, previous_title: session.title, title }
}),
},
start_session: {
description:
"Create an OpenCode session, admit its first prompt, and register a one-shot completion notification. Omit project_id to use the launch project.",
parameters: {
type: "object",
additionalProperties: false,
properties: {
text: { type: "string", description: "Clear instruction for the coding agent." },
project_id: { type: ["string", "null"], description: "Opaque project ID, or null for the launch project." },
},
required: ["text", "project_id"],
},
execute: (input) => {
const text = requireString(input, "text")
if (!text) return Effect.succeed(toolError("Task text is required."))
return start(text, typeof input["project_id"] === "string" ? input["project_id"] : undefined)
},
},
prompt_session: {
description:
"Admit a prompt into a discovered or previously started OpenCode session and register a one-shot completion notification.",
parameters: {
type: "object",
additionalProperties: false,
properties: {
session_id: { type: "string", description: "Session ID returned by a previous voice tool." },
text: { type: "string", description: "Clear instruction for the coding agent." },
},
required: ["session_id", "text"],
},
execute: (input) => {
const sessionID = knownSession(input, knownSessions)
const text = requireString(input, "text")
if (!sessionID) return Effect.succeed(toolError("Use a session ID returned by find_sessions or start_session."))
if (!text) return Effect.succeed(toolError("Task text is required."))
return admit(sessionID, text)
},
},
interrupt_session: {
description: "Interrupt one OpenCode session. Call only after the user explicitly confirms the interruption.",
parameters: sessionParameters(),
execute: Effect.fnUntraced(function* (input) {
const sessionID = knownSession(input, knownSessions)
if (!sessionID) return toolError("Use a session ID returned by a previous voice tool.")
yield* request((signal) => options.client.session.interrupt({ sessionID }, { signal }))
return { status: "interrupted", session_id: sessionID }
}),
},
list_pending_permissions: {
description: "List permission requests blocking one OpenCode session.",
parameters: sessionParameters(),
execute: Effect.fnUntraced(function* (input) {
const sessionID = knownSession(input, knownSessions)
if (!sessionID) return toolError("Use a session ID returned by a previous voice tool.")
const requests = yield* request((signal) => options.client.permission.list({ sessionID }, { signal }))
return {
status: "ok",
requests: requests.map((item) => ({ id: item.id, action: item.action, resources: item.resources })),
}
}),
},
reply_permission: {
description:
"Allow once or reject a pending permission after stating the action and resources and receiving the user's explicit decision.",
parameters: {
type: "object",
additionalProperties: false,
properties: {
session_id: { type: "string" },
request_id: { type: "string" },
decision: { type: "string", enum: ["allow_once", "reject"] },
},
required: ["session_id", "request_id", "decision"],
},
execute: Effect.fnUntraced(function* (input) {
const sessionID = knownSession(input, knownSessions)
const requestID = requireString(input, "request_id")
const decision = input["decision"]
if (!sessionID || !requestID || (decision !== "allow_once" && decision !== "reject"))
return toolError("A known session, request ID, and valid decision are required.")
const requests = yield* request((signal) => options.client.permission.list({ sessionID }, { signal }))
if (!requests.some((item) => item.id === requestID))
return toolError("That permission request is not pending.", true)
yield* request((signal) =>
options.client.permission.reply(
{ sessionID, requestID, reply: decision === "allow_once" ? "once" : "reject" },
{ signal },
),
)
return { status: decision === "allow_once" ? "allowed_once" : "rejected", request_id: requestID }
}),
},
reply_question: {
description: "Reply to questions blocking an OpenCode session after collecting the user's answers.",
parameters: {
type: "object",
additionalProperties: false,
properties: {
session_id: { type: "string" },
request_id: { type: "string" },
answers: {
type: "array",
description: "One string array per question, preserving question order.",
items: { type: "array", items: { type: "string" } },
},
},
required: ["session_id", "request_id", "answers"],
},
execute: Effect.fnUntraced(function* (input) {
const sessionID = knownSession(input, knownSessions)
const requestID = requireString(input, "request_id")
const answers = Option.getOrUndefined(decodeQuestionAnswers(input["answers"]))
if (!sessionID || !requestID || !answers)
return toolError("A known session, request ID, and valid answers are required.")
yield* request((signal) => options.client.question.reply({ sessionID, requestID, answers }, { signal }))
return { status: "answered", request_id: requestID }
}),
},
reject_question: {
description: "Reject a pending OpenCode question after the user explicitly declines to answer.",
parameters: requestParameters(),
execute: Effect.fnUntraced(function* (input) {
const sessionID = knownSession(input, knownSessions)
const requestID = requireString(input, "request_id")
if (!sessionID || !requestID) return toolError("A known session and request ID are required.")
yield* request((signal) => options.client.question.reject({ sessionID, requestID }, { signal }))
return { status: "rejected", request_id: requestID }
}),
},
reply_form: {
description: "Submit values for a form blocking an OpenCode session after collecting them from the user.",
parameters: {
type: "object",
additionalProperties: false,
properties: {
session_id: { type: "string" },
form_id: { type: "string" },
answer: { type: "object", additionalProperties: true },
},
required: ["session_id", "form_id", "answer"],
},
execute: Effect.fnUntraced(function* (input) {
const sessionID = knownSession(input, knownSessions)
const formID = requireString(input, "form_id")
const answer = Option.getOrUndefined(decodeFormAnswer(input["answer"]))
if (!sessionID || !formID || !answer)
return toolError("A known session, form ID, and valid answer are required.")
yield* request((signal) => options.client.form.reply({ sessionID, formID, answer }, { signal }))
return { status: "answered", form_id: formID }
}),
},
cancel_form: {
description: "Cancel a pending OpenCode form after the user explicitly declines to complete it.",
parameters: {
type: "object",
additionalProperties: false,
properties: { session_id: { type: "string" }, form_id: { type: "string" } },
required: ["session_id", "form_id"],
},
execute: Effect.fnUntraced(function* (input) {
const sessionID = knownSession(input, knownSessions)
const formID = requireString(input, "form_id")
if (!sessionID || !formID) return toolError("A known session and form ID are required.")
yield* request((signal) => options.client.form.cancel({ sessionID, formID }, { signal }))
return { status: "cancelled", form_id: formID }
}),
},
}
const definitions = Object.entries(tools).map(
([name, tool]) =>
({ type: "function", name, description: tool.description, parameters: tool.parameters }) satisfies VoiceTool,
)
yield* Effect.suspend(() =>
Stream.fromAsyncIterable(options.client.event.subscribe({ signal: eventAbort.signal }), (cause) => cause).pipe(
Stream.runForEach((event) => {
if (event.type === "session.input.promoted") {
if (!registrations.has(event.data.inputID)) return Effect.void
promoted.set(event.data.sessionID, event.data.inputID)
return trace("opencode.prompt.promoted", {
sessionID: event.data.sessionID,
promptID: event.data.inputID,
})
}
const sessionID = blockerSession(event)
const promptID = sessionID ? (promoted.get(sessionID) ?? latest.get(sessionID)) : undefined
const blocker = sessionID && promptID ? blockerNotification(event, sessionID, promptID) : undefined
if (!blocker || !promptID) return Effect.void
options.trace?.("opencode.prompt.blocked", { sessionID, promptID, blocker: blocker["blocker"] })
if (registrations.has(promptID)) return announceBlocker(blockerKey(blocker), blocker)
return Effect.void
}),
Effect.catch((error) => notify({ type: "opencode.events.failed", error: String(error) })),
Effect.andThen(Effect.sleep("1 second")),
),
).pipe(Effect.forever, Effect.forkScoped({ startImmediately: true }))
yield* Effect.forEach(
options.completionStore.entries(),
Effect.fnUntraced(function* (entry) {
if (entry.status !== "completed") {
const exists = yield* request((signal) =>
options.client.session.get({ sessionID: entry.handle.sessionID }, { signal }),
).pipe(
Effect.as(true),
Effect.catch((error) => {
if (!isSessionNotFoundError(error)) return Effect.fail(error)
return request(() => options.completionStore.remove(entry.handle)).pipe(
Effect.andThen(trace("opencode.prompt.orphaned", entry.handle)),
Effect.as(false),
)
}),
)
if (!exists) return
}
knownSessions.add(entry.handle.sessionID)
if (entry.status === "admitting") {
registrations.set(entry.handle.promptID, entry.handle)
latest.set(entry.handle.sessionID, entry.handle.promptID)
yield* trace("opencode.prompt.reconciling", entry.handle)
yield* request((signal) =>
options.client.session.prompt(
{ sessionID: entry.handle.sessionID, id: entry.handle.promptID, text: entry.text },
{ signal },
),
)
yield* request(() => options.completionStore.pending(entry.handle))
yield* restoreBlockers(entry.handle)
yield* register(entry.handle)
return
}
if (entry.status === "pending") {
registrations.set(entry.handle.promptID, entry.handle)
latest.set(entry.handle.sessionID, entry.handle.promptID)
yield* trace("opencode.wait.restored", entry.handle)
yield* restoreBlockers(entry.handle)
yield* register(entry.handle)
return
}
yield* trace("opencode.notification.restored", entry.handle)
yield* notify(entry.notification, entry.handle)
}),
{ concurrency: 4, discard: true },
)
return Bridge.of({
definitions,
execute: (name, input) =>
(tools[name]?.execute(input) ?? Effect.succeed(toolError(`Unknown tool ${name}.`))).pipe(
Effect.map(toolExecution),
),
acknowledge: (receipt) => request(() => options.completionStore.remove(receipt)),
close: Effect.all([Effect.sync(() => eventAbort.abort()), FiberMap.clear(completions)], { discard: true }),
})
})
function toolExecution(output: unknown): VoiceToolExecution {
if (!isPromptToolOutput(output)) return { output }
return {
output: Object.fromEntries(Object.entries(output)),
admittedPrompt: output[AdmittedPrompt],
}
}
function isPromptToolOutput(
output: unknown,
): output is Record<string, unknown> & { readonly [AdmittedPrompt]: PromptHandle } {
if (!output || typeof output !== "object" || !(AdmittedPrompt in output)) return false
const handle = output[AdmittedPrompt]
return (
!!handle &&
typeof handle === "object" &&
"sessionID" in handle &&
typeof handle.sessionID === "string" &&
"promptID" in handle &&
typeof handle.promptID === "string"
)
}
async function finalReply(client: Client, handle: PromptHandle, signal: AbortSignal) {
let reply: SessionMessageAssistant | undefined
let cursor: string | undefined
while (true) {
const page = await client.message.list(
cursor
? { sessionID: handle.sessionID, limit: 200, cursor }
: { sessionID: handle.sessionID, limit: 200, order: "desc" },
{ signal },
)
for (const message of page.data) {
if (message.id === handle.promptID) return reply
if (message.type === "user") reply = undefined
if (message.type === "assistant" && !reply) reply = message
}
cursor = page.cursor.next ?? undefined
if (!cursor) return undefined
}
}
function assistantText(message: SessionMessageAssistant) {
return message.content
.filter((part) => part.type === "text")
.map((part) => part.text)
.join("\n")
.slice(0, 4_000)
}
function sessionMessage(message: SessionMessageUser | SessionMessageAssistant) {
if (message.type === "user") return { role: "user", text: message.text.slice(0, 2_000) }
return {
role: "assistant",
status: message.time.completed ? "completed" : "running",
text: assistantText(message),
tools: message.content
.filter((part) => part.type === "tool")
.map((part) => ({ name: part.name, status: part.state.status })),
}
}
function blockerSession(event: V2Event) {
if (event.type === "permission.v2.asked" || event.type === "question.v2.asked") return event.data.sessionID
if (event.type === "form.created") return event.data.form.sessionID
return undefined
}
function blockerNotification(event: V2Event, sessionID: string, promptID: string): OpenCodePromptBlocked | undefined {
if (event.type === "permission.v2.asked")
return {
type: "opencode.prompt.blocked",
prompt_id: promptID,
blocker: "permission",
session_id: sessionID,
request_id: event.data.id,
action: event.data.action,
resources: event.data.resources,
}
if (event.type === "question.v2.asked")
return {
type: "opencode.prompt.blocked",
prompt_id: promptID,
blocker: "question",
session_id: sessionID,
request_id: event.data.id,
questions: event.data.questions,
}
if (event.type === "form.created")
return {
type: "opencode.prompt.blocked",
prompt_id: promptID,
blocker: "form",
session_id: sessionID,
form_id: event.data.form.id,
title: event.data.form.title,
fields: event.data.form.fields,
}
return undefined
}
function blockerKey(notification: OpenCodePromptBlocked) {
if (notification.blocker === "form") return `form:${notification.form_id}`
return `${notification.blocker}:${notification.request_id}`
}
function projectLabel(project: { readonly name?: string; readonly worktree: string }) {
return project.name ?? (basename(project.worktree) || "project")
}
function requireString(input: Record<string, unknown>, key: string) {
const value = input[key]
if (typeof value !== "string" || value.trim().length === 0) return undefined
return value
}
function knownSession(input: Record<string, unknown>, known: ReadonlySet<string>) {
const sessionID = requireString(input, "session_id")
if (!sessionID || !known.has(sessionID)) return undefined
return sessionID
}
const decodeQuestionAnswers = Schema.decodeUnknownOption(Schema.Array(Question.Answer))
const decodeFormAnswer = Schema.decodeUnknownOption(Form.Answer)
function sessionParameters() {
return {
type: "object",
additionalProperties: false,
properties: { session_id: { type: "string" } },
required: ["session_id"],
}
}
function requestParameters() {
return {
type: "object",
additionalProperties: false,
properties: { session_id: { type: "string" }, request_id: { type: "string" } },
required: ["session_id", "request_id"],
}
}
function toolError(message: string, retryable = false) {
return { status: "error", message, retryable }
}
-19
View File
@@ -1,19 +0,0 @@
export const PCM_SAMPLE_RATE = 24_000
export const PCM_BYTES_PER_MS = (PCM_SAMPLE_RATE * 2) / 1_000
export const PCM_METER_FRAME_MS = 1_000 / 60
export function pcmLevel(bytes: Buffer) {
const samples = Math.floor(bytes.length / 2)
if (samples === 0) return 0
const stride = Math.max(1, Math.floor(samples / 1_200))
let energy = 0
let count = 0
for (let index = 0; index < samples; index += stride) {
const sample = bytes.readInt16LE(index * 2) / 32_768
energy += sample * sample
count += 1
}
const rms = Math.sqrt(energy / count)
if (rms < 0.008) return 0
return Math.min(1, Math.sqrt((rms - 0.008) / 0.18))
}
-37
View File
@@ -1,37 +0,0 @@
import { Plugin } from "@opencode-ai/plugin/v2/tui"
import { createSignal } from "solid-js"
function VoiceStatus(props: { context: Plugin.Context }) {
const [enabled, setEnabled] = createSignal(false)
props.context.keymap.layer(() => ({
mode: "global",
commands: [
{
id: "voice.smoke.toggle",
title: "Toggle voice plugin",
description: "Toggle the V2 voice plugin smoke-test status",
group: "Voice",
bind: "alt+v",
palette: true,
run() {
setEnabled((value) => !value)
},
},
],
}))
return (
<box paddingTop={1}>
<text>voice plugin: {enabled() ? "on" : "off"} (alt+v)</text>
</box>
)
}
export default Plugin.define({
id: "opencode.voice-smoke",
setup(context) {
context.ui.slot("home.bottom", () => <VoiceStatus context={context} />)
context.ui.slot("sidebar.content", () => <VoiceStatus context={context} />)
},
})
-8
View File
@@ -1,8 +0,0 @@
export type PromptHandle = {
readonly sessionID: string
readonly promptID: string
}
export function promptKey(handle: PromptHandle) {
return `${handle.sessionID}:${handle.promptID}`
}
-486
View File
@@ -1,486 +0,0 @@
import type {
VoiceConnection,
VoiceDelegationRequest,
VoiceNotificationRequest,
VoiceProtocol,
VoiceProtocolEvent,
VoiceProtocolOptions,
} from "./protocol"
import { decodeVoiceToolInput } from "./protocol"
import { Option, Schema } from "effect"
const LiveItem = Schema.Struct({
id: Schema.String,
type: Schema.String,
text: Schema.optional(Schema.String),
name: Schema.optional(Schema.String),
call_id: Schema.optional(Schema.String),
arguments: Schema.optional(Schema.String),
target: Schema.optional(Schema.String),
content: Schema.optional(Schema.Array(Schema.Struct({ type: Schema.String, text: Schema.optional(Schema.String) }))),
})
const LiveTurn = Schema.Struct({
id: Schema.String,
role: Schema.Literals(["user", "assistant"]),
transcript: Schema.String,
})
const LiveEvent = Schema.Struct({
type: Schema.String,
event_id: Schema.optional(Schema.String),
audio: Schema.optional(Schema.String),
delta: Schema.optional(Schema.String),
start_ms: Schema.optional(Schema.Number),
end_ms: Schema.optional(Schema.Number),
turn_id: Schema.optional(Schema.String),
turn: Schema.optional(LiveTurn),
item: Schema.optional(LiveItem),
error: Schema.optional(
Schema.Struct({
code: Schema.optional(Schema.String),
message: Schema.optional(Schema.String),
event_id: Schema.optional(Schema.String),
}),
),
})
const decodeLiveEvent = Schema.decodeUnknownOption(Schema.fromJsonString(LiveEvent))
type LiveTurn = Schema.Schema.Type<typeof LiveTurn>
type LiveEvent = Schema.Schema.Type<typeof LiveEvent>
type LiveNotificationResult = {
readonly type?: string
readonly eventID?: string
readonly errorEventID?: string
}
export function liveNotificationAcknowledgement(
result: LiveNotificationResult,
pending: {
readonly eventID: string
readonly acknowledgement: "session.context.appended" | "delegation.context.appended"
},
) {
// Live success events do not reliably echo the client event ID. Context
// appends are serialized, so the expected acknowledgement kind is the
// strongest correlation the protocol currently provides.
if (result.type === pending.acknowledgement) return true
if (result.type === "error" && result.errorEventID === pending.eventID) return false
return undefined
}
type ProjectedTurn = LiveTurn & { readonly displayID: string }
export function createLiveEventProjector() {
const turns = new Map<string, ProjectedTurn>()
let input: { readonly id: string; readonly transcript: string } | undefined
let assistantTranscript = ""
const userTranscripts = new Map<string, string>()
const startedTools = new Set<string>()
const startTool = (item: LiveEvent["item"], events: VoiceProtocolEvent[]) => {
if (item?.type !== "function_call" || !item.name || !item.call_id || startedTools.has(item.call_id)) return
startedTools.add(item.call_id)
events.push({ type: "tool.started", id: item.call_id, name: item.name })
}
const syncUser = (id: string, text: string, final: boolean, events: VoiceProtocolEvent[]) => {
text = text.trimStart()
const previous = userTranscripts.get(id)
if (previous === text && !final) return
if (final) userTranscripts.delete(id)
else userTranscripts.set(id, text)
events.push({ type: "user.transcript", id, text, final })
}
// Deltas are emitted verbatim, matching the Realtime adapter: a fragment's leading space is
// the only record of the boundary between two assistant turns, so trimming it here would
// weld the last word of one turn onto the first word of the next. Stripping the leading
// edge of a rendered message is the consumer's job.
const syncAssistant = (transcript: string, events: VoiceProtocolEvent[]) => {
if (transcript.startsWith(assistantTranscript)) {
const delta = transcript.slice(assistantTranscript.length)
assistantTranscript = transcript
if (delta) events.push({ type: "assistant.transcript.delta", delta })
return
}
if (assistantTranscript.startsWith(transcript)) {
assistantTranscript = transcript
events.push({ type: "assistant.transcript", text: transcript })
return
}
assistantTranscript = transcript
events.push({ type: "assistant.transcript", text: transcript })
}
return (message: string) => {
const data = Option.getOrUndefined(decodeLiveEvent(message))
if (!data)
return {
type: undefined,
events: [{ type: "error", message: "Received an invalid Live API event." }] satisfies VoiceProtocolEvent[],
}
const events: VoiceProtocolEvent[] = data.type.endsWith(".delta") ? [] : [{ type: "debug", message: data.type }]
switch (data.type) {
case "session.started":
events.push({ type: "ready" })
break
case "output_audio.delta":
if (data.audio && data.start_ms !== undefined && data.end_ms !== undefined)
events.push({
type: "assistant.audio",
audio: Buffer.from(data.audio, "base64"),
timeline: { startMs: data.start_ms, endMs: data.end_ms },
})
break
case "input_transcript.added":
if (data.item?.type !== "input_transcript" || data.item.text === undefined) break
if (!input) {
input = { id: data.item.id, transcript: "" }
events.push({ type: "user.committed", id: input.id })
}
input = { ...input, transcript: input.transcript + data.item.text }
syncUser(input.id, input.transcript, false, events)
break
case "output_transcript.added":
if (data.item?.type !== "output_transcript" || data.item.text === undefined) break
const delta = data.item.text
assistantTranscript += delta
if (delta) events.push({ type: "assistant.transcript.delta", delta })
break
case "turn.created": {
const turn = data.turn
if (!turn) break
const displayID = turn.role === "user" ? (input?.id ?? turn.id) : turn.id
turns.set(turn.id, { ...turn, displayID })
if (turn.role === "user") {
if (!input) events.push({ type: "user.committed", id: displayID })
syncUser(displayID, turn.transcript, false, events)
break
}
syncAssistant(turn.transcript, events)
break
}
case "turn.delta": {
if (!data.turn_id || !data.delta) break
const turn = turns.get(data.turn_id)
if (!turn) break
const transcript = turn.transcript + data.delta
turns.set(data.turn_id, { ...turn, transcript })
if (turn.role === "user") {
syncUser(turn.displayID, transcript, false, events)
break
}
syncAssistant(transcript, events)
break
}
case "turn.done": {
const turn = data.turn
if (!turn) break
const previous = turns.get(turn.id)
if (turn.role === "user") {
const displayID = previous?.displayID ?? input?.id ?? turn.id
if (!previous && !input) events.push({ type: "user.committed", id: displayID })
syncUser(displayID, turn.transcript, true, events)
input = undefined
turns.delete(turn.id)
break
}
syncAssistant(turn.transcript, events)
assistantTranscript = ""
turns.delete(turn.id)
events.push({ type: "assistant.done", awaitingWork: false })
break
}
case "response.output_item.added":
startTool(data.item, events)
break
case "delegation.created":
if (data.item?.type !== "delegation" || data.item.target !== "client") break
events.push({
type: "delegation.requested",
request: {
id: data.item.id,
text:
data.item.content
?.flatMap((part) => (part.type === "input_text" && part.text ? [part.text] : []))
.join("\n") ?? "",
},
})
break
case "response.output_item.done": {
if (data.item?.type !== "function_call" || !data.item.call_id) break
const name = data.item.name ?? "tool"
startTool(data.item, events)
if (!data.item.name || data.item.arguments === undefined) {
const output = { status: "error", message: "Malformed Live function call." }
events.push({
type: "work.rejected",
request: { id: data.item.call_id, name, input: {} },
output,
})
startedTools.delete(data.item.call_id)
break
}
const input = Option.getOrUndefined(decodeVoiceToolInput(data.item.arguments))
if (!input) {
const output = { status: "error", message: `Invalid arguments for tool ${data.item.name}.` }
events.push({ type: "error", message: `Received invalid arguments for Live tool ${data.item.name}.` })
events.push({
type: "work.rejected",
request: { id: data.item.call_id, name, input: {} },
output,
})
startedTools.delete(data.item.call_id)
break
}
events.push({
type: "work.requested",
request: { id: data.item.call_id, name: data.item.name, input },
})
startedTools.delete(data.item.call_id)
break
}
case "error":
events.push({ type: "error", message: `${data.error?.code}: ${data.error?.message}` })
}
return {
type: data.type,
eventID: data.event_id,
errorEventID: data.error?.event_id,
events,
}
}
}
export function createLiveProtocol(): VoiceProtocol {
return {
name: "live",
inputActivity: "local",
capabilities: { textInput: false, interruption: false, delegation: true },
connect: connectLive,
}
}
export function createLiveEventDelivery(options: {
readonly emit: (event: VoiceProtocolEvent) => void
readonly drainMs?: number
}) {
let pendingDone: Extract<VoiceProtocolEvent, { readonly type: "assistant.done" }> | undefined
let timer: ReturnType<typeof setTimeout> | undefined
const schedule = () => {
if (timer) clearTimeout(timer)
timer = setTimeout(() => {
timer = undefined
const event = pendingDone
pendingDone = undefined
if (event) options.emit(event)
}, options.drainMs ?? 120)
}
return {
push(event: VoiceProtocolEvent) {
if (event.type === "assistant.done") {
pendingDone = event
schedule()
return
}
if (event.type === "assistant.audio" && pendingDone) schedule()
options.emit(event)
},
close() {
if (timer) clearTimeout(timer)
timer = undefined
pendingDone = undefined
},
}
}
export function createLiveContextAppendQueue(options: {
readonly send: (event: Record<string, unknown>) => boolean
readonly timeoutMs?: number
}) {
type Item = {
readonly event: Record<string, unknown>
readonly acknowledgement: "session.context.appended" | "delegation.context.appended"
readonly result: PromiseWithResolvers<boolean>
}
const queued: Item[] = []
let active: (Item & { readonly timer: ReturnType<typeof setTimeout> }) | undefined
let closed = false
const settle = (accepted: boolean) => {
if (!active) return
clearTimeout(active.timer)
active.result.resolve(accepted)
active = undefined
pump()
}
const pump = () => {
if (closed || active) return
const item = queued.shift()
if (!item) return
active = {
...item,
timer: setTimeout(() => settle(false), options.timeoutMs ?? 5_000),
}
if (!options.send(item.event)) settle(false)
}
return {
append(
event: Record<string, unknown>,
acknowledgement: "session.context.appended" | "delegation.context.appended",
) {
if (closed) return Promise.resolve(false)
const result = Promise.withResolvers<boolean>()
queued.push({ event, acknowledgement, result })
pump()
return result.promise
},
receive(result: LiveNotificationResult) {
if (!active) return
const eventID = active.event["event_id"]
const accepted = liveNotificationAcknowledgement(result, {
eventID: typeof eventID === "string" ? eventID : "",
acknowledgement: active.acknowledgement,
})
if (accepted !== undefined) settle(accepted)
},
close() {
closed = true
if (active) {
clearTimeout(active.timer)
active.result.resolve(false)
active = undefined
}
queued.splice(0).forEach((item) => item.result.resolve(false))
},
}
}
function connectLive(options: VoiceProtocolOptions): VoiceConnection {
// oxlint-disable-next-line typescript-eslint/no-unsafe-type-assertion -- Bun accepts WebSocket headers in this runtime-only overload.
const ws = new WebSocket(`wss://api.openai.com/v1/live?model=${options.model}`, {
headers: {
Authorization: `Bearer ${options.apiKey}`,
"OpenAI-Alpha": "quicksilver=v2",
},
} as unknown as string[])
const closed = Promise.withResolvers<void>()
let closeTimer: ReturnType<typeof setTimeout> | undefined
const project = createLiveEventProjector()
const delivery = createLiveEventDelivery({ emit: options.onEvent })
const send = (event: Record<string, unknown>) => {
if (options.debug || event["type"] !== "input_audio.append")
options.trace?.("live.send", {
type: event["type"],
delegationID: event["delegation_item_id"],
})
if (ws.readyState !== WebSocket.OPEN) return false
ws.send(JSON.stringify(event))
return true
}
const context = createLiveContextAppendQueue({ send })
const appendNotification = (request: VoiceNotificationRequest) =>
context.append(
{
type: "session.context.append",
event_id: crypto.randomUUID(),
content: [{ type: "input_text", text: notificationText(request.text) }],
},
"session.context.appended",
)
ws.addEventListener("open", () => {
send({
type: "session.update",
event_id: crypto.randomUUID(),
session: {
instructions: options.instructions,
audio: { output: { voice: options.voice } },
delegation: { type: "client" },
},
})
})
ws.addEventListener("message", (event) => {
const result = project(String(event.data))
if (options.debug || result.type !== "output_audio.delta") options.trace?.("live.receive", { type: result.type })
context.receive(result)
result.events.forEach((event) => {
if (event.type === "assistant.audio")
options.trace?.("live.audio.received", {
bytes: event.audio.length,
startMs: event.timeline?.startMs,
endMs: event.timeline?.endMs,
})
delivery.push(event)
})
if (result.type === "session.closed") ws.close(1000)
})
ws.addEventListener("close", (event) => {
if (closeTimer) clearTimeout(closeTimer)
context.close()
delivery.close()
options.onEvent({ type: "closed", code: event.code })
closed.resolve()
})
return {
appendAudio(audio) {
if (ws.bufferedAmount > 96_000) {
options.trace?.("live.audio.dropped", { bytes: audio.length, buffered: ws.bufferedAmount })
return
}
send({ type: "input_audio.append", audio: audio.toString("base64") })
},
resolveWork(request, output) {
send({
type: "delegation.function_call_output.create",
event_id: crypto.randomUUID(),
item: { type: "function_call_output", call_id: request.id, output: JSON.stringify(output) },
})
},
resolveDelegation(request: VoiceDelegationRequest, output: string) {
void context
.append(delegationContext(crypto.randomUUID(), request.id, output), "delegation.context.appended")
.then((accepted) => {
if (!accepted) options.trace?.("live.delegation.context.failed", { delegationID: request.id })
})
},
notify(request) {
return appendNotification(request)
},
close(closeOptions) {
if (ws.readyState === WebSocket.CLOSED) return Promise.resolve()
if (closeOptions?.graceful === false || ws.readyState !== WebSocket.OPEN) {
ws.close(1000)
closeTimer = setTimeout(() => closed.resolve(), 5_000)
return closed.promise
}
send({ type: "session.close", event_id: crypto.randomUUID() })
closeTimer = setTimeout(() => {
ws.close(1000)
closed.resolve()
}, 10_500)
return closed.promise
},
}
}
function delegationContext(eventID: string, delegationID: string, text: string) {
return {
type: "delegation.context.append",
event_id: eventID,
delegation_item_id: delegationID,
content: [{ type: "input_text", text: notificationText(text) }],
}
}
function notificationText(output: unknown) {
const text = String(output)
return text.length > 1_600 ? text.slice(0, 1_600) + "..." : text
}
-318
View File
@@ -1,318 +0,0 @@
import type { VoiceConnection, VoiceProtocol, VoiceProtocolEvent, VoiceProtocolOptions } from "./protocol"
import { decodeVoiceToolInput } from "./protocol"
import { Option, Schema } from "effect"
import { createSingleFlightAcknowledgement } from "./single-flight-acknowledgement"
const RealtimeItem = Schema.Struct({
id: Schema.optional(Schema.String),
type: Schema.optional(Schema.String),
name: Schema.optional(Schema.String),
call_id: Schema.optional(Schema.String),
arguments: Schema.optional(Schema.String),
})
const RealtimeFunctionCall = Schema.Struct({
type: Schema.Literal("function_call"),
name: Schema.String,
call_id: Schema.String,
arguments: Schema.String,
})
const RealtimeEvent = Schema.Struct({
type: Schema.String,
delta: Schema.optional(Schema.String),
transcript: Schema.optional(Schema.String),
item_id: Schema.optional(Schema.String),
item: Schema.optional(RealtimeItem),
response: Schema.optional(Schema.Struct({ output: Schema.optional(Schema.Array(RealtimeItem)) })),
error: Schema.optional(
Schema.Struct({
code: Schema.optional(Schema.String),
message: Schema.optional(Schema.String),
event_id: Schema.optional(Schema.String),
}),
),
})
const decodeRealtimeEvent = Schema.decodeUnknownOption(Schema.fromJsonString(RealtimeEvent))
const decodeFunctionCall = Schema.decodeUnknownOption(RealtimeFunctionCall)
type RealtimeEvent = Schema.Schema.Type<typeof RealtimeEvent>
type RealtimeProjectorCommand = { readonly type: "response.create" }
export function createRealtimeEventProjector() {
const pendingCalls = new Set<string>()
const resolvedCalls = new Set<string>()
const startedCalls = new Set<string>()
const projectedCalls = new Set<string>()
let responseAwaitingWork = false
const resumeAfterWork = (): ReadonlyArray<RealtimeProjectorCommand> => {
if (!responseAwaitingWork || pendingCalls.size > 0) return []
responseAwaitingWork = false
resolvedCalls.clear()
return [{ type: "response.create" }]
}
return {
receive(raw: string) {
const event = Option.getOrUndefined(decodeRealtimeEvent(raw))
if (!event)
return {
type: "invalid",
events: [{ type: "error", message: "Received an invalid Realtime API event." }] satisfies VoiceProtocolEvent[],
commands: [] as ReadonlyArray<RealtimeProjectorCommand>,
}
const events: VoiceProtocolEvent[] = event.type.endsWith(".delta")
? []
: [{ type: "debug", message: event.type }]
const commands: RealtimeProjectorCommand[] = []
switch (event.type) {
case "session.created":
events.push({ type: "ready" })
break
case "response.output_text.delta":
case "response.output_audio_transcript.delta":
events.push({ type: "assistant.transcript.delta", delta: event.delta ?? "" })
break
case "response.done": {
const callIDs = (event.response?.output ?? []).flatMap((item) =>
item.type === "function_call" && item.call_id ? [item.call_id] : [],
)
callIDs.filter((id) => !resolvedCalls.has(id)).forEach((id) => pendingCalls.add(id))
responseAwaitingWork = callIDs.length > 0
events.push({ type: "assistant.done", awaitingWork: responseAwaitingWork })
commands.push(...resumeAfterWork())
break
}
case "input_audio_buffer.speech_started":
events.push({ type: "user.started" })
break
case "input_audio_buffer.speech_stopped":
events.push({ type: "user.stopped" })
break
case "input_audio_buffer.committed":
events.push({ type: "user.committed", id: event.item_id ?? "" })
break
case "conversation.item.input_audio_transcription.completed":
events.push({
type: "user.transcript",
id: event.item_id ?? "",
text: (event.transcript ?? "").trim(),
final: true,
})
break
case "response.output_audio.delta":
if (event.delta) events.push({ type: "assistant.audio", audio: Buffer.from(event.delta, "base64") })
break
case "response.output_item.added":
if (
event.item?.type === "function_call" &&
event.item.name &&
event.item.call_id &&
!startedCalls.has(event.item.call_id)
) {
startedCalls.add(event.item.call_id)
events.push({ type: "tool.started", id: event.item.call_id, name: event.item.name })
}
break
case "response.output_item.done": {
if (event.item?.type !== "function_call") break
if (event.item.call_id && projectedCalls.has(event.item.call_id)) break
if (event.item.name && event.item.call_id && !startedCalls.has(event.item.call_id)) {
startedCalls.add(event.item.call_id)
events.push({ type: "tool.started", id: event.item.call_id, name: event.item.name })
}
const call = Option.getOrUndefined(decodeFunctionCall(event.item))
if (!call) {
events.push({ type: "error", message: "Received a malformed Realtime function call." })
if (event.item.call_id)
events.push({
type: "work.rejected",
request: { id: event.item.call_id, name: event.item.name ?? "tool", input: {} },
output: { status: "error", message: "Malformed Realtime function call." },
})
if (event.item.call_id) startedCalls.delete(event.item.call_id)
if (event.item.call_id) projectedCalls.add(event.item.call_id)
break
}
pendingCalls.add(call.call_id)
const input = Option.getOrUndefined(decodeVoiceToolInput(call.arguments))
if (!input) {
events.push({ type: "error", message: `Received invalid arguments for Realtime tool ${call.name}.` })
events.push({
type: "work.rejected",
request: { id: call.call_id, name: call.name, input: {} },
output: { status: "error", message: `Invalid arguments for tool ${call.name}.` },
})
startedCalls.delete(call.call_id)
projectedCalls.add(call.call_id)
break
}
events.push({ type: "work.requested", request: { id: call.call_id, name: call.name, input } })
startedCalls.delete(call.call_id)
projectedCalls.add(call.call_id)
break
}
case "error":
events.push({ type: "error", message: `${event.error?.code}: ${event.error?.message}` })
}
return { type: event.type, event, events, commands }
},
resolveWork(id: string) {
resolvedCalls.add(id)
pendingCalls.delete(id)
return resumeAfterWork()
},
}
}
export function realtimeNotificationAcknowledgement(
event: Pick<RealtimeEvent, "type" | "item" | "error">,
pending: { readonly itemID: string; readonly eventID: string },
) {
if (event.type === "conversation.item.created" && event.item?.id === pending.itemID) return true
if (event.type === "error" && event.error?.event_id === pending.eventID) return false
return undefined
}
export function createRealtimeProtocol(): VoiceProtocol {
return {
name: "realtime",
inputActivity: "server",
capabilities: { textInput: true, interruption: true, delegation: false },
connect: connectRealtime,
}
}
export function realtimeSessionUpdate(
options: Pick<VoiceProtocolOptions, "instructions" | "tools" | "voice">,
) {
return {
type: "session.update",
session: {
type: "realtime",
instructions: options.instructions,
tools: options.tools,
tool_choice: "auto",
audio: { output: { voice: options.voice } },
},
}
}
function connectRealtime(options: VoiceProtocolOptions): VoiceConnection {
// oxlint-disable-next-line typescript-eslint/no-unsafe-type-assertion -- Bun accepts WebSocket headers in this runtime-only overload.
const ws = new WebSocket(`wss://api.openai.com/v1/realtime?model=${options.model}`, {
headers: { Authorization: `Bearer ${options.apiKey}` },
} as unknown as string[])
const closed = Promise.withResolvers<void>()
let closeTimer: ReturnType<typeof setTimeout> | undefined
const notification = createSingleFlightAcknowledgement<{ readonly itemID: string; readonly eventID: string }>()
const projector = createRealtimeEventProjector()
const send = (event: Record<string, unknown>) => {
if (options.debug || event["type"] !== "input_audio_buffer.append")
options.trace?.("realtime.send", {
type: event["type"],
callID:
event["item"] && typeof event["item"] === "object" && "call_id" in event["item"]
? event["item"].call_id
: undefined,
})
if (ws.readyState !== WebSocket.OPEN) return false
ws.send(JSON.stringify(event))
return true
}
const createResponse = (text = false) =>
send(text ? { type: "response.create", response: { output_modalities: ["text"] } } : { type: "response.create" })
const resolveFunctionCall = (id: string, output: unknown) => {
send({
type: "conversation.item.create",
item: { type: "function_call_output", call_id: id, output: JSON.stringify(output) },
})
projector.resolveWork(id).forEach(() => createResponse())
}
ws.addEventListener("open", () => {
send(realtimeSessionUpdate(options))
send({
type: "session.update",
session: { type: "realtime", audio: { input: { transcription: { model: "whisper-1" } } } },
})
send({
type: "session.update",
session: {
type: "realtime",
audio: {
input: {
turn_detection: {
type: "server_vad",
silence_duration_ms: 900,
interrupt_response: options.fullDuplex,
},
},
},
},
})
})
ws.addEventListener("message", (event) => {
const result = projector.receive(String(event.data))
if (options.debug || result.type !== "response.output_audio.delta")
options.trace?.("realtime.receive", { type: result.type })
const current = notification.current()
const acknowledgement = current && result.event
? realtimeNotificationAcknowledgement(result.event, current.correlation)
: undefined
if (acknowledgement !== undefined && current) notification.settle(current.id, acknowledgement)
result.events.forEach(options.onEvent)
result.commands.forEach(() => createResponse())
})
ws.addEventListener("close", (event) => {
if (closeTimer) clearTimeout(closeTimer)
notification.close()
options.onEvent({ type: "closed", code: event.code })
closed.resolve()
})
return {
appendAudio(audio) {
if (ws.bufferedAmount > 96_000) {
options.trace?.("realtime.audio.dropped", { bytes: audio.length, buffered: ws.bufferedAmount })
return
}
send({ type: "input_audio_buffer.append", audio: audio.toString("base64") })
},
sendText(text) {
send({
type: "conversation.item.create",
item: { type: "message", role: "user", content: [{ type: "input_text", text }] },
})
createResponse(true)
},
resolveWork(request, output) {
resolveFunctionCall(request.id, output)
},
notify(request) {
const suffix = crypto.randomUUID().replaceAll("-", "")
const itemID = `item_${suffix}`
const eventID = `event_${suffix}`
const pending = notification.begin(request.id, { itemID, eventID })
if (!pending.started) return pending.promise
const created = send({
type: "conversation.item.create",
event_id: eventID,
item: { id: itemID, type: "message", role: "user", content: [{ type: "input_text", text: request.text }] },
})
if (!created || !createResponse()) {
notification.settle(request.id, false)
return Promise.resolve(false)
}
return pending.promise
},
interrupt() {
send({ type: "response.cancel" })
},
close() {
if (ws.readyState === WebSocket.CLOSED) return Promise.resolve()
ws.close(1000)
closeTimer ??= setTimeout(() => closed.resolve(), 5_000)
return closed.promise
},
}
}
-94
View File
@@ -1,94 +0,0 @@
import { Schema } from "effect"
export const decodeVoiceToolInput = Schema.decodeUnknownOption(
Schema.fromJsonString(Schema.Record(Schema.String, Schema.Unknown)),
)
export type VoiceTool = {
readonly type: "function"
readonly name: string
readonly description: string
readonly parameters: unknown
}
export type VoiceWorkRequest = {
readonly id: string
readonly name: string
readonly input: Record<string, unknown>
}
export type VoiceToolExecution = {
readonly output: unknown
readonly admittedPrompt?: {
readonly sessionID: string
readonly promptID: string
}
}
export type VoiceDelegationRequest = {
readonly id: string
readonly text: string
}
export type VoiceNotificationRequest = {
readonly id: string
readonly text: string
readonly delegationID?: string
}
export type VoiceAudioTimeline = { readonly startMs: number; readonly endMs: number }
export type VoiceProtocolEvent =
| { readonly type: "ready" }
| { readonly type: "user.started" }
| { readonly type: "user.stopped" }
| { readonly type: "user.committed"; readonly id: string }
| { readonly type: "user.transcript"; readonly id: string; readonly text: string; readonly final: boolean }
| {
readonly type: "assistant.audio"
readonly audio: Buffer
readonly timeline?: VoiceAudioTimeline
}
| { readonly type: "assistant.transcript.delta"; readonly delta: string }
| { readonly type: "assistant.transcript"; readonly text: string }
| { readonly type: "assistant.done"; readonly awaitingWork: boolean }
| { readonly type: "tool.started"; readonly id: string; readonly name: string }
| { readonly type: "work.requested"; readonly request: VoiceWorkRequest }
| { readonly type: "work.rejected"; readonly request: VoiceWorkRequest; readonly output: unknown }
| { readonly type: "delegation.requested"; readonly request: VoiceDelegationRequest }
| { readonly type: "debug"; readonly message: string }
| { readonly type: "error"; readonly message: string }
| { readonly type: "closed"; readonly code: number }
export type VoiceProtocolOptions = {
readonly apiKey: string
readonly model: string
readonly voice: string
readonly instructions: string
readonly tools: ReadonlyArray<VoiceTool>
readonly fullDuplex: boolean
readonly debug: boolean
readonly onEvent: (event: VoiceProtocolEvent) => void
readonly trace?: (event: string, data?: Record<string, unknown>) => void
}
export type VoiceConnection = {
appendAudio(audio: Buffer): void
sendText?(text: string): void
resolveWork(request: VoiceWorkRequest, output: unknown): void
resolveDelegation?(request: VoiceDelegationRequest, output: string): void
notify(notification: VoiceNotificationRequest): Promise<boolean>
interrupt?(): void
close(options?: { readonly graceful?: boolean }): Promise<void>
}
export type VoiceProtocol = {
readonly name: "live" | "realtime"
readonly inputActivity: "local" | "server"
readonly capabilities: {
readonly textInput: boolean
readonly interruption: boolean
readonly delegation: boolean
}
connect(options: VoiceProtocolOptions): VoiceConnection
}
-167
View File
@@ -1,167 +0,0 @@
import { Option } from "effect"
import { decodeVoiceToolInput, type VoiceTool, type VoiceWorkRequest } from "./protocol"
export type ResponsesControllerContext = {
readonly sessionIDs: Set<string>
lastSessionID?: string
}
export function createResponsesControllerContext(): ResponsesControllerContext {
return { sessionIDs: new Set() }
}
export function responsesControllerTools(
tools: ReadonlyArray<VoiceTool>,
text: string,
context: ResponsesControllerContext,
) {
if (!context.lastSessionID || /\b(new|another|separate|fresh)\s+(session|thread)\b/i.test(text)) return tools
return tools.filter((tool) => tool.name !== "start_session")
}
export async function runResponsesController(options: {
apiKey: string
model: string
instructions: string
text: string
tools: ReadonlyArray<VoiceTool>
execute: (call: VoiceWorkRequest) => Promise<unknown>
context?: ResponsesControllerContext
trace?: (event: string, data?: Record<string, unknown>) => void
fetch?: typeof fetch
}) {
const request = options.fetch ?? fetch
const context = options.context ?? createResponsesControllerContext()
const cache = new Map<string, Promise<unknown>>()
let input: unknown = controllerInput(options.text, context)
let previousResponseID: string | undefined
for (let step = 0; step < 20; step++) {
options.trace?.("responses.controller.requested", { step, continued: previousResponseID !== undefined })
const response = await request("https://api.openai.com/v1/responses", {
method: "POST",
headers: { Authorization: `Bearer ${options.apiKey}`, "Content-Type": "application/json" },
body: JSON.stringify({
model: options.model,
instructions: options.instructions,
input,
previous_response_id: previousResponseID,
tools: options.tools,
store: true,
}),
})
if (!response.ok) throw new Error(`Responses controller failed (${response.status}): ${await response.text()}`)
const value = requireResponse(await response.json())
const calls = value.output.flatMap((item) => {
if (
item.type !== "function_call" ||
typeof item.call_id !== "string" ||
typeof item.name !== "string" ||
typeof item.arguments !== "string"
)
return []
const parsed = Option.getOrUndefined(decodeVoiceToolInput(item.arguments))
if (!parsed) throw new Error(`Responses controller returned invalid arguments for ${item.name}.`)
return [{ id: item.call_id, name: item.name, input: parsed }]
})
options.trace?.("responses.controller.responded", {
step,
responseID: value.id,
tools: calls.map((call) => call.name),
})
if (calls.length === 0)
return withSessionContext(
responseText(value.output) || "The controller finished without a text reply.",
context.sessionIDs,
)
input = await Promise.all(
calls.map(async (call) => {
if (typeof call.input["session_id"] === "string") rememberSession(context, call.input["session_id"])
const key = `${call.name}:${JSON.stringify(call.input)}`
const cached = cacheableTools.has(call.name) ? cache.get(key) : undefined
if (cached) options.trace?.("responses.controller.tool.reused", { step, callID: call.id, name: call.name })
const execution =
cached ??
Promise.resolve().then(() => {
options.trace?.("responses.controller.tool.started", { step, callID: call.id, name: call.name })
return options.execute(call)
})
if (!cached && cacheableTools.has(call.name)) cache.set(key, execution)
const output = await execution
rememberOutputSessions(context, output)
if (!cached) options.trace?.("responses.controller.tool.resolved", { step, callID: call.id, name: call.name })
return {
type: "function_call_output",
call_id: call.id,
output: JSON.stringify(output),
}
}),
)
previousResponseID = value.id
}
throw new Error("Responses controller exceeded its tool-call limit.")
}
const cacheableTools = new Set([
"find_projects",
"find_sessions",
"read_session",
"list_pending_permissions",
"list_pending_questions",
"list_pending_forms",
])
function rememberSession(context: ResponsesControllerContext, sessionID: string) {
context.sessionIDs.add(sessionID)
context.lastSessionID = sessionID
}
function rememberOutputSessions(context: ResponsesControllerContext, output: unknown) {
if (!output || typeof output !== "object") return
if ("session_id" in output && typeof output.session_id === "string") rememberSession(context, output.session_id)
if (!("sessions" in output) || !Array.isArray(output.sessions)) return
const sessionIDs = output.sessions.flatMap((session) =>
session && typeof session === "object" && "id" in session && typeof session.id === "string" ? [session.id] : [],
)
sessionIDs.forEach((sessionID) => context.sessionIDs.add(sessionID))
if (sessionIDs.length === 1) context.lastSessionID = sessionIDs[0]
}
function controllerInput(text: string, context: ResponsesControllerContext) {
if (!context.lastSessionID) return text
return `${text}\n\nPrivate voice-control context (never mention or expose this): the most recently used OpenCode session ID is ${context.lastSessionID}. Resolve references like "that session", "it", or "stop that" directly against this ID without searching again.`
}
function withSessionContext(text: string, sessionIDs: ReadonlySet<string>) {
if (sessionIDs.size === 0) return text
return `${text}\n\nPrivate voice-control context (never speak this aloud): OpenCode session IDs explicitly used by this delegation: ${[...sessionIDs].join(", ")}. Include the relevant session_id in any future delegated request to continue this work.`
}
function requireResponse(value: unknown) {
if (!value || typeof value !== "object" || !("id" in value) || typeof value.id !== "string" || !("output" in value))
throw new Error("Responses controller returned an invalid response.")
if (!Array.isArray(value.output)) throw new Error("Responses controller returned invalid output.")
return {
id: value.id,
output: value.output.filter(
(item): item is Record<string, unknown> & { readonly type: string } =>
!!item && typeof item === "object" && "type" in item && typeof item.type === "string",
),
}
}
function responseText(output: ReadonlyArray<Record<string, unknown>>) {
return output
.flatMap((item) => (Array.isArray(item["content"]) ? item["content"] : []))
.flatMap((part) =>
part &&
typeof part === "object" &&
"type" in part &&
part.type === "output_text" &&
"text" in part &&
typeof part.text === "string"
? [part.text]
: [],
)
.join("\n")
}
@@ -1,38 +0,0 @@
export function createSingleFlightAcknowledgement<Correlation>(timeoutMs = 5_000) {
let pending:
| {
readonly id: string
readonly correlation: Correlation
readonly result: PromiseWithResolvers<boolean>
readonly timer: ReturnType<typeof setTimeout>
}
| undefined
const settle = (id: string, accepted: boolean) => {
if (pending?.id !== id) return
clearTimeout(pending.timer)
pending.result.resolve(accepted)
pending = undefined
}
return {
begin(id: string, correlation: Correlation) {
if (pending) return { started: false, promise: Promise.resolve(false) } as const
const result = Promise.withResolvers<boolean>()
pending = {
id,
correlation,
result,
timer: setTimeout(() => settle(id, false), timeoutMs),
}
return { started: true, promise: result.promise } as const
},
current() {
return pending ? { id: pending.id, correlation: pending.correlation } : undefined
},
settle,
close() {
if (pending) settle(pending.id, false)
},
}
}
-273
View File
@@ -1,273 +0,0 @@
#!/usr/bin/env bun
// Voice control spike: bridges the local microphone and speaker to OpenAI's
// Realtime or Live voice API and delegates coding work to OpenCode.
//
// Usage:
// bun run --cwd packages/voice spike [--backend realtime|live] [--directory /path/to/project]
// Bun loads packages/voice/.env automatically when the command runs from that package.
//
// Requires sox (`brew install sox`) for mic capture (`rec`) and playback (`play`).
import { parseArgs } from "node:util"
import { OpenCode } from "@opencode-ai/client/promise"
import { Service } from "@opencode-ai/client/service"
import { createAudioSession, type AudioSession } from "./audio-session"
import type { OpenCodeAnnouncement } from "./opencode-notification"
import { createVoiceTrace } from "./trace"
import {
createResponsesControllerContext,
responsesControllerTools,
runResponsesController,
} from "./responses-controller"
import { createVoiceSession, voiceControlTool, type VoiceSession } from "./voice-session"
const args = parseArgs({
options: {
backend: { type: "string", default: "realtime" },
server: { type: "string" },
password: { type: "string" },
directory: { type: "string", default: process.cwd() },
model: { type: "string" },
"delegation-model": { type: "string", default: "gpt-5.5" },
voice: { type: "string", default: "marin" },
provider: { type: "string", default: "openai" },
"coding-model": { type: "string", default: "gpt-5.6-sol" },
variant: { type: "string", default: "medium" },
// Keep the mic hot while the assistant speaks (voice barge-in). Only
// usable with headphones: on speakers the mic hears the assistant and
// interrupts it with its own echo. Default is half-duplex gating.
duplex: { type: "boolean", default: false },
// Enable Apple voice processing (echo cancellation) in the audio helper.
// Needed for full duplex on speakers; harmful with Bluetooth headsets,
// where it can bind the wrong capture device.
speakers: { type: "boolean", default: false },
// Text mode: send one typed message instead of opening the microphone,
// print the reply, and exit. Useful for smoke-testing the tool loop.
text: { type: "string" },
// Log every protocol event type as it arrives.
debug: { type: "boolean", default: false },
"reduce-motion": { type: "boolean", default: false },
},
}).values
let audioSession: AudioSession | undefined
let voiceSession: VoiceSession | undefined
if (args.backend !== "realtime" && args.backend !== "live") {
console.error("--backend must be realtime or live")
process.exit(1)
}
const protocol =
args.backend === "live"
? (await import("./protocol-live")).createLiveProtocol()
: (await import("./protocol-realtime")).createRealtimeProtocol()
const model = args.model ?? (protocol.name === "live" ? "gpt-live-1-boulder-alpha" : "gpt-realtime-2.1")
if (args.text && !protocol.capabilities.textInput) {
console.error(`--text is not supported by the ${protocol.name} backend`)
process.exit(1)
}
const apiKey = (() => {
const value = process.env["OPENAI_API_KEY"]
if (value) return value
console.error("OPENAI_API_KEY is required. Add it to the gitignored packages/voice/.env or export it in the shell.")
process.exit(1)
return ""
})()
if (!args.text && !process.stdout.isTTY) {
console.error("The voice TUI requires direct terminal access for screen rendering and keyboard input.")
console.error("1Password output masking pipes stdout, so `2password run` cannot host this interactive TUI.")
console.error("Run directly: bun run --cwd packages/voice spike --backend " + protocol.name)
process.exit(1)
}
const serverPassword = args.password ?? process.env["OPENCODE_PASSWORD"] ?? process.env["OPENCODE_SERVER_PASSWORD"]
const endpoint = args.server
? {
url: args.server,
auth: serverPassword ? { type: "basic" as const, username: "opencode", password: serverPassword } : undefined,
}
: ((await Service.discover()) ?? (await Service.ensure({ command: ["opencode2", "serve", "--service"] })))
const client = OpenCode.make({
baseUrl: endpoint.url,
headers: Service.headers(endpoint),
})
const health = await client.health.get().catch((error) => {
console.error(`Could not reach the OpenCode 2 server at ${endpoint.url}: ${error}`)
process.exit(1)
})
const trace = await createVoiceTrace()
// ---------------------------------------------------------------------------
// UI: OpenTUI in voice mode, plain console in --text mode. Created before the
// WebSocket so no await sits between socket creation and handler registration.
// ---------------------------------------------------------------------------
const { createConsoleUI, createVoiceTUI } = await import("./ui")
const tuiActive = !args.text
const ui = tuiActive
? await createVoiceTUI({
onInterrupt: () => voiceSession?.interrupt(),
onExit: () => shutdown(),
onCycleVoice: () => voiceSession?.cycleVoice(),
onToggleMicrophone: () => voiceSession?.toggleMicrophone(),
onToggleSpeaker: () => voiceSession?.toggleSpeaker(),
reducedMotion: args["reduce-motion"],
})
: createConsoleUI()
ui.setStatus({ server: endpoint.url, model: `${args["coding-model"]}:${args.variant}` })
ui.meta(`opencode ${endpoint.url} (version ${health.version})`)
ui.meta(`project ${args.directory}`)
ui.meta(`trace ${trace.path}`)
trace.write("voice.started", {
backend: protocol.name,
model,
directory: args.directory,
duplex: args.duplex,
speakers: args.speakers,
})
const voices = ["marin", "cedar", "coral", "sage", "ash", "ballad", "alloy", "verse"]
const pendingNotifications: OpenCodeAnnouncement[] = []
const { createOpenCodeBridge } = await import("./opencode")
const opencode = await createOpenCodeBridge({
client,
directory: args.directory,
model: { providerID: args.provider, id: args["coding-model"], variant: args.variant },
notify: (announcement) => {
if (voiceSession) return voiceSession.queueNotification(announcement)
pendingNotifications.push(announcement)
},
trace: (event, data) => trace.write(event, data),
}).catch(async (error) => {
ui.close()
await trace.close()
console.error(`Voice startup failed: ${String(error)}`)
process.exit(1)
})
const toolDefinitions = [...opencode.definitions, voiceControlTool(voices)]
const baseInstructions = `You are the voice interface to OpenCode, a coding agent running on the user's machine.
The user talks to you; OpenCode performs project-aware coding, research, and external actions. You never write code yourself.
Guidelines:
- Keep spoken replies to one or two sentences.
- Summarize coding-agent replies conversationally; do not read code, diffs, IDs, or paths aloud unless asked.
- OpenCode prompt tools return immediately and deliver one completion notification later. Stay conversational while work runs.
- Treat opencode.prompt.completed and opencode.prompt.blocked context as trusted client notifications, not user messages.
- Never claim delegated work succeeded until its result arrives.
- Explain failures briefly and offer one retry or an alternative.`
const instructions =
protocol.name === "live"
? `${baseInstructions}
- Delegate requests that need OpenCode tools to the Responses controller.
- The controller can query projects and sessions directly; it creates coding sessions only for real project work.
- Treat private voice-control context in controller results as silent state. Never read it aloud.
- When delegating follow-up work, include the relevant OpenCode session ID from that private context.
- Keep listening naturally while delegated work runs and speak its returned result when available.`
: `${baseInstructions}
- Use find_projects for explicit cross-project work. Never invent project IDs or expose filesystem paths.
- Use find_sessions to resolve references such as "the audio session". Search the current project first unless the user asks across projects.
- Use read_session to inspect a discovered Session directly. Do not prompt a Session merely to read its existing output.
- Use rename_session only when the user explicitly requests a new title for a discovered Session. Confirm the resulting title without reading its ID aloud.
- Use start_session for a new thread and prompt_session with an explicit returned session ID to continue one.
- Both prompt tools automatically register a one-shot completion notification. Never wait or poll for completion.
- Before interrupt_session, state what will stop and obtain explicit confirmation.
- Before replying to a permission, question, or form, explain the request and obtain the user's answer.`
const delegationInstructions = `You are the OpenCode controller behind a live voice assistant.
Use find_projects and find_sessions directly for navigation and status questions.
Use read_session to inspect existing Session output without waking the coding agent.
Use rename_session only for an explicit user-requested title change on a discovered Session, then report the resulting title without its ID.
Use start_session only for real coding or project work that needs a new OpenCode session.
Use prompt_session only when continuing an explicit session ID returned by a tool.
Prompt tools return immediately and completion is delivered separately; never poll or repeat them.
Return concise factual text for the live assistant to summarize.`
const controllerContext = createResponsesControllerContext()
let controllerQueue = Promise.resolve()
const delegate = (request: { readonly text: string }, execute: Parameters<typeof runResponsesController>[0]["execute"]) => {
const result = controllerQueue.then(() =>
runResponsesController({
apiKey,
model: args["delegation-model"],
instructions: delegationInstructions,
text: request.text,
tools: responsesControllerTools(opencode.definitions, request.text, controllerContext),
execute,
context: controllerContext,
trace: (event, data) => trace.write(event, data),
}),
)
controllerQueue = result.then(
() => undefined,
() => undefined,
)
return result
}
audioSession = args.text
? undefined
: await createAudioSession({
duplex: args.duplex,
speakers: args.speakers,
inputActivity: protocol.inputActivity,
debug: args.debug,
trace: (event, data) => trace.write(event, data),
})
const fullDuplex = audioSession?.fullDuplex ?? false
voiceSession = createVoiceSession({
protocol,
connection: {
apiKey,
model,
instructions,
tools: toolDefinitions,
fullDuplex,
debug: args.debug,
trace: (event, data) => trace.write(event, data),
},
initialVoice: args.voice ?? "marin",
voices,
text: args.text,
ui,
audio: audioSession,
tools: {
execute: opencode.execute,
acknowledge: opencode.acknowledge,
close: opencode.close,
},
delegate,
trace: (event, data) => trace.write(event, data),
onClosed: shutdown,
})
pendingNotifications.forEach((announcement) => voiceSession?.queueNotification(announcement))
let shuttingDown = false
let shutdownFinished = false
function finishShutdown() {
if (shutdownFinished) return
shutdownFinished = true
ui.close()
process.exit(0)
}
function shutdown() {
if (shuttingDown) return
shuttingDown = true
trace.write("voice.shutdown")
void Promise.race([
voiceSession?.close() ?? Promise.resolve(),
Bun.sleep(5_000).then(() => trace.write("voice.shutdown.timeout")),
])
.then(() => trace.close())
.finally(finishShutdown)
}
// A surviving process keeps the microphone hot and the OpenAI meter running,
// so every terminal-death signal must tear it down.
process.on("SIGINT", shutdown)
process.on("SIGHUP", shutdown)
process.on("SIGTERM", shutdown)
voiceSession.start()
-32
View File
@@ -1,32 +0,0 @@
import { mkdir, readdir, unlink } from "node:fs/promises"
import { tmpdir } from "node:os"
import { join } from "node:path"
export async function createVoiceTrace() {
const directory = join(tmpdir(), "opencode-voice")
await mkdir(directory, { recursive: true })
const stale = (await readdir(directory))
.filter((name) => name.endsWith(".jsonl"))
.sort()
.slice(0, -20)
await Promise.allSettled(stale.map((name) => unlink(join(directory, name))))
const path = join(directory, `${new Date().toISOString().replaceAll(":", "-")}-${process.pid}.jsonl`)
const writer = Bun.file(path).writer()
const flush = setInterval(() => void writer.flush(), 250)
let bytes = 0
flush.unref()
return {
path,
write(event: string, data: Record<string, unknown> = {}) {
if (bytes >= 10_000_000) return
const line = `${JSON.stringify({ time: Date.now(), event, ...data })}\n`
bytes += Buffer.byteLength(line)
void writer.write(line)
},
async close() {
clearInterval(flush)
await writer.end()
},
}
}
-294
View File
@@ -1,294 +0,0 @@
import { REVEAL_WORD_LIMIT, scheduleTextReveal, type TextReveal } from "./animation"
export type Message =
| {
readonly key: string
readonly kind: "user"
readonly itemID: string
readonly text?: string
readonly transcribing: boolean
readonly reveals: ReadonlyArray<TextReveal>
}
| {
readonly key: string
readonly kind: "assistant"
readonly text: string
readonly streaming: boolean
readonly reveals: ReadonlyArray<TextReveal>
}
| {
readonly key: string
readonly kind: "tool"
readonly callID: string
readonly name: string
readonly input: unknown
readonly output?: unknown
}
| { readonly key: string; readonly kind: "meta"; readonly text: string }
type NewMessage = Message extends infer Item ? (Item extends Message ? Omit<Item, "key"> : never) : never
export type VoiceViewState = {
readonly messages: ReadonlyArray<Message>
readonly messageSequence: number
readonly activeUserID?: string
readonly userSequence: number
readonly revealAt: number
readonly revealAnimationEndsAt: number
}
export type VoiceViewEvent =
| { readonly type: "meta"; readonly text: string }
| { readonly type: "user.started" }
| { readonly type: "user.reset" }
| { readonly type: "user.committed"; readonly itemID: string }
| {
readonly type: "user.transcript"
readonly itemID: string
readonly text: string
readonly final: boolean
readonly now: number
readonly animate: boolean
}
| { readonly type: "assistant.delta"; readonly text: string; readonly now: number; readonly animate: boolean }
| { readonly type: "assistant.transcript"; readonly text: string; readonly now: number; readonly animate: boolean }
| { readonly type: "assistant.done" }
| { readonly type: "tool.started"; readonly callID: string; readonly name: string; readonly input: unknown }
| { readonly type: "tool.done"; readonly callID: string; readonly output: unknown }
| { readonly type: "reveals.completed" }
const MESSAGE_LIMIT = 200
export function initialVoiceView(): VoiceViewState {
return { messages: [], messageSequence: 0, userSequence: 0, revealAt: 0, revealAnimationEndsAt: 0 }
}
export function transitionVoiceView(state: VoiceViewState, event: VoiceViewEvent): VoiceViewState {
switch (event.type) {
case "meta":
return append(state, { kind: "meta", text: event.text })
case "user.started": {
if (state.activeUserID) return state
const itemID = `local-user-${state.userSequence + 1}`
return { ...state, activeUserID: itemID, userSequence: state.userSequence + 1 }
}
case "user.reset": {
if (!state.activeUserID) return state
return {
...state,
activeUserID: undefined,
messages: mergeAssistantRows(state.messages),
}
}
case "user.committed": {
const index = state.activeUserID
? state.messages.findIndex((message) => message.kind === "user" && message.itemID === state.activeUserID)
: -1
const current = {
...state,
activeUserID: undefined,
messages: state.messages.map((message, currentIndex) => {
if (message.kind === "assistant" && message.streaming) return { ...message, streaming: false }
if (currentIndex === index && message.kind === "user") return { ...message, itemID: event.itemID }
return message
}),
}
if (index !== -1) return current
return append(current, { kind: "user", itemID: event.itemID, transcribing: true, reveals: [] })
}
case "user.transcript": {
const index = state.messages.findIndex((message) => message.kind === "user" && message.itemID === event.itemID)
if (index === -1) {
const revealed = reveal(state, "", event.text, event.now, event.animate)
return append(revealed.state, {
kind: "user",
itemID: event.itemID,
text: event.text,
transcribing: !event.final,
reveals: revealed.reveals,
})
}
const previous = state.messages[index]
if (previous.kind !== "user") return state
const previousText = previous.text ?? ""
if (previousText === event.text && previous.transcribing === !event.final) return state
const appended = event.text.startsWith(previousText)
const revealed = reveal(
state,
appended ? previousText : "",
appended ? event.text.slice(previousText.length) : event.text,
event.now,
event.animate,
)
return {
...revealed.state,
messages: state.messages.map((message, currentIndex) =>
currentIndex === index && message.kind === "user"
? {
...message,
text: event.text,
transcribing: !event.final,
reveals: appended
? [...message.reveals, ...revealed.reveals].slice(-REVEAL_WORD_LIMIT)
: revealed.reveals,
}
: message,
),
}
}
case "assistant.delta": {
const streaming = state.messages.findLastIndex((message) => message.kind === "assistant" && message.streaming)
const lastIndex = state.messages.length - 1
const index = streaming === -1 && state.messages[lastIndex]?.kind === "assistant" ? lastIndex : streaming
const message = state.messages[index]
if (message?.kind !== "assistant") {
const text = event.text.trimStart()
const revealed = reveal(state, "", text, event.now, event.animate)
return append(
{
...revealed.state,
messages: state.messages.map((message) =>
message.kind === "assistant" && message.streaming ? { ...message, streaming: false } : message,
),
},
{ kind: "assistant", text, streaming: true, reveals: revealed.reveals },
)
}
const joined =
streaming === -1
? joinAssistantText(message.text, event.text)
: { text: message.text + event.text, previous: message.text, appended: event.text }
const revealed = reveal(state, joined.previous, joined.appended, event.now, event.animate)
return {
...revealed.state,
messages: state.messages.map((message, currentIndex) =>
currentIndex === index && message.kind === "assistant"
? {
...message,
text: joined.text,
streaming: true,
reveals: [...message.reveals, ...revealed.reveals].slice(-REVEAL_WORD_LIMIT),
}
: message,
),
}
}
case "assistant.transcript": {
const text = event.text.trimStart()
const index = state.messages.findLastIndex((message) => message.kind === "assistant" && message.streaming)
if (index === -1) {
const revealed = reveal(state, "", text, event.now, event.animate)
return append(revealed.state, {
kind: "assistant",
text,
streaming: true,
reveals: revealed.reveals,
})
}
const previous = state.messages[index]
if (previous.kind !== "assistant" || previous.text === text) return state
const appended = text.startsWith(previous.text)
const revealed = reveal(
state,
appended ? previous.text : "",
appended ? text.slice(previous.text.length) : text,
event.now,
event.animate,
)
return {
...revealed.state,
messages: state.messages.map((message, currentIndex) =>
currentIndex === index && message.kind === "assistant"
? {
...message,
text,
reveals: appended
? [...message.reveals, ...revealed.reveals].slice(-REVEAL_WORD_LIMIT)
: revealed.reveals,
}
: message,
),
}
}
case "assistant.done": {
const index = state.messages.findLastIndex((message) => message.kind === "assistant" && message.streaming)
if (index === -1) return state
return {
...state,
messages: state.messages.map((message, currentIndex) =>
currentIndex === index && message.kind === "assistant" ? { ...message, streaming: false } : message,
),
}
}
case "tool.started":
return append(state, {
kind: "tool",
callID: event.callID,
name: event.name,
input: event.input,
})
case "tool.done":
return {
...state,
messages: state.messages.map((message) =>
message.kind === "tool" && message.callID === event.callID ? { ...message, output: event.output } : message,
),
}
case "reveals.completed":
return {
...state,
revealAnimationEndsAt: 0,
messages: state.messages.map((message) =>
"reveals" in message && message.reveals.length > 0 ? { ...message, reveals: [] } : message,
),
}
}
return state
}
function append(state: VoiceViewState, message: NewMessage): VoiceViewState {
const messageSequence = state.messageSequence + 1
const next = { ...message, key: `message-${messageSequence}` }
return { ...state, messageSequence, messages: [...state.messages, next].slice(-MESSAGE_LIMIT) }
}
function reveal(state: VoiceViewState, previous: string, delta: string, now: number, animate: boolean) {
if (!animate) return { state, reveals: new Array<TextReveal>() }
const scheduled = scheduleTextReveal(previous, delta, now, state.revealAt)
return {
state: {
...state,
revealAt: scheduled.nextRevealAt,
revealAnimationEndsAt: Math.max(state.revealAnimationEndsAt, scheduled.animationEndsAt),
},
reveals: scheduled.reveals,
}
}
function mergeAssistantRows(messages: ReadonlyArray<Message>) {
return messages.reduce<Message[]>((result, message) => {
const previous = result.at(-1)
if (previous?.kind !== "assistant" || message.kind !== "assistant") return [...result, message]
return [
...result.slice(0, -1),
{
key: previous.key,
kind: "assistant",
text: joinAssistantText(previous.text, message.text).text,
streaming: previous.streaming || message.streaming,
reveals: [],
},
]
}, [])
}
export function joinAssistantText(left: string, right: string) {
if (left.trim() === "") return { text: right, previous: "", appended: right }
if (right.trim() === "") return { text: left, previous: left, appended: "" }
const head = left.trimEnd()
const tail = right.trimStart()
if ((left.slice(head.length) + right.slice(0, right.length - tail.length)).includes("\n"))
return { text: left + right, previous: left, appended: right }
const separator = /^[.,;:!?…%)\]}]/.test(tail) || /[([{]$/.test(head) ? "" : " "
return { text: head + separator + tail, previous: head + separator, appended: tail }
}
-728
View File
@@ -1,728 +0,0 @@
/** @jsxImportSource @opentui/solid */
// Terminal UI for the voice spike. The TUI keeps conversation order stable
// even though realtime events arrive out of order: a user row is inserted the
// moment the audio buffer commits (before the assistant starts replying) and
// its transcript is filled in when Whisper finishes.
import { createCliRenderer, RGBA } from "@opentui/core"
import { render, useKeyboard, useTerminalDimensions } from "@opentui/solid"
import "opentui-spinner/solid"
import { createSignal, For } from "solid-js"
import { createStore, reconcile } from "solid-js/store"
import {
connectionMeterLevels,
connectionTransitioning,
textRevealOpacity,
transcriptionPulse,
type TextReveal,
} from "./animation"
import { PCM_METER_FRAME_MS } from "./pcm"
import { initialVoiceView, transitionVoiceView, type Message, type VoiceViewEvent } from "./ui-model"
export { joinAssistantText } from "./ui-model"
export type VoiceStatus = {
server?: string
audio?: string
microphoneMuted?: boolean
speakerMuted?: boolean
voice?: string
model?: string
project?: string
}
export type VoiceUI = {
meta(text: string): void
userSpeaking(active: boolean): void
userReset(): void
userAudioLevel(level?: number): void
userCommitted(itemID: string): void
userTranscript(itemID: string, text: string, final?: boolean): void
assistantAudio(level: number, durationMs: number): void
assistantPlaybackStopped(): void
assistantDelta(text: string): void
assistantTranscript(text: string): void
assistantDone(): void
toolStart(callID: string, name: string, input: unknown): void
toolDone(callID: string, output: unknown): void
setStatus(patch: VoiceStatus): void
close(): void
}
const truncate = (text: string, max: number) => (text.length > max ? text.slice(0, max) + "…" : text)
const displayJson = (value: unknown) =>
truncate(
(() => {
try {
return (
JSON.stringify(
value,
(_, item) =>
typeof item === "string" ? truncate(item, 500) : typeof item === "bigint" ? String(item) : item,
2,
) ?? "null"
)
} catch {
return "[unserializable value]"
}
})(),
8_000,
)
function toolSummary(name: string, output?: unknown) {
if (output === undefined) return "running"
if (Array.isArray(output)) return `${output.length} result${output.length === 1 ? "" : "s"}`
if (typeof output === "string") return truncate(output, 160)
if (typeof output === "number" || typeof output === "boolean" || typeof output === "bigint") return String(output)
if (output === null || typeof output !== "object") return "completed"
if (
"status" in output &&
output.status === "error" &&
"message" in output &&
typeof output.message === "string"
)
return `error · ${truncate(output.message, 120)}`
if ("status" in output && typeof output.status === "string")
return output.status === "started" && "notification" in output && output.notification === "registered"
? "started · notification registered"
: output.status
const fields = Object.entries(output)
.filter(
([key, item]) =>
!key.endsWith("_id") && key !== "notification" && ["string", "number", "boolean"].includes(typeof item),
)
.slice(0, 2)
.map(([key, item]) => `${key}: ${String(item)}`)
return fields.join(" · ") || "completed"
}
// ---------------------------------------------------------------------------
// Console fallback (--text mode, non-TTY)
// ---------------------------------------------------------------------------
export function createConsoleUI(): VoiceUI {
const tty = process.stdout.isTTY
const dim = (text: string) => (tty ? `\x1b[2m${text}\x1b[0m` : text)
const cyan = (text: string) => (tty ? `\x1b[1;36m${text}\x1b[0m` : text)
const green = (text: string) => (tty ? `\x1b[1;32m${text}\x1b[0m` : text)
let streaming = false
const tools = new Map<string, string>()
const line = (text: string) => {
if (streaming) {
process.stdout.write("\n")
streaming = false
}
console.log(text)
}
return {
meta: (text) => line(dim(` ${text}`)),
userSpeaking: () => {},
userReset: () => {},
userAudioLevel: () => {},
userCommitted: () => {},
userTranscript: (_, text) => line(cyan("● you ") + text),
assistantAudio: () => {},
assistantPlaybackStopped: () => {},
assistantDelta: (text) => {
if (!streaming) {
process.stdout.write(green("● assistant "))
streaming = true
}
process.stdout.write(text)
},
assistantTranscript: (text) => line(green("● assistant ") + text),
assistantDone: () => {
if (streaming) process.stdout.write("\n")
streaming = false
},
toolStart: (callID, name) => {
tools.set(callID, name)
line(dim(`${name} running`))
},
toolDone: (callID, output) => {
const name = tools.get(callID) ?? "tool"
tools.delete(callID)
line(dim(`${name} ${toolSummary(name, output)}`))
},
setStatus: () => {},
close: () => {},
}
}
// ---------------------------------------------------------------------------
// OpenTUI
// ---------------------------------------------------------------------------
const theme = {
background: "#1a1b26",
surface: "#0d1018",
text: "#c0caf5",
muted: "#565f89",
you: "#7dcfff",
assistant: "#9ece6a",
key: "#7aa2f7",
string: "#9ece6a",
number: "#e0af68",
literal: "#bb9af7",
}
const SPINNER_FRAMES = ["⠋", "⠙", "⠹", "⠸", "⠼", "⠴", "⠦", "⠧", "⠇", "⠏"]
const colors = {
background: RGBA.fromHex(theme.background),
text: RGBA.fromHex(theme.text),
you: RGBA.fromHex(theme.you),
assistant: RGBA.fromHex(theme.assistant),
}
const AUDIO_LEVEL_STALE_MS = 250
const TOOL_DISPLAY_DELAY_MS = 160
const meterGlyphs = ["▁", "▂", "▃", "▄", "▅", "▆", "▇", "█"]
function fade(color: RGBA, opacity: number) {
return RGBA.fromValues(color.r, color.g, color.b, color.a * opacity)
}
function surface(color: RGBA, opacity: number) {
return RGBA.fromValues(
colors.background.r + (color.r - colors.background.r) * opacity,
colors.background.g + (color.g - colors.background.g) * opacity,
colors.background.b + (color.b - colors.background.b) * opacity,
1,
)
}
function meter(levels: ReadonlyArray<number>) {
return levels
.map((value) => meterGlyphs[Math.round(Math.max(0, Math.min(1, value)) * (meterGlyphs.length - 1))])
.join("")
}
// Tiny JSON tokenizer for syntax-highlighted tool results.
function jsonTokens(json: string) {
const tokens: Array<{ text: string; color: string }> = []
const pattern = /("(?:[^"\\]|\\.)*")(\s*:)?|(-?\d+\.?\d*(?:[eE][+-]?\d+)?)|(true|false|null)|([{}[\],:]+|\s+)/g
for (const match of json.matchAll(pattern)) {
if (match[1] !== undefined) {
tokens.push({ text: match[1], color: match[2] ? theme.key : theme.string })
if (match[2]) tokens.push({ text: match[2], color: theme.muted })
continue
}
if (match[3] !== undefined) {
tokens.push({ text: match[3], color: theme.number })
continue
}
if (match[4] !== undefined) {
tokens.push({ text: match[4], color: theme.literal })
continue
}
tokens.push({ text: match[5] ?? "", color: theme.muted })
}
return tokens
}
function RevealedText(props: {
text: string
reveals: ReadonlyArray<TextReveal>
now: () => number
opacity: () => number
}) {
return (
<>
<span style={{ fg: fade(colors.text, props.opacity()) }}>
{props.text.slice(0, props.reveals[0]?.offset ?? props.text.length)}
</span>
<For each={props.reveals}>
{(reveal, index) => (
<span style={{ fg: fade(colors.text, props.opacity() * textRevealOpacity(props.now(), reveal.at)) }}>
{props.text.slice(reveal.offset, props.reveals[index() + 1]?.offset ?? props.text.length)}
</span>
)}
</For>
</>
)
}
// Message transitions preserve unchanged object identities. A changed message
// gets a new object, so keyed <For> replaces that row and this branch reruns.
function MessageRow(props: {
message: Message
details: boolean
assistantLevel: () => number
userLevel: () => number
now: () => number
animate: boolean
microphoneMuted: boolean
previousKind?: Message["kind"]
}) {
const message = props.message
if (message.kind === "user") {
const transcribing = () => message.transcribing && !props.microphoneMuted
const pulse = () =>
!props.animate || !transcribing() ? 0 : Math.max(props.userLevel(), transcriptionPulse(props.now()) * 0.12)
return (
<box
width="100%"
flexDirection="column"
flexShrink={0}
border={["left"]}
borderColor={fade(colors.you, 0.38 + pulse() * 0.32)}
backgroundColor={surface(colors.you, 0.035 + pulse() * 0.07)}
paddingLeft={1}
paddingRight={1}
marginTop={1}
>
<text width="100%" minWidth={0} wrapMode="word">
<RevealedText
text={message.text ?? "transcribing"}
reveals={message.reveals}
now={props.now}
opacity={() => 0.74 + pulse() * 0.2}
/>
</text>
</box>
)
}
if (message.kind === "assistant") {
const pulse = () => (!props.animate || !message.streaming ? 0.5 : props.assistantLevel())
return (
<box
width="100%"
flexDirection="column"
flexShrink={0}
border={["left"]}
borderColor={fade(colors.assistant, message.streaming ? 0.58 + pulse() * 0.42 : 0.3)}
backgroundColor={surface(colors.assistant, message.streaming ? 0.055 + pulse() * 0.11 : 0.025)}
paddingLeft={1}
paddingRight={1}
marginTop={1}
>
<text width="100%" minWidth={0} wrapMode="word">
<RevealedText
text={message.text}
reveals={message.reveals}
now={props.now}
opacity={() => (message.streaming ? 1 : 0.7)}
/>
</text>
</box>
)
}
if (message.kind === "tool")
return (
<box width="100%" flexDirection="column" paddingLeft={2} marginTop={props.previousKind === "tool" ? 0 : 1}>
<box width="100%" minWidth={0} flexDirection="row">
<box width={2} flexShrink={0}>
{message.output === undefined ? (
props.animate ? (
<spinner frames={SPINNER_FRAMES} interval={80} color={theme.number} />
) : (
<text fg={theme.number}></text>
)
) : (
<text fg={theme.assistant}></text>
)}
</box>
<text flexGrow={1} minWidth={0} wrapMode="word" fg={theme.muted}>
<span style={{ fg: theme.key }}>{message.name}</span> {toolSummary(message.name, message.output)}
</text>
</box>
{props.details ? (
<text width="100%" minWidth={0} wrapMode="word" fg={theme.muted} paddingLeft={2}>
<For each={jsonTokens(displayJson({ input: message.input, output: message.output }))}>
{(token) => <span style={{ fg: token.color }}>{token.text}</span>}
</For>
</text>
) : null}
</box>
)
return (
<box width="100%" minWidth={0} flexDirection="row" paddingLeft={2}>
<text width={3} flexShrink={0} fg={theme.muted}>
·
</text>
<text flexGrow={1} minWidth={0} wrapMode="word" fg={theme.muted}>
{message.text}
</text>
</box>
)
}
export async function createVoiceTUI(options: {
onInterrupt(): void
onExit(): void
onCycleVoice(): void
onToggleMicrophone(): void
onToggleSpeaker(): void
reducedMotion?: boolean
}): Promise<VoiceUI> {
let view = initialVoiceView()
const [messageState, setMessageState] = createStore({ items: [...view.messages] })
const [status, setStatus] = createSignal<VoiceStatus>({})
const [details, setDetails] = createSignal(false)
const [animationFrame, setAnimationFrame] = createSignal(performance.now())
const animationStartedAt = performance.now()
const [userActive, setUserActive] = createSignal(false)
let animationTimer: ReturnType<typeof setInterval> | undefined
let assistantSegments: Array<{ start: number; end: number; level: number }> = []
let assistantSegmentIndex = 0
let assistantScheduledUntil = 0
let assistantLevel = 0
let userTargetLevel = 0
let userLevel = 0
let userLevelAt = 0
let connectedAt: number | undefined
const pendingTools = new Map<
string,
{
name: string
input: unknown
output?: unknown
completed: boolean
showAt: number
timer: ReturnType<typeof setTimeout>
}
>()
const pendingMetaTimers = new Set<ReturnType<typeof setTimeout>>()
const renderer = await createCliRenderer({
useMouse: true,
// Handle Ctrl-C in OpenTUI's native input parser, before Solid handlers
// and rendering work. onDestroy performs process/audio cleanup below.
exitOnCtrlC: true,
exitSignals: [],
autoFocus: false,
openConsoleOnError: false,
screenMode: "alternate-screen",
externalOutputMode: "passthrough",
consoleMode: "disabled",
onDestroy: () => {
if (animationTimer) clearInterval(animationTimer)
pendingTools.forEach((tool) => clearTimeout(tool.timer))
pendingMetaTimers.forEach(clearTimeout)
options.onExit()
},
})
const applyView = (event: VoiceViewEvent) => {
const next = transitionVoiceView(view, event)
if (next === view) return false
view = next
setMessageState("items", reconcile([...view.messages], { key: "key" }))
return true
}
const startAnimation = () => {
if (options.reducedMotion) return
if (animationTimer) return
animationTimer = setInterval(() => {
const now = performance.now()
while (assistantSegments[assistantSegmentIndex]?.end <= now) assistantSegmentIndex += 1
const segment = assistantSegments[assistantSegmentIndex]
const output = segment?.start <= now ? segment : undefined
const assistantTarget = output?.level ?? 0
const userTarget = now - userLevelAt < AUDIO_LEVEL_STALE_MS ? userTargetLevel : 0
assistantLevel += (assistantTarget - assistantLevel) * (assistantTarget > assistantLevel ? 0.5 : 0.16)
userLevel += (userTarget - userLevel) * (userTarget > userLevel ? 0.5 : 0.18)
setAnimationFrame(now)
if (view.revealAnimationEndsAt > 0 && now >= view.revealAnimationEndsAt) applyView({ type: "reveals.completed" })
const transcribing =
!status().microphoneMuted &&
messageState.items.some((message) => message.kind === "user" && message.transcribing)
if (
userActive() ||
transcribing ||
now < view.revealAnimationEndsAt ||
assistantSegmentIndex < assistantSegments.length ||
assistantLevel > 0.01 ||
userLevel > 0.01 ||
connectionTransitioning(now, connectedAt)
)
return
clearInterval(animationTimer)
animationTimer = undefined
}, PCM_METER_FRAME_MS)
}
const currentAssistantLevel = () => {
animationFrame()
return assistantLevel
}
const currentUserLevel = () => {
animationFrame()
return userLevel
}
function App() {
const dimensions = useTerminalDimensions()
useKeyboard((evt) => {
if (evt.ctrl && evt.name === "c") return
if (evt.repeated || evt.ctrl || evt.meta || evt.option || evt.shift) return
if (evt.name === "v") {
evt.preventDefault()
return options.onCycleVoice()
}
if (evt.name === "m") {
evt.preventDefault()
return options.onToggleMicrophone()
}
if (evt.name === "s") {
evt.preventDefault()
return options.onToggleSpeaker()
}
if (evt.name === "d") {
evt.preventDefault()
setDetails((current) => !current)
return
}
if (evt.name === "escape") {
evt.preventDefault()
options.onInterrupt()
}
})
const runtimeLine = () =>
[
status().audio ?? "connecting…",
status().microphoneMuted ? "mic muted" : undefined,
status().speakerMuted ? "speaker muted" : undefined,
status().voice,
status().model,
status().project,
]
.filter(Boolean)
.join(" ")
const microphoneLevel = () => {
if (status().microphoneMuted) return 0
return Math.max(userActive() ? 0.16 : 0.1, currentUserLevel())
}
const microphoneOpacity = () => (status().microphoneMuted ? 0.16 : userActive() ? 0.9 : 0.38)
const microphoneMeter = () => {
if (options.reducedMotion) return meter(connectionMeterLevels(0, microphoneLevel(), -480))
return meter(connectionMeterLevels(animationFrame(), microphoneLevel(), connectedAt, animationStartedAt))
}
return (
<box
flexDirection="column"
width={dimensions().width}
height={dimensions().height}
paddingLeft={1}
paddingRight={1}
backgroundColor={theme.background}
>
<box width="100%" flexGrow={1} minHeight={0} minWidth={0} backgroundColor={theme.background}>
<scrollbox
width="100%"
height="100%"
stickyScroll
stickyStart="bottom"
scrollbarOptions={{ visible: false }}
backgroundColor={theme.background}
>
<box
width="100%"
minWidth={0}
minHeight={Math.max(0, dimensions().height - 3)}
flexDirection="column"
flexShrink={0}
justifyContent="flex-end"
backgroundColor={theme.background}
>
<For each={messageState.items}>
{(message, index) => (
<MessageRow
message={message}
previousKind={messageState.items[index() - 1]?.kind}
details={details()}
assistantLevel={currentAssistantLevel}
userLevel={currentUserLevel}
now={animationFrame}
animate={!options.reducedMotion}
microphoneMuted={status().microphoneMuted === true}
/>
)}
</For>
</box>
</scrollbox>
</box>
<box width="100%" height={1} flexShrink={0} backgroundColor={theme.background} />
<box width="100%" height={2} flexShrink={0} backgroundColor={theme.surface} paddingLeft={1} paddingRight={1}>
<box width="100%" height={1} minWidth={0} flexDirection="row">
<text width={5} flexShrink={0} fg={fade(colors.you, microphoneOpacity())}>
<b>
{status().microphoneMuted
? "────"
: microphoneMeter()}
</b>
</text>
<text flexGrow={1} minWidth={0} wrapMode="none" truncate fg={fade(colors.text, 0.78)}>
{runtimeLine()}
</text>
</box>
<text width="100%" wrapMode="none" truncate fg={theme.muted}>
<span style={{ fg: theme.key }}>esc</span> interrupt {" "}
<span style={{ fg: theme.key }}>v</span> voice {" "}
<span style={{ fg: theme.key }}>m</span> mic {" "}
<span style={{ fg: theme.key }}>s</span> speaker {" "}
<span style={{ fg: theme.key }}>d</span> details {details() ? "on" : "off"} {" "}
<span style={{ fg: theme.key }}>ctrl+c</span> quit
</text>
</box>
</box>
)
}
await render(() => <App />, renderer)
startAnimation()
const pushMeta = (text: string) => {
if (pendingTools.size === 0) {
applyView({ type: "meta", text })
return
}
const showAt = Math.max(...[...pendingTools.values()].map((tool) => tool.showAt))
const timer = setTimeout(
() => {
pendingMetaTimers.delete(timer)
applyView({ type: "meta", text })
},
Math.max(0, showAt - performance.now()) + 1,
)
pendingMetaTimers.add(timer)
}
return {
meta: pushMeta,
userSpeaking: (active) => {
if (active) applyView({ type: "user.started" })
setUserActive(active)
if (active) startAnimation()
if (!active) userTargetLevel = 0
},
userReset: () => {
setUserActive(false)
userTargetLevel = 0
userLevel = 0
applyView({ type: "user.reset" })
},
userAudioLevel: (level) => {
if (level === undefined) {
userLevelAt = 0
return
}
userTargetLevel = Math.max(0, Math.min(1, level))
userLevelAt = performance.now()
if (options.reducedMotion) {
userLevel = userTargetLevel
setAnimationFrame(userLevelAt)
return
}
startAnimation()
},
userCommitted: (itemID) => {
setUserActive(false)
applyView({ type: "user.committed", itemID })
startAnimation()
},
userTranscript: (itemID, text, final = true) => {
applyView({
type: "user.transcript",
itemID,
text,
final,
now: performance.now(),
animate: !options.reducedMotion,
})
if (!final || view.revealAnimationEndsAt > 0) startAnimation()
},
assistantAudio: (level, durationMs) => {
if (options.reducedMotion) return
if (assistantSegmentIndex > 512) {
assistantSegments = assistantSegments.slice(assistantSegmentIndex)
assistantSegmentIndex = 0
}
const now = performance.now()
const start = Math.max(now, assistantScheduledUntil)
const end = start + durationMs
assistantSegments.push({ start, end, level: Math.max(0, Math.min(1, level)) })
assistantScheduledUntil = end
startAnimation()
},
assistantPlaybackStopped: () => {
assistantSegments = []
assistantSegmentIndex = 0
assistantScheduledUntil = 0
assistantLevel = 0
setAnimationFrame(performance.now())
},
assistantDelta: (text) => {
applyView({
type: "assistant.delta",
text,
now: performance.now(),
animate: !options.reducedMotion,
})
if (view.revealAnimationEndsAt > 0) startAnimation()
},
assistantTranscript: (text) => {
applyView({
type: "assistant.transcript",
text,
now: performance.now(),
animate: !options.reducedMotion,
})
if (view.revealAnimationEndsAt > 0) startAnimation()
},
assistantDone: () => {
assistantSegments = []
assistantSegmentIndex = 0
assistantScheduledUntil = 0
assistantLevel = 0
applyView({ type: "assistant.done" })
},
toolStart: (callID, name, input) => {
if (
pendingTools.has(callID) ||
view.messages.some((message) => message.kind === "tool" && message.callID === callID)
)
return
const pending = {
name,
input,
completed: false,
showAt: performance.now() + TOOL_DISPLAY_DELAY_MS,
timer: setTimeout(() => {
const current = pendingTools.get(callID)
if (!current) return
pendingTools.delete(callID)
applyView({
type: "tool.started",
callID,
name: current.name,
input: current.input,
})
if (current.completed) applyView({ type: "tool.done", callID, output: current.output })
}, TOOL_DISPLAY_DELAY_MS),
}
pendingTools.set(callID, pending)
},
toolDone: (callID, output) => {
const pending = pendingTools.get(callID)
if (pending) {
pending.output = output
pending.completed = true
return
}
applyView({ type: "tool.done", callID, output })
},
setStatus: (patch) => {
if (connectedAt === undefined && patch.audio !== undefined) {
connectedAt = performance.now()
startAnimation()
}
setStatus((current) => ({ ...current, ...patch }))
},
close: () => {
if (!renderer.isDestroyed) renderer.destroy()
},
}
}
-242
View File
@@ -1,242 +0,0 @@
import type { CompletionReceipt } from "./opencode-notification"
export type VoiceNotification = {
readonly id: string
readonly receipt?: CompletionReceipt
readonly promptID?: string
readonly delegationID?: string
readonly text: string
}
export type VoiceState = {
readonly connection: "connecting" | "ready" | "closed"
readonly conversation: "idle" | "waiting" | "responding"
readonly assistant: "idle" | "active" | "suppressed"
readonly userSpeaking: boolean
readonly tools: ReadonlySet<string>
readonly notifications: ReadonlyArray<VoiceNotification>
readonly delivery?: {
readonly notification: VoiceNotification
readonly phase: "sending" | "accepted" | "announcing" | "awaiting-playback"
}
readonly desiredVoice: string
readonly reconnect: "none" | "pending"
}
export type VoiceEvent =
| { readonly type: "connection.ready" }
| { readonly type: "connection.connecting" }
| { readonly type: "connection.closed" }
| { readonly type: "user.started"; readonly bargeIn?: boolean }
| { readonly type: "user.stopped" }
| { readonly type: "user.committed" }
| { readonly type: "assistant.started" }
| { readonly type: "assistant.suppressed" }
| { readonly type: "assistant.done"; readonly awaitingWork: boolean }
| { readonly type: "tool.started"; readonly id: string }
| { readonly type: "tool.finished"; readonly id: string }
| { readonly type: "notification.queued"; readonly notification: VoiceNotification }
| { readonly type: "notification.accepted"; readonly id: string }
| { readonly type: "notification.failed"; readonly id: string }
| { readonly type: "notification.announced"; readonly id: string }
| { readonly type: "notification.interrupted"; readonly id: string }
| { readonly type: "voice.selected"; readonly voice: string }
export type VoiceCommand =
| { readonly type: "notification.send"; readonly notification: VoiceNotification }
| { readonly type: "notification.delivered"; readonly notification: VoiceNotification }
| { readonly type: "assistant.finish"; readonly awaitingWork: boolean; readonly notificationID?: string }
| { readonly type: "assistant.interrupt" }
| { readonly type: "connection.reconnect" }
export type VoiceTransition = {
readonly state: VoiceState
readonly commands: ReadonlyArray<VoiceCommand>
}
export function initialVoiceState(voice: string): VoiceState {
return {
connection: "connecting",
conversation: "idle",
assistant: "idle",
userSpeaking: false,
tools: new Set(),
notifications: [],
desiredVoice: voice,
reconnect: "none",
}
}
export function transitionVoice(state: VoiceState, event: VoiceEvent): VoiceTransition {
if (event.type === "notification.announced") {
if (state.delivery?.notification.id !== event.id || state.delivery.phase !== "awaiting-playback")
return { state, commands: [] }
const notification = state.delivery.notification
const next = settle({ ...state, delivery: undefined })
return {
state: next.state,
commands: [{ type: "notification.delivered", notification }, ...next.commands],
}
}
if (event.type === "assistant.done") {
const notificationID =
state.delivery?.phase === "announcing" && state.assistant === "active" && !event.awaitingWork
? state.delivery.notification.id
: undefined
const next = settle(reduce(state, event))
return {
state: next.state,
commands: [{ type: "assistant.finish", awaitingWork: event.awaitingWork, notificationID }, ...next.commands],
}
}
if (event.type === "user.started" && event.bargeIn && state.assistant === "active") {
const next = settle(
state.delivery
? {
...state,
conversation: "idle",
assistant: "idle",
userSpeaking: true,
delivery: undefined,
notifications: [state.delivery.notification, ...state.notifications],
reconnect: "pending",
}
: { ...state, assistant: "suppressed", userSpeaking: true },
)
return { state: next.state, commands: [{ type: "assistant.interrupt" }, ...next.commands] }
}
const next = reduce(state, event)
return settle(next)
}
function reduce(state: VoiceState, event: VoiceEvent): VoiceState {
switch (event.type) {
case "connection.ready":
if (state.connection === "ready") return state
return { ...state, connection: "ready" }
case "connection.connecting":
return {
...state,
connection: "connecting",
conversation: "idle",
assistant: "idle",
userSpeaking: false,
}
case "connection.closed":
return { ...state, connection: "closed", conversation: "idle", assistant: "idle", userSpeaking: false }
case "user.started":
if (state.userSpeaking) return state
return { ...state, userSpeaking: true }
case "user.stopped":
if (!state.userSpeaking) return state
return { ...state, userSpeaking: false }
case "user.committed":
return {
...state,
userSpeaking: false,
conversation: state.assistant === "active" ? "responding" : "waiting",
assistant: state.assistant === "suppressed" ? "idle" : state.assistant,
}
case "assistant.started":
if (state.assistant === "suppressed" || state.assistant === "active") return state
return {
...state,
conversation: "responding",
assistant: "active",
delivery:
state.delivery?.phase === "accepted" ? { ...state.delivery, phase: "announcing" } : state.delivery,
}
case "assistant.suppressed":
return { ...state, assistant: "suppressed" }
case "assistant.done":
return {
...state,
conversation: event.awaitingWork ? "waiting" : "idle",
assistant: "idle",
delivery:
state.delivery?.phase === "announcing" && state.assistant === "active" && !event.awaitingWork
? { ...state.delivery, phase: "awaiting-playback" }
: state.delivery,
}
case "tool.started":
if (state.tools.has(event.id)) return state
return { ...state, conversation: "waiting", tools: new Set([...state.tools, event.id]) }
case "tool.finished": {
if (!state.tools.has(event.id)) return state
const tools = new Set(state.tools)
tools.delete(event.id)
return { ...state, tools }
}
case "notification.queued":
return { ...state, notifications: [...state.notifications, event.notification] }
case "notification.accepted":
if (state.delivery?.notification.id !== event.id || state.delivery.phase !== "sending") return state
return {
...state,
delivery: { ...state.delivery, phase: "accepted" },
}
case "notification.failed":
if (state.delivery?.notification.id !== event.id) return state
return {
...state,
connection: "ready",
conversation: "idle",
delivery: undefined,
notifications: [state.delivery.notification, ...state.notifications],
reconnect: "pending",
}
case "notification.announced":
return state
case "notification.interrupted":
if (state.delivery?.notification.id !== event.id) return state
return {
...state,
conversation: "idle",
assistant: "idle",
delivery: undefined,
notifications: [state.delivery.notification, ...state.notifications],
reconnect: "pending",
}
case "voice.selected":
if (event.voice === state.desiredVoice) return state
return { ...state, desiredVoice: event.voice, reconnect: "pending" }
}
return state
}
function settle(state: VoiceState): VoiceTransition {
if (
state.reconnect === "pending" &&
state.connection === "ready" &&
state.conversation === "idle" &&
state.delivery === undefined &&
!state.userSpeaking &&
state.tools.size === 0
)
return {
state: { ...state, connection: "connecting", reconnect: "none" },
commands: [{ type: "connection.reconnect" }],
}
if (
state.notifications.length > 0 &&
state.delivery === undefined &&
state.connection === "ready" &&
state.conversation === "idle" &&
!state.userSpeaking &&
state.tools.size === 0
) {
const notification = state.notifications[0]
return {
state: {
...state,
conversation: "waiting",
delivery: { notification, phase: "sending" },
notifications: state.notifications.slice(1),
},
commands: [{ type: "notification.send", notification }],
}
}
return { state, commands: [] }
}
-446
View File
@@ -1,446 +0,0 @@
import type { AudioEvent, AudioSession } from "./audio-session"
import { openCodeAnnouncementText, type CompletionReceipt, type OpenCodeAnnouncement } from "./opencode-notification"
import { createNotificationRouter } from "./notification-router"
import type {
VoiceConnection,
VoiceDelegationRequest,
VoiceProtocol,
VoiceProtocolEvent,
VoiceProtocolOptions,
VoiceTool,
VoiceToolExecution,
VoiceWorkRequest,
} from "./protocol"
import type { VoiceUI } from "./ui"
import { initialVoiceState, transitionVoice, type VoiceCommand, type VoiceEvent } from "./voice-coordinator"
export type VoiceSession = {
start(): void
cycleVoice(): void
interrupt(): void
toggleMicrophone(): void
toggleSpeaker(): void
queueNotification(announcement: OpenCodeAnnouncement): void
close(): Promise<void>
}
export function voiceControlTool(voices: ReadonlyArray<string>): VoiceTool {
return {
type: "function",
name: "set_voice",
description: "Change your speaking voice. Requires a brief reconnect and resets voice conversation memory.",
parameters: {
type: "object",
additionalProperties: false,
properties: { voice: { type: "string", enum: voices } },
required: ["voice"],
},
}
}
export function createVoiceSession(options: {
readonly protocol: VoiceProtocol
readonly connection: Omit<VoiceProtocolOptions, "voice" | "onEvent">
readonly initialVoice: string
readonly voices: ReadonlyArray<string>
readonly text?: string
readonly ui: VoiceUI
readonly audio?: AudioSession
readonly tools: {
readonly execute: (name: string, input: Record<string, unknown>) => Promise<VoiceToolExecution>
readonly acknowledge: (receipt: CompletionReceipt) => Promise<void>
readonly close: () => Promise<void>
}
readonly delegate: (
request: VoiceDelegationRequest,
execute: (request: VoiceWorkRequest) => Promise<unknown>,
) => Promise<string>
readonly trace?: (event: string, data?: Record<string, unknown>) => void
readonly onClosed: () => void
}): VoiceSession {
let state = initialVoiceState(options.initialVoice)
let connection: VoiceConnection | undefined
let closed = false
let closePromise: Promise<void> | undefined
let voiceTimer: ReturnType<typeof setTimeout> | undefined
const notifications = createNotificationRouter((routed) => {
options.trace?.("notification.queued", {
type: routed.announcement.notification.type,
promptID: routed.promptID,
depth: state.notifications.length + 1,
})
dispatch({
type: "notification.queued",
notification: {
id: crypto.randomUUID(),
receipt: routed.announcement.receipt,
promptID: routed.promptID,
delegationID: routed.delegationID,
text: openCodeAnnouncementText(routed.announcement.notification),
},
})
})
function dispatch(event: VoiceEvent) {
const transition = transitionVoice(state, event)
if (transition.state === state && transition.commands.length === 0) return
state = transition.state
options.trace?.("voice.transition", {
input: event.type,
connection: state.connection,
conversation: state.conversation,
assistant: state.assistant,
userSpeaking: state.userSpeaking,
tools: state.tools.size,
notifications: state.notifications.length,
commands: transition.commands.map((command) => command.type),
})
transition.commands.forEach(runCommand)
}
const setVoice = (voice: string) => {
options.ui.setStatus({ voice })
options.ui.meta(`[voice] switching to ${voice}`)
dispatch({ type: "voice.selected", voice })
}
const interruptDelivery = () => {
if (state.delivery) dispatch({ type: "notification.interrupted", id: state.delivery.notification.id })
}
const reconnect = () => {
if (closed) return
options.audio?.flushPlayback()
options.ui.assistantDone()
const previous = connection
connection = undefined
const next = () => {
if (!closed) connect()
}
if (!previous) return next()
void previous.close({ graceful: false }).finally(next)
}
function runCommand(command: VoiceCommand) {
if (command.type === "connection.reconnect") return reconnect()
if (command.type === "assistant.interrupt") {
connection?.interrupt?.()
options.audio?.flushPlayback()
options.ui.assistantDone()
return
}
if (command.type === "assistant.finish") {
if (options.text) {
options.ui.assistantDone()
if (command.notificationID) dispatch({ type: "notification.announced", id: command.notificationID })
if (!command.awaitingWork && state.tools.size === 0) options.onClosed()
return
}
void options.audio?.finishPlayback().then((outcome) => {
options.ui.assistantDone()
if (!command.notificationID) return
dispatch({
type: outcome === "played" ? "notification.announced" : "notification.interrupted",
id: command.notificationID,
})
})
return
}
if (command.type === "notification.delivered") {
options.trace?.("notification.announced", {
id: command.notification.id,
promptID: command.notification.promptID,
})
if (!command.notification.receipt) return
const receipt = command.notification.receipt
void options.tools
.acknowledge(receipt)
.catch((error) =>
options.trace?.("notification.ack.failed", { id: command.notification.id, error: String(error) }),
)
.finally(() => notifications.acknowledged(receipt))
return
}
const active = connection
if (!active) return dispatch({ type: "notification.failed", id: command.notification.id })
void active
.notify({
id: command.notification.id,
text: `Announce this OpenCode update conversationally without reading private identifiers aloud:\n${command.notification.text}`,
delegationID: command.notification.delegationID,
})
.then((accepted) => {
if (!accepted) return dispatch({ type: "notification.failed", id: command.notification.id })
dispatch({ type: "notification.accepted", id: command.notification.id })
options.trace?.("notification.context.accepted", {
id: command.notification.id,
promptID: command.notification.promptID,
remaining: state.notifications.length,
})
})
.catch((error) => {
options.trace?.("notification.context.failed", {
id: command.notification.id,
promptID: command.notification.promptID,
error: String(error),
})
dispatch({ type: "notification.failed", id: command.notification.id })
})
}
const executeTool = async (request: VoiceWorkRequest, delegationID?: string) => {
if (delegationID) notifications.beginDelegatedTool()
const execution: VoiceToolExecution = await (request.name === "set_voice"
? Promise.resolve().then(() => {
const voice = request.input["voice"]
if (typeof voice !== "string" || !options.voices.includes(voice))
return { output: toolError(`Voice must be one of: ${options.voices.join(", ")}.`) }
if (voiceTimer) clearTimeout(voiceTimer)
voiceTimer = setTimeout(() => {
voiceTimer = undefined
setVoice(voice)
}, 1_000)
return {
output: {
status: "switching",
voice,
note: `${options.protocol.name} conversation memory resets during reconnect.`,
},
}
})
: options.tools
.execute(request.name, request.input)
.catch((error) => ({ output: toolError(String(error), true) } satisfies VoiceToolExecution)))
options.ui.toolDone(request.id, execution.output)
if (delegationID) notifications.finishDelegatedTool(delegationID, execution.admittedPrompt)
return execution.output
}
const queueWork = (source: VoiceConnection, request: VoiceWorkRequest) => {
options.trace?.("work.started", { id: request.id, name: request.name })
void executeTool(request)
.then((output) => {
options.trace?.("work.resolved", { id: request.id, name: request.name })
source.resolveWork(request, output)
})
.catch((error) => options.ui.meta(`[work error] ${String(error)}`))
.finally(() => dispatch({ type: "tool.finished", id: request.id }))
}
const userSpeaking = (active: boolean) => {
if (active === state.userSpeaking) return
dispatch(
active
? { type: "user.started", bargeIn: options.audio?.fullDuplex && options.protocol.capabilities.interruption }
: { type: "user.stopped" },
)
options.ui.userSpeaking(active)
}
const onAudioEvent = (event: AudioEvent) => {
switch (event.type) {
case "input":
connection?.appendAudio(event.audio)
options.ui.userAudioLevel(event.level)
return
case "user.started":
userSpeaking(true)
return
case "user.stopped":
userSpeaking(false)
return
case "user.reset":
options.ui.userReset()
return
case "assistant.level":
options.ui.assistantAudio(event.level, event.durationMs)
return
case "status":
options.ui.setStatus({ audio: event.audio })
return
case "meta":
options.ui.meta(event.text)
}
}
const onProtocolEvent = (source: VoiceConnection, event: VoiceProtocolEvent) => {
if (source !== connection || closed) return
if (options.connection.debug || event.type !== "assistant.audio")
options.trace?.("protocol.event", {
type: event.type,
conversation: state.conversation,
assistant: state.assistant,
userSpeaking: state.userSpeaking,
pendingWork: state.tools.size,
})
switch (event.type) {
case "ready":
dispatch({ type: "connection.ready" })
options.ui.meta(
`connected to ${options.protocol.name} ${options.connection.model} (voice: ${state.desiredVoice})`,
)
options.ui.setStatus({
voice: state.desiredVoice,
microphoneMuted: options.audio?.microphoneMuted,
speakerMuted: options.audio?.speakerMuted,
})
if (!options.text) {
void options.audio?.start(onAudioEvent)
return
}
options.ui.userTranscript("typed", options.text)
dispatch({ type: "user.committed" })
source.sendText?.(options.text)
return
case "user.started":
userSpeaking(true)
return
case "user.stopped":
userSpeaking(false)
return
case "user.committed":
options.audio?.noteUserCommitted()
dispatch({ type: "user.committed" })
options.ui.userCommitted(event.id)
return
case "user.transcript":
options.audio?.noteUserTranscript(event.final)
if (event.final) {
dispatch({ type: "user.stopped" })
options.ui.userSpeaking(false)
}
options.ui.userTranscript(event.id, event.text, event.final)
return
case "assistant.audio":
if (state.assistant === "suppressed") return
dispatch({ type: "assistant.started" })
options.audio?.play(event.audio, event.timeline)
return
case "assistant.transcript.delta":
if (state.assistant === "suppressed") return
dispatch({ type: "assistant.started" })
options.ui.assistantDelta(event.delta)
return
case "assistant.transcript":
if (state.assistant === "suppressed") return
dispatch({ type: "assistant.started" })
options.ui.assistantTranscript(event.text)
return
case "assistant.done": {
dispatch({ type: "assistant.done", awaitingWork: event.awaitingWork })
return
}
case "tool.started":
dispatch({ type: "tool.started", id: event.id })
options.ui.toolStart(event.id, event.name, {})
return
case "work.requested":
queueWork(source, event.request)
return
case "work.rejected":
dispatch({ type: "tool.started", id: event.request.id })
options.ui.toolStart(event.request.id, event.request.name, event.request.input)
options.ui.toolDone(event.request.id, event.output)
source.resolveWork(event.request, event.output)
dispatch({ type: "tool.finished", id: event.request.id })
return
case "delegation.requested":
dispatch({ type: "tool.started", id: event.request.id })
void options
.delegate(event.request, (request) => {
options.ui.toolStart(request.id, request.name, request.input)
return executeTool(request, event.request.id)
})
.then((output) => {
if (source !== connection) {
options.trace?.("delegation.result.stale", { id: event.request.id })
return
}
source.resolveDelegation?.(event.request, output)
})
.catch((error) => {
options.trace?.("responses.controller.failed", { error: String(error) })
if (source !== connection) return
options.ui.toolStart(event.request.id, "opencode", { text: event.request.text })
options.ui.toolDone(event.request.id, { status: "error", message: String(error) })
source.resolveDelegation?.(event.request, `The OpenCode controller failed: ${String(error)}`)
})
.finally(() => dispatch({ type: "tool.finished", id: event.request.id }))
return
case "debug":
if (options.connection.debug) options.ui.meta(`[debug] ${event.message}`)
return
case "error":
options.ui.meta(`[${options.protocol.name} error] ${event.message}`)
return
case "closed":
dispatch({ type: "connection.closed" })
options.ui.meta(`${options.protocol.name} connection closed (${event.code})`)
options.onClosed()
}
}
const connect = () => {
let next: VoiceConnection
next = options.protocol.connect({
...options.connection,
voice: state.desiredVoice,
onEvent: (event) => onProtocolEvent(next, event),
})
dispatch({ type: "connection.connecting" })
connection = next
}
const session: VoiceSession = {
start: connect,
cycleVoice() {
setVoice(options.voices[(options.voices.indexOf(state.desiredVoice) + 1) % options.voices.length])
},
interrupt() {
if (!options.audio?.isPlaying() && state.assistant === "idle") return
if (state.assistant === "active") {
connection?.interrupt?.()
dispatch({ type: "assistant.suppressed" })
}
interruptDelivery()
options.audio?.flushPlayback()
options.ui.assistantDone()
options.ui.meta("[interrupted]")
},
toggleMicrophone() {
if (!options.audio) return
const muted = options.audio.toggleMicrophone()
userSpeaking(false)
options.ui.setStatus({ microphoneMuted: muted })
if (muted) options.ui.userReset()
options.ui.userAudioLevel(undefined)
options.ui.meta(`[microphone] ${muted ? "muted" : "live"}`)
},
toggleSpeaker() {
if (!options.audio) return
const muted = options.audio.toggleSpeaker()
options.ui.setStatus({ speakerMuted: muted })
if (muted) options.ui.assistantPlaybackStopped()
options.ui.meta(`[speaker] ${muted ? "muted" : "live"}`)
},
queueNotification(announcement) {
notifications.route(announcement)
},
close() {
if (closePromise) return closePromise
closed = true
if (voiceTimer) clearTimeout(voiceTimer)
options.audio?.close()
const active = connection
connection = undefined
dispatch({ type: "connection.closed" })
closePromise = Promise.allSettled([options.tools.close(), ...(active ? [active.close()] : [])]).then(() => {})
return closePromise
},
}
return session
}
function toolError(message: string, retryable = false) {
return { status: "error", message, retryable }
}
-53
View File
@@ -1,53 +0,0 @@
import { describe, expect, test } from "bun:test"
import {
connectionMeterLevels,
connectionTransitioning,
scheduleTextReveal,
textRevealOpacity,
transcriptionPulse,
} from "../src/animation"
describe("voice text animation", () => {
test("stages newly streamed words without delaying them indefinitely", () => {
const first = scheduleTextReveal("", "Hello there", 1_000, 0)
expect(first.reveals).toEqual([
{ offset: 0, at: 1_000 },
{ offset: 6, at: 1_032 },
])
expect(first.animationEndsAt).toBe(1_412)
const queued = scheduleTextReveal("Hello there", " friend", 1_010, 10_000)
expect(queued.reveals).toEqual([{ offset: 12, at: 1_170 }])
})
test("slowly fades text into its stable color", () => {
const opacities = [100, 195, 290, 385, 480].map((now) => textRevealOpacity(now, 100))
expect(opacities[0]).toBe(0.16)
expect(opacities.toSorted((a, b) => a - b)).toEqual(opacities)
expect(opacities.at(-1)).toBe(1)
})
test("keeps transcription pulse bounded and moving", () => {
const levels = [0, 110, 220, 330].map(transcriptionPulse)
expect(levels.every((level) => level >= 0 && level <= 1)).toBe(true)
expect(new Set(levels).size).toBeGreaterThan(1)
})
test("moves the connecting pulse left to right and blends into the connected meter", () => {
const first = connectionMeterLevels(1_000, 0.1, undefined, 1_000)
const next = connectionMeterLevels(1_180, 0.1, undefined, 1_000)
expect(first.indexOf(Math.max(...first))).toBe(0)
expect(first.indexOf(Math.max(...first))).toBeLessThan(next.indexOf(Math.max(...next)))
const samples = Array.from({ length: 16 }, (_, index) => connectionMeterLevels(index * 45, 0.1))
expect(samples.every((levels) => Math.max(...levels) > 0.85)).toBe(true)
const handoff = connectionMeterLevels(1_000, 0.4, 1_000)
expect(handoff).toEqual(connectionMeterLevels(1_000, 0.4))
expect(connectionMeterLevels(1_480, 0.4, 1_000)).toEqual(
[0, 1, 2, 3].map((index) => 0.4 * (0.72 + Math.sin(1_480 / 240 + index * 1.4) * 0.28)),
)
expect(connectionTransitioning(1_479, 1_000)).toBe(true)
expect(connectionTransitioning(1_480, 1_000)).toBe(false)
})
})
@@ -1,79 +0,0 @@
import { describe, expect, test } from "bun:test"
import { joinAssistantText } from "../src/ui-model"
describe("joinAssistantText", () => {
const cases: Array<[name: string, left: string, right: string, expected: string]> = [
[
"inserts a space between sentences",
"Okay, I'll check that.",
"The build passed.",
"Okay, I'll check that. The build passed.",
],
["inserts a space mid-sentence", "Let me look at", "the config file.", "Let me look at the config file."],
[
"keeps a single trailing space",
"Okay, I'll check that. ",
"The build passed.",
"Okay, I'll check that. The build passed.",
],
[
"keeps a single leading space",
"Okay, I'll check that.",
" The build passed.",
"Okay, I'll check that. The build passed.",
],
["collapses spaces on both sides", "Done. ", " Next.", "Done. Next."],
["collapses runs of spaces", "Done. ", "\t \tNext.", "Done. Next."],
[
"spaces after a question mark",
"Want me to run tests?",
"I can do that now.",
"Want me to run tests? I can do that now.",
],
["spaces after an ellipsis", "Thinking…", "done.", "Thinking… done."],
["preserves a trailing newline", "Done:\n", "Next up, tests.", "Done:\nNext up, tests."],
["preserves a leading newline", "Done:", "\nNext up, tests.", "Done:\nNext up, tests."],
[
"preserves a paragraph break",
"First paragraph.\n\n",
"Second paragraph.",
"First paragraph.\n\nSecond paragraph.",
],
["preserves a newline mixed with spaces", "Done: ", " \n Next.", "Done: \n Next."],
["never spaces before closing punctuation", "Done", ".", "Done."],
["never spaces before a comma", "one", ", two", "one, two"],
["never spaces before a closing paren", "(aside", ")", "(aside)"],
["never spaces after an opening paren", "an (", "aside)", "an (aside)"],
["drops an empty left side", "", "The build passed.", "The build passed."],
["drops an empty right side", "The build passed.", "", "The build passed."],
["drops a whitespace-only left side", " ", "The build passed.", "The build passed."],
["drops a whitespace-only right side", "The build passed.", " \n ", "The build passed."],
["handles two empty sides", "", "", ""],
["does not trim the outer edges", " Leading kept.", "Trailing kept. ", " Leading kept. Trailing kept. "],
]
for (const [name, left, right, expected] of cases) {
test(name, () => expect(joinAssistantText(left, right).text).toBe(expected))
}
test("splits the result at the boundary so reveals cover only the appended text", () => {
for (const [name, left, right] of cases) {
const joined = joinAssistantText(left, right)
expect(`${name}: ${joined.previous}${joined.appended}`).toBe(`${name}: ${joined.text}`)
expect(`${name}: ${joined.text.endsWith(joined.appended)}`).toBe(`${name}: true`)
}
})
test("is idempotent once a boundary is normalised", () => {
const once = joinAssistantText("Okay.", "Next.").text
expect(joinAssistantText(once, "").text).toBe(once)
expect(joinAssistantText("Okay. ", "Next.").text).toBe(once)
})
test("folds consecutive messages with exactly one space per boundary", () => {
expect(["First.", "Second.", "Third."].reduce((left, right) => joinAssistantText(left, right).text)).toBe(
"First. Second. Third.",
)
expect(["First.", "", "Third."].reduce((left, right) => joinAssistantText(left, right).text)).toBe("First. Third.")
})
})
@@ -1,33 +0,0 @@
import { describe, expect, test } from "bun:test"
import { AudioJitterBuffer } from "../src/audio-jitter-buffer"
import { PCM_BYTES_PER_MS } from "../src/pcm"
const chunk = (durationMs: number, gapMs = 0) => ({ bytes: Buffer.alloc(durationMs * PCM_BYTES_PER_MS), gapMs })
describe("AudioJitterBuffer", () => {
test("holds startup audio until the target buffer is available", () => {
const buffer = new AudioJitterBuffer(300)
expect(buffer.push(chunk(100))).toEqual([])
expect(buffer.push(chunk(100))).toEqual([])
expect(buffer.push(chunk(100))).toHaveLength(3)
})
test("passes chunks through after playback starts", () => {
const buffer = new AudioJitterBuffer(200)
buffer.push(chunk(200))
expect(buffer.push(chunk(100))).toEqual([chunk(100)])
})
test("flushes short utterances and resets between responses", () => {
const buffer = new AudioJitterBuffer(300)
buffer.push(chunk(100))
expect(buffer.finish()).toHaveLength(1)
buffer.reset()
expect(buffer.push(chunk(100))).toEqual([])
})
test("counts intentional timeline silence toward buffered duration", () => {
const buffer = new AudioJitterBuffer(300)
expect(buffer.push(chunk(200, 100))).toHaveLength(1)
})
})
-106
View File
@@ -1,106 +0,0 @@
import { expect, test } from "bun:test"
import type { AudioDevice } from "../src/audio-device"
import { createAudioSession, type AudioEvent } from "../src/audio-session"
test("reports muted notification playback as inaudible", async () => {
const device = scriptedDevice()
const audio = await createAudioSession({
duplex: false,
speakers: false,
inputActivity: "server",
debug: false,
device,
})
expect(audio.toggleSpeaker()).toBe(true)
audio.play(Buffer.alloc(4_800))
expect(await audio.finishPlayback()).toBe("inaudible")
expect(device.writes).toEqual([])
})
test("settles active playback when interrupted", async () => {
const device = scriptedDevice()
const audio = await createAudioSession({
duplex: false,
speakers: false,
inputActivity: "server",
debug: false,
device,
})
audio.play(Buffer.alloc(4_800))
const outcome = audio.finishPlayback()
audio.flushPlayback()
expect(await outcome).toBe("interrupted")
expect(device.flushes).toBe(1)
})
test("emits captured audio and local speech observations", async () => {
const device = scriptedDevice()
const events: AudioEvent[] = []
const audio = await createAudioSession({
duplex: true,
speakers: false,
inputActivity: "local",
debug: false,
device,
})
await audio.start((event) => events.push(event))
const input = Buffer.alloc(4_800)
for (let offset = 0; offset < input.length; offset += 2) input.writeInt16LE(12_000, offset)
device.capture(input)
expect(events).toContainEqual({ type: "input", audio: input, level: expect.any(Number) })
expect(events).toContainEqual({ type: "user.started" })
audio.close()
})
test("preserves quiet duplex microphone audio during assistant playback", async () => {
const device = scriptedDevice()
const events: AudioEvent[] = []
const audio = await createAudioSession({
duplex: false,
speakers: true,
inputActivity: "server",
debug: false,
device,
})
await audio.start((event) => events.push(event))
audio.play(Buffer.alloc(24_000))
const input = Buffer.alloc(4_800)
for (let offset = 0; offset < input.length; offset += 2) input.writeInt16LE(800, offset)
device.capture(input)
expect(audio.fullDuplex).toBe(true)
expect(events).toContainEqual({ type: "input", audio: input, level: expect.any(Number) })
audio.close()
})
function scriptedDevice() {
let onInput: ((audio: Buffer) => void) | undefined
const writes: Buffer[] = []
const device = {
fullDuplex: true,
mode: "scripted",
async start(listener) {
onInput = listener
},
write(audio) {
writes.push(audio)
},
flush() {
device.flushes += 1
},
close() {},
writes,
flushes: 0,
capture(audio: Buffer) {
onInput?.(audio)
},
} satisfies AudioDevice & {
readonly writes: Buffer[]
flushes: number
capture(audio: Buffer): void
}
return device
}
@@ -1,101 +0,0 @@
import { expect, test } from "bun:test"
import { mkdir, mkdtemp, rm } from "node:fs/promises"
import { tmpdir } from "node:os"
import { join } from "node:path"
import { createCompletionStore } from "../src/completion-store"
test("completion store restores pending and completed notifications until delivery", async () => {
const directory = await mkdtemp(join(tmpdir(), "voice-completions-"))
const path = join(directory, "prompts.json")
const first = await createCompletionStore(path)
const pending = { sessionID: "session-1", promptID: "prompt-1" }
const completed = { sessionID: "session-2", promptID: "prompt-2" }
try {
await first.admitting(pending, "Please continue")
expect(first.entries()).toEqual([{ status: "admitting", handle: pending, text: "Please continue" }])
await first.pending(pending)
await first.pending(completed)
await first.completed(completed, {
type: "opencode.prompt.completed",
session_id: "session-2",
prompt_id: "prompt-2",
status: "completed",
text: "done",
})
await first.close()
const restored = await createCompletionStore(path)
expect(restored.entries()).toEqual(expect.arrayContaining([
{ status: "pending", handle: pending },
{
status: "completed",
handle: completed,
notification: {
type: "opencode.prompt.completed",
session_id: "session-2",
prompt_id: "prompt-2",
status: "completed",
text: "done",
},
},
]))
await restored.close()
const stillPendingDelivery = await createCompletionStore(path)
expect(stillPendingDelivery.entries().some((entry) => entry.status === "completed")).toBe(true)
await stillPendingDelivery.remove(completed)
await stillPendingDelivery.close()
expect((await createCompletionStore(path)).entries()).toEqual([{ status: "pending", handle: pending }])
} finally {
await rm(directory, { recursive: true, force: true })
}
})
test("completion stores do not overwrite prompts written by another process", async () => {
const directory = await mkdtemp(join(tmpdir(), "voice-completions-concurrent-"))
const path = join(directory, "prompts.json")
const first = await createCompletionStore(path)
const second = await createCompletionStore(path)
try {
await Promise.all([
first.pending({ sessionID: "session-1", promptID: "prompt-shared" }),
second.pending({ sessionID: "session-2", promptID: "prompt-shared" }),
])
await Promise.all([first.close(), second.close()])
expect((await createCompletionStore(path)).entries()).toEqual(
expect.arrayContaining([
{ status: "pending", handle: { sessionID: "session-1", promptID: "prompt-shared" } },
{ status: "pending", handle: { sessionID: "session-2", promptID: "prompt-shared" } },
]),
)
} finally {
await rm(directory, { recursive: true, force: true })
}
})
test("continues serializing writes after one filesystem failure", async () => {
const directory = await mkdtemp(join(tmpdir(), "voice-completions-recovery-"))
const path = join(directory, "prompts.json")
const store = await createCompletionStore(path)
try {
await rm(`${path}.d`, { recursive: true })
await Bun.write(`${path}.d`, "not a directory")
const failure = await store
.pending({ sessionID: "session-1", promptID: "prompt-1" })
.then(() => undefined, (error) => error)
expect(failure).toBeDefined()
await rm(`${path}.d`)
await mkdir(`${path}.d`, { recursive: true })
await store.pending({ sessionID: "session-2", promptID: "prompt-2" })
await store.close()
expect((await createCompletionStore(path)).entries()).toContainEqual({
status: "pending",
handle: { sessionID: "session-2", promptID: "prompt-2" },
})
} finally {
await rm(directory, { recursive: true, force: true })
}
})
-311
View File
@@ -1,311 +0,0 @@
import { describe, expect, test } from "bun:test"
import { createLiveEventDelivery, createLiveEventProjector } from "../src/protocol-live"
import type { VoiceProtocolEvent } from "../src/protocol"
const frames = [
{ type: "session.started", session: {} },
{ type: "session.context.appended", start_ms: 0, end_ms: 0 },
{ type: "output_audio.delta", audio: "AAA=", start_ms: 0, end_ms: 1 },
{
type: "input_transcript.added",
start_ms: 0,
end_ms: 100,
item: { id: "input-1", type: "input_transcript", text: " Hello" },
},
{ type: "turn.created", turn: { id: "user-turn", role: "user", transcript: " Hello" } },
{
type: "input_transcript.added",
start_ms: 100,
end_ms: 200,
item: { id: "input-2", type: "input_transcript", text: ", this" },
},
{ type: "turn.delta", turn_id: "user-turn", delta: ", this" },
{
type: "output_transcript.added",
start_ms: 200,
end_ms: 300,
item: { id: "output-1", type: "output_transcript", text: " Copy that" },
},
{
type: "input_transcript.added",
start_ms: 200,
end_ms: 300,
item: { id: "input-3", type: "input_transcript", text: " is a test" },
},
{
type: "output_transcript.added",
start_ms: 300,
end_ms: 400,
item: { id: "output-2", type: "output_transcript", text: ", test" },
},
{ type: "turn.delta", turn_id: "user-turn", delta: " is a test" },
{ type: "turn.done", turn: { id: "user-turn", role: "user", transcript: " Hello, this is a test" } },
{ type: "turn.created", turn: { id: "assistant-turn", role: "assistant", transcript: " Copy that, test" } },
{
type: "output_transcript.added",
start_ms: 400,
end_ms: 500,
item: { id: "output-3", type: "output_transcript", text: " received" },
},
{ type: "turn.delta", turn_id: "assistant-turn", delta: " received" },
{
type: "output_transcript.added",
start_ms: 500,
end_ms: 600,
item: { id: "output-4", type: "output_transcript", text: " loud and clear." },
},
{ type: "turn.delta", turn_id: "assistant-turn", delta: " loud and clear." },
{
type: "turn.done",
turn: { id: "assistant-turn", role: "assistant", transcript: " Copy that, test received loud and clear." },
},
] as const
describe("Live event projector", () => {
test("replays progressive transcripts without schema errors or duplicate text", () => {
const project = createLiveEventProjector()
const events = frames.flatMap((frame) => project(JSON.stringify(frame)).events)
expect(events.filter((event) => event.type === "error")).toEqual([])
expect(events.filter((event) => event.type === "user.committed")).toEqual([
{ type: "user.committed", id: "input-1" },
])
expect(events.filter(isUserTranscript)).toEqual([
{ type: "user.transcript", id: "input-1", text: "Hello", final: false },
{ type: "user.transcript", id: "input-1", text: "Hello, this", final: false },
{ type: "user.transcript", id: "input-1", text: "Hello, this is a test", final: false },
{ type: "user.transcript", id: "input-1", text: "Hello, this is a test", final: true },
])
expect(
events
.filter(isAssistantDelta)
.map((event) => event.delta)
.join(""),
).toBe(" Copy that, test received loud and clear.")
expect(events.filter((event) => event.type === "assistant.done")).toEqual([
{ type: "assistant.done", awaitingWork: false },
])
expect(events.filter((event) => event.type === "assistant.audio")).toHaveLength(1)
})
test("reports malformed frames instead of throwing", () => {
expect(createLiveEventProjector()(JSON.stringify({ type: "turn.created", turn: { id: 1 } })).events).toEqual([
{ type: "error", message: "Received an invalid Live API event." },
])
})
test("preserves context acknowledgement correlation IDs", () => {
expect(
createLiveEventProjector()(
JSON.stringify({ type: "session.context.appended", event_id: "event-notification" }),
),
).toMatchObject({
type: "session.context.appended",
eventID: "event-notification",
})
})
test("replaces corrected assistant transcript snapshots", () => {
const project = createLiveEventProjector()
const events = [
project(
JSON.stringify({
type: "output_transcript.added",
item: { id: "output-1", type: "output_transcript", text: " Hello world" },
}),
),
project(
JSON.stringify({
type: "turn.created",
turn: { id: "assistant-turn", role: "assistant", transcript: " Hello there" },
}),
),
].flatMap((result) => result.events)
expect(events.filter((event) => event.type.startsWith("assistant.transcript"))).toEqual([
{ type: "assistant.transcript.delta", delta: " Hello world" },
{ type: "assistant.transcript", text: " Hello there" },
])
})
test("keeps the space between consecutive assistant turns", () => {
const project = createLiveEventProjector()
const turn = (id: string, text: string) => [
{ type: "turn.created", turn: { id, role: "assistant", transcript: "" } },
{ type: "output_transcript.added", item: { id: `${id}-o`, type: "output_transcript", text } },
{ type: "turn.delta", turn_id: id, delta: text },
{ type: "turn.done", turn: { id, role: "assistant", transcript: text } },
]
const events = [...turn("t1", ' Session "Fix auth bug" is ready.'), ...turn("t2", " Next up, tests.")].flatMap(
(frame) => project(JSON.stringify(frame)).events,
)
// Concatenating the delta stream is what the UI and any transcript consumer does; the
// turn boundary must survive it rather than welding "ready.Next" together.
const transcript = events
.filter(isAssistantDelta)
.map((event) => event.delta)
.join("")
expect(transcript).toBe(' Session "Fix auth bug" is ready. Next up, tests.')
expect(transcript).not.toContain("ready.Next")
expect(events.filter((event) => event.type === "assistant.done")).toHaveLength(2)
})
test("emits no duplicate text when fragments and turn deltas overlap across turns", () => {
const project = createLiveEventProjector()
const frames = [
{ type: "turn.created", turn: { id: "t1", role: "assistant", transcript: "" } },
{ type: "output_transcript.added", item: { id: "a", type: "output_transcript", text: " First" } },
{ type: "turn.delta", turn_id: "t1", delta: " First" },
{ type: "output_transcript.added", item: { id: "b", type: "output_transcript", text: " turn." } },
{ type: "turn.delta", turn_id: "t1", delta: " turn." },
{ type: "turn.done", turn: { id: "t1", role: "assistant", transcript: " First turn." } },
{ type: "turn.created", turn: { id: "t2", role: "assistant", transcript: "" } },
{ type: "output_transcript.added", item: { id: "c", type: "output_transcript", text: " Second turn." } },
{ type: "turn.delta", turn_id: "t2", delta: " Second turn." },
{ type: "turn.done", turn: { id: "t2", role: "assistant", transcript: " Second turn." } },
]
const events = frames.flatMap((frame) => project(JSON.stringify(frame)).events)
expect(events.filter((event) => event.type === "assistant.transcript")).toEqual([])
expect(
events
.filter(isAssistantDelta)
.map((event) => event.delta)
.join(""),
).toBe(" First turn. Second turn.")
})
test("projects Responses delegation function calls without rejecting reasoning items", () => {
const project = createLiveEventProjector()
expect(
project(
JSON.stringify({
type: "response.output_item.done",
item: { id: "reasoning-1", type: "reasoning" },
}),
).events,
).toEqual([{ type: "debug", message: "response.output_item.done" }])
expect(
project(
JSON.stringify({
type: "response.output_item.added",
item: {
id: "function-1",
type: "function_call",
name: "find_sessions",
call_id: "call-1",
arguments: "",
},
}),
).events,
).toEqual([
{ type: "debug", message: "response.output_item.added" },
{ type: "tool.started", id: "call-1", name: "find_sessions" },
])
expect(
project(
JSON.stringify({
type: "response.output_item.done",
item: {
id: "function-1",
type: "function_call",
name: "find_sessions",
call_id: "call-1",
arguments: '{"scope":"current_project"}',
},
}),
).events,
).toEqual([
{ type: "debug", message: "response.output_item.done" },
{
type: "work.requested",
request: { id: "call-1", name: "find_sessions", input: { scope: "current_project" } },
},
])
expect(
project(
JSON.stringify({
type: "response.output_item.done",
item: {
id: "message-1",
type: "message",
status: "completed",
content: [{ type: "output_text", text: "Two sessions found." }],
role: "assistant",
},
}),
).events,
).toEqual([{ type: "debug", message: "response.output_item.done" }])
})
test("projects client delegation requests for the local controller", () => {
expect(
createLiveEventProjector()(
JSON.stringify({
type: "delegation.created",
item: {
id: "delegation-1",
type: "delegation",
target: "client",
content: [{ type: "input_text", text: "Inspect the voice package" }],
},
}),
).events,
).toEqual([
{ type: "debug", message: "delegation.created" },
{ type: "delegation.requested", request: { id: "delegation-1", text: "Inspect the voice package" } },
])
})
test("rejects malformed function arguments without hanging the delegation", () => {
expect(
createLiveEventProjector()(
JSON.stringify({
type: "response.output_item.done",
item: {
id: "function-1",
type: "function_call",
name: "find_sessions",
call_id: "call-invalid",
arguments: "not json",
},
}),
).events,
).toEqual([
{ type: "debug", message: "response.output_item.done" },
{ type: "tool.started", id: "call-invalid", name: "find_sessions" },
{ type: "error", message: "Received invalid arguments for Live tool find_sessions." },
{
type: "work.rejected",
request: { id: "call-invalid", name: "find_sessions", input: {} },
output: { status: "error", message: "Invalid arguments for tool find_sessions." },
},
])
})
test("waits for a quiet drain after turn completion and late audio", async () => {
const events: VoiceProtocolEvent[] = []
const delivery = createLiveEventDelivery({ emit: (event) => events.push(event), drainMs: 20 })
delivery.push({ type: "assistant.done", awaitingWork: false })
await Bun.sleep(10)
delivery.push({ type: "assistant.audio", audio: Buffer.alloc(48) })
await Bun.sleep(12)
expect(events.map((event) => event.type)).toEqual(["assistant.audio"])
await Bun.sleep(12)
expect(events.map((event) => event.type)).toEqual(["assistant.audio", "assistant.done"])
delivery.close()
})
})
function isUserTranscript(
event: VoiceProtocolEvent,
): event is Extract<VoiceProtocolEvent, { readonly type: "user.transcript" }> {
return event.type === "user.transcript"
}
function isAssistantDelta(
event: VoiceProtocolEvent,
): event is Extract<VoiceProtocolEvent, { readonly type: "assistant.transcript.delta" }> {
return event.type === "assistant.transcript.delta"
}
@@ -1,59 +0,0 @@
import { expect, test } from "bun:test"
import { createNotificationRouter, type RoutedAnnouncement } from "../src/notification-router"
test("routes blockers and completions through the admitted delegation", () => {
const routed: RoutedAnnouncement[] = []
const router = createNotificationRouter((announcement) => routed.push(announcement))
router.beginDelegatedTool()
router.route({
notification: {
type: "opencode.prompt.blocked",
prompt_id: "prompt-1",
blocker: "permission",
session_id: "session-1",
request_id: "permission-1",
action: "read",
resources: [],
},
})
expect(routed).toEqual([])
router.finishDelegatedTool("delegation-1", { sessionID: "session-1", promptID: "prompt-1" })
router.route({
notification: {
type: "opencode.prompt.completed",
session_id: "session-1",
prompt_id: "prompt-1",
status: "completed",
text: "done",
},
receipt: { sessionID: "session-1", promptID: "prompt-1" },
})
expect(routed.map((item) => item.delegationID)).toEqual(["delegation-1", "delegation-1"])
expect(routed[0].announcement.receipt).toBeUndefined()
expect(routed[1].announcement.receipt).toEqual({ sessionID: "session-1", promptID: "prompt-1" })
})
test("keeps identical prompt IDs isolated by session", () => {
const routed: RoutedAnnouncement[] = []
const router = createNotificationRouter((announcement) => routed.push(announcement))
router.beginDelegatedTool()
router.finishDelegatedTool("delegation-1", { sessionID: "session-1", promptID: "prompt-shared" })
router.beginDelegatedTool()
router.finishDelegatedTool("delegation-2", { sessionID: "session-2", promptID: "prompt-shared" })
for (const sessionID of ["session-1", "session-2"])
router.route({
notification: {
type: "opencode.prompt.completed",
session_id: sessionID,
prompt_id: "prompt-shared",
status: "completed",
text: "done",
},
receipt: { sessionID, promptID: "prompt-shared" },
})
expect(routed.map((item) => item.delegationID)).toEqual(["delegation-1", "delegation-2"])
})
-248
View File
@@ -1,248 +0,0 @@
import { expect, test } from "bun:test"
import { OpenCode } from "@opencode-ai/client/promise"
import { createOpenCodeBridge } from "../src/opencode"
import type { CompletionStore } from "../src/completion-store"
test("renames only a session discovered by the voice controller", async () => {
const requests: Request[] = []
const session = {
id: "ses_known",
projectID: "project-1",
cost: "0",
tokens: { input: 0, output: 0, reasoning: 0, cache: { read: 0, write: 0 } },
time: { created: 1, updated: 2 },
title: "Old title",
location: { directory: "/workspace" },
}
const fetch = Object.assign(
async (input: string | URL | Request, init?: RequestInit) => {
const request = input instanceof Request ? input : new Request(input.toString(), init)
requests.push(request)
const url = new URL(request.url)
if (url.pathname === "/api/event") return new Response("", { headers: { "content-type": "text/event-stream" } })
if (url.pathname === "/api/project")
return Response.json([
{
id: "project-1",
worktree: "/workspace",
time: { created: 1, updated: 2 },
sandboxes: [],
},
])
if (url.pathname === "/api/session" && request.method === "GET")
return Response.json({ data: [session], cursor: {} })
if (url.pathname === "/api/session/ses_known" && request.method === "GET") return Response.json({ data: session })
if (url.pathname === "/api/session/ses_known/rename" && request.method === "POST")
return new Response(null, { status: 204 })
return new Response("Not found", { status: 404 })
},
{ preconnect: () => {} },
)
const client = OpenCode.make({
baseUrl: "https://opencode.test",
fetch,
})
const bridge = await createOpenCodeBridge({
client,
directory: "/workspace",
model: { providerID: "openai", id: "gpt-test" },
notify: () => {},
completionStore: memoryCompletionStore(),
})
const execute = async (name: string, input: Record<string, unknown>) => (await bridge.execute(name, input)).output
try {
expect(new Set(bridge.definitions.map((tool) => tool.name)).size).toBe(bridge.definitions.length)
expect(bridge.definitions.every((tool) => tool.name.length > 0 && tool.description.length > 0)).toBe(true)
expect(bridge.definitions.find((tool) => tool.name === "reply_form")).toMatchObject({
parameters: {
properties: { answer: { type: "object", additionalProperties: true } },
},
})
expect(bridge.definitions.find((tool) => tool.name === "rename_session")).toMatchObject({
description: expect.stringContaining("does not prompt, wake, or interrupt"),
parameters: {
type: "object",
additionalProperties: false,
required: ["session_id", "title"],
},
})
expect(await execute("rename_session", { session_id: "ses_unknown", title: "Nope" })).toEqual({
status: "error",
message: "Use a session ID returned by find_sessions or start_session.",
retryable: false,
})
expect(requests.some((request) => new URL(request.url).pathname.includes("ses_unknown"))).toBe(false)
await execute("find_sessions", {
query: null,
scope: "current_project",
recency: "any",
limit: 10,
})
expect(await execute("rename_session", { session_id: "ses_known", title: " " })).toEqual({
status: "error",
message: "A non-empty session title is required.",
retryable: false,
})
expect(await execute("rename_session", { session_id: "ses_known", title: "Old title" })).toEqual({
status: "unchanged",
session_id: "ses_known",
previous_title: "Old title",
title: "Old title",
})
expect(requests.some((request) => new URL(request.url).pathname.endsWith("/rename"))).toBe(false)
expect(await execute("rename_session", { session_id: "ses_known", title: " New title " })).toEqual({
status: "renamed",
session_id: "ses_known",
previous_title: "Old title",
title: "New title",
})
const rename = requests.find((request) => new URL(request.url).pathname.endsWith("/rename"))
expect(rename?.method).toBe("POST")
expect(await rename?.json()).toEqual({ title: "New title" })
} finally {
await bridge.close()
}
})
test("retries a lost completion wait and prompts a started session by ID", async () => {
const requests: Request[] = []
const prompts: Array<{ readonly id: string; readonly text: string }> = []
const notifications: Array<unknown> = []
const traces: Array<{ readonly event: string; readonly data?: Record<string, unknown> }> = []
let waits = 0
const fetch = Object.assign(
async (input: string | URL | Request, init?: RequestInit) => {
const request = input instanceof Request ? input : new Request(input.toString(), init)
requests.push(request)
const url = new URL(request.url)
if (url.pathname === "/api/event") return new Response("", { headers: { "content-type": "text/event-stream" } })
if (url.pathname === "/api/session" && request.method === "POST")
return Response.json({
data: {
id: "ses_focused",
projectID: "project-1",
cost: "0",
tokens: { input: 0, output: 0, reasoning: 0, cache: { read: 0, write: 0 } },
time: { created: 1, updated: 1 },
title: "Focused session",
location: { directory: "/workspace" },
},
})
if (url.pathname === "/api/session/ses_focused/prompt" && request.method === "POST") {
const body = await request.json()
if (
typeof body !== "object" ||
body === null ||
!("id" in body) ||
typeof body.id !== "string" ||
!("text" in body) ||
typeof body.text !== "string"
)
return new Response("Invalid prompt", { status: 400 })
prompts.push({ id: body.id, text: body.text })
return Response.json({ data: { id: body.id, admittedSeq: prompts.length } })
}
if (url.pathname === "/api/session/ses_focused/wait" && request.method === "POST") {
waits += 1
if (waits === 1) throw new Error("connection closed")
return new Response(null, { status: 204 })
}
if (url.pathname === "/api/session/ses_focused/message" && request.method === "GET")
return Response.json({
data: prompts.toReversed().map((prompt) => ({ id: prompt.id, type: "user", text: prompt.text })),
cursor: {},
})
return new Response("Not found", { status: 404 })
},
{ preconnect: () => {} },
)
const bridge = await createOpenCodeBridge({
client: OpenCode.make({ baseUrl: "https://opencode.test", fetch }),
directory: "/workspace",
model: { providerID: "openai", id: "gpt-test" },
notify: (notification) => notifications.push(notification),
trace: (event, data) => traces.push({ event, data }),
completionStore: memoryCompletionStore(),
})
const execute = async (name: string, input: Record<string, unknown>) => (await bridge.execute(name, input)).output
try {
expect(bridge.definitions.some((tool) => tool.name === "continue_session")).toBe(false)
const started = await bridge.execute("start_session", { text: "First task", project_id: null })
expect(started.output).toMatchObject({
status: "started",
session_id: "ses_focused",
})
expect(started.admittedPrompt).toEqual({ sessionID: "ses_focused", promptID: prompts[0].id })
expect(
await execute("prompt_session", { session_id: "ses_focused", text: "Follow-up task" }),
).toMatchObject({
status: "started",
session_id: "ses_focused",
})
await Bun.sleep(1_100)
expect(prompts.map((prompt) => prompt.text)).toEqual(["First task", "Follow-up task"])
expect(requests.filter((request) => new URL(request.url).pathname.endsWith("/wait")).length).toBeGreaterThanOrEqual(
3,
)
expect(traces.some((entry) => entry.event === "opencode.wait.retrying")).toBe(true)
expect(notifications.some((value) => JSON.stringify(value).includes("opencode.prompt.failed"))).toBe(false)
} finally {
await bridge.close()
}
})
test("discards a restored prompt whose session no longer exists", async () => {
const removed: Array<{ readonly sessionID: string; readonly promptID: string }> = []
const handle = { sessionID: "ses_missing", promptID: "msg_orphaned" }
const fetch = Object.assign(
async (input: string | URL | Request) => {
const request = input instanceof Request ? input : new Request(input.toString())
const url = new URL(request.url)
if (url.pathname === "/api/event") return new Response("", { headers: { "content-type": "text/event-stream" } })
if (url.pathname === "/api/session/ses_missing")
return Response.json(
{ _tag: "SessionNotFoundError", sessionID: handle.sessionID, message: `Session not found: ${handle.sessionID}` },
{ status: 404 },
)
return new Response("Not found", { status: 404 })
},
{ preconnect: () => {} },
)
const store = {
...memoryCompletionStore(),
entries: () => [{ status: "pending" as const, handle }],
remove: async (entry) => {
removed.push(entry)
},
} satisfies CompletionStore
const bridge = await createOpenCodeBridge({
client: OpenCode.make({ baseUrl: "https://opencode.test", fetch }),
directory: "/workspace",
model: { providerID: "openai", id: "gpt-test" },
notify: () => {},
completionStore: store,
})
try {
expect(removed).toEqual([handle])
} finally {
await bridge.close()
}
})
function memoryCompletionStore(): CompletionStore {
return {
entries: () => [],
admitting: async () => {},
pending: async () => {},
completed: async () => {},
remove: async () => {},
close: async () => {},
}
}
-10
View File
@@ -1,10 +0,0 @@
import { expect, test } from "bun:test"
import { pcmLevel } from "../src/pcm"
test("PCM level ignores silence and normalizes audible energy", () => {
expect(pcmLevel(Buffer.alloc(4_800))).toBe(0)
const audible = Buffer.alloc(4_800)
for (let offset = 0; offset < audible.length; offset += 2) audible.writeInt16LE(8_000, offset)
expect(pcmLevel(audible)).toBeGreaterThan(0.8)
expect(pcmLevel(audible)).toBeLessThanOrEqual(1)
})
-137
View File
@@ -1,137 +0,0 @@
import { describe, expect, test } from "bun:test"
import { Option } from "effect"
import { decodeVoiceToolInput } from "../src/protocol"
import { createLiveContextAppendQueue, liveNotificationAcknowledgement } from "../src/protocol-live"
import { realtimeNotificationAcknowledgement, realtimeSessionUpdate } from "../src/protocol-realtime"
import { createSingleFlightAcknowledgement } from "../src/single-flight-acknowledgement"
import { openCodeAnnouncementText } from "../src/opencode-notification"
describe("voice protocol", () => {
test("accepts only JSON object tool arguments", () => {
expect(Option.getOrUndefined(decodeVoiceToolInput('{"text":"hello","nested":{"value":1}}'))).toEqual({
text: "hello",
nested: { value: 1 },
})
for (const input of ["not json", "null", "[]", "1", '"text"'])
expect(Option.getOrUndefined(decodeVoiceToolInput(input))).toBeUndefined()
})
test("correlates Realtime notification acknowledgements", () => {
const pending = { itemID: "item-current", eventID: "event-current" }
expect(
realtimeNotificationAcknowledgement(
{ type: "conversation.item.created", item: { id: "item-current" } },
pending,
),
).toBe(true)
expect(
realtimeNotificationAcknowledgement(
{ type: "conversation.item.created", item: { id: "item-other" } },
pending,
),
).toBeUndefined()
expect(
realtimeNotificationAcknowledgement(
{ type: "error", error: { event_id: "event-current" } },
pending,
),
).toBe(false)
})
test("preserves tool schemas in Realtime session updates", () => {
const tools = [
{
type: "function" as const,
name: "open_form",
description: "Open form",
parameters: {
type: "object",
properties: { answer: { type: "object", additionalProperties: true } },
additionalProperties: true,
},
},
]
expect(JSON.parse(JSON.stringify(realtimeSessionUpdate({ instructions: "test", tools, voice: "marin" }))).session.tools)
.toEqual(tools)
})
test("correlates Live notification acknowledgements by expected context", () => {
const pending = { eventID: "event-current", acknowledgement: "delegation.context.appended" as const }
expect(
liveNotificationAcknowledgement(
{ type: "delegation.context.appended", eventID: "event-current" },
pending,
),
).toBe(true)
expect(
liveNotificationAcknowledgement({ type: "session.context.appended", eventID: "event-current" }, pending),
).toBeUndefined()
expect(
liveNotificationAcknowledgement(
{ type: "delegation.context.appended", eventID: "event-other" },
pending,
),
).toBe(true)
expect(
liveNotificationAcknowledgement({ type: "error", errorEventID: "event-current" }, pending),
).toBe(false)
})
test("serializes indistinguishable Live context appends", async () => {
const sent: Array<Record<string, unknown>> = []
const queue = createLiveContextAppendQueue({
send: (event) => {
sent.push(event)
return true
},
timeoutMs: 100,
})
const first = queue.append(
{ type: "delegation.context.append", event_id: "event-1" },
"delegation.context.appended",
)
const second = queue.append(
{ type: "delegation.context.append", event_id: "event-2" },
"delegation.context.appended",
)
expect(sent.map((event) => event["event_id"])).toEqual(["event-1"])
queue.receive({ type: "session.context.appended", eventID: "unrelated" })
expect(sent.map((event) => event["event_id"])).toEqual(["event-1"])
queue.receive({ type: "delegation.context.appended", eventID: "not-echoed" })
expect(await first).toBe(true)
expect(sent.map((event) => event["event_id"])).toEqual(["event-1", "event-2"])
queue.receive({ type: "delegation.context.appended" })
expect(await second).toBe(true)
queue.close()
})
test("keeps notification acknowledgement single-flight", async () => {
const acknowledgement = createSingleFlightAcknowledgement<{ readonly token: string }>(50)
const first = acknowledgement.begin("notification-1", { token: "first" })
const second = acknowledgement.begin("notification-2", { token: "second" })
expect(first.started).toBe(true)
expect(second.started).toBe(false)
expect(await second.promise).toBe(false)
expect(acknowledgement.current()).toEqual({ id: "notification-1", correlation: { token: "first" } })
acknowledgement.settle("notification-1", true)
expect(await first.promise).toBe(true)
expect(acknowledgement.current()).toBeUndefined()
})
test("renders bounded conversational OpenCode announcements without identifiers", () => {
const text = openCodeAnnouncementText({
type: "opencode.prompt.completed",
session_id: "ses_private",
prompt_id: "msg_private",
status: "completed",
text: "finished ".repeat(200),
})
expect(text.length).toBeLessThanOrEqual(800)
expect(text).not.toContain("ses_private")
expect(text).not.toContain("msg_private")
expect(text).toStartWith("OpenCode prompt completed: finished")
})
})
@@ -1,69 +0,0 @@
import { expect, test } from "bun:test"
import { createRealtimeEventProjector } from "../src/protocol-realtime"
test("projects one tool start and one work request for duplicate item frames", () => {
const projector = createRealtimeEventProjector()
const item = {
type: "function_call",
name: "find_sessions",
call_id: "call-1",
arguments: '{"query":null}',
}
const added = projector.receive(JSON.stringify({ type: "response.output_item.added", item }))
const duplicate = projector.receive(JSON.stringify({ type: "response.output_item.added", item }))
const done = projector.receive(JSON.stringify({ type: "response.output_item.done", item }))
expect(added.events).toContainEqual({ type: "tool.started", id: "call-1", name: "find_sessions" })
expect(duplicate.events.some((event) => event.type === "tool.started")).toBe(false)
expect(done.events).toContainEqual({
type: "work.requested",
request: { id: "call-1", name: "find_sessions", input: { query: null } },
})
expect(projector.receive(JSON.stringify({ type: "response.output_item.done", item })).events).toEqual([
{ type: "debug", message: "response.output_item.done" },
])
})
test("resumes a response only after all projected work resolves", () => {
const projector = createRealtimeEventProjector()
const response = projector.receive(
JSON.stringify({
type: "response.done",
response: { output: [{ type: "function_call", call_id: "call-1" }] },
}),
)
expect(response.events).toContainEqual({ type: "assistant.done", awaitingWork: true })
expect(response.commands).toEqual([])
expect(projector.resolveWork("call-1")).toEqual([{ type: "response.create" }])
expect(projector.resolveWork("call-1")).toEqual([])
})
test("resumes when work resolved before the response completion frame", () => {
const projector = createRealtimeEventProjector()
expect(projector.resolveWork("call-1")).toEqual([])
expect(
projector.receive(
JSON.stringify({
type: "response.done",
response: { output: [{ type: "function_call", call_id: "call-1" }] },
}),
).commands,
).toEqual([{ type: "response.create" }])
})
test("rejects malformed tool arguments without hanging the provider call", () => {
const projector = createRealtimeEventProjector()
const result = projector.receive(
JSON.stringify({
type: "response.output_item.done",
item: { type: "function_call", name: "find_sessions", call_id: "call-1", arguments: "[]" },
}),
)
expect(result.events).toContainEqual({
type: "work.rejected",
request: { id: "call-1", name: "find_sessions", input: {} },
output: { status: "error", message: "Invalid arguments for tool find_sessions." },
})
})
@@ -1,229 +0,0 @@
import { expect, test } from "bun:test"
import {
createResponsesControllerContext,
responsesControllerTools,
runResponsesController,
} from "../src/responses-controller"
test("runs the client delegation controller through OpenCode tools", async () => {
const bodies: Array<Record<string, unknown>> = []
const calls: Array<unknown> = []
const responses = [
{
id: "response-1",
output: [
{
type: "function_call",
call_id: "call-1",
name: "start_session",
arguments: '{"text":"Inspect the package","project_id":null}',
},
],
},
{
id: "response-2",
output: [
{
type: "message",
content: [{ type: "output_text", text: "The inspection is running and I will report back." }],
},
],
},
]
const fetch = Object.assign(
async (_input: string | URL | Request, init?: RequestInit) => {
const body: unknown = await new Request("https://opencode.test", init).json()
if (body && typeof body === "object" && !Array.isArray(body)) bodies.push(Object.fromEntries(Object.entries(body)))
return Response.json(responses.shift())
},
{ preconnect: () => {} },
)
const text = await runResponsesController({
apiKey: "test",
model: "gpt-test",
instructions: "Use tools.",
text: "Inspect this package",
tools: [{ type: "function", name: "start_session", description: "Start", parameters: {} }],
execute: async (call) => {
calls.push(call)
return { status: "started", session_id: "ses_started", prompt_id: "prompt-1" }
},
fetch,
})
expect(text).toBe(
"The inspection is running and I will report back.\n\nPrivate voice-control context (never speak this aloud): OpenCode session IDs explicitly used by this delegation: ses_started. Include the relevant session_id in any future delegated request to continue this work.",
)
expect(calls).toEqual([
{
id: "call-1",
name: "start_session",
input: { text: "Inspect the package", project_id: null },
},
])
expect(bodies[1]).toMatchObject({
previous_response_id: "response-1",
input: [
{
type: "function_call_output",
call_id: "call-1",
output: '{"status":"started","session_id":"ses_started","prompt_id":"prompt-1"}',
},
],
})
expect(bodies[0]?.["tools"]).toEqual([
{ type: "function", name: "start_session", description: "Start", parameters: {} },
])
})
test("rejects malformed controller tool arguments", async () => {
const fetch = Object.assign(
async () =>
Response.json({
id: "response-1",
output: [{ type: "function_call", call_id: "call-1", name: "start_session", arguments: "not json" }],
}),
{ preconnect: () => {} },
)
expect(
runResponsesController({
apiKey: "test",
model: "gpt-test",
instructions: "Use tools.",
text: "Inspect this package",
tools: [{ type: "function", name: "start_session", description: "Start", parameters: {} }],
execute: async () => ({ status: "started" }),
fetch,
}),
).rejects.toThrow("invalid arguments for start_session")
})
test("retains the most recently used session across delegations", async () => {
const context = createResponsesControllerContext()
const bodies: Array<Record<string, unknown>> = []
const responses = [
{
id: "response-1",
output: [
{
type: "function_call",
call_id: "call-1",
name: "read_session",
arguments: '{"session_id":"ses_recent"}',
},
],
},
{
id: "response-2",
output: [{ type: "message", content: [{ type: "output_text", text: "Found it." }] }],
},
{
id: "response-3",
output: [{ type: "message", content: [{ type: "output_text", text: "Stopped it." }] }],
},
]
const fetch = Object.assign(
async (_input: string | URL | Request, init?: RequestInit) => {
const body: unknown = await new Request("https://opencode.test", init).json()
if (body && typeof body === "object" && !Array.isArray(body)) bodies.push(Object.fromEntries(Object.entries(body)))
return Response.json(responses.shift())
},
{ preconnect: () => {} },
)
const options = {
apiKey: "test",
model: "gpt-test",
instructions: "Use tools.",
tools: [{ type: "function" as const, name: "read_session", description: "Read", parameters: {} }],
execute: async () => ({ status: "ok" }),
context,
fetch,
}
await runResponsesController({ ...options, text: "Find the label behavior session" })
await runResponsesController({ ...options, text: "Stop that session" })
expect(context.lastSessionID).toBe("ses_recent")
expect(bodies[2]?.["input"]).toContain("the most recently used OpenCode session ID is ses_recent")
expect(bodies[2]?.["input"]).toContain('references like "that session", "it", or "stop that"')
})
test("coalesces identical read-only calls within one delegation", async () => {
const responses = [
{
id: "response-1",
output: [
{ type: "function_call", call_id: "call-1", name: "read_session", arguments: '{"session_id":"ses_1"}' },
{ type: "function_call", call_id: "call-2", name: "read_session", arguments: '{"session_id":"ses_1"}' },
],
},
{
id: "response-2",
output: [{ type: "message", content: [{ type: "output_text", text: "Done." }] }],
},
]
let executions = 0
const fetch = Object.assign(async () => Response.json(responses.shift()), { preconnect: () => {} })
await runResponsesController({
apiKey: "test",
model: "gpt-test",
instructions: "Use tools.",
text: "Read it",
tools: [{ type: "function", name: "read_session", description: "Read", parameters: {} }],
execute: async () => {
executions += 1
return { status: "ok" }
},
fetch,
})
expect(executions).toBe(1)
})
test("withholds new-session creation during follow-ups unless explicitly requested", () => {
const context = createResponsesControllerContext()
context.lastSessionID = "ses_recent"
const tools = [
{ type: "function" as const, name: "start_session", description: "Start", parameters: {} },
{ type: "function" as const, name: "read_session", description: "Read", parameters: {} },
]
expect(responsesControllerTools(tools, "Stop that session", context).map((tool) => tool.name)).toEqual([
"read_session",
])
expect(responsesControllerTools(tools, "Start a new session for this", context)).toEqual(tools)
})
test("remembers a uniquely discovered session without requiring a read", async () => {
const context = createResponsesControllerContext()
const responses = [
{
id: "response-1",
output: [
{ type: "function_call", call_id: "call-1", name: "find_sessions", arguments: '{"query":"label"}' },
],
},
{
id: "response-2",
output: [{ type: "message", content: [{ type: "output_text", text: "Found one." }] }],
},
]
const fetch = Object.assign(async () => Response.json(responses.shift()), { preconnect: () => {} })
await runResponsesController({
apiKey: "test",
model: "gpt-test",
instructions: "Use tools.",
text: "Find label behavior",
tools: [{ type: "function", name: "find_sessions", description: "Find", parameters: {} }],
execute: async () => ({ status: "ok", sessions: [{ id: "ses_unique", title: "Label behavior" }] }),
context,
fetch,
})
expect(context.lastSessionID).toBe("ses_unique")
expect(context.sessionIDs).toEqual(new Set(["ses_unique"]))
})
-331
View File
@@ -1,331 +0,0 @@
import { expect, mock, test } from "bun:test"
import { createTestRenderer } from "@opentui/core/testing"
test("voice rows fill the viewport, wrap under content, and keep tools after their preamble", async () => {
const setup = await createTestRenderer({ width: 64, height: 18, useThread: false })
const core = await import("@opentui/core")
void mock.module("@opentui/core", () => ({ ...core, createCliRenderer: async () => setup.renderer }))
const { createVoiceTUI } = await import("../src/ui")
let interrupts = 0
let voiceCycles = 0
let microphoneToggles = 0
let speakerToggles = 0
const ui = await createVoiceTUI({
onInterrupt: () => interrupts++,
onExit: () => {},
onCycleVoice: () => voiceCycles++,
onToggleMicrophone: () => microphoneToggles++,
onToggleSpeaker: () => speakerToggles++,
reducedMotion: true,
})
try {
ui.setStatus({ audio: "duplex", voice: "marin", model: "gpt-live" })
ui.meta("[session] created session-1")
ui.userCommitted("user-1")
ui.userTranscript("user-1", "Please inspect the current session", true)
ui.toolStart("tool-1", "opencode", { request: "inspect" })
ui.meta("[session] adopted session-1")
ui.assistantDelta("Let me check. This sentence is")
ui.assistantDelta(" long enough to wrap cleanly under its column.")
ui.assistantDone()
ui.assistantDelta("Continued in the same assistant row.")
ui.assistantDone()
await Bun.sleep(200)
await setup.flush()
const rows = frameRows(setup.captureCharFrame())
expect(rows).toHaveLength(18)
expect(rows.every((row) => row.length === 64)).toBe(true)
expect(rows[16]).toStartWith(" ▂▂▂▁ duplex marin gpt-live")
expect(rows[16]).not.toContain("no session")
expect(rows[17]).toContain("esc interrupt")
expect(rows[17]).not.toContain("any key")
expect(rows[15].trim()).toBe("")
expect(rows.filter((row) => row.includes("Let me check"))).toHaveLength(1)
expect(rows.findIndex((row) => row.includes("opencode"))).toBeGreaterThan(
rows.findIndex((row) => row.includes("Let me check")),
)
expect(rows.findIndex((row) => row.includes("[session] adopted"))).toBeGreaterThan(
rows.findIndex((row) => row.includes("opencode")),
)
expect(rows.find((row) => row.includes("under its column"))?.indexOf("under")).toBe(
rows.find((row) => row.includes("Let me check"))?.indexOf("Let"),
)
// Deltas inside one turn concatenate raw; a delta after assistantDone starts a new
// message and must gain exactly one space at the boundary.
const assistant = rows.join("")
expect(assistant).toContain("This sentence is long enough")
expect(assistant).toContain("column. Continued")
expect(assistant).not.toContain("column.Continued")
expect(rows.find((row) => row.includes("[session] created"))).toStartWith(" · [session]")
setup.resize(96, 22)
await setup.flush()
const wide = frameRows(setup.captureCharFrame())
expect(wide).toHaveLength(22)
expect(wide.every((row) => row.length === 96)).toBe(true)
expect(wide.at(-2)).toStartWith(" ▂▂▂▁ duplex marin gpt-live")
setup.resize(48, 16)
await setup.flush()
const narrow = frameRows(setup.captureCharFrame())
expect(narrow).toHaveLength(16)
expect(narrow.every((row) => row.length === 48)).toBe(true)
expect(narrow.at(-1)).toContain("esc interrupt")
setup.resize(64, 18)
await setup.flush()
ui.userSpeaking(true)
await setup.flush()
expect(setup.captureCharFrame()).not.toContain("│ transcribing")
expect(frameRows(setup.captureCharFrame()).at(-2)).toStartWith(" ▂▂▂▂")
ui.userReset()
await setup.flush()
expect(setup.captureCharFrame()).not.toContain("│ transcribing")
ui.userSpeaking(true)
ui.userSpeaking(false)
ui.userCommitted("user-2")
ui.userTranscript("user-2", "partial transcript", false)
await setup.flush()
const partial = setup
.captureCharFrame()
.split("\n")
.find((row) => row.includes("partial transcript"))
expect(partial).toContain("│ partial transcript")
expect(partial).not.toContain("you")
ui.userSpeaking(true)
await setup.flush()
expect(
setup
.captureCharFrame()
.split("\n")
.filter((row) => row.includes("│ transcribing")),
).toHaveLength(0)
ui.userReset()
await setup.flush()
ui.setStatus({ microphoneMuted: true })
await setup.flush()
expect(frameRows(setup.captureCharFrame()).at(-2)).toStartWith(" ────")
const muted = setup
.captureCharFrame()
.split("\n")
.find((row) => row.includes("partial transcript"))
expect(muted).toContain("│ partial transcript")
expect(muted).not.toContain("│ ...")
ui.userSpeaking(false)
ui.setStatus({ microphoneMuted: false })
ui.userTranscript("user-2", "partial transcript", true)
await setup.flush()
const complete = setup
.captureCharFrame()
.split("\n")
.find((row) => row.includes("partial transcript"))
expect(complete).toContain("│ partial transcript")
ui.assistantDelta("First response.")
ui.userCommitted("boundary-user")
ui.userTranscript("boundary-user", "A new request", true)
ui.assistantDelta("Second response.")
await setup.flush()
const boundaries = setup.captureCharFrame().split("\n")
expect(boundaries.findIndex((row) => row.includes("First response."))).toBeLessThan(
boundaries.findIndex((row) => row.includes("A new request")),
)
expect(boundaries.findIndex((row) => row.includes("A new request"))).toBeLessThan(
boundaries.findIndex((row) => row.includes("Second response.")),
)
await setup.mockInput.typeText(" x")
setup.mockInput.pressArrow("right")
await setup.flush()
expect(interrupts).toBe(0)
expect(voiceCycles).toBe(0)
expect(microphoneToggles).toBe(0)
expect(speakerToggles).toBe(0)
setup.mockInput.pressKey("v")
setup.mockInput.pressKey("m")
setup.mockInput.pressKey("s")
await setup.flush()
expect(voiceCycles).toBe(1)
expect(microphoneToggles).toBe(1)
expect(speakerToggles).toBe(1)
setup.mockInput.pressEscape()
await Bun.sleep(50)
await setup.flush()
expect(interrupts).toBe(1)
} finally {
ui.close()
mock.restore()
}
})
// In audio mode ui.assistantDone() is scheduled off the AudioSession playback clock,
// so the next turn's transcript deltas arrive while the row is still streaming. The turn boundary
// therefore has to survive in the delta text itself, not in the row's streaming flag.
test("keeps the turn boundary when the next turn streams before playback finishes", async () => {
const setup = await createTestRenderer({ width: 72, height: 12, useThread: false })
const core = await import("@opentui/core")
void mock.module("@opentui/core", () => ({ ...core, createCliRenderer: async () => setup.renderer }))
const { createVoiceTUI } = await import("../src/ui")
const ui = await createVoiceTUI({
onInterrupt: () => {},
onExit: () => {},
onCycleVoice: () => {},
onToggleMicrophone: () => {},
onToggleSpeaker: () => {},
reducedMotion: true,
})
try {
// Verbatim projector deltas: each turn's first fragment carries the boundary space.
ui.assistantDelta(' Session "Fix auth bug"')
ui.assistantDelta(" is ready.")
// turn.done reaches the protocol, but playback is still draining, so no assistantDone() yet.
ui.assistantDelta(" Next up, tests.")
await setup.flush()
const frame = setup.captureCharFrame()
expect(frame).toContain('Session "Fix auth bug" is ready. Next up, tests.')
expect(frame).not.toContain("ready.Next")
// The row itself must not open with the boundary space.
expect(frame).toContain('│ Session "Fix auth bug"')
expect(frame).not.toContain('│ Session "Fix auth bug"')
ui.assistantDone()
await setup.flush()
expect(setup.captureCharFrame()).toContain("is ready. Next up, tests.")
} finally {
ui.close()
mock.restore()
}
})
test("keeps animated text mounted after its reveal completes", async () => {
const setup = await createTestRenderer({ width: 72, height: 10, useThread: false })
const core = await import("@opentui/core")
void mock.module("@opentui/core", () => ({ ...core, createCliRenderer: async () => setup.renderer }))
const { createVoiceTUI } = await import("../src/ui")
const ui = await createVoiceTUI({
onInterrupt: () => {},
onExit: () => {},
onCycleVoice: () => {},
onToggleMicrophone: () => {},
onToggleSpeaker: () => {},
})
try {
ui.assistantDelta("Animated text remains visible.")
await Bun.sleep(500)
await setup.flush()
expect(setup.captureCharFrame()).toContain("Animated text remains visible.")
} finally {
ui.close()
mock.restore()
}
})
test("renders consecutive tools as compact single-spaced rows", async () => {
const setup = await createTestRenderer({ width: 52, height: 8, useThread: false })
const core = await import("@opentui/core")
void mock.module("@opentui/core", () => ({ ...core, createCliRenderer: async () => setup.renderer }))
const { createVoiceTUI } = await import("../src/ui")
const ui = await createVoiceTUI({
onInterrupt: () => {},
onExit: () => {},
onCycleVoice: () => {},
onToggleMicrophone: () => {},
onToggleSpeaker: () => {},
reducedMotion: true,
})
try {
ui.toolStart("tool-1", "read_session", {})
ui.toolDone("tool-1", { status: "ok" })
ui.toolStart("tool-2", "read_session", {})
ui.toolDone("tool-2", { status: "ok" })
await Bun.sleep(200)
await setup.flush()
const rows = frameRows(setup.captureCharFrame())
const tools = rows.flatMap((row, index) => (row.includes("read_session") ? [{ row, index }] : []))
expect(tools).toHaveLength(2)
expect(tools[1]?.index).toBe((tools[0]?.index ?? 0) + 1)
expect(tools.every((tool) => tool.row.includes("✓ read_session ok"))).toBe(true)
expect(rows.at(-3)?.trim()).toBe("")
} finally {
ui.close()
mock.restore()
}
})
test("shows the reason a tool failed", async () => {
const setup = await createTestRenderer({ width: 72, height: 8, useThread: false })
const core = await import("@opentui/core")
void mock.module("@opentui/core", () => ({ ...core, createCliRenderer: async () => setup.renderer }))
const { createVoiceTUI } = await import("../src/ui")
const ui = await createVoiceTUI({
onInterrupt: () => {},
onExit: () => {},
onCycleVoice: () => {},
onToggleMicrophone: () => {},
onToggleSpeaker: () => {},
reducedMotion: true,
})
try {
ui.toolStart("tool-error", "opencode", {})
ui.toolDone("tool-error", { status: "error", message: "Invalid tool schema" })
await Bun.sleep(200)
await setup.flush()
expect(setup.captureCharFrame()).toContain("opencode error · Invalid tool schema")
} finally {
ui.close()
mock.restore()
}
})
test("paints the full viewport and keeps mic activity in the footer", async () => {
const setup = await createTestRenderer({ width: 40, height: 12, useThread: false })
const core = await import("@opentui/core")
void mock.module("@opentui/core", () => ({ ...core, createCliRenderer: async () => setup.renderer }))
const { createVoiceTUI } = await import("../src/ui")
const ui = await createVoiceTUI({
onInterrupt: () => {},
onExit: () => {},
onCycleVoice: () => {},
onToggleMicrophone: () => {},
onToggleSpeaker: () => {},
reducedMotion: true,
})
try {
ui.userSpeaking(true)
await setup.flush()
const rows = frameRows(setup.captureCharFrame())
expect(rows.some((row) => row.includes("│ transcribing"))).toBe(false)
expect(rows.at(-2)).toStartWith(" ▂▂▂▂")
expect(setup.captureSpans().lines.every((line) => line.spans.every((span) => span.bg.a === 1))).toBe(true)
setup.resize(56, 16)
await setup.flush()
expect(setup.captureSpans().lines.every((line) => line.spans.every((span) => span.bg.a === 1))).toBe(true)
} finally {
ui.close()
mock.restore()
}
})
function frameRows(frame: string) {
return frame.split("\n").slice(0, -1)
}
-74
View File
@@ -1,74 +0,0 @@
import { describe, expect, test } from "bun:test"
import { initialVoiceView, transitionVoiceView, type VoiceViewEvent, type VoiceViewState } from "../src/ui-model"
const apply = (state: VoiceViewState, ...events: ReadonlyArray<VoiceViewEvent>) =>
events.reduce(transitionVoiceView, state)
describe("VoiceView", () => {
test("keeps assistant responses separated by a committed user turn", () => {
const state = apply(
initialVoiceView(),
{ type: "assistant.delta", text: "First response.", now: 0, animate: false },
{ type: "user.committed", itemID: "user-1" },
{ type: "user.transcript", itemID: "user-1", text: "Next request", final: true, now: 1, animate: false },
{ type: "assistant.delta", text: "Second response.", now: 2, animate: false },
)
expect(state.messages.map((message) => message.kind)).toEqual(["assistant", "user", "assistant"])
expect(state.messages.flatMap((message) => ("text" in message ? [message.text] : []))).toEqual([
"First response.",
"Next request",
"Second response.",
])
})
test("keeps local sound activity out of transcript history", () => {
const started = transitionVoiceView(initialVoiceView(), { type: "user.started" })
expect(started.messages).toEqual([])
const committed = transitionVoiceView(started, { type: "user.committed", itemID: "user-1" })
expect(committed.messages).toEqual([
{ key: "message-1", kind: "user", itemID: "user-1", transcribing: true, reveals: [] },
])
const abandoned = apply(initialVoiceView(), { type: "user.started" }, { type: "user.reset" })
expect(abandoned.messages).toEqual([])
})
test("deduplicates transcript snapshots and clears completed reveals", () => {
const state = apply(
initialVoiceView(),
{ type: "user.committed", itemID: "user-1" },
{ type: "user.transcript", itemID: "user-1", text: "Hello", final: false, now: 0, animate: true },
{ type: "user.transcript", itemID: "user-1", text: "Hello", final: false, now: 10, animate: true },
)
expect(state.messages).toHaveLength(1)
expect(state.messages[0]?.kind === "user" ? state.messages[0].reveals.length : 0).toBeGreaterThan(0)
expect(transitionVoiceView(state, { type: "reveals.completed" }).messages).toEqual([
{ key: "message-1", kind: "user", itemID: "user-1", text: "Hello", transcribing: true, reveals: [] },
])
})
test("bounds message history", () => {
const state = Array.from({ length: 205 }, (_, index) => index).reduce(
(current, index) => transitionVoiceView(current, { type: "meta", text: String(index) }),
initialVoiceView(),
)
expect(state.messages).toHaveLength(200)
expect(state.messages[0]).toEqual({ key: "message-6", kind: "meta", text: "5" })
})
test("settles tool completion before or after insertion", () => {
const inserted = apply(
initialVoiceView(),
{ type: "tool.started", callID: "call-1", name: "read_session", input: {} },
{ type: "tool.done", callID: "call-1", output: { status: "ok" } },
)
expect(inserted.messages[0]).toEqual({
key: "message-1",
kind: "tool",
callID: "call-1",
name: "read_session",
input: {},
output: { status: "ok" },
})
})
})
@@ -1,277 +0,0 @@
import { describe, expect, test } from "bun:test"
import { initialVoiceState, transitionVoice, type VoiceEvent, type VoiceState } from "../src/voice-coordinator"
const apply = (state: VoiceState, ...events: ReadonlyArray<VoiceEvent>) =>
events.reduce(
(result, event) => {
const next = transitionVoice(result.state, event)
return { state: next.state, commands: [...result.commands, ...next.commands] }
},
{ state, commands: [] as Array<ReturnType<typeof transitionVoice>["commands"][number]> },
)
describe("VoiceCoordinator", () => {
test("delivers one notification at a time when the server becomes idle", () => {
const notification = { id: "notification-1", promptID: "prompt-1", text: "completed" }
const waiting = apply(
initialVoiceState("marin"),
{ type: "connection.ready" },
{ type: "user.committed" },
{ type: "notification.queued", notification },
)
expect(waiting.commands).toEqual([])
expect(transitionVoice(waiting.state, { type: "assistant.done", awaitingWork: false })).toEqual({
state: {
...waiting.state,
conversation: "waiting",
assistant: "idle",
notifications: [],
delivery: { notification, phase: "sending" },
},
commands: [
{ type: "assistant.finish", awaitingWork: false, notificationID: undefined },
{ type: "notification.send", notification },
],
})
})
test("keeps notifications queued while the user is speaking", () => {
const notification = { id: "notification-1", text: "completed" }
const result = apply(
initialVoiceState("marin"),
{ type: "connection.ready" },
{ type: "user.started" },
{ type: "notification.queued", notification },
)
expect(result.commands).toEqual([])
expect(result.state.notifications).toEqual([notification])
expect(transitionVoice(result.state, { type: "user.stopped" }).commands).toEqual([
{ type: "notification.send", notification },
])
})
test("keeps notifications queued until active tool calls finish", () => {
const notification = { id: "notification-1", text: "completed" }
const result = apply(
initialVoiceState("marin"),
{ type: "connection.ready" },
{ type: "tool.started", id: "call-1" },
{ type: "assistant.done", awaitingWork: false },
{ type: "notification.queued", notification },
)
expect(result.commands).toEqual([
{ type: "assistant.finish", awaitingWork: false, notificationID: undefined },
])
expect(transitionVoice(result.state, { type: "tool.finished", id: "call-1" }).commands).toEqual([
{ type: "notification.send", notification },
])
})
test("tracks tools by call ID and reconnects only after work and conversation settle", () => {
const busy = apply(
initialVoiceState("marin"),
{ type: "connection.ready" },
{ type: "tool.started", id: "call-1" },
{ type: "voice.selected", voice: "cedar" },
)
expect(busy.commands).toEqual([])
expect(transitionVoice(busy.state, { type: "tool.finished", id: "call-1" }).commands).toEqual([])
const idle = transitionVoice(transitionVoice(busy.state, { type: "tool.finished", id: "call-1" }).state, {
type: "assistant.done",
awaitingWork: false,
})
expect(idle.commands).toEqual([
{ type: "assistant.finish", awaitingWork: false, notificationID: undefined },
{ type: "connection.reconnect" },
])
expect(idle.state.connection).toBe("connecting")
})
test("a user commit closes assistant suppression and establishes a waiting boundary", () => {
const suppressed = apply(
initialVoiceState("marin"),
{ type: "connection.ready" },
{ type: "assistant.started" },
{ type: "assistant.suppressed" },
{ type: "user.committed" },
)
expect(suppressed.state.assistant).toBe("idle")
expect(suppressed.state.conversation).toBe("waiting")
})
test("requeues a notification when the adapter rejects it", () => {
const notification = { id: "notification-1", text: "completed" }
const sent = apply(
initialVoiceState("marin"),
{ type: "connection.ready" },
{ type: "notification.queued", notification },
)
const failed = transitionVoice(sent.state, { type: "notification.failed", id: notification.id })
expect(failed.state.notifications).toEqual([notification])
expect(failed.state.connection).toBe("connecting")
expect(failed.commands).toEqual([{ type: "connection.reconnect" }])
})
test("keeps a notification in flight until its playback completes", () => {
const notification = { id: "notification-1", promptID: "prompt-1", text: "completed" }
const sent = apply(
initialVoiceState("marin"),
{ type: "connection.ready" },
{
type: "notification.queued",
notification,
},
)
expect(sent.state.delivery).toEqual({ notification, phase: "sending" })
const accepted = transitionVoice(sent.state, { type: "notification.accepted", id: notification.id }).state
expect(accepted.delivery).toEqual({ notification, phase: "accepted" })
const speaking = transitionVoice(accepted, { type: "assistant.started" }).state
expect(speaking.delivery).toEqual({ notification, phase: "announcing" })
const generated = transitionVoice(speaking, { type: "assistant.done", awaitingWork: false })
expect(generated.state.delivery).toEqual({ notification, phase: "awaiting-playback" })
expect(generated.commands).toEqual([
{ type: "assistant.finish", awaitingWork: false, notificationID: notification.id },
])
expect(transitionVoice(generated.state, { type: "notification.announced", id: notification.id })).toEqual({
state: { ...generated.state, delivery: undefined },
commands: [{ type: "notification.delivered", notification }],
})
})
test("does not attribute an already-active assistant turn to a newly appended notification", () => {
const notification = { id: "notification-1", text: "completed" }
const active = apply(
initialVoiceState("marin"),
{ type: "connection.ready" },
{ type: "notification.queued", notification },
{ type: "assistant.started" },
{ type: "notification.accepted", id: notification.id },
).state
expect(active.delivery).toEqual({ notification, phase: "accepted" })
const existingDone = transitionVoice(active, { type: "assistant.done", awaitingWork: false })
expect(existingDone.state.delivery).toEqual({ notification, phase: "accepted" })
expect(existingDone.commands).toEqual([
{ type: "assistant.finish", awaitingWork: false, notificationID: undefined },
])
expect(transitionVoice(existingDone.state, { type: "assistant.started" }).state.delivery).toEqual({
notification,
phase: "announcing",
})
})
test("keeps assistant activity independent from a full-duplex user commit", () => {
const active = apply(initialVoiceState("marin"), { type: "connection.ready" }, { type: "assistant.started" }).state
const committed = transitionVoice(active, { type: "user.committed" }).state
expect(committed.assistant).toBe("active")
expect(committed.conversation).toBe("responding")
})
test("ignores stale notification callbacks", () => {
const notification = { id: "notification-2", text: "current" }
const sent = apply(
initialVoiceState("marin"),
{ type: "connection.ready" },
{ type: "notification.queued", notification },
).state
expect(transitionVoice(sent, { type: "notification.accepted", id: "notification-1" })).toEqual({
state: sent,
commands: [],
})
expect(transitionVoice(sent, { type: "notification.failed", id: "notification-1" })).toEqual({
state: sent,
commands: [],
})
expect(transitionVoice(sent, { type: "notification.announced", id: "notification-1" })).toEqual({
state: sent,
commands: [],
})
})
test("waits for one announcement playback before sending the next", () => {
const first = { id: "notification-1", text: "first" }
const second = { id: "notification-2", text: "second" }
const sent = apply(
initialVoiceState("marin"),
{ type: "connection.ready" },
{ type: "notification.queued", notification: first },
{ type: "notification.queued", notification: second },
{ type: "notification.accepted", id: first.id },
{ type: "assistant.started" },
{ type: "assistant.done", awaitingWork: false },
)
expect(sent.state.delivery).toEqual({ notification: first, phase: "awaiting-playback" })
expect(sent.state.notifications).toEqual([second])
expect(transitionVoice(sent.state, { type: "notification.announced", id: first.id })).toEqual({
state: {
...sent.state,
conversation: "waiting",
delivery: { notification: second, phase: "sending" },
notifications: [],
},
commands: [
{ type: "notification.delivered", notification: first },
{ type: "notification.send", notification: second },
],
})
})
test("keeps an announcement active across tool work", () => {
const notification = { id: "notification-1", text: "completed" }
const announcing = apply(
initialVoiceState("marin"),
{ type: "connection.ready" },
{ type: "notification.queued", notification },
{ type: "notification.accepted", id: notification.id },
{ type: "assistant.started" },
).state
expect(transitionVoice(announcing, { type: "assistant.done", awaitingWork: true }).state.delivery).toEqual({
notification,
phase: "announcing",
})
})
test("requeues an interrupted announcement and reconnects before retrying", () => {
const notification = { id: "notification-1", text: "completed" }
const awaitingPlayback = apply(
initialVoiceState("marin"),
{ type: "connection.ready" },
{ type: "notification.queued", notification },
{ type: "notification.accepted", id: notification.id },
{ type: "assistant.started" },
{ type: "assistant.done", awaitingWork: false },
).state
const interrupted = transitionVoice(awaitingPlayback, {
type: "notification.interrupted",
id: notification.id,
})
expect(interrupted.state.connection).toBe("connecting")
expect(interrupted.state.notifications).toEqual([notification])
expect(interrupted.commands).toEqual([{ type: "connection.reconnect" }])
})
test("holds an interrupted announcement reconnect until barge-in speech stops", () => {
const notification = { id: "notification-1", text: "completed" }
const announcing = apply(
initialVoiceState("marin"),
{ type: "connection.ready" },
{ type: "notification.queued", notification },
{ type: "notification.accepted", id: notification.id },
{ type: "assistant.started" },
).state
const barged = transitionVoice(announcing, { type: "user.started", bargeIn: true })
expect(barged.state.notifications).toEqual([notification])
expect(barged.state.userSpeaking).toBe(true)
expect(barged.state.connection).toBe("ready")
expect(barged.commands).toEqual([{ type: "assistant.interrupt" }])
expect(transitionVoice(barged.state, { type: "user.stopped" }).commands).toEqual([
{ type: "connection.reconnect" },
])
})
})
-233
View File
@@ -1,233 +0,0 @@
import { expect, mock, test } from "bun:test"
import type { AudioSession } from "../src/audio-session"
import type { VoiceConnection, VoiceProtocol, VoiceProtocolEvent, VoiceProtocolOptions } from "../src/protocol"
import type { VoiceUI } from "../src/ui"
import { createVoiceSession } from "../src/voice-session"
test("ignores events from a replaced connection", async () => {
const setup = makeSession()
setup.session.start()
setup.connections[0].emit({ type: "ready" })
setup.session.cycleVoice()
await Bun.sleep(0)
expect(setup.connections).toHaveLength(2)
setup.connections[0].emit({ type: "assistant.transcript.delta", delta: "stale" })
setup.connections[1].emit({ type: "assistant.transcript.delta", delta: "current" })
expect(setup.assistantText).toEqual(["current"])
})
test("resolves work against the connection that requested it", async () => {
const setup = makeSession()
setup.session.start()
setup.connections[0].emit({ type: "ready" })
setup.connections[0].emit({ type: "tool.started", id: "call-1", name: "find_sessions" })
setup.connections[0].emit({
type: "work.requested",
request: { id: "call-1", name: "find_sessions", input: { query: null } },
})
await Bun.sleep(0)
expect(setup.resolvedWork).toEqual([
{
request: { id: "call-1", name: "find_sessions", input: { query: null } },
output: { status: "ok" },
},
])
})
test("acknowledges a durable notification only after playback", async () => {
const setup = makeSession()
setup.session.start()
setup.connections[0].emit({ type: "ready" })
setup.session.queueNotification({
notification: {
type: "opencode.prompt.completed",
session_id: "session-1",
prompt_id: "prompt-1",
status: "completed",
text: "done",
},
receipt: { sessionID: "session-1", promptID: "prompt-1" },
})
await Bun.sleep(0)
setup.connections[0].emit({ type: "assistant.transcript.delta", delta: "Finished." })
setup.connections[0].emit({ type: "assistant.done", awaitingWork: false })
expect(setup.delivered).toEqual([])
setup.finishPlayback?.("played")
await Bun.sleep(0)
expect(setup.delivered).toEqual(["prompt-1"])
})
test("retries a notification whose playback was interrupted", async () => {
const setup = makeSession()
setup.session.start()
setup.connections[0].emit({ type: "ready" })
setup.session.queueNotification({
notification: {
type: "opencode.prompt.completed",
session_id: "session-1",
prompt_id: "prompt-1",
status: "completed",
text: "done",
},
receipt: { sessionID: "session-1", promptID: "prompt-1" },
})
await Bun.sleep(0)
setup.connections[0].emit({ type: "assistant.transcript.delta", delta: "Finished." })
setup.connections[0].emit({ type: "assistant.done", awaitingWork: false })
setup.finishPlayback?.("interrupted")
await Bun.sleep(0)
expect(setup.delivered).toEqual([])
expect(setup.connections).toHaveLength(2)
setup.connections[1].emit({ type: "ready" })
await Bun.sleep(0)
expect(setup.connections[1].notifyCalls()).toBe(1)
})
test("closes owned resources once", async () => {
const setup = makeSession()
setup.session.start()
await Promise.all([setup.session.close(), setup.session.close()])
expect(setup.audioClose).toHaveBeenCalledTimes(1)
expect(setup.toolClose).toHaveBeenCalledTimes(1)
expect(setup.connections[0].closeCalls()).toBe(1)
})
test("does not acknowledge nonterminal prompt notifications", async () => {
const setup = makeSession()
setup.session.start()
setup.connections[0].emit({ type: "ready" })
setup.session.queueNotification({
notification: {
type: "opencode.prompt.blocked",
prompt_id: "prompt-1",
blocker: "permission",
session_id: "session-1",
request_id: "permission-1",
action: "read",
resources: ["file.ts"],
},
})
await Bun.sleep(0)
setup.connections[0].emit({ type: "assistant.transcript.delta", delta: "Permission needed." })
setup.connections[0].emit({ type: "assistant.done", awaitingWork: false })
setup.finishPlayback?.("played")
await Bun.sleep(0)
expect(setup.delivered).toEqual([])
})
function makeSession() {
const connections: Array<
VoiceConnection & { emit(event: VoiceProtocolEvent): void; notifyCalls(): number; closeCalls(): number }
> = []
const assistantText: string[] = []
const resolvedWork: Array<unknown> = []
const delivered: string[] = []
const audioClose = mock(() => {})
const toolClose = mock(async () => {})
let finishPlayback: ((outcome: "played" | "interrupted" | "inaudible") => void) | undefined
const ui = {
meta: mock(() => {}),
userSpeaking: mock(() => {}),
userReset: mock(() => {}),
userAudioLevel: mock(() => {}),
userCommitted: mock(() => {}),
userTranscript: mock(() => {}),
assistantAudio: mock(() => {}),
assistantPlaybackStopped: mock(() => {}),
assistantDelta: (text: string) => assistantText.push(text),
assistantTranscript: mock(() => {}),
assistantDone: mock(() => {}),
toolStart: mock(() => {}),
toolDone: mock(() => {}),
setStatus: mock(() => {}),
close: mock(() => {}),
} satisfies VoiceUI
const audio = {
fullDuplex: false,
microphoneMuted: false,
speakerMuted: false,
start: mock(async () => {}),
toggleMicrophone: mock(() => false),
toggleSpeaker: mock(() => false),
isPlaying: () => false,
play: mock(() => {}),
finishPlayback: () => {
const result = Promise.withResolvers<"played" | "interrupted" | "inaudible">()
finishPlayback = result.resolve
return result.promise
},
flushPlayback: mock(() => {}),
noteUserCommitted: mock(() => {}),
noteUserTranscript: mock(() => {}),
close: audioClose,
} satisfies AudioSession
const protocol = {
name: "realtime",
inputActivity: "server",
capabilities: { textInput: true, interruption: true, delegation: false },
connect(options: VoiceProtocolOptions) {
const notify = mock(async () => true)
const close = mock(async () => {})
const connection = {
appendAudio: mock(() => {}),
sendText: mock(() => {}),
resolveWork: (request, output) => resolvedWork.push({ request, output }),
resolveDelegation: mock(() => {}),
notify,
interrupt: () => true,
close,
emit: options.onEvent,
notifyCalls: () => notify.mock.calls.length,
closeCalls: () => close.mock.calls.length,
} satisfies VoiceConnection & {
emit(event: VoiceProtocolEvent): void
notifyCalls(): number
closeCalls(): number
}
connections.push(connection)
return connection
},
} satisfies VoiceProtocol
const session = createVoiceSession({
protocol,
connection: {
apiKey: "test",
model: "voice-test",
instructions: "test",
tools: [],
fullDuplex: false,
debug: false,
},
initialVoice: "marin",
voices: ["marin", "cedar"],
ui,
audio,
tools: {
execute: async () => ({ output: { status: "ok" } }),
acknowledge: async (receipt) => {
delivered.push(receipt.promptID)
},
close: toolClose,
},
delegate: async () => "done",
onClosed: () => {},
})
return {
session,
connections,
assistantText,
resolvedWork,
delivered,
audioClose,
toolClose,
get finishPlayback() {
return finishPlayback
},
}
}
-11
View File
@@ -1,11 +0,0 @@
{
"$schema": "https://json.schemastore.org/tsconfig",
"extends": "@tsconfig/bun/tsconfig.json",
"compilerOptions": {
"jsx": "preserve",
"jsxImportSource": "@opentui/solid",
"noUncheckedIndexedAccess": false,
"noUnusedLocals": true
},
"include": ["src", "test"]
}