mirror of
https://github.com/anomalyco/opencode.git
synced 2026-07-31 07:26:47 -04:00
Compare commits
1 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| b1a61aaf55 |
@@ -986,25 +986,6 @@
|
||||
"@typescript/native-preview": "catalog:",
|
||||
},
|
||||
},
|
||||
"packages/voice": {
|
||||
"name": "@opencode-ai/voice",
|
||||
"version": "0.0.0",
|
||||
"dependencies": {
|
||||
"@opencode-ai/client": "workspace:*",
|
||||
"@opencode-ai/plugin": "workspace:*",
|
||||
"@opentui/core": "catalog:",
|
||||
"@opentui/solid": "catalog:",
|
||||
"effect": "catalog:",
|
||||
"opentui-spinner": "catalog:",
|
||||
"solid-js": "catalog:",
|
||||
},
|
||||
"devDependencies": {
|
||||
"@tsconfig/bun": "catalog:",
|
||||
"@types/bun": "catalog:",
|
||||
"@typescript/native-preview": "catalog:",
|
||||
"typescript": "catalog:",
|
||||
},
|
||||
},
|
||||
"packages/web": {
|
||||
"name": "@opencode-ai/web",
|
||||
"version": "1.18.4",
|
||||
@@ -2096,8 +2077,6 @@
|
||||
|
||||
"@opencode-ai/util": ["@opencode-ai/util@workspace:packages/util"],
|
||||
|
||||
"@opencode-ai/voice": ["@opencode-ai/voice@workspace:packages/voice"],
|
||||
|
||||
"@opencode-ai/web": ["@opencode-ai/web@workspace:packages/web"],
|
||||
|
||||
"@opencode-ai/www": ["@opencode-ai/www@workspace:packages/www"],
|
||||
|
||||
@@ -123,6 +123,7 @@ export const mergeGenerationOptions = (...items: ReadonlyArray<GenerationOptions
|
||||
|
||||
export class ModelLimits extends Schema.Class<ModelLimits>("LLM.ModelLimits")({
|
||||
context: Schema.optional(Schema.Number),
|
||||
input: Schema.optional(Schema.Number),
|
||||
output: Schema.optional(Schema.Number),
|
||||
}) {}
|
||||
|
||||
|
||||
@@ -88,12 +88,8 @@ function SessionTabSlot(props: {
|
||||
data-titlebar-tab-slot
|
||||
data-tab-key={props.id}
|
||||
data-active={props.active()}
|
||||
class="relative flex min-w-7 flex-shrink"
|
||||
classList={{
|
||||
hidden: !session() && !missingSession() && !persisted()?.title,
|
||||
"w-72 max-w-72": props.active(),
|
||||
"w-56 max-w-56": !props.active(),
|
||||
}}
|
||||
class="relative flex w-56 min-w-7 max-w-56 flex-shrink"
|
||||
classList={{ hidden: !session() && !missingSession() && !persisted()?.title }}
|
||||
>
|
||||
<TabNavItem
|
||||
ref={(el) => {
|
||||
@@ -148,11 +144,7 @@ function DraftTabSlot(props: {
|
||||
data-titlebar-tab-slot
|
||||
data-tab-key={props.id}
|
||||
data-active={props.active()}
|
||||
class="relative flex min-w-7 flex-shrink"
|
||||
classList={{
|
||||
"w-72 max-w-72": props.active(),
|
||||
"w-56 max-w-56": !props.active(),
|
||||
}}
|
||||
class="relative flex w-56 min-w-7 max-w-56 flex-shrink"
|
||||
>
|
||||
<DraftTabItem
|
||||
ref={(el) => {
|
||||
|
||||
@@ -1,6 +1,5 @@
|
||||
#!/usr/bin/env bun
|
||||
|
||||
import "@opentui/solid/runtime-plugin-support"
|
||||
import { NodeRuntime, NodeServices } from "@effect/platform-node"
|
||||
import { Effect } from "effect"
|
||||
import { Commands } from "./commands/commands"
|
||||
|
||||
@@ -37,6 +37,7 @@ export async function ensure(options: EnsureOptions = {}): Promise<Endpoint> {
|
||||
let announced = false
|
||||
let lastSpawn = 0
|
||||
let spawnDelay = 5_000
|
||||
let ownerHeld = false
|
||||
|
||||
const announce = (reason: "missing" | "version-mismatch", previousVersion?: string) => {
|
||||
if (announced) return
|
||||
@@ -64,6 +65,7 @@ export async function ensure(options: EnsureOptions = {}): Promise<Endpoint> {
|
||||
const registration = await registered(options.file, true)
|
||||
|
||||
if (registration.service !== undefined) {
|
||||
ownerHeld = false
|
||||
spawnDelay = 5_000
|
||||
const service = registration.service
|
||||
const compatible = !service.legacy && (options.version === undefined || service.version === options.version)
|
||||
@@ -80,6 +82,7 @@ export async function ensure(options: EnsureOptions = {}): Promise<Endpoint> {
|
||||
if (failure !== undefined) throw failure
|
||||
const finished = [...contenders].filter(contenderFinished)
|
||||
if (finished.some((item) => item.child.exitCode === 0)) {
|
||||
ownerHeld = true
|
||||
spawnDelay = Math.min(spawnDelay * 2, 30_000)
|
||||
}
|
||||
finished.forEach((item) => contenders.delete(item))
|
||||
|
||||
@@ -332,7 +332,7 @@ function modelFromLanguage(info: ModelV2.Info, language: LanguageModelV3) {
|
||||
body: projected.body === undefined ? undefined : { ...projected.body },
|
||||
headers: info.headers,
|
||||
},
|
||||
limits: { context: info.limit.context, output: info.limit.output },
|
||||
limits: { context: info.limit.context, input: info.limit.input, output: info.limit.output },
|
||||
providerOptions,
|
||||
},
|
||||
body: {
|
||||
|
||||
@@ -81,7 +81,7 @@ const withDefaults = (model: ModelV2.Info, route: AnyRoute) =>
|
||||
headers: providerHeaders(model),
|
||||
providerOptions: providerOptions(model),
|
||||
http: model.body === undefined ? undefined : { body: model.body },
|
||||
limits: { context: model.limit.context, output: model.limit.output },
|
||||
limits: { context: model.limit.context, input: model.limit.input, output: model.limit.output },
|
||||
})
|
||||
|
||||
const providerHeaders = (model: ModelV2.Info) => {
|
||||
@@ -206,7 +206,7 @@ export const fromCatalogModel = (
|
||||
...nativeCredentialSettings(specifier, credential),
|
||||
headers: resolved.headers,
|
||||
body: resolved.body,
|
||||
limits: { context: resolved.limit.context, output: resolved.limit.output },
|
||||
limits: { context: resolved.limit.context, input: resolved.limit.input, output: resolved.limit.output },
|
||||
}
|
||||
return yield* Effect.try({
|
||||
try: () => {
|
||||
|
||||
@@ -180,6 +180,12 @@ export const OpenAIPlugin = define({
|
||||
})
|
||||
yield* load()
|
||||
yield* ctx.catalog.transform((evt) => {
|
||||
const codex = evt.provider.get(ProviderV2.ID.make("openai-codex"))
|
||||
if (codex)
|
||||
for (const model of codex.models.values())
|
||||
evt.model.update(codex.provider.id, model.id, (draft) => {
|
||||
draft.enabled = false
|
||||
})
|
||||
for (const item of evt.provider.list()) {
|
||||
if (!ProviderV2.isAISDK(item.provider.package)) continue
|
||||
if (ProviderV2.packageName(item.provider.package) !== "@ai-sdk/openai") continue
|
||||
@@ -205,6 +211,8 @@ export const OpenAIPlugin = define({
|
||||
draft.enabled = false
|
||||
return
|
||||
}
|
||||
const route = codex?.models.get(draft.id)
|
||||
if (route) draft.limit = { ...route.limit }
|
||||
draft.cost = []
|
||||
})
|
||||
}
|
||||
|
||||
@@ -351,10 +351,12 @@ const make = (dependencies: Dependencies) => {
|
||||
)
|
||||
if (!last) return false
|
||||
const output = Math.min(input.model.route.defaults.limits?.output ?? 0, OUTPUT_TOKEN_MAX)
|
||||
const inputLimit = input.model.route.defaults.limits?.input
|
||||
const limit = inputLimit === undefined ? context - (output || config.buffer) : Math.max(0, inputLimit - config.buffer)
|
||||
const used =
|
||||
last.tokens.input + last.tokens.output + last.tokens.reasoning + last.tokens.cache.read + last.tokens.cache.write
|
||||
if (used <= 0) return false
|
||||
return used >= context - (output || config.buffer)
|
||||
return used >= limit
|
||||
}
|
||||
const compactManual = Effect.fn("SessionCompaction.compactManual")(function* (input: ManualInput) {
|
||||
const content = planContent(input.messages, config.tokens)
|
||||
|
||||
@@ -654,12 +654,6 @@ const layer = Layer.effectDiscard(
|
||||
input: event.data.input,
|
||||
timeCreated: event.created,
|
||||
})
|
||||
yield* db
|
||||
.update(SessionTable)
|
||||
.set({ time_updated: DateTime.toEpochMillis(event.created) })
|
||||
.where(eq(SessionTable.id, event.data.sessionID))
|
||||
.run()
|
||||
.pipe(Effect.orDie)
|
||||
}),
|
||||
)
|
||||
yield* events.project(SessionEvent.Compaction.Admitted, (event) =>
|
||||
|
||||
@@ -17,6 +17,7 @@ interface ModelOptions {
|
||||
readonly headers?: ModelV2.Info["headers"]
|
||||
readonly body?: ModelV2.Info["body"]
|
||||
readonly variants?: ModelV2.Info["variants"]
|
||||
readonly limit?: ModelV2.Info["limit"]
|
||||
}
|
||||
|
||||
const model = (packageName: string | undefined, options: ModelOptions = {}) =>
|
||||
@@ -36,7 +37,7 @@ const model = (packageName: string | undefined, options: ModelOptions = {}) =>
|
||||
cost: [],
|
||||
status: "active",
|
||||
enabled: true,
|
||||
limit: { context: 100, output: 20 },
|
||||
limit: options.limit ?? { context: 100, output: 20 },
|
||||
})
|
||||
|
||||
describe("ModelResolver", () => {
|
||||
@@ -44,6 +45,7 @@ describe("ModelResolver", () => {
|
||||
Effect.gen(function* () {
|
||||
const catalog = model(ProviderV2.aisdk("@ai-sdk/openai"), {
|
||||
settings: { baseURL: "https://openai.example/v1" },
|
||||
limit: { context: 100, input: 70, output: 20 },
|
||||
})
|
||||
const resolved = yield* ModelResolver.fromCatalogModel(catalog)
|
||||
|
||||
@@ -55,7 +57,7 @@ describe("ModelResolver", () => {
|
||||
endpoint: { baseURL: "https://openai.example/v1" },
|
||||
defaults: {
|
||||
headers: { "x-test": "header" },
|
||||
limits: { context: 100, output: 20 },
|
||||
limits: { context: 100, input: 70, output: 20 },
|
||||
http: { body: { custom_extension: { enabled: true } } },
|
||||
},
|
||||
})
|
||||
|
||||
@@ -164,6 +164,16 @@ describe("OpenAIPlugin", () => {
|
||||
const catalog = yield* Catalog.Service
|
||||
const credentials = yield* Credential.Service
|
||||
yield* catalog.transform((catalog) => {
|
||||
const codex = ProviderV2.Info.make({
|
||||
...ProviderV2.Info.empty(ProviderV2.ID.make("openai-codex")),
|
||||
package: ProviderV2.aisdk("@ai-sdk/openai"),
|
||||
})
|
||||
catalog.provider.update(codex.id, (draft) => {
|
||||
draft.package = codex.package
|
||||
})
|
||||
catalog.model.update(codex.id, ModelV2.ID.make("gpt-5.6-sol"), (model) => {
|
||||
model.limit = { context: 400_000, input: 272_000, output: 128_000 }
|
||||
})
|
||||
const item = ProviderV2.Info.make({
|
||||
...ProviderV2.Info.empty(ProviderV2.ID.openai),
|
||||
package: ProviderV2.aisdk("@ai-sdk/openai"),
|
||||
@@ -189,7 +199,9 @@ describe("OpenAIPlugin", () => {
|
||||
model.body = { reasoning: { mode: "pro" } }
|
||||
})
|
||||
catalog.model.update(item.id, ModelV2.ID.make("gpt-5.6"), () => {})
|
||||
catalog.model.update(item.id, ModelV2.ID.make("gpt-5.6-sol"), () => {})
|
||||
catalog.model.update(item.id, ModelV2.ID.make("gpt-5.6-sol"), (model) => {
|
||||
model.limit = { context: 1_050_000, input: 922_000, output: 128_000 }
|
||||
})
|
||||
catalog.model.update(item.id, ModelV2.ID.make("gpt-4.1"), () => {})
|
||||
})
|
||||
yield* credentials.create({
|
||||
@@ -215,9 +227,13 @@ describe("OpenAIPlugin", () => {
|
||||
false,
|
||||
)
|
||||
expect(required(yield* catalog.model.get(ProviderV2.ID.openai, ModelV2.ID.make("gpt-5.6"))).enabled).toBe(false)
|
||||
expect(required(yield* catalog.model.get(ProviderV2.ID.openai, ModelV2.ID.make("gpt-5.6-sol"))).enabled).toBe(
|
||||
true,
|
||||
)
|
||||
const sol = required(yield* catalog.model.get(ProviderV2.ID.openai, ModelV2.ID.make("gpt-5.6-sol")))
|
||||
expect(sol.enabled).toBe(true)
|
||||
expect(sol.limit).toEqual({ context: 400_000, input: 272_000, output: 128_000 })
|
||||
expect(
|
||||
required(yield* catalog.model.get(ProviderV2.ID.make("openai-codex"), ModelV2.ID.make("gpt-5.6-sol")))
|
||||
.enabled,
|
||||
).toBe(false)
|
||||
expect(required(yield* catalog.model.get(ProviderV2.ID.openai, ModelV2.ID.make("gpt-4.1"))).enabled).toBe(false)
|
||||
}),
|
||||
)
|
||||
@@ -227,6 +243,16 @@ describe("OpenAIPlugin", () => {
|
||||
const catalog = yield* Catalog.Service
|
||||
const credentials = yield* Credential.Service
|
||||
yield* catalog.transform((catalog) => {
|
||||
const codex = ProviderV2.Info.make({
|
||||
...ProviderV2.Info.empty(ProviderV2.ID.make("openai-codex")),
|
||||
package: ProviderV2.aisdk("@ai-sdk/openai"),
|
||||
})
|
||||
catalog.provider.update(codex.id, (draft) => {
|
||||
draft.package = codex.package
|
||||
})
|
||||
catalog.model.update(codex.id, ModelV2.ID.make("gpt-5.5"), (model) => {
|
||||
model.limit = { context: 400_000, input: 272_000, output: 128_000 }
|
||||
})
|
||||
const item = ProviderV2.Info.make({
|
||||
...ProviderV2.Info.empty(ProviderV2.ID.openai),
|
||||
package: ProviderV2.aisdk("@ai-sdk/openai"),
|
||||
@@ -234,7 +260,9 @@ describe("OpenAIPlugin", () => {
|
||||
catalog.provider.update(item.id, (draft) => {
|
||||
draft.package = item.package
|
||||
})
|
||||
catalog.model.update(item.id, ModelV2.ID.make("gpt-5.5"), () => {})
|
||||
catalog.model.update(item.id, ModelV2.ID.make("gpt-5.5"), (model) => {
|
||||
model.limit = { context: 1_050_000, input: 922_000, output: 128_000 }
|
||||
})
|
||||
catalog.model.update(item.id, ModelV2.ID.make("gpt-4.1"), () => {})
|
||||
})
|
||||
yield* credentials.create({
|
||||
@@ -243,7 +271,9 @@ describe("OpenAIPlugin", () => {
|
||||
})
|
||||
yield* addPlugin()
|
||||
|
||||
expect(required(yield* catalog.model.get(ProviderV2.ID.openai, ModelV2.ID.make("gpt-5.5"))).enabled).toBe(true)
|
||||
const model = required(yield* catalog.model.get(ProviderV2.ID.openai, ModelV2.ID.make("gpt-5.5")))
|
||||
expect(model.enabled).toBe(true)
|
||||
expect(model.limit).toEqual({ context: 1_050_000, input: 922_000, output: 128_000 })
|
||||
expect(required(yield* catalog.model.get(ProviderV2.ID.openai, ModelV2.ID.make("gpt-4.1"))).enabled).toBe(true)
|
||||
}),
|
||||
)
|
||||
|
||||
@@ -16,11 +16,15 @@ import { SessionRunnerModel } from "@opencode-ai/core/session/runner/model"
|
||||
import { SessionTable } from "@opencode-ai/core/session/sql"
|
||||
import { SessionStore } from "@opencode-ai/core/session/store"
|
||||
import { SessionV2 } from "@opencode-ai/core/session"
|
||||
import { AgentV2 } from "@opencode-ai/core/agent"
|
||||
import { Location } from "@opencode-ai/core/location"
|
||||
import { Project } from "@opencode-ai/core/project"
|
||||
import { ProjectTable } from "@opencode-ai/core/project/sql"
|
||||
import { App } from "@opencode-ai/core/app"
|
||||
import { AbsolutePath } from "@opencode-ai/core/schema"
|
||||
import { Money } from "@opencode-ai/schema/money"
|
||||
import { ModelV2 } from "@opencode-ai/core/model"
|
||||
import { ProviderV2 } from "@opencode-ai/core/provider"
|
||||
import { DateTime, Effect, Fiber, Layer, Stream } from "effect"
|
||||
import { asc, eq } from "drizzle-orm"
|
||||
import { testEffect } from "./lib/effect"
|
||||
@@ -131,6 +135,40 @@ test("compaction prompt requires the checkpoint headings in order", () => {
|
||||
expect(prompt).toContain("Keep every section, even when empty.")
|
||||
})
|
||||
|
||||
it.effect("auto compaction uses the model input limit", () =>
|
||||
Effect.gen(function* () {
|
||||
const compaction = yield* SessionCompaction.Service
|
||||
const session = SessionV2.Info.make({
|
||||
id: SessionV2.ID.make("ses_input_limit"),
|
||||
projectID: Project.ID.global,
|
||||
title: "Input limit",
|
||||
cost: Money.USD.zero,
|
||||
tokens: { input: 0, output: 0, reasoning: 0, cache: { read: 0, write: 0 } },
|
||||
time: { created: DateTime.makeUnsafe(0), updated: DateTime.makeUnsafe(0) },
|
||||
location: Location.Ref.make({ directory: AbsolutePath.make("/project") }),
|
||||
})
|
||||
const route = Model.make({
|
||||
id: "codex-model",
|
||||
provider: "openai",
|
||||
route: OpenAIChat.route.with({ limits: { context: 400_000, input: 272_000, output: 128_000 } }),
|
||||
})
|
||||
const message = (input: number) =>
|
||||
SessionMessage.Assistant.make({
|
||||
id: SessionMessage.ID.create(),
|
||||
type: "assistant",
|
||||
agent: AgentV2.defaultID,
|
||||
model: { id: ModelV2.ID.make("codex-model"), providerID: ProviderV2.ID.openai },
|
||||
content: [],
|
||||
cost: Money.USD.zero,
|
||||
tokens: { input, output: 0, reasoning: 0, cache: { read: 0, write: 0 } },
|
||||
time: { created: DateTime.makeUnsafe(0) },
|
||||
})
|
||||
|
||||
expect(compaction.required({ session, model: route, cost: [], messages: [message(251_999)] })).toBe(false)
|
||||
expect(compaction.required({ session, model: route, cost: [], messages: [message(252_000)] })).toBe(true)
|
||||
}),
|
||||
)
|
||||
|
||||
it.effect("manual compaction summarizes short context instead of no-op", () =>
|
||||
Effect.gen(function* () {
|
||||
requests = []
|
||||
|
||||
@@ -171,28 +171,6 @@ describe("SessionV2.create", () => {
|
||||
}),
|
||||
)
|
||||
|
||||
it.effect("orders reused sessions by prompt admission time", () =>
|
||||
Effect.gen(function* () {
|
||||
const session = yield* SessionV2.Service
|
||||
const events = yield* EventV2.Service
|
||||
const { db } = yield* Database.Service
|
||||
const reused = yield* session.create({ location, title: "reused" })
|
||||
const newer = yield* session.create({ location, title: "newer" })
|
||||
yield* db.update(SessionTable).set({ time_updated: -2 }).where(eq(SessionTable.id, reused.id)).run()
|
||||
yield* db.update(SessionTable).set({ time_updated: -1 }).where(eq(SessionTable.id, newer.id)).run()
|
||||
|
||||
const admitted = yield* events.publish(SessionEvent.InputAdmitted, {
|
||||
sessionID: reused.id,
|
||||
inputID: SessionMessage.ID.create(),
|
||||
input: { type: "user", data: { text: "continue" }, delivery: "steer" },
|
||||
})
|
||||
|
||||
const page = yield* session.list({ directory: location.directory, parentID: null, order: "desc" })
|
||||
expect(page.data.map((item) => item.id)).toEqual([reused.id, newer.id])
|
||||
expect(page.data[0]!.time.updated).toEqual(admitted.created)
|
||||
}),
|
||||
)
|
||||
|
||||
it.effect("filters direct child sessions by parent ID", () =>
|
||||
Effect.gen(function* () {
|
||||
const session = yield* SessionV2.Service
|
||||
|
||||
@@ -29,9 +29,10 @@ export function contextUsage(
|
||||
last.tokens.input + last.tokens.output + last.tokens.reasoning + last.tokens.cache.read + last.tokens.cache.write
|
||||
if (tokens <= 0) return
|
||||
const model = models?.find((model) => model.providerID === last.model.providerID && model.id === last.model.id)
|
||||
const limit = model?.limit.input ?? model?.limit.context
|
||||
return {
|
||||
tokens,
|
||||
percent: model?.limit.context ? Math.round((tokens / model.limit.context) * 100) : undefined,
|
||||
percent: limit ? Math.round((tokens / limit) * 100) : undefined,
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
import { describe, expect, test } from "bun:test"
|
||||
import type { SessionMessageInfo } from "@opencode-ai/client"
|
||||
import { isDefaultTitle, lastAssistantWithUsage } from "../../src/util/session"
|
||||
import type { ModelInfo, SessionMessageInfo } from "@opencode-ai/client"
|
||||
import { contextUsage, isDefaultTitle, lastAssistantWithUsage } from "../../src/util/session"
|
||||
|
||||
const assistant = (id: string, input: number): SessionMessageInfo => ({
|
||||
id,
|
||||
@@ -45,4 +45,19 @@ describe("util.session", () => {
|
||||
messages.push(assistant("msg_after", 5))
|
||||
expect(lastAssistantWithUsage(messages)?.tokens.input).toBe(5)
|
||||
})
|
||||
|
||||
test("reports usage against the effective input window", () => {
|
||||
const models = [
|
||||
{
|
||||
id: "model",
|
||||
providerID: "provider",
|
||||
limit: { context: 400_000, input: 272_000, output: 128_000 },
|
||||
} as unknown as ModelInfo,
|
||||
]
|
||||
|
||||
expect(contextUsage([assistant("msg_input_limit", 252_000)], models)).toEqual({
|
||||
tokens: 252_000,
|
||||
percent: 93,
|
||||
})
|
||||
})
|
||||
})
|
||||
|
||||
@@ -1 +0,0 @@
|
||||
.build/
|
||||
@@ -1,4 +0,0 @@
|
||||
preload = ["@opentui/solid/preload"]
|
||||
|
||||
[test]
|
||||
preload = ["@opentui/solid/preload"]
|
||||
@@ -1,29 +0,0 @@
|
||||
{
|
||||
"$schema": "https://json.schemastore.org/package.json",
|
||||
"name": "@opencode-ai/voice",
|
||||
"version": "0.0.0",
|
||||
"private": true,
|
||||
"description": "Prototype voice control for OpenCode via the OpenAI Realtime and Live APIs",
|
||||
"type": "module",
|
||||
"scripts": {
|
||||
"bench:audio": "bun scripts/bench-speaker-queue.ts",
|
||||
"spike": "bun run --no-orphans --conditions=browser src/spike.ts",
|
||||
"test": "bun test",
|
||||
"typecheck": "tsgo --noEmit"
|
||||
},
|
||||
"devDependencies": {
|
||||
"@tsconfig/bun": "catalog:",
|
||||
"@types/bun": "catalog:",
|
||||
"@typescript/native-preview": "catalog:",
|
||||
"typescript": "catalog:"
|
||||
},
|
||||
"dependencies": {
|
||||
"@opencode-ai/client": "workspace:*",
|
||||
"@opencode-ai/plugin": "workspace:*",
|
||||
"@opentui/core": "catalog:",
|
||||
"@opentui/solid": "catalog:",
|
||||
"effect": "catalog:",
|
||||
"opentui-spinner": "catalog:",
|
||||
"solid-js": "catalog:"
|
||||
}
|
||||
}
|
||||
@@ -1,38 +0,0 @@
|
||||
# Speaker Queue Performance
|
||||
|
||||
## Goal
|
||||
|
||||
Reduce render-path latency and variance that can produce audible output glitches.
|
||||
|
||||
## Benchmark
|
||||
|
||||
Run `bun run bench:audio` from `packages/voice`.
|
||||
|
||||
The benchmark compiles the production `SpeakerQueue`, performs one warmup run and seven measured runs, and reports callback latency for a 512-frame pop followed by a same-sized producer push. The queue starts with one second of 24 kHz PCM audio.
|
||||
|
||||
## Metrics
|
||||
|
||||
- Primary: `speaker_queue_p99_ns`
|
||||
- Secondary: `speaker_queue_median_ns`, `speaker_queue_worst_ns`
|
||||
|
||||
## Scope
|
||||
|
||||
- `src/duplex-audio.swift`
|
||||
- `scripts/bench-speaker-queue.ts`
|
||||
|
||||
## Experiments
|
||||
|
||||
### Baseline: allocating `Data` queue
|
||||
|
||||
- Median: 3,750 ns
|
||||
- p99: 5,833 ns
|
||||
- Worst: 92,209 ns median across runs; one run reached 7,497,334 ns
|
||||
- Decision: baseline
|
||||
|
||||
### Experiment 1: preallocated circular queue
|
||||
|
||||
- Hypothesis: rendering directly from preallocated storage will reduce callback tail latency by avoiding an array allocation and `Data.removeFirst()` on every callback.
|
||||
- Median: 1,417 ns, down 62.2%
|
||||
- p99: 1,917 ns, down 67.1%
|
||||
- Worst: 32,709 ns median across runs, down 64.5%
|
||||
- Decision: keep
|
||||
@@ -1,43 +0,0 @@
|
||||
import { mkdtemp, rm } from "node:fs/promises"
|
||||
import { tmpdir } from "node:os"
|
||||
import { join } from "node:path"
|
||||
|
||||
const directory = await mkdtemp(join(tmpdir(), "opencode-voice-bench-"))
|
||||
const binary = join(directory, "speaker-queue")
|
||||
const source = Bun.fileURLToPath(new URL("../src/duplex-audio.swift", import.meta.url))
|
||||
const compile = Bun.spawn(["swiftc", "-O", "-D", "QUEUE_BENCHMARK", source, "-o", binary], {
|
||||
stdout: "ignore",
|
||||
stderr: "pipe",
|
||||
})
|
||||
if ((await compile.exited) !== 0) {
|
||||
console.error(await new Response(compile.stderr).text())
|
||||
await rm(directory, { recursive: true })
|
||||
process.exit(1)
|
||||
}
|
||||
|
||||
const runs: Array<Record<string, number>> = []
|
||||
for (let index = 0; index < 8; index++) {
|
||||
const process = Bun.spawn([binary], { stdout: "pipe", stderr: "inherit" })
|
||||
const output = await new Response(process.stdout).text()
|
||||
if ((await process.exited) !== 0) throw new Error(`benchmark run ${index + 1} failed`)
|
||||
if (index === 0) continue
|
||||
runs.push(
|
||||
Object.fromEntries(
|
||||
output
|
||||
.split("\n")
|
||||
.filter((line) => line.startsWith("METRIC "))
|
||||
.map((line) => {
|
||||
const [name, value] = line.slice(7).split("=")
|
||||
return [name!, Number(value)]
|
||||
}),
|
||||
),
|
||||
)
|
||||
}
|
||||
await rm(directory, { recursive: true })
|
||||
|
||||
for (const name of ["speaker_queue_median_ns", "speaker_queue_p99_ns", "speaker_queue_worst_ns"]) {
|
||||
const values = runs.map((run) => run[name]!).sort((a, b) => a - b)
|
||||
const median = values[Math.floor(values.length / 2)]!
|
||||
console.log(`${name}: median=${median} range=${values[0]}..${values.at(-1)}`)
|
||||
console.log(`METRIC ${name}=${median}`)
|
||||
}
|
||||
@@ -1,49 +0,0 @@
|
||||
export type TextReveal = { readonly offset: number; readonly at: number }
|
||||
|
||||
const CONNECTION_TRANSITION_MS = 480
|
||||
const REVEAL_DURATION_MS = 380
|
||||
const REVEAL_STAGGER_MS = 32
|
||||
const REVEAL_MAX_QUEUE_MS = 160
|
||||
export const REVEAL_WORD_LIMIT = 24
|
||||
|
||||
export function textRevealOpacity(now: number, start: number) {
|
||||
const progress = Math.max(0, Math.min(1, (now - start) / REVEAL_DURATION_MS))
|
||||
const eased = 1 - (1 - progress) ** 2
|
||||
return 0.16 + eased * 0.84
|
||||
}
|
||||
|
||||
export function transcriptionPulse(now: number) {
|
||||
return 0.5 + Math.sin(now / 220) * 0.5
|
||||
}
|
||||
|
||||
export function connectionMeterLevels(now: number, level: number, connectedAt?: number, startedAt = 0) {
|
||||
const transition = connectedAt === undefined ? 0 : Math.max(0, Math.min(1, (now - connectedAt) / CONNECTION_TRANSITION_MS))
|
||||
const head = ((now - startedAt) / 180) % 4
|
||||
return [0, 1, 2, 3].map((index) => {
|
||||
const distance = Math.abs(index - head)
|
||||
const wrappedDistance = Math.min(distance, 4 - distance)
|
||||
const loading = 0.08 + Math.max(0, Math.cos((wrappedDistance / 1.5) * (Math.PI / 2))) * 0.92
|
||||
const connected = level * (0.72 + Math.sin(now / 240 + index * 1.4) * 0.28)
|
||||
return loading + (connected - loading) * transition
|
||||
})
|
||||
}
|
||||
|
||||
export function connectionTransitioning(now: number, connectedAt?: number) {
|
||||
return connectedAt === undefined || now - connectedAt < CONNECTION_TRANSITION_MS
|
||||
}
|
||||
|
||||
export function scheduleTextReveal(previous: string, delta: string, now: number, revealAt: number) {
|
||||
const text = previous + delta
|
||||
const offsets = Array.from(delta.matchAll(/\S+/g))
|
||||
.map((match) => previous.length + match.index)
|
||||
.filter((offset) => offset === 0 || /\s/.test(text[offset - 1] ?? ""))
|
||||
.slice(-REVEAL_WORD_LIMIT)
|
||||
const start = Math.min(Math.max(revealAt, now), now + REVEAL_MAX_QUEUE_MS)
|
||||
const reveals = offsets.map((offset, word) => ({ offset, at: start + word * REVEAL_STAGGER_MS }))
|
||||
const nextRevealAt = reveals.length === 0 ? revealAt : reveals.at(-1)!.at + REVEAL_STAGGER_MS
|
||||
return {
|
||||
reveals,
|
||||
nextRevealAt,
|
||||
animationEndsAt: reveals.length === 0 ? now : reveals.at(-1)!.at + REVEAL_DURATION_MS,
|
||||
}
|
||||
}
|
||||
@@ -1,136 +0,0 @@
|
||||
import { mkdir } from "node:fs/promises"
|
||||
import { PCM_SAMPLE_RATE } from "./pcm"
|
||||
|
||||
export type AudioDevice = {
|
||||
readonly fullDuplex: boolean
|
||||
readonly mode: string
|
||||
start(onInput: (audio: Buffer) => void, onMeta: (text: string) => void): Promise<void>
|
||||
write(audio: Buffer): void
|
||||
flush(): void
|
||||
close(): void
|
||||
}
|
||||
|
||||
export async function createAudioDevice(options: {
|
||||
readonly duplex: boolean
|
||||
readonly speakers: boolean
|
||||
readonly debug: boolean
|
||||
}): Promise<AudioDevice> {
|
||||
const helper = await prepareHelper()
|
||||
if (helper) return appleDevice(helper, options)
|
||||
return soxDevice(options)
|
||||
}
|
||||
|
||||
function appleDevice(
|
||||
binary: string,
|
||||
options: { readonly duplex: boolean; readonly speakers: boolean },
|
||||
): AudioDevice {
|
||||
let process: ReturnType<typeof Bun.spawn> | undefined
|
||||
let sink: import("bun").FileSink | undefined
|
||||
return {
|
||||
fullDuplex: options.speakers || options.duplex,
|
||||
mode: options.speakers ? "duplex+aec" : options.duplex ? "duplex" : "half-duplex",
|
||||
async start(onInput, onMeta) {
|
||||
if (process) return
|
||||
const child = Bun.spawn([binary, ...(options.speakers ? ["--aec"] : [])], {
|
||||
stdin: "pipe",
|
||||
stdout: "pipe",
|
||||
stderr: "pipe",
|
||||
})
|
||||
process = child
|
||||
sink = child.stdin
|
||||
void forwardLines(child.stderr, onMeta)
|
||||
for await (const chunk of child.stdout) onInput(Buffer.from(chunk))
|
||||
process = undefined
|
||||
sink = undefined
|
||||
},
|
||||
write(audio) {
|
||||
void sink?.write(audio)
|
||||
void sink?.flush()
|
||||
},
|
||||
flush() {
|
||||
process?.kill("SIGUSR1")
|
||||
},
|
||||
close() {
|
||||
process?.kill()
|
||||
process = undefined
|
||||
sink = undefined
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
function soxDevice(options: { readonly duplex: boolean; readonly debug: boolean }): AudioDevice {
|
||||
const format = [
|
||||
"-q",
|
||||
"-t",
|
||||
"raw",
|
||||
"-r",
|
||||
String(PCM_SAMPLE_RATE),
|
||||
"-e",
|
||||
"signed-integer",
|
||||
"-b",
|
||||
"16",
|
||||
"-c",
|
||||
"1",
|
||||
]
|
||||
let recorder: ReturnType<typeof Bun.spawn> | undefined
|
||||
let player: ReturnType<typeof Bun.spawn> | undefined
|
||||
let sink: import("bun").FileSink | undefined
|
||||
let onMeta: ((text: string) => void) | undefined
|
||||
return {
|
||||
fullDuplex: options.duplex,
|
||||
mode: options.duplex ? "duplex (sox)" : "half-duplex (sox)",
|
||||
async start(onInput, report) {
|
||||
if (recorder) return
|
||||
onMeta = report
|
||||
const child = Bun.spawn(["rec", ...format, "-"], { stdout: "pipe", stderr: "ignore" })
|
||||
recorder = child
|
||||
for await (const chunk of child.stdout) onInput(Buffer.from(chunk))
|
||||
recorder = undefined
|
||||
},
|
||||
write(audio) {
|
||||
if (!player) {
|
||||
const child = Bun.spawn(["play", ...format, "-"], { stdin: "pipe", stderr: "ignore" })
|
||||
player = child
|
||||
sink = child.stdin
|
||||
if (options.debug) void child.exited.then((code) => onMeta?.(`[debug] play exited (${code})`))
|
||||
}
|
||||
void sink?.write(audio)
|
||||
void sink?.flush()
|
||||
},
|
||||
flush() {
|
||||
player?.kill()
|
||||
player = undefined
|
||||
sink = undefined
|
||||
},
|
||||
close() {
|
||||
recorder?.kill()
|
||||
player?.kill()
|
||||
recorder = undefined
|
||||
player = undefined
|
||||
sink = undefined
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
async function forwardLines(stream: ReadableStream<Uint8Array>, onLine: (line: string) => void) {
|
||||
const decoder = new TextDecoder()
|
||||
let remainder = ""
|
||||
for await (const chunk of stream) {
|
||||
const lines = (remainder + decoder.decode(chunk, { stream: true })).split("\n")
|
||||
remainder = lines.pop() ?? ""
|
||||
lines.filter((line) => line.trim()).forEach((line) => onLine(line.trim()))
|
||||
}
|
||||
remainder += decoder.decode()
|
||||
if (remainder.trim()) onLine(remainder.trim())
|
||||
}
|
||||
|
||||
async function prepareHelper() {
|
||||
if (process.platform !== "darwin") return undefined
|
||||
const source = Bun.fileURLToPath(new URL("./duplex-audio.swift", import.meta.url))
|
||||
const binary = Bun.fileURLToPath(new URL("../.build/duplex-audio", import.meta.url))
|
||||
if ((await Bun.file(binary).exists()) && Bun.file(binary).lastModified > Bun.file(source).lastModified) return binary
|
||||
await mkdir(Bun.fileURLToPath(new URL("../.build", import.meta.url)), { recursive: true })
|
||||
const compile = Bun.spawn(["swiftc", "-O", source, "-o", binary], { stdout: "ignore", stderr: "pipe" })
|
||||
if ((await compile.exited) === 0) return binary
|
||||
return undefined
|
||||
}
|
||||
@@ -1,41 +0,0 @@
|
||||
import { PCM_BYTES_PER_MS } from "./pcm"
|
||||
|
||||
export type PlaybackChunk = {
|
||||
readonly bytes: Buffer
|
||||
readonly gapMs: number
|
||||
}
|
||||
|
||||
export class AudioJitterBuffer {
|
||||
private readonly pending: PlaybackChunk[] = []
|
||||
private durationMs = 0
|
||||
private started = false
|
||||
|
||||
constructor(private readonly targetMs = 500) {}
|
||||
|
||||
push(chunk: PlaybackChunk): ReadonlyArray<PlaybackChunk> {
|
||||
if (this.started) return [chunk]
|
||||
this.pending.push(chunk)
|
||||
this.durationMs += chunk.gapMs + chunk.bytes.length / PCM_BYTES_PER_MS
|
||||
if (this.durationMs < this.targetMs) return []
|
||||
this.started = true
|
||||
return this.drain()
|
||||
}
|
||||
|
||||
finish(): ReadonlyArray<PlaybackChunk> {
|
||||
if (this.pending.length === 0) return []
|
||||
this.started = true
|
||||
return this.drain()
|
||||
}
|
||||
|
||||
reset() {
|
||||
this.pending.length = 0
|
||||
this.durationMs = 0
|
||||
this.started = false
|
||||
}
|
||||
|
||||
private drain() {
|
||||
const chunks = this.pending.splice(0)
|
||||
this.durationMs = 0
|
||||
return chunks
|
||||
}
|
||||
}
|
||||
@@ -1,217 +0,0 @@
|
||||
import { createAudioDevice, type AudioDevice } from "./audio-device"
|
||||
import { AudioJitterBuffer, type PlaybackChunk } from "./audio-jitter-buffer"
|
||||
import { pcmLevel, PCM_BYTES_PER_MS, PCM_METER_FRAME_MS } from "./pcm"
|
||||
import type { VoiceAudioTimeline } from "./protocol"
|
||||
|
||||
export type AudioEvent =
|
||||
| { readonly type: "input"; readonly audio: Buffer; readonly level: number }
|
||||
| { readonly type: "user.started" }
|
||||
| { readonly type: "user.stopped" }
|
||||
| { readonly type: "user.reset" }
|
||||
| { readonly type: "assistant.level"; readonly level: number; readonly durationMs: number }
|
||||
| { readonly type: "status"; readonly audio: string }
|
||||
| { readonly type: "meta"; readonly text: string }
|
||||
|
||||
export type PlaybackOutcome = "played" | "interrupted" | "inaudible"
|
||||
|
||||
export type AudioSession = {
|
||||
readonly fullDuplex: boolean
|
||||
readonly microphoneMuted: boolean
|
||||
readonly speakerMuted: boolean
|
||||
start(emit: (event: AudioEvent) => void): Promise<void>
|
||||
toggleMicrophone(): boolean
|
||||
toggleSpeaker(): boolean
|
||||
isPlaying(): boolean
|
||||
play(bytes: Buffer, timeline?: VoiceAudioTimeline): void
|
||||
finishPlayback(): Promise<PlaybackOutcome>
|
||||
flushPlayback(): void
|
||||
noteUserCommitted(): void
|
||||
noteUserTranscript(final: boolean): void
|
||||
close(): void
|
||||
}
|
||||
|
||||
export async function createAudioSession(options: {
|
||||
readonly duplex: boolean
|
||||
readonly speakers: boolean
|
||||
readonly inputActivity: "local" | "server"
|
||||
readonly debug: boolean
|
||||
readonly device?: AudioDevice
|
||||
readonly trace?: (event: string, data?: Record<string, unknown>) => void
|
||||
}): Promise<AudioSession> {
|
||||
const device = options.device ?? (await createAudioDevice(options))
|
||||
const fullDuplex = device.fullDuplex
|
||||
const playbackBuffer = new AudioJitterBuffer()
|
||||
let emit: ((event: AudioEvent) => void) | undefined
|
||||
let microphoneMuted = false
|
||||
let speakerMuted = false
|
||||
let playbackEndsAt = 0
|
||||
let playbackAudible = false
|
||||
let playbackDoneTimer: ReturnType<typeof setTimeout> | undefined
|
||||
let playbackCompletion: PromiseWithResolvers<PlaybackOutcome> | undefined
|
||||
let outputTimelineEnd: number | undefined
|
||||
let userFinalizedAt = 0
|
||||
let userSpeechTimer: ReturnType<typeof setTimeout> | undefined
|
||||
let userDraftTimer: ReturnType<typeof setTimeout> | undefined
|
||||
let microphoneStarted = false
|
||||
let closed = false
|
||||
|
||||
const isPlaying = () => Date.now() < playbackEndsAt
|
||||
|
||||
const settlePlayback = (outcome: PlaybackOutcome) => {
|
||||
if (playbackDoneTimer) clearTimeout(playbackDoneTimer)
|
||||
playbackDoneTimer = undefined
|
||||
const completion = playbackCompletion
|
||||
playbackCompletion = undefined
|
||||
playbackEndsAt = 0
|
||||
playbackAudible = false
|
||||
playbackBuffer.reset()
|
||||
outputTimelineEnd = undefined
|
||||
completion?.resolve(outcome)
|
||||
}
|
||||
|
||||
const flushPlayback = () => {
|
||||
device.flush()
|
||||
settlePlayback("interrupted")
|
||||
}
|
||||
|
||||
const observeUserAudio = (level: number) => {
|
||||
if (options.inputActivity !== "local" || level < 0.2 || Date.now() - userFinalizedAt < 500) return
|
||||
if (userDraftTimer) clearTimeout(userDraftTimer)
|
||||
userDraftTimer = undefined
|
||||
emit?.({ type: "user.started" })
|
||||
outputTimelineEnd = undefined
|
||||
if (userSpeechTimer) clearTimeout(userSpeechTimer)
|
||||
userSpeechTimer = setTimeout(() => {
|
||||
userSpeechTimer = undefined
|
||||
emit?.({ type: "user.stopped" })
|
||||
userDraftTimer = setTimeout(() => {
|
||||
userDraftTimer = undefined
|
||||
emit?.({ type: "user.reset" })
|
||||
}, 1_200)
|
||||
}, 500)
|
||||
}
|
||||
|
||||
const publishInput = (bytes: Buffer) => {
|
||||
const level = pcmLevel(bytes)
|
||||
emit?.({ type: "input", audio: bytes, level })
|
||||
observeUserAudio(level)
|
||||
}
|
||||
|
||||
const start = async (listener: (event: AudioEvent) => void) => {
|
||||
if (microphoneStarted || closed) return
|
||||
emit = listener
|
||||
microphoneStarted = true
|
||||
emit({ type: "status", audio: device.mode })
|
||||
emit({ type: "meta", text: "mic live - start talking" })
|
||||
if (!fullDuplex)
|
||||
emit({ type: "meta", text: "mic mutes while the assistant speaks; press Esc to interrupt" })
|
||||
await device.start(
|
||||
(chunk) => {
|
||||
if (closed || microphoneMuted || (!fullDuplex && Date.now() < playbackEndsAt + 300)) return
|
||||
publishInput(chunk)
|
||||
},
|
||||
(text) => {
|
||||
emit?.({ type: "meta", text })
|
||||
options.trace?.("audio.helper", { message: text })
|
||||
},
|
||||
)
|
||||
}
|
||||
|
||||
const writeAudio = (chunk: PlaybackChunk) => {
|
||||
playbackAudible = true
|
||||
if (chunk.gapMs > 0) {
|
||||
emit?.({ type: "assistant.level", level: 0, durationMs: chunk.gapMs })
|
||||
device.write(Buffer.alloc(Math.round(chunk.gapMs * PCM_BYTES_PER_MS)))
|
||||
}
|
||||
const meterBytes = PCM_METER_FRAME_MS * PCM_BYTES_PER_MS
|
||||
for (let offset = 0; offset < chunk.bytes.length; offset += meterBytes) {
|
||||
const window = chunk.bytes.subarray(offset, offset + meterBytes)
|
||||
emit?.({ type: "assistant.level", level: pcmLevel(window), durationMs: window.length / PCM_BYTES_PER_MS })
|
||||
}
|
||||
device.write(chunk.bytes)
|
||||
playbackEndsAt = Math.max(playbackEndsAt, Date.now()) + chunk.gapMs + chunk.bytes.length / PCM_BYTES_PER_MS
|
||||
}
|
||||
|
||||
const schedulePlaybackCompletion = () => {
|
||||
if (!playbackCompletion) return
|
||||
if (!playbackAudible) return settlePlayback("inaudible")
|
||||
if (playbackDoneTimer) clearTimeout(playbackDoneTimer)
|
||||
playbackDoneTimer = setTimeout(
|
||||
() => settlePlayback("played"),
|
||||
Math.max(0, playbackEndsAt - Date.now()) + 180,
|
||||
)
|
||||
}
|
||||
|
||||
return {
|
||||
fullDuplex,
|
||||
get microphoneMuted() {
|
||||
return microphoneMuted
|
||||
},
|
||||
get speakerMuted() {
|
||||
return speakerMuted
|
||||
},
|
||||
start,
|
||||
toggleMicrophone() {
|
||||
microphoneMuted = !microphoneMuted
|
||||
if (userSpeechTimer) clearTimeout(userSpeechTimer)
|
||||
userSpeechTimer = undefined
|
||||
emit?.({ type: "user.stopped" })
|
||||
if (microphoneMuted) emit?.({ type: "user.reset" })
|
||||
return microphoneMuted
|
||||
},
|
||||
toggleSpeaker() {
|
||||
speakerMuted = !speakerMuted
|
||||
if (speakerMuted) flushPlayback()
|
||||
return speakerMuted
|
||||
},
|
||||
isPlaying,
|
||||
play(bytes, timeline) {
|
||||
if (speakerMuted || closed) return
|
||||
const timelineGap = timeline && outputTimelineEnd !== undefined ? Math.max(0, timeline.startMs - outputTimelineEnd) : 0
|
||||
const gapMs = timelineGap < 2_000 ? timelineGap : 0
|
||||
if (options.debug)
|
||||
options.trace?.("audio.output", {
|
||||
bytes: bytes.length,
|
||||
durationMs: bytes.length / PCM_BYTES_PER_MS,
|
||||
level: pcmLevel(bytes),
|
||||
timelineStart: timeline?.startMs,
|
||||
timelineEnd: timeline?.endMs,
|
||||
timelineGap,
|
||||
gapMs,
|
||||
})
|
||||
outputTimelineEnd = timeline?.endMs
|
||||
playbackBuffer.push({ bytes, gapMs }).forEach(writeAudio)
|
||||
schedulePlaybackCompletion()
|
||||
},
|
||||
finishPlayback() {
|
||||
if (playbackCompletion) return playbackCompletion.promise
|
||||
playbackCompletion = Promise.withResolvers<PlaybackOutcome>()
|
||||
playbackBuffer.finish().forEach(writeAudio)
|
||||
schedulePlaybackCompletion()
|
||||
return playbackCompletion?.promise ?? Promise.resolve("inaudible")
|
||||
},
|
||||
flushPlayback,
|
||||
noteUserCommitted() {
|
||||
if (userDraftTimer) clearTimeout(userDraftTimer)
|
||||
userDraftTimer = undefined
|
||||
},
|
||||
noteUserTranscript(final) {
|
||||
if (userDraftTimer) clearTimeout(userDraftTimer)
|
||||
userDraftTimer = undefined
|
||||
if (!final) return
|
||||
if (userSpeechTimer) clearTimeout(userSpeechTimer)
|
||||
userSpeechTimer = undefined
|
||||
userFinalizedAt = Date.now()
|
||||
emit?.({ type: "user.stopped" })
|
||||
},
|
||||
close() {
|
||||
if (closed) return
|
||||
closed = true
|
||||
device.close()
|
||||
if (userSpeechTimer) clearTimeout(userSpeechTimer)
|
||||
if (userDraftTimer) clearTimeout(userDraftTimer)
|
||||
settlePlayback("interrupted")
|
||||
emit = undefined
|
||||
},
|
||||
}
|
||||
}
|
||||
@@ -1,109 +0,0 @@
|
||||
import { mkdir, readdir, rename, rm } from "node:fs/promises"
|
||||
import { homedir } from "node:os"
|
||||
import { dirname, join } from "node:path"
|
||||
import { Option, Schema } from "effect"
|
||||
import { OpenCodeNotification } from "./opencode-notification"
|
||||
import { promptKey, type PromptHandle } from "./prompt-handle"
|
||||
|
||||
export type CompletionHandle = PromptHandle
|
||||
export type StoredCompletion =
|
||||
| { readonly status: "admitting"; readonly handle: CompletionHandle; readonly text: string }
|
||||
| { readonly status: "pending"; readonly handle: CompletionHandle }
|
||||
| {
|
||||
readonly status: "completed"
|
||||
readonly handle: CompletionHandle
|
||||
readonly notification: OpenCodeNotification
|
||||
}
|
||||
|
||||
const CompletionHandle = Schema.Struct({ sessionID: Schema.String, promptID: Schema.String })
|
||||
const StoredCompletion = Schema.Union([
|
||||
Schema.Struct({ status: Schema.Literal("admitting"), handle: CompletionHandle, text: Schema.String }),
|
||||
Schema.Struct({ status: Schema.Literal("pending"), handle: CompletionHandle }),
|
||||
Schema.Struct({ status: Schema.Literal("completed"), handle: CompletionHandle, notification: OpenCodeNotification }),
|
||||
])
|
||||
const StoredCompletions = Schema.Array(StoredCompletion)
|
||||
|
||||
export async function createCompletionStore(
|
||||
path = join(process.env["XDG_STATE_HOME"] ?? join(homedir(), ".local", "state"), "opencode", "voice-prompts.json"),
|
||||
) {
|
||||
await mkdir(dirname(path), { recursive: true })
|
||||
const directory = `${path}.d`
|
||||
await mkdir(directory, { recursive: true })
|
||||
const files = (await readdir(directory)).filter((name) => name.endsWith(".json"))
|
||||
const decoded = files.length > 0
|
||||
? await Promise.all(
|
||||
files.map(async (name) => {
|
||||
const entry = Option.getOrUndefined(
|
||||
Schema.decodeUnknownOption(StoredCompletion)(await Bun.file(join(directory, name)).json()),
|
||||
)
|
||||
if (!entry) throw new Error(`Invalid voice completion entry: ${join(directory, name)}`)
|
||||
return entry
|
||||
}),
|
||||
)
|
||||
: (await Bun.file(path).exists())
|
||||
? Option.getOrUndefined(Schema.decodeUnknownOption(StoredCompletions)(await Bun.file(path).json()))
|
||||
: []
|
||||
if (!decoded) throw new Error(`Invalid voice completion store: ${path}`)
|
||||
const entries = new Map(decoded.map((entry) => [promptKey(entry.handle), entry]))
|
||||
let writes = Promise.resolve()
|
||||
|
||||
const enqueue = (operation: () => Promise<void>) => {
|
||||
const result = writes.then(operation)
|
||||
writes = result.catch(() => {})
|
||||
return result
|
||||
}
|
||||
|
||||
const save = (entry: StoredCompletion) => {
|
||||
return enqueue(async () => {
|
||||
const target = completionPath(directory, entry.handle)
|
||||
const temporary = `${target}.${process.pid}.${crypto.randomUUID()}.tmp`
|
||||
await Bun.write(temporary, JSON.stringify(entry))
|
||||
await rename(temporary, target)
|
||||
})
|
||||
}
|
||||
|
||||
if (files.length === 0 && decoded.length > 0) {
|
||||
await Promise.all(decoded.map(save))
|
||||
await rm(path, { force: true })
|
||||
}
|
||||
if (files.length > 0) {
|
||||
const existing = new Set(files)
|
||||
await Promise.all(decoded.filter((entry) => !existing.has(completionFilename(entry.handle))).map(save))
|
||||
const current = new Set(decoded.map((entry) => completionFilename(entry.handle)))
|
||||
await Promise.all(files.filter((name) => !current.has(name)).map((name) => rm(join(directory, name), { force: true })))
|
||||
}
|
||||
|
||||
return {
|
||||
entries: () => [...entries.entries()].toSorted(([left], [right]) => left.localeCompare(right)).map(([, entry]) => entry),
|
||||
admitting(handle: CompletionHandle, text: string) {
|
||||
const entry = { status: "admitting", handle, text } as const
|
||||
entries.set(promptKey(handle), entry)
|
||||
return save(entry)
|
||||
},
|
||||
pending(handle: CompletionHandle) {
|
||||
const entry = { status: "pending", handle } as const
|
||||
entries.set(promptKey(handle), entry)
|
||||
return save(entry)
|
||||
},
|
||||
completed(handle: CompletionHandle, notification: OpenCodeNotification) {
|
||||
const entry = { status: "completed", handle, notification } as const
|
||||
entries.set(promptKey(handle), entry)
|
||||
return save(entry)
|
||||
},
|
||||
remove(handle: CompletionHandle) {
|
||||
if (!entries.delete(promptKey(handle))) return writes
|
||||
return enqueue(() => rm(completionPath(directory, handle), { force: true }))
|
||||
},
|
||||
close: () => writes,
|
||||
}
|
||||
}
|
||||
|
||||
export type CompletionStore = Awaited<ReturnType<typeof createCompletionStore>>
|
||||
|
||||
function completionFilename(handle: CompletionHandle) {
|
||||
return `${encodeURIComponent(promptKey(handle))}.json`
|
||||
}
|
||||
|
||||
function completionPath(directory: string, handle: CompletionHandle) {
|
||||
return join(directory, completionFilename(handle))
|
||||
}
|
||||
@@ -1,288 +0,0 @@
|
||||
// Full-duplex terminal audio bridge with Apple voice processing (AEC).
|
||||
//
|
||||
// stdin <- raw PCM16 mono 24kHz to play through the speakers
|
||||
// stdout -> raw PCM16 mono 24kHz captured from the microphone
|
||||
// SIGUSR1: drop any queued speaker audio (barge-in flush)
|
||||
//
|
||||
// Two independent engines: attaching a speaker source to a voice-processed
|
||||
// engine silently kills its input tap, so playback runs on its own plain
|
||||
// engine. Voice processing (echo cancellation) is attempted on the input
|
||||
// engine and abandoned if the mic delivers nothing: on some devices
|
||||
// (Bluetooth headsets mid-negotiation) the VP tap never fires. Without VP
|
||||
// there is no echo cancellation, which is acceptable exactly when it happens:
|
||||
// headphones have no echo path.
|
||||
//
|
||||
// Compiled on demand by spike.ts: swiftc -O duplex-audio.swift -o duplex-audio
|
||||
|
||||
import AVFoundation
|
||||
|
||||
let stderr = FileHandle.standardError
|
||||
func log(_ message: String) { stderr.write(Data("[audio] \(message)\n".utf8)) }
|
||||
|
||||
final class SpeakerQueue {
|
||||
private static let capacity = 24_000 * 2 * 30
|
||||
private var data = [UInt8](repeating: 0, count: capacity)
|
||||
private var readIndex = 0
|
||||
private var writeIndex = 0
|
||||
private var count = 0
|
||||
private let lock = NSLock()
|
||||
func push(_ chunk: Data) {
|
||||
if chunk.isEmpty { return }
|
||||
lock.lock()
|
||||
defer { lock.unlock() }
|
||||
chunk.withUnsafeBytes { source in
|
||||
let skipped = max(0, source.count - Self.capacity)
|
||||
let start = skipped + skipped % 2
|
||||
let bytes = source.count - start
|
||||
let overflow = max(0, count + bytes - Self.capacity)
|
||||
let dropped = min(count, overflow + overflow % 2)
|
||||
readIndex = (readIndex + dropped) % Self.capacity
|
||||
count -= dropped
|
||||
let first = min(bytes, Self.capacity - writeIndex)
|
||||
data.withUnsafeMutableBytes { destination in
|
||||
destination.baseAddress!.advanced(by: writeIndex).copyMemory(
|
||||
from: source.baseAddress!.advanced(by: start), byteCount: first)
|
||||
destination.baseAddress!.copyMemory(
|
||||
from: source.baseAddress!.advanced(by: start + first), byteCount: bytes - first)
|
||||
}
|
||||
writeIndex = (writeIndex + bytes) % Self.capacity
|
||||
count += bytes
|
||||
}
|
||||
}
|
||||
func render(frames: Int, into output: UnsafeMutablePointer<Float>) {
|
||||
lock.lock()
|
||||
defer { lock.unlock() }
|
||||
let samples = min(frames, count / 2)
|
||||
for index in 0..<samples {
|
||||
let low = UInt16(data[readIndex])
|
||||
let high = UInt16(data[(readIndex + 1) % Self.capacity])
|
||||
output[index] = Float(Int16(bitPattern: low | high << 8)) / 32768.0
|
||||
readIndex = (readIndex + 2) % Self.capacity
|
||||
}
|
||||
count -= samples * 2
|
||||
for index in samples..<frames { output[index] = 0 }
|
||||
}
|
||||
func flush() {
|
||||
lock.lock()
|
||||
readIndex = writeIndex
|
||||
count = 0
|
||||
lock.unlock()
|
||||
}
|
||||
}
|
||||
|
||||
#if QUEUE_BENCHMARK
|
||||
let correctnessQueue = SpeakerQueue()
|
||||
correctnessQueue.push(Data([0x00, 0x80, 0xff, 0x7f]))
|
||||
var correctnessOutput = [Float](repeating: 1, count: 3)
|
||||
correctnessOutput.withUnsafeMutableBufferPointer {
|
||||
correctnessQueue.render(frames: 3, into: $0.baseAddress!)
|
||||
}
|
||||
precondition(correctnessOutput[0] == -1)
|
||||
precondition(abs(correctnessOutput[1] - Float(Int16.max) / 32768) < 0.000_001)
|
||||
precondition(correctnessOutput[2] == 0)
|
||||
correctnessQueue.push(Data([1, 0]))
|
||||
correctnessQueue.flush()
|
||||
correctnessOutput.withUnsafeMutableBufferPointer {
|
||||
correctnessQueue.render(frames: 3, into: $0.baseAddress!)
|
||||
}
|
||||
precondition(correctnessOutput.allSatisfy { $0 == 0 })
|
||||
|
||||
let benchmarkQueue = SpeakerQueue()
|
||||
let benchmarkFrames = 512
|
||||
let benchmarkChunk = Data(repeating: 1, count: benchmarkFrames * 2)
|
||||
var benchmarkOutput = [Float](repeating: 0, count: benchmarkFrames)
|
||||
for _ in 0..<48 { benchmarkQueue.push(benchmarkChunk) }
|
||||
for _ in 0..<2_000 {
|
||||
benchmarkOutput.withUnsafeMutableBufferPointer {
|
||||
benchmarkQueue.render(frames: benchmarkFrames, into: $0.baseAddress!)
|
||||
}
|
||||
benchmarkQueue.push(benchmarkChunk)
|
||||
}
|
||||
var benchmarkDurations = [UInt64]()
|
||||
benchmarkDurations.reserveCapacity(30_000)
|
||||
var benchmarkChecksum: Float = 0
|
||||
for _ in 0..<30_000 {
|
||||
let started = DispatchTime.now().uptimeNanoseconds
|
||||
benchmarkOutput.withUnsafeMutableBufferPointer {
|
||||
benchmarkQueue.render(frames: benchmarkFrames, into: $0.baseAddress!)
|
||||
}
|
||||
benchmarkQueue.push(benchmarkChunk)
|
||||
benchmarkDurations.append(DispatchTime.now().uptimeNanoseconds - started)
|
||||
benchmarkChecksum += benchmarkOutput[0]
|
||||
}
|
||||
benchmarkDurations.sort()
|
||||
print("METRIC speaker_queue_median_ns=\(benchmarkDurations[benchmarkDurations.count / 2])")
|
||||
print("METRIC speaker_queue_p99_ns=\(benchmarkDurations[benchmarkDurations.count * 99 / 100])")
|
||||
print("METRIC speaker_queue_worst_ns=\(benchmarkDurations.last!)")
|
||||
print("CHECKSUM \(benchmarkChecksum)")
|
||||
exit(0)
|
||||
#endif
|
||||
|
||||
let queue = SpeakerQueue()
|
||||
let playFormat = AVAudioFormat(commonFormat: .pcmFormatFloat32, sampleRate: 24000, channels: 1, interleaved: false)!
|
||||
let captureFormat = AVAudioFormat(commonFormat: .pcmFormatInt16, sampleRate: 24000, channels: 1, interleaved: true)!
|
||||
var rebuildScheduled = false
|
||||
var ignoreRouteChangesUntil = Date.distantPast
|
||||
|
||||
func scheduleRebuild(_ message: String, after delay: TimeInterval = 0.5) {
|
||||
if rebuildScheduled { return }
|
||||
rebuildScheduled = true
|
||||
DispatchQueue.main.asyncAfter(deadline: .now() + delay) {
|
||||
rebuildScheduled = false
|
||||
// Starting an engine emits configuration changes of its own. AirPods can
|
||||
// otherwise turn one real route change into a perpetual rebuild loop.
|
||||
ignoreRouteChangesUntil = Date().addingTimeInterval(1.5)
|
||||
log(message)
|
||||
startInput(voiceProcessing: useAEC)
|
||||
startOutput()
|
||||
}
|
||||
}
|
||||
|
||||
// Speaker: its own engine, pulling PCM16 from the stdin-fed queue.
|
||||
var outputEngine: AVAudioEngine?
|
||||
|
||||
func startOutput() {
|
||||
outputEngine?.stop()
|
||||
let engine = AVAudioEngine()
|
||||
outputEngine = engine
|
||||
let source = AVAudioSourceNode(format: playFormat) { _, _, frameCount, audioBufferList -> OSStatus in
|
||||
let out = UnsafeMutableAudioBufferListPointer(audioBufferList)[0].mData!.assumingMemoryBound(to: Float.self)
|
||||
queue.render(frames: Int(frameCount), into: out)
|
||||
return noErr
|
||||
}
|
||||
engine.attach(source)
|
||||
engine.connect(source, to: engine.mainMixerNode, format: playFormat)
|
||||
do {
|
||||
try engine.start()
|
||||
let format = engine.outputNode.outputFormat(forBus: 0)
|
||||
log("output active (\(Int(format.sampleRate))Hz, \(format.channelCount)ch)")
|
||||
} catch {
|
||||
log("output engine failed: \(error)")
|
||||
scheduleRebuild("audio engines unavailable — retrying")
|
||||
}
|
||||
}
|
||||
|
||||
// Microphone: take channel 0 of whatever the hardware provides, resample to
|
||||
// 24kHz PCM16 for stdout. Channel extraction is manual because hardware
|
||||
// channel counts vary wildly (1, 2, 3, 22...) and AVAudioConverter cannot
|
||||
// downmix all of them.
|
||||
var tapCount = 0
|
||||
var inputEngine: AVAudioEngine?
|
||||
var useAEC = CommandLine.arguments.contains("--aec")
|
||||
|
||||
func startInput(voiceProcessing: Bool) {
|
||||
inputEngine?.stop()
|
||||
let engine = AVAudioEngine()
|
||||
inputEngine = engine
|
||||
if voiceProcessing {
|
||||
do {
|
||||
try engine.inputNode.setVoiceProcessingEnabled(true)
|
||||
} catch {
|
||||
log("voice processing unavailable (\(error)) — no echo cancellation")
|
||||
useAEC = false
|
||||
return startInput(voiceProcessing: false)
|
||||
}
|
||||
if #available(macOS 14.0, *) {
|
||||
engine.inputNode.voiceProcessingOtherAudioDuckingConfiguration = .init(
|
||||
enableAdvancedDucking: false,
|
||||
duckingLevel: .min
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
let micFormat = engine.inputNode.outputFormat(forBus: 0)
|
||||
guard micFormat.sampleRate > 0, micFormat.channelCount > 0, let monoFormat = AVAudioFormat(
|
||||
commonFormat: .pcmFormatFloat32, sampleRate: micFormat.sampleRate, channels: 1, interleaved: false)
|
||||
else {
|
||||
log("microphone route is not ready — retrying")
|
||||
scheduleRebuild("audio engines unavailable — retrying")
|
||||
return
|
||||
}
|
||||
guard let converter = AVAudioConverter(from: monoFormat, to: captureFormat) else {
|
||||
log("cannot convert \(micFormat.sampleRate)Hz to 24kHz — retrying")
|
||||
scheduleRebuild("audio engines unavailable — retrying")
|
||||
return
|
||||
}
|
||||
|
||||
engine.inputNode.installTap(onBus: 0, bufferSize: 2400, format: micFormat) { buffer, _ in
|
||||
tapCount += 1
|
||||
if tapCount == 1 { log("mic active (echo cancellation \(voiceProcessing ? "on" : "off"))") }
|
||||
guard let channel = buffer.floatChannelData?[0], buffer.frameLength > 0 else { return }
|
||||
guard let mono = AVAudioPCMBuffer(pcmFormat: monoFormat, frameCapacity: buffer.frameLength) else { return }
|
||||
memcpy(mono.floatChannelData![0], channel, Int(buffer.frameLength) * 4)
|
||||
mono.frameLength = buffer.frameLength
|
||||
|
||||
let capacity = AVAudioFrameCount(Double(buffer.frameLength) * 24000.0 / micFormat.sampleRate) + 32
|
||||
guard let converted = AVAudioPCMBuffer(pcmFormat: captureFormat, frameCapacity: capacity) else { return }
|
||||
var consumed = false
|
||||
converter.convert(to: converted, error: nil) { _, status in
|
||||
if consumed {
|
||||
status.pointee = .noDataNow
|
||||
return nil
|
||||
}
|
||||
consumed = true
|
||||
status.pointee = .haveData
|
||||
return mono
|
||||
}
|
||||
guard converted.frameLength > 0, let out = converted.int16ChannelData?[0] else { return }
|
||||
FileHandle.standardOutput.write(Data(bytes: out, count: Int(converted.frameLength) * 2))
|
||||
}
|
||||
|
||||
do {
|
||||
try engine.start()
|
||||
log("input active (\(Int(micFormat.sampleRate))Hz, \(micFormat.channelCount)ch, echo cancellation \(voiceProcessing ? "on" : "off"))")
|
||||
} catch {
|
||||
if voiceProcessing {
|
||||
log("input engine failed with voice processing (\(error)) — retrying without")
|
||||
useAEC = false
|
||||
return startInput(voiceProcessing: false)
|
||||
}
|
||||
log("input engine failed: \(error)")
|
||||
scheduleRebuild("audio engines unavailable — retrying")
|
||||
return
|
||||
}
|
||||
|
||||
// Watchdog: on some devices the voice-processed tap simply never fires.
|
||||
// Fall back to a plain tap; without VP the mic reliably delivers.
|
||||
if voiceProcessing {
|
||||
DispatchQueue.main.asyncAfter(deadline: .now() + 2.5) {
|
||||
if tapCount > 0 { return }
|
||||
log("voice-processed mic delivered nothing — restarting without echo cancellation")
|
||||
useAEC = false
|
||||
startInput(voiceProcessing: false)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Voice processing is opt-in (--aec): it is only needed on speakers, and on
|
||||
// some machines (observed with Bluetooth headsets active) the VP engine binds
|
||||
// to the wrong capture device entirely, delivering noise instead of the mic.
|
||||
// Mic first: activating the mic flips Bluetooth headsets from music mode to
|
||||
// headset mode, reconfiguring the output device. Starting output afterwards
|
||||
// (and rebuilding on any route change below) keeps playback on the live device.
|
||||
startInput(voiceProcessing: useAEC)
|
||||
startOutput()
|
||||
|
||||
// Device switches (Bluetooth profile flips, headphones plugged/unplugged,
|
||||
// default device changes) stop engines silently. Rebuild both, debounced.
|
||||
NotificationCenter.default.addObserver(
|
||||
forName: .AVAudioEngineConfigurationChange, object: nil, queue: .main
|
||||
) { _ in
|
||||
if Date() < ignoreRouteChangesUntil { return }
|
||||
scheduleRebuild("audio route changed — rebuilding engines")
|
||||
}
|
||||
|
||||
signal(SIGUSR1, SIG_IGN)
|
||||
let flushSignal = DispatchSource.makeSignalSource(signal: SIGUSR1, queue: .main)
|
||||
flushSignal.setEventHandler { queue.flush() }
|
||||
flushSignal.resume()
|
||||
|
||||
DispatchQueue.global().async {
|
||||
while true {
|
||||
let chunk = FileHandle.standardInput.availableData
|
||||
if chunk.isEmpty { exit(0) } // parent closed stdin
|
||||
queue.push(chunk)
|
||||
}
|
||||
}
|
||||
|
||||
RunLoop.main.run()
|
||||
@@ -1,48 +0,0 @@
|
||||
import type { CompletionReceipt, OpenCodeAnnouncement } from "./opencode-notification"
|
||||
import { promptKey } from "./prompt-handle"
|
||||
|
||||
export type RoutedAnnouncement = {
|
||||
readonly announcement: OpenCodeAnnouncement
|
||||
readonly promptID?: string
|
||||
readonly delegationID?: string
|
||||
}
|
||||
|
||||
export function createNotificationRouter(deliver: (announcement: RoutedAnnouncement) => void) {
|
||||
const delegations = new Map<string, string>()
|
||||
const deferred: OpenCodeAnnouncement[] = []
|
||||
let pendingDelegatedTools = 0
|
||||
|
||||
const route = (announcement: OpenCodeAnnouncement) => {
|
||||
const prompt = "prompt_id" in announcement.notification
|
||||
? { sessionID: announcement.notification.session_id, promptID: announcement.notification.prompt_id }
|
||||
: undefined
|
||||
const promptID = prompt?.promptID
|
||||
if (prompt && pendingDelegatedTools > 0 && !delegations.has(promptKey(prompt))) {
|
||||
deferred.push(announcement)
|
||||
return
|
||||
}
|
||||
deliver({
|
||||
announcement,
|
||||
promptID,
|
||||
delegationID: prompt ? delegations.get(promptKey(prompt)) : undefined,
|
||||
})
|
||||
}
|
||||
|
||||
return {
|
||||
route,
|
||||
beginDelegatedTool() {
|
||||
pendingDelegatedTools += 1
|
||||
},
|
||||
finishDelegatedTool(
|
||||
delegationID: string,
|
||||
admittedPrompt?: { readonly sessionID: string; readonly promptID: string },
|
||||
) {
|
||||
if (admittedPrompt) delegations.set(promptKey(admittedPrompt), delegationID)
|
||||
pendingDelegatedTools -= 1
|
||||
if (pendingDelegatedTools === 0) deferred.splice(0).forEach(route)
|
||||
},
|
||||
acknowledged(receipt: CompletionReceipt) {
|
||||
delegations.delete(promptKey(receipt))
|
||||
},
|
||||
}
|
||||
}
|
||||
@@ -1,97 +0,0 @@
|
||||
import { Schema } from "effect"
|
||||
import type { PromptHandle } from "./prompt-handle"
|
||||
|
||||
const Completed = Schema.Struct({
|
||||
type: Schema.Literal("opencode.prompt.completed"),
|
||||
session_id: Schema.String,
|
||||
prompt_id: Schema.String,
|
||||
status: Schema.Literals(["completed", "failed"]),
|
||||
text: Schema.String,
|
||||
error: Schema.optional(Schema.String),
|
||||
})
|
||||
|
||||
const Failed = Schema.Struct({
|
||||
type: Schema.Literal("opencode.prompt.failed"),
|
||||
session_id: Schema.String,
|
||||
prompt_id: Schema.String,
|
||||
status: Schema.Literal("failed"),
|
||||
error: Schema.String,
|
||||
})
|
||||
|
||||
const PermissionBlocked = Schema.Struct({
|
||||
type: Schema.Literal("opencode.prompt.blocked"),
|
||||
prompt_id: Schema.String,
|
||||
blocker: Schema.Literal("permission"),
|
||||
session_id: Schema.String,
|
||||
request_id: Schema.String,
|
||||
action: Schema.String,
|
||||
resources: Schema.Unknown,
|
||||
})
|
||||
|
||||
const QuestionBlocked = Schema.Struct({
|
||||
type: Schema.Literal("opencode.prompt.blocked"),
|
||||
prompt_id: Schema.String,
|
||||
blocker: Schema.Literal("question"),
|
||||
session_id: Schema.String,
|
||||
request_id: Schema.String,
|
||||
questions: Schema.Unknown,
|
||||
})
|
||||
|
||||
const FormBlocked = Schema.Struct({
|
||||
type: Schema.Literal("opencode.prompt.blocked"),
|
||||
prompt_id: Schema.String,
|
||||
blocker: Schema.Literal("form"),
|
||||
session_id: Schema.String,
|
||||
form_id: Schema.String,
|
||||
title: Schema.String,
|
||||
fields: Schema.Unknown,
|
||||
})
|
||||
|
||||
const EventsFailed = Schema.Struct({
|
||||
type: Schema.Literal("opencode.events.failed"),
|
||||
error: Schema.String,
|
||||
})
|
||||
|
||||
export const OpenCodeNotification = Schema.Union([
|
||||
Completed,
|
||||
Failed,
|
||||
PermissionBlocked,
|
||||
QuestionBlocked,
|
||||
FormBlocked,
|
||||
EventsFailed,
|
||||
])
|
||||
export type OpenCodeNotification = typeof OpenCodeNotification.Type
|
||||
export type OpenCodePromptBlocked = Extract<OpenCodeNotification, { readonly type: "opencode.prompt.blocked" }>
|
||||
|
||||
export type CompletionReceipt = PromptHandle
|
||||
|
||||
export type OpenCodeAnnouncement = {
|
||||
readonly notification: OpenCodeNotification
|
||||
readonly receipt?: CompletionReceipt
|
||||
}
|
||||
|
||||
export function openCodeAnnouncementText(notification: OpenCodeNotification) {
|
||||
const text = (() => {
|
||||
switch (notification.type) {
|
||||
case "opencode.prompt.completed":
|
||||
return `OpenCode prompt ${notification.status}: ${notification.text}${notification.error ? ` Error: ${notification.error}` : ""}`
|
||||
case "opencode.prompt.failed":
|
||||
return `OpenCode prompt failed: ${notification.error}`
|
||||
case "opencode.prompt.blocked":
|
||||
if (notification.blocker === "permission")
|
||||
return `OpenCode needs permission for ${notification.action}: ${compact(notification.resources)}`
|
||||
if (notification.blocker === "question")
|
||||
return `OpenCode needs the user to answer: ${compact(notification.questions)}`
|
||||
return `OpenCode needs the user to complete the form “${notification.title}”: ${compact(notification.fields)}`
|
||||
case "opencode.events.failed":
|
||||
return `OpenCode event delivery failed: ${notification.error}`
|
||||
}
|
||||
return "OpenCode has an update."
|
||||
})()
|
||||
return text.length > 800 ? text.slice(0, 797) + "..." : text
|
||||
}
|
||||
|
||||
function compact(value: unknown) {
|
||||
if (typeof value === "string") return value
|
||||
return JSON.stringify(value) ?? String(value)
|
||||
}
|
||||
@@ -1,811 +0,0 @@
|
||||
import { realpathSync } from "node:fs"
|
||||
import { tmpdir } from "node:os"
|
||||
import { basename, isAbsolute, relative } from "node:path"
|
||||
import { ClientError, isSessionNotFoundError, OpenCode } from "@opencode-ai/client/promise"
|
||||
import type { SessionMessageAssistant, SessionMessageUser, V2Event } from "@opencode-ai/client/promise"
|
||||
import { Form, Question, SessionMessage } from "@opencode-ai/client/effect"
|
||||
import { Context, Effect, FiberMap, Layer, ManagedRuntime, Option, Schema, Stream } from "effect"
|
||||
import { createCompletionStore, type CompletionStore } from "./completion-store"
|
||||
import type {
|
||||
CompletionReceipt,
|
||||
OpenCodeAnnouncement,
|
||||
OpenCodeNotification,
|
||||
OpenCodePromptBlocked,
|
||||
} from "./opencode-notification"
|
||||
import type { VoiceTool, VoiceToolExecution } from "./protocol"
|
||||
import type { PromptHandle } from "./prompt-handle"
|
||||
|
||||
type Client = ReturnType<typeof OpenCode.make>
|
||||
const AdmittedPrompt = Symbol("AdmittedPrompt")
|
||||
type Tool = {
|
||||
readonly description: string
|
||||
readonly parameters: unknown
|
||||
readonly execute: (input: Record<string, unknown>) => Effect.Effect<unknown, unknown>
|
||||
}
|
||||
type BridgeApi = {
|
||||
readonly definitions: ReadonlyArray<VoiceTool>
|
||||
readonly execute: (name: string, input: Record<string, unknown>) => Effect.Effect<VoiceToolExecution, unknown>
|
||||
readonly acknowledge: (receipt: CompletionReceipt) => Effect.Effect<void, unknown>
|
||||
readonly close: Effect.Effect<void>
|
||||
}
|
||||
|
||||
class Bridge extends Context.Service<Bridge, BridgeApi>()("@opencode-ai/voice/OpenCodeBridge") {}
|
||||
|
||||
export async function createOpenCodeBridge(options: {
|
||||
client: Client
|
||||
directory: string
|
||||
model: { readonly providerID: string; readonly id: string; readonly variant?: string }
|
||||
notify: (announcement: OpenCodeAnnouncement) => void
|
||||
trace?: (event: string, data?: Record<string, unknown>) => void
|
||||
completionStore?: CompletionStore
|
||||
}) {
|
||||
const completionStore = options.completionStore ?? (await createCompletionStore())
|
||||
const runtime = ManagedRuntime.make(Layer.effect(Bridge, makeBridge({ ...options, completionStore })))
|
||||
const bridge = await runtime.runPromise(Bridge)
|
||||
return {
|
||||
definitions: bridge.definitions,
|
||||
execute: (name: string, input: Record<string, unknown>) => runtime.runPromise(bridge.execute(name, input)),
|
||||
acknowledge: (receipt: CompletionReceipt) => runtime.runPromise(bridge.acknowledge(receipt)),
|
||||
close: async () => {
|
||||
await runtime.runPromise(bridge.close)
|
||||
await Promise.race([runtime.dispose(), Bun.sleep(1_000)])
|
||||
await completionStore.close()
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
const makeBridge = Effect.fnUntraced(function* (options: {
|
||||
client: Client
|
||||
directory: string
|
||||
model: { readonly providerID: string; readonly id: string; readonly variant?: string }
|
||||
notify: (announcement: OpenCodeAnnouncement) => void
|
||||
trace?: (event: string, data?: Record<string, unknown>) => void
|
||||
completionStore: CompletionStore
|
||||
}) {
|
||||
const knownProjects = new Set<string>()
|
||||
const knownSessions = new Set<string>()
|
||||
const registrations = new Map<string, PromptHandle>()
|
||||
const promoted = new Map<string, string>()
|
||||
const latest = new Map<string, string>()
|
||||
const announcedBlockers = new Set<string>()
|
||||
const completions = yield* FiberMap.make<string>()
|
||||
const eventAbort = yield* Effect.acquireRelease(
|
||||
Effect.sync(() => new AbortController()),
|
||||
(controller) => Effect.sync(() => controller.abort()),
|
||||
)
|
||||
|
||||
const notify = (notification: OpenCodeNotification, receipt?: CompletionReceipt) =>
|
||||
Effect.sync(() => options.notify({ notification, receipt }))
|
||||
const trace = (event: string, data: Record<string, unknown>) => Effect.sync(() => options.trace?.(event, data))
|
||||
const request = <A>(run: (signal: AbortSignal) => PromiseLike<A>) =>
|
||||
Effect.tryPromise({ try: run, catch: (cause) => cause })
|
||||
const announceBlocker = (key: string, notification: OpenCodeNotification) => {
|
||||
if (announcedBlockers.has(key)) return Effect.void
|
||||
announcedBlockers.add(key)
|
||||
return notify(notification)
|
||||
}
|
||||
const clearRegistration = (handle: PromptHandle) =>
|
||||
Effect.sync(() => {
|
||||
registrations.delete(handle.promptID)
|
||||
if (promoted.get(handle.sessionID) === handle.promptID) promoted.delete(handle.sessionID)
|
||||
if (latest.get(handle.sessionID) === handle.promptID) latest.delete(handle.sessionID)
|
||||
})
|
||||
|
||||
const listProjects = Effect.fnUntraced(function* () {
|
||||
const seen = new Set<string>()
|
||||
const temporary = realpathSync(tmpdir())
|
||||
return (yield* request((signal) => options.client.project.list({ signal })))
|
||||
.sort((a, b) => b.time.updated - a.time.updated)
|
||||
.filter((project) => {
|
||||
const fromTemporary = relative(temporary, project.worktree)
|
||||
if (
|
||||
project.id === "global" ||
|
||||
fromTemporary === "" ||
|
||||
(!fromTemporary.startsWith("..") && !isAbsolute(fromTemporary)) ||
|
||||
seen.has(project.worktree)
|
||||
)
|
||||
return false
|
||||
seen.add(project.worktree)
|
||||
return true
|
||||
})
|
||||
})
|
||||
|
||||
const projectDirectory = Effect.fnUntraced(function* (projectID: string) {
|
||||
if (!knownProjects.has(projectID)) return undefined
|
||||
return (yield* listProjects()).find((project) => project.id === projectID)?.worktree
|
||||
})
|
||||
|
||||
const complete = Effect.fnUntraced(function* (handle: PromptHandle) {
|
||||
yield* trace("opencode.wait.started", handle)
|
||||
yield* waitForIdle(handle)
|
||||
const reply = yield* request((signal) => finalReply(options.client, handle, signal))
|
||||
yield* trace("opencode.wait.completed", { ...handle, replyID: reply?.id, error: reply?.error?.message })
|
||||
const notification: OpenCodeNotification = {
|
||||
type: "opencode.prompt.completed",
|
||||
session_id: handle.sessionID,
|
||||
prompt_id: handle.promptID,
|
||||
status: reply?.error ? "failed" : "completed",
|
||||
text: reply
|
||||
? assistantText(reply) || "OpenCode finished without a text reply."
|
||||
: "OpenCode finished without a reply.",
|
||||
error: reply?.error?.message,
|
||||
}
|
||||
yield* request(() => options.completionStore.completed(handle, notification))
|
||||
yield* notify(notification, handle)
|
||||
})
|
||||
|
||||
const waitForIdle = Effect.fnUntraced(function* (handle: PromptHandle, attempt = 1): Effect.fn.Return<void, unknown> {
|
||||
return yield* request((signal) => options.client.session.wait({ sessionID: handle.sessionID }, { signal })).pipe(
|
||||
Effect.catch((error) => {
|
||||
if (!(error instanceof ClientError) || error.reason !== "Transport") return Effect.fail(error)
|
||||
return trace("opencode.wait.retrying", {
|
||||
...handle,
|
||||
attempt,
|
||||
reason: error.reason,
|
||||
cause: error.cause instanceof Error ? error.cause.name : typeof error.cause,
|
||||
}).pipe(Effect.andThen(Effect.sleep("1 second")), Effect.andThen(waitForIdle(handle, attempt + 1)))
|
||||
}),
|
||||
)
|
||||
})
|
||||
|
||||
const register = Effect.fnUntraced(function* (handle: PromptHandle) {
|
||||
yield* complete(handle).pipe(
|
||||
Effect.catch((error) => {
|
||||
const notification: OpenCodeNotification = {
|
||||
type: "opencode.prompt.failed",
|
||||
session_id: handle.sessionID,
|
||||
prompt_id: handle.promptID,
|
||||
status: "failed",
|
||||
error: String(error),
|
||||
}
|
||||
return request(() => options.completionStore.completed(handle, notification)).pipe(
|
||||
Effect.andThen(notify(notification, handle)),
|
||||
)
|
||||
}),
|
||||
Effect.ensuring(clearRegistration(handle)),
|
||||
FiberMap.run(completions, handle.promptID, { onlyIfMissing: true, startImmediately: true }),
|
||||
)
|
||||
})
|
||||
|
||||
const restoreBlockers = Effect.fnUntraced(function* (handle: PromptHandle) {
|
||||
const [permissions, questions, forms] = yield* Effect.all(
|
||||
[
|
||||
request((signal) => options.client.permission.list({ sessionID: handle.sessionID }, { signal })),
|
||||
request((signal) => options.client.question.list({ sessionID: handle.sessionID }, { signal })),
|
||||
request((signal) => options.client.form.list({ sessionID: handle.sessionID }, { signal })),
|
||||
],
|
||||
{ concurrency: "unbounded" },
|
||||
)
|
||||
yield* Effect.forEach(
|
||||
[
|
||||
...permissions.map((item) => ({
|
||||
key: `permission:${item.id}`,
|
||||
notification: {
|
||||
type: "opencode.prompt.blocked",
|
||||
prompt_id: handle.promptID,
|
||||
blocker: "permission",
|
||||
session_id: handle.sessionID,
|
||||
request_id: item.id,
|
||||
action: item.action,
|
||||
resources: item.resources,
|
||||
} satisfies OpenCodeNotification,
|
||||
})),
|
||||
...questions.map((item) => ({
|
||||
key: `question:${item.id}`,
|
||||
notification: {
|
||||
type: "opencode.prompt.blocked",
|
||||
prompt_id: handle.promptID,
|
||||
blocker: "question",
|
||||
session_id: handle.sessionID,
|
||||
request_id: item.id,
|
||||
questions: item.questions,
|
||||
} satisfies OpenCodeNotification,
|
||||
})),
|
||||
...forms.map((item) => ({
|
||||
key: `form:${item.id}`,
|
||||
notification: {
|
||||
type: "opencode.prompt.blocked",
|
||||
prompt_id: handle.promptID,
|
||||
blocker: "form",
|
||||
session_id: handle.sessionID,
|
||||
form_id: item.id,
|
||||
title: item.title,
|
||||
fields: item.fields,
|
||||
} satisfies OpenCodeNotification,
|
||||
})),
|
||||
],
|
||||
(item) => announceBlocker(item.key, item.notification),
|
||||
{ discard: true },
|
||||
)
|
||||
})
|
||||
|
||||
const admit = Effect.fnUntraced(function* (sessionID: string, text: string) {
|
||||
const handle = { sessionID, promptID: SessionMessage.ID.create() }
|
||||
registrations.set(handle.promptID, handle)
|
||||
latest.set(sessionID, handle.promptID)
|
||||
yield* trace("opencode.prompt.admitting", handle)
|
||||
yield* request(() => options.completionStore.admitting(handle, text))
|
||||
const admitted = yield* request((signal) =>
|
||||
options.client.session.prompt({ sessionID, id: handle.promptID, text }, { signal }),
|
||||
).pipe(
|
||||
Effect.tapError(() =>
|
||||
request(() => options.completionStore.remove(handle)).pipe(
|
||||
Effect.andThen(clearRegistration(handle)),
|
||||
),
|
||||
),
|
||||
)
|
||||
knownSessions.add(sessionID)
|
||||
yield* request(() => options.completionStore.pending(handle))
|
||||
yield* trace("opencode.prompt.admitted", { ...handle, admittedSeq: admitted.admittedSeq })
|
||||
yield* register(handle)
|
||||
return {
|
||||
status: "started",
|
||||
session_id: sessionID,
|
||||
prompt_id: admitted.id,
|
||||
notification: "registered",
|
||||
message: "OpenCode accepted the prompt and will send a completion notification. Continue the conversation now.",
|
||||
[AdmittedPrompt]: handle,
|
||||
}
|
||||
})
|
||||
|
||||
const start = Effect.fnUntraced(function* (text: string, projectID?: string) {
|
||||
const directory = projectID ? yield* projectDirectory(projectID) : options.directory
|
||||
if (!directory) return toolError("Use a project ID returned by find_projects.")
|
||||
const session = yield* request((signal) =>
|
||||
options.client.session.create({ location: { directory }, model: options.model }, { signal }),
|
||||
)
|
||||
return yield* admit(session.id, text)
|
||||
})
|
||||
|
||||
const tools: Record<string, Tool> = {
|
||||
find_projects: {
|
||||
description: "Find known OpenCode projects by display name. Returns opaque IDs, never filesystem paths.",
|
||||
parameters: {
|
||||
type: "object",
|
||||
additionalProperties: false,
|
||||
properties: {
|
||||
query: { type: ["string", "null"], description: "Name fragment, or null for recent projects." },
|
||||
limit: { type: "integer", minimum: 1, maximum: 20 },
|
||||
},
|
||||
required: ["query", "limit"],
|
||||
},
|
||||
execute: Effect.fnUntraced(function* (input) {
|
||||
const query = typeof input["query"] === "string" ? input["query"].toLowerCase() : undefined
|
||||
const limit = typeof input["limit"] === "number" ? input["limit"] : 10
|
||||
const projects = (yield* listProjects())
|
||||
.filter((project) => !query || projectLabel(project).toLowerCase().includes(query))
|
||||
.slice(0, limit)
|
||||
projects.forEach((project) => knownProjects.add(project.id))
|
||||
return {
|
||||
status: "ok",
|
||||
projects: projects.map((project) => ({
|
||||
id: project.id,
|
||||
name: projectLabel(project),
|
||||
directories: 1 + project.sandboxes.length,
|
||||
updated: new Date(project.time.updated).toISOString(),
|
||||
})),
|
||||
}
|
||||
}),
|
||||
},
|
||||
find_sessions: {
|
||||
description:
|
||||
"Find root OpenCode sessions by title and recency. Search the launch project by default; use all_projects only when requested or the local search misses.",
|
||||
parameters: {
|
||||
type: "object",
|
||||
additionalProperties: false,
|
||||
properties: {
|
||||
query: { type: ["string", "null"], description: "Title words, or null for recent sessions." },
|
||||
scope: { type: "string", enum: ["current_project", "all_projects"] },
|
||||
recency: { type: "string", enum: ["day", "week", "month", "any"] },
|
||||
limit: { type: "integer", minimum: 1, maximum: 20 },
|
||||
},
|
||||
required: ["query", "scope", "recency", "limit"],
|
||||
},
|
||||
execute: Effect.fnUntraced(function* (input) {
|
||||
const query = typeof input["query"] === "string" ? input["query"] : undefined
|
||||
const scope = input["scope"] === "all_projects" ? "all_projects" : "current_project"
|
||||
const recency = typeof input["recency"] === "string" ? input["recency"] : "any"
|
||||
const limit = typeof input["limit"] === "number" ? input["limit"] : 10
|
||||
const durations: Record<string, number> = { day: 86_400_000, week: 604_800_000, month: 2_592_000_000 }
|
||||
const threshold = recency === "any" ? 0 : Date.now() - (durations[recency] ?? 0)
|
||||
const [result, projectList] = yield* Effect.all(
|
||||
[
|
||||
request((signal) =>
|
||||
options.client.session.list(
|
||||
{
|
||||
...(scope === "current_project" ? { directory: options.directory } : {}),
|
||||
search: query,
|
||||
parentID: null,
|
||||
limit: recency === "any" ? limit : Math.min(limit * 5, 100),
|
||||
order: "desc",
|
||||
},
|
||||
{ signal },
|
||||
),
|
||||
),
|
||||
listProjects(),
|
||||
],
|
||||
{ concurrency: "unbounded" },
|
||||
)
|
||||
const projects = new Map(projectList.map((project) => [project.id, projectLabel(project)]))
|
||||
const sessions = result.data
|
||||
.filter((session) => projects.has(session.projectID) && session.time.updated >= threshold)
|
||||
.slice(0, limit)
|
||||
sessions.forEach((session) => knownSessions.add(session.id))
|
||||
return {
|
||||
status: "ok",
|
||||
scope,
|
||||
sessions: sessions.map((session) => ({
|
||||
id: session.id,
|
||||
title: session.title,
|
||||
project: projects.get(session.projectID) ?? "project",
|
||||
updated: new Date(session.time.updated).toISOString(),
|
||||
})),
|
||||
}
|
||||
}),
|
||||
},
|
||||
read_session: {
|
||||
description:
|
||||
"Read bounded recent user and assistant text from a session returned by find_sessions or start_session. This never prompts or wakes the coding agent.",
|
||||
parameters: {
|
||||
type: "object",
|
||||
additionalProperties: false,
|
||||
properties: {
|
||||
session_id: { type: "string", description: "Session ID returned by a previous voice tool." },
|
||||
limit: { type: "integer", minimum: 1, maximum: 20 },
|
||||
},
|
||||
required: ["session_id", "limit"],
|
||||
},
|
||||
execute: Effect.fnUntraced(function* (input) {
|
||||
const sessionID = knownSession(input, knownSessions)
|
||||
if (!sessionID) return toolError("Use a session ID returned by find_sessions or start_session.")
|
||||
const limit = typeof input["limit"] === "number" ? input["limit"] : 10
|
||||
const [session, messages] = yield* Effect.all(
|
||||
[
|
||||
request((signal) => options.client.session.get({ sessionID }, { signal })),
|
||||
request((signal) => options.client.message.list({ sessionID, order: "desc", limit }, { signal })),
|
||||
],
|
||||
{ concurrency: "unbounded" },
|
||||
)
|
||||
const latestAssistant = messages.data.find(
|
||||
(message): message is SessionMessageAssistant => message.type === "assistant",
|
||||
)
|
||||
return {
|
||||
status: "ok",
|
||||
title: session.title,
|
||||
running: latestAssistant ? !latestAssistant.time.completed : false,
|
||||
messages: messages.data
|
||||
.toReversed()
|
||||
.flatMap((message) =>
|
||||
message.type === "user" || message.type === "assistant" ? [sessionMessage(message)] : [],
|
||||
),
|
||||
}
|
||||
}),
|
||||
},
|
||||
rename_session: {
|
||||
description:
|
||||
"Rename a session returned by find_sessions or start_session after the user requests a new title. This changes only the display title and does not prompt, wake, or interrupt the coding agent.",
|
||||
parameters: {
|
||||
type: "object",
|
||||
additionalProperties: false,
|
||||
properties: {
|
||||
session_id: { type: "string", description: "Session ID returned by a previous voice tool." },
|
||||
title: { type: "string", description: "The complete new session title." },
|
||||
},
|
||||
required: ["session_id", "title"],
|
||||
},
|
||||
execute: Effect.fnUntraced(function* (input) {
|
||||
const sessionID = knownSession(input, knownSessions)
|
||||
const title = requireString(input, "title")?.trim()
|
||||
if (!sessionID) return toolError("Use a session ID returned by find_sessions or start_session.")
|
||||
if (!title) return toolError("A non-empty session title is required.")
|
||||
const session = yield* request((signal) => options.client.session.get({ sessionID }, { signal }))
|
||||
if (session.title === title)
|
||||
return { status: "unchanged", session_id: sessionID, previous_title: session.title, title }
|
||||
yield* request((signal) => options.client.session.rename({ sessionID, title }, { signal }))
|
||||
return { status: "renamed", session_id: sessionID, previous_title: session.title, title }
|
||||
}),
|
||||
},
|
||||
start_session: {
|
||||
description:
|
||||
"Create an OpenCode session, admit its first prompt, and register a one-shot completion notification. Omit project_id to use the launch project.",
|
||||
parameters: {
|
||||
type: "object",
|
||||
additionalProperties: false,
|
||||
properties: {
|
||||
text: { type: "string", description: "Clear instruction for the coding agent." },
|
||||
project_id: { type: ["string", "null"], description: "Opaque project ID, or null for the launch project." },
|
||||
},
|
||||
required: ["text", "project_id"],
|
||||
},
|
||||
execute: (input) => {
|
||||
const text = requireString(input, "text")
|
||||
if (!text) return Effect.succeed(toolError("Task text is required."))
|
||||
return start(text, typeof input["project_id"] === "string" ? input["project_id"] : undefined)
|
||||
},
|
||||
},
|
||||
prompt_session: {
|
||||
description:
|
||||
"Admit a prompt into a discovered or previously started OpenCode session and register a one-shot completion notification.",
|
||||
parameters: {
|
||||
type: "object",
|
||||
additionalProperties: false,
|
||||
properties: {
|
||||
session_id: { type: "string", description: "Session ID returned by a previous voice tool." },
|
||||
text: { type: "string", description: "Clear instruction for the coding agent." },
|
||||
},
|
||||
required: ["session_id", "text"],
|
||||
},
|
||||
execute: (input) => {
|
||||
const sessionID = knownSession(input, knownSessions)
|
||||
const text = requireString(input, "text")
|
||||
if (!sessionID) return Effect.succeed(toolError("Use a session ID returned by find_sessions or start_session."))
|
||||
if (!text) return Effect.succeed(toolError("Task text is required."))
|
||||
return admit(sessionID, text)
|
||||
},
|
||||
},
|
||||
interrupt_session: {
|
||||
description: "Interrupt one OpenCode session. Call only after the user explicitly confirms the interruption.",
|
||||
parameters: sessionParameters(),
|
||||
execute: Effect.fnUntraced(function* (input) {
|
||||
const sessionID = knownSession(input, knownSessions)
|
||||
if (!sessionID) return toolError("Use a session ID returned by a previous voice tool.")
|
||||
yield* request((signal) => options.client.session.interrupt({ sessionID }, { signal }))
|
||||
return { status: "interrupted", session_id: sessionID }
|
||||
}),
|
||||
},
|
||||
list_pending_permissions: {
|
||||
description: "List permission requests blocking one OpenCode session.",
|
||||
parameters: sessionParameters(),
|
||||
execute: Effect.fnUntraced(function* (input) {
|
||||
const sessionID = knownSession(input, knownSessions)
|
||||
if (!sessionID) return toolError("Use a session ID returned by a previous voice tool.")
|
||||
const requests = yield* request((signal) => options.client.permission.list({ sessionID }, { signal }))
|
||||
return {
|
||||
status: "ok",
|
||||
requests: requests.map((item) => ({ id: item.id, action: item.action, resources: item.resources })),
|
||||
}
|
||||
}),
|
||||
},
|
||||
reply_permission: {
|
||||
description:
|
||||
"Allow once or reject a pending permission after stating the action and resources and receiving the user's explicit decision.",
|
||||
parameters: {
|
||||
type: "object",
|
||||
additionalProperties: false,
|
||||
properties: {
|
||||
session_id: { type: "string" },
|
||||
request_id: { type: "string" },
|
||||
decision: { type: "string", enum: ["allow_once", "reject"] },
|
||||
},
|
||||
required: ["session_id", "request_id", "decision"],
|
||||
},
|
||||
execute: Effect.fnUntraced(function* (input) {
|
||||
const sessionID = knownSession(input, knownSessions)
|
||||
const requestID = requireString(input, "request_id")
|
||||
const decision = input["decision"]
|
||||
if (!sessionID || !requestID || (decision !== "allow_once" && decision !== "reject"))
|
||||
return toolError("A known session, request ID, and valid decision are required.")
|
||||
const requests = yield* request((signal) => options.client.permission.list({ sessionID }, { signal }))
|
||||
if (!requests.some((item) => item.id === requestID))
|
||||
return toolError("That permission request is not pending.", true)
|
||||
yield* request((signal) =>
|
||||
options.client.permission.reply(
|
||||
{ sessionID, requestID, reply: decision === "allow_once" ? "once" : "reject" },
|
||||
{ signal },
|
||||
),
|
||||
)
|
||||
return { status: decision === "allow_once" ? "allowed_once" : "rejected", request_id: requestID }
|
||||
}),
|
||||
},
|
||||
reply_question: {
|
||||
description: "Reply to questions blocking an OpenCode session after collecting the user's answers.",
|
||||
parameters: {
|
||||
type: "object",
|
||||
additionalProperties: false,
|
||||
properties: {
|
||||
session_id: { type: "string" },
|
||||
request_id: { type: "string" },
|
||||
answers: {
|
||||
type: "array",
|
||||
description: "One string array per question, preserving question order.",
|
||||
items: { type: "array", items: { type: "string" } },
|
||||
},
|
||||
},
|
||||
required: ["session_id", "request_id", "answers"],
|
||||
},
|
||||
execute: Effect.fnUntraced(function* (input) {
|
||||
const sessionID = knownSession(input, knownSessions)
|
||||
const requestID = requireString(input, "request_id")
|
||||
const answers = Option.getOrUndefined(decodeQuestionAnswers(input["answers"]))
|
||||
if (!sessionID || !requestID || !answers)
|
||||
return toolError("A known session, request ID, and valid answers are required.")
|
||||
yield* request((signal) => options.client.question.reply({ sessionID, requestID, answers }, { signal }))
|
||||
return { status: "answered", request_id: requestID }
|
||||
}),
|
||||
},
|
||||
reject_question: {
|
||||
description: "Reject a pending OpenCode question after the user explicitly declines to answer.",
|
||||
parameters: requestParameters(),
|
||||
execute: Effect.fnUntraced(function* (input) {
|
||||
const sessionID = knownSession(input, knownSessions)
|
||||
const requestID = requireString(input, "request_id")
|
||||
if (!sessionID || !requestID) return toolError("A known session and request ID are required.")
|
||||
yield* request((signal) => options.client.question.reject({ sessionID, requestID }, { signal }))
|
||||
return { status: "rejected", request_id: requestID }
|
||||
}),
|
||||
},
|
||||
reply_form: {
|
||||
description: "Submit values for a form blocking an OpenCode session after collecting them from the user.",
|
||||
parameters: {
|
||||
type: "object",
|
||||
additionalProperties: false,
|
||||
properties: {
|
||||
session_id: { type: "string" },
|
||||
form_id: { type: "string" },
|
||||
answer: { type: "object", additionalProperties: true },
|
||||
},
|
||||
required: ["session_id", "form_id", "answer"],
|
||||
},
|
||||
execute: Effect.fnUntraced(function* (input) {
|
||||
const sessionID = knownSession(input, knownSessions)
|
||||
const formID = requireString(input, "form_id")
|
||||
const answer = Option.getOrUndefined(decodeFormAnswer(input["answer"]))
|
||||
if (!sessionID || !formID || !answer)
|
||||
return toolError("A known session, form ID, and valid answer are required.")
|
||||
yield* request((signal) => options.client.form.reply({ sessionID, formID, answer }, { signal }))
|
||||
return { status: "answered", form_id: formID }
|
||||
}),
|
||||
},
|
||||
cancel_form: {
|
||||
description: "Cancel a pending OpenCode form after the user explicitly declines to complete it.",
|
||||
parameters: {
|
||||
type: "object",
|
||||
additionalProperties: false,
|
||||
properties: { session_id: { type: "string" }, form_id: { type: "string" } },
|
||||
required: ["session_id", "form_id"],
|
||||
},
|
||||
execute: Effect.fnUntraced(function* (input) {
|
||||
const sessionID = knownSession(input, knownSessions)
|
||||
const formID = requireString(input, "form_id")
|
||||
if (!sessionID || !formID) return toolError("A known session and form ID are required.")
|
||||
yield* request((signal) => options.client.form.cancel({ sessionID, formID }, { signal }))
|
||||
return { status: "cancelled", form_id: formID }
|
||||
}),
|
||||
},
|
||||
}
|
||||
|
||||
const definitions = Object.entries(tools).map(
|
||||
([name, tool]) =>
|
||||
({ type: "function", name, description: tool.description, parameters: tool.parameters }) satisfies VoiceTool,
|
||||
)
|
||||
|
||||
yield* Effect.suspend(() =>
|
||||
Stream.fromAsyncIterable(options.client.event.subscribe({ signal: eventAbort.signal }), (cause) => cause).pipe(
|
||||
Stream.runForEach((event) => {
|
||||
if (event.type === "session.input.promoted") {
|
||||
if (!registrations.has(event.data.inputID)) return Effect.void
|
||||
promoted.set(event.data.sessionID, event.data.inputID)
|
||||
return trace("opencode.prompt.promoted", {
|
||||
sessionID: event.data.sessionID,
|
||||
promptID: event.data.inputID,
|
||||
})
|
||||
}
|
||||
const sessionID = blockerSession(event)
|
||||
const promptID = sessionID ? (promoted.get(sessionID) ?? latest.get(sessionID)) : undefined
|
||||
const blocker = sessionID && promptID ? blockerNotification(event, sessionID, promptID) : undefined
|
||||
if (!blocker || !promptID) return Effect.void
|
||||
options.trace?.("opencode.prompt.blocked", { sessionID, promptID, blocker: blocker["blocker"] })
|
||||
if (registrations.has(promptID)) return announceBlocker(blockerKey(blocker), blocker)
|
||||
return Effect.void
|
||||
}),
|
||||
Effect.catch((error) => notify({ type: "opencode.events.failed", error: String(error) })),
|
||||
Effect.andThen(Effect.sleep("1 second")),
|
||||
),
|
||||
).pipe(Effect.forever, Effect.forkScoped({ startImmediately: true }))
|
||||
|
||||
yield* Effect.forEach(
|
||||
options.completionStore.entries(),
|
||||
Effect.fnUntraced(function* (entry) {
|
||||
if (entry.status !== "completed") {
|
||||
const exists = yield* request((signal) =>
|
||||
options.client.session.get({ sessionID: entry.handle.sessionID }, { signal }),
|
||||
).pipe(
|
||||
Effect.as(true),
|
||||
Effect.catch((error) => {
|
||||
if (!isSessionNotFoundError(error)) return Effect.fail(error)
|
||||
return request(() => options.completionStore.remove(entry.handle)).pipe(
|
||||
Effect.andThen(trace("opencode.prompt.orphaned", entry.handle)),
|
||||
Effect.as(false),
|
||||
)
|
||||
}),
|
||||
)
|
||||
if (!exists) return
|
||||
}
|
||||
knownSessions.add(entry.handle.sessionID)
|
||||
if (entry.status === "admitting") {
|
||||
registrations.set(entry.handle.promptID, entry.handle)
|
||||
latest.set(entry.handle.sessionID, entry.handle.promptID)
|
||||
yield* trace("opencode.prompt.reconciling", entry.handle)
|
||||
yield* request((signal) =>
|
||||
options.client.session.prompt(
|
||||
{ sessionID: entry.handle.sessionID, id: entry.handle.promptID, text: entry.text },
|
||||
{ signal },
|
||||
),
|
||||
)
|
||||
yield* request(() => options.completionStore.pending(entry.handle))
|
||||
yield* restoreBlockers(entry.handle)
|
||||
yield* register(entry.handle)
|
||||
return
|
||||
}
|
||||
if (entry.status === "pending") {
|
||||
registrations.set(entry.handle.promptID, entry.handle)
|
||||
latest.set(entry.handle.sessionID, entry.handle.promptID)
|
||||
yield* trace("opencode.wait.restored", entry.handle)
|
||||
yield* restoreBlockers(entry.handle)
|
||||
yield* register(entry.handle)
|
||||
return
|
||||
}
|
||||
yield* trace("opencode.notification.restored", entry.handle)
|
||||
yield* notify(entry.notification, entry.handle)
|
||||
}),
|
||||
{ concurrency: 4, discard: true },
|
||||
)
|
||||
|
||||
return Bridge.of({
|
||||
definitions,
|
||||
execute: (name, input) =>
|
||||
(tools[name]?.execute(input) ?? Effect.succeed(toolError(`Unknown tool ${name}.`))).pipe(
|
||||
Effect.map(toolExecution),
|
||||
),
|
||||
acknowledge: (receipt) => request(() => options.completionStore.remove(receipt)),
|
||||
close: Effect.all([Effect.sync(() => eventAbort.abort()), FiberMap.clear(completions)], { discard: true }),
|
||||
})
|
||||
})
|
||||
|
||||
function toolExecution(output: unknown): VoiceToolExecution {
|
||||
if (!isPromptToolOutput(output)) return { output }
|
||||
return {
|
||||
output: Object.fromEntries(Object.entries(output)),
|
||||
admittedPrompt: output[AdmittedPrompt],
|
||||
}
|
||||
}
|
||||
|
||||
function isPromptToolOutput(
|
||||
output: unknown,
|
||||
): output is Record<string, unknown> & { readonly [AdmittedPrompt]: PromptHandle } {
|
||||
if (!output || typeof output !== "object" || !(AdmittedPrompt in output)) return false
|
||||
const handle = output[AdmittedPrompt]
|
||||
return (
|
||||
!!handle &&
|
||||
typeof handle === "object" &&
|
||||
"sessionID" in handle &&
|
||||
typeof handle.sessionID === "string" &&
|
||||
"promptID" in handle &&
|
||||
typeof handle.promptID === "string"
|
||||
)
|
||||
}
|
||||
|
||||
async function finalReply(client: Client, handle: PromptHandle, signal: AbortSignal) {
|
||||
let reply: SessionMessageAssistant | undefined
|
||||
let cursor: string | undefined
|
||||
while (true) {
|
||||
const page = await client.message.list(
|
||||
cursor
|
||||
? { sessionID: handle.sessionID, limit: 200, cursor }
|
||||
: { sessionID: handle.sessionID, limit: 200, order: "desc" },
|
||||
{ signal },
|
||||
)
|
||||
for (const message of page.data) {
|
||||
if (message.id === handle.promptID) return reply
|
||||
if (message.type === "user") reply = undefined
|
||||
if (message.type === "assistant" && !reply) reply = message
|
||||
}
|
||||
cursor = page.cursor.next ?? undefined
|
||||
if (!cursor) return undefined
|
||||
}
|
||||
}
|
||||
|
||||
function assistantText(message: SessionMessageAssistant) {
|
||||
return message.content
|
||||
.filter((part) => part.type === "text")
|
||||
.map((part) => part.text)
|
||||
.join("\n")
|
||||
.slice(0, 4_000)
|
||||
}
|
||||
|
||||
function sessionMessage(message: SessionMessageUser | SessionMessageAssistant) {
|
||||
if (message.type === "user") return { role: "user", text: message.text.slice(0, 2_000) }
|
||||
return {
|
||||
role: "assistant",
|
||||
status: message.time.completed ? "completed" : "running",
|
||||
text: assistantText(message),
|
||||
tools: message.content
|
||||
.filter((part) => part.type === "tool")
|
||||
.map((part) => ({ name: part.name, status: part.state.status })),
|
||||
}
|
||||
}
|
||||
|
||||
function blockerSession(event: V2Event) {
|
||||
if (event.type === "permission.v2.asked" || event.type === "question.v2.asked") return event.data.sessionID
|
||||
if (event.type === "form.created") return event.data.form.sessionID
|
||||
return undefined
|
||||
}
|
||||
|
||||
function blockerNotification(event: V2Event, sessionID: string, promptID: string): OpenCodePromptBlocked | undefined {
|
||||
if (event.type === "permission.v2.asked")
|
||||
return {
|
||||
type: "opencode.prompt.blocked",
|
||||
prompt_id: promptID,
|
||||
blocker: "permission",
|
||||
session_id: sessionID,
|
||||
request_id: event.data.id,
|
||||
action: event.data.action,
|
||||
resources: event.data.resources,
|
||||
}
|
||||
if (event.type === "question.v2.asked")
|
||||
return {
|
||||
type: "opencode.prompt.blocked",
|
||||
prompt_id: promptID,
|
||||
blocker: "question",
|
||||
session_id: sessionID,
|
||||
request_id: event.data.id,
|
||||
questions: event.data.questions,
|
||||
}
|
||||
if (event.type === "form.created")
|
||||
return {
|
||||
type: "opencode.prompt.blocked",
|
||||
prompt_id: promptID,
|
||||
blocker: "form",
|
||||
session_id: sessionID,
|
||||
form_id: event.data.form.id,
|
||||
title: event.data.form.title,
|
||||
fields: event.data.form.fields,
|
||||
}
|
||||
return undefined
|
||||
}
|
||||
|
||||
function blockerKey(notification: OpenCodePromptBlocked) {
|
||||
if (notification.blocker === "form") return `form:${notification.form_id}`
|
||||
return `${notification.blocker}:${notification.request_id}`
|
||||
}
|
||||
|
||||
function projectLabel(project: { readonly name?: string; readonly worktree: string }) {
|
||||
return project.name ?? (basename(project.worktree) || "project")
|
||||
}
|
||||
|
||||
function requireString(input: Record<string, unknown>, key: string) {
|
||||
const value = input[key]
|
||||
if (typeof value !== "string" || value.trim().length === 0) return undefined
|
||||
return value
|
||||
}
|
||||
|
||||
function knownSession(input: Record<string, unknown>, known: ReadonlySet<string>) {
|
||||
const sessionID = requireString(input, "session_id")
|
||||
if (!sessionID || !known.has(sessionID)) return undefined
|
||||
return sessionID
|
||||
}
|
||||
|
||||
const decodeQuestionAnswers = Schema.decodeUnknownOption(Schema.Array(Question.Answer))
|
||||
const decodeFormAnswer = Schema.decodeUnknownOption(Form.Answer)
|
||||
|
||||
function sessionParameters() {
|
||||
return {
|
||||
type: "object",
|
||||
additionalProperties: false,
|
||||
properties: { session_id: { type: "string" } },
|
||||
required: ["session_id"],
|
||||
}
|
||||
}
|
||||
|
||||
function requestParameters() {
|
||||
return {
|
||||
type: "object",
|
||||
additionalProperties: false,
|
||||
properties: { session_id: { type: "string" }, request_id: { type: "string" } },
|
||||
required: ["session_id", "request_id"],
|
||||
}
|
||||
}
|
||||
|
||||
function toolError(message: string, retryable = false) {
|
||||
return { status: "error", message, retryable }
|
||||
}
|
||||
@@ -1,19 +0,0 @@
|
||||
export const PCM_SAMPLE_RATE = 24_000
|
||||
export const PCM_BYTES_PER_MS = (PCM_SAMPLE_RATE * 2) / 1_000
|
||||
export const PCM_METER_FRAME_MS = 1_000 / 60
|
||||
|
||||
export function pcmLevel(bytes: Buffer) {
|
||||
const samples = Math.floor(bytes.length / 2)
|
||||
if (samples === 0) return 0
|
||||
const stride = Math.max(1, Math.floor(samples / 1_200))
|
||||
let energy = 0
|
||||
let count = 0
|
||||
for (let index = 0; index < samples; index += stride) {
|
||||
const sample = bytes.readInt16LE(index * 2) / 32_768
|
||||
energy += sample * sample
|
||||
count += 1
|
||||
}
|
||||
const rms = Math.sqrt(energy / count)
|
||||
if (rms < 0.008) return 0
|
||||
return Math.min(1, Math.sqrt((rms - 0.008) / 0.18))
|
||||
}
|
||||
@@ -1,37 +0,0 @@
|
||||
import { Plugin } from "@opencode-ai/plugin/v2/tui"
|
||||
import { createSignal } from "solid-js"
|
||||
|
||||
function VoiceStatus(props: { context: Plugin.Context }) {
|
||||
const [enabled, setEnabled] = createSignal(false)
|
||||
|
||||
props.context.keymap.layer(() => ({
|
||||
mode: "global",
|
||||
commands: [
|
||||
{
|
||||
id: "voice.smoke.toggle",
|
||||
title: "Toggle voice plugin",
|
||||
description: "Toggle the V2 voice plugin smoke-test status",
|
||||
group: "Voice",
|
||||
bind: "alt+v",
|
||||
palette: true,
|
||||
run() {
|
||||
setEnabled((value) => !value)
|
||||
},
|
||||
},
|
||||
],
|
||||
}))
|
||||
|
||||
return (
|
||||
<box paddingTop={1}>
|
||||
<text>voice plugin: {enabled() ? "on" : "off"} (alt+v)</text>
|
||||
</box>
|
||||
)
|
||||
}
|
||||
|
||||
export default Plugin.define({
|
||||
id: "opencode.voice-smoke",
|
||||
setup(context) {
|
||||
context.ui.slot("home.bottom", () => <VoiceStatus context={context} />)
|
||||
context.ui.slot("sidebar.content", () => <VoiceStatus context={context} />)
|
||||
},
|
||||
})
|
||||
@@ -1,8 +0,0 @@
|
||||
export type PromptHandle = {
|
||||
readonly sessionID: string
|
||||
readonly promptID: string
|
||||
}
|
||||
|
||||
export function promptKey(handle: PromptHandle) {
|
||||
return `${handle.sessionID}:${handle.promptID}`
|
||||
}
|
||||
@@ -1,486 +0,0 @@
|
||||
import type {
|
||||
VoiceConnection,
|
||||
VoiceDelegationRequest,
|
||||
VoiceNotificationRequest,
|
||||
VoiceProtocol,
|
||||
VoiceProtocolEvent,
|
||||
VoiceProtocolOptions,
|
||||
} from "./protocol"
|
||||
import { decodeVoiceToolInput } from "./protocol"
|
||||
import { Option, Schema } from "effect"
|
||||
|
||||
const LiveItem = Schema.Struct({
|
||||
id: Schema.String,
|
||||
type: Schema.String,
|
||||
text: Schema.optional(Schema.String),
|
||||
name: Schema.optional(Schema.String),
|
||||
call_id: Schema.optional(Schema.String),
|
||||
arguments: Schema.optional(Schema.String),
|
||||
target: Schema.optional(Schema.String),
|
||||
content: Schema.optional(Schema.Array(Schema.Struct({ type: Schema.String, text: Schema.optional(Schema.String) }))),
|
||||
})
|
||||
const LiveTurn = Schema.Struct({
|
||||
id: Schema.String,
|
||||
role: Schema.Literals(["user", "assistant"]),
|
||||
transcript: Schema.String,
|
||||
})
|
||||
const LiveEvent = Schema.Struct({
|
||||
type: Schema.String,
|
||||
event_id: Schema.optional(Schema.String),
|
||||
audio: Schema.optional(Schema.String),
|
||||
delta: Schema.optional(Schema.String),
|
||||
start_ms: Schema.optional(Schema.Number),
|
||||
end_ms: Schema.optional(Schema.Number),
|
||||
turn_id: Schema.optional(Schema.String),
|
||||
turn: Schema.optional(LiveTurn),
|
||||
item: Schema.optional(LiveItem),
|
||||
error: Schema.optional(
|
||||
Schema.Struct({
|
||||
code: Schema.optional(Schema.String),
|
||||
message: Schema.optional(Schema.String),
|
||||
event_id: Schema.optional(Schema.String),
|
||||
}),
|
||||
),
|
||||
})
|
||||
const decodeLiveEvent = Schema.decodeUnknownOption(Schema.fromJsonString(LiveEvent))
|
||||
type LiveTurn = Schema.Schema.Type<typeof LiveTurn>
|
||||
type LiveEvent = Schema.Schema.Type<typeof LiveEvent>
|
||||
|
||||
type LiveNotificationResult = {
|
||||
readonly type?: string
|
||||
readonly eventID?: string
|
||||
readonly errorEventID?: string
|
||||
}
|
||||
|
||||
export function liveNotificationAcknowledgement(
|
||||
result: LiveNotificationResult,
|
||||
pending: {
|
||||
readonly eventID: string
|
||||
readonly acknowledgement: "session.context.appended" | "delegation.context.appended"
|
||||
},
|
||||
) {
|
||||
// Live success events do not reliably echo the client event ID. Context
|
||||
// appends are serialized, so the expected acknowledgement kind is the
|
||||
// strongest correlation the protocol currently provides.
|
||||
if (result.type === pending.acknowledgement) return true
|
||||
if (result.type === "error" && result.errorEventID === pending.eventID) return false
|
||||
return undefined
|
||||
}
|
||||
|
||||
type ProjectedTurn = LiveTurn & { readonly displayID: string }
|
||||
|
||||
export function createLiveEventProjector() {
|
||||
const turns = new Map<string, ProjectedTurn>()
|
||||
let input: { readonly id: string; readonly transcript: string } | undefined
|
||||
let assistantTranscript = ""
|
||||
const userTranscripts = new Map<string, string>()
|
||||
const startedTools = new Set<string>()
|
||||
|
||||
const startTool = (item: LiveEvent["item"], events: VoiceProtocolEvent[]) => {
|
||||
if (item?.type !== "function_call" || !item.name || !item.call_id || startedTools.has(item.call_id)) return
|
||||
startedTools.add(item.call_id)
|
||||
events.push({ type: "tool.started", id: item.call_id, name: item.name })
|
||||
}
|
||||
|
||||
const syncUser = (id: string, text: string, final: boolean, events: VoiceProtocolEvent[]) => {
|
||||
text = text.trimStart()
|
||||
const previous = userTranscripts.get(id)
|
||||
if (previous === text && !final) return
|
||||
if (final) userTranscripts.delete(id)
|
||||
else userTranscripts.set(id, text)
|
||||
events.push({ type: "user.transcript", id, text, final })
|
||||
}
|
||||
|
||||
// Deltas are emitted verbatim, matching the Realtime adapter: a fragment's leading space is
|
||||
// the only record of the boundary between two assistant turns, so trimming it here would
|
||||
// weld the last word of one turn onto the first word of the next. Stripping the leading
|
||||
// edge of a rendered message is the consumer's job.
|
||||
const syncAssistant = (transcript: string, events: VoiceProtocolEvent[]) => {
|
||||
if (transcript.startsWith(assistantTranscript)) {
|
||||
const delta = transcript.slice(assistantTranscript.length)
|
||||
assistantTranscript = transcript
|
||||
if (delta) events.push({ type: "assistant.transcript.delta", delta })
|
||||
return
|
||||
}
|
||||
if (assistantTranscript.startsWith(transcript)) {
|
||||
assistantTranscript = transcript
|
||||
events.push({ type: "assistant.transcript", text: transcript })
|
||||
return
|
||||
}
|
||||
assistantTranscript = transcript
|
||||
events.push({ type: "assistant.transcript", text: transcript })
|
||||
}
|
||||
|
||||
return (message: string) => {
|
||||
const data = Option.getOrUndefined(decodeLiveEvent(message))
|
||||
if (!data)
|
||||
return {
|
||||
type: undefined,
|
||||
events: [{ type: "error", message: "Received an invalid Live API event." }] satisfies VoiceProtocolEvent[],
|
||||
}
|
||||
|
||||
const events: VoiceProtocolEvent[] = data.type.endsWith(".delta") ? [] : [{ type: "debug", message: data.type }]
|
||||
switch (data.type) {
|
||||
case "session.started":
|
||||
events.push({ type: "ready" })
|
||||
break
|
||||
case "output_audio.delta":
|
||||
if (data.audio && data.start_ms !== undefined && data.end_ms !== undefined)
|
||||
events.push({
|
||||
type: "assistant.audio",
|
||||
audio: Buffer.from(data.audio, "base64"),
|
||||
timeline: { startMs: data.start_ms, endMs: data.end_ms },
|
||||
})
|
||||
break
|
||||
case "input_transcript.added":
|
||||
if (data.item?.type !== "input_transcript" || data.item.text === undefined) break
|
||||
if (!input) {
|
||||
input = { id: data.item.id, transcript: "" }
|
||||
events.push({ type: "user.committed", id: input.id })
|
||||
}
|
||||
input = { ...input, transcript: input.transcript + data.item.text }
|
||||
syncUser(input.id, input.transcript, false, events)
|
||||
break
|
||||
case "output_transcript.added":
|
||||
if (data.item?.type !== "output_transcript" || data.item.text === undefined) break
|
||||
const delta = data.item.text
|
||||
assistantTranscript += delta
|
||||
if (delta) events.push({ type: "assistant.transcript.delta", delta })
|
||||
break
|
||||
case "turn.created": {
|
||||
const turn = data.turn
|
||||
if (!turn) break
|
||||
const displayID = turn.role === "user" ? (input?.id ?? turn.id) : turn.id
|
||||
turns.set(turn.id, { ...turn, displayID })
|
||||
if (turn.role === "user") {
|
||||
if (!input) events.push({ type: "user.committed", id: displayID })
|
||||
syncUser(displayID, turn.transcript, false, events)
|
||||
break
|
||||
}
|
||||
syncAssistant(turn.transcript, events)
|
||||
break
|
||||
}
|
||||
case "turn.delta": {
|
||||
if (!data.turn_id || !data.delta) break
|
||||
const turn = turns.get(data.turn_id)
|
||||
if (!turn) break
|
||||
const transcript = turn.transcript + data.delta
|
||||
turns.set(data.turn_id, { ...turn, transcript })
|
||||
if (turn.role === "user") {
|
||||
syncUser(turn.displayID, transcript, false, events)
|
||||
break
|
||||
}
|
||||
syncAssistant(transcript, events)
|
||||
break
|
||||
}
|
||||
case "turn.done": {
|
||||
const turn = data.turn
|
||||
if (!turn) break
|
||||
const previous = turns.get(turn.id)
|
||||
if (turn.role === "user") {
|
||||
const displayID = previous?.displayID ?? input?.id ?? turn.id
|
||||
if (!previous && !input) events.push({ type: "user.committed", id: displayID })
|
||||
syncUser(displayID, turn.transcript, true, events)
|
||||
input = undefined
|
||||
turns.delete(turn.id)
|
||||
break
|
||||
}
|
||||
syncAssistant(turn.transcript, events)
|
||||
assistantTranscript = ""
|
||||
turns.delete(turn.id)
|
||||
events.push({ type: "assistant.done", awaitingWork: false })
|
||||
break
|
||||
}
|
||||
case "response.output_item.added":
|
||||
startTool(data.item, events)
|
||||
break
|
||||
case "delegation.created":
|
||||
if (data.item?.type !== "delegation" || data.item.target !== "client") break
|
||||
events.push({
|
||||
type: "delegation.requested",
|
||||
request: {
|
||||
id: data.item.id,
|
||||
text:
|
||||
data.item.content
|
||||
?.flatMap((part) => (part.type === "input_text" && part.text ? [part.text] : []))
|
||||
.join("\n") ?? "",
|
||||
},
|
||||
})
|
||||
break
|
||||
case "response.output_item.done": {
|
||||
if (data.item?.type !== "function_call" || !data.item.call_id) break
|
||||
const name = data.item.name ?? "tool"
|
||||
startTool(data.item, events)
|
||||
if (!data.item.name || data.item.arguments === undefined) {
|
||||
const output = { status: "error", message: "Malformed Live function call." }
|
||||
events.push({
|
||||
type: "work.rejected",
|
||||
request: { id: data.item.call_id, name, input: {} },
|
||||
output,
|
||||
})
|
||||
startedTools.delete(data.item.call_id)
|
||||
break
|
||||
}
|
||||
const input = Option.getOrUndefined(decodeVoiceToolInput(data.item.arguments))
|
||||
if (!input) {
|
||||
const output = { status: "error", message: `Invalid arguments for tool ${data.item.name}.` }
|
||||
events.push({ type: "error", message: `Received invalid arguments for Live tool ${data.item.name}.` })
|
||||
events.push({
|
||||
type: "work.rejected",
|
||||
request: { id: data.item.call_id, name, input: {} },
|
||||
output,
|
||||
})
|
||||
startedTools.delete(data.item.call_id)
|
||||
break
|
||||
}
|
||||
events.push({
|
||||
type: "work.requested",
|
||||
request: { id: data.item.call_id, name: data.item.name, input },
|
||||
})
|
||||
startedTools.delete(data.item.call_id)
|
||||
break
|
||||
}
|
||||
case "error":
|
||||
events.push({ type: "error", message: `${data.error?.code}: ${data.error?.message}` })
|
||||
}
|
||||
return {
|
||||
type: data.type,
|
||||
eventID: data.event_id,
|
||||
errorEventID: data.error?.event_id,
|
||||
events,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
export function createLiveProtocol(): VoiceProtocol {
|
||||
return {
|
||||
name: "live",
|
||||
inputActivity: "local",
|
||||
capabilities: { textInput: false, interruption: false, delegation: true },
|
||||
connect: connectLive,
|
||||
}
|
||||
}
|
||||
|
||||
export function createLiveEventDelivery(options: {
|
||||
readonly emit: (event: VoiceProtocolEvent) => void
|
||||
readonly drainMs?: number
|
||||
}) {
|
||||
let pendingDone: Extract<VoiceProtocolEvent, { readonly type: "assistant.done" }> | undefined
|
||||
let timer: ReturnType<typeof setTimeout> | undefined
|
||||
|
||||
const schedule = () => {
|
||||
if (timer) clearTimeout(timer)
|
||||
timer = setTimeout(() => {
|
||||
timer = undefined
|
||||
const event = pendingDone
|
||||
pendingDone = undefined
|
||||
if (event) options.emit(event)
|
||||
}, options.drainMs ?? 120)
|
||||
}
|
||||
|
||||
return {
|
||||
push(event: VoiceProtocolEvent) {
|
||||
if (event.type === "assistant.done") {
|
||||
pendingDone = event
|
||||
schedule()
|
||||
return
|
||||
}
|
||||
if (event.type === "assistant.audio" && pendingDone) schedule()
|
||||
options.emit(event)
|
||||
},
|
||||
close() {
|
||||
if (timer) clearTimeout(timer)
|
||||
timer = undefined
|
||||
pendingDone = undefined
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
export function createLiveContextAppendQueue(options: {
|
||||
readonly send: (event: Record<string, unknown>) => boolean
|
||||
readonly timeoutMs?: number
|
||||
}) {
|
||||
type Item = {
|
||||
readonly event: Record<string, unknown>
|
||||
readonly acknowledgement: "session.context.appended" | "delegation.context.appended"
|
||||
readonly result: PromiseWithResolvers<boolean>
|
||||
}
|
||||
const queued: Item[] = []
|
||||
let active: (Item & { readonly timer: ReturnType<typeof setTimeout> }) | undefined
|
||||
let closed = false
|
||||
|
||||
const settle = (accepted: boolean) => {
|
||||
if (!active) return
|
||||
clearTimeout(active.timer)
|
||||
active.result.resolve(accepted)
|
||||
active = undefined
|
||||
pump()
|
||||
}
|
||||
|
||||
const pump = () => {
|
||||
if (closed || active) return
|
||||
const item = queued.shift()
|
||||
if (!item) return
|
||||
active = {
|
||||
...item,
|
||||
timer: setTimeout(() => settle(false), options.timeoutMs ?? 5_000),
|
||||
}
|
||||
if (!options.send(item.event)) settle(false)
|
||||
}
|
||||
|
||||
return {
|
||||
append(
|
||||
event: Record<string, unknown>,
|
||||
acknowledgement: "session.context.appended" | "delegation.context.appended",
|
||||
) {
|
||||
if (closed) return Promise.resolve(false)
|
||||
const result = Promise.withResolvers<boolean>()
|
||||
queued.push({ event, acknowledgement, result })
|
||||
pump()
|
||||
return result.promise
|
||||
},
|
||||
receive(result: LiveNotificationResult) {
|
||||
if (!active) return
|
||||
const eventID = active.event["event_id"]
|
||||
const accepted = liveNotificationAcknowledgement(result, {
|
||||
eventID: typeof eventID === "string" ? eventID : "",
|
||||
acknowledgement: active.acknowledgement,
|
||||
})
|
||||
if (accepted !== undefined) settle(accepted)
|
||||
},
|
||||
close() {
|
||||
closed = true
|
||||
if (active) {
|
||||
clearTimeout(active.timer)
|
||||
active.result.resolve(false)
|
||||
active = undefined
|
||||
}
|
||||
queued.splice(0).forEach((item) => item.result.resolve(false))
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
function connectLive(options: VoiceProtocolOptions): VoiceConnection {
|
||||
// oxlint-disable-next-line typescript-eslint/no-unsafe-type-assertion -- Bun accepts WebSocket headers in this runtime-only overload.
|
||||
const ws = new WebSocket(`wss://api.openai.com/v1/live?model=${options.model}`, {
|
||||
headers: {
|
||||
Authorization: `Bearer ${options.apiKey}`,
|
||||
"OpenAI-Alpha": "quicksilver=v2",
|
||||
},
|
||||
} as unknown as string[])
|
||||
const closed = Promise.withResolvers<void>()
|
||||
let closeTimer: ReturnType<typeof setTimeout> | undefined
|
||||
const project = createLiveEventProjector()
|
||||
const delivery = createLiveEventDelivery({ emit: options.onEvent })
|
||||
|
||||
const send = (event: Record<string, unknown>) => {
|
||||
if (options.debug || event["type"] !== "input_audio.append")
|
||||
options.trace?.("live.send", {
|
||||
type: event["type"],
|
||||
delegationID: event["delegation_item_id"],
|
||||
})
|
||||
if (ws.readyState !== WebSocket.OPEN) return false
|
||||
ws.send(JSON.stringify(event))
|
||||
return true
|
||||
}
|
||||
const context = createLiveContextAppendQueue({ send })
|
||||
|
||||
const appendNotification = (request: VoiceNotificationRequest) =>
|
||||
context.append(
|
||||
{
|
||||
type: "session.context.append",
|
||||
event_id: crypto.randomUUID(),
|
||||
content: [{ type: "input_text", text: notificationText(request.text) }],
|
||||
},
|
||||
"session.context.appended",
|
||||
)
|
||||
|
||||
ws.addEventListener("open", () => {
|
||||
send({
|
||||
type: "session.update",
|
||||
event_id: crypto.randomUUID(),
|
||||
session: {
|
||||
instructions: options.instructions,
|
||||
audio: { output: { voice: options.voice } },
|
||||
delegation: { type: "client" },
|
||||
},
|
||||
})
|
||||
})
|
||||
ws.addEventListener("message", (event) => {
|
||||
const result = project(String(event.data))
|
||||
if (options.debug || result.type !== "output_audio.delta") options.trace?.("live.receive", { type: result.type })
|
||||
context.receive(result)
|
||||
result.events.forEach((event) => {
|
||||
if (event.type === "assistant.audio")
|
||||
options.trace?.("live.audio.received", {
|
||||
bytes: event.audio.length,
|
||||
startMs: event.timeline?.startMs,
|
||||
endMs: event.timeline?.endMs,
|
||||
})
|
||||
delivery.push(event)
|
||||
})
|
||||
if (result.type === "session.closed") ws.close(1000)
|
||||
})
|
||||
ws.addEventListener("close", (event) => {
|
||||
if (closeTimer) clearTimeout(closeTimer)
|
||||
context.close()
|
||||
delivery.close()
|
||||
options.onEvent({ type: "closed", code: event.code })
|
||||
closed.resolve()
|
||||
})
|
||||
|
||||
return {
|
||||
appendAudio(audio) {
|
||||
if (ws.bufferedAmount > 96_000) {
|
||||
options.trace?.("live.audio.dropped", { bytes: audio.length, buffered: ws.bufferedAmount })
|
||||
return
|
||||
}
|
||||
send({ type: "input_audio.append", audio: audio.toString("base64") })
|
||||
},
|
||||
resolveWork(request, output) {
|
||||
send({
|
||||
type: "delegation.function_call_output.create",
|
||||
event_id: crypto.randomUUID(),
|
||||
item: { type: "function_call_output", call_id: request.id, output: JSON.stringify(output) },
|
||||
})
|
||||
},
|
||||
resolveDelegation(request: VoiceDelegationRequest, output: string) {
|
||||
void context
|
||||
.append(delegationContext(crypto.randomUUID(), request.id, output), "delegation.context.appended")
|
||||
.then((accepted) => {
|
||||
if (!accepted) options.trace?.("live.delegation.context.failed", { delegationID: request.id })
|
||||
})
|
||||
},
|
||||
notify(request) {
|
||||
return appendNotification(request)
|
||||
},
|
||||
close(closeOptions) {
|
||||
if (ws.readyState === WebSocket.CLOSED) return Promise.resolve()
|
||||
if (closeOptions?.graceful === false || ws.readyState !== WebSocket.OPEN) {
|
||||
ws.close(1000)
|
||||
closeTimer = setTimeout(() => closed.resolve(), 5_000)
|
||||
return closed.promise
|
||||
}
|
||||
send({ type: "session.close", event_id: crypto.randomUUID() })
|
||||
closeTimer = setTimeout(() => {
|
||||
ws.close(1000)
|
||||
closed.resolve()
|
||||
}, 10_500)
|
||||
return closed.promise
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
function delegationContext(eventID: string, delegationID: string, text: string) {
|
||||
return {
|
||||
type: "delegation.context.append",
|
||||
event_id: eventID,
|
||||
delegation_item_id: delegationID,
|
||||
content: [{ type: "input_text", text: notificationText(text) }],
|
||||
}
|
||||
}
|
||||
|
||||
function notificationText(output: unknown) {
|
||||
const text = String(output)
|
||||
return text.length > 1_600 ? text.slice(0, 1_600) + "..." : text
|
||||
}
|
||||
@@ -1,318 +0,0 @@
|
||||
import type { VoiceConnection, VoiceProtocol, VoiceProtocolEvent, VoiceProtocolOptions } from "./protocol"
|
||||
import { decodeVoiceToolInput } from "./protocol"
|
||||
import { Option, Schema } from "effect"
|
||||
import { createSingleFlightAcknowledgement } from "./single-flight-acknowledgement"
|
||||
|
||||
const RealtimeItem = Schema.Struct({
|
||||
id: Schema.optional(Schema.String),
|
||||
type: Schema.optional(Schema.String),
|
||||
name: Schema.optional(Schema.String),
|
||||
call_id: Schema.optional(Schema.String),
|
||||
arguments: Schema.optional(Schema.String),
|
||||
})
|
||||
const RealtimeFunctionCall = Schema.Struct({
|
||||
type: Schema.Literal("function_call"),
|
||||
name: Schema.String,
|
||||
call_id: Schema.String,
|
||||
arguments: Schema.String,
|
||||
})
|
||||
const RealtimeEvent = Schema.Struct({
|
||||
type: Schema.String,
|
||||
delta: Schema.optional(Schema.String),
|
||||
transcript: Schema.optional(Schema.String),
|
||||
item_id: Schema.optional(Schema.String),
|
||||
item: Schema.optional(RealtimeItem),
|
||||
response: Schema.optional(Schema.Struct({ output: Schema.optional(Schema.Array(RealtimeItem)) })),
|
||||
error: Schema.optional(
|
||||
Schema.Struct({
|
||||
code: Schema.optional(Schema.String),
|
||||
message: Schema.optional(Schema.String),
|
||||
event_id: Schema.optional(Schema.String),
|
||||
}),
|
||||
),
|
||||
})
|
||||
const decodeRealtimeEvent = Schema.decodeUnknownOption(Schema.fromJsonString(RealtimeEvent))
|
||||
const decodeFunctionCall = Schema.decodeUnknownOption(RealtimeFunctionCall)
|
||||
type RealtimeEvent = Schema.Schema.Type<typeof RealtimeEvent>
|
||||
type RealtimeProjectorCommand = { readonly type: "response.create" }
|
||||
|
||||
export function createRealtimeEventProjector() {
|
||||
const pendingCalls = new Set<string>()
|
||||
const resolvedCalls = new Set<string>()
|
||||
const startedCalls = new Set<string>()
|
||||
const projectedCalls = new Set<string>()
|
||||
let responseAwaitingWork = false
|
||||
|
||||
const resumeAfterWork = (): ReadonlyArray<RealtimeProjectorCommand> => {
|
||||
if (!responseAwaitingWork || pendingCalls.size > 0) return []
|
||||
responseAwaitingWork = false
|
||||
resolvedCalls.clear()
|
||||
return [{ type: "response.create" }]
|
||||
}
|
||||
|
||||
return {
|
||||
receive(raw: string) {
|
||||
const event = Option.getOrUndefined(decodeRealtimeEvent(raw))
|
||||
if (!event)
|
||||
return {
|
||||
type: "invalid",
|
||||
events: [{ type: "error", message: "Received an invalid Realtime API event." }] satisfies VoiceProtocolEvent[],
|
||||
commands: [] as ReadonlyArray<RealtimeProjectorCommand>,
|
||||
}
|
||||
const events: VoiceProtocolEvent[] = event.type.endsWith(".delta")
|
||||
? []
|
||||
: [{ type: "debug", message: event.type }]
|
||||
const commands: RealtimeProjectorCommand[] = []
|
||||
switch (event.type) {
|
||||
case "session.created":
|
||||
events.push({ type: "ready" })
|
||||
break
|
||||
case "response.output_text.delta":
|
||||
case "response.output_audio_transcript.delta":
|
||||
events.push({ type: "assistant.transcript.delta", delta: event.delta ?? "" })
|
||||
break
|
||||
case "response.done": {
|
||||
const callIDs = (event.response?.output ?? []).flatMap((item) =>
|
||||
item.type === "function_call" && item.call_id ? [item.call_id] : [],
|
||||
)
|
||||
callIDs.filter((id) => !resolvedCalls.has(id)).forEach((id) => pendingCalls.add(id))
|
||||
responseAwaitingWork = callIDs.length > 0
|
||||
events.push({ type: "assistant.done", awaitingWork: responseAwaitingWork })
|
||||
commands.push(...resumeAfterWork())
|
||||
break
|
||||
}
|
||||
case "input_audio_buffer.speech_started":
|
||||
events.push({ type: "user.started" })
|
||||
break
|
||||
case "input_audio_buffer.speech_stopped":
|
||||
events.push({ type: "user.stopped" })
|
||||
break
|
||||
case "input_audio_buffer.committed":
|
||||
events.push({ type: "user.committed", id: event.item_id ?? "" })
|
||||
break
|
||||
case "conversation.item.input_audio_transcription.completed":
|
||||
events.push({
|
||||
type: "user.transcript",
|
||||
id: event.item_id ?? "",
|
||||
text: (event.transcript ?? "").trim(),
|
||||
final: true,
|
||||
})
|
||||
break
|
||||
case "response.output_audio.delta":
|
||||
if (event.delta) events.push({ type: "assistant.audio", audio: Buffer.from(event.delta, "base64") })
|
||||
break
|
||||
case "response.output_item.added":
|
||||
if (
|
||||
event.item?.type === "function_call" &&
|
||||
event.item.name &&
|
||||
event.item.call_id &&
|
||||
!startedCalls.has(event.item.call_id)
|
||||
) {
|
||||
startedCalls.add(event.item.call_id)
|
||||
events.push({ type: "tool.started", id: event.item.call_id, name: event.item.name })
|
||||
}
|
||||
break
|
||||
case "response.output_item.done": {
|
||||
if (event.item?.type !== "function_call") break
|
||||
if (event.item.call_id && projectedCalls.has(event.item.call_id)) break
|
||||
if (event.item.name && event.item.call_id && !startedCalls.has(event.item.call_id)) {
|
||||
startedCalls.add(event.item.call_id)
|
||||
events.push({ type: "tool.started", id: event.item.call_id, name: event.item.name })
|
||||
}
|
||||
const call = Option.getOrUndefined(decodeFunctionCall(event.item))
|
||||
if (!call) {
|
||||
events.push({ type: "error", message: "Received a malformed Realtime function call." })
|
||||
if (event.item.call_id)
|
||||
events.push({
|
||||
type: "work.rejected",
|
||||
request: { id: event.item.call_id, name: event.item.name ?? "tool", input: {} },
|
||||
output: { status: "error", message: "Malformed Realtime function call." },
|
||||
})
|
||||
if (event.item.call_id) startedCalls.delete(event.item.call_id)
|
||||
if (event.item.call_id) projectedCalls.add(event.item.call_id)
|
||||
break
|
||||
}
|
||||
pendingCalls.add(call.call_id)
|
||||
const input = Option.getOrUndefined(decodeVoiceToolInput(call.arguments))
|
||||
if (!input) {
|
||||
events.push({ type: "error", message: `Received invalid arguments for Realtime tool ${call.name}.` })
|
||||
events.push({
|
||||
type: "work.rejected",
|
||||
request: { id: call.call_id, name: call.name, input: {} },
|
||||
output: { status: "error", message: `Invalid arguments for tool ${call.name}.` },
|
||||
})
|
||||
startedCalls.delete(call.call_id)
|
||||
projectedCalls.add(call.call_id)
|
||||
break
|
||||
}
|
||||
events.push({ type: "work.requested", request: { id: call.call_id, name: call.name, input } })
|
||||
startedCalls.delete(call.call_id)
|
||||
projectedCalls.add(call.call_id)
|
||||
break
|
||||
}
|
||||
case "error":
|
||||
events.push({ type: "error", message: `${event.error?.code}: ${event.error?.message}` })
|
||||
}
|
||||
return { type: event.type, event, events, commands }
|
||||
},
|
||||
resolveWork(id: string) {
|
||||
resolvedCalls.add(id)
|
||||
pendingCalls.delete(id)
|
||||
return resumeAfterWork()
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
export function realtimeNotificationAcknowledgement(
|
||||
event: Pick<RealtimeEvent, "type" | "item" | "error">,
|
||||
pending: { readonly itemID: string; readonly eventID: string },
|
||||
) {
|
||||
if (event.type === "conversation.item.created" && event.item?.id === pending.itemID) return true
|
||||
if (event.type === "error" && event.error?.event_id === pending.eventID) return false
|
||||
return undefined
|
||||
}
|
||||
|
||||
export function createRealtimeProtocol(): VoiceProtocol {
|
||||
return {
|
||||
name: "realtime",
|
||||
inputActivity: "server",
|
||||
capabilities: { textInput: true, interruption: true, delegation: false },
|
||||
connect: connectRealtime,
|
||||
}
|
||||
}
|
||||
|
||||
export function realtimeSessionUpdate(
|
||||
options: Pick<VoiceProtocolOptions, "instructions" | "tools" | "voice">,
|
||||
) {
|
||||
return {
|
||||
type: "session.update",
|
||||
session: {
|
||||
type: "realtime",
|
||||
instructions: options.instructions,
|
||||
tools: options.tools,
|
||||
tool_choice: "auto",
|
||||
audio: { output: { voice: options.voice } },
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
function connectRealtime(options: VoiceProtocolOptions): VoiceConnection {
|
||||
// oxlint-disable-next-line typescript-eslint/no-unsafe-type-assertion -- Bun accepts WebSocket headers in this runtime-only overload.
|
||||
const ws = new WebSocket(`wss://api.openai.com/v1/realtime?model=${options.model}`, {
|
||||
headers: { Authorization: `Bearer ${options.apiKey}` },
|
||||
} as unknown as string[])
|
||||
const closed = Promise.withResolvers<void>()
|
||||
let closeTimer: ReturnType<typeof setTimeout> | undefined
|
||||
const notification = createSingleFlightAcknowledgement<{ readonly itemID: string; readonly eventID: string }>()
|
||||
const projector = createRealtimeEventProjector()
|
||||
|
||||
const send = (event: Record<string, unknown>) => {
|
||||
if (options.debug || event["type"] !== "input_audio_buffer.append")
|
||||
options.trace?.("realtime.send", {
|
||||
type: event["type"],
|
||||
callID:
|
||||
event["item"] && typeof event["item"] === "object" && "call_id" in event["item"]
|
||||
? event["item"].call_id
|
||||
: undefined,
|
||||
})
|
||||
if (ws.readyState !== WebSocket.OPEN) return false
|
||||
ws.send(JSON.stringify(event))
|
||||
return true
|
||||
}
|
||||
const createResponse = (text = false) =>
|
||||
send(text ? { type: "response.create", response: { output_modalities: ["text"] } } : { type: "response.create" })
|
||||
const resolveFunctionCall = (id: string, output: unknown) => {
|
||||
send({
|
||||
type: "conversation.item.create",
|
||||
item: { type: "function_call_output", call_id: id, output: JSON.stringify(output) },
|
||||
})
|
||||
projector.resolveWork(id).forEach(() => createResponse())
|
||||
}
|
||||
|
||||
ws.addEventListener("open", () => {
|
||||
send(realtimeSessionUpdate(options))
|
||||
send({
|
||||
type: "session.update",
|
||||
session: { type: "realtime", audio: { input: { transcription: { model: "whisper-1" } } } },
|
||||
})
|
||||
send({
|
||||
type: "session.update",
|
||||
session: {
|
||||
type: "realtime",
|
||||
audio: {
|
||||
input: {
|
||||
turn_detection: {
|
||||
type: "server_vad",
|
||||
silence_duration_ms: 900,
|
||||
interrupt_response: options.fullDuplex,
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
})
|
||||
})
|
||||
ws.addEventListener("message", (event) => {
|
||||
const result = projector.receive(String(event.data))
|
||||
if (options.debug || result.type !== "response.output_audio.delta")
|
||||
options.trace?.("realtime.receive", { type: result.type })
|
||||
const current = notification.current()
|
||||
const acknowledgement = current && result.event
|
||||
? realtimeNotificationAcknowledgement(result.event, current.correlation)
|
||||
: undefined
|
||||
if (acknowledgement !== undefined && current) notification.settle(current.id, acknowledgement)
|
||||
result.events.forEach(options.onEvent)
|
||||
result.commands.forEach(() => createResponse())
|
||||
})
|
||||
ws.addEventListener("close", (event) => {
|
||||
if (closeTimer) clearTimeout(closeTimer)
|
||||
notification.close()
|
||||
options.onEvent({ type: "closed", code: event.code })
|
||||
closed.resolve()
|
||||
})
|
||||
|
||||
return {
|
||||
appendAudio(audio) {
|
||||
if (ws.bufferedAmount > 96_000) {
|
||||
options.trace?.("realtime.audio.dropped", { bytes: audio.length, buffered: ws.bufferedAmount })
|
||||
return
|
||||
}
|
||||
send({ type: "input_audio_buffer.append", audio: audio.toString("base64") })
|
||||
},
|
||||
sendText(text) {
|
||||
send({
|
||||
type: "conversation.item.create",
|
||||
item: { type: "message", role: "user", content: [{ type: "input_text", text }] },
|
||||
})
|
||||
createResponse(true)
|
||||
},
|
||||
resolveWork(request, output) {
|
||||
resolveFunctionCall(request.id, output)
|
||||
},
|
||||
notify(request) {
|
||||
const suffix = crypto.randomUUID().replaceAll("-", "")
|
||||
const itemID = `item_${suffix}`
|
||||
const eventID = `event_${suffix}`
|
||||
const pending = notification.begin(request.id, { itemID, eventID })
|
||||
if (!pending.started) return pending.promise
|
||||
const created = send({
|
||||
type: "conversation.item.create",
|
||||
event_id: eventID,
|
||||
item: { id: itemID, type: "message", role: "user", content: [{ type: "input_text", text: request.text }] },
|
||||
})
|
||||
if (!created || !createResponse()) {
|
||||
notification.settle(request.id, false)
|
||||
return Promise.resolve(false)
|
||||
}
|
||||
return pending.promise
|
||||
},
|
||||
interrupt() {
|
||||
send({ type: "response.cancel" })
|
||||
},
|
||||
close() {
|
||||
if (ws.readyState === WebSocket.CLOSED) return Promise.resolve()
|
||||
ws.close(1000)
|
||||
closeTimer ??= setTimeout(() => closed.resolve(), 5_000)
|
||||
return closed.promise
|
||||
},
|
||||
}
|
||||
}
|
||||
@@ -1,94 +0,0 @@
|
||||
import { Schema } from "effect"
|
||||
|
||||
export const decodeVoiceToolInput = Schema.decodeUnknownOption(
|
||||
Schema.fromJsonString(Schema.Record(Schema.String, Schema.Unknown)),
|
||||
)
|
||||
|
||||
export type VoiceTool = {
|
||||
readonly type: "function"
|
||||
readonly name: string
|
||||
readonly description: string
|
||||
readonly parameters: unknown
|
||||
}
|
||||
|
||||
export type VoiceWorkRequest = {
|
||||
readonly id: string
|
||||
readonly name: string
|
||||
readonly input: Record<string, unknown>
|
||||
}
|
||||
|
||||
export type VoiceToolExecution = {
|
||||
readonly output: unknown
|
||||
readonly admittedPrompt?: {
|
||||
readonly sessionID: string
|
||||
readonly promptID: string
|
||||
}
|
||||
}
|
||||
|
||||
export type VoiceDelegationRequest = {
|
||||
readonly id: string
|
||||
readonly text: string
|
||||
}
|
||||
|
||||
export type VoiceNotificationRequest = {
|
||||
readonly id: string
|
||||
readonly text: string
|
||||
readonly delegationID?: string
|
||||
}
|
||||
|
||||
export type VoiceAudioTimeline = { readonly startMs: number; readonly endMs: number }
|
||||
|
||||
export type VoiceProtocolEvent =
|
||||
| { readonly type: "ready" }
|
||||
| { readonly type: "user.started" }
|
||||
| { readonly type: "user.stopped" }
|
||||
| { readonly type: "user.committed"; readonly id: string }
|
||||
| { readonly type: "user.transcript"; readonly id: string; readonly text: string; readonly final: boolean }
|
||||
| {
|
||||
readonly type: "assistant.audio"
|
||||
readonly audio: Buffer
|
||||
readonly timeline?: VoiceAudioTimeline
|
||||
}
|
||||
| { readonly type: "assistant.transcript.delta"; readonly delta: string }
|
||||
| { readonly type: "assistant.transcript"; readonly text: string }
|
||||
| { readonly type: "assistant.done"; readonly awaitingWork: boolean }
|
||||
| { readonly type: "tool.started"; readonly id: string; readonly name: string }
|
||||
| { readonly type: "work.requested"; readonly request: VoiceWorkRequest }
|
||||
| { readonly type: "work.rejected"; readonly request: VoiceWorkRequest; readonly output: unknown }
|
||||
| { readonly type: "delegation.requested"; readonly request: VoiceDelegationRequest }
|
||||
| { readonly type: "debug"; readonly message: string }
|
||||
| { readonly type: "error"; readonly message: string }
|
||||
| { readonly type: "closed"; readonly code: number }
|
||||
|
||||
export type VoiceProtocolOptions = {
|
||||
readonly apiKey: string
|
||||
readonly model: string
|
||||
readonly voice: string
|
||||
readonly instructions: string
|
||||
readonly tools: ReadonlyArray<VoiceTool>
|
||||
readonly fullDuplex: boolean
|
||||
readonly debug: boolean
|
||||
readonly onEvent: (event: VoiceProtocolEvent) => void
|
||||
readonly trace?: (event: string, data?: Record<string, unknown>) => void
|
||||
}
|
||||
|
||||
export type VoiceConnection = {
|
||||
appendAudio(audio: Buffer): void
|
||||
sendText?(text: string): void
|
||||
resolveWork(request: VoiceWorkRequest, output: unknown): void
|
||||
resolveDelegation?(request: VoiceDelegationRequest, output: string): void
|
||||
notify(notification: VoiceNotificationRequest): Promise<boolean>
|
||||
interrupt?(): void
|
||||
close(options?: { readonly graceful?: boolean }): Promise<void>
|
||||
}
|
||||
|
||||
export type VoiceProtocol = {
|
||||
readonly name: "live" | "realtime"
|
||||
readonly inputActivity: "local" | "server"
|
||||
readonly capabilities: {
|
||||
readonly textInput: boolean
|
||||
readonly interruption: boolean
|
||||
readonly delegation: boolean
|
||||
}
|
||||
connect(options: VoiceProtocolOptions): VoiceConnection
|
||||
}
|
||||
@@ -1,167 +0,0 @@
|
||||
import { Option } from "effect"
|
||||
import { decodeVoiceToolInput, type VoiceTool, type VoiceWorkRequest } from "./protocol"
|
||||
|
||||
export type ResponsesControllerContext = {
|
||||
readonly sessionIDs: Set<string>
|
||||
lastSessionID?: string
|
||||
}
|
||||
|
||||
export function createResponsesControllerContext(): ResponsesControllerContext {
|
||||
return { sessionIDs: new Set() }
|
||||
}
|
||||
|
||||
export function responsesControllerTools(
|
||||
tools: ReadonlyArray<VoiceTool>,
|
||||
text: string,
|
||||
context: ResponsesControllerContext,
|
||||
) {
|
||||
if (!context.lastSessionID || /\b(new|another|separate|fresh)\s+(session|thread)\b/i.test(text)) return tools
|
||||
return tools.filter((tool) => tool.name !== "start_session")
|
||||
}
|
||||
|
||||
export async function runResponsesController(options: {
|
||||
apiKey: string
|
||||
model: string
|
||||
instructions: string
|
||||
text: string
|
||||
tools: ReadonlyArray<VoiceTool>
|
||||
execute: (call: VoiceWorkRequest) => Promise<unknown>
|
||||
context?: ResponsesControllerContext
|
||||
trace?: (event: string, data?: Record<string, unknown>) => void
|
||||
fetch?: typeof fetch
|
||||
}) {
|
||||
const request = options.fetch ?? fetch
|
||||
const context = options.context ?? createResponsesControllerContext()
|
||||
const cache = new Map<string, Promise<unknown>>()
|
||||
let input: unknown = controllerInput(options.text, context)
|
||||
let previousResponseID: string | undefined
|
||||
|
||||
for (let step = 0; step < 20; step++) {
|
||||
options.trace?.("responses.controller.requested", { step, continued: previousResponseID !== undefined })
|
||||
const response = await request("https://api.openai.com/v1/responses", {
|
||||
method: "POST",
|
||||
headers: { Authorization: `Bearer ${options.apiKey}`, "Content-Type": "application/json" },
|
||||
body: JSON.stringify({
|
||||
model: options.model,
|
||||
instructions: options.instructions,
|
||||
input,
|
||||
previous_response_id: previousResponseID,
|
||||
tools: options.tools,
|
||||
store: true,
|
||||
}),
|
||||
})
|
||||
if (!response.ok) throw new Error(`Responses controller failed (${response.status}): ${await response.text()}`)
|
||||
const value = requireResponse(await response.json())
|
||||
const calls = value.output.flatMap((item) => {
|
||||
if (
|
||||
item.type !== "function_call" ||
|
||||
typeof item.call_id !== "string" ||
|
||||
typeof item.name !== "string" ||
|
||||
typeof item.arguments !== "string"
|
||||
)
|
||||
return []
|
||||
const parsed = Option.getOrUndefined(decodeVoiceToolInput(item.arguments))
|
||||
if (!parsed) throw new Error(`Responses controller returned invalid arguments for ${item.name}.`)
|
||||
return [{ id: item.call_id, name: item.name, input: parsed }]
|
||||
})
|
||||
options.trace?.("responses.controller.responded", {
|
||||
step,
|
||||
responseID: value.id,
|
||||
tools: calls.map((call) => call.name),
|
||||
})
|
||||
if (calls.length === 0)
|
||||
return withSessionContext(
|
||||
responseText(value.output) || "The controller finished without a text reply.",
|
||||
context.sessionIDs,
|
||||
)
|
||||
input = await Promise.all(
|
||||
calls.map(async (call) => {
|
||||
if (typeof call.input["session_id"] === "string") rememberSession(context, call.input["session_id"])
|
||||
const key = `${call.name}:${JSON.stringify(call.input)}`
|
||||
const cached = cacheableTools.has(call.name) ? cache.get(key) : undefined
|
||||
if (cached) options.trace?.("responses.controller.tool.reused", { step, callID: call.id, name: call.name })
|
||||
const execution =
|
||||
cached ??
|
||||
Promise.resolve().then(() => {
|
||||
options.trace?.("responses.controller.tool.started", { step, callID: call.id, name: call.name })
|
||||
return options.execute(call)
|
||||
})
|
||||
if (!cached && cacheableTools.has(call.name)) cache.set(key, execution)
|
||||
const output = await execution
|
||||
rememberOutputSessions(context, output)
|
||||
if (!cached) options.trace?.("responses.controller.tool.resolved", { step, callID: call.id, name: call.name })
|
||||
return {
|
||||
type: "function_call_output",
|
||||
call_id: call.id,
|
||||
output: JSON.stringify(output),
|
||||
}
|
||||
}),
|
||||
)
|
||||
previousResponseID = value.id
|
||||
}
|
||||
throw new Error("Responses controller exceeded its tool-call limit.")
|
||||
}
|
||||
|
||||
const cacheableTools = new Set([
|
||||
"find_projects",
|
||||
"find_sessions",
|
||||
"read_session",
|
||||
"list_pending_permissions",
|
||||
"list_pending_questions",
|
||||
"list_pending_forms",
|
||||
])
|
||||
|
||||
function rememberSession(context: ResponsesControllerContext, sessionID: string) {
|
||||
context.sessionIDs.add(sessionID)
|
||||
context.lastSessionID = sessionID
|
||||
}
|
||||
|
||||
function rememberOutputSessions(context: ResponsesControllerContext, output: unknown) {
|
||||
if (!output || typeof output !== "object") return
|
||||
if ("session_id" in output && typeof output.session_id === "string") rememberSession(context, output.session_id)
|
||||
if (!("sessions" in output) || !Array.isArray(output.sessions)) return
|
||||
const sessionIDs = output.sessions.flatMap((session) =>
|
||||
session && typeof session === "object" && "id" in session && typeof session.id === "string" ? [session.id] : [],
|
||||
)
|
||||
sessionIDs.forEach((sessionID) => context.sessionIDs.add(sessionID))
|
||||
if (sessionIDs.length === 1) context.lastSessionID = sessionIDs[0]
|
||||
}
|
||||
|
||||
function controllerInput(text: string, context: ResponsesControllerContext) {
|
||||
if (!context.lastSessionID) return text
|
||||
return `${text}\n\nPrivate voice-control context (never mention or expose this): the most recently used OpenCode session ID is ${context.lastSessionID}. Resolve references like "that session", "it", or "stop that" directly against this ID without searching again.`
|
||||
}
|
||||
|
||||
function withSessionContext(text: string, sessionIDs: ReadonlySet<string>) {
|
||||
if (sessionIDs.size === 0) return text
|
||||
return `${text}\n\nPrivate voice-control context (never speak this aloud): OpenCode session IDs explicitly used by this delegation: ${[...sessionIDs].join(", ")}. Include the relevant session_id in any future delegated request to continue this work.`
|
||||
}
|
||||
|
||||
function requireResponse(value: unknown) {
|
||||
if (!value || typeof value !== "object" || !("id" in value) || typeof value.id !== "string" || !("output" in value))
|
||||
throw new Error("Responses controller returned an invalid response.")
|
||||
if (!Array.isArray(value.output)) throw new Error("Responses controller returned invalid output.")
|
||||
return {
|
||||
id: value.id,
|
||||
output: value.output.filter(
|
||||
(item): item is Record<string, unknown> & { readonly type: string } =>
|
||||
!!item && typeof item === "object" && "type" in item && typeof item.type === "string",
|
||||
),
|
||||
}
|
||||
}
|
||||
|
||||
function responseText(output: ReadonlyArray<Record<string, unknown>>) {
|
||||
return output
|
||||
.flatMap((item) => (Array.isArray(item["content"]) ? item["content"] : []))
|
||||
.flatMap((part) =>
|
||||
part &&
|
||||
typeof part === "object" &&
|
||||
"type" in part &&
|
||||
part.type === "output_text" &&
|
||||
"text" in part &&
|
||||
typeof part.text === "string"
|
||||
? [part.text]
|
||||
: [],
|
||||
)
|
||||
.join("\n")
|
||||
}
|
||||
@@ -1,38 +0,0 @@
|
||||
export function createSingleFlightAcknowledgement<Correlation>(timeoutMs = 5_000) {
|
||||
let pending:
|
||||
| {
|
||||
readonly id: string
|
||||
readonly correlation: Correlation
|
||||
readonly result: PromiseWithResolvers<boolean>
|
||||
readonly timer: ReturnType<typeof setTimeout>
|
||||
}
|
||||
| undefined
|
||||
|
||||
const settle = (id: string, accepted: boolean) => {
|
||||
if (pending?.id !== id) return
|
||||
clearTimeout(pending.timer)
|
||||
pending.result.resolve(accepted)
|
||||
pending = undefined
|
||||
}
|
||||
|
||||
return {
|
||||
begin(id: string, correlation: Correlation) {
|
||||
if (pending) return { started: false, promise: Promise.resolve(false) } as const
|
||||
const result = Promise.withResolvers<boolean>()
|
||||
pending = {
|
||||
id,
|
||||
correlation,
|
||||
result,
|
||||
timer: setTimeout(() => settle(id, false), timeoutMs),
|
||||
}
|
||||
return { started: true, promise: result.promise } as const
|
||||
},
|
||||
current() {
|
||||
return pending ? { id: pending.id, correlation: pending.correlation } : undefined
|
||||
},
|
||||
settle,
|
||||
close() {
|
||||
if (pending) settle(pending.id, false)
|
||||
},
|
||||
}
|
||||
}
|
||||
@@ -1,273 +0,0 @@
|
||||
#!/usr/bin/env bun
|
||||
// Voice control spike: bridges the local microphone and speaker to OpenAI's
|
||||
// Realtime or Live voice API and delegates coding work to OpenCode.
|
||||
//
|
||||
// Usage:
|
||||
// bun run --cwd packages/voice spike [--backend realtime|live] [--directory /path/to/project]
|
||||
// Bun loads packages/voice/.env automatically when the command runs from that package.
|
||||
//
|
||||
// Requires sox (`brew install sox`) for mic capture (`rec`) and playback (`play`).
|
||||
|
||||
import { parseArgs } from "node:util"
|
||||
import { OpenCode } from "@opencode-ai/client/promise"
|
||||
import { Service } from "@opencode-ai/client/service"
|
||||
import { createAudioSession, type AudioSession } from "./audio-session"
|
||||
import type { OpenCodeAnnouncement } from "./opencode-notification"
|
||||
import { createVoiceTrace } from "./trace"
|
||||
import {
|
||||
createResponsesControllerContext,
|
||||
responsesControllerTools,
|
||||
runResponsesController,
|
||||
} from "./responses-controller"
|
||||
import { createVoiceSession, voiceControlTool, type VoiceSession } from "./voice-session"
|
||||
|
||||
const args = parseArgs({
|
||||
options: {
|
||||
backend: { type: "string", default: "realtime" },
|
||||
server: { type: "string" },
|
||||
password: { type: "string" },
|
||||
directory: { type: "string", default: process.cwd() },
|
||||
model: { type: "string" },
|
||||
"delegation-model": { type: "string", default: "gpt-5.5" },
|
||||
voice: { type: "string", default: "marin" },
|
||||
provider: { type: "string", default: "openai" },
|
||||
"coding-model": { type: "string", default: "gpt-5.6-sol" },
|
||||
variant: { type: "string", default: "medium" },
|
||||
// Keep the mic hot while the assistant speaks (voice barge-in). Only
|
||||
// usable with headphones: on speakers the mic hears the assistant and
|
||||
// interrupts it with its own echo. Default is half-duplex gating.
|
||||
duplex: { type: "boolean", default: false },
|
||||
// Enable Apple voice processing (echo cancellation) in the audio helper.
|
||||
// Needed for full duplex on speakers; harmful with Bluetooth headsets,
|
||||
// where it can bind the wrong capture device.
|
||||
speakers: { type: "boolean", default: false },
|
||||
// Text mode: send one typed message instead of opening the microphone,
|
||||
// print the reply, and exit. Useful for smoke-testing the tool loop.
|
||||
text: { type: "string" },
|
||||
// Log every protocol event type as it arrives.
|
||||
debug: { type: "boolean", default: false },
|
||||
"reduce-motion": { type: "boolean", default: false },
|
||||
},
|
||||
}).values
|
||||
let audioSession: AudioSession | undefined
|
||||
let voiceSession: VoiceSession | undefined
|
||||
|
||||
if (args.backend !== "realtime" && args.backend !== "live") {
|
||||
console.error("--backend must be realtime or live")
|
||||
process.exit(1)
|
||||
}
|
||||
const protocol =
|
||||
args.backend === "live"
|
||||
? (await import("./protocol-live")).createLiveProtocol()
|
||||
: (await import("./protocol-realtime")).createRealtimeProtocol()
|
||||
const model = args.model ?? (protocol.name === "live" ? "gpt-live-1-boulder-alpha" : "gpt-realtime-2.1")
|
||||
if (args.text && !protocol.capabilities.textInput) {
|
||||
console.error(`--text is not supported by the ${protocol.name} backend`)
|
||||
process.exit(1)
|
||||
}
|
||||
const apiKey = (() => {
|
||||
const value = process.env["OPENAI_API_KEY"]
|
||||
if (value) return value
|
||||
console.error("OPENAI_API_KEY is required. Add it to the gitignored packages/voice/.env or export it in the shell.")
|
||||
process.exit(1)
|
||||
return ""
|
||||
})()
|
||||
if (!args.text && !process.stdout.isTTY) {
|
||||
console.error("The voice TUI requires direct terminal access for screen rendering and keyboard input.")
|
||||
console.error("1Password output masking pipes stdout, so `2password run` cannot host this interactive TUI.")
|
||||
console.error("Run directly: bun run --cwd packages/voice spike --backend " + protocol.name)
|
||||
process.exit(1)
|
||||
}
|
||||
|
||||
const serverPassword = args.password ?? process.env["OPENCODE_PASSWORD"] ?? process.env["OPENCODE_SERVER_PASSWORD"]
|
||||
const endpoint = args.server
|
||||
? {
|
||||
url: args.server,
|
||||
auth: serverPassword ? { type: "basic" as const, username: "opencode", password: serverPassword } : undefined,
|
||||
}
|
||||
: ((await Service.discover()) ?? (await Service.ensure({ command: ["opencode2", "serve", "--service"] })))
|
||||
const client = OpenCode.make({
|
||||
baseUrl: endpoint.url,
|
||||
headers: Service.headers(endpoint),
|
||||
})
|
||||
const health = await client.health.get().catch((error) => {
|
||||
console.error(`Could not reach the OpenCode 2 server at ${endpoint.url}: ${error}`)
|
||||
process.exit(1)
|
||||
})
|
||||
const trace = await createVoiceTrace()
|
||||
// ---------------------------------------------------------------------------
|
||||
// UI: OpenTUI in voice mode, plain console in --text mode. Created before the
|
||||
// WebSocket so no await sits between socket creation and handler registration.
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
const { createConsoleUI, createVoiceTUI } = await import("./ui")
|
||||
const tuiActive = !args.text
|
||||
const ui = tuiActive
|
||||
? await createVoiceTUI({
|
||||
onInterrupt: () => voiceSession?.interrupt(),
|
||||
onExit: () => shutdown(),
|
||||
onCycleVoice: () => voiceSession?.cycleVoice(),
|
||||
onToggleMicrophone: () => voiceSession?.toggleMicrophone(),
|
||||
onToggleSpeaker: () => voiceSession?.toggleSpeaker(),
|
||||
reducedMotion: args["reduce-motion"],
|
||||
})
|
||||
: createConsoleUI()
|
||||
ui.setStatus({ server: endpoint.url, model: `${args["coding-model"]}:${args.variant}` })
|
||||
ui.meta(`opencode ${endpoint.url} (version ${health.version})`)
|
||||
ui.meta(`project ${args.directory}`)
|
||||
ui.meta(`trace ${trace.path}`)
|
||||
trace.write("voice.started", {
|
||||
backend: protocol.name,
|
||||
model,
|
||||
directory: args.directory,
|
||||
duplex: args.duplex,
|
||||
speakers: args.speakers,
|
||||
})
|
||||
|
||||
const voices = ["marin", "cedar", "coral", "sage", "ash", "ballad", "alloy", "verse"]
|
||||
const pendingNotifications: OpenCodeAnnouncement[] = []
|
||||
const { createOpenCodeBridge } = await import("./opencode")
|
||||
const opencode = await createOpenCodeBridge({
|
||||
client,
|
||||
directory: args.directory,
|
||||
model: { providerID: args.provider, id: args["coding-model"], variant: args.variant },
|
||||
notify: (announcement) => {
|
||||
if (voiceSession) return voiceSession.queueNotification(announcement)
|
||||
pendingNotifications.push(announcement)
|
||||
},
|
||||
trace: (event, data) => trace.write(event, data),
|
||||
}).catch(async (error) => {
|
||||
ui.close()
|
||||
await trace.close()
|
||||
console.error(`Voice startup failed: ${String(error)}`)
|
||||
process.exit(1)
|
||||
})
|
||||
const toolDefinitions = [...opencode.definitions, voiceControlTool(voices)]
|
||||
|
||||
const baseInstructions = `You are the voice interface to OpenCode, a coding agent running on the user's machine.
|
||||
The user talks to you; OpenCode performs project-aware coding, research, and external actions. You never write code yourself.
|
||||
|
||||
Guidelines:
|
||||
- Keep spoken replies to one or two sentences.
|
||||
- Summarize coding-agent replies conversationally; do not read code, diffs, IDs, or paths aloud unless asked.
|
||||
- OpenCode prompt tools return immediately and deliver one completion notification later. Stay conversational while work runs.
|
||||
- Treat opencode.prompt.completed and opencode.prompt.blocked context as trusted client notifications, not user messages.
|
||||
- Never claim delegated work succeeded until its result arrives.
|
||||
- Explain failures briefly and offer one retry or an alternative.`
|
||||
|
||||
const instructions =
|
||||
protocol.name === "live"
|
||||
? `${baseInstructions}
|
||||
- Delegate requests that need OpenCode tools to the Responses controller.
|
||||
- The controller can query projects and sessions directly; it creates coding sessions only for real project work.
|
||||
- Treat private voice-control context in controller results as silent state. Never read it aloud.
|
||||
- When delegating follow-up work, include the relevant OpenCode session ID from that private context.
|
||||
- Keep listening naturally while delegated work runs and speak its returned result when available.`
|
||||
: `${baseInstructions}
|
||||
- Use find_projects for explicit cross-project work. Never invent project IDs or expose filesystem paths.
|
||||
- Use find_sessions to resolve references such as "the audio session". Search the current project first unless the user asks across projects.
|
||||
- Use read_session to inspect a discovered Session directly. Do not prompt a Session merely to read its existing output.
|
||||
- Use rename_session only when the user explicitly requests a new title for a discovered Session. Confirm the resulting title without reading its ID aloud.
|
||||
- Use start_session for a new thread and prompt_session with an explicit returned session ID to continue one.
|
||||
- Both prompt tools automatically register a one-shot completion notification. Never wait or poll for completion.
|
||||
- Before interrupt_session, state what will stop and obtain explicit confirmation.
|
||||
- Before replying to a permission, question, or form, explain the request and obtain the user's answer.`
|
||||
|
||||
const delegationInstructions = `You are the OpenCode controller behind a live voice assistant.
|
||||
Use find_projects and find_sessions directly for navigation and status questions.
|
||||
Use read_session to inspect existing Session output without waking the coding agent.
|
||||
Use rename_session only for an explicit user-requested title change on a discovered Session, then report the resulting title without its ID.
|
||||
Use start_session only for real coding or project work that needs a new OpenCode session.
|
||||
Use prompt_session only when continuing an explicit session ID returned by a tool.
|
||||
Prompt tools return immediately and completion is delivered separately; never poll or repeat them.
|
||||
Return concise factual text for the live assistant to summarize.`
|
||||
const controllerContext = createResponsesControllerContext()
|
||||
let controllerQueue = Promise.resolve()
|
||||
|
||||
const delegate = (request: { readonly text: string }, execute: Parameters<typeof runResponsesController>[0]["execute"]) => {
|
||||
const result = controllerQueue.then(() =>
|
||||
runResponsesController({
|
||||
apiKey,
|
||||
model: args["delegation-model"],
|
||||
instructions: delegationInstructions,
|
||||
text: request.text,
|
||||
tools: responsesControllerTools(opencode.definitions, request.text, controllerContext),
|
||||
execute,
|
||||
context: controllerContext,
|
||||
trace: (event, data) => trace.write(event, data),
|
||||
}),
|
||||
)
|
||||
controllerQueue = result.then(
|
||||
() => undefined,
|
||||
() => undefined,
|
||||
)
|
||||
return result
|
||||
}
|
||||
|
||||
audioSession = args.text
|
||||
? undefined
|
||||
: await createAudioSession({
|
||||
duplex: args.duplex,
|
||||
speakers: args.speakers,
|
||||
inputActivity: protocol.inputActivity,
|
||||
debug: args.debug,
|
||||
trace: (event, data) => trace.write(event, data),
|
||||
})
|
||||
const fullDuplex = audioSession?.fullDuplex ?? false
|
||||
|
||||
voiceSession = createVoiceSession({
|
||||
protocol,
|
||||
connection: {
|
||||
apiKey,
|
||||
model,
|
||||
instructions,
|
||||
tools: toolDefinitions,
|
||||
fullDuplex,
|
||||
debug: args.debug,
|
||||
trace: (event, data) => trace.write(event, data),
|
||||
},
|
||||
initialVoice: args.voice ?? "marin",
|
||||
voices,
|
||||
text: args.text,
|
||||
ui,
|
||||
audio: audioSession,
|
||||
tools: {
|
||||
execute: opencode.execute,
|
||||
acknowledge: opencode.acknowledge,
|
||||
close: opencode.close,
|
||||
},
|
||||
delegate,
|
||||
trace: (event, data) => trace.write(event, data),
|
||||
onClosed: shutdown,
|
||||
})
|
||||
pendingNotifications.forEach((announcement) => voiceSession?.queueNotification(announcement))
|
||||
|
||||
let shuttingDown = false
|
||||
let shutdownFinished = false
|
||||
|
||||
function finishShutdown() {
|
||||
if (shutdownFinished) return
|
||||
shutdownFinished = true
|
||||
ui.close()
|
||||
process.exit(0)
|
||||
}
|
||||
|
||||
function shutdown() {
|
||||
if (shuttingDown) return
|
||||
shuttingDown = true
|
||||
trace.write("voice.shutdown")
|
||||
void Promise.race([
|
||||
voiceSession?.close() ?? Promise.resolve(),
|
||||
Bun.sleep(5_000).then(() => trace.write("voice.shutdown.timeout")),
|
||||
])
|
||||
.then(() => trace.close())
|
||||
.finally(finishShutdown)
|
||||
}
|
||||
|
||||
// A surviving process keeps the microphone hot and the OpenAI meter running,
|
||||
// so every terminal-death signal must tear it down.
|
||||
process.on("SIGINT", shutdown)
|
||||
process.on("SIGHUP", shutdown)
|
||||
process.on("SIGTERM", shutdown)
|
||||
|
||||
voiceSession.start()
|
||||
@@ -1,32 +0,0 @@
|
||||
import { mkdir, readdir, unlink } from "node:fs/promises"
|
||||
import { tmpdir } from "node:os"
|
||||
import { join } from "node:path"
|
||||
|
||||
export async function createVoiceTrace() {
|
||||
const directory = join(tmpdir(), "opencode-voice")
|
||||
await mkdir(directory, { recursive: true })
|
||||
const stale = (await readdir(directory))
|
||||
.filter((name) => name.endsWith(".jsonl"))
|
||||
.sort()
|
||||
.slice(0, -20)
|
||||
await Promise.allSettled(stale.map((name) => unlink(join(directory, name))))
|
||||
const path = join(directory, `${new Date().toISOString().replaceAll(":", "-")}-${process.pid}.jsonl`)
|
||||
const writer = Bun.file(path).writer()
|
||||
const flush = setInterval(() => void writer.flush(), 250)
|
||||
let bytes = 0
|
||||
flush.unref()
|
||||
|
||||
return {
|
||||
path,
|
||||
write(event: string, data: Record<string, unknown> = {}) {
|
||||
if (bytes >= 10_000_000) return
|
||||
const line = `${JSON.stringify({ time: Date.now(), event, ...data })}\n`
|
||||
bytes += Buffer.byteLength(line)
|
||||
void writer.write(line)
|
||||
},
|
||||
async close() {
|
||||
clearInterval(flush)
|
||||
await writer.end()
|
||||
},
|
||||
}
|
||||
}
|
||||
@@ -1,294 +0,0 @@
|
||||
import { REVEAL_WORD_LIMIT, scheduleTextReveal, type TextReveal } from "./animation"
|
||||
|
||||
export type Message =
|
||||
| {
|
||||
readonly key: string
|
||||
readonly kind: "user"
|
||||
readonly itemID: string
|
||||
readonly text?: string
|
||||
readonly transcribing: boolean
|
||||
readonly reveals: ReadonlyArray<TextReveal>
|
||||
}
|
||||
| {
|
||||
readonly key: string
|
||||
readonly kind: "assistant"
|
||||
readonly text: string
|
||||
readonly streaming: boolean
|
||||
readonly reveals: ReadonlyArray<TextReveal>
|
||||
}
|
||||
| {
|
||||
readonly key: string
|
||||
readonly kind: "tool"
|
||||
readonly callID: string
|
||||
readonly name: string
|
||||
readonly input: unknown
|
||||
readonly output?: unknown
|
||||
}
|
||||
| { readonly key: string; readonly kind: "meta"; readonly text: string }
|
||||
|
||||
type NewMessage = Message extends infer Item ? (Item extends Message ? Omit<Item, "key"> : never) : never
|
||||
|
||||
export type VoiceViewState = {
|
||||
readonly messages: ReadonlyArray<Message>
|
||||
readonly messageSequence: number
|
||||
readonly activeUserID?: string
|
||||
readonly userSequence: number
|
||||
readonly revealAt: number
|
||||
readonly revealAnimationEndsAt: number
|
||||
}
|
||||
|
||||
export type VoiceViewEvent =
|
||||
| { readonly type: "meta"; readonly text: string }
|
||||
| { readonly type: "user.started" }
|
||||
| { readonly type: "user.reset" }
|
||||
| { readonly type: "user.committed"; readonly itemID: string }
|
||||
| {
|
||||
readonly type: "user.transcript"
|
||||
readonly itemID: string
|
||||
readonly text: string
|
||||
readonly final: boolean
|
||||
readonly now: number
|
||||
readonly animate: boolean
|
||||
}
|
||||
| { readonly type: "assistant.delta"; readonly text: string; readonly now: number; readonly animate: boolean }
|
||||
| { readonly type: "assistant.transcript"; readonly text: string; readonly now: number; readonly animate: boolean }
|
||||
| { readonly type: "assistant.done" }
|
||||
| { readonly type: "tool.started"; readonly callID: string; readonly name: string; readonly input: unknown }
|
||||
| { readonly type: "tool.done"; readonly callID: string; readonly output: unknown }
|
||||
| { readonly type: "reveals.completed" }
|
||||
|
||||
const MESSAGE_LIMIT = 200
|
||||
|
||||
export function initialVoiceView(): VoiceViewState {
|
||||
return { messages: [], messageSequence: 0, userSequence: 0, revealAt: 0, revealAnimationEndsAt: 0 }
|
||||
}
|
||||
|
||||
export function transitionVoiceView(state: VoiceViewState, event: VoiceViewEvent): VoiceViewState {
|
||||
switch (event.type) {
|
||||
case "meta":
|
||||
return append(state, { kind: "meta", text: event.text })
|
||||
case "user.started": {
|
||||
if (state.activeUserID) return state
|
||||
const itemID = `local-user-${state.userSequence + 1}`
|
||||
return { ...state, activeUserID: itemID, userSequence: state.userSequence + 1 }
|
||||
}
|
||||
case "user.reset": {
|
||||
if (!state.activeUserID) return state
|
||||
return {
|
||||
...state,
|
||||
activeUserID: undefined,
|
||||
messages: mergeAssistantRows(state.messages),
|
||||
}
|
||||
}
|
||||
case "user.committed": {
|
||||
const index = state.activeUserID
|
||||
? state.messages.findIndex((message) => message.kind === "user" && message.itemID === state.activeUserID)
|
||||
: -1
|
||||
const current = {
|
||||
...state,
|
||||
activeUserID: undefined,
|
||||
messages: state.messages.map((message, currentIndex) => {
|
||||
if (message.kind === "assistant" && message.streaming) return { ...message, streaming: false }
|
||||
if (currentIndex === index && message.kind === "user") return { ...message, itemID: event.itemID }
|
||||
return message
|
||||
}),
|
||||
}
|
||||
if (index !== -1) return current
|
||||
return append(current, { kind: "user", itemID: event.itemID, transcribing: true, reveals: [] })
|
||||
}
|
||||
case "user.transcript": {
|
||||
const index = state.messages.findIndex((message) => message.kind === "user" && message.itemID === event.itemID)
|
||||
if (index === -1) {
|
||||
const revealed = reveal(state, "", event.text, event.now, event.animate)
|
||||
return append(revealed.state, {
|
||||
kind: "user",
|
||||
itemID: event.itemID,
|
||||
text: event.text,
|
||||
transcribing: !event.final,
|
||||
reveals: revealed.reveals,
|
||||
})
|
||||
}
|
||||
const previous = state.messages[index]
|
||||
if (previous.kind !== "user") return state
|
||||
const previousText = previous.text ?? ""
|
||||
if (previousText === event.text && previous.transcribing === !event.final) return state
|
||||
const appended = event.text.startsWith(previousText)
|
||||
const revealed = reveal(
|
||||
state,
|
||||
appended ? previousText : "",
|
||||
appended ? event.text.slice(previousText.length) : event.text,
|
||||
event.now,
|
||||
event.animate,
|
||||
)
|
||||
return {
|
||||
...revealed.state,
|
||||
messages: state.messages.map((message, currentIndex) =>
|
||||
currentIndex === index && message.kind === "user"
|
||||
? {
|
||||
...message,
|
||||
text: event.text,
|
||||
transcribing: !event.final,
|
||||
reveals: appended
|
||||
? [...message.reveals, ...revealed.reveals].slice(-REVEAL_WORD_LIMIT)
|
||||
: revealed.reveals,
|
||||
}
|
||||
: message,
|
||||
),
|
||||
}
|
||||
}
|
||||
case "assistant.delta": {
|
||||
const streaming = state.messages.findLastIndex((message) => message.kind === "assistant" && message.streaming)
|
||||
const lastIndex = state.messages.length - 1
|
||||
const index = streaming === -1 && state.messages[lastIndex]?.kind === "assistant" ? lastIndex : streaming
|
||||
const message = state.messages[index]
|
||||
if (message?.kind !== "assistant") {
|
||||
const text = event.text.trimStart()
|
||||
const revealed = reveal(state, "", text, event.now, event.animate)
|
||||
return append(
|
||||
{
|
||||
...revealed.state,
|
||||
messages: state.messages.map((message) =>
|
||||
message.kind === "assistant" && message.streaming ? { ...message, streaming: false } : message,
|
||||
),
|
||||
},
|
||||
{ kind: "assistant", text, streaming: true, reveals: revealed.reveals },
|
||||
)
|
||||
}
|
||||
const joined =
|
||||
streaming === -1
|
||||
? joinAssistantText(message.text, event.text)
|
||||
: { text: message.text + event.text, previous: message.text, appended: event.text }
|
||||
const revealed = reveal(state, joined.previous, joined.appended, event.now, event.animate)
|
||||
return {
|
||||
...revealed.state,
|
||||
messages: state.messages.map((message, currentIndex) =>
|
||||
currentIndex === index && message.kind === "assistant"
|
||||
? {
|
||||
...message,
|
||||
text: joined.text,
|
||||
streaming: true,
|
||||
reveals: [...message.reveals, ...revealed.reveals].slice(-REVEAL_WORD_LIMIT),
|
||||
}
|
||||
: message,
|
||||
),
|
||||
}
|
||||
}
|
||||
case "assistant.transcript": {
|
||||
const text = event.text.trimStart()
|
||||
const index = state.messages.findLastIndex((message) => message.kind === "assistant" && message.streaming)
|
||||
if (index === -1) {
|
||||
const revealed = reveal(state, "", text, event.now, event.animate)
|
||||
return append(revealed.state, {
|
||||
kind: "assistant",
|
||||
text,
|
||||
streaming: true,
|
||||
reveals: revealed.reveals,
|
||||
})
|
||||
}
|
||||
const previous = state.messages[index]
|
||||
if (previous.kind !== "assistant" || previous.text === text) return state
|
||||
const appended = text.startsWith(previous.text)
|
||||
const revealed = reveal(
|
||||
state,
|
||||
appended ? previous.text : "",
|
||||
appended ? text.slice(previous.text.length) : text,
|
||||
event.now,
|
||||
event.animate,
|
||||
)
|
||||
return {
|
||||
...revealed.state,
|
||||
messages: state.messages.map((message, currentIndex) =>
|
||||
currentIndex === index && message.kind === "assistant"
|
||||
? {
|
||||
...message,
|
||||
text,
|
||||
reveals: appended
|
||||
? [...message.reveals, ...revealed.reveals].slice(-REVEAL_WORD_LIMIT)
|
||||
: revealed.reveals,
|
||||
}
|
||||
: message,
|
||||
),
|
||||
}
|
||||
}
|
||||
case "assistant.done": {
|
||||
const index = state.messages.findLastIndex((message) => message.kind === "assistant" && message.streaming)
|
||||
if (index === -1) return state
|
||||
return {
|
||||
...state,
|
||||
messages: state.messages.map((message, currentIndex) =>
|
||||
currentIndex === index && message.kind === "assistant" ? { ...message, streaming: false } : message,
|
||||
),
|
||||
}
|
||||
}
|
||||
case "tool.started":
|
||||
return append(state, {
|
||||
kind: "tool",
|
||||
callID: event.callID,
|
||||
name: event.name,
|
||||
input: event.input,
|
||||
})
|
||||
case "tool.done":
|
||||
return {
|
||||
...state,
|
||||
messages: state.messages.map((message) =>
|
||||
message.kind === "tool" && message.callID === event.callID ? { ...message, output: event.output } : message,
|
||||
),
|
||||
}
|
||||
case "reveals.completed":
|
||||
return {
|
||||
...state,
|
||||
revealAnimationEndsAt: 0,
|
||||
messages: state.messages.map((message) =>
|
||||
"reveals" in message && message.reveals.length > 0 ? { ...message, reveals: [] } : message,
|
||||
),
|
||||
}
|
||||
}
|
||||
return state
|
||||
}
|
||||
|
||||
function append(state: VoiceViewState, message: NewMessage): VoiceViewState {
|
||||
const messageSequence = state.messageSequence + 1
|
||||
const next = { ...message, key: `message-${messageSequence}` }
|
||||
return { ...state, messageSequence, messages: [...state.messages, next].slice(-MESSAGE_LIMIT) }
|
||||
}
|
||||
|
||||
function reveal(state: VoiceViewState, previous: string, delta: string, now: number, animate: boolean) {
|
||||
if (!animate) return { state, reveals: new Array<TextReveal>() }
|
||||
const scheduled = scheduleTextReveal(previous, delta, now, state.revealAt)
|
||||
return {
|
||||
state: {
|
||||
...state,
|
||||
revealAt: scheduled.nextRevealAt,
|
||||
revealAnimationEndsAt: Math.max(state.revealAnimationEndsAt, scheduled.animationEndsAt),
|
||||
},
|
||||
reveals: scheduled.reveals,
|
||||
}
|
||||
}
|
||||
|
||||
function mergeAssistantRows(messages: ReadonlyArray<Message>) {
|
||||
return messages.reduce<Message[]>((result, message) => {
|
||||
const previous = result.at(-1)
|
||||
if (previous?.kind !== "assistant" || message.kind !== "assistant") return [...result, message]
|
||||
return [
|
||||
...result.slice(0, -1),
|
||||
{
|
||||
key: previous.key,
|
||||
kind: "assistant",
|
||||
text: joinAssistantText(previous.text, message.text).text,
|
||||
streaming: previous.streaming || message.streaming,
|
||||
reveals: [],
|
||||
},
|
||||
]
|
||||
}, [])
|
||||
}
|
||||
|
||||
export function joinAssistantText(left: string, right: string) {
|
||||
if (left.trim() === "") return { text: right, previous: "", appended: right }
|
||||
if (right.trim() === "") return { text: left, previous: left, appended: "" }
|
||||
const head = left.trimEnd()
|
||||
const tail = right.trimStart()
|
||||
if ((left.slice(head.length) + right.slice(0, right.length - tail.length)).includes("\n"))
|
||||
return { text: left + right, previous: left, appended: right }
|
||||
const separator = /^[.,;:!?…%)\]}]/.test(tail) || /[([{]$/.test(head) ? "" : " "
|
||||
return { text: head + separator + tail, previous: head + separator, appended: tail }
|
||||
}
|
||||
@@ -1,728 +0,0 @@
|
||||
/** @jsxImportSource @opentui/solid */
|
||||
// Terminal UI for the voice spike. The TUI keeps conversation order stable
|
||||
// even though realtime events arrive out of order: a user row is inserted the
|
||||
// moment the audio buffer commits (before the assistant starts replying) and
|
||||
// its transcript is filled in when Whisper finishes.
|
||||
import { createCliRenderer, RGBA } from "@opentui/core"
|
||||
import { render, useKeyboard, useTerminalDimensions } from "@opentui/solid"
|
||||
import "opentui-spinner/solid"
|
||||
import { createSignal, For } from "solid-js"
|
||||
import { createStore, reconcile } from "solid-js/store"
|
||||
import {
|
||||
connectionMeterLevels,
|
||||
connectionTransitioning,
|
||||
textRevealOpacity,
|
||||
transcriptionPulse,
|
||||
type TextReveal,
|
||||
} from "./animation"
|
||||
import { PCM_METER_FRAME_MS } from "./pcm"
|
||||
import { initialVoiceView, transitionVoiceView, type Message, type VoiceViewEvent } from "./ui-model"
|
||||
|
||||
export { joinAssistantText } from "./ui-model"
|
||||
|
||||
export type VoiceStatus = {
|
||||
server?: string
|
||||
audio?: string
|
||||
microphoneMuted?: boolean
|
||||
speakerMuted?: boolean
|
||||
voice?: string
|
||||
model?: string
|
||||
project?: string
|
||||
}
|
||||
|
||||
export type VoiceUI = {
|
||||
meta(text: string): void
|
||||
userSpeaking(active: boolean): void
|
||||
userReset(): void
|
||||
userAudioLevel(level?: number): void
|
||||
userCommitted(itemID: string): void
|
||||
userTranscript(itemID: string, text: string, final?: boolean): void
|
||||
assistantAudio(level: number, durationMs: number): void
|
||||
assistantPlaybackStopped(): void
|
||||
assistantDelta(text: string): void
|
||||
assistantTranscript(text: string): void
|
||||
assistantDone(): void
|
||||
toolStart(callID: string, name: string, input: unknown): void
|
||||
toolDone(callID: string, output: unknown): void
|
||||
setStatus(patch: VoiceStatus): void
|
||||
close(): void
|
||||
}
|
||||
|
||||
const truncate = (text: string, max: number) => (text.length > max ? text.slice(0, max) + "…" : text)
|
||||
|
||||
const displayJson = (value: unknown) =>
|
||||
truncate(
|
||||
(() => {
|
||||
try {
|
||||
return (
|
||||
JSON.stringify(
|
||||
value,
|
||||
(_, item) =>
|
||||
typeof item === "string" ? truncate(item, 500) : typeof item === "bigint" ? String(item) : item,
|
||||
2,
|
||||
) ?? "null"
|
||||
)
|
||||
} catch {
|
||||
return "[unserializable value]"
|
||||
}
|
||||
})(),
|
||||
8_000,
|
||||
)
|
||||
|
||||
function toolSummary(name: string, output?: unknown) {
|
||||
if (output === undefined) return "running"
|
||||
if (Array.isArray(output)) return `${output.length} result${output.length === 1 ? "" : "s"}`
|
||||
if (typeof output === "string") return truncate(output, 160)
|
||||
if (typeof output === "number" || typeof output === "boolean" || typeof output === "bigint") return String(output)
|
||||
if (output === null || typeof output !== "object") return "completed"
|
||||
if (
|
||||
"status" in output &&
|
||||
output.status === "error" &&
|
||||
"message" in output &&
|
||||
typeof output.message === "string"
|
||||
)
|
||||
return `error · ${truncate(output.message, 120)}`
|
||||
if ("status" in output && typeof output.status === "string")
|
||||
return output.status === "started" && "notification" in output && output.notification === "registered"
|
||||
? "started · notification registered"
|
||||
: output.status
|
||||
const fields = Object.entries(output)
|
||||
.filter(
|
||||
([key, item]) =>
|
||||
!key.endsWith("_id") && key !== "notification" && ["string", "number", "boolean"].includes(typeof item),
|
||||
)
|
||||
.slice(0, 2)
|
||||
.map(([key, item]) => `${key}: ${String(item)}`)
|
||||
return fields.join(" · ") || "completed"
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Console fallback (--text mode, non-TTY)
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
export function createConsoleUI(): VoiceUI {
|
||||
const tty = process.stdout.isTTY
|
||||
const dim = (text: string) => (tty ? `\x1b[2m${text}\x1b[0m` : text)
|
||||
const cyan = (text: string) => (tty ? `\x1b[1;36m${text}\x1b[0m` : text)
|
||||
const green = (text: string) => (tty ? `\x1b[1;32m${text}\x1b[0m` : text)
|
||||
|
||||
let streaming = false
|
||||
const tools = new Map<string, string>()
|
||||
const line = (text: string) => {
|
||||
if (streaming) {
|
||||
process.stdout.write("\n")
|
||||
streaming = false
|
||||
}
|
||||
console.log(text)
|
||||
}
|
||||
|
||||
return {
|
||||
meta: (text) => line(dim(` ${text}`)),
|
||||
userSpeaking: () => {},
|
||||
userReset: () => {},
|
||||
userAudioLevel: () => {},
|
||||
userCommitted: () => {},
|
||||
userTranscript: (_, text) => line(cyan("● you ") + text),
|
||||
assistantAudio: () => {},
|
||||
assistantPlaybackStopped: () => {},
|
||||
assistantDelta: (text) => {
|
||||
if (!streaming) {
|
||||
process.stdout.write(green("● assistant "))
|
||||
streaming = true
|
||||
}
|
||||
process.stdout.write(text)
|
||||
},
|
||||
assistantTranscript: (text) => line(green("● assistant ") + text),
|
||||
assistantDone: () => {
|
||||
if (streaming) process.stdout.write("\n")
|
||||
streaming = false
|
||||
},
|
||||
toolStart: (callID, name) => {
|
||||
tools.set(callID, name)
|
||||
line(dim(` ◌ ${name} running`))
|
||||
},
|
||||
toolDone: (callID, output) => {
|
||||
const name = tools.get(callID) ?? "tool"
|
||||
tools.delete(callID)
|
||||
line(dim(` ✓ ${name} ${toolSummary(name, output)}`))
|
||||
},
|
||||
setStatus: () => {},
|
||||
close: () => {},
|
||||
}
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// OpenTUI
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
const theme = {
|
||||
background: "#1a1b26",
|
||||
surface: "#0d1018",
|
||||
text: "#c0caf5",
|
||||
muted: "#565f89",
|
||||
you: "#7dcfff",
|
||||
assistant: "#9ece6a",
|
||||
key: "#7aa2f7",
|
||||
string: "#9ece6a",
|
||||
number: "#e0af68",
|
||||
literal: "#bb9af7",
|
||||
}
|
||||
|
||||
const SPINNER_FRAMES = ["⠋", "⠙", "⠹", "⠸", "⠼", "⠴", "⠦", "⠧", "⠇", "⠏"]
|
||||
const colors = {
|
||||
background: RGBA.fromHex(theme.background),
|
||||
text: RGBA.fromHex(theme.text),
|
||||
you: RGBA.fromHex(theme.you),
|
||||
assistant: RGBA.fromHex(theme.assistant),
|
||||
}
|
||||
const AUDIO_LEVEL_STALE_MS = 250
|
||||
const TOOL_DISPLAY_DELAY_MS = 160
|
||||
const meterGlyphs = ["▁", "▂", "▃", "▄", "▅", "▆", "▇", "█"]
|
||||
|
||||
function fade(color: RGBA, opacity: number) {
|
||||
return RGBA.fromValues(color.r, color.g, color.b, color.a * opacity)
|
||||
}
|
||||
|
||||
function surface(color: RGBA, opacity: number) {
|
||||
return RGBA.fromValues(
|
||||
colors.background.r + (color.r - colors.background.r) * opacity,
|
||||
colors.background.g + (color.g - colors.background.g) * opacity,
|
||||
colors.background.b + (color.b - colors.background.b) * opacity,
|
||||
1,
|
||||
)
|
||||
}
|
||||
|
||||
function meter(levels: ReadonlyArray<number>) {
|
||||
return levels
|
||||
.map((value) => meterGlyphs[Math.round(Math.max(0, Math.min(1, value)) * (meterGlyphs.length - 1))])
|
||||
.join("")
|
||||
}
|
||||
|
||||
// Tiny JSON tokenizer for syntax-highlighted tool results.
|
||||
function jsonTokens(json: string) {
|
||||
const tokens: Array<{ text: string; color: string }> = []
|
||||
const pattern = /("(?:[^"\\]|\\.)*")(\s*:)?|(-?\d+\.?\d*(?:[eE][+-]?\d+)?)|(true|false|null)|([{}[\],:]+|\s+)/g
|
||||
for (const match of json.matchAll(pattern)) {
|
||||
if (match[1] !== undefined) {
|
||||
tokens.push({ text: match[1], color: match[2] ? theme.key : theme.string })
|
||||
if (match[2]) tokens.push({ text: match[2], color: theme.muted })
|
||||
continue
|
||||
}
|
||||
if (match[3] !== undefined) {
|
||||
tokens.push({ text: match[3], color: theme.number })
|
||||
continue
|
||||
}
|
||||
if (match[4] !== undefined) {
|
||||
tokens.push({ text: match[4], color: theme.literal })
|
||||
continue
|
||||
}
|
||||
tokens.push({ text: match[5] ?? "", color: theme.muted })
|
||||
}
|
||||
return tokens
|
||||
}
|
||||
|
||||
function RevealedText(props: {
|
||||
text: string
|
||||
reveals: ReadonlyArray<TextReveal>
|
||||
now: () => number
|
||||
opacity: () => number
|
||||
}) {
|
||||
return (
|
||||
<>
|
||||
<span style={{ fg: fade(colors.text, props.opacity()) }}>
|
||||
{props.text.slice(0, props.reveals[0]?.offset ?? props.text.length)}
|
||||
</span>
|
||||
<For each={props.reveals}>
|
||||
{(reveal, index) => (
|
||||
<span style={{ fg: fade(colors.text, props.opacity() * textRevealOpacity(props.now(), reveal.at)) }}>
|
||||
{props.text.slice(reveal.offset, props.reveals[index() + 1]?.offset ?? props.text.length)}
|
||||
</span>
|
||||
)}
|
||||
</For>
|
||||
</>
|
||||
)
|
||||
}
|
||||
|
||||
// Message transitions preserve unchanged object identities. A changed message
|
||||
// gets a new object, so keyed <For> replaces that row and this branch reruns.
|
||||
function MessageRow(props: {
|
||||
message: Message
|
||||
details: boolean
|
||||
assistantLevel: () => number
|
||||
userLevel: () => number
|
||||
now: () => number
|
||||
animate: boolean
|
||||
microphoneMuted: boolean
|
||||
previousKind?: Message["kind"]
|
||||
}) {
|
||||
const message = props.message
|
||||
if (message.kind === "user") {
|
||||
const transcribing = () => message.transcribing && !props.microphoneMuted
|
||||
const pulse = () =>
|
||||
!props.animate || !transcribing() ? 0 : Math.max(props.userLevel(), transcriptionPulse(props.now()) * 0.12)
|
||||
return (
|
||||
<box
|
||||
width="100%"
|
||||
flexDirection="column"
|
||||
flexShrink={0}
|
||||
border={["left"]}
|
||||
borderColor={fade(colors.you, 0.38 + pulse() * 0.32)}
|
||||
backgroundColor={surface(colors.you, 0.035 + pulse() * 0.07)}
|
||||
paddingLeft={1}
|
||||
paddingRight={1}
|
||||
marginTop={1}
|
||||
>
|
||||
<text width="100%" minWidth={0} wrapMode="word">
|
||||
<RevealedText
|
||||
text={message.text ?? "transcribing"}
|
||||
reveals={message.reveals}
|
||||
now={props.now}
|
||||
opacity={() => 0.74 + pulse() * 0.2}
|
||||
/>
|
||||
</text>
|
||||
</box>
|
||||
)
|
||||
}
|
||||
if (message.kind === "assistant") {
|
||||
const pulse = () => (!props.animate || !message.streaming ? 0.5 : props.assistantLevel())
|
||||
return (
|
||||
<box
|
||||
width="100%"
|
||||
flexDirection="column"
|
||||
flexShrink={0}
|
||||
border={["left"]}
|
||||
borderColor={fade(colors.assistant, message.streaming ? 0.58 + pulse() * 0.42 : 0.3)}
|
||||
backgroundColor={surface(colors.assistant, message.streaming ? 0.055 + pulse() * 0.11 : 0.025)}
|
||||
paddingLeft={1}
|
||||
paddingRight={1}
|
||||
marginTop={1}
|
||||
>
|
||||
<text width="100%" minWidth={0} wrapMode="word">
|
||||
<RevealedText
|
||||
text={message.text}
|
||||
reveals={message.reveals}
|
||||
now={props.now}
|
||||
opacity={() => (message.streaming ? 1 : 0.7)}
|
||||
/>
|
||||
</text>
|
||||
</box>
|
||||
)
|
||||
}
|
||||
if (message.kind === "tool")
|
||||
return (
|
||||
<box width="100%" flexDirection="column" paddingLeft={2} marginTop={props.previousKind === "tool" ? 0 : 1}>
|
||||
<box width="100%" minWidth={0} flexDirection="row">
|
||||
<box width={2} flexShrink={0}>
|
||||
{message.output === undefined ? (
|
||||
props.animate ? (
|
||||
<spinner frames={SPINNER_FRAMES} interval={80} color={theme.number} />
|
||||
) : (
|
||||
<text fg={theme.number}>⋯</text>
|
||||
)
|
||||
) : (
|
||||
<text fg={theme.assistant}>✓</text>
|
||||
)}
|
||||
</box>
|
||||
<text flexGrow={1} minWidth={0} wrapMode="word" fg={theme.muted}>
|
||||
<span style={{ fg: theme.key }}>{message.name}</span> {toolSummary(message.name, message.output)}
|
||||
</text>
|
||||
</box>
|
||||
{props.details ? (
|
||||
<text width="100%" minWidth={0} wrapMode="word" fg={theme.muted} paddingLeft={2}>
|
||||
<For each={jsonTokens(displayJson({ input: message.input, output: message.output }))}>
|
||||
{(token) => <span style={{ fg: token.color }}>{token.text}</span>}
|
||||
</For>
|
||||
</text>
|
||||
) : null}
|
||||
</box>
|
||||
)
|
||||
return (
|
||||
<box width="100%" minWidth={0} flexDirection="row" paddingLeft={2}>
|
||||
<text width={3} flexShrink={0} fg={theme.muted}>
|
||||
·
|
||||
</text>
|
||||
<text flexGrow={1} minWidth={0} wrapMode="word" fg={theme.muted}>
|
||||
{message.text}
|
||||
</text>
|
||||
</box>
|
||||
)
|
||||
}
|
||||
|
||||
export async function createVoiceTUI(options: {
|
||||
onInterrupt(): void
|
||||
onExit(): void
|
||||
onCycleVoice(): void
|
||||
onToggleMicrophone(): void
|
||||
onToggleSpeaker(): void
|
||||
reducedMotion?: boolean
|
||||
}): Promise<VoiceUI> {
|
||||
let view = initialVoiceView()
|
||||
const [messageState, setMessageState] = createStore({ items: [...view.messages] })
|
||||
const [status, setStatus] = createSignal<VoiceStatus>({})
|
||||
const [details, setDetails] = createSignal(false)
|
||||
const [animationFrame, setAnimationFrame] = createSignal(performance.now())
|
||||
const animationStartedAt = performance.now()
|
||||
const [userActive, setUserActive] = createSignal(false)
|
||||
let animationTimer: ReturnType<typeof setInterval> | undefined
|
||||
let assistantSegments: Array<{ start: number; end: number; level: number }> = []
|
||||
let assistantSegmentIndex = 0
|
||||
let assistantScheduledUntil = 0
|
||||
let assistantLevel = 0
|
||||
let userTargetLevel = 0
|
||||
let userLevel = 0
|
||||
let userLevelAt = 0
|
||||
let connectedAt: number | undefined
|
||||
const pendingTools = new Map<
|
||||
string,
|
||||
{
|
||||
name: string
|
||||
input: unknown
|
||||
output?: unknown
|
||||
completed: boolean
|
||||
showAt: number
|
||||
timer: ReturnType<typeof setTimeout>
|
||||
}
|
||||
>()
|
||||
const pendingMetaTimers = new Set<ReturnType<typeof setTimeout>>()
|
||||
|
||||
const renderer = await createCliRenderer({
|
||||
useMouse: true,
|
||||
// Handle Ctrl-C in OpenTUI's native input parser, before Solid handlers
|
||||
// and rendering work. onDestroy performs process/audio cleanup below.
|
||||
exitOnCtrlC: true,
|
||||
exitSignals: [],
|
||||
autoFocus: false,
|
||||
openConsoleOnError: false,
|
||||
screenMode: "alternate-screen",
|
||||
externalOutputMode: "passthrough",
|
||||
consoleMode: "disabled",
|
||||
onDestroy: () => {
|
||||
if (animationTimer) clearInterval(animationTimer)
|
||||
pendingTools.forEach((tool) => clearTimeout(tool.timer))
|
||||
pendingMetaTimers.forEach(clearTimeout)
|
||||
options.onExit()
|
||||
},
|
||||
})
|
||||
|
||||
const applyView = (event: VoiceViewEvent) => {
|
||||
const next = transitionVoiceView(view, event)
|
||||
if (next === view) return false
|
||||
view = next
|
||||
setMessageState("items", reconcile([...view.messages], { key: "key" }))
|
||||
return true
|
||||
}
|
||||
|
||||
const startAnimation = () => {
|
||||
if (options.reducedMotion) return
|
||||
if (animationTimer) return
|
||||
animationTimer = setInterval(() => {
|
||||
const now = performance.now()
|
||||
while (assistantSegments[assistantSegmentIndex]?.end <= now) assistantSegmentIndex += 1
|
||||
const segment = assistantSegments[assistantSegmentIndex]
|
||||
const output = segment?.start <= now ? segment : undefined
|
||||
const assistantTarget = output?.level ?? 0
|
||||
const userTarget = now - userLevelAt < AUDIO_LEVEL_STALE_MS ? userTargetLevel : 0
|
||||
assistantLevel += (assistantTarget - assistantLevel) * (assistantTarget > assistantLevel ? 0.5 : 0.16)
|
||||
userLevel += (userTarget - userLevel) * (userTarget > userLevel ? 0.5 : 0.18)
|
||||
setAnimationFrame(now)
|
||||
if (view.revealAnimationEndsAt > 0 && now >= view.revealAnimationEndsAt) applyView({ type: "reveals.completed" })
|
||||
const transcribing =
|
||||
!status().microphoneMuted &&
|
||||
messageState.items.some((message) => message.kind === "user" && message.transcribing)
|
||||
if (
|
||||
userActive() ||
|
||||
transcribing ||
|
||||
now < view.revealAnimationEndsAt ||
|
||||
assistantSegmentIndex < assistantSegments.length ||
|
||||
assistantLevel > 0.01 ||
|
||||
userLevel > 0.01 ||
|
||||
connectionTransitioning(now, connectedAt)
|
||||
)
|
||||
return
|
||||
clearInterval(animationTimer)
|
||||
animationTimer = undefined
|
||||
}, PCM_METER_FRAME_MS)
|
||||
}
|
||||
|
||||
const currentAssistantLevel = () => {
|
||||
animationFrame()
|
||||
return assistantLevel
|
||||
}
|
||||
const currentUserLevel = () => {
|
||||
animationFrame()
|
||||
return userLevel
|
||||
}
|
||||
|
||||
function App() {
|
||||
const dimensions = useTerminalDimensions()
|
||||
useKeyboard((evt) => {
|
||||
if (evt.ctrl && evt.name === "c") return
|
||||
if (evt.repeated || evt.ctrl || evt.meta || evt.option || evt.shift) return
|
||||
if (evt.name === "v") {
|
||||
evt.preventDefault()
|
||||
return options.onCycleVoice()
|
||||
}
|
||||
if (evt.name === "m") {
|
||||
evt.preventDefault()
|
||||
return options.onToggleMicrophone()
|
||||
}
|
||||
if (evt.name === "s") {
|
||||
evt.preventDefault()
|
||||
return options.onToggleSpeaker()
|
||||
}
|
||||
if (evt.name === "d") {
|
||||
evt.preventDefault()
|
||||
setDetails((current) => !current)
|
||||
return
|
||||
}
|
||||
if (evt.name === "escape") {
|
||||
evt.preventDefault()
|
||||
options.onInterrupt()
|
||||
}
|
||||
})
|
||||
const runtimeLine = () =>
|
||||
[
|
||||
status().audio ?? "connecting…",
|
||||
status().microphoneMuted ? "mic muted" : undefined,
|
||||
status().speakerMuted ? "speaker muted" : undefined,
|
||||
status().voice,
|
||||
status().model,
|
||||
status().project,
|
||||
]
|
||||
.filter(Boolean)
|
||||
.join(" ")
|
||||
const microphoneLevel = () => {
|
||||
if (status().microphoneMuted) return 0
|
||||
return Math.max(userActive() ? 0.16 : 0.1, currentUserLevel())
|
||||
}
|
||||
const microphoneOpacity = () => (status().microphoneMuted ? 0.16 : userActive() ? 0.9 : 0.38)
|
||||
const microphoneMeter = () => {
|
||||
if (options.reducedMotion) return meter(connectionMeterLevels(0, microphoneLevel(), -480))
|
||||
return meter(connectionMeterLevels(animationFrame(), microphoneLevel(), connectedAt, animationStartedAt))
|
||||
}
|
||||
return (
|
||||
<box
|
||||
flexDirection="column"
|
||||
width={dimensions().width}
|
||||
height={dimensions().height}
|
||||
paddingLeft={1}
|
||||
paddingRight={1}
|
||||
backgroundColor={theme.background}
|
||||
>
|
||||
<box width="100%" flexGrow={1} minHeight={0} minWidth={0} backgroundColor={theme.background}>
|
||||
<scrollbox
|
||||
width="100%"
|
||||
height="100%"
|
||||
stickyScroll
|
||||
stickyStart="bottom"
|
||||
scrollbarOptions={{ visible: false }}
|
||||
backgroundColor={theme.background}
|
||||
>
|
||||
<box
|
||||
width="100%"
|
||||
minWidth={0}
|
||||
minHeight={Math.max(0, dimensions().height - 3)}
|
||||
flexDirection="column"
|
||||
flexShrink={0}
|
||||
justifyContent="flex-end"
|
||||
backgroundColor={theme.background}
|
||||
>
|
||||
<For each={messageState.items}>
|
||||
{(message, index) => (
|
||||
<MessageRow
|
||||
message={message}
|
||||
previousKind={messageState.items[index() - 1]?.kind}
|
||||
details={details()}
|
||||
assistantLevel={currentAssistantLevel}
|
||||
userLevel={currentUserLevel}
|
||||
now={animationFrame}
|
||||
animate={!options.reducedMotion}
|
||||
microphoneMuted={status().microphoneMuted === true}
|
||||
/>
|
||||
)}
|
||||
</For>
|
||||
</box>
|
||||
</scrollbox>
|
||||
</box>
|
||||
<box width="100%" height={1} flexShrink={0} backgroundColor={theme.background} />
|
||||
<box width="100%" height={2} flexShrink={0} backgroundColor={theme.surface} paddingLeft={1} paddingRight={1}>
|
||||
<box width="100%" height={1} minWidth={0} flexDirection="row">
|
||||
<text width={5} flexShrink={0} fg={fade(colors.you, microphoneOpacity())}>
|
||||
<b>
|
||||
{status().microphoneMuted
|
||||
? "────"
|
||||
: microphoneMeter()}
|
||||
</b>
|
||||
</text>
|
||||
<text flexGrow={1} minWidth={0} wrapMode="none" truncate fg={fade(colors.text, 0.78)}>
|
||||
{runtimeLine()}
|
||||
</text>
|
||||
</box>
|
||||
<text width="100%" wrapMode="none" truncate fg={theme.muted}>
|
||||
<span style={{ fg: theme.key }}>esc</span> interrupt {" "}
|
||||
<span style={{ fg: theme.key }}>v</span> voice {" "}
|
||||
<span style={{ fg: theme.key }}>m</span> mic {" "}
|
||||
<span style={{ fg: theme.key }}>s</span> speaker {" "}
|
||||
<span style={{ fg: theme.key }}>d</span> details {details() ? "on" : "off"} {" "}
|
||||
<span style={{ fg: theme.key }}>ctrl+c</span> quit
|
||||
</text>
|
||||
</box>
|
||||
</box>
|
||||
)
|
||||
}
|
||||
|
||||
await render(() => <App />, renderer)
|
||||
startAnimation()
|
||||
|
||||
const pushMeta = (text: string) => {
|
||||
if (pendingTools.size === 0) {
|
||||
applyView({ type: "meta", text })
|
||||
return
|
||||
}
|
||||
const showAt = Math.max(...[...pendingTools.values()].map((tool) => tool.showAt))
|
||||
const timer = setTimeout(
|
||||
() => {
|
||||
pendingMetaTimers.delete(timer)
|
||||
applyView({ type: "meta", text })
|
||||
},
|
||||
Math.max(0, showAt - performance.now()) + 1,
|
||||
)
|
||||
pendingMetaTimers.add(timer)
|
||||
}
|
||||
|
||||
return {
|
||||
meta: pushMeta,
|
||||
userSpeaking: (active) => {
|
||||
if (active) applyView({ type: "user.started" })
|
||||
setUserActive(active)
|
||||
if (active) startAnimation()
|
||||
if (!active) userTargetLevel = 0
|
||||
},
|
||||
userReset: () => {
|
||||
setUserActive(false)
|
||||
userTargetLevel = 0
|
||||
userLevel = 0
|
||||
applyView({ type: "user.reset" })
|
||||
},
|
||||
userAudioLevel: (level) => {
|
||||
if (level === undefined) {
|
||||
userLevelAt = 0
|
||||
return
|
||||
}
|
||||
userTargetLevel = Math.max(0, Math.min(1, level))
|
||||
userLevelAt = performance.now()
|
||||
if (options.reducedMotion) {
|
||||
userLevel = userTargetLevel
|
||||
setAnimationFrame(userLevelAt)
|
||||
return
|
||||
}
|
||||
startAnimation()
|
||||
},
|
||||
userCommitted: (itemID) => {
|
||||
setUserActive(false)
|
||||
applyView({ type: "user.committed", itemID })
|
||||
startAnimation()
|
||||
},
|
||||
userTranscript: (itemID, text, final = true) => {
|
||||
applyView({
|
||||
type: "user.transcript",
|
||||
itemID,
|
||||
text,
|
||||
final,
|
||||
now: performance.now(),
|
||||
animate: !options.reducedMotion,
|
||||
})
|
||||
if (!final || view.revealAnimationEndsAt > 0) startAnimation()
|
||||
},
|
||||
assistantAudio: (level, durationMs) => {
|
||||
if (options.reducedMotion) return
|
||||
if (assistantSegmentIndex > 512) {
|
||||
assistantSegments = assistantSegments.slice(assistantSegmentIndex)
|
||||
assistantSegmentIndex = 0
|
||||
}
|
||||
const now = performance.now()
|
||||
const start = Math.max(now, assistantScheduledUntil)
|
||||
const end = start + durationMs
|
||||
assistantSegments.push({ start, end, level: Math.max(0, Math.min(1, level)) })
|
||||
assistantScheduledUntil = end
|
||||
startAnimation()
|
||||
},
|
||||
assistantPlaybackStopped: () => {
|
||||
assistantSegments = []
|
||||
assistantSegmentIndex = 0
|
||||
assistantScheduledUntil = 0
|
||||
assistantLevel = 0
|
||||
setAnimationFrame(performance.now())
|
||||
},
|
||||
assistantDelta: (text) => {
|
||||
applyView({
|
||||
type: "assistant.delta",
|
||||
text,
|
||||
now: performance.now(),
|
||||
animate: !options.reducedMotion,
|
||||
})
|
||||
if (view.revealAnimationEndsAt > 0) startAnimation()
|
||||
},
|
||||
assistantTranscript: (text) => {
|
||||
applyView({
|
||||
type: "assistant.transcript",
|
||||
text,
|
||||
now: performance.now(),
|
||||
animate: !options.reducedMotion,
|
||||
})
|
||||
if (view.revealAnimationEndsAt > 0) startAnimation()
|
||||
},
|
||||
assistantDone: () => {
|
||||
assistantSegments = []
|
||||
assistantSegmentIndex = 0
|
||||
assistantScheduledUntil = 0
|
||||
assistantLevel = 0
|
||||
applyView({ type: "assistant.done" })
|
||||
},
|
||||
toolStart: (callID, name, input) => {
|
||||
if (
|
||||
pendingTools.has(callID) ||
|
||||
view.messages.some((message) => message.kind === "tool" && message.callID === callID)
|
||||
)
|
||||
return
|
||||
const pending = {
|
||||
name,
|
||||
input,
|
||||
completed: false,
|
||||
showAt: performance.now() + TOOL_DISPLAY_DELAY_MS,
|
||||
timer: setTimeout(() => {
|
||||
const current = pendingTools.get(callID)
|
||||
if (!current) return
|
||||
pendingTools.delete(callID)
|
||||
applyView({
|
||||
type: "tool.started",
|
||||
callID,
|
||||
name: current.name,
|
||||
input: current.input,
|
||||
})
|
||||
if (current.completed) applyView({ type: "tool.done", callID, output: current.output })
|
||||
}, TOOL_DISPLAY_DELAY_MS),
|
||||
}
|
||||
pendingTools.set(callID, pending)
|
||||
},
|
||||
toolDone: (callID, output) => {
|
||||
const pending = pendingTools.get(callID)
|
||||
if (pending) {
|
||||
pending.output = output
|
||||
pending.completed = true
|
||||
return
|
||||
}
|
||||
applyView({ type: "tool.done", callID, output })
|
||||
},
|
||||
setStatus: (patch) => {
|
||||
if (connectedAt === undefined && patch.audio !== undefined) {
|
||||
connectedAt = performance.now()
|
||||
startAnimation()
|
||||
}
|
||||
setStatus((current) => ({ ...current, ...patch }))
|
||||
},
|
||||
close: () => {
|
||||
if (!renderer.isDestroyed) renderer.destroy()
|
||||
},
|
||||
}
|
||||
}
|
||||
@@ -1,242 +0,0 @@
|
||||
import type { CompletionReceipt } from "./opencode-notification"
|
||||
|
||||
export type VoiceNotification = {
|
||||
readonly id: string
|
||||
readonly receipt?: CompletionReceipt
|
||||
readonly promptID?: string
|
||||
readonly delegationID?: string
|
||||
readonly text: string
|
||||
}
|
||||
|
||||
export type VoiceState = {
|
||||
readonly connection: "connecting" | "ready" | "closed"
|
||||
readonly conversation: "idle" | "waiting" | "responding"
|
||||
readonly assistant: "idle" | "active" | "suppressed"
|
||||
readonly userSpeaking: boolean
|
||||
readonly tools: ReadonlySet<string>
|
||||
readonly notifications: ReadonlyArray<VoiceNotification>
|
||||
readonly delivery?: {
|
||||
readonly notification: VoiceNotification
|
||||
readonly phase: "sending" | "accepted" | "announcing" | "awaiting-playback"
|
||||
}
|
||||
readonly desiredVoice: string
|
||||
readonly reconnect: "none" | "pending"
|
||||
}
|
||||
|
||||
export type VoiceEvent =
|
||||
| { readonly type: "connection.ready" }
|
||||
| { readonly type: "connection.connecting" }
|
||||
| { readonly type: "connection.closed" }
|
||||
| { readonly type: "user.started"; readonly bargeIn?: boolean }
|
||||
| { readonly type: "user.stopped" }
|
||||
| { readonly type: "user.committed" }
|
||||
| { readonly type: "assistant.started" }
|
||||
| { readonly type: "assistant.suppressed" }
|
||||
| { readonly type: "assistant.done"; readonly awaitingWork: boolean }
|
||||
| { readonly type: "tool.started"; readonly id: string }
|
||||
| { readonly type: "tool.finished"; readonly id: string }
|
||||
| { readonly type: "notification.queued"; readonly notification: VoiceNotification }
|
||||
| { readonly type: "notification.accepted"; readonly id: string }
|
||||
| { readonly type: "notification.failed"; readonly id: string }
|
||||
| { readonly type: "notification.announced"; readonly id: string }
|
||||
| { readonly type: "notification.interrupted"; readonly id: string }
|
||||
| { readonly type: "voice.selected"; readonly voice: string }
|
||||
|
||||
export type VoiceCommand =
|
||||
| { readonly type: "notification.send"; readonly notification: VoiceNotification }
|
||||
| { readonly type: "notification.delivered"; readonly notification: VoiceNotification }
|
||||
| { readonly type: "assistant.finish"; readonly awaitingWork: boolean; readonly notificationID?: string }
|
||||
| { readonly type: "assistant.interrupt" }
|
||||
| { readonly type: "connection.reconnect" }
|
||||
|
||||
export type VoiceTransition = {
|
||||
readonly state: VoiceState
|
||||
readonly commands: ReadonlyArray<VoiceCommand>
|
||||
}
|
||||
|
||||
export function initialVoiceState(voice: string): VoiceState {
|
||||
return {
|
||||
connection: "connecting",
|
||||
conversation: "idle",
|
||||
assistant: "idle",
|
||||
userSpeaking: false,
|
||||
tools: new Set(),
|
||||
notifications: [],
|
||||
desiredVoice: voice,
|
||||
reconnect: "none",
|
||||
}
|
||||
}
|
||||
|
||||
export function transitionVoice(state: VoiceState, event: VoiceEvent): VoiceTransition {
|
||||
if (event.type === "notification.announced") {
|
||||
if (state.delivery?.notification.id !== event.id || state.delivery.phase !== "awaiting-playback")
|
||||
return { state, commands: [] }
|
||||
const notification = state.delivery.notification
|
||||
const next = settle({ ...state, delivery: undefined })
|
||||
return {
|
||||
state: next.state,
|
||||
commands: [{ type: "notification.delivered", notification }, ...next.commands],
|
||||
}
|
||||
}
|
||||
if (event.type === "assistant.done") {
|
||||
const notificationID =
|
||||
state.delivery?.phase === "announcing" && state.assistant === "active" && !event.awaitingWork
|
||||
? state.delivery.notification.id
|
||||
: undefined
|
||||
const next = settle(reduce(state, event))
|
||||
return {
|
||||
state: next.state,
|
||||
commands: [{ type: "assistant.finish", awaitingWork: event.awaitingWork, notificationID }, ...next.commands],
|
||||
}
|
||||
}
|
||||
if (event.type === "user.started" && event.bargeIn && state.assistant === "active") {
|
||||
const next = settle(
|
||||
state.delivery
|
||||
? {
|
||||
...state,
|
||||
conversation: "idle",
|
||||
assistant: "idle",
|
||||
userSpeaking: true,
|
||||
delivery: undefined,
|
||||
notifications: [state.delivery.notification, ...state.notifications],
|
||||
reconnect: "pending",
|
||||
}
|
||||
: { ...state, assistant: "suppressed", userSpeaking: true },
|
||||
)
|
||||
return { state: next.state, commands: [{ type: "assistant.interrupt" }, ...next.commands] }
|
||||
}
|
||||
const next = reduce(state, event)
|
||||
return settle(next)
|
||||
}
|
||||
|
||||
function reduce(state: VoiceState, event: VoiceEvent): VoiceState {
|
||||
switch (event.type) {
|
||||
case "connection.ready":
|
||||
if (state.connection === "ready") return state
|
||||
return { ...state, connection: "ready" }
|
||||
case "connection.connecting":
|
||||
return {
|
||||
...state,
|
||||
connection: "connecting",
|
||||
conversation: "idle",
|
||||
assistant: "idle",
|
||||
userSpeaking: false,
|
||||
}
|
||||
case "connection.closed":
|
||||
return { ...state, connection: "closed", conversation: "idle", assistant: "idle", userSpeaking: false }
|
||||
case "user.started":
|
||||
if (state.userSpeaking) return state
|
||||
return { ...state, userSpeaking: true }
|
||||
case "user.stopped":
|
||||
if (!state.userSpeaking) return state
|
||||
return { ...state, userSpeaking: false }
|
||||
case "user.committed":
|
||||
return {
|
||||
...state,
|
||||
userSpeaking: false,
|
||||
conversation: state.assistant === "active" ? "responding" : "waiting",
|
||||
assistant: state.assistant === "suppressed" ? "idle" : state.assistant,
|
||||
}
|
||||
case "assistant.started":
|
||||
if (state.assistant === "suppressed" || state.assistant === "active") return state
|
||||
return {
|
||||
...state,
|
||||
conversation: "responding",
|
||||
assistant: "active",
|
||||
delivery:
|
||||
state.delivery?.phase === "accepted" ? { ...state.delivery, phase: "announcing" } : state.delivery,
|
||||
}
|
||||
case "assistant.suppressed":
|
||||
return { ...state, assistant: "suppressed" }
|
||||
case "assistant.done":
|
||||
return {
|
||||
...state,
|
||||
conversation: event.awaitingWork ? "waiting" : "idle",
|
||||
assistant: "idle",
|
||||
delivery:
|
||||
state.delivery?.phase === "announcing" && state.assistant === "active" && !event.awaitingWork
|
||||
? { ...state.delivery, phase: "awaiting-playback" }
|
||||
: state.delivery,
|
||||
}
|
||||
case "tool.started":
|
||||
if (state.tools.has(event.id)) return state
|
||||
return { ...state, conversation: "waiting", tools: new Set([...state.tools, event.id]) }
|
||||
case "tool.finished": {
|
||||
if (!state.tools.has(event.id)) return state
|
||||
const tools = new Set(state.tools)
|
||||
tools.delete(event.id)
|
||||
return { ...state, tools }
|
||||
}
|
||||
case "notification.queued":
|
||||
return { ...state, notifications: [...state.notifications, event.notification] }
|
||||
case "notification.accepted":
|
||||
if (state.delivery?.notification.id !== event.id || state.delivery.phase !== "sending") return state
|
||||
return {
|
||||
...state,
|
||||
delivery: { ...state.delivery, phase: "accepted" },
|
||||
}
|
||||
case "notification.failed":
|
||||
if (state.delivery?.notification.id !== event.id) return state
|
||||
return {
|
||||
...state,
|
||||
connection: "ready",
|
||||
conversation: "idle",
|
||||
delivery: undefined,
|
||||
notifications: [state.delivery.notification, ...state.notifications],
|
||||
reconnect: "pending",
|
||||
}
|
||||
case "notification.announced":
|
||||
return state
|
||||
case "notification.interrupted":
|
||||
if (state.delivery?.notification.id !== event.id) return state
|
||||
return {
|
||||
...state,
|
||||
conversation: "idle",
|
||||
assistant: "idle",
|
||||
delivery: undefined,
|
||||
notifications: [state.delivery.notification, ...state.notifications],
|
||||
reconnect: "pending",
|
||||
}
|
||||
case "voice.selected":
|
||||
if (event.voice === state.desiredVoice) return state
|
||||
return { ...state, desiredVoice: event.voice, reconnect: "pending" }
|
||||
}
|
||||
return state
|
||||
}
|
||||
|
||||
function settle(state: VoiceState): VoiceTransition {
|
||||
if (
|
||||
state.reconnect === "pending" &&
|
||||
state.connection === "ready" &&
|
||||
state.conversation === "idle" &&
|
||||
state.delivery === undefined &&
|
||||
!state.userSpeaking &&
|
||||
state.tools.size === 0
|
||||
)
|
||||
return {
|
||||
state: { ...state, connection: "connecting", reconnect: "none" },
|
||||
commands: [{ type: "connection.reconnect" }],
|
||||
}
|
||||
|
||||
if (
|
||||
state.notifications.length > 0 &&
|
||||
state.delivery === undefined &&
|
||||
state.connection === "ready" &&
|
||||
state.conversation === "idle" &&
|
||||
!state.userSpeaking &&
|
||||
state.tools.size === 0
|
||||
) {
|
||||
const notification = state.notifications[0]
|
||||
return {
|
||||
state: {
|
||||
...state,
|
||||
conversation: "waiting",
|
||||
delivery: { notification, phase: "sending" },
|
||||
notifications: state.notifications.slice(1),
|
||||
},
|
||||
commands: [{ type: "notification.send", notification }],
|
||||
}
|
||||
}
|
||||
|
||||
return { state, commands: [] }
|
||||
}
|
||||
@@ -1,446 +0,0 @@
|
||||
import type { AudioEvent, AudioSession } from "./audio-session"
|
||||
import { openCodeAnnouncementText, type CompletionReceipt, type OpenCodeAnnouncement } from "./opencode-notification"
|
||||
import { createNotificationRouter } from "./notification-router"
|
||||
import type {
|
||||
VoiceConnection,
|
||||
VoiceDelegationRequest,
|
||||
VoiceProtocol,
|
||||
VoiceProtocolEvent,
|
||||
VoiceProtocolOptions,
|
||||
VoiceTool,
|
||||
VoiceToolExecution,
|
||||
VoiceWorkRequest,
|
||||
} from "./protocol"
|
||||
import type { VoiceUI } from "./ui"
|
||||
import { initialVoiceState, transitionVoice, type VoiceCommand, type VoiceEvent } from "./voice-coordinator"
|
||||
|
||||
export type VoiceSession = {
|
||||
start(): void
|
||||
cycleVoice(): void
|
||||
interrupt(): void
|
||||
toggleMicrophone(): void
|
||||
toggleSpeaker(): void
|
||||
queueNotification(announcement: OpenCodeAnnouncement): void
|
||||
close(): Promise<void>
|
||||
}
|
||||
|
||||
export function voiceControlTool(voices: ReadonlyArray<string>): VoiceTool {
|
||||
return {
|
||||
type: "function",
|
||||
name: "set_voice",
|
||||
description: "Change your speaking voice. Requires a brief reconnect and resets voice conversation memory.",
|
||||
parameters: {
|
||||
type: "object",
|
||||
additionalProperties: false,
|
||||
properties: { voice: { type: "string", enum: voices } },
|
||||
required: ["voice"],
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
export function createVoiceSession(options: {
|
||||
readonly protocol: VoiceProtocol
|
||||
readonly connection: Omit<VoiceProtocolOptions, "voice" | "onEvent">
|
||||
readonly initialVoice: string
|
||||
readonly voices: ReadonlyArray<string>
|
||||
readonly text?: string
|
||||
readonly ui: VoiceUI
|
||||
readonly audio?: AudioSession
|
||||
readonly tools: {
|
||||
readonly execute: (name: string, input: Record<string, unknown>) => Promise<VoiceToolExecution>
|
||||
readonly acknowledge: (receipt: CompletionReceipt) => Promise<void>
|
||||
readonly close: () => Promise<void>
|
||||
}
|
||||
readonly delegate: (
|
||||
request: VoiceDelegationRequest,
|
||||
execute: (request: VoiceWorkRequest) => Promise<unknown>,
|
||||
) => Promise<string>
|
||||
readonly trace?: (event: string, data?: Record<string, unknown>) => void
|
||||
readonly onClosed: () => void
|
||||
}): VoiceSession {
|
||||
let state = initialVoiceState(options.initialVoice)
|
||||
let connection: VoiceConnection | undefined
|
||||
let closed = false
|
||||
let closePromise: Promise<void> | undefined
|
||||
let voiceTimer: ReturnType<typeof setTimeout> | undefined
|
||||
const notifications = createNotificationRouter((routed) => {
|
||||
options.trace?.("notification.queued", {
|
||||
type: routed.announcement.notification.type,
|
||||
promptID: routed.promptID,
|
||||
depth: state.notifications.length + 1,
|
||||
})
|
||||
dispatch({
|
||||
type: "notification.queued",
|
||||
notification: {
|
||||
id: crypto.randomUUID(),
|
||||
receipt: routed.announcement.receipt,
|
||||
promptID: routed.promptID,
|
||||
delegationID: routed.delegationID,
|
||||
text: openCodeAnnouncementText(routed.announcement.notification),
|
||||
},
|
||||
})
|
||||
})
|
||||
|
||||
function dispatch(event: VoiceEvent) {
|
||||
const transition = transitionVoice(state, event)
|
||||
if (transition.state === state && transition.commands.length === 0) return
|
||||
state = transition.state
|
||||
options.trace?.("voice.transition", {
|
||||
input: event.type,
|
||||
connection: state.connection,
|
||||
conversation: state.conversation,
|
||||
assistant: state.assistant,
|
||||
userSpeaking: state.userSpeaking,
|
||||
tools: state.tools.size,
|
||||
notifications: state.notifications.length,
|
||||
commands: transition.commands.map((command) => command.type),
|
||||
})
|
||||
transition.commands.forEach(runCommand)
|
||||
}
|
||||
|
||||
const setVoice = (voice: string) => {
|
||||
options.ui.setStatus({ voice })
|
||||
options.ui.meta(`[voice] switching to ${voice}…`)
|
||||
dispatch({ type: "voice.selected", voice })
|
||||
}
|
||||
|
||||
const interruptDelivery = () => {
|
||||
if (state.delivery) dispatch({ type: "notification.interrupted", id: state.delivery.notification.id })
|
||||
}
|
||||
|
||||
const reconnect = () => {
|
||||
if (closed) return
|
||||
options.audio?.flushPlayback()
|
||||
options.ui.assistantDone()
|
||||
const previous = connection
|
||||
connection = undefined
|
||||
const next = () => {
|
||||
if (!closed) connect()
|
||||
}
|
||||
if (!previous) return next()
|
||||
void previous.close({ graceful: false }).finally(next)
|
||||
}
|
||||
|
||||
function runCommand(command: VoiceCommand) {
|
||||
if (command.type === "connection.reconnect") return reconnect()
|
||||
if (command.type === "assistant.interrupt") {
|
||||
connection?.interrupt?.()
|
||||
options.audio?.flushPlayback()
|
||||
options.ui.assistantDone()
|
||||
return
|
||||
}
|
||||
if (command.type === "assistant.finish") {
|
||||
if (options.text) {
|
||||
options.ui.assistantDone()
|
||||
if (command.notificationID) dispatch({ type: "notification.announced", id: command.notificationID })
|
||||
if (!command.awaitingWork && state.tools.size === 0) options.onClosed()
|
||||
return
|
||||
}
|
||||
void options.audio?.finishPlayback().then((outcome) => {
|
||||
options.ui.assistantDone()
|
||||
if (!command.notificationID) return
|
||||
dispatch({
|
||||
type: outcome === "played" ? "notification.announced" : "notification.interrupted",
|
||||
id: command.notificationID,
|
||||
})
|
||||
})
|
||||
return
|
||||
}
|
||||
if (command.type === "notification.delivered") {
|
||||
options.trace?.("notification.announced", {
|
||||
id: command.notification.id,
|
||||
promptID: command.notification.promptID,
|
||||
})
|
||||
if (!command.notification.receipt) return
|
||||
const receipt = command.notification.receipt
|
||||
void options.tools
|
||||
.acknowledge(receipt)
|
||||
.catch((error) =>
|
||||
options.trace?.("notification.ack.failed", { id: command.notification.id, error: String(error) }),
|
||||
)
|
||||
.finally(() => notifications.acknowledged(receipt))
|
||||
return
|
||||
}
|
||||
const active = connection
|
||||
if (!active) return dispatch({ type: "notification.failed", id: command.notification.id })
|
||||
void active
|
||||
.notify({
|
||||
id: command.notification.id,
|
||||
text: `Announce this OpenCode update conversationally without reading private identifiers aloud:\n${command.notification.text}`,
|
||||
delegationID: command.notification.delegationID,
|
||||
})
|
||||
.then((accepted) => {
|
||||
if (!accepted) return dispatch({ type: "notification.failed", id: command.notification.id })
|
||||
dispatch({ type: "notification.accepted", id: command.notification.id })
|
||||
options.trace?.("notification.context.accepted", {
|
||||
id: command.notification.id,
|
||||
promptID: command.notification.promptID,
|
||||
remaining: state.notifications.length,
|
||||
})
|
||||
})
|
||||
.catch((error) => {
|
||||
options.trace?.("notification.context.failed", {
|
||||
id: command.notification.id,
|
||||
promptID: command.notification.promptID,
|
||||
error: String(error),
|
||||
})
|
||||
dispatch({ type: "notification.failed", id: command.notification.id })
|
||||
})
|
||||
}
|
||||
|
||||
const executeTool = async (request: VoiceWorkRequest, delegationID?: string) => {
|
||||
if (delegationID) notifications.beginDelegatedTool()
|
||||
const execution: VoiceToolExecution = await (request.name === "set_voice"
|
||||
? Promise.resolve().then(() => {
|
||||
const voice = request.input["voice"]
|
||||
if (typeof voice !== "string" || !options.voices.includes(voice))
|
||||
return { output: toolError(`Voice must be one of: ${options.voices.join(", ")}.`) }
|
||||
if (voiceTimer) clearTimeout(voiceTimer)
|
||||
voiceTimer = setTimeout(() => {
|
||||
voiceTimer = undefined
|
||||
setVoice(voice)
|
||||
}, 1_000)
|
||||
return {
|
||||
output: {
|
||||
status: "switching",
|
||||
voice,
|
||||
note: `${options.protocol.name} conversation memory resets during reconnect.`,
|
||||
},
|
||||
}
|
||||
})
|
||||
: options.tools
|
||||
.execute(request.name, request.input)
|
||||
.catch((error) => ({ output: toolError(String(error), true) } satisfies VoiceToolExecution)))
|
||||
options.ui.toolDone(request.id, execution.output)
|
||||
if (delegationID) notifications.finishDelegatedTool(delegationID, execution.admittedPrompt)
|
||||
return execution.output
|
||||
}
|
||||
|
||||
const queueWork = (source: VoiceConnection, request: VoiceWorkRequest) => {
|
||||
options.trace?.("work.started", { id: request.id, name: request.name })
|
||||
void executeTool(request)
|
||||
.then((output) => {
|
||||
options.trace?.("work.resolved", { id: request.id, name: request.name })
|
||||
source.resolveWork(request, output)
|
||||
})
|
||||
.catch((error) => options.ui.meta(`[work error] ${String(error)}`))
|
||||
.finally(() => dispatch({ type: "tool.finished", id: request.id }))
|
||||
}
|
||||
|
||||
const userSpeaking = (active: boolean) => {
|
||||
if (active === state.userSpeaking) return
|
||||
dispatch(
|
||||
active
|
||||
? { type: "user.started", bargeIn: options.audio?.fullDuplex && options.protocol.capabilities.interruption }
|
||||
: { type: "user.stopped" },
|
||||
)
|
||||
options.ui.userSpeaking(active)
|
||||
}
|
||||
|
||||
const onAudioEvent = (event: AudioEvent) => {
|
||||
switch (event.type) {
|
||||
case "input":
|
||||
connection?.appendAudio(event.audio)
|
||||
options.ui.userAudioLevel(event.level)
|
||||
return
|
||||
case "user.started":
|
||||
userSpeaking(true)
|
||||
return
|
||||
case "user.stopped":
|
||||
userSpeaking(false)
|
||||
return
|
||||
case "user.reset":
|
||||
options.ui.userReset()
|
||||
return
|
||||
case "assistant.level":
|
||||
options.ui.assistantAudio(event.level, event.durationMs)
|
||||
return
|
||||
case "status":
|
||||
options.ui.setStatus({ audio: event.audio })
|
||||
return
|
||||
case "meta":
|
||||
options.ui.meta(event.text)
|
||||
}
|
||||
}
|
||||
|
||||
const onProtocolEvent = (source: VoiceConnection, event: VoiceProtocolEvent) => {
|
||||
if (source !== connection || closed) return
|
||||
if (options.connection.debug || event.type !== "assistant.audio")
|
||||
options.trace?.("protocol.event", {
|
||||
type: event.type,
|
||||
conversation: state.conversation,
|
||||
assistant: state.assistant,
|
||||
userSpeaking: state.userSpeaking,
|
||||
pendingWork: state.tools.size,
|
||||
})
|
||||
switch (event.type) {
|
||||
case "ready":
|
||||
dispatch({ type: "connection.ready" })
|
||||
options.ui.meta(
|
||||
`connected to ${options.protocol.name} ${options.connection.model} (voice: ${state.desiredVoice})`,
|
||||
)
|
||||
options.ui.setStatus({
|
||||
voice: state.desiredVoice,
|
||||
microphoneMuted: options.audio?.microphoneMuted,
|
||||
speakerMuted: options.audio?.speakerMuted,
|
||||
})
|
||||
if (!options.text) {
|
||||
void options.audio?.start(onAudioEvent)
|
||||
return
|
||||
}
|
||||
options.ui.userTranscript("typed", options.text)
|
||||
dispatch({ type: "user.committed" })
|
||||
source.sendText?.(options.text)
|
||||
return
|
||||
case "user.started":
|
||||
userSpeaking(true)
|
||||
return
|
||||
case "user.stopped":
|
||||
userSpeaking(false)
|
||||
return
|
||||
case "user.committed":
|
||||
options.audio?.noteUserCommitted()
|
||||
dispatch({ type: "user.committed" })
|
||||
options.ui.userCommitted(event.id)
|
||||
return
|
||||
case "user.transcript":
|
||||
options.audio?.noteUserTranscript(event.final)
|
||||
if (event.final) {
|
||||
dispatch({ type: "user.stopped" })
|
||||
options.ui.userSpeaking(false)
|
||||
}
|
||||
options.ui.userTranscript(event.id, event.text, event.final)
|
||||
return
|
||||
case "assistant.audio":
|
||||
if (state.assistant === "suppressed") return
|
||||
dispatch({ type: "assistant.started" })
|
||||
options.audio?.play(event.audio, event.timeline)
|
||||
return
|
||||
case "assistant.transcript.delta":
|
||||
if (state.assistant === "suppressed") return
|
||||
dispatch({ type: "assistant.started" })
|
||||
options.ui.assistantDelta(event.delta)
|
||||
return
|
||||
case "assistant.transcript":
|
||||
if (state.assistant === "suppressed") return
|
||||
dispatch({ type: "assistant.started" })
|
||||
options.ui.assistantTranscript(event.text)
|
||||
return
|
||||
case "assistant.done": {
|
||||
dispatch({ type: "assistant.done", awaitingWork: event.awaitingWork })
|
||||
return
|
||||
}
|
||||
case "tool.started":
|
||||
dispatch({ type: "tool.started", id: event.id })
|
||||
options.ui.toolStart(event.id, event.name, {})
|
||||
return
|
||||
case "work.requested":
|
||||
queueWork(source, event.request)
|
||||
return
|
||||
case "work.rejected":
|
||||
dispatch({ type: "tool.started", id: event.request.id })
|
||||
options.ui.toolStart(event.request.id, event.request.name, event.request.input)
|
||||
options.ui.toolDone(event.request.id, event.output)
|
||||
source.resolveWork(event.request, event.output)
|
||||
dispatch({ type: "tool.finished", id: event.request.id })
|
||||
return
|
||||
case "delegation.requested":
|
||||
dispatch({ type: "tool.started", id: event.request.id })
|
||||
void options
|
||||
.delegate(event.request, (request) => {
|
||||
options.ui.toolStart(request.id, request.name, request.input)
|
||||
return executeTool(request, event.request.id)
|
||||
})
|
||||
.then((output) => {
|
||||
if (source !== connection) {
|
||||
options.trace?.("delegation.result.stale", { id: event.request.id })
|
||||
return
|
||||
}
|
||||
source.resolveDelegation?.(event.request, output)
|
||||
})
|
||||
.catch((error) => {
|
||||
options.trace?.("responses.controller.failed", { error: String(error) })
|
||||
if (source !== connection) return
|
||||
options.ui.toolStart(event.request.id, "opencode", { text: event.request.text })
|
||||
options.ui.toolDone(event.request.id, { status: "error", message: String(error) })
|
||||
source.resolveDelegation?.(event.request, `The OpenCode controller failed: ${String(error)}`)
|
||||
})
|
||||
.finally(() => dispatch({ type: "tool.finished", id: event.request.id }))
|
||||
return
|
||||
case "debug":
|
||||
if (options.connection.debug) options.ui.meta(`[debug] ${event.message}`)
|
||||
return
|
||||
case "error":
|
||||
options.ui.meta(`[${options.protocol.name} error] ${event.message}`)
|
||||
return
|
||||
case "closed":
|
||||
dispatch({ type: "connection.closed" })
|
||||
options.ui.meta(`${options.protocol.name} connection closed (${event.code})`)
|
||||
options.onClosed()
|
||||
}
|
||||
}
|
||||
|
||||
const connect = () => {
|
||||
let next: VoiceConnection
|
||||
next = options.protocol.connect({
|
||||
...options.connection,
|
||||
voice: state.desiredVoice,
|
||||
onEvent: (event) => onProtocolEvent(next, event),
|
||||
})
|
||||
dispatch({ type: "connection.connecting" })
|
||||
connection = next
|
||||
}
|
||||
|
||||
const session: VoiceSession = {
|
||||
start: connect,
|
||||
cycleVoice() {
|
||||
setVoice(options.voices[(options.voices.indexOf(state.desiredVoice) + 1) % options.voices.length])
|
||||
},
|
||||
interrupt() {
|
||||
if (!options.audio?.isPlaying() && state.assistant === "idle") return
|
||||
if (state.assistant === "active") {
|
||||
connection?.interrupt?.()
|
||||
dispatch({ type: "assistant.suppressed" })
|
||||
}
|
||||
interruptDelivery()
|
||||
options.audio?.flushPlayback()
|
||||
options.ui.assistantDone()
|
||||
options.ui.meta("[interrupted]")
|
||||
},
|
||||
toggleMicrophone() {
|
||||
if (!options.audio) return
|
||||
const muted = options.audio.toggleMicrophone()
|
||||
userSpeaking(false)
|
||||
options.ui.setStatus({ microphoneMuted: muted })
|
||||
if (muted) options.ui.userReset()
|
||||
options.ui.userAudioLevel(undefined)
|
||||
options.ui.meta(`[microphone] ${muted ? "muted" : "live"}`)
|
||||
},
|
||||
toggleSpeaker() {
|
||||
if (!options.audio) return
|
||||
const muted = options.audio.toggleSpeaker()
|
||||
options.ui.setStatus({ speakerMuted: muted })
|
||||
if (muted) options.ui.assistantPlaybackStopped()
|
||||
options.ui.meta(`[speaker] ${muted ? "muted" : "live"}`)
|
||||
},
|
||||
queueNotification(announcement) {
|
||||
notifications.route(announcement)
|
||||
},
|
||||
close() {
|
||||
if (closePromise) return closePromise
|
||||
closed = true
|
||||
if (voiceTimer) clearTimeout(voiceTimer)
|
||||
options.audio?.close()
|
||||
const active = connection
|
||||
connection = undefined
|
||||
dispatch({ type: "connection.closed" })
|
||||
closePromise = Promise.allSettled([options.tools.close(), ...(active ? [active.close()] : [])]).then(() => {})
|
||||
return closePromise
|
||||
},
|
||||
}
|
||||
return session
|
||||
}
|
||||
|
||||
function toolError(message: string, retryable = false) {
|
||||
return { status: "error", message, retryable }
|
||||
}
|
||||
@@ -1,53 +0,0 @@
|
||||
import { describe, expect, test } from "bun:test"
|
||||
import {
|
||||
connectionMeterLevels,
|
||||
connectionTransitioning,
|
||||
scheduleTextReveal,
|
||||
textRevealOpacity,
|
||||
transcriptionPulse,
|
||||
} from "../src/animation"
|
||||
|
||||
describe("voice text animation", () => {
|
||||
test("stages newly streamed words without delaying them indefinitely", () => {
|
||||
const first = scheduleTextReveal("", "Hello there", 1_000, 0)
|
||||
expect(first.reveals).toEqual([
|
||||
{ offset: 0, at: 1_000 },
|
||||
{ offset: 6, at: 1_032 },
|
||||
])
|
||||
expect(first.animationEndsAt).toBe(1_412)
|
||||
|
||||
const queued = scheduleTextReveal("Hello there", " friend", 1_010, 10_000)
|
||||
expect(queued.reveals).toEqual([{ offset: 12, at: 1_170 }])
|
||||
})
|
||||
|
||||
test("slowly fades text into its stable color", () => {
|
||||
const opacities = [100, 195, 290, 385, 480].map((now) => textRevealOpacity(now, 100))
|
||||
expect(opacities[0]).toBe(0.16)
|
||||
expect(opacities.toSorted((a, b) => a - b)).toEqual(opacities)
|
||||
expect(opacities.at(-1)).toBe(1)
|
||||
})
|
||||
|
||||
test("keeps transcription pulse bounded and moving", () => {
|
||||
const levels = [0, 110, 220, 330].map(transcriptionPulse)
|
||||
expect(levels.every((level) => level >= 0 && level <= 1)).toBe(true)
|
||||
expect(new Set(levels).size).toBeGreaterThan(1)
|
||||
})
|
||||
|
||||
test("moves the connecting pulse left to right and blends into the connected meter", () => {
|
||||
const first = connectionMeterLevels(1_000, 0.1, undefined, 1_000)
|
||||
const next = connectionMeterLevels(1_180, 0.1, undefined, 1_000)
|
||||
expect(first.indexOf(Math.max(...first))).toBe(0)
|
||||
expect(first.indexOf(Math.max(...first))).toBeLessThan(next.indexOf(Math.max(...next)))
|
||||
|
||||
const samples = Array.from({ length: 16 }, (_, index) => connectionMeterLevels(index * 45, 0.1))
|
||||
expect(samples.every((levels) => Math.max(...levels) > 0.85)).toBe(true)
|
||||
|
||||
const handoff = connectionMeterLevels(1_000, 0.4, 1_000)
|
||||
expect(handoff).toEqual(connectionMeterLevels(1_000, 0.4))
|
||||
expect(connectionMeterLevels(1_480, 0.4, 1_000)).toEqual(
|
||||
[0, 1, 2, 3].map((index) => 0.4 * (0.72 + Math.sin(1_480 / 240 + index * 1.4) * 0.28)),
|
||||
)
|
||||
expect(connectionTransitioning(1_479, 1_000)).toBe(true)
|
||||
expect(connectionTransitioning(1_480, 1_000)).toBe(false)
|
||||
})
|
||||
})
|
||||
@@ -1,79 +0,0 @@
|
||||
import { describe, expect, test } from "bun:test"
|
||||
import { joinAssistantText } from "../src/ui-model"
|
||||
|
||||
describe("joinAssistantText", () => {
|
||||
const cases: Array<[name: string, left: string, right: string, expected: string]> = [
|
||||
[
|
||||
"inserts a space between sentences",
|
||||
"Okay, I'll check that.",
|
||||
"The build passed.",
|
||||
"Okay, I'll check that. The build passed.",
|
||||
],
|
||||
["inserts a space mid-sentence", "Let me look at", "the config file.", "Let me look at the config file."],
|
||||
[
|
||||
"keeps a single trailing space",
|
||||
"Okay, I'll check that. ",
|
||||
"The build passed.",
|
||||
"Okay, I'll check that. The build passed.",
|
||||
],
|
||||
[
|
||||
"keeps a single leading space",
|
||||
"Okay, I'll check that.",
|
||||
" The build passed.",
|
||||
"Okay, I'll check that. The build passed.",
|
||||
],
|
||||
["collapses spaces on both sides", "Done. ", " Next.", "Done. Next."],
|
||||
["collapses runs of spaces", "Done. ", "\t \tNext.", "Done. Next."],
|
||||
[
|
||||
"spaces after a question mark",
|
||||
"Want me to run tests?",
|
||||
"I can do that now.",
|
||||
"Want me to run tests? I can do that now.",
|
||||
],
|
||||
["spaces after an ellipsis", "Thinking…", "done.", "Thinking… done."],
|
||||
["preserves a trailing newline", "Done:\n", "Next up, tests.", "Done:\nNext up, tests."],
|
||||
["preserves a leading newline", "Done:", "\nNext up, tests.", "Done:\nNext up, tests."],
|
||||
[
|
||||
"preserves a paragraph break",
|
||||
"First paragraph.\n\n",
|
||||
"Second paragraph.",
|
||||
"First paragraph.\n\nSecond paragraph.",
|
||||
],
|
||||
["preserves a newline mixed with spaces", "Done: ", " \n Next.", "Done: \n Next."],
|
||||
["never spaces before closing punctuation", "Done", ".", "Done."],
|
||||
["never spaces before a comma", "one", ", two", "one, two"],
|
||||
["never spaces before a closing paren", "(aside", ")", "(aside)"],
|
||||
["never spaces after an opening paren", "an (", "aside)", "an (aside)"],
|
||||
["drops an empty left side", "", "The build passed.", "The build passed."],
|
||||
["drops an empty right side", "The build passed.", "", "The build passed."],
|
||||
["drops a whitespace-only left side", " ", "The build passed.", "The build passed."],
|
||||
["drops a whitespace-only right side", "The build passed.", " \n ", "The build passed."],
|
||||
["handles two empty sides", "", "", ""],
|
||||
["does not trim the outer edges", " Leading kept.", "Trailing kept. ", " Leading kept. Trailing kept. "],
|
||||
]
|
||||
|
||||
for (const [name, left, right, expected] of cases) {
|
||||
test(name, () => expect(joinAssistantText(left, right).text).toBe(expected))
|
||||
}
|
||||
|
||||
test("splits the result at the boundary so reveals cover only the appended text", () => {
|
||||
for (const [name, left, right] of cases) {
|
||||
const joined = joinAssistantText(left, right)
|
||||
expect(`${name}: ${joined.previous}${joined.appended}`).toBe(`${name}: ${joined.text}`)
|
||||
expect(`${name}: ${joined.text.endsWith(joined.appended)}`).toBe(`${name}: true`)
|
||||
}
|
||||
})
|
||||
|
||||
test("is idempotent once a boundary is normalised", () => {
|
||||
const once = joinAssistantText("Okay.", "Next.").text
|
||||
expect(joinAssistantText(once, "").text).toBe(once)
|
||||
expect(joinAssistantText("Okay. ", "Next.").text).toBe(once)
|
||||
})
|
||||
|
||||
test("folds consecutive messages with exactly one space per boundary", () => {
|
||||
expect(["First.", "Second.", "Third."].reduce((left, right) => joinAssistantText(left, right).text)).toBe(
|
||||
"First. Second. Third.",
|
||||
)
|
||||
expect(["First.", "", "Third."].reduce((left, right) => joinAssistantText(left, right).text)).toBe("First. Third.")
|
||||
})
|
||||
})
|
||||
@@ -1,33 +0,0 @@
|
||||
import { describe, expect, test } from "bun:test"
|
||||
import { AudioJitterBuffer } from "../src/audio-jitter-buffer"
|
||||
import { PCM_BYTES_PER_MS } from "../src/pcm"
|
||||
|
||||
const chunk = (durationMs: number, gapMs = 0) => ({ bytes: Buffer.alloc(durationMs * PCM_BYTES_PER_MS), gapMs })
|
||||
|
||||
describe("AudioJitterBuffer", () => {
|
||||
test("holds startup audio until the target buffer is available", () => {
|
||||
const buffer = new AudioJitterBuffer(300)
|
||||
expect(buffer.push(chunk(100))).toEqual([])
|
||||
expect(buffer.push(chunk(100))).toEqual([])
|
||||
expect(buffer.push(chunk(100))).toHaveLength(3)
|
||||
})
|
||||
|
||||
test("passes chunks through after playback starts", () => {
|
||||
const buffer = new AudioJitterBuffer(200)
|
||||
buffer.push(chunk(200))
|
||||
expect(buffer.push(chunk(100))).toEqual([chunk(100)])
|
||||
})
|
||||
|
||||
test("flushes short utterances and resets between responses", () => {
|
||||
const buffer = new AudioJitterBuffer(300)
|
||||
buffer.push(chunk(100))
|
||||
expect(buffer.finish()).toHaveLength(1)
|
||||
buffer.reset()
|
||||
expect(buffer.push(chunk(100))).toEqual([])
|
||||
})
|
||||
|
||||
test("counts intentional timeline silence toward buffered duration", () => {
|
||||
const buffer = new AudioJitterBuffer(300)
|
||||
expect(buffer.push(chunk(200, 100))).toHaveLength(1)
|
||||
})
|
||||
})
|
||||
@@ -1,106 +0,0 @@
|
||||
import { expect, test } from "bun:test"
|
||||
import type { AudioDevice } from "../src/audio-device"
|
||||
import { createAudioSession, type AudioEvent } from "../src/audio-session"
|
||||
|
||||
test("reports muted notification playback as inaudible", async () => {
|
||||
const device = scriptedDevice()
|
||||
const audio = await createAudioSession({
|
||||
duplex: false,
|
||||
speakers: false,
|
||||
inputActivity: "server",
|
||||
debug: false,
|
||||
device,
|
||||
})
|
||||
|
||||
expect(audio.toggleSpeaker()).toBe(true)
|
||||
audio.play(Buffer.alloc(4_800))
|
||||
expect(await audio.finishPlayback()).toBe("inaudible")
|
||||
expect(device.writes).toEqual([])
|
||||
})
|
||||
|
||||
test("settles active playback when interrupted", async () => {
|
||||
const device = scriptedDevice()
|
||||
const audio = await createAudioSession({
|
||||
duplex: false,
|
||||
speakers: false,
|
||||
inputActivity: "server",
|
||||
debug: false,
|
||||
device,
|
||||
})
|
||||
|
||||
audio.play(Buffer.alloc(4_800))
|
||||
const outcome = audio.finishPlayback()
|
||||
audio.flushPlayback()
|
||||
expect(await outcome).toBe("interrupted")
|
||||
expect(device.flushes).toBe(1)
|
||||
})
|
||||
|
||||
test("emits captured audio and local speech observations", async () => {
|
||||
const device = scriptedDevice()
|
||||
const events: AudioEvent[] = []
|
||||
const audio = await createAudioSession({
|
||||
duplex: true,
|
||||
speakers: false,
|
||||
inputActivity: "local",
|
||||
debug: false,
|
||||
device,
|
||||
})
|
||||
await audio.start((event) => events.push(event))
|
||||
const input = Buffer.alloc(4_800)
|
||||
for (let offset = 0; offset < input.length; offset += 2) input.writeInt16LE(12_000, offset)
|
||||
device.capture(input)
|
||||
|
||||
expect(events).toContainEqual({ type: "input", audio: input, level: expect.any(Number) })
|
||||
expect(events).toContainEqual({ type: "user.started" })
|
||||
audio.close()
|
||||
})
|
||||
|
||||
test("preserves quiet duplex microphone audio during assistant playback", async () => {
|
||||
const device = scriptedDevice()
|
||||
const events: AudioEvent[] = []
|
||||
const audio = await createAudioSession({
|
||||
duplex: false,
|
||||
speakers: true,
|
||||
inputActivity: "server",
|
||||
debug: false,
|
||||
device,
|
||||
})
|
||||
await audio.start((event) => events.push(event))
|
||||
audio.play(Buffer.alloc(24_000))
|
||||
const input = Buffer.alloc(4_800)
|
||||
for (let offset = 0; offset < input.length; offset += 2) input.writeInt16LE(800, offset)
|
||||
device.capture(input)
|
||||
|
||||
expect(audio.fullDuplex).toBe(true)
|
||||
expect(events).toContainEqual({ type: "input", audio: input, level: expect.any(Number) })
|
||||
audio.close()
|
||||
})
|
||||
|
||||
function scriptedDevice() {
|
||||
let onInput: ((audio: Buffer) => void) | undefined
|
||||
const writes: Buffer[] = []
|
||||
const device = {
|
||||
fullDuplex: true,
|
||||
mode: "scripted",
|
||||
async start(listener) {
|
||||
onInput = listener
|
||||
},
|
||||
write(audio) {
|
||||
writes.push(audio)
|
||||
},
|
||||
flush() {
|
||||
device.flushes += 1
|
||||
},
|
||||
close() {},
|
||||
writes,
|
||||
flushes: 0,
|
||||
capture(audio: Buffer) {
|
||||
onInput?.(audio)
|
||||
},
|
||||
} satisfies AudioDevice & {
|
||||
readonly writes: Buffer[]
|
||||
flushes: number
|
||||
capture(audio: Buffer): void
|
||||
}
|
||||
return device
|
||||
}
|
||||
@@ -1,101 +0,0 @@
|
||||
import { expect, test } from "bun:test"
|
||||
import { mkdir, mkdtemp, rm } from "node:fs/promises"
|
||||
import { tmpdir } from "node:os"
|
||||
import { join } from "node:path"
|
||||
import { createCompletionStore } from "../src/completion-store"
|
||||
|
||||
test("completion store restores pending and completed notifications until delivery", async () => {
|
||||
const directory = await mkdtemp(join(tmpdir(), "voice-completions-"))
|
||||
const path = join(directory, "prompts.json")
|
||||
const first = await createCompletionStore(path)
|
||||
const pending = { sessionID: "session-1", promptID: "prompt-1" }
|
||||
const completed = { sessionID: "session-2", promptID: "prompt-2" }
|
||||
|
||||
try {
|
||||
await first.admitting(pending, "Please continue")
|
||||
expect(first.entries()).toEqual([{ status: "admitting", handle: pending, text: "Please continue" }])
|
||||
await first.pending(pending)
|
||||
await first.pending(completed)
|
||||
await first.completed(completed, {
|
||||
type: "opencode.prompt.completed",
|
||||
session_id: "session-2",
|
||||
prompt_id: "prompt-2",
|
||||
status: "completed",
|
||||
text: "done",
|
||||
})
|
||||
await first.close()
|
||||
|
||||
const restored = await createCompletionStore(path)
|
||||
expect(restored.entries()).toEqual(expect.arrayContaining([
|
||||
{ status: "pending", handle: pending },
|
||||
{
|
||||
status: "completed",
|
||||
handle: completed,
|
||||
notification: {
|
||||
type: "opencode.prompt.completed",
|
||||
session_id: "session-2",
|
||||
prompt_id: "prompt-2",
|
||||
status: "completed",
|
||||
text: "done",
|
||||
},
|
||||
},
|
||||
]))
|
||||
await restored.close()
|
||||
|
||||
const stillPendingDelivery = await createCompletionStore(path)
|
||||
expect(stillPendingDelivery.entries().some((entry) => entry.status === "completed")).toBe(true)
|
||||
|
||||
await stillPendingDelivery.remove(completed)
|
||||
await stillPendingDelivery.close()
|
||||
expect((await createCompletionStore(path)).entries()).toEqual([{ status: "pending", handle: pending }])
|
||||
} finally {
|
||||
await rm(directory, { recursive: true, force: true })
|
||||
}
|
||||
})
|
||||
|
||||
test("completion stores do not overwrite prompts written by another process", async () => {
|
||||
const directory = await mkdtemp(join(tmpdir(), "voice-completions-concurrent-"))
|
||||
const path = join(directory, "prompts.json")
|
||||
const first = await createCompletionStore(path)
|
||||
const second = await createCompletionStore(path)
|
||||
|
||||
try {
|
||||
await Promise.all([
|
||||
first.pending({ sessionID: "session-1", promptID: "prompt-shared" }),
|
||||
second.pending({ sessionID: "session-2", promptID: "prompt-shared" }),
|
||||
])
|
||||
await Promise.all([first.close(), second.close()])
|
||||
expect((await createCompletionStore(path)).entries()).toEqual(
|
||||
expect.arrayContaining([
|
||||
{ status: "pending", handle: { sessionID: "session-1", promptID: "prompt-shared" } },
|
||||
{ status: "pending", handle: { sessionID: "session-2", promptID: "prompt-shared" } },
|
||||
]),
|
||||
)
|
||||
} finally {
|
||||
await rm(directory, { recursive: true, force: true })
|
||||
}
|
||||
})
|
||||
|
||||
test("continues serializing writes after one filesystem failure", async () => {
|
||||
const directory = await mkdtemp(join(tmpdir(), "voice-completions-recovery-"))
|
||||
const path = join(directory, "prompts.json")
|
||||
const store = await createCompletionStore(path)
|
||||
try {
|
||||
await rm(`${path}.d`, { recursive: true })
|
||||
await Bun.write(`${path}.d`, "not a directory")
|
||||
const failure = await store
|
||||
.pending({ sessionID: "session-1", promptID: "prompt-1" })
|
||||
.then(() => undefined, (error) => error)
|
||||
expect(failure).toBeDefined()
|
||||
await rm(`${path}.d`)
|
||||
await mkdir(`${path}.d`, { recursive: true })
|
||||
await store.pending({ sessionID: "session-2", promptID: "prompt-2" })
|
||||
await store.close()
|
||||
expect((await createCompletionStore(path)).entries()).toContainEqual({
|
||||
status: "pending",
|
||||
handle: { sessionID: "session-2", promptID: "prompt-2" },
|
||||
})
|
||||
} finally {
|
||||
await rm(directory, { recursive: true, force: true })
|
||||
}
|
||||
})
|
||||
@@ -1,311 +0,0 @@
|
||||
import { describe, expect, test } from "bun:test"
|
||||
import { createLiveEventDelivery, createLiveEventProjector } from "../src/protocol-live"
|
||||
import type { VoiceProtocolEvent } from "../src/protocol"
|
||||
|
||||
const frames = [
|
||||
{ type: "session.started", session: {} },
|
||||
{ type: "session.context.appended", start_ms: 0, end_ms: 0 },
|
||||
{ type: "output_audio.delta", audio: "AAA=", start_ms: 0, end_ms: 1 },
|
||||
{
|
||||
type: "input_transcript.added",
|
||||
start_ms: 0,
|
||||
end_ms: 100,
|
||||
item: { id: "input-1", type: "input_transcript", text: " Hello" },
|
||||
},
|
||||
{ type: "turn.created", turn: { id: "user-turn", role: "user", transcript: " Hello" } },
|
||||
{
|
||||
type: "input_transcript.added",
|
||||
start_ms: 100,
|
||||
end_ms: 200,
|
||||
item: { id: "input-2", type: "input_transcript", text: ", this" },
|
||||
},
|
||||
{ type: "turn.delta", turn_id: "user-turn", delta: ", this" },
|
||||
{
|
||||
type: "output_transcript.added",
|
||||
start_ms: 200,
|
||||
end_ms: 300,
|
||||
item: { id: "output-1", type: "output_transcript", text: " Copy that" },
|
||||
},
|
||||
{
|
||||
type: "input_transcript.added",
|
||||
start_ms: 200,
|
||||
end_ms: 300,
|
||||
item: { id: "input-3", type: "input_transcript", text: " is a test" },
|
||||
},
|
||||
{
|
||||
type: "output_transcript.added",
|
||||
start_ms: 300,
|
||||
end_ms: 400,
|
||||
item: { id: "output-2", type: "output_transcript", text: ", test" },
|
||||
},
|
||||
{ type: "turn.delta", turn_id: "user-turn", delta: " is a test" },
|
||||
{ type: "turn.done", turn: { id: "user-turn", role: "user", transcript: " Hello, this is a test" } },
|
||||
{ type: "turn.created", turn: { id: "assistant-turn", role: "assistant", transcript: " Copy that, test" } },
|
||||
{
|
||||
type: "output_transcript.added",
|
||||
start_ms: 400,
|
||||
end_ms: 500,
|
||||
item: { id: "output-3", type: "output_transcript", text: " received" },
|
||||
},
|
||||
{ type: "turn.delta", turn_id: "assistant-turn", delta: " received" },
|
||||
{
|
||||
type: "output_transcript.added",
|
||||
start_ms: 500,
|
||||
end_ms: 600,
|
||||
item: { id: "output-4", type: "output_transcript", text: " loud and clear." },
|
||||
},
|
||||
{ type: "turn.delta", turn_id: "assistant-turn", delta: " loud and clear." },
|
||||
{
|
||||
type: "turn.done",
|
||||
turn: { id: "assistant-turn", role: "assistant", transcript: " Copy that, test received loud and clear." },
|
||||
},
|
||||
] as const
|
||||
|
||||
describe("Live event projector", () => {
|
||||
test("replays progressive transcripts without schema errors or duplicate text", () => {
|
||||
const project = createLiveEventProjector()
|
||||
const events = frames.flatMap((frame) => project(JSON.stringify(frame)).events)
|
||||
|
||||
expect(events.filter((event) => event.type === "error")).toEqual([])
|
||||
expect(events.filter((event) => event.type === "user.committed")).toEqual([
|
||||
{ type: "user.committed", id: "input-1" },
|
||||
])
|
||||
expect(events.filter(isUserTranscript)).toEqual([
|
||||
{ type: "user.transcript", id: "input-1", text: "Hello", final: false },
|
||||
{ type: "user.transcript", id: "input-1", text: "Hello, this", final: false },
|
||||
{ type: "user.transcript", id: "input-1", text: "Hello, this is a test", final: false },
|
||||
{ type: "user.transcript", id: "input-1", text: "Hello, this is a test", final: true },
|
||||
])
|
||||
expect(
|
||||
events
|
||||
.filter(isAssistantDelta)
|
||||
.map((event) => event.delta)
|
||||
.join(""),
|
||||
).toBe(" Copy that, test received loud and clear.")
|
||||
expect(events.filter((event) => event.type === "assistant.done")).toEqual([
|
||||
{ type: "assistant.done", awaitingWork: false },
|
||||
])
|
||||
expect(events.filter((event) => event.type === "assistant.audio")).toHaveLength(1)
|
||||
})
|
||||
|
||||
test("reports malformed frames instead of throwing", () => {
|
||||
expect(createLiveEventProjector()(JSON.stringify({ type: "turn.created", turn: { id: 1 } })).events).toEqual([
|
||||
{ type: "error", message: "Received an invalid Live API event." },
|
||||
])
|
||||
})
|
||||
|
||||
test("preserves context acknowledgement correlation IDs", () => {
|
||||
expect(
|
||||
createLiveEventProjector()(
|
||||
JSON.stringify({ type: "session.context.appended", event_id: "event-notification" }),
|
||||
),
|
||||
).toMatchObject({
|
||||
type: "session.context.appended",
|
||||
eventID: "event-notification",
|
||||
})
|
||||
})
|
||||
|
||||
test("replaces corrected assistant transcript snapshots", () => {
|
||||
const project = createLiveEventProjector()
|
||||
const events = [
|
||||
project(
|
||||
JSON.stringify({
|
||||
type: "output_transcript.added",
|
||||
item: { id: "output-1", type: "output_transcript", text: " Hello world" },
|
||||
}),
|
||||
),
|
||||
project(
|
||||
JSON.stringify({
|
||||
type: "turn.created",
|
||||
turn: { id: "assistant-turn", role: "assistant", transcript: " Hello there" },
|
||||
}),
|
||||
),
|
||||
].flatMap((result) => result.events)
|
||||
|
||||
expect(events.filter((event) => event.type.startsWith("assistant.transcript"))).toEqual([
|
||||
{ type: "assistant.transcript.delta", delta: " Hello world" },
|
||||
{ type: "assistant.transcript", text: " Hello there" },
|
||||
])
|
||||
})
|
||||
|
||||
test("keeps the space between consecutive assistant turns", () => {
|
||||
const project = createLiveEventProjector()
|
||||
const turn = (id: string, text: string) => [
|
||||
{ type: "turn.created", turn: { id, role: "assistant", transcript: "" } },
|
||||
{ type: "output_transcript.added", item: { id: `${id}-o`, type: "output_transcript", text } },
|
||||
{ type: "turn.delta", turn_id: id, delta: text },
|
||||
{ type: "turn.done", turn: { id, role: "assistant", transcript: text } },
|
||||
]
|
||||
const events = [...turn("t1", ' Session "Fix auth bug" is ready.'), ...turn("t2", " Next up, tests.")].flatMap(
|
||||
(frame) => project(JSON.stringify(frame)).events,
|
||||
)
|
||||
|
||||
// Concatenating the delta stream is what the UI and any transcript consumer does; the
|
||||
// turn boundary must survive it rather than welding "ready.Next" together.
|
||||
const transcript = events
|
||||
.filter(isAssistantDelta)
|
||||
.map((event) => event.delta)
|
||||
.join("")
|
||||
expect(transcript).toBe(' Session "Fix auth bug" is ready. Next up, tests.')
|
||||
expect(transcript).not.toContain("ready.Next")
|
||||
expect(events.filter((event) => event.type === "assistant.done")).toHaveLength(2)
|
||||
})
|
||||
|
||||
test("emits no duplicate text when fragments and turn deltas overlap across turns", () => {
|
||||
const project = createLiveEventProjector()
|
||||
const frames = [
|
||||
{ type: "turn.created", turn: { id: "t1", role: "assistant", transcript: "" } },
|
||||
{ type: "output_transcript.added", item: { id: "a", type: "output_transcript", text: " First" } },
|
||||
{ type: "turn.delta", turn_id: "t1", delta: " First" },
|
||||
{ type: "output_transcript.added", item: { id: "b", type: "output_transcript", text: " turn." } },
|
||||
{ type: "turn.delta", turn_id: "t1", delta: " turn." },
|
||||
{ type: "turn.done", turn: { id: "t1", role: "assistant", transcript: " First turn." } },
|
||||
{ type: "turn.created", turn: { id: "t2", role: "assistant", transcript: "" } },
|
||||
{ type: "output_transcript.added", item: { id: "c", type: "output_transcript", text: " Second turn." } },
|
||||
{ type: "turn.delta", turn_id: "t2", delta: " Second turn." },
|
||||
{ type: "turn.done", turn: { id: "t2", role: "assistant", transcript: " Second turn." } },
|
||||
]
|
||||
const events = frames.flatMap((frame) => project(JSON.stringify(frame)).events)
|
||||
|
||||
expect(events.filter((event) => event.type === "assistant.transcript")).toEqual([])
|
||||
expect(
|
||||
events
|
||||
.filter(isAssistantDelta)
|
||||
.map((event) => event.delta)
|
||||
.join(""),
|
||||
).toBe(" First turn. Second turn.")
|
||||
})
|
||||
|
||||
test("projects Responses delegation function calls without rejecting reasoning items", () => {
|
||||
const project = createLiveEventProjector()
|
||||
expect(
|
||||
project(
|
||||
JSON.stringify({
|
||||
type: "response.output_item.done",
|
||||
item: { id: "reasoning-1", type: "reasoning" },
|
||||
}),
|
||||
).events,
|
||||
).toEqual([{ type: "debug", message: "response.output_item.done" }])
|
||||
expect(
|
||||
project(
|
||||
JSON.stringify({
|
||||
type: "response.output_item.added",
|
||||
item: {
|
||||
id: "function-1",
|
||||
type: "function_call",
|
||||
name: "find_sessions",
|
||||
call_id: "call-1",
|
||||
arguments: "",
|
||||
},
|
||||
}),
|
||||
).events,
|
||||
).toEqual([
|
||||
{ type: "debug", message: "response.output_item.added" },
|
||||
{ type: "tool.started", id: "call-1", name: "find_sessions" },
|
||||
])
|
||||
expect(
|
||||
project(
|
||||
JSON.stringify({
|
||||
type: "response.output_item.done",
|
||||
item: {
|
||||
id: "function-1",
|
||||
type: "function_call",
|
||||
name: "find_sessions",
|
||||
call_id: "call-1",
|
||||
arguments: '{"scope":"current_project"}',
|
||||
},
|
||||
}),
|
||||
).events,
|
||||
).toEqual([
|
||||
{ type: "debug", message: "response.output_item.done" },
|
||||
{
|
||||
type: "work.requested",
|
||||
request: { id: "call-1", name: "find_sessions", input: { scope: "current_project" } },
|
||||
},
|
||||
])
|
||||
expect(
|
||||
project(
|
||||
JSON.stringify({
|
||||
type: "response.output_item.done",
|
||||
item: {
|
||||
id: "message-1",
|
||||
type: "message",
|
||||
status: "completed",
|
||||
content: [{ type: "output_text", text: "Two sessions found." }],
|
||||
role: "assistant",
|
||||
},
|
||||
}),
|
||||
).events,
|
||||
).toEqual([{ type: "debug", message: "response.output_item.done" }])
|
||||
})
|
||||
|
||||
test("projects client delegation requests for the local controller", () => {
|
||||
expect(
|
||||
createLiveEventProjector()(
|
||||
JSON.stringify({
|
||||
type: "delegation.created",
|
||||
item: {
|
||||
id: "delegation-1",
|
||||
type: "delegation",
|
||||
target: "client",
|
||||
content: [{ type: "input_text", text: "Inspect the voice package" }],
|
||||
},
|
||||
}),
|
||||
).events,
|
||||
).toEqual([
|
||||
{ type: "debug", message: "delegation.created" },
|
||||
{ type: "delegation.requested", request: { id: "delegation-1", text: "Inspect the voice package" } },
|
||||
])
|
||||
})
|
||||
|
||||
test("rejects malformed function arguments without hanging the delegation", () => {
|
||||
expect(
|
||||
createLiveEventProjector()(
|
||||
JSON.stringify({
|
||||
type: "response.output_item.done",
|
||||
item: {
|
||||
id: "function-1",
|
||||
type: "function_call",
|
||||
name: "find_sessions",
|
||||
call_id: "call-invalid",
|
||||
arguments: "not json",
|
||||
},
|
||||
}),
|
||||
).events,
|
||||
).toEqual([
|
||||
{ type: "debug", message: "response.output_item.done" },
|
||||
{ type: "tool.started", id: "call-invalid", name: "find_sessions" },
|
||||
{ type: "error", message: "Received invalid arguments for Live tool find_sessions." },
|
||||
{
|
||||
type: "work.rejected",
|
||||
request: { id: "call-invalid", name: "find_sessions", input: {} },
|
||||
output: { status: "error", message: "Invalid arguments for tool find_sessions." },
|
||||
},
|
||||
])
|
||||
})
|
||||
|
||||
test("waits for a quiet drain after turn completion and late audio", async () => {
|
||||
const events: VoiceProtocolEvent[] = []
|
||||
const delivery = createLiveEventDelivery({ emit: (event) => events.push(event), drainMs: 20 })
|
||||
delivery.push({ type: "assistant.done", awaitingWork: false })
|
||||
await Bun.sleep(10)
|
||||
delivery.push({ type: "assistant.audio", audio: Buffer.alloc(48) })
|
||||
await Bun.sleep(12)
|
||||
expect(events.map((event) => event.type)).toEqual(["assistant.audio"])
|
||||
await Bun.sleep(12)
|
||||
expect(events.map((event) => event.type)).toEqual(["assistant.audio", "assistant.done"])
|
||||
delivery.close()
|
||||
})
|
||||
})
|
||||
|
||||
function isUserTranscript(
|
||||
event: VoiceProtocolEvent,
|
||||
): event is Extract<VoiceProtocolEvent, { readonly type: "user.transcript" }> {
|
||||
return event.type === "user.transcript"
|
||||
}
|
||||
|
||||
function isAssistantDelta(
|
||||
event: VoiceProtocolEvent,
|
||||
): event is Extract<VoiceProtocolEvent, { readonly type: "assistant.transcript.delta" }> {
|
||||
return event.type === "assistant.transcript.delta"
|
||||
}
|
||||
@@ -1,59 +0,0 @@
|
||||
import { expect, test } from "bun:test"
|
||||
import { createNotificationRouter, type RoutedAnnouncement } from "../src/notification-router"
|
||||
|
||||
test("routes blockers and completions through the admitted delegation", () => {
|
||||
const routed: RoutedAnnouncement[] = []
|
||||
const router = createNotificationRouter((announcement) => routed.push(announcement))
|
||||
router.beginDelegatedTool()
|
||||
router.route({
|
||||
notification: {
|
||||
type: "opencode.prompt.blocked",
|
||||
prompt_id: "prompt-1",
|
||||
blocker: "permission",
|
||||
session_id: "session-1",
|
||||
request_id: "permission-1",
|
||||
action: "read",
|
||||
resources: [],
|
||||
},
|
||||
})
|
||||
expect(routed).toEqual([])
|
||||
|
||||
router.finishDelegatedTool("delegation-1", { sessionID: "session-1", promptID: "prompt-1" })
|
||||
router.route({
|
||||
notification: {
|
||||
type: "opencode.prompt.completed",
|
||||
session_id: "session-1",
|
||||
prompt_id: "prompt-1",
|
||||
status: "completed",
|
||||
text: "done",
|
||||
},
|
||||
receipt: { sessionID: "session-1", promptID: "prompt-1" },
|
||||
})
|
||||
|
||||
expect(routed.map((item) => item.delegationID)).toEqual(["delegation-1", "delegation-1"])
|
||||
expect(routed[0].announcement.receipt).toBeUndefined()
|
||||
expect(routed[1].announcement.receipt).toEqual({ sessionID: "session-1", promptID: "prompt-1" })
|
||||
})
|
||||
|
||||
test("keeps identical prompt IDs isolated by session", () => {
|
||||
const routed: RoutedAnnouncement[] = []
|
||||
const router = createNotificationRouter((announcement) => routed.push(announcement))
|
||||
router.beginDelegatedTool()
|
||||
router.finishDelegatedTool("delegation-1", { sessionID: "session-1", promptID: "prompt-shared" })
|
||||
router.beginDelegatedTool()
|
||||
router.finishDelegatedTool("delegation-2", { sessionID: "session-2", promptID: "prompt-shared" })
|
||||
|
||||
for (const sessionID of ["session-1", "session-2"])
|
||||
router.route({
|
||||
notification: {
|
||||
type: "opencode.prompt.completed",
|
||||
session_id: sessionID,
|
||||
prompt_id: "prompt-shared",
|
||||
status: "completed",
|
||||
text: "done",
|
||||
},
|
||||
receipt: { sessionID, promptID: "prompt-shared" },
|
||||
})
|
||||
|
||||
expect(routed.map((item) => item.delegationID)).toEqual(["delegation-1", "delegation-2"])
|
||||
})
|
||||
@@ -1,248 +0,0 @@
|
||||
import { expect, test } from "bun:test"
|
||||
import { OpenCode } from "@opencode-ai/client/promise"
|
||||
import { createOpenCodeBridge } from "../src/opencode"
|
||||
import type { CompletionStore } from "../src/completion-store"
|
||||
|
||||
test("renames only a session discovered by the voice controller", async () => {
|
||||
const requests: Request[] = []
|
||||
const session = {
|
||||
id: "ses_known",
|
||||
projectID: "project-1",
|
||||
cost: "0",
|
||||
tokens: { input: 0, output: 0, reasoning: 0, cache: { read: 0, write: 0 } },
|
||||
time: { created: 1, updated: 2 },
|
||||
title: "Old title",
|
||||
location: { directory: "/workspace" },
|
||||
}
|
||||
const fetch = Object.assign(
|
||||
async (input: string | URL | Request, init?: RequestInit) => {
|
||||
const request = input instanceof Request ? input : new Request(input.toString(), init)
|
||||
requests.push(request)
|
||||
const url = new URL(request.url)
|
||||
if (url.pathname === "/api/event") return new Response("", { headers: { "content-type": "text/event-stream" } })
|
||||
if (url.pathname === "/api/project")
|
||||
return Response.json([
|
||||
{
|
||||
id: "project-1",
|
||||
worktree: "/workspace",
|
||||
time: { created: 1, updated: 2 },
|
||||
sandboxes: [],
|
||||
},
|
||||
])
|
||||
if (url.pathname === "/api/session" && request.method === "GET")
|
||||
return Response.json({ data: [session], cursor: {} })
|
||||
if (url.pathname === "/api/session/ses_known" && request.method === "GET") return Response.json({ data: session })
|
||||
if (url.pathname === "/api/session/ses_known/rename" && request.method === "POST")
|
||||
return new Response(null, { status: 204 })
|
||||
return new Response("Not found", { status: 404 })
|
||||
},
|
||||
{ preconnect: () => {} },
|
||||
)
|
||||
const client = OpenCode.make({
|
||||
baseUrl: "https://opencode.test",
|
||||
fetch,
|
||||
})
|
||||
const bridge = await createOpenCodeBridge({
|
||||
client,
|
||||
directory: "/workspace",
|
||||
model: { providerID: "openai", id: "gpt-test" },
|
||||
notify: () => {},
|
||||
completionStore: memoryCompletionStore(),
|
||||
})
|
||||
const execute = async (name: string, input: Record<string, unknown>) => (await bridge.execute(name, input)).output
|
||||
|
||||
try {
|
||||
expect(new Set(bridge.definitions.map((tool) => tool.name)).size).toBe(bridge.definitions.length)
|
||||
expect(bridge.definitions.every((tool) => tool.name.length > 0 && tool.description.length > 0)).toBe(true)
|
||||
expect(bridge.definitions.find((tool) => tool.name === "reply_form")).toMatchObject({
|
||||
parameters: {
|
||||
properties: { answer: { type: "object", additionalProperties: true } },
|
||||
},
|
||||
})
|
||||
expect(bridge.definitions.find((tool) => tool.name === "rename_session")).toMatchObject({
|
||||
description: expect.stringContaining("does not prompt, wake, or interrupt"),
|
||||
parameters: {
|
||||
type: "object",
|
||||
additionalProperties: false,
|
||||
required: ["session_id", "title"],
|
||||
},
|
||||
})
|
||||
|
||||
expect(await execute("rename_session", { session_id: "ses_unknown", title: "Nope" })).toEqual({
|
||||
status: "error",
|
||||
message: "Use a session ID returned by find_sessions or start_session.",
|
||||
retryable: false,
|
||||
})
|
||||
expect(requests.some((request) => new URL(request.url).pathname.includes("ses_unknown"))).toBe(false)
|
||||
|
||||
await execute("find_sessions", {
|
||||
query: null,
|
||||
scope: "current_project",
|
||||
recency: "any",
|
||||
limit: 10,
|
||||
})
|
||||
expect(await execute("rename_session", { session_id: "ses_known", title: " " })).toEqual({
|
||||
status: "error",
|
||||
message: "A non-empty session title is required.",
|
||||
retryable: false,
|
||||
})
|
||||
expect(await execute("rename_session", { session_id: "ses_known", title: "Old title" })).toEqual({
|
||||
status: "unchanged",
|
||||
session_id: "ses_known",
|
||||
previous_title: "Old title",
|
||||
title: "Old title",
|
||||
})
|
||||
expect(requests.some((request) => new URL(request.url).pathname.endsWith("/rename"))).toBe(false)
|
||||
|
||||
expect(await execute("rename_session", { session_id: "ses_known", title: " New title " })).toEqual({
|
||||
status: "renamed",
|
||||
session_id: "ses_known",
|
||||
previous_title: "Old title",
|
||||
title: "New title",
|
||||
})
|
||||
const rename = requests.find((request) => new URL(request.url).pathname.endsWith("/rename"))
|
||||
expect(rename?.method).toBe("POST")
|
||||
expect(await rename?.json()).toEqual({ title: "New title" })
|
||||
} finally {
|
||||
await bridge.close()
|
||||
}
|
||||
})
|
||||
|
||||
test("retries a lost completion wait and prompts a started session by ID", async () => {
|
||||
const requests: Request[] = []
|
||||
const prompts: Array<{ readonly id: string; readonly text: string }> = []
|
||||
const notifications: Array<unknown> = []
|
||||
const traces: Array<{ readonly event: string; readonly data?: Record<string, unknown> }> = []
|
||||
let waits = 0
|
||||
const fetch = Object.assign(
|
||||
async (input: string | URL | Request, init?: RequestInit) => {
|
||||
const request = input instanceof Request ? input : new Request(input.toString(), init)
|
||||
requests.push(request)
|
||||
const url = new URL(request.url)
|
||||
if (url.pathname === "/api/event") return new Response("", { headers: { "content-type": "text/event-stream" } })
|
||||
if (url.pathname === "/api/session" && request.method === "POST")
|
||||
return Response.json({
|
||||
data: {
|
||||
id: "ses_focused",
|
||||
projectID: "project-1",
|
||||
cost: "0",
|
||||
tokens: { input: 0, output: 0, reasoning: 0, cache: { read: 0, write: 0 } },
|
||||
time: { created: 1, updated: 1 },
|
||||
title: "Focused session",
|
||||
location: { directory: "/workspace" },
|
||||
},
|
||||
})
|
||||
if (url.pathname === "/api/session/ses_focused/prompt" && request.method === "POST") {
|
||||
const body = await request.json()
|
||||
if (
|
||||
typeof body !== "object" ||
|
||||
body === null ||
|
||||
!("id" in body) ||
|
||||
typeof body.id !== "string" ||
|
||||
!("text" in body) ||
|
||||
typeof body.text !== "string"
|
||||
)
|
||||
return new Response("Invalid prompt", { status: 400 })
|
||||
prompts.push({ id: body.id, text: body.text })
|
||||
return Response.json({ data: { id: body.id, admittedSeq: prompts.length } })
|
||||
}
|
||||
if (url.pathname === "/api/session/ses_focused/wait" && request.method === "POST") {
|
||||
waits += 1
|
||||
if (waits === 1) throw new Error("connection closed")
|
||||
return new Response(null, { status: 204 })
|
||||
}
|
||||
if (url.pathname === "/api/session/ses_focused/message" && request.method === "GET")
|
||||
return Response.json({
|
||||
data: prompts.toReversed().map((prompt) => ({ id: prompt.id, type: "user", text: prompt.text })),
|
||||
cursor: {},
|
||||
})
|
||||
return new Response("Not found", { status: 404 })
|
||||
},
|
||||
{ preconnect: () => {} },
|
||||
)
|
||||
const bridge = await createOpenCodeBridge({
|
||||
client: OpenCode.make({ baseUrl: "https://opencode.test", fetch }),
|
||||
directory: "/workspace",
|
||||
model: { providerID: "openai", id: "gpt-test" },
|
||||
notify: (notification) => notifications.push(notification),
|
||||
trace: (event, data) => traces.push({ event, data }),
|
||||
completionStore: memoryCompletionStore(),
|
||||
})
|
||||
const execute = async (name: string, input: Record<string, unknown>) => (await bridge.execute(name, input)).output
|
||||
|
||||
try {
|
||||
expect(bridge.definitions.some((tool) => tool.name === "continue_session")).toBe(false)
|
||||
const started = await bridge.execute("start_session", { text: "First task", project_id: null })
|
||||
expect(started.output).toMatchObject({
|
||||
status: "started",
|
||||
session_id: "ses_focused",
|
||||
})
|
||||
expect(started.admittedPrompt).toEqual({ sessionID: "ses_focused", promptID: prompts[0].id })
|
||||
expect(
|
||||
await execute("prompt_session", { session_id: "ses_focused", text: "Follow-up task" }),
|
||||
).toMatchObject({
|
||||
status: "started",
|
||||
session_id: "ses_focused",
|
||||
})
|
||||
|
||||
await Bun.sleep(1_100)
|
||||
expect(prompts.map((prompt) => prompt.text)).toEqual(["First task", "Follow-up task"])
|
||||
expect(requests.filter((request) => new URL(request.url).pathname.endsWith("/wait")).length).toBeGreaterThanOrEqual(
|
||||
3,
|
||||
)
|
||||
expect(traces.some((entry) => entry.event === "opencode.wait.retrying")).toBe(true)
|
||||
expect(notifications.some((value) => JSON.stringify(value).includes("opencode.prompt.failed"))).toBe(false)
|
||||
} finally {
|
||||
await bridge.close()
|
||||
}
|
||||
})
|
||||
|
||||
test("discards a restored prompt whose session no longer exists", async () => {
|
||||
const removed: Array<{ readonly sessionID: string; readonly promptID: string }> = []
|
||||
const handle = { sessionID: "ses_missing", promptID: "msg_orphaned" }
|
||||
const fetch = Object.assign(
|
||||
async (input: string | URL | Request) => {
|
||||
const request = input instanceof Request ? input : new Request(input.toString())
|
||||
const url = new URL(request.url)
|
||||
if (url.pathname === "/api/event") return new Response("", { headers: { "content-type": "text/event-stream" } })
|
||||
if (url.pathname === "/api/session/ses_missing")
|
||||
return Response.json(
|
||||
{ _tag: "SessionNotFoundError", sessionID: handle.sessionID, message: `Session not found: ${handle.sessionID}` },
|
||||
{ status: 404 },
|
||||
)
|
||||
return new Response("Not found", { status: 404 })
|
||||
},
|
||||
{ preconnect: () => {} },
|
||||
)
|
||||
const store = {
|
||||
...memoryCompletionStore(),
|
||||
entries: () => [{ status: "pending" as const, handle }],
|
||||
remove: async (entry) => {
|
||||
removed.push(entry)
|
||||
},
|
||||
} satisfies CompletionStore
|
||||
|
||||
const bridge = await createOpenCodeBridge({
|
||||
client: OpenCode.make({ baseUrl: "https://opencode.test", fetch }),
|
||||
directory: "/workspace",
|
||||
model: { providerID: "openai", id: "gpt-test" },
|
||||
notify: () => {},
|
||||
completionStore: store,
|
||||
})
|
||||
try {
|
||||
expect(removed).toEqual([handle])
|
||||
} finally {
|
||||
await bridge.close()
|
||||
}
|
||||
})
|
||||
|
||||
function memoryCompletionStore(): CompletionStore {
|
||||
return {
|
||||
entries: () => [],
|
||||
admitting: async () => {},
|
||||
pending: async () => {},
|
||||
completed: async () => {},
|
||||
remove: async () => {},
|
||||
close: async () => {},
|
||||
}
|
||||
}
|
||||
@@ -1,10 +0,0 @@
|
||||
import { expect, test } from "bun:test"
|
||||
import { pcmLevel } from "../src/pcm"
|
||||
|
||||
test("PCM level ignores silence and normalizes audible energy", () => {
|
||||
expect(pcmLevel(Buffer.alloc(4_800))).toBe(0)
|
||||
const audible = Buffer.alloc(4_800)
|
||||
for (let offset = 0; offset < audible.length; offset += 2) audible.writeInt16LE(8_000, offset)
|
||||
expect(pcmLevel(audible)).toBeGreaterThan(0.8)
|
||||
expect(pcmLevel(audible)).toBeLessThanOrEqual(1)
|
||||
})
|
||||
@@ -1,137 +0,0 @@
|
||||
import { describe, expect, test } from "bun:test"
|
||||
import { Option } from "effect"
|
||||
import { decodeVoiceToolInput } from "../src/protocol"
|
||||
import { createLiveContextAppendQueue, liveNotificationAcknowledgement } from "../src/protocol-live"
|
||||
import { realtimeNotificationAcknowledgement, realtimeSessionUpdate } from "../src/protocol-realtime"
|
||||
import { createSingleFlightAcknowledgement } from "../src/single-flight-acknowledgement"
|
||||
import { openCodeAnnouncementText } from "../src/opencode-notification"
|
||||
|
||||
describe("voice protocol", () => {
|
||||
test("accepts only JSON object tool arguments", () => {
|
||||
expect(Option.getOrUndefined(decodeVoiceToolInput('{"text":"hello","nested":{"value":1}}'))).toEqual({
|
||||
text: "hello",
|
||||
nested: { value: 1 },
|
||||
})
|
||||
for (const input of ["not json", "null", "[]", "1", '"text"'])
|
||||
expect(Option.getOrUndefined(decodeVoiceToolInput(input))).toBeUndefined()
|
||||
})
|
||||
|
||||
test("correlates Realtime notification acknowledgements", () => {
|
||||
const pending = { itemID: "item-current", eventID: "event-current" }
|
||||
expect(
|
||||
realtimeNotificationAcknowledgement(
|
||||
{ type: "conversation.item.created", item: { id: "item-current" } },
|
||||
pending,
|
||||
),
|
||||
).toBe(true)
|
||||
expect(
|
||||
realtimeNotificationAcknowledgement(
|
||||
{ type: "conversation.item.created", item: { id: "item-other" } },
|
||||
pending,
|
||||
),
|
||||
).toBeUndefined()
|
||||
expect(
|
||||
realtimeNotificationAcknowledgement(
|
||||
{ type: "error", error: { event_id: "event-current" } },
|
||||
pending,
|
||||
),
|
||||
).toBe(false)
|
||||
})
|
||||
|
||||
test("preserves tool schemas in Realtime session updates", () => {
|
||||
const tools = [
|
||||
{
|
||||
type: "function" as const,
|
||||
name: "open_form",
|
||||
description: "Open form",
|
||||
parameters: {
|
||||
type: "object",
|
||||
properties: { answer: { type: "object", additionalProperties: true } },
|
||||
additionalProperties: true,
|
||||
},
|
||||
},
|
||||
]
|
||||
expect(JSON.parse(JSON.stringify(realtimeSessionUpdate({ instructions: "test", tools, voice: "marin" }))).session.tools)
|
||||
.toEqual(tools)
|
||||
})
|
||||
|
||||
test("correlates Live notification acknowledgements by expected context", () => {
|
||||
const pending = { eventID: "event-current", acknowledgement: "delegation.context.appended" as const }
|
||||
expect(
|
||||
liveNotificationAcknowledgement(
|
||||
{ type: "delegation.context.appended", eventID: "event-current" },
|
||||
pending,
|
||||
),
|
||||
).toBe(true)
|
||||
expect(
|
||||
liveNotificationAcknowledgement({ type: "session.context.appended", eventID: "event-current" }, pending),
|
||||
).toBeUndefined()
|
||||
expect(
|
||||
liveNotificationAcknowledgement(
|
||||
{ type: "delegation.context.appended", eventID: "event-other" },
|
||||
pending,
|
||||
),
|
||||
).toBe(true)
|
||||
expect(
|
||||
liveNotificationAcknowledgement({ type: "error", errorEventID: "event-current" }, pending),
|
||||
).toBe(false)
|
||||
})
|
||||
|
||||
test("serializes indistinguishable Live context appends", async () => {
|
||||
const sent: Array<Record<string, unknown>> = []
|
||||
const queue = createLiveContextAppendQueue({
|
||||
send: (event) => {
|
||||
sent.push(event)
|
||||
return true
|
||||
},
|
||||
timeoutMs: 100,
|
||||
})
|
||||
const first = queue.append(
|
||||
{ type: "delegation.context.append", event_id: "event-1" },
|
||||
"delegation.context.appended",
|
||||
)
|
||||
const second = queue.append(
|
||||
{ type: "delegation.context.append", event_id: "event-2" },
|
||||
"delegation.context.appended",
|
||||
)
|
||||
expect(sent.map((event) => event["event_id"])).toEqual(["event-1"])
|
||||
|
||||
queue.receive({ type: "session.context.appended", eventID: "unrelated" })
|
||||
expect(sent.map((event) => event["event_id"])).toEqual(["event-1"])
|
||||
queue.receive({ type: "delegation.context.appended", eventID: "not-echoed" })
|
||||
expect(await first).toBe(true)
|
||||
expect(sent.map((event) => event["event_id"])).toEqual(["event-1", "event-2"])
|
||||
queue.receive({ type: "delegation.context.appended" })
|
||||
expect(await second).toBe(true)
|
||||
queue.close()
|
||||
})
|
||||
|
||||
test("keeps notification acknowledgement single-flight", async () => {
|
||||
const acknowledgement = createSingleFlightAcknowledgement<{ readonly token: string }>(50)
|
||||
const first = acknowledgement.begin("notification-1", { token: "first" })
|
||||
const second = acknowledgement.begin("notification-2", { token: "second" })
|
||||
|
||||
expect(first.started).toBe(true)
|
||||
expect(second.started).toBe(false)
|
||||
expect(await second.promise).toBe(false)
|
||||
expect(acknowledgement.current()).toEqual({ id: "notification-1", correlation: { token: "first" } })
|
||||
acknowledgement.settle("notification-1", true)
|
||||
expect(await first.promise).toBe(true)
|
||||
expect(acknowledgement.current()).toBeUndefined()
|
||||
})
|
||||
|
||||
test("renders bounded conversational OpenCode announcements without identifiers", () => {
|
||||
const text = openCodeAnnouncementText({
|
||||
type: "opencode.prompt.completed",
|
||||
session_id: "ses_private",
|
||||
prompt_id: "msg_private",
|
||||
status: "completed",
|
||||
text: "finished ".repeat(200),
|
||||
})
|
||||
expect(text.length).toBeLessThanOrEqual(800)
|
||||
expect(text).not.toContain("ses_private")
|
||||
expect(text).not.toContain("msg_private")
|
||||
expect(text).toStartWith("OpenCode prompt completed: finished")
|
||||
})
|
||||
|
||||
})
|
||||
@@ -1,69 +0,0 @@
|
||||
import { expect, test } from "bun:test"
|
||||
import { createRealtimeEventProjector } from "../src/protocol-realtime"
|
||||
|
||||
test("projects one tool start and one work request for duplicate item frames", () => {
|
||||
const projector = createRealtimeEventProjector()
|
||||
const item = {
|
||||
type: "function_call",
|
||||
name: "find_sessions",
|
||||
call_id: "call-1",
|
||||
arguments: '{"query":null}',
|
||||
}
|
||||
const added = projector.receive(JSON.stringify({ type: "response.output_item.added", item }))
|
||||
const duplicate = projector.receive(JSON.stringify({ type: "response.output_item.added", item }))
|
||||
const done = projector.receive(JSON.stringify({ type: "response.output_item.done", item }))
|
||||
|
||||
expect(added.events).toContainEqual({ type: "tool.started", id: "call-1", name: "find_sessions" })
|
||||
expect(duplicate.events.some((event) => event.type === "tool.started")).toBe(false)
|
||||
expect(done.events).toContainEqual({
|
||||
type: "work.requested",
|
||||
request: { id: "call-1", name: "find_sessions", input: { query: null } },
|
||||
})
|
||||
expect(projector.receive(JSON.stringify({ type: "response.output_item.done", item })).events).toEqual([
|
||||
{ type: "debug", message: "response.output_item.done" },
|
||||
])
|
||||
})
|
||||
|
||||
test("resumes a response only after all projected work resolves", () => {
|
||||
const projector = createRealtimeEventProjector()
|
||||
const response = projector.receive(
|
||||
JSON.stringify({
|
||||
type: "response.done",
|
||||
response: { output: [{ type: "function_call", call_id: "call-1" }] },
|
||||
}),
|
||||
)
|
||||
|
||||
expect(response.events).toContainEqual({ type: "assistant.done", awaitingWork: true })
|
||||
expect(response.commands).toEqual([])
|
||||
expect(projector.resolveWork("call-1")).toEqual([{ type: "response.create" }])
|
||||
expect(projector.resolveWork("call-1")).toEqual([])
|
||||
})
|
||||
|
||||
test("resumes when work resolved before the response completion frame", () => {
|
||||
const projector = createRealtimeEventProjector()
|
||||
expect(projector.resolveWork("call-1")).toEqual([])
|
||||
expect(
|
||||
projector.receive(
|
||||
JSON.stringify({
|
||||
type: "response.done",
|
||||
response: { output: [{ type: "function_call", call_id: "call-1" }] },
|
||||
}),
|
||||
).commands,
|
||||
).toEqual([{ type: "response.create" }])
|
||||
})
|
||||
|
||||
test("rejects malformed tool arguments without hanging the provider call", () => {
|
||||
const projector = createRealtimeEventProjector()
|
||||
const result = projector.receive(
|
||||
JSON.stringify({
|
||||
type: "response.output_item.done",
|
||||
item: { type: "function_call", name: "find_sessions", call_id: "call-1", arguments: "[]" },
|
||||
}),
|
||||
)
|
||||
|
||||
expect(result.events).toContainEqual({
|
||||
type: "work.rejected",
|
||||
request: { id: "call-1", name: "find_sessions", input: {} },
|
||||
output: { status: "error", message: "Invalid arguments for tool find_sessions." },
|
||||
})
|
||||
})
|
||||
@@ -1,229 +0,0 @@
|
||||
import { expect, test } from "bun:test"
|
||||
import {
|
||||
createResponsesControllerContext,
|
||||
responsesControllerTools,
|
||||
runResponsesController,
|
||||
} from "../src/responses-controller"
|
||||
|
||||
test("runs the client delegation controller through OpenCode tools", async () => {
|
||||
const bodies: Array<Record<string, unknown>> = []
|
||||
const calls: Array<unknown> = []
|
||||
const responses = [
|
||||
{
|
||||
id: "response-1",
|
||||
output: [
|
||||
{
|
||||
type: "function_call",
|
||||
call_id: "call-1",
|
||||
name: "start_session",
|
||||
arguments: '{"text":"Inspect the package","project_id":null}',
|
||||
},
|
||||
],
|
||||
},
|
||||
{
|
||||
id: "response-2",
|
||||
output: [
|
||||
{
|
||||
type: "message",
|
||||
content: [{ type: "output_text", text: "The inspection is running and I will report back." }],
|
||||
},
|
||||
],
|
||||
},
|
||||
]
|
||||
const fetch = Object.assign(
|
||||
async (_input: string | URL | Request, init?: RequestInit) => {
|
||||
const body: unknown = await new Request("https://opencode.test", init).json()
|
||||
if (body && typeof body === "object" && !Array.isArray(body)) bodies.push(Object.fromEntries(Object.entries(body)))
|
||||
return Response.json(responses.shift())
|
||||
},
|
||||
{ preconnect: () => {} },
|
||||
)
|
||||
|
||||
const text = await runResponsesController({
|
||||
apiKey: "test",
|
||||
model: "gpt-test",
|
||||
instructions: "Use tools.",
|
||||
text: "Inspect this package",
|
||||
tools: [{ type: "function", name: "start_session", description: "Start", parameters: {} }],
|
||||
execute: async (call) => {
|
||||
calls.push(call)
|
||||
return { status: "started", session_id: "ses_started", prompt_id: "prompt-1" }
|
||||
},
|
||||
fetch,
|
||||
})
|
||||
|
||||
expect(text).toBe(
|
||||
"The inspection is running and I will report back.\n\nPrivate voice-control context (never speak this aloud): OpenCode session IDs explicitly used by this delegation: ses_started. Include the relevant session_id in any future delegated request to continue this work.",
|
||||
)
|
||||
expect(calls).toEqual([
|
||||
{
|
||||
id: "call-1",
|
||||
name: "start_session",
|
||||
input: { text: "Inspect the package", project_id: null },
|
||||
},
|
||||
])
|
||||
expect(bodies[1]).toMatchObject({
|
||||
previous_response_id: "response-1",
|
||||
input: [
|
||||
{
|
||||
type: "function_call_output",
|
||||
call_id: "call-1",
|
||||
output: '{"status":"started","session_id":"ses_started","prompt_id":"prompt-1"}',
|
||||
},
|
||||
],
|
||||
})
|
||||
expect(bodies[0]?.["tools"]).toEqual([
|
||||
{ type: "function", name: "start_session", description: "Start", parameters: {} },
|
||||
])
|
||||
})
|
||||
|
||||
test("rejects malformed controller tool arguments", async () => {
|
||||
const fetch = Object.assign(
|
||||
async () =>
|
||||
Response.json({
|
||||
id: "response-1",
|
||||
output: [{ type: "function_call", call_id: "call-1", name: "start_session", arguments: "not json" }],
|
||||
}),
|
||||
{ preconnect: () => {} },
|
||||
)
|
||||
|
||||
expect(
|
||||
runResponsesController({
|
||||
apiKey: "test",
|
||||
model: "gpt-test",
|
||||
instructions: "Use tools.",
|
||||
text: "Inspect this package",
|
||||
tools: [{ type: "function", name: "start_session", description: "Start", parameters: {} }],
|
||||
execute: async () => ({ status: "started" }),
|
||||
fetch,
|
||||
}),
|
||||
).rejects.toThrow("invalid arguments for start_session")
|
||||
})
|
||||
|
||||
test("retains the most recently used session across delegations", async () => {
|
||||
const context = createResponsesControllerContext()
|
||||
const bodies: Array<Record<string, unknown>> = []
|
||||
const responses = [
|
||||
{
|
||||
id: "response-1",
|
||||
output: [
|
||||
{
|
||||
type: "function_call",
|
||||
call_id: "call-1",
|
||||
name: "read_session",
|
||||
arguments: '{"session_id":"ses_recent"}',
|
||||
},
|
||||
],
|
||||
},
|
||||
{
|
||||
id: "response-2",
|
||||
output: [{ type: "message", content: [{ type: "output_text", text: "Found it." }] }],
|
||||
},
|
||||
{
|
||||
id: "response-3",
|
||||
output: [{ type: "message", content: [{ type: "output_text", text: "Stopped it." }] }],
|
||||
},
|
||||
]
|
||||
const fetch = Object.assign(
|
||||
async (_input: string | URL | Request, init?: RequestInit) => {
|
||||
const body: unknown = await new Request("https://opencode.test", init).json()
|
||||
if (body && typeof body === "object" && !Array.isArray(body)) bodies.push(Object.fromEntries(Object.entries(body)))
|
||||
return Response.json(responses.shift())
|
||||
},
|
||||
{ preconnect: () => {} },
|
||||
)
|
||||
const options = {
|
||||
apiKey: "test",
|
||||
model: "gpt-test",
|
||||
instructions: "Use tools.",
|
||||
tools: [{ type: "function" as const, name: "read_session", description: "Read", parameters: {} }],
|
||||
execute: async () => ({ status: "ok" }),
|
||||
context,
|
||||
fetch,
|
||||
}
|
||||
|
||||
await runResponsesController({ ...options, text: "Find the label behavior session" })
|
||||
await runResponsesController({ ...options, text: "Stop that session" })
|
||||
|
||||
expect(context.lastSessionID).toBe("ses_recent")
|
||||
expect(bodies[2]?.["input"]).toContain("the most recently used OpenCode session ID is ses_recent")
|
||||
expect(bodies[2]?.["input"]).toContain('references like "that session", "it", or "stop that"')
|
||||
})
|
||||
|
||||
test("coalesces identical read-only calls within one delegation", async () => {
|
||||
const responses = [
|
||||
{
|
||||
id: "response-1",
|
||||
output: [
|
||||
{ type: "function_call", call_id: "call-1", name: "read_session", arguments: '{"session_id":"ses_1"}' },
|
||||
{ type: "function_call", call_id: "call-2", name: "read_session", arguments: '{"session_id":"ses_1"}' },
|
||||
],
|
||||
},
|
||||
{
|
||||
id: "response-2",
|
||||
output: [{ type: "message", content: [{ type: "output_text", text: "Done." }] }],
|
||||
},
|
||||
]
|
||||
let executions = 0
|
||||
const fetch = Object.assign(async () => Response.json(responses.shift()), { preconnect: () => {} })
|
||||
|
||||
await runResponsesController({
|
||||
apiKey: "test",
|
||||
model: "gpt-test",
|
||||
instructions: "Use tools.",
|
||||
text: "Read it",
|
||||
tools: [{ type: "function", name: "read_session", description: "Read", parameters: {} }],
|
||||
execute: async () => {
|
||||
executions += 1
|
||||
return { status: "ok" }
|
||||
},
|
||||
fetch,
|
||||
})
|
||||
|
||||
expect(executions).toBe(1)
|
||||
})
|
||||
|
||||
test("withholds new-session creation during follow-ups unless explicitly requested", () => {
|
||||
const context = createResponsesControllerContext()
|
||||
context.lastSessionID = "ses_recent"
|
||||
const tools = [
|
||||
{ type: "function" as const, name: "start_session", description: "Start", parameters: {} },
|
||||
{ type: "function" as const, name: "read_session", description: "Read", parameters: {} },
|
||||
]
|
||||
|
||||
expect(responsesControllerTools(tools, "Stop that session", context).map((tool) => tool.name)).toEqual([
|
||||
"read_session",
|
||||
])
|
||||
expect(responsesControllerTools(tools, "Start a new session for this", context)).toEqual(tools)
|
||||
})
|
||||
|
||||
test("remembers a uniquely discovered session without requiring a read", async () => {
|
||||
const context = createResponsesControllerContext()
|
||||
const responses = [
|
||||
{
|
||||
id: "response-1",
|
||||
output: [
|
||||
{ type: "function_call", call_id: "call-1", name: "find_sessions", arguments: '{"query":"label"}' },
|
||||
],
|
||||
},
|
||||
{
|
||||
id: "response-2",
|
||||
output: [{ type: "message", content: [{ type: "output_text", text: "Found one." }] }],
|
||||
},
|
||||
]
|
||||
const fetch = Object.assign(async () => Response.json(responses.shift()), { preconnect: () => {} })
|
||||
|
||||
await runResponsesController({
|
||||
apiKey: "test",
|
||||
model: "gpt-test",
|
||||
instructions: "Use tools.",
|
||||
text: "Find label behavior",
|
||||
tools: [{ type: "function", name: "find_sessions", description: "Find", parameters: {} }],
|
||||
execute: async () => ({ status: "ok", sessions: [{ id: "ses_unique", title: "Label behavior" }] }),
|
||||
context,
|
||||
fetch,
|
||||
})
|
||||
|
||||
expect(context.lastSessionID).toBe("ses_unique")
|
||||
expect(context.sessionIDs).toEqual(new Set(["ses_unique"]))
|
||||
})
|
||||
@@ -1,331 +0,0 @@
|
||||
import { expect, mock, test } from "bun:test"
|
||||
import { createTestRenderer } from "@opentui/core/testing"
|
||||
|
||||
test("voice rows fill the viewport, wrap under content, and keep tools after their preamble", async () => {
|
||||
const setup = await createTestRenderer({ width: 64, height: 18, useThread: false })
|
||||
const core = await import("@opentui/core")
|
||||
void mock.module("@opentui/core", () => ({ ...core, createCliRenderer: async () => setup.renderer }))
|
||||
const { createVoiceTUI } = await import("../src/ui")
|
||||
let interrupts = 0
|
||||
let voiceCycles = 0
|
||||
let microphoneToggles = 0
|
||||
let speakerToggles = 0
|
||||
const ui = await createVoiceTUI({
|
||||
onInterrupt: () => interrupts++,
|
||||
onExit: () => {},
|
||||
onCycleVoice: () => voiceCycles++,
|
||||
onToggleMicrophone: () => microphoneToggles++,
|
||||
onToggleSpeaker: () => speakerToggles++,
|
||||
reducedMotion: true,
|
||||
})
|
||||
|
||||
try {
|
||||
ui.setStatus({ audio: "duplex", voice: "marin", model: "gpt-live" })
|
||||
ui.meta("[session] created session-1")
|
||||
ui.userCommitted("user-1")
|
||||
ui.userTranscript("user-1", "Please inspect the current session", true)
|
||||
ui.toolStart("tool-1", "opencode", { request: "inspect" })
|
||||
ui.meta("[session] adopted session-1")
|
||||
ui.assistantDelta("Let me check. This sentence is")
|
||||
ui.assistantDelta(" long enough to wrap cleanly under its column.")
|
||||
ui.assistantDone()
|
||||
ui.assistantDelta("Continued in the same assistant row.")
|
||||
ui.assistantDone()
|
||||
await Bun.sleep(200)
|
||||
await setup.flush()
|
||||
|
||||
const rows = frameRows(setup.captureCharFrame())
|
||||
expect(rows).toHaveLength(18)
|
||||
expect(rows.every((row) => row.length === 64)).toBe(true)
|
||||
expect(rows[16]).toStartWith(" ▂▂▂▁ duplex marin gpt-live")
|
||||
expect(rows[16]).not.toContain("no session")
|
||||
expect(rows[17]).toContain("esc interrupt")
|
||||
expect(rows[17]).not.toContain("any key")
|
||||
expect(rows[15].trim()).toBe("")
|
||||
expect(rows.filter((row) => row.includes("Let me check"))).toHaveLength(1)
|
||||
expect(rows.findIndex((row) => row.includes("opencode"))).toBeGreaterThan(
|
||||
rows.findIndex((row) => row.includes("Let me check")),
|
||||
)
|
||||
expect(rows.findIndex((row) => row.includes("[session] adopted"))).toBeGreaterThan(
|
||||
rows.findIndex((row) => row.includes("opencode")),
|
||||
)
|
||||
expect(rows.find((row) => row.includes("under its column"))?.indexOf("under")).toBe(
|
||||
rows.find((row) => row.includes("Let me check"))?.indexOf("Let"),
|
||||
)
|
||||
// Deltas inside one turn concatenate raw; a delta after assistantDone starts a new
|
||||
// message and must gain exactly one space at the boundary.
|
||||
const assistant = rows.join("")
|
||||
expect(assistant).toContain("This sentence is long enough")
|
||||
expect(assistant).toContain("column. Continued")
|
||||
expect(assistant).not.toContain("column.Continued")
|
||||
expect(rows.find((row) => row.includes("[session] created"))).toStartWith(" · [session]")
|
||||
|
||||
setup.resize(96, 22)
|
||||
await setup.flush()
|
||||
const wide = frameRows(setup.captureCharFrame())
|
||||
expect(wide).toHaveLength(22)
|
||||
expect(wide.every((row) => row.length === 96)).toBe(true)
|
||||
expect(wide.at(-2)).toStartWith(" ▂▂▂▁ duplex marin gpt-live")
|
||||
|
||||
setup.resize(48, 16)
|
||||
await setup.flush()
|
||||
const narrow = frameRows(setup.captureCharFrame())
|
||||
expect(narrow).toHaveLength(16)
|
||||
expect(narrow.every((row) => row.length === 48)).toBe(true)
|
||||
expect(narrow.at(-1)).toContain("esc interrupt")
|
||||
|
||||
setup.resize(64, 18)
|
||||
await setup.flush()
|
||||
|
||||
ui.userSpeaking(true)
|
||||
await setup.flush()
|
||||
expect(setup.captureCharFrame()).not.toContain("│ transcribing")
|
||||
expect(frameRows(setup.captureCharFrame()).at(-2)).toStartWith(" ▂▂▂▂")
|
||||
|
||||
ui.userReset()
|
||||
await setup.flush()
|
||||
expect(setup.captureCharFrame()).not.toContain("│ transcribing")
|
||||
|
||||
ui.userSpeaking(true)
|
||||
ui.userSpeaking(false)
|
||||
ui.userCommitted("user-2")
|
||||
ui.userTranscript("user-2", "partial transcript", false)
|
||||
await setup.flush()
|
||||
const partial = setup
|
||||
.captureCharFrame()
|
||||
.split("\n")
|
||||
.find((row) => row.includes("partial transcript"))
|
||||
expect(partial).toContain("│ partial transcript")
|
||||
expect(partial).not.toContain("you")
|
||||
|
||||
ui.userSpeaking(true)
|
||||
await setup.flush()
|
||||
expect(
|
||||
setup
|
||||
.captureCharFrame()
|
||||
.split("\n")
|
||||
.filter((row) => row.includes("│ transcribing")),
|
||||
).toHaveLength(0)
|
||||
|
||||
ui.userReset()
|
||||
await setup.flush()
|
||||
|
||||
ui.setStatus({ microphoneMuted: true })
|
||||
await setup.flush()
|
||||
expect(frameRows(setup.captureCharFrame()).at(-2)).toStartWith(" ────")
|
||||
const muted = setup
|
||||
.captureCharFrame()
|
||||
.split("\n")
|
||||
.find((row) => row.includes("partial transcript"))
|
||||
expect(muted).toContain("│ partial transcript")
|
||||
expect(muted).not.toContain("│ ...")
|
||||
|
||||
ui.userSpeaking(false)
|
||||
ui.setStatus({ microphoneMuted: false })
|
||||
ui.userTranscript("user-2", "partial transcript", true)
|
||||
await setup.flush()
|
||||
const complete = setup
|
||||
.captureCharFrame()
|
||||
.split("\n")
|
||||
.find((row) => row.includes("partial transcript"))
|
||||
expect(complete).toContain("│ partial transcript")
|
||||
|
||||
ui.assistantDelta("First response.")
|
||||
ui.userCommitted("boundary-user")
|
||||
ui.userTranscript("boundary-user", "A new request", true)
|
||||
ui.assistantDelta("Second response.")
|
||||
await setup.flush()
|
||||
const boundaries = setup.captureCharFrame().split("\n")
|
||||
expect(boundaries.findIndex((row) => row.includes("First response."))).toBeLessThan(
|
||||
boundaries.findIndex((row) => row.includes("A new request")),
|
||||
)
|
||||
expect(boundaries.findIndex((row) => row.includes("A new request"))).toBeLessThan(
|
||||
boundaries.findIndex((row) => row.includes("Second response.")),
|
||||
)
|
||||
|
||||
await setup.mockInput.typeText(" x")
|
||||
setup.mockInput.pressArrow("right")
|
||||
await setup.flush()
|
||||
expect(interrupts).toBe(0)
|
||||
expect(voiceCycles).toBe(0)
|
||||
expect(microphoneToggles).toBe(0)
|
||||
expect(speakerToggles).toBe(0)
|
||||
|
||||
setup.mockInput.pressKey("v")
|
||||
setup.mockInput.pressKey("m")
|
||||
setup.mockInput.pressKey("s")
|
||||
await setup.flush()
|
||||
expect(voiceCycles).toBe(1)
|
||||
expect(microphoneToggles).toBe(1)
|
||||
expect(speakerToggles).toBe(1)
|
||||
|
||||
setup.mockInput.pressEscape()
|
||||
await Bun.sleep(50)
|
||||
await setup.flush()
|
||||
expect(interrupts).toBe(1)
|
||||
} finally {
|
||||
ui.close()
|
||||
mock.restore()
|
||||
}
|
||||
})
|
||||
|
||||
// In audio mode ui.assistantDone() is scheduled off the AudioSession playback clock,
|
||||
// so the next turn's transcript deltas arrive while the row is still streaming. The turn boundary
|
||||
// therefore has to survive in the delta text itself, not in the row's streaming flag.
|
||||
test("keeps the turn boundary when the next turn streams before playback finishes", async () => {
|
||||
const setup = await createTestRenderer({ width: 72, height: 12, useThread: false })
|
||||
const core = await import("@opentui/core")
|
||||
void mock.module("@opentui/core", () => ({ ...core, createCliRenderer: async () => setup.renderer }))
|
||||
const { createVoiceTUI } = await import("../src/ui")
|
||||
const ui = await createVoiceTUI({
|
||||
onInterrupt: () => {},
|
||||
onExit: () => {},
|
||||
onCycleVoice: () => {},
|
||||
onToggleMicrophone: () => {},
|
||||
onToggleSpeaker: () => {},
|
||||
reducedMotion: true,
|
||||
})
|
||||
|
||||
try {
|
||||
// Verbatim projector deltas: each turn's first fragment carries the boundary space.
|
||||
ui.assistantDelta(' Session "Fix auth bug"')
|
||||
ui.assistantDelta(" is ready.")
|
||||
// turn.done reaches the protocol, but playback is still draining, so no assistantDone() yet.
|
||||
ui.assistantDelta(" Next up, tests.")
|
||||
await setup.flush()
|
||||
|
||||
const frame = setup.captureCharFrame()
|
||||
expect(frame).toContain('Session "Fix auth bug" is ready. Next up, tests.')
|
||||
expect(frame).not.toContain("ready.Next")
|
||||
// The row itself must not open with the boundary space.
|
||||
expect(frame).toContain('│ Session "Fix auth bug"')
|
||||
expect(frame).not.toContain('│ Session "Fix auth bug"')
|
||||
|
||||
ui.assistantDone()
|
||||
await setup.flush()
|
||||
expect(setup.captureCharFrame()).toContain("is ready. Next up, tests.")
|
||||
} finally {
|
||||
ui.close()
|
||||
mock.restore()
|
||||
}
|
||||
})
|
||||
|
||||
test("keeps animated text mounted after its reveal completes", async () => {
|
||||
const setup = await createTestRenderer({ width: 72, height: 10, useThread: false })
|
||||
const core = await import("@opentui/core")
|
||||
void mock.module("@opentui/core", () => ({ ...core, createCliRenderer: async () => setup.renderer }))
|
||||
const { createVoiceTUI } = await import("../src/ui")
|
||||
const ui = await createVoiceTUI({
|
||||
onInterrupt: () => {},
|
||||
onExit: () => {},
|
||||
onCycleVoice: () => {},
|
||||
onToggleMicrophone: () => {},
|
||||
onToggleSpeaker: () => {},
|
||||
})
|
||||
|
||||
try {
|
||||
ui.assistantDelta("Animated text remains visible.")
|
||||
await Bun.sleep(500)
|
||||
await setup.flush()
|
||||
expect(setup.captureCharFrame()).toContain("Animated text remains visible.")
|
||||
} finally {
|
||||
ui.close()
|
||||
mock.restore()
|
||||
}
|
||||
})
|
||||
|
||||
test("renders consecutive tools as compact single-spaced rows", async () => {
|
||||
const setup = await createTestRenderer({ width: 52, height: 8, useThread: false })
|
||||
const core = await import("@opentui/core")
|
||||
void mock.module("@opentui/core", () => ({ ...core, createCliRenderer: async () => setup.renderer }))
|
||||
const { createVoiceTUI } = await import("../src/ui")
|
||||
const ui = await createVoiceTUI({
|
||||
onInterrupt: () => {},
|
||||
onExit: () => {},
|
||||
onCycleVoice: () => {},
|
||||
onToggleMicrophone: () => {},
|
||||
onToggleSpeaker: () => {},
|
||||
reducedMotion: true,
|
||||
})
|
||||
|
||||
try {
|
||||
ui.toolStart("tool-1", "read_session", {})
|
||||
ui.toolDone("tool-1", { status: "ok" })
|
||||
ui.toolStart("tool-2", "read_session", {})
|
||||
ui.toolDone("tool-2", { status: "ok" })
|
||||
await Bun.sleep(200)
|
||||
await setup.flush()
|
||||
|
||||
const rows = frameRows(setup.captureCharFrame())
|
||||
const tools = rows.flatMap((row, index) => (row.includes("read_session") ? [{ row, index }] : []))
|
||||
expect(tools).toHaveLength(2)
|
||||
expect(tools[1]?.index).toBe((tools[0]?.index ?? 0) + 1)
|
||||
expect(tools.every((tool) => tool.row.includes("✓ read_session ok"))).toBe(true)
|
||||
expect(rows.at(-3)?.trim()).toBe("")
|
||||
} finally {
|
||||
ui.close()
|
||||
mock.restore()
|
||||
}
|
||||
})
|
||||
|
||||
test("shows the reason a tool failed", async () => {
|
||||
const setup = await createTestRenderer({ width: 72, height: 8, useThread: false })
|
||||
const core = await import("@opentui/core")
|
||||
void mock.module("@opentui/core", () => ({ ...core, createCliRenderer: async () => setup.renderer }))
|
||||
const { createVoiceTUI } = await import("../src/ui")
|
||||
const ui = await createVoiceTUI({
|
||||
onInterrupt: () => {},
|
||||
onExit: () => {},
|
||||
onCycleVoice: () => {},
|
||||
onToggleMicrophone: () => {},
|
||||
onToggleSpeaker: () => {},
|
||||
reducedMotion: true,
|
||||
})
|
||||
|
||||
try {
|
||||
ui.toolStart("tool-error", "opencode", {})
|
||||
ui.toolDone("tool-error", { status: "error", message: "Invalid tool schema" })
|
||||
await Bun.sleep(200)
|
||||
await setup.flush()
|
||||
expect(setup.captureCharFrame()).toContain("opencode error · Invalid tool schema")
|
||||
} finally {
|
||||
ui.close()
|
||||
mock.restore()
|
||||
}
|
||||
})
|
||||
|
||||
test("paints the full viewport and keeps mic activity in the footer", async () => {
|
||||
const setup = await createTestRenderer({ width: 40, height: 12, useThread: false })
|
||||
const core = await import("@opentui/core")
|
||||
void mock.module("@opentui/core", () => ({ ...core, createCliRenderer: async () => setup.renderer }))
|
||||
const { createVoiceTUI } = await import("../src/ui")
|
||||
const ui = await createVoiceTUI({
|
||||
onInterrupt: () => {},
|
||||
onExit: () => {},
|
||||
onCycleVoice: () => {},
|
||||
onToggleMicrophone: () => {},
|
||||
onToggleSpeaker: () => {},
|
||||
reducedMotion: true,
|
||||
})
|
||||
|
||||
try {
|
||||
ui.userSpeaking(true)
|
||||
await setup.flush()
|
||||
|
||||
const rows = frameRows(setup.captureCharFrame())
|
||||
expect(rows.some((row) => row.includes("│ transcribing"))).toBe(false)
|
||||
expect(rows.at(-2)).toStartWith(" ▂▂▂▂")
|
||||
expect(setup.captureSpans().lines.every((line) => line.spans.every((span) => span.bg.a === 1))).toBe(true)
|
||||
|
||||
setup.resize(56, 16)
|
||||
await setup.flush()
|
||||
expect(setup.captureSpans().lines.every((line) => line.spans.every((span) => span.bg.a === 1))).toBe(true)
|
||||
} finally {
|
||||
ui.close()
|
||||
mock.restore()
|
||||
}
|
||||
})
|
||||
|
||||
function frameRows(frame: string) {
|
||||
return frame.split("\n").slice(0, -1)
|
||||
}
|
||||
@@ -1,74 +0,0 @@
|
||||
import { describe, expect, test } from "bun:test"
|
||||
import { initialVoiceView, transitionVoiceView, type VoiceViewEvent, type VoiceViewState } from "../src/ui-model"
|
||||
|
||||
const apply = (state: VoiceViewState, ...events: ReadonlyArray<VoiceViewEvent>) =>
|
||||
events.reduce(transitionVoiceView, state)
|
||||
|
||||
describe("VoiceView", () => {
|
||||
test("keeps assistant responses separated by a committed user turn", () => {
|
||||
const state = apply(
|
||||
initialVoiceView(),
|
||||
{ type: "assistant.delta", text: "First response.", now: 0, animate: false },
|
||||
{ type: "user.committed", itemID: "user-1" },
|
||||
{ type: "user.transcript", itemID: "user-1", text: "Next request", final: true, now: 1, animate: false },
|
||||
{ type: "assistant.delta", text: "Second response.", now: 2, animate: false },
|
||||
)
|
||||
expect(state.messages.map((message) => message.kind)).toEqual(["assistant", "user", "assistant"])
|
||||
expect(state.messages.flatMap((message) => ("text" in message ? [message.text] : []))).toEqual([
|
||||
"First response.",
|
||||
"Next request",
|
||||
"Second response.",
|
||||
])
|
||||
})
|
||||
|
||||
test("keeps local sound activity out of transcript history", () => {
|
||||
const started = transitionVoiceView(initialVoiceView(), { type: "user.started" })
|
||||
expect(started.messages).toEqual([])
|
||||
const committed = transitionVoiceView(started, { type: "user.committed", itemID: "user-1" })
|
||||
expect(committed.messages).toEqual([
|
||||
{ key: "message-1", kind: "user", itemID: "user-1", transcribing: true, reveals: [] },
|
||||
])
|
||||
|
||||
const abandoned = apply(initialVoiceView(), { type: "user.started" }, { type: "user.reset" })
|
||||
expect(abandoned.messages).toEqual([])
|
||||
})
|
||||
|
||||
test("deduplicates transcript snapshots and clears completed reveals", () => {
|
||||
const state = apply(
|
||||
initialVoiceView(),
|
||||
{ type: "user.committed", itemID: "user-1" },
|
||||
{ type: "user.transcript", itemID: "user-1", text: "Hello", final: false, now: 0, animate: true },
|
||||
{ type: "user.transcript", itemID: "user-1", text: "Hello", final: false, now: 10, animate: true },
|
||||
)
|
||||
expect(state.messages).toHaveLength(1)
|
||||
expect(state.messages[0]?.kind === "user" ? state.messages[0].reveals.length : 0).toBeGreaterThan(0)
|
||||
expect(transitionVoiceView(state, { type: "reveals.completed" }).messages).toEqual([
|
||||
{ key: "message-1", kind: "user", itemID: "user-1", text: "Hello", transcribing: true, reveals: [] },
|
||||
])
|
||||
})
|
||||
|
||||
test("bounds message history", () => {
|
||||
const state = Array.from({ length: 205 }, (_, index) => index).reduce(
|
||||
(current, index) => transitionVoiceView(current, { type: "meta", text: String(index) }),
|
||||
initialVoiceView(),
|
||||
)
|
||||
expect(state.messages).toHaveLength(200)
|
||||
expect(state.messages[0]).toEqual({ key: "message-6", kind: "meta", text: "5" })
|
||||
})
|
||||
|
||||
test("settles tool completion before or after insertion", () => {
|
||||
const inserted = apply(
|
||||
initialVoiceView(),
|
||||
{ type: "tool.started", callID: "call-1", name: "read_session", input: {} },
|
||||
{ type: "tool.done", callID: "call-1", output: { status: "ok" } },
|
||||
)
|
||||
expect(inserted.messages[0]).toEqual({
|
||||
key: "message-1",
|
||||
kind: "tool",
|
||||
callID: "call-1",
|
||||
name: "read_session",
|
||||
input: {},
|
||||
output: { status: "ok" },
|
||||
})
|
||||
})
|
||||
})
|
||||
@@ -1,277 +0,0 @@
|
||||
import { describe, expect, test } from "bun:test"
|
||||
import { initialVoiceState, transitionVoice, type VoiceEvent, type VoiceState } from "../src/voice-coordinator"
|
||||
|
||||
const apply = (state: VoiceState, ...events: ReadonlyArray<VoiceEvent>) =>
|
||||
events.reduce(
|
||||
(result, event) => {
|
||||
const next = transitionVoice(result.state, event)
|
||||
return { state: next.state, commands: [...result.commands, ...next.commands] }
|
||||
},
|
||||
{ state, commands: [] as Array<ReturnType<typeof transitionVoice>["commands"][number]> },
|
||||
)
|
||||
|
||||
describe("VoiceCoordinator", () => {
|
||||
test("delivers one notification at a time when the server becomes idle", () => {
|
||||
const notification = { id: "notification-1", promptID: "prompt-1", text: "completed" }
|
||||
const waiting = apply(
|
||||
initialVoiceState("marin"),
|
||||
{ type: "connection.ready" },
|
||||
{ type: "user.committed" },
|
||||
{ type: "notification.queued", notification },
|
||||
)
|
||||
expect(waiting.commands).toEqual([])
|
||||
|
||||
expect(transitionVoice(waiting.state, { type: "assistant.done", awaitingWork: false })).toEqual({
|
||||
state: {
|
||||
...waiting.state,
|
||||
conversation: "waiting",
|
||||
assistant: "idle",
|
||||
notifications: [],
|
||||
delivery: { notification, phase: "sending" },
|
||||
},
|
||||
commands: [
|
||||
{ type: "assistant.finish", awaitingWork: false, notificationID: undefined },
|
||||
{ type: "notification.send", notification },
|
||||
],
|
||||
})
|
||||
})
|
||||
|
||||
test("keeps notifications queued while the user is speaking", () => {
|
||||
const notification = { id: "notification-1", text: "completed" }
|
||||
const result = apply(
|
||||
initialVoiceState("marin"),
|
||||
{ type: "connection.ready" },
|
||||
{ type: "user.started" },
|
||||
{ type: "notification.queued", notification },
|
||||
)
|
||||
expect(result.commands).toEqual([])
|
||||
expect(result.state.notifications).toEqual([notification])
|
||||
expect(transitionVoice(result.state, { type: "user.stopped" }).commands).toEqual([
|
||||
{ type: "notification.send", notification },
|
||||
])
|
||||
})
|
||||
|
||||
test("keeps notifications queued until active tool calls finish", () => {
|
||||
const notification = { id: "notification-1", text: "completed" }
|
||||
const result = apply(
|
||||
initialVoiceState("marin"),
|
||||
{ type: "connection.ready" },
|
||||
{ type: "tool.started", id: "call-1" },
|
||||
{ type: "assistant.done", awaitingWork: false },
|
||||
{ type: "notification.queued", notification },
|
||||
)
|
||||
expect(result.commands).toEqual([
|
||||
{ type: "assistant.finish", awaitingWork: false, notificationID: undefined },
|
||||
])
|
||||
expect(transitionVoice(result.state, { type: "tool.finished", id: "call-1" }).commands).toEqual([
|
||||
{ type: "notification.send", notification },
|
||||
])
|
||||
})
|
||||
|
||||
test("tracks tools by call ID and reconnects only after work and conversation settle", () => {
|
||||
const busy = apply(
|
||||
initialVoiceState("marin"),
|
||||
{ type: "connection.ready" },
|
||||
{ type: "tool.started", id: "call-1" },
|
||||
{ type: "voice.selected", voice: "cedar" },
|
||||
)
|
||||
expect(busy.commands).toEqual([])
|
||||
expect(transitionVoice(busy.state, { type: "tool.finished", id: "call-1" }).commands).toEqual([])
|
||||
|
||||
const idle = transitionVoice(transitionVoice(busy.state, { type: "tool.finished", id: "call-1" }).state, {
|
||||
type: "assistant.done",
|
||||
awaitingWork: false,
|
||||
})
|
||||
expect(idle.commands).toEqual([
|
||||
{ type: "assistant.finish", awaitingWork: false, notificationID: undefined },
|
||||
{ type: "connection.reconnect" },
|
||||
])
|
||||
expect(idle.state.connection).toBe("connecting")
|
||||
})
|
||||
|
||||
test("a user commit closes assistant suppression and establishes a waiting boundary", () => {
|
||||
const suppressed = apply(
|
||||
initialVoiceState("marin"),
|
||||
{ type: "connection.ready" },
|
||||
{ type: "assistant.started" },
|
||||
{ type: "assistant.suppressed" },
|
||||
{ type: "user.committed" },
|
||||
)
|
||||
expect(suppressed.state.assistant).toBe("idle")
|
||||
expect(suppressed.state.conversation).toBe("waiting")
|
||||
})
|
||||
|
||||
test("requeues a notification when the adapter rejects it", () => {
|
||||
const notification = { id: "notification-1", text: "completed" }
|
||||
const sent = apply(
|
||||
initialVoiceState("marin"),
|
||||
{ type: "connection.ready" },
|
||||
{ type: "notification.queued", notification },
|
||||
)
|
||||
const failed = transitionVoice(sent.state, { type: "notification.failed", id: notification.id })
|
||||
expect(failed.state.notifications).toEqual([notification])
|
||||
expect(failed.state.connection).toBe("connecting")
|
||||
expect(failed.commands).toEqual([{ type: "connection.reconnect" }])
|
||||
})
|
||||
|
||||
test("keeps a notification in flight until its playback completes", () => {
|
||||
const notification = { id: "notification-1", promptID: "prompt-1", text: "completed" }
|
||||
const sent = apply(
|
||||
initialVoiceState("marin"),
|
||||
{ type: "connection.ready" },
|
||||
{
|
||||
type: "notification.queued",
|
||||
notification,
|
||||
},
|
||||
)
|
||||
expect(sent.state.delivery).toEqual({ notification, phase: "sending" })
|
||||
|
||||
const accepted = transitionVoice(sent.state, { type: "notification.accepted", id: notification.id }).state
|
||||
expect(accepted.delivery).toEqual({ notification, phase: "accepted" })
|
||||
const speaking = transitionVoice(accepted, { type: "assistant.started" }).state
|
||||
expect(speaking.delivery).toEqual({ notification, phase: "announcing" })
|
||||
const generated = transitionVoice(speaking, { type: "assistant.done", awaitingWork: false })
|
||||
expect(generated.state.delivery).toEqual({ notification, phase: "awaiting-playback" })
|
||||
expect(generated.commands).toEqual([
|
||||
{ type: "assistant.finish", awaitingWork: false, notificationID: notification.id },
|
||||
])
|
||||
expect(transitionVoice(generated.state, { type: "notification.announced", id: notification.id })).toEqual({
|
||||
state: { ...generated.state, delivery: undefined },
|
||||
commands: [{ type: "notification.delivered", notification }],
|
||||
})
|
||||
})
|
||||
|
||||
test("does not attribute an already-active assistant turn to a newly appended notification", () => {
|
||||
const notification = { id: "notification-1", text: "completed" }
|
||||
const active = apply(
|
||||
initialVoiceState("marin"),
|
||||
{ type: "connection.ready" },
|
||||
{ type: "notification.queued", notification },
|
||||
{ type: "assistant.started" },
|
||||
{ type: "notification.accepted", id: notification.id },
|
||||
).state
|
||||
|
||||
expect(active.delivery).toEqual({ notification, phase: "accepted" })
|
||||
const existingDone = transitionVoice(active, { type: "assistant.done", awaitingWork: false })
|
||||
expect(existingDone.state.delivery).toEqual({ notification, phase: "accepted" })
|
||||
expect(existingDone.commands).toEqual([
|
||||
{ type: "assistant.finish", awaitingWork: false, notificationID: undefined },
|
||||
])
|
||||
expect(transitionVoice(existingDone.state, { type: "assistant.started" }).state.delivery).toEqual({
|
||||
notification,
|
||||
phase: "announcing",
|
||||
})
|
||||
})
|
||||
|
||||
test("keeps assistant activity independent from a full-duplex user commit", () => {
|
||||
const active = apply(initialVoiceState("marin"), { type: "connection.ready" }, { type: "assistant.started" }).state
|
||||
const committed = transitionVoice(active, { type: "user.committed" }).state
|
||||
expect(committed.assistant).toBe("active")
|
||||
expect(committed.conversation).toBe("responding")
|
||||
})
|
||||
|
||||
test("ignores stale notification callbacks", () => {
|
||||
const notification = { id: "notification-2", text: "current" }
|
||||
const sent = apply(
|
||||
initialVoiceState("marin"),
|
||||
{ type: "connection.ready" },
|
||||
{ type: "notification.queued", notification },
|
||||
).state
|
||||
|
||||
expect(transitionVoice(sent, { type: "notification.accepted", id: "notification-1" })).toEqual({
|
||||
state: sent,
|
||||
commands: [],
|
||||
})
|
||||
expect(transitionVoice(sent, { type: "notification.failed", id: "notification-1" })).toEqual({
|
||||
state: sent,
|
||||
commands: [],
|
||||
})
|
||||
expect(transitionVoice(sent, { type: "notification.announced", id: "notification-1" })).toEqual({
|
||||
state: sent,
|
||||
commands: [],
|
||||
})
|
||||
})
|
||||
|
||||
test("waits for one announcement playback before sending the next", () => {
|
||||
const first = { id: "notification-1", text: "first" }
|
||||
const second = { id: "notification-2", text: "second" }
|
||||
const sent = apply(
|
||||
initialVoiceState("marin"),
|
||||
{ type: "connection.ready" },
|
||||
{ type: "notification.queued", notification: first },
|
||||
{ type: "notification.queued", notification: second },
|
||||
{ type: "notification.accepted", id: first.id },
|
||||
{ type: "assistant.started" },
|
||||
{ type: "assistant.done", awaitingWork: false },
|
||||
)
|
||||
expect(sent.state.delivery).toEqual({ notification: first, phase: "awaiting-playback" })
|
||||
expect(sent.state.notifications).toEqual([second])
|
||||
|
||||
expect(transitionVoice(sent.state, { type: "notification.announced", id: first.id })).toEqual({
|
||||
state: {
|
||||
...sent.state,
|
||||
conversation: "waiting",
|
||||
delivery: { notification: second, phase: "sending" },
|
||||
notifications: [],
|
||||
},
|
||||
commands: [
|
||||
{ type: "notification.delivered", notification: first },
|
||||
{ type: "notification.send", notification: second },
|
||||
],
|
||||
})
|
||||
})
|
||||
|
||||
test("keeps an announcement active across tool work", () => {
|
||||
const notification = { id: "notification-1", text: "completed" }
|
||||
const announcing = apply(
|
||||
initialVoiceState("marin"),
|
||||
{ type: "connection.ready" },
|
||||
{ type: "notification.queued", notification },
|
||||
{ type: "notification.accepted", id: notification.id },
|
||||
{ type: "assistant.started" },
|
||||
).state
|
||||
expect(transitionVoice(announcing, { type: "assistant.done", awaitingWork: true }).state.delivery).toEqual({
|
||||
notification,
|
||||
phase: "announcing",
|
||||
})
|
||||
})
|
||||
|
||||
test("requeues an interrupted announcement and reconnects before retrying", () => {
|
||||
const notification = { id: "notification-1", text: "completed" }
|
||||
const awaitingPlayback = apply(
|
||||
initialVoiceState("marin"),
|
||||
{ type: "connection.ready" },
|
||||
{ type: "notification.queued", notification },
|
||||
{ type: "notification.accepted", id: notification.id },
|
||||
{ type: "assistant.started" },
|
||||
{ type: "assistant.done", awaitingWork: false },
|
||||
).state
|
||||
const interrupted = transitionVoice(awaitingPlayback, {
|
||||
type: "notification.interrupted",
|
||||
id: notification.id,
|
||||
})
|
||||
expect(interrupted.state.connection).toBe("connecting")
|
||||
expect(interrupted.state.notifications).toEqual([notification])
|
||||
expect(interrupted.commands).toEqual([{ type: "connection.reconnect" }])
|
||||
})
|
||||
|
||||
test("holds an interrupted announcement reconnect until barge-in speech stops", () => {
|
||||
const notification = { id: "notification-1", text: "completed" }
|
||||
const announcing = apply(
|
||||
initialVoiceState("marin"),
|
||||
{ type: "connection.ready" },
|
||||
{ type: "notification.queued", notification },
|
||||
{ type: "notification.accepted", id: notification.id },
|
||||
{ type: "assistant.started" },
|
||||
).state
|
||||
const barged = transitionVoice(announcing, { type: "user.started", bargeIn: true })
|
||||
expect(barged.state.notifications).toEqual([notification])
|
||||
expect(barged.state.userSpeaking).toBe(true)
|
||||
expect(barged.state.connection).toBe("ready")
|
||||
expect(barged.commands).toEqual([{ type: "assistant.interrupt" }])
|
||||
|
||||
expect(transitionVoice(barged.state, { type: "user.stopped" }).commands).toEqual([
|
||||
{ type: "connection.reconnect" },
|
||||
])
|
||||
})
|
||||
})
|
||||
@@ -1,233 +0,0 @@
|
||||
import { expect, mock, test } from "bun:test"
|
||||
import type { AudioSession } from "../src/audio-session"
|
||||
import type { VoiceConnection, VoiceProtocol, VoiceProtocolEvent, VoiceProtocolOptions } from "../src/protocol"
|
||||
import type { VoiceUI } from "../src/ui"
|
||||
import { createVoiceSession } from "../src/voice-session"
|
||||
|
||||
test("ignores events from a replaced connection", async () => {
|
||||
const setup = makeSession()
|
||||
setup.session.start()
|
||||
setup.connections[0].emit({ type: "ready" })
|
||||
setup.session.cycleVoice()
|
||||
await Bun.sleep(0)
|
||||
|
||||
expect(setup.connections).toHaveLength(2)
|
||||
setup.connections[0].emit({ type: "assistant.transcript.delta", delta: "stale" })
|
||||
setup.connections[1].emit({ type: "assistant.transcript.delta", delta: "current" })
|
||||
expect(setup.assistantText).toEqual(["current"])
|
||||
})
|
||||
|
||||
test("resolves work against the connection that requested it", async () => {
|
||||
const setup = makeSession()
|
||||
setup.session.start()
|
||||
setup.connections[0].emit({ type: "ready" })
|
||||
setup.connections[0].emit({ type: "tool.started", id: "call-1", name: "find_sessions" })
|
||||
setup.connections[0].emit({
|
||||
type: "work.requested",
|
||||
request: { id: "call-1", name: "find_sessions", input: { query: null } },
|
||||
})
|
||||
await Bun.sleep(0)
|
||||
|
||||
expect(setup.resolvedWork).toEqual([
|
||||
{
|
||||
request: { id: "call-1", name: "find_sessions", input: { query: null } },
|
||||
output: { status: "ok" },
|
||||
},
|
||||
])
|
||||
})
|
||||
|
||||
test("acknowledges a durable notification only after playback", async () => {
|
||||
const setup = makeSession()
|
||||
setup.session.start()
|
||||
setup.connections[0].emit({ type: "ready" })
|
||||
setup.session.queueNotification({
|
||||
notification: {
|
||||
type: "opencode.prompt.completed",
|
||||
session_id: "session-1",
|
||||
prompt_id: "prompt-1",
|
||||
status: "completed",
|
||||
text: "done",
|
||||
},
|
||||
receipt: { sessionID: "session-1", promptID: "prompt-1" },
|
||||
})
|
||||
await Bun.sleep(0)
|
||||
|
||||
setup.connections[0].emit({ type: "assistant.transcript.delta", delta: "Finished." })
|
||||
setup.connections[0].emit({ type: "assistant.done", awaitingWork: false })
|
||||
expect(setup.delivered).toEqual([])
|
||||
|
||||
setup.finishPlayback?.("played")
|
||||
await Bun.sleep(0)
|
||||
expect(setup.delivered).toEqual(["prompt-1"])
|
||||
})
|
||||
|
||||
test("retries a notification whose playback was interrupted", async () => {
|
||||
const setup = makeSession()
|
||||
setup.session.start()
|
||||
setup.connections[0].emit({ type: "ready" })
|
||||
setup.session.queueNotification({
|
||||
notification: {
|
||||
type: "opencode.prompt.completed",
|
||||
session_id: "session-1",
|
||||
prompt_id: "prompt-1",
|
||||
status: "completed",
|
||||
text: "done",
|
||||
},
|
||||
receipt: { sessionID: "session-1", promptID: "prompt-1" },
|
||||
})
|
||||
await Bun.sleep(0)
|
||||
setup.connections[0].emit({ type: "assistant.transcript.delta", delta: "Finished." })
|
||||
setup.connections[0].emit({ type: "assistant.done", awaitingWork: false })
|
||||
setup.finishPlayback?.("interrupted")
|
||||
await Bun.sleep(0)
|
||||
|
||||
expect(setup.delivered).toEqual([])
|
||||
expect(setup.connections).toHaveLength(2)
|
||||
setup.connections[1].emit({ type: "ready" })
|
||||
await Bun.sleep(0)
|
||||
expect(setup.connections[1].notifyCalls()).toBe(1)
|
||||
})
|
||||
|
||||
test("closes owned resources once", async () => {
|
||||
const setup = makeSession()
|
||||
setup.session.start()
|
||||
await Promise.all([setup.session.close(), setup.session.close()])
|
||||
expect(setup.audioClose).toHaveBeenCalledTimes(1)
|
||||
expect(setup.toolClose).toHaveBeenCalledTimes(1)
|
||||
expect(setup.connections[0].closeCalls()).toBe(1)
|
||||
})
|
||||
|
||||
test("does not acknowledge nonterminal prompt notifications", async () => {
|
||||
const setup = makeSession()
|
||||
setup.session.start()
|
||||
setup.connections[0].emit({ type: "ready" })
|
||||
setup.session.queueNotification({
|
||||
notification: {
|
||||
type: "opencode.prompt.blocked",
|
||||
prompt_id: "prompt-1",
|
||||
blocker: "permission",
|
||||
session_id: "session-1",
|
||||
request_id: "permission-1",
|
||||
action: "read",
|
||||
resources: ["file.ts"],
|
||||
},
|
||||
})
|
||||
await Bun.sleep(0)
|
||||
setup.connections[0].emit({ type: "assistant.transcript.delta", delta: "Permission needed." })
|
||||
setup.connections[0].emit({ type: "assistant.done", awaitingWork: false })
|
||||
setup.finishPlayback?.("played")
|
||||
await Bun.sleep(0)
|
||||
|
||||
expect(setup.delivered).toEqual([])
|
||||
})
|
||||
|
||||
function makeSession() {
|
||||
const connections: Array<
|
||||
VoiceConnection & { emit(event: VoiceProtocolEvent): void; notifyCalls(): number; closeCalls(): number }
|
||||
> = []
|
||||
const assistantText: string[] = []
|
||||
const resolvedWork: Array<unknown> = []
|
||||
const delivered: string[] = []
|
||||
const audioClose = mock(() => {})
|
||||
const toolClose = mock(async () => {})
|
||||
let finishPlayback: ((outcome: "played" | "interrupted" | "inaudible") => void) | undefined
|
||||
const ui = {
|
||||
meta: mock(() => {}),
|
||||
userSpeaking: mock(() => {}),
|
||||
userReset: mock(() => {}),
|
||||
userAudioLevel: mock(() => {}),
|
||||
userCommitted: mock(() => {}),
|
||||
userTranscript: mock(() => {}),
|
||||
assistantAudio: mock(() => {}),
|
||||
assistantPlaybackStopped: mock(() => {}),
|
||||
assistantDelta: (text: string) => assistantText.push(text),
|
||||
assistantTranscript: mock(() => {}),
|
||||
assistantDone: mock(() => {}),
|
||||
toolStart: mock(() => {}),
|
||||
toolDone: mock(() => {}),
|
||||
setStatus: mock(() => {}),
|
||||
close: mock(() => {}),
|
||||
} satisfies VoiceUI
|
||||
const audio = {
|
||||
fullDuplex: false,
|
||||
microphoneMuted: false,
|
||||
speakerMuted: false,
|
||||
start: mock(async () => {}),
|
||||
toggleMicrophone: mock(() => false),
|
||||
toggleSpeaker: mock(() => false),
|
||||
isPlaying: () => false,
|
||||
play: mock(() => {}),
|
||||
finishPlayback: () => {
|
||||
const result = Promise.withResolvers<"played" | "interrupted" | "inaudible">()
|
||||
finishPlayback = result.resolve
|
||||
return result.promise
|
||||
},
|
||||
flushPlayback: mock(() => {}),
|
||||
noteUserCommitted: mock(() => {}),
|
||||
noteUserTranscript: mock(() => {}),
|
||||
close: audioClose,
|
||||
} satisfies AudioSession
|
||||
const protocol = {
|
||||
name: "realtime",
|
||||
inputActivity: "server",
|
||||
capabilities: { textInput: true, interruption: true, delegation: false },
|
||||
connect(options: VoiceProtocolOptions) {
|
||||
const notify = mock(async () => true)
|
||||
const close = mock(async () => {})
|
||||
const connection = {
|
||||
appendAudio: mock(() => {}),
|
||||
sendText: mock(() => {}),
|
||||
resolveWork: (request, output) => resolvedWork.push({ request, output }),
|
||||
resolveDelegation: mock(() => {}),
|
||||
notify,
|
||||
interrupt: () => true,
|
||||
close,
|
||||
emit: options.onEvent,
|
||||
notifyCalls: () => notify.mock.calls.length,
|
||||
closeCalls: () => close.mock.calls.length,
|
||||
} satisfies VoiceConnection & {
|
||||
emit(event: VoiceProtocolEvent): void
|
||||
notifyCalls(): number
|
||||
closeCalls(): number
|
||||
}
|
||||
connections.push(connection)
|
||||
return connection
|
||||
},
|
||||
} satisfies VoiceProtocol
|
||||
const session = createVoiceSession({
|
||||
protocol,
|
||||
connection: {
|
||||
apiKey: "test",
|
||||
model: "voice-test",
|
||||
instructions: "test",
|
||||
tools: [],
|
||||
fullDuplex: false,
|
||||
debug: false,
|
||||
},
|
||||
initialVoice: "marin",
|
||||
voices: ["marin", "cedar"],
|
||||
ui,
|
||||
audio,
|
||||
tools: {
|
||||
execute: async () => ({ output: { status: "ok" } }),
|
||||
acknowledge: async (receipt) => {
|
||||
delivered.push(receipt.promptID)
|
||||
},
|
||||
close: toolClose,
|
||||
},
|
||||
delegate: async () => "done",
|
||||
onClosed: () => {},
|
||||
})
|
||||
return {
|
||||
session,
|
||||
connections,
|
||||
assistantText,
|
||||
resolvedWork,
|
||||
delivered,
|
||||
audioClose,
|
||||
toolClose,
|
||||
get finishPlayback() {
|
||||
return finishPlayback
|
||||
},
|
||||
}
|
||||
}
|
||||
@@ -1,11 +0,0 @@
|
||||
{
|
||||
"$schema": "https://json.schemastore.org/tsconfig",
|
||||
"extends": "@tsconfig/bun/tsconfig.json",
|
||||
"compilerOptions": {
|
||||
"jsx": "preserve",
|
||||
"jsxImportSource": "@opentui/solid",
|
||||
"noUncheckedIndexedAccess": false,
|
||||
"noUnusedLocals": true
|
||||
},
|
||||
"include": ["src", "test"]
|
||||
}
|
||||
Reference in New Issue
Block a user