From a44946dab8a8f6e55f10d43d4c8fc76d2837931d Mon Sep 17 00:00:00 2001 From: Andre Brait Date: Sat, 19 Sep 2026 19:05:54 +0200 Subject: [PATCH] feat(dictation): recording deck for voice dictation (timer, waveform, pause, retry, send modes) (#109) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * feat(dictation): recording deck with timer, waveform, pause, retry and send modes * fix(dictation): remove stray pulse ring and let waveform shrink on narrow composers * test(systemd): scrub ambient OMP_WEB_OMP_BIN so install test is hermetic * feat(dictation): mic becomes cancel while recording, composer Send transcribes+queues, deck keeps only pause and stop * fix(dictation): apply review findings — theme tokens, audio lifecycle, attachment staleness, dynamic waveform width - Use the theme's --status-error token instead of undefined --danger custom properties so dictation colors follow light/dark themes. - Close the AudioContext and release the analyser when capture finishes, not only on cleanup, so transcription no longer holds the mic graph. - Read attachments through refs inside handleSend/sendQueued so files attached mid-recording are included when the transcript dispatches. - Surface 'No speech detected' when a capture yields no audio instead of failing silently. - Stretch the waveform canvas to the free composer width (dynamic bar count, newest samples at the buttons) and freeze it gray while paused. - Make the transcribe-and-send wiring test assert the contiguous branch. - Drop the orphaned chatInput.stopDictation i18n key. * fix(dictation): address coderabbit findings — audio context ownership, state-gated finish, enter passthrough * feat(dictation): play recording before transcribe Stop enters review with playback instead of sending immediately. Paused capture keeps a left-side preview play button. * feat(dictation): fix audio playback lifecycle, timeslice, and replay reset * fix(dictation): address review findings (timer corruption, audio leak, icon, test honesty) * fix(dictation): restore setIsRecording(true) on capture start and add behavioral test suite * fix(dictation): allow blob: media in CSP so preview playback works; mic icon for resume * fix(dictation): draw recorded bars and playback progress in review-mode waveform * fix(dictation): stop replaced preview audio; neutral confirm label * chore(ci): retrigger windows verify after flaky MessageView copy test --- CHANGELOG.md | 1 + app/globals.css | 16 + components/ChatInput.tsx | 310 ++++++++++++------ components/RecordingDeck.tsx | 343 ++++++++++++++++++++ hooks/useDictation.behavior.test.mjs | 274 ++++++++++++++++ hooks/useDictation.test.mjs | 106 ++++++- hooks/useDictation.ts | 459 ++++++++++++++++++++++----- lib/i18n/locales/en.json | 14 +- lib/i18n/locales/ja.json | 14 +- lib/i18n/locales/zh-CN.json | 14 +- next.config.ts | 2 +- 11 files changed, 1367 insertions(+), 186 deletions(-) create mode 100644 components/RecordingDeck.tsx create mode 100644 hooks/useDictation.behavior.test.mjs diff --git a/CHANGELOG.md b/CHANGELOG.md index 99420f9..5c35cd4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,6 +11,7 @@ All notable changes to **omp-web** (`@kahme247/ompweb`) are documented in this f - Add **Copy** and **Copy as Markdown** below user messages and completed assistant replies, with keyboard access and touch-sized controls. Copy only message text, excluding thinking, tool output, and renderer controls; preserve full source for oversized raw-text messages. - Scope Ctrl+A / Cmd+A to the selected message, currently loaded chat, or active file contents instead of the whole page. Message selection includes collapsed extension previews and expanded details without toolbar labels. Newer pane focus takes precedence over retained child selections. Text fields and IME composition retain native behavior; browser-menu commands and embedded viewers remain browser-controlled. - Add an off-by-default **Scope native Select All (experimental)** switch in Settings → Interface & Behavior. The per-browser preference narrows whole-page selections from native menus while leaving keyboard scoping independent. Disable it if browser selection handles or menus behave unexpectedly; intentional whole-page selections can also be narrowed. +- Play back a voice recording before transcribing or sending it. Pause keeps a left-side preview control; Stop opens a review deck with play, discard, and transcribe-and-send. ### Fixes & Improvements diff --git a/app/globals.css b/app/globals.css index fcd03f0..39f7b21 100644 --- a/app/globals.css +++ b/app/globals.css @@ -3584,3 +3584,19 @@ html.dark .dropdown-surface :is(select, option) { padding-bottom: max(10px, env(safe-area-inset-bottom)) !important; } } + +/* Dictation deck: indeterminate transcription progress. */ +@keyframes dictation-indeterminate { + 0% { transform: translateX(-100%); } + 100% { transform: translateX(350%); } +} +.dictation-indeterminate { + animation: dictation-indeterminate 1.2s ease-in-out infinite; +} +@media (prefers-reduced-motion: reduce) { + .dictation-indeterminate { + animation: none; + left: 30%; + width: 40% !important; + } +} diff --git a/components/ChatInput.tsx b/components/ChatInput.tsx index 60c198c..c0f58ad 100644 --- a/components/ChatInput.tsx +++ b/components/ChatInput.tsx @@ -1,7 +1,7 @@ "use client"; import React, { useRef, useState, useCallback, useEffect, useLayoutEffect, useImperativeHandle, forwardRef, memo, KeyboardEvent } from "react"; -import { ChevronDown, ListChecks, Loader2, Mic, Paperclip, Plus, Shrink, Sparkles, Wrench, Zap } from "lucide-react"; +import { ChevronDown, ListChecks, Loader2, Mic, Paperclip, Plus, Shrink, Sparkles, Wrench, X, Zap } from "lucide-react"; import { getSubmitDuringRunBehavior } from "@/lib/composer-prefs"; import type { BuiltinSlashCommandResult, CompactResultInfo, QueuedMessages, SlashCommandInfo } from "@/hooks/useAgentSession"; import type { ActiveGoal, ActivePlan } from "@/lib/web-mode-state"; @@ -11,6 +11,7 @@ import { useDictation } from "@/hooks/useDictation"; import type { GenerationSpeedInfo, SessionStatsInfo } from "@/lib/pi-types"; import { formatCompactNumber, formatPercent } from "@/lib/format"; import { ContextDetailPanel } from "./ComposerPanels"; +import { RecordingDeck } from "./RecordingDeck"; import { clearDraft, getDraft, setDraft } from "@/lib/draft-store"; import { expandWebSlashCommand } from "@/lib/web-slash-commands"; import type { AttachedImage, AttachedTextFile } from "./ChatInput-draft-attachments"; @@ -359,22 +360,6 @@ export const ChatInput = memo(forwardRef(function ChatIn }); }, []); - const { isRecording, isTranscribing, toggle: toggleDictation, cancel: cancelDictation, stop: stopDictation } = useDictation({ - onTranscript: insertTextAtCursor, - onError: (err) => { - const msg = - err === "Microphone not supported in this browser or context" - ? t("chatInput.dictationNotSupported") - : err === "Microphone access denied" - ? t("chatInput.dictationPermissionDenied") - : err === "No speech detected" - ? t("chatInput.dictationNoSpeech") - : err === "Transcription failed" - ? t("chatInput.dictationFailed") - : err; - toast.error(msg); - }, - }); useImperativeHandle(ref, () => ({ focus() { @@ -663,28 +648,128 @@ export const ChatInput = memo(forwardRef(function ChatIn return error !== null; }, []); - const handleSend = useCallback(async () => { - const msg = value.trim(); - if (!msg && !attachedImages.length && !attachedTextFiles.length) return; + const handleSend = useCallback(async (overrideText?: string) => { + const raw = overrideText ?? value; + const msg = raw.trim(); + if (!msg && !attachedImagesRef.current.length && !attachedTextFilesRef.current.length) return; if (isStreaming) return; onAudioUnlock?.(); - const composedMessage = composeMessageWithTextAttachments(msg, attachedTextFiles); - if (!attachedImages.length && !attachedTextFiles.length && msg.startsWith("/") && onBuiltinCommand) { + const composedMessage = composeMessageWithTextAttachments(msg, attachedTextFilesRef.current); + if (!attachedImagesRef.current.length && !attachedTextFilesRef.current.length && msg.startsWith("/") && onBuiltinCommand) { const expansion = expandWebSlashCommand(msg); - if (expansion.kind === "expand" && rejectsOversizedPrompt(expansion.prompt, attachedImages)) return; - const sentValue = value; + if (expansion.kind === "expand" && rejectsOversizedPrompt(expansion.prompt, attachedImagesRef.current)) return; + const sentValue = overrideText ?? value; const result = await onBuiltinCommand(msg); if (result.handled) { // The user may have started typing while the command ran; only clear // if the composer still holds what was sent. - if (!result.error && !result.retainInput && valueRef.current === sentValue) clearInput(); + if (!result.error && !result.retainInput && (overrideText !== undefined || valueRef.current === sentValue)) clearInput(); return; } } - if (rejectsOversizedPrompt(composedMessage, attachedImages)) return; - onSend(composedMessage, attachedImages.length ? attachedImages : undefined); + if (rejectsOversizedPrompt(composedMessage, attachedImagesRef.current)) return; + onSend(composedMessage, attachedImagesRef.current.length ? attachedImagesRef.current : undefined); clearInput(); - }, [value, attachedImages, attachedTextFiles, isStreaming, onBuiltinCommand, onSend, clearInput, onAudioUnlock, rejectsOversizedPrompt]); + }, [value, isStreaming, onBuiltinCommand, onSend, clearInput, onAudioUnlock, rejectsOversizedPrompt]); + /** What happens to the composer after the transcript lands: null inserts it + * for editing; "send" dispatches immediately; "steer"/"followup" queue it + * into the running agent. */ + type DictationAfterMode = "send" | "steer" | "followup"; + const dictationAfterRef = useRef(null); + const { + isRecording, + isPaused, + isReviewing, + isTranscribing, + isPlayingPreview, + previewCurrentTime, + previewDuration, + transcribeError, + captureRef, + toggle: toggleDictation, + cancel: cancelDictation, + stop: stopDictation, + togglePause: togglePauseDictation, + retry: retryDictation, + playPreview: playPreviewDictation, + pausePreview: pausePreviewDictation, + seekPreview: seekPreviewDictation, + confirmTranscribe: confirmTranscribeDictation, + } = useDictation({ + onTranscript: (text) => { + const after = dictationAfterRef.current; + dictationAfterRef.current = null; + const base = valueRef.current; + const sep = base.length > 0 && !base.endsWith(" ") ? " " : ""; + const finalText = base + sep + text; + insertTextAtCursor(text); + if (after === "send") { + void handleSend(finalText); + } else if (after === "steer" || after === "followup") { + sendQueued(after, finalText); + } else { + toast.success(t("chatInput.dictationSuccess")); + } + }, + onError: (err) => { + const msg = + err === "Microphone not supported in this browser or context" + ? t("chatInput.dictationNotSupported") + : err === "Microphone access denied" + ? t("chatInput.dictationPermissionDenied") + : err === "No speech detected" + ? t("chatInput.dictationNoSpeech") + : err === "Transcription timed out" + ? t("chatInput.dictationTimedOut") + : err === "Transcription failed" + ? t("chatInput.dictationFailed") + : err; + toast.error(msg); + }, + }); + const stopAndInsertDictation = useCallback(() => { + dictationAfterRef.current = null; + stopDictation(); + }, [stopDictation]); + const stopAndSendDictation = useCallback(() => { + dictationAfterRef.current = "send"; + stopDictation({ immediateSend: true }); + }, [stopDictation]); + const stopAndQueueDictation = useCallback((mode: "steer" | "followup") => { + dictationAfterRef.current = mode; + stopDictation({ immediateSend: true }); + }, [stopDictation]); + const cancelDictationAndReset = useCallback(() => { + dictationAfterRef.current = null; + cancelDictation(); + }, [cancelDictation]); + const startFreshDictation = useCallback(() => { + dictationAfterRef.current = null; + toggleDictation(); + }, [toggleDictation]); + // While the recording deck replaces the textarea there is no focused input, + // so Escape/Enter are handled at window level: Escape cancels/discard, Enter + // retries after an error or converts the recording to text. + useEffect(() => { + if (!(isRecording || isPaused || isReviewing || isTranscribing || transcribeError)) return; + const onKeyDown = (e: globalThis.KeyboardEvent) => { + if (e.key === "Escape") { + e.preventDefault(); + cancelDictationAndReset(); + } else if (e.key === "Enter" && !e.shiftKey && !isTranscribing) { + // Let focused deck/toolbar controls keep their own activation; + // only hijack Enter from the non-interactive page context. + const target = e.target as HTMLElement | null; + if (target && target.closest("button, input, textarea, select, a, [role='button']")) return; + e.preventDefault(); + if (transcribeError) retryDictation(); + else if (isReviewing) confirmTranscribeDictation(); + else stopAndInsertDictation(); + } + }; + window.addEventListener("keydown", onKeyDown); + return () => window.removeEventListener("keydown", onKeyDown); + }, [isRecording, isPaused, isReviewing, isTranscribing, transcribeError, cancelDictationAndReset, retryDictation, confirmTranscribeDictation, stopAndInsertDictation]); const slashQuery = value.startsWith("/") && !/\s/.test(value.slice(1)) ? value.slice(1).toLowerCase() @@ -964,10 +1049,11 @@ export const ChatInput = memo(forwardRef(function ChatIn }); }, []); - const sendQueued = useCallback((mode: "steer" | "followup") => { - const msg = value.trim(); - if (!msg && !attachedImages.length && !attachedTextFiles.length) return; - if (attachedImages.length || attachedTextFiles.length) return; + const sendQueued = useCallback((mode: "steer" | "followup", overrideText?: string) => { + const raw = overrideText ?? value; + const msg = raw.trim(); + if (!msg && !attachedImagesRef.current.length && !attachedTextFilesRef.current.length) return; + if (attachedImagesRef.current.length || attachedTextFilesRef.current.length) return; onAudioUnlock?.(); const streamingBehavior = mode === "steer" ? "steer" : "followUp"; if (msg.startsWith("/") && onPromptWithStreamingBehavior) { @@ -984,8 +1070,8 @@ export const ChatInput = memo(forwardRef(function ChatIn // own ACP handlers can run them. const expansion = expandWebSlashCommand(msg); if (expansion.kind === "expand") { - if (rejectsOversizedPrompt(expansion.prompt, attachedImages)) return; - onPromptWithStreamingBehavior(expansion.prompt, streamingBehavior, attachedImages.length ? attachedImages : undefined); + if (rejectsOversizedPrompt(expansion.prompt, attachedImagesRef.current)) return; + onPromptWithStreamingBehavior(expansion.prompt, streamingBehavior, attachedImagesRef.current.length ? attachedImagesRef.current : undefined); clearInput(); return; } @@ -996,24 +1082,27 @@ export const ChatInput = memo(forwardRef(function ChatIn })); return; } - if (rejectsOversizedPrompt(msg, attachedImages)) return; - onPromptWithStreamingBehavior(msg, streamingBehavior, attachedImages.length ? attachedImages : undefined); + if (rejectsOversizedPrompt(msg, attachedImagesRef.current)) return; + onPromptWithStreamingBehavior(msg, streamingBehavior, attachedImagesRef.current.length ? attachedImagesRef.current : undefined); clearInput(); return; } - if (rejectsOversizedPrompt(msg, attachedImages)) return; + if (rejectsOversizedPrompt(msg, attachedImagesRef.current)) return; if (mode === "steer" && onSteer) { - onSteer(msg, attachedImages.length ? attachedImages : undefined); + onSteer(msg, attachedImagesRef.current.length ? attachedImagesRef.current : undefined); } else if (mode === "followup" && onFollowUp) { - onFollowUp(msg, attachedImages.length ? attachedImages : undefined); + onFollowUp(msg, attachedImagesRef.current.length ? attachedImagesRef.current : undefined); } clearInput(); - }, [value, attachedImages, attachedTextFiles, onPromptWithStreamingBehavior, onSteer, onFollowUp, clearInput, onAudioUnlock, t, advisorEnabled, rejectsOversizedPrompt]); - // A typed, text-only message during a run is a queued follow-up. Keep Stop - // as the action while the composer is empty or contains attachments. + }, [value, onPromptWithStreamingBehavior, onSteer, onFollowUp, clearInput, onAudioUnlock, t, advisorEnabled, rejectsOversizedPrompt]); + // A typed, text-only message during a run is a queued follow-up — and so is + // a dictation in progress: the primary button must take the same state it + // would have if the composer already held text. Keep Stop as the action + // while the composer is empty and nothing is being recorded. + const dictationCapturing = isRecording || isPaused; const primaryActionQueuesMessage = isStreaming - && Boolean(value.trim()) + && (Boolean(value.trim()) || dictationCapturing) && attachedImages.length === 0 && attachedTextFiles.length === 0 && Boolean(onFollowUp); @@ -1130,21 +1219,9 @@ export const ChatInput = memo(forwardRef(function ChatIn if (recentlyComposed) e.preventDefault(); return; } - if (isRecording || isTranscribing) { - if (e.key === "Escape") { - e.preventDefault(); - cancelDictation(); - return; - } - if (isRecording && e.key === "Enter" && !e.shiftKey) { - e.preventDefault(); - stopDictation(); - return; - } - } if ((e.metaKey || e.ctrlKey) && !e.altKey && !e.shiftKey && e.key.toLowerCase() === "m") { e.preventDefault(); - toggleDictation(); + startFreshDictation(); return; } @@ -1268,7 +1345,7 @@ export const ChatInput = memo(forwardRef(function ChatIn } } }, - [isMobile, isStreaming, onSteer, onFollowUp, onAbort, onMinimize, slashMenuOpen, slashQuery, filteredSlashCommands, slashActiveIndex, applySlashCommand, sendQueued, handleSend, getNextSlashIndex, atMenuOpen, atQuery, atMatches, atActiveIndex, applyAtCompletion, historyMenuOpen, inputHistory, historyActiveIndex, applyHistoryInput, value, isRecording, isTranscribing, cancelDictation, stopDictation, toggleDictation] + [isMobile, isStreaming, onSteer, onFollowUp, onAbort, onMinimize, slashMenuOpen, slashQuery, filteredSlashCommands, slashActiveIndex, applySlashCommand, sendQueued, handleSend, getNextSlashIndex, atMenuOpen, atQuery, atMatches, atActiveIndex, applyAtCompletion, historyMenuOpen, inputHistory, historyActiveIndex, applyHistoryInput, value, startFreshDictation] ); const handleInput = useCallback(() => { @@ -2241,6 +2318,25 @@ export const ChatInput = memo(forwardRef(function ChatIn transition: "border-color var(--dur-fast) var(--ease-out-warm), background var(--dur-fast) var(--ease-out-warm), box-shadow var(--dur-fast) var(--ease-out-warm)", } as React.CSSProperties} > + {isRecording || isPaused || isReviewing || isTranscribing || transcribeError ? ( + + ) : (