mirror of
https://github.com/Heretek-AI/ompweb.git
synced 2026-11-07 10:56:00 -10:00
feat(dictation): recording deck for voice dictation (timer, waveform, pause, retry, send modes) (#109)
* feat(dictation): recording deck with timer, waveform, pause, retry and send modes * fix(dictation): remove stray pulse ring and let waveform shrink on narrow composers * test(systemd): scrub ambient OMP_WEB_OMP_BIN so install test is hermetic * feat(dictation): mic becomes cancel while recording, composer Send transcribes+queues, deck keeps only pause and stop * fix(dictation): apply review findings — theme tokens, audio lifecycle, attachment staleness, dynamic waveform width - Use the theme's --status-error token instead of undefined --danger custom properties so dictation colors follow light/dark themes. - Close the AudioContext and release the analyser when capture finishes, not only on cleanup, so transcription no longer holds the mic graph. - Read attachments through refs inside handleSend/sendQueued so files attached mid-recording are included when the transcript dispatches. - Surface 'No speech detected' when a capture yields no audio instead of failing silently. - Stretch the waveform canvas to the free composer width (dynamic bar count, newest samples at the buttons) and freeze it gray while paused. - Make the transcribe-and-send wiring test assert the contiguous branch. - Drop the orphaned chatInput.stopDictation i18n key. * fix(dictation): address coderabbit findings — audio context ownership, state-gated finish, enter passthrough * feat(dictation): play recording before transcribe Stop enters review with playback instead of sending immediately. Paused capture keeps a left-side preview play button. * feat(dictation): fix audio playback lifecycle, timeslice, and replay reset * fix(dictation): address review findings (timer corruption, audio leak, icon, test honesty) * fix(dictation): restore setIsRecording(true) on capture start and add behavioral test suite * fix(dictation): allow blob: media in CSP so preview playback works; mic icon for resume * fix(dictation): draw recorded bars and playback progress in review-mode waveform * fix(dictation): stop replaced preview audio; neutral confirm label * chore(ci): retrigger windows verify after flaky MessageView copy test
This commit is contained in:
Andre Brait authored and GitHub committed
1 parent
40295c8c75
commit
a44946dab8
11 files changed
+1367
-186
No files matched your search
@@ -11,6 +11,7 @@ All notable changes to **omp-web** (`@kahme247/ompweb`) are documented in this f
|
||||
- Add **Copy** and **Copy as Markdown** below user messages and completed assistant replies, with keyboard access and touch-sized controls. Copy only message text, excluding thinking, tool output, and renderer controls; preserve full source for oversized raw-text messages.
|
||||
- Scope Ctrl+A / Cmd+A to the selected message, currently loaded chat, or active file contents instead of the whole page. Message selection includes collapsed extension previews and expanded details without toolbar labels. Newer pane focus takes precedence over retained child selections. Text fields and IME composition retain native behavior; browser-menu commands and embedded viewers remain browser-controlled.
|
||||
- Add an off-by-default **Scope native Select All (experimental)** switch in Settings → Interface & Behavior. The per-browser preference narrows whole-page selections from native menus while leaving keyboard scoping independent. Disable it if browser selection handles or menus behave unexpectedly; intentional whole-page selections can also be narrowed.
|
||||
- Play back a voice recording before transcribing or sending it. Pause keeps a left-side preview control; Stop opens a review deck with play, discard, and transcribe-and-send.
|
||||
|
||||
### Fixes & Improvements
|
||||
|
||||
|
||||
@@ -3584,3 +3584,19 @@ html.dark .dropdown-surface :is(select, option) {
|
||||
padding-bottom: max(10px, env(safe-area-inset-bottom)) !important;
|
||||
}
|
||||
}
|
||||
|
||||
/* Dictation deck: indeterminate transcription progress. */
|
||||
@keyframes dictation-indeterminate {
|
||||
0% { transform: translateX(-100%); }
|
||||
100% { transform: translateX(350%); }
|
||||
}
|
||||
.dictation-indeterminate {
|
||||
animation: dictation-indeterminate 1.2s ease-in-out infinite;
|
||||
}
|
||||
@media (prefers-reduced-motion: reduce) {
|
||||
.dictation-indeterminate {
|
||||
animation: none;
|
||||
left: 30%;
|
||||
width: 40% !important;
|
||||
}
|
||||
}
|
||||
+218
-92
@@ -1,7 +1,7 @@
|
||||
"use client";
|
||||
|
||||
import React, { useRef, useState, useCallback, useEffect, useLayoutEffect, useImperativeHandle, forwardRef, memo, KeyboardEvent } from "react";
|
||||
import { ChevronDown, ListChecks, Loader2, Mic, Paperclip, Plus, Shrink, Sparkles, Wrench, Zap } from "lucide-react";
|
||||
import { ChevronDown, ListChecks, Loader2, Mic, Paperclip, Plus, Shrink, Sparkles, Wrench, X, Zap } from "lucide-react";
|
||||
import { getSubmitDuringRunBehavior } from "@/lib/composer-prefs";
|
||||
import type { BuiltinSlashCommandResult, CompactResultInfo, QueuedMessages, SlashCommandInfo } from "@/hooks/useAgentSession";
|
||||
import type { ActiveGoal, ActivePlan } from "@/lib/web-mode-state";
|
||||
@@ -11,6 +11,7 @@ import { useDictation } from "@/hooks/useDictation";
|
||||
import type { GenerationSpeedInfo, SessionStatsInfo } from "@/lib/pi-types";
|
||||
import { formatCompactNumber, formatPercent } from "@/lib/format";
|
||||
import { ContextDetailPanel } from "./ComposerPanels";
|
||||
import { RecordingDeck } from "./RecordingDeck";
|
||||
import { clearDraft, getDraft, setDraft } from "@/lib/draft-store";
|
||||
import { expandWebSlashCommand } from "@/lib/web-slash-commands";
|
||||
import type { AttachedImage, AttachedTextFile } from "./ChatInput-draft-attachments";
|
||||
@@ -359,22 +360,6 @@ export const ChatInput = memo(forwardRef<ChatInputHandle, Props>(function ChatIn
|
||||
});
|
||||
}, []);
|
||||
|
||||
const { isRecording, isTranscribing, toggle: toggleDictation, cancel: cancelDictation, stop: stopDictation } = useDictation({
|
||||
onTranscript: insertTextAtCursor,
|
||||
onError: (err) => {
|
||||
const msg =
|
||||
err === "Microphone not supported in this browser or context"
|
||||
? t("chatInput.dictationNotSupported")
|
||||
: err === "Microphone access denied"
|
||||
? t("chatInput.dictationPermissionDenied")
|
||||
: err === "No speech detected"
|
||||
? t("chatInput.dictationNoSpeech")
|
||||
: err === "Transcription failed"
|
||||
? t("chatInput.dictationFailed")
|
||||
: err;
|
||||
toast.error(msg);
|
||||
},
|
||||
});
|
||||
|
||||
useImperativeHandle(ref, () => ({
|
||||
focus() {
|
||||
@@ -663,28 +648,128 @@ export const ChatInput = memo(forwardRef<ChatInputHandle, Props>(function ChatIn
|
||||
return error !== null;
|
||||
}, []);
|
||||
|
||||
const handleSend = useCallback(async () => {
|
||||
const msg = value.trim();
|
||||
if (!msg && !attachedImages.length && !attachedTextFiles.length) return;
|
||||
const handleSend = useCallback(async (overrideText?: string) => {
|
||||
const raw = overrideText ?? value;
|
||||
const msg = raw.trim();
|
||||
if (!msg && !attachedImagesRef.current.length && !attachedTextFilesRef.current.length) return;
|
||||
if (isStreaming) return;
|
||||
onAudioUnlock?.();
|
||||
const composedMessage = composeMessageWithTextAttachments(msg, attachedTextFiles);
|
||||
if (!attachedImages.length && !attachedTextFiles.length && msg.startsWith("/") && onBuiltinCommand) {
|
||||
const composedMessage = composeMessageWithTextAttachments(msg, attachedTextFilesRef.current);
|
||||
if (!attachedImagesRef.current.length && !attachedTextFilesRef.current.length && msg.startsWith("/") && onBuiltinCommand) {
|
||||
const expansion = expandWebSlashCommand(msg);
|
||||
if (expansion.kind === "expand" && rejectsOversizedPrompt(expansion.prompt, attachedImages)) return;
|
||||
const sentValue = value;
|
||||
if (expansion.kind === "expand" && rejectsOversizedPrompt(expansion.prompt, attachedImagesRef.current)) return;
|
||||
const sentValue = overrideText ?? value;
|
||||
const result = await onBuiltinCommand(msg);
|
||||
if (result.handled) {
|
||||
// The user may have started typing while the command ran; only clear
|
||||
// if the composer still holds what was sent.
|
||||
if (!result.error && !result.retainInput && valueRef.current === sentValue) clearInput();
|
||||
if (!result.error && !result.retainInput && (overrideText !== undefined || valueRef.current === sentValue)) clearInput();
|
||||
return;
|
||||
}
|
||||
}
|
||||
if (rejectsOversizedPrompt(composedMessage, attachedImages)) return;
|
||||
onSend(composedMessage, attachedImages.length ? attachedImages : undefined);
|
||||
if (rejectsOversizedPrompt(composedMessage, attachedImagesRef.current)) return;
|
||||
onSend(composedMessage, attachedImagesRef.current.length ? attachedImagesRef.current : undefined);
|
||||
clearInput();
|
||||
}, [value, attachedImages, attachedTextFiles, isStreaming, onBuiltinCommand, onSend, clearInput, onAudioUnlock, rejectsOversizedPrompt]);
|
||||
}, [value, isStreaming, onBuiltinCommand, onSend, clearInput, onAudioUnlock, rejectsOversizedPrompt]);
|
||||
/** What happens to the composer after the transcript lands: null inserts it
|
||||
* for editing; "send" dispatches immediately; "steer"/"followup" queue it
|
||||
* into the running agent. */
|
||||
type DictationAfterMode = "send" | "steer" | "followup";
|
||||
const dictationAfterRef = useRef<DictationAfterMode | null>(null);
|
||||
const {
|
||||
isRecording,
|
||||
isPaused,
|
||||
isReviewing,
|
||||
isTranscribing,
|
||||
isPlayingPreview,
|
||||
previewCurrentTime,
|
||||
previewDuration,
|
||||
transcribeError,
|
||||
captureRef,
|
||||
toggle: toggleDictation,
|
||||
cancel: cancelDictation,
|
||||
stop: stopDictation,
|
||||
togglePause: togglePauseDictation,
|
||||
retry: retryDictation,
|
||||
playPreview: playPreviewDictation,
|
||||
pausePreview: pausePreviewDictation,
|
||||
seekPreview: seekPreviewDictation,
|
||||
confirmTranscribe: confirmTranscribeDictation,
|
||||
} = useDictation({
|
||||
onTranscript: (text) => {
|
||||
const after = dictationAfterRef.current;
|
||||
dictationAfterRef.current = null;
|
||||
const base = valueRef.current;
|
||||
const sep = base.length > 0 && !base.endsWith(" ") ? " " : "";
|
||||
const finalText = base + sep + text;
|
||||
insertTextAtCursor(text);
|
||||
if (after === "send") {
|
||||
void handleSend(finalText);
|
||||
} else if (after === "steer" || after === "followup") {
|
||||
sendQueued(after, finalText);
|
||||
} else {
|
||||
toast.success(t("chatInput.dictationSuccess"));
|
||||
}
|
||||
},
|
||||
onError: (err) => {
|
||||
const msg =
|
||||
err === "Microphone not supported in this browser or context"
|
||||
? t("chatInput.dictationNotSupported")
|
||||
: err === "Microphone access denied"
|
||||
? t("chatInput.dictationPermissionDenied")
|
||||
: err === "No speech detected"
|
||||
? t("chatInput.dictationNoSpeech")
|
||||
: err === "Transcription timed out"
|
||||
? t("chatInput.dictationTimedOut")
|
||||
: err === "Transcription failed"
|
||||
? t("chatInput.dictationFailed")
|
||||
: err;
|
||||
toast.error(msg);
|
||||
},
|
||||
});
|
||||
const stopAndInsertDictation = useCallback(() => {
|
||||
dictationAfterRef.current = null;
|
||||
stopDictation();
|
||||
}, [stopDictation]);
|
||||
const stopAndSendDictation = useCallback(() => {
|
||||
dictationAfterRef.current = "send";
|
||||
stopDictation({ immediateSend: true });
|
||||
}, [stopDictation]);
|
||||
const stopAndQueueDictation = useCallback((mode: "steer" | "followup") => {
|
||||
dictationAfterRef.current = mode;
|
||||
stopDictation({ immediateSend: true });
|
||||
}, [stopDictation]);
|
||||
const cancelDictationAndReset = useCallback(() => {
|
||||
dictationAfterRef.current = null;
|
||||
cancelDictation();
|
||||
}, [cancelDictation]);
|
||||
const startFreshDictation = useCallback(() => {
|
||||
dictationAfterRef.current = null;
|
||||
toggleDictation();
|
||||
}, [toggleDictation]);
|
||||
// While the recording deck replaces the textarea there is no focused input,
|
||||
// so Escape/Enter are handled at window level: Escape cancels/discard, Enter
|
||||
// retries after an error or converts the recording to text.
|
||||
useEffect(() => {
|
||||
if (!(isRecording || isPaused || isReviewing || isTranscribing || transcribeError)) return;
|
||||
const onKeyDown = (e: globalThis.KeyboardEvent) => {
|
||||
if (e.key === "Escape") {
|
||||
e.preventDefault();
|
||||
cancelDictationAndReset();
|
||||
} else if (e.key === "Enter" && !e.shiftKey && !isTranscribing) {
|
||||
// Let focused deck/toolbar controls keep their own activation;
|
||||
// only hijack Enter from the non-interactive page context.
|
||||
const target = e.target as HTMLElement | null;
|
||||
if (target && target.closest("button, input, textarea, select, a, [role='button']")) return;
|
||||
e.preventDefault();
|
||||
if (transcribeError) retryDictation();
|
||||
else if (isReviewing) confirmTranscribeDictation();
|
||||
else stopAndInsertDictation();
|
||||
}
|
||||
};
|
||||
window.addEventListener("keydown", onKeyDown);
|
||||
return () => window.removeEventListener("keydown", onKeyDown);
|
||||
}, [isRecording, isPaused, isReviewing, isTranscribing, transcribeError, cancelDictationAndReset, retryDictation, confirmTranscribeDictation, stopAndInsertDictation]);
|
||||
|
||||
const slashQuery = value.startsWith("/") && !/\s/.test(value.slice(1))
|
||||
? value.slice(1).toLowerCase()
|
||||
@@ -964,10 +1049,11 @@ export const ChatInput = memo(forwardRef<ChatInputHandle, Props>(function ChatIn
|
||||
});
|
||||
}, []);
|
||||
|
||||
const sendQueued = useCallback((mode: "steer" | "followup") => {
|
||||
const msg = value.trim();
|
||||
if (!msg && !attachedImages.length && !attachedTextFiles.length) return;
|
||||
if (attachedImages.length || attachedTextFiles.length) return;
|
||||
const sendQueued = useCallback((mode: "steer" | "followup", overrideText?: string) => {
|
||||
const raw = overrideText ?? value;
|
||||
const msg = raw.trim();
|
||||
if (!msg && !attachedImagesRef.current.length && !attachedTextFilesRef.current.length) return;
|
||||
if (attachedImagesRef.current.length || attachedTextFilesRef.current.length) return;
|
||||
onAudioUnlock?.();
|
||||
const streamingBehavior = mode === "steer" ? "steer" : "followUp";
|
||||
if (msg.startsWith("/") && onPromptWithStreamingBehavior) {
|
||||
@@ -984,8 +1070,8 @@ export const ChatInput = memo(forwardRef<ChatInputHandle, Props>(function ChatIn
|
||||
// own ACP handlers can run them.
|
||||
const expansion = expandWebSlashCommand(msg);
|
||||
if (expansion.kind === "expand") {
|
||||
if (rejectsOversizedPrompt(expansion.prompt, attachedImages)) return;
|
||||
onPromptWithStreamingBehavior(expansion.prompt, streamingBehavior, attachedImages.length ? attachedImages : undefined);
|
||||
if (rejectsOversizedPrompt(expansion.prompt, attachedImagesRef.current)) return;
|
||||
onPromptWithStreamingBehavior(expansion.prompt, streamingBehavior, attachedImagesRef.current.length ? attachedImagesRef.current : undefined);
|
||||
clearInput();
|
||||
return;
|
||||
}
|
||||
@@ -996,24 +1082,27 @@ export const ChatInput = memo(forwardRef<ChatInputHandle, Props>(function ChatIn
|
||||
}));
|
||||
return;
|
||||
}
|
||||
if (rejectsOversizedPrompt(msg, attachedImages)) return;
|
||||
onPromptWithStreamingBehavior(msg, streamingBehavior, attachedImages.length ? attachedImages : undefined);
|
||||
if (rejectsOversizedPrompt(msg, attachedImagesRef.current)) return;
|
||||
onPromptWithStreamingBehavior(msg, streamingBehavior, attachedImagesRef.current.length ? attachedImagesRef.current : undefined);
|
||||
clearInput();
|
||||
return;
|
||||
}
|
||||
if (rejectsOversizedPrompt(msg, attachedImages)) return;
|
||||
if (rejectsOversizedPrompt(msg, attachedImagesRef.current)) return;
|
||||
if (mode === "steer" && onSteer) {
|
||||
onSteer(msg, attachedImages.length ? attachedImages : undefined);
|
||||
onSteer(msg, attachedImagesRef.current.length ? attachedImagesRef.current : undefined);
|
||||
} else if (mode === "followup" && onFollowUp) {
|
||||
onFollowUp(msg, attachedImages.length ? attachedImages : undefined);
|
||||
onFollowUp(msg, attachedImagesRef.current.length ? attachedImagesRef.current : undefined);
|
||||
}
|
||||
clearInput();
|
||||
}, [value, attachedImages, attachedTextFiles, onPromptWithStreamingBehavior, onSteer, onFollowUp, clearInput, onAudioUnlock, t, advisorEnabled, rejectsOversizedPrompt]);
|
||||
// A typed, text-only message during a run is a queued follow-up. Keep Stop
|
||||
// as the action while the composer is empty or contains attachments.
|
||||
}, [value, onPromptWithStreamingBehavior, onSteer, onFollowUp, clearInput, onAudioUnlock, t, advisorEnabled, rejectsOversizedPrompt]);
|
||||
// A typed, text-only message during a run is a queued follow-up — and so is
|
||||
// a dictation in progress: the primary button must take the same state it
|
||||
// would have if the composer already held text. Keep Stop as the action
|
||||
// while the composer is empty and nothing is being recorded.
|
||||
const dictationCapturing = isRecording || isPaused;
|
||||
const primaryActionQueuesMessage =
|
||||
isStreaming
|
||||
&& Boolean(value.trim())
|
||||
&& (Boolean(value.trim()) || dictationCapturing)
|
||||
&& attachedImages.length === 0
|
||||
&& attachedTextFiles.length === 0
|
||||
&& Boolean(onFollowUp);
|
||||
@@ -1130,21 +1219,9 @@ export const ChatInput = memo(forwardRef<ChatInputHandle, Props>(function ChatIn
|
||||
if (recentlyComposed) e.preventDefault();
|
||||
return;
|
||||
}
|
||||
if (isRecording || isTranscribing) {
|
||||
if (e.key === "Escape") {
|
||||
e.preventDefault();
|
||||
cancelDictation();
|
||||
return;
|
||||
}
|
||||
if (isRecording && e.key === "Enter" && !e.shiftKey) {
|
||||
e.preventDefault();
|
||||
stopDictation();
|
||||
return;
|
||||
}
|
||||
}
|
||||
if ((e.metaKey || e.ctrlKey) && !e.altKey && !e.shiftKey && e.key.toLowerCase() === "m") {
|
||||
e.preventDefault();
|
||||
toggleDictation();
|
||||
startFreshDictation();
|
||||
return;
|
||||
}
|
||||
|
||||
@@ -1268,7 +1345,7 @@ export const ChatInput = memo(forwardRef<ChatInputHandle, Props>(function ChatIn
|
||||
}
|
||||
}
|
||||
},
|
||||
[isMobile, isStreaming, onSteer, onFollowUp, onAbort, onMinimize, slashMenuOpen, slashQuery, filteredSlashCommands, slashActiveIndex, applySlashCommand, sendQueued, handleSend, getNextSlashIndex, atMenuOpen, atQuery, atMatches, atActiveIndex, applyAtCompletion, historyMenuOpen, inputHistory, historyActiveIndex, applyHistoryInput, value, isRecording, isTranscribing, cancelDictation, stopDictation, toggleDictation]
|
||||
[isMobile, isStreaming, onSteer, onFollowUp, onAbort, onMinimize, slashMenuOpen, slashQuery, filteredSlashCommands, slashActiveIndex, applySlashCommand, sendQueued, handleSend, getNextSlashIndex, atMenuOpen, atQuery, atMatches, atActiveIndex, applyAtCompletion, historyMenuOpen, inputHistory, historyActiveIndex, applyHistoryInput, value, startFreshDictation]
|
||||
);
|
||||
|
||||
const handleInput = useCallback(() => {
|
||||
@@ -2241,6 +2318,25 @@ export const ChatInput = memo(forwardRef<ChatInputHandle, Props>(function ChatIn
|
||||
transition: "border-color var(--dur-fast) var(--ease-out-warm), background var(--dur-fast) var(--ease-out-warm), box-shadow var(--dur-fast) var(--ease-out-warm)",
|
||||
} as React.CSSProperties}
|
||||
>
|
||||
{isRecording || isPaused || isReviewing || isTranscribing || transcribeError ? (
|
||||
<RecordingDeck
|
||||
captureRef={captureRef}
|
||||
isPaused={isPaused}
|
||||
isReviewing={isReviewing}
|
||||
isTranscribing={isTranscribing}
|
||||
isPlayingPreview={isPlayingPreview}
|
||||
previewCurrentTime={previewCurrentTime}
|
||||
previewDuration={previewDuration}
|
||||
transcribeError={transcribeError}
|
||||
onPauseResume={togglePauseDictation}
|
||||
onConvert={stopAndInsertDictation}
|
||||
onRetry={retryDictation}
|
||||
onPlayPreview={isPlayingPreview ? pausePreviewDictation : playPreviewDictation}
|
||||
onSeekPreview={seekPreviewDictation}
|
||||
onConfirmTranscribe={confirmTranscribeDictation}
|
||||
onDiscard={cancelDictationAndReset}
|
||||
/>
|
||||
) : (
|
||||
<textarea
|
||||
ref={textareaRef}
|
||||
value={value}
|
||||
@@ -2282,6 +2378,7 @@ export const ChatInput = memo(forwardRef<ChatInputHandle, Props>(function ChatIn
|
||||
overflow: "auto",
|
||||
}}
|
||||
/>
|
||||
)}
|
||||
|
||||
{/* Toolbar: plus menu · model · reasoning · fast · compact · send/queue/stop */}
|
||||
<div className="composer-toolbar" style={{
|
||||
@@ -2784,37 +2881,61 @@ export const ChatInput = memo(forwardRef<ChatInputHandle, Props>(function ChatIn
|
||||
)}
|
||||
|
||||
{/* Dictation */}
|
||||
<button
|
||||
type="button"
|
||||
onClick={toggleDictation}
|
||||
disabled={isTranscribing}
|
||||
title={isRecording ? t("chatInput.stopDictation") : isTranscribing ? t("chatInput.transcribing") : t("chatInput.startDictation")}
|
||||
aria-label={isRecording ? t("chatInput.stopDictation") : isTranscribing ? t("chatInput.transcribing") : t("chatInput.startDictation")}
|
||||
style={{
|
||||
display: "flex", alignItems: "center", justifyContent: "center",
|
||||
width: 28, height: 28, padding: 0,
|
||||
background: isRecording ? "var(--danger-subtle, rgba(239, 68, 68, 0.15))" : "none",
|
||||
border: isRecording ? "1px solid var(--danger, #ef4444)" : "none",
|
||||
borderRadius: 7,
|
||||
color: isRecording ? "var(--danger, #ef4444)" : isTranscribing ? "var(--accent)" : "var(--text-muted)",
|
||||
cursor: isTranscribing ? "wait" : "pointer",
|
||||
transition: "background var(--dur-fast) var(--ease-out-warm), color var(--dur-fast) var(--ease-out-warm)",
|
||||
}}
|
||||
onMouseEnter={(e) => { if (!isRecording && !isTranscribing) e.currentTarget.style.background = "var(--bg-hover)"; }}
|
||||
onMouseLeave={(e) => { e.currentTarget.style.background = isRecording ? "var(--danger-subtle, rgba(239, 68, 68, 0.15))" : "none"; }}
|
||||
>
|
||||
{isTranscribing ? (
|
||||
<Loader2 size={14} strokeWidth={1.8} className="animate-spin" aria-hidden="true" />
|
||||
) : (
|
||||
{isRecording || isPaused || isReviewing || isTranscribing || transcribeError ? (
|
||||
<button
|
||||
type="button"
|
||||
onClick={cancelDictationAndReset}
|
||||
title={transcribeError ? t("chatInput.discardDictation") : t("chatInput.cancelDictation")}
|
||||
aria-label={transcribeError ? t("chatInput.discardDictation") : t("chatInput.cancelDictation")}
|
||||
style={{
|
||||
display: "flex", alignItems: "center", justifyContent: "center",
|
||||
width: 28, height: 28, padding: 0,
|
||||
background: "color-mix(in srgb, var(--status-error) 15%, transparent)",
|
||||
border: "1px solid var(--status-error)",
|
||||
borderRadius: 7,
|
||||
color: "var(--status-error)",
|
||||
cursor: "pointer",
|
||||
transition: "background var(--dur-fast) var(--ease-out-warm), color var(--dur-fast) var(--ease-out-warm)",
|
||||
}}
|
||||
>
|
||||
<X size={14} strokeWidth={1.8} aria-hidden="true" />
|
||||
</button>
|
||||
) : (
|
||||
<button
|
||||
type="button"
|
||||
onClick={startFreshDictation}
|
||||
title={t("chatInput.startDictation")}
|
||||
aria-label={t("chatInput.startDictation")}
|
||||
style={{
|
||||
display: "flex", alignItems: "center", justifyContent: "center",
|
||||
width: 28, height: 28, padding: 0,
|
||||
background: "none",
|
||||
border: "none",
|
||||
borderRadius: 7,
|
||||
color: "var(--text-muted)",
|
||||
cursor: "pointer",
|
||||
transition: "background var(--dur-fast) var(--ease-out-warm), color var(--dur-fast) var(--ease-out-warm)",
|
||||
}}
|
||||
onMouseEnter={(e) => { e.currentTarget.style.background = "var(--bg-hover)"; }}
|
||||
onMouseLeave={(e) => { e.currentTarget.style.background = "none"; }}
|
||||
>
|
||||
<Mic size={14} strokeWidth={1.8} aria-hidden="true" />
|
||||
)}
|
||||
</button>
|
||||
</button>
|
||||
)}
|
||||
{/* Primary action: Send (idle) / Queue (typed while running) / Stop (running) */}
|
||||
{primaryActionQueuesMessage ? (
|
||||
<button
|
||||
type="button"
|
||||
className="composer-primary-action"
|
||||
onClick={() => sendQueued("followup")}
|
||||
onClick={() => {
|
||||
if (dictationCapturing) {
|
||||
const behavior = getSubmitDuringRunBehavior();
|
||||
stopAndQueueDictation(behavior === "steer" && onSteer ? "steer" : "followup");
|
||||
} else {
|
||||
sendQueued("followup");
|
||||
}
|
||||
}}
|
||||
disabled={isTranscribing}
|
||||
title={t("chatInput.queueMessage")}
|
||||
style={{
|
||||
display: "flex", alignItems: "center", gap: 6,
|
||||
@@ -2858,25 +2979,30 @@ export const ChatInput = memo(forwardRef<ChatInputHandle, Props>(function ChatIn
|
||||
<button
|
||||
type="button"
|
||||
className="composer-primary-action"
|
||||
onClick={handleSend}
|
||||
disabled={!value.trim() && !attachedImages.length && !attachedTextFiles.length}
|
||||
onClick={isRecording || isPaused || isReviewing ? stopAndSendDictation : () => void handleSend()}
|
||||
disabled={isTranscribing || !(isRecording || isPaused || isReviewing || value.trim() || attachedImages.length || attachedTextFiles.length)}
|
||||
title={isRecording || isPaused || isReviewing ? t("chatInput.sendDictation") : t("chatInput.send")}
|
||||
style={{
|
||||
display: "flex", alignItems: "center", gap: 6,
|
||||
background: (value.trim() || attachedImages.length || attachedTextFiles.length) ? "var(--accent-strong)" : "var(--bg-panel)",
|
||||
background: (isRecording || isPaused || isReviewing || isTranscribing || value.trim() || attachedImages.length || attachedTextFiles.length) ? "var(--accent-strong)" : "var(--bg-panel)",
|
||||
border: "none",
|
||||
borderRadius: 8,
|
||||
color: (value.trim() || attachedImages.length || attachedTextFiles.length) ? "var(--on-accent)" : "var(--text-dim)",
|
||||
cursor: (value.trim() || attachedImages.length || attachedTextFiles.length) ? "pointer" : "not-allowed",
|
||||
color: (isRecording || isPaused || isReviewing || isTranscribing || value.trim() || attachedImages.length || attachedTextFiles.length) ? "var(--on-accent)" : "var(--text-dim)",
|
||||
cursor: isTranscribing ? "wait" : (isRecording || isPaused || value.trim() || attachedImages.length || attachedTextFiles.length) ? "pointer" : "not-allowed",
|
||||
fontSize: 12,
|
||||
fontWeight: 600,
|
||||
boxShadow: (value.trim() || attachedImages.length || attachedTextFiles.length) ? "var(--shadow-card)" : "none",
|
||||
boxShadow: (isRecording || isPaused || isTranscribing || value.trim() || attachedImages.length || attachedTextFiles.length) ? "var(--shadow-card)" : "none",
|
||||
transition: "background var(--dur-fast) var(--ease-out-warm), box-shadow var(--dur-fast) var(--ease-out-warm)",
|
||||
}}
|
||||
>
|
||||
<svg width="12" height="12" viewBox="0 0 14 14" fill="none" stroke="currentColor" strokeWidth="2" strokeLinecap="round" strokeLinejoin="round">
|
||||
<line x1="2" y1="7" x2="11" y2="7" />
|
||||
<polyline points="7.5 3 12 7 7.5 11" />
|
||||
</svg>
|
||||
{isTranscribing ? (
|
||||
<Loader2 size={12} strokeWidth={2} className="animate-spin" aria-hidden="true" />
|
||||
) : (
|
||||
<svg width="12" height="12" viewBox="0 0 14 14" fill="none" stroke="currentColor" strokeWidth="2" strokeLinecap="round" strokeLinejoin="round" aria-hidden="true">
|
||||
<line x1="2" y1="7" x2="11" y2="7" />
|
||||
<polyline points="7.5 3 12 7 7.5 11" />
|
||||
</svg>
|
||||
)}
|
||||
{t("chatInput.send")}
|
||||
</button>
|
||||
)}
|
||||
|
||||
@@ -0,0 +1,343 @@
|
||||
"use client";
|
||||
|
||||
import React, { useEffect, useRef, useState } from "react";
|
||||
import { AlertCircle, ArrowUp, Loader2, Mic, Pause, Play, RotateCw, Square, Trash2 } from "lucide-react";
|
||||
import { MAX_RECORDING_MS, type DictationCapture } from "@/hooks/useDictation";
|
||||
import { useI18n } from "@/lib/i18n";
|
||||
|
||||
const WAVE_WIDTH = 160;
|
||||
const WAVE_HEIGHT = 28;
|
||||
const SAMPLE_INTERVAL_MS = 50;
|
||||
const BAR_SCALE = 3;
|
||||
|
||||
interface RecordingDeckProps {
|
||||
captureRef: React.RefObject<DictationCapture>;
|
||||
isPaused: boolean;
|
||||
isReviewing?: boolean;
|
||||
isTranscribing: boolean;
|
||||
isPlayingPreview?: boolean;
|
||||
previewCurrentTime?: number;
|
||||
previewDuration?: number;
|
||||
transcribeError: string | null;
|
||||
onPauseResume: () => void;
|
||||
onConvert: () => void;
|
||||
onRetry: () => void;
|
||||
onPlayPreview?: () => void;
|
||||
onSeekPreview?: (seconds: number) => void;
|
||||
onConfirmTranscribe?: () => void;
|
||||
onDiscard?: () => void;
|
||||
}
|
||||
|
||||
function formatElapsed(ms: number): string {
|
||||
const total = Math.max(0, Math.floor(ms / 1000));
|
||||
return `${Math.floor(total / 60)}:${String(total % 60).padStart(2, "0")}`;
|
||||
}
|
||||
|
||||
function DeckIconButton({
|
||||
children,
|
||||
onClick,
|
||||
title,
|
||||
tone,
|
||||
}: {
|
||||
children: React.ReactNode;
|
||||
onClick: () => void;
|
||||
title: string;
|
||||
tone?: "danger" | "accent";
|
||||
}) {
|
||||
return (
|
||||
<button
|
||||
type="button"
|
||||
onClick={onClick}
|
||||
title={title}
|
||||
aria-label={title}
|
||||
style={{
|
||||
display: "flex", alignItems: "center", justifyContent: "center",
|
||||
width: 28, height: 28, padding: 0, flexShrink: 0,
|
||||
background:
|
||||
tone === "danger" ? "color-mix(in srgb, var(--status-error) 15%, transparent)"
|
||||
: tone === "accent" ? "var(--bg-subtle)"
|
||||
: "var(--bg-subtle)",
|
||||
border: `1px solid ${tone === "danger" ? "var(--status-error)" : tone === "accent" ? "var(--accent)" : "var(--border)"}`,
|
||||
borderRadius: 7,
|
||||
color:
|
||||
tone === "danger" ? "var(--status-error)"
|
||||
: tone === "accent" ? "var(--accent)"
|
||||
: "var(--text-muted)",
|
||||
cursor: "pointer",
|
||||
}}
|
||||
>
|
||||
{children}
|
||||
</button>
|
||||
);
|
||||
}
|
||||
|
||||
export function RecordingDeck({
|
||||
captureRef,
|
||||
isPaused,
|
||||
isReviewing = false,
|
||||
isTranscribing,
|
||||
isPlayingPreview = false,
|
||||
previewCurrentTime = 0,
|
||||
previewDuration = 0,
|
||||
transcribeError,
|
||||
onPauseResume,
|
||||
onConvert,
|
||||
onRetry,
|
||||
onPlayPreview,
|
||||
onSeekPreview,
|
||||
onConfirmTranscribe,
|
||||
onDiscard,
|
||||
}: RecordingDeckProps) {
|
||||
const { t } = useI18n();
|
||||
const canvasRef = useRef<HTMLCanvasElement | null>(null);
|
||||
const barsRef = useRef<number[]>([]);
|
||||
const [elapsed, setElapsed] = useState(0);
|
||||
|
||||
const captureActive = !isTranscribing && !transcribeError && !isReviewing;
|
||||
|
||||
useEffect(() => {
|
||||
const compute = () => {
|
||||
const capture = captureRef.current;
|
||||
if (isReviewing && capture?.finalDurationMs != null) {
|
||||
setElapsed(capture.finalDurationMs);
|
||||
return;
|
||||
}
|
||||
setElapsed(
|
||||
capture
|
||||
? Math.min(
|
||||
MAX_RECORDING_MS,
|
||||
Math.max(
|
||||
0,
|
||||
performance.now() -
|
||||
capture.startedAt -
|
||||
capture.pausedAccum -
|
||||
(capture.pausedAt !== null ? performance.now() - capture.pausedAt : 0),
|
||||
),
|
||||
)
|
||||
: 0,
|
||||
);
|
||||
};
|
||||
compute();
|
||||
const id = window.setInterval(compute, 100);
|
||||
return () => window.clearInterval(id);
|
||||
}, [captureRef, isReviewing]);
|
||||
|
||||
useEffect(() => {
|
||||
const canvas = canvasRef.current;
|
||||
// Review mode has no live capture but still draws the recorded bars and
|
||||
// the playback progress overlay.
|
||||
if (!canvas || (!captureActive && !isReviewing)) return;
|
||||
const ctx = canvas.getContext("2d");
|
||||
if (!ctx) return;
|
||||
|
||||
const dpr = window.devicePixelRatio || 1;
|
||||
let lastW = 0;
|
||||
|
||||
const style = getComputedStyle(canvas);
|
||||
const barColor = (isPaused || isReviewing ? style.getPropertyValue("--text-muted") : style.getPropertyValue("--status-error")).trim();
|
||||
|
||||
const data = new Uint8Array(256);
|
||||
let raf = 0;
|
||||
let lastPush = 0;
|
||||
|
||||
const draw = (now: number) => {
|
||||
raf = requestAnimationFrame(draw);
|
||||
// The canvas flexes with the composer; re-fit the backing store whenever
|
||||
// its CSS width changes (mobile <-> desktop, panel resizes).
|
||||
const cssW = Math.max(1, canvas.clientWidth || WAVE_WIDTH);
|
||||
if (cssW !== lastW) {
|
||||
lastW = cssW;
|
||||
canvas.width = cssW * dpr;
|
||||
canvas.height = WAVE_HEIGHT * dpr;
|
||||
ctx.setTransform(dpr, 0, 0, dpr, 0, 0);
|
||||
}
|
||||
const barCount = Math.max(24, Math.min(80, Math.floor(cssW / 6)));
|
||||
const bars = barsRef.current;
|
||||
const analyser = captureRef.current?.analyser ?? null;
|
||||
if (analyser && !isPaused && now - lastPush >= SAMPLE_INTERVAL_MS) {
|
||||
lastPush = now;
|
||||
analyser.getByteTimeDomainData(data);
|
||||
let sum = 0;
|
||||
for (let i = 0; i < data.length; i++) {
|
||||
const v = (data[i] - 128) / 128;
|
||||
sum += v * v;
|
||||
}
|
||||
const rms = Math.sqrt(sum / data.length);
|
||||
bars.push(Math.min(1, rms * BAR_SCALE));
|
||||
while (bars.length > barCount) bars.shift();
|
||||
}
|
||||
while (bars.length > barCount) bars.shift();
|
||||
|
||||
ctx.clearRect(0, 0, cssW, WAVE_HEIGHT);
|
||||
const gap = 2;
|
||||
const barW = Math.max(1, (cssW - gap * (barCount - 1)) / barCount);
|
||||
const visible = bars.slice(-barCount);
|
||||
const offset = cssW - visible.length * (barW + gap) + gap;
|
||||
|
||||
// Determine playback progress ratio if previewing or reviewing
|
||||
const isPreviewActive = (isReviewing || isPaused) && (isPlayingPreview || previewCurrentTime > 0);
|
||||
const activeDuration = previewDuration > 0 ? previewDuration : Math.max(1, elapsed / 1000);
|
||||
const progressRatio = isPreviewActive ? Math.min(1, Math.max(0, previewCurrentTime / activeDuration)) : 0;
|
||||
const progressCutoff = isPreviewActive ? offset + progressRatio * (cssW - offset) : -1;
|
||||
|
||||
const accentColor = style.getPropertyValue("--accent").trim() || "var(--accent)";
|
||||
|
||||
for (let i = 0; i < visible.length; i++) {
|
||||
const h = Math.max(2, visible[i] * (WAVE_HEIGHT - 2));
|
||||
const x = offset + i * (barW + gap);
|
||||
ctx.fillStyle = isPreviewActive && x <= progressCutoff ? accentColor : barColor;
|
||||
ctx.fillRect(x, (WAVE_HEIGHT - h) / 2, barW, h);
|
||||
}
|
||||
|
||||
if (isPreviewActive && progressCutoff >= offset) {
|
||||
ctx.fillStyle = accentColor;
|
||||
ctx.fillRect(Math.min(cssW - 2, progressCutoff), 2, 2, WAVE_HEIGHT - 4);
|
||||
}
|
||||
};
|
||||
raf = requestAnimationFrame(draw);
|
||||
return () => cancelAnimationFrame(raf);
|
||||
}, [captureRef, captureActive, isPaused, isReviewing, isPlayingPreview, previewCurrentTime, previewDuration]);
|
||||
|
||||
return (
|
||||
<div
|
||||
role="status"
|
||||
aria-label={
|
||||
isTranscribing
|
||||
? t("chatInput.transcribing")
|
||||
: transcribeError
|
||||
? transcribeError
|
||||
: isReviewing
|
||||
? t("chatInput.discardDictation")
|
||||
: isPaused
|
||||
? t("chatInput.resumeDictation")
|
||||
: t("chatInput.dictationRecording")
|
||||
}
|
||||
style={{
|
||||
display: "flex",
|
||||
flexDirection: "column",
|
||||
justifyContent: "center",
|
||||
gap: 6,
|
||||
minHeight: 24,
|
||||
padding: "2px 0",
|
||||
}}
|
||||
>
|
||||
{isTranscribing ? (
|
||||
<div style={{ display: "flex", alignItems: "center", gap: 8, color: "var(--text-muted)", fontSize: 12 }}>
|
||||
<Loader2 size={14} strokeWidth={1.8} className="animate-spin" aria-hidden="true" />
|
||||
<span style={{ flexShrink: 0 }}>{t("chatInput.transcribing")}</span>
|
||||
<div style={{ flex: 1, height: 3, borderRadius: 2, background: "var(--border)", overflow: "hidden", position: "relative" }}>
|
||||
<div className="dictation-indeterminate" style={{ position: "absolute", top: 0, bottom: 0, width: "40%", background: "var(--accent)", borderRadius: 2 }} />
|
||||
</div>
|
||||
</div>
|
||||
) : transcribeError ? (
|
||||
<div style={{ display: "flex", alignItems: "center", gap: 8, fontSize: 12, color: "var(--text)", minWidth: 0 }}>
|
||||
<AlertCircle size={14} strokeWidth={1.8} aria-hidden="true" style={{ color: "var(--status-error)", flexShrink: 0 }} />
|
||||
<span title={transcribeError} style={{ overflow: "hidden", textOverflow: "ellipsis", whiteSpace: "nowrap", flex: 1 }}>
|
||||
{transcribeError}
|
||||
</span>
|
||||
<DeckIconButton onClick={onRetry} title={t("chatInput.retryDictation")} tone="accent">
|
||||
<RotateCw size={14} strokeWidth={1.8} aria-hidden="true" />
|
||||
</DeckIconButton>
|
||||
</div>
|
||||
) : (
|
||||
<>
|
||||
<div style={{ display: "flex", alignItems: "center", gap: 8, minWidth: 0 }}>
|
||||
{isReviewing ? (
|
||||
<DeckIconButton
|
||||
onClick={onPlayPreview ?? (() => {})}
|
||||
title={isPlayingPreview ? t("chatInput.pausePreview") : t("chatInput.playPreview")}
|
||||
tone="accent"
|
||||
>
|
||||
{isPlayingPreview ? <Pause size={14} strokeWidth={1.8} aria-hidden="true" /> : <Play size={14} strokeWidth={1.8} aria-hidden="true" style={{ marginLeft: 1 }} />}
|
||||
</DeckIconButton>
|
||||
) : isPaused ? (
|
||||
<DeckIconButton
|
||||
onClick={onPlayPreview ?? (() => {})}
|
||||
title={isPlayingPreview ? t("chatInput.pausePreview") : t("chatInput.playPreview")}
|
||||
>
|
||||
{isPlayingPreview ? <Pause size={12} strokeWidth={2} aria-hidden="true" /> : <Play size={12} strokeWidth={2} aria-hidden="true" style={{ marginLeft: 1 }} />}
|
||||
</DeckIconButton>
|
||||
) : (
|
||||
<span
|
||||
aria-hidden="true"
|
||||
style={{ width: 8, height: 8, borderRadius: "50%", background: "var(--status-error)", flexShrink: 0 }}
|
||||
/>
|
||||
)}
|
||||
<span
|
||||
style={{
|
||||
fontSize: 12,
|
||||
fontVariantNumeric: "tabular-nums",
|
||||
color: "var(--text)",
|
||||
flexShrink: 0,
|
||||
minWidth: isReviewing ? 70 : 34,
|
||||
}}
|
||||
>
|
||||
{isReviewing
|
||||
? `${formatElapsed(previewCurrentTime * 1000)} / ${formatElapsed((previewDuration || elapsed / 1000) * 1000)}`
|
||||
: isPlayingPreview
|
||||
? `${formatElapsed(previewCurrentTime * 1000)} / ${formatElapsed(elapsed)}`
|
||||
: formatElapsed(elapsed)}
|
||||
</span>
|
||||
<canvas
|
||||
ref={canvasRef}
|
||||
style={{ flex: 1, minWidth: 0, height: WAVE_HEIGHT, alignSelf: "center", cursor: isReviewing || isPaused ? "pointer" : "default" }}
|
||||
aria-hidden="true"
|
||||
onClick={(e) => {
|
||||
if (!isReviewing && !isPaused) return;
|
||||
const rect = e.currentTarget.getBoundingClientRect();
|
||||
const ratio = Math.min(1, Math.max(0, (e.clientX - rect.left) / rect.width));
|
||||
const totalSec = previewDuration > 0 ? previewDuration : elapsed / 1000;
|
||||
onSeekPreview?.(ratio * totalSec);
|
||||
}}
|
||||
/>
|
||||
<div style={{ display: "flex", alignItems: "center", gap: 4, marginLeft: "auto", flexShrink: 0 }}>
|
||||
{isReviewing ? (
|
||||
<>
|
||||
<DeckIconButton onClick={onDiscard ?? onConvert} title={t("chatInput.discardDictation")} tone="danger">
|
||||
<Trash2 size={14} strokeWidth={1.8} aria-hidden="true" />
|
||||
</DeckIconButton>
|
||||
<DeckIconButton onClick={onConfirmTranscribe ?? onConvert} title={t("chatInput.transcribeDictation")} tone="accent">
|
||||
<ArrowUp size={14} strokeWidth={2} aria-hidden="true" />
|
||||
</DeckIconButton>
|
||||
</>
|
||||
) : (
|
||||
<>
|
||||
<DeckIconButton
|
||||
onClick={onPauseResume}
|
||||
title={isPaused ? t("chatInput.resumeDictation") : t("chatInput.pauseDictation")}
|
||||
tone={isPaused ? "accent" : undefined}
|
||||
>
|
||||
{isPaused ? (
|
||||
<Mic size={14} strokeWidth={1.8} aria-hidden="true" />
|
||||
) : (
|
||||
<Pause size={14} strokeWidth={1.8} aria-hidden="true" />
|
||||
)}
|
||||
</DeckIconButton>
|
||||
<DeckIconButton onClick={onConvert} title={t("chatInput.stopDictation")}>
|
||||
<Square size={11} strokeWidth={2} aria-hidden="true" />
|
||||
</DeckIconButton>
|
||||
</>
|
||||
)}
|
||||
</div>
|
||||
</div>
|
||||
<div
|
||||
role="progressbar"
|
||||
aria-valuemin={0}
|
||||
aria-valuemax={MAX_RECORDING_MS}
|
||||
aria-valuenow={Math.round(elapsed)}
|
||||
style={{ height: 2, borderRadius: 1, background: "var(--border)", overflow: "hidden" }}
|
||||
>
|
||||
<div
|
||||
style={{
|
||||
height: "100%",
|
||||
width: isReviewing ? "100%" : `${Math.min(100, (elapsed / MAX_RECORDING_MS) * 100)}%`,
|
||||
background: isReviewing ? "var(--accent)" : isPaused ? "var(--text-muted)" : "var(--status-error)",
|
||||
transition: "width 0.1s linear",
|
||||
}}
|
||||
/>
|
||||
</div>
|
||||
</>
|
||||
)}
|
||||
</div>
|
||||
);
|
||||
}
|
||||
@@ -0,0 +1,274 @@
|
||||
import "../tests/setup-dom.mjs";
|
||||
import assert from "node:assert/strict";
|
||||
import test, { afterEach, beforeEach } from "node:test";
|
||||
import { fileURLToPath } from "node:url";
|
||||
import { createJiti } from "jiti";
|
||||
import { act, cleanup, renderHook } from "@testing-library/react/pure.js";
|
||||
|
||||
// These tests execute the dictation state machine against fake
|
||||
// getUserMedia/MediaRecorder/AudioContext implementations. They assert the
|
||||
// observable hook state a consumer renders from (isRecording/isPaused/
|
||||
// isReviewing, live analyser, active-time accounting, blob-URL lifetime) —
|
||||
// the states the composer and RecordingDeck actually read.
|
||||
|
||||
const jiti = createJiti(import.meta.url, {
|
||||
tryNative: false,
|
||||
alias: { "@/": fileURLToPath(new URL("../", import.meta.url)) },
|
||||
});
|
||||
const { useDictation } = await jiti.import("../hooks/useDictation.ts");
|
||||
|
||||
const sleep = (ms) => new Promise((resolve) => setTimeout(resolve, ms));
|
||||
async function settle(ms = 20) {
|
||||
await act(async () => {
|
||||
await sleep(ms);
|
||||
});
|
||||
}
|
||||
|
||||
let world;
|
||||
|
||||
class FakeTrack {
|
||||
constructor() {
|
||||
this.stopped = false;
|
||||
}
|
||||
stop() {
|
||||
this.stopped = true;
|
||||
}
|
||||
}
|
||||
|
||||
class FakeMediaRecorder {
|
||||
constructor(stream) {
|
||||
this.stream = stream;
|
||||
this.state = "inactive";
|
||||
this.mimeType = "audio/webm";
|
||||
this.timeslice = null;
|
||||
this.ondataavailable = null;
|
||||
this.onstop = null;
|
||||
world.recorders.push(this);
|
||||
}
|
||||
start(timeslice) {
|
||||
this.state = "recording";
|
||||
this.timeslice = timeslice ?? null;
|
||||
}
|
||||
requestData() {
|
||||
this.ondataavailable?.({ data: new Blob(["chunk"], { type: this.mimeType }) });
|
||||
}
|
||||
pause() {
|
||||
this.state = "paused";
|
||||
}
|
||||
resume() {
|
||||
this.state = "recording";
|
||||
}
|
||||
stop() {
|
||||
this.state = "inactive";
|
||||
this.ondataavailable?.({ data: new Blob(["chunk"], { type: this.mimeType }) });
|
||||
this.onstop?.();
|
||||
}
|
||||
}
|
||||
|
||||
class FakeAudioContext {
|
||||
constructor() {
|
||||
this.state = "running";
|
||||
this.closed = false;
|
||||
}
|
||||
createMediaStreamSource() {
|
||||
return { connect: () => {} };
|
||||
}
|
||||
createAnalyser() {
|
||||
return {
|
||||
fftSize: 2048,
|
||||
frequencyBinCount: 128,
|
||||
getByteFrequencyData: () => {},
|
||||
connect: () => {},
|
||||
};
|
||||
}
|
||||
resume() {
|
||||
return Promise.resolve();
|
||||
}
|
||||
close() {
|
||||
this.closed = true;
|
||||
return Promise.resolve();
|
||||
}
|
||||
}
|
||||
|
||||
class FakePreviewAudio {
|
||||
constructor(src) {
|
||||
this.src = src;
|
||||
this.currentTime = 0;
|
||||
this.duration = 1;
|
||||
this.ended = false;
|
||||
this.paused = true;
|
||||
}
|
||||
play() {
|
||||
this.paused = false;
|
||||
this.onplay?.();
|
||||
return Promise.resolve();
|
||||
}
|
||||
pause() {
|
||||
this.paused = true;
|
||||
this.onpause?.();
|
||||
}
|
||||
}
|
||||
|
||||
const overrides = [];
|
||||
function override(target, key, replacement) {
|
||||
overrides.push({ target, key, original: Object.getOwnPropertyDescriptor(target, key) });
|
||||
Object.defineProperty(target, key, { configurable: true, writable: true, value: replacement });
|
||||
}
|
||||
|
||||
beforeEach(() => {
|
||||
world = { recorders: [], createdUrls: [], revokedUrls: [], tracks: [] };
|
||||
const track = new FakeTrack();
|
||||
world.tracks.push(track);
|
||||
override(navigator, "mediaDevices", {
|
||||
getUserMedia: async () => ({ getTracks: () => [track] }),
|
||||
});
|
||||
override(window, "MediaRecorder", FakeMediaRecorder);
|
||||
override(globalThis, "MediaRecorder", FakeMediaRecorder);
|
||||
override(window, "AudioContext", FakeAudioContext);
|
||||
override(globalThis, "AudioContext", FakeAudioContext);
|
||||
override(window, "Audio", FakePreviewAudio);
|
||||
override(globalThis, "Audio", FakePreviewAudio);
|
||||
override(URL, "createObjectURL", (blob) => {
|
||||
const url = `blob:fake/${world.createdUrls.length}`;
|
||||
world.createdUrls.push({ url, blob });
|
||||
return url;
|
||||
});
|
||||
override(URL, "revokeObjectURL", (url) => {
|
||||
world.revokedUrls.push(url);
|
||||
});
|
||||
override(globalThis, "fetch", async () => ({
|
||||
ok: true,
|
||||
json: async () => ({ text: "transcribed words" }),
|
||||
}));
|
||||
});
|
||||
|
||||
afterEach(() => {
|
||||
try {
|
||||
cleanup();
|
||||
} finally {
|
||||
while (overrides.length) {
|
||||
const { target, key, original } = overrides.pop();
|
||||
if (original) Object.defineProperty(target, key, original);
|
||||
else delete target[key];
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
function mountDictation() {
|
||||
const transcripts = [];
|
||||
const errors = [];
|
||||
const view = renderHook(() =>
|
||||
useDictation({
|
||||
onTranscript: (text) => transcripts.push(text),
|
||||
onError: (message) => errors.push(message),
|
||||
}),
|
||||
);
|
||||
return { view, transcripts, errors };
|
||||
}
|
||||
|
||||
test("starting capture puts the hook in the recording state with a live analyser", async () => {
|
||||
const { view, errors } = mountDictation();
|
||||
|
||||
await act(async () => {
|
||||
view.result.current.toggle();
|
||||
});
|
||||
await settle();
|
||||
|
||||
assert.equal(errors.length, 0, `unexpected dictation errors: ${errors.join(" | ")}`);
|
||||
assert.equal(view.result.current.isRecording, true, "consumer must see isRecording after start");
|
||||
assert.equal(view.result.current.isPaused, false);
|
||||
assert.equal(view.result.current.isReviewing, false);
|
||||
assert.equal(world.recorders.length, 1);
|
||||
assert.equal(world.recorders[0].state, "recording");
|
||||
assert.equal(world.recorders[0].timeslice, 100, "timeslice keeps chunks flowing for preview");
|
||||
assert.notEqual(view.result.current.captureRef.current.analyser, null, "waveform needs a live analyser");
|
||||
assert.ok(view.result.current.captureRef.current.startedAt > 0, "timer needs a start stamp");
|
||||
});
|
||||
|
||||
test("pause then resume keeps the timer base and analyser while accumulating paused time", async () => {
|
||||
const { view } = mountDictation();
|
||||
|
||||
await act(async () => {
|
||||
view.result.current.toggle();
|
||||
});
|
||||
await settle();
|
||||
|
||||
const { startedAt, analyser } = view.result.current.captureRef.current;
|
||||
|
||||
await act(async () => {
|
||||
view.result.current.togglePause();
|
||||
});
|
||||
await settle(40);
|
||||
|
||||
assert.equal(view.result.current.isPaused, true);
|
||||
assert.ok(view.result.current.captureRef.current.pausedAt !== null);
|
||||
|
||||
await act(async () => {
|
||||
view.result.current.togglePause();
|
||||
});
|
||||
await settle();
|
||||
|
||||
const capture = view.result.current.captureRef.current;
|
||||
assert.equal(view.result.current.isPaused, false);
|
||||
assert.equal(capture.startedAt, startedAt, "resume must not reset the elapsed base");
|
||||
assert.equal(capture.analyser, analyser, "resume must not drop the live analyser");
|
||||
assert.equal(capture.pausedAt, null);
|
||||
assert.ok(capture.pausedAccum > 0, "paused time must be subtracted from active elapsed time");
|
||||
assert.equal(world.recorders[0].state, "recording");
|
||||
});
|
||||
|
||||
test("stopping capture enters review with a playable preview, and confirming releases it", async () => {
|
||||
const { view, transcripts } = mountDictation();
|
||||
|
||||
await act(async () => {
|
||||
view.result.current.toggle();
|
||||
});
|
||||
await settle();
|
||||
|
||||
await act(async () => {
|
||||
view.result.current.stop();
|
||||
});
|
||||
await settle();
|
||||
|
||||
assert.equal(view.result.current.isReviewing, true);
|
||||
assert.equal(view.result.current.isRecording, false);
|
||||
assert.equal(world.createdUrls.length, 1, "review mode needs a preview blob URL");
|
||||
assert.ok(world.tracks[0].stopped, "microphone tracks must be released on stop");
|
||||
|
||||
await act(async () => {
|
||||
view.result.current.playPreview();
|
||||
});
|
||||
await settle();
|
||||
assert.equal(view.result.current.isPlayingPreview, true);
|
||||
|
||||
await act(async () => {
|
||||
view.result.current.confirmTranscribe();
|
||||
});
|
||||
await settle(60);
|
||||
|
||||
assert.deepEqual(world.revokedUrls, [world.createdUrls[0].url], "confirming must revoke the preview URL");
|
||||
assert.equal(view.result.current.isReviewing, false);
|
||||
assert.deepEqual(transcripts, ["transcribed words"]);
|
||||
});
|
||||
|
||||
test("cancelling during capture clears state and releases the microphone", async () => {
|
||||
const { view, transcripts } = mountDictation();
|
||||
|
||||
await act(async () => {
|
||||
view.result.current.toggle();
|
||||
});
|
||||
await settle();
|
||||
|
||||
await act(async () => {
|
||||
view.result.current.cancel();
|
||||
});
|
||||
await settle();
|
||||
|
||||
assert.equal(view.result.current.isRecording, false);
|
||||
assert.equal(view.result.current.isPaused, false);
|
||||
assert.equal(view.result.current.isReviewing, false);
|
||||
assert.equal(view.result.current.isTranscribing, false);
|
||||
assert.equal(view.result.current.captureRef.current.analyser, null);
|
||||
assert.ok(world.tracks[0].stopped);
|
||||
assert.deepEqual(transcripts, []);
|
||||
});
|
||||
+102
-4
@@ -15,12 +15,110 @@ test("useDictation aborts in-flight transcription and silences transcript on can
|
||||
assert.match(source, /return \(\) => \{\s*\n\s*cancelledRef\.current = true;/);
|
||||
|
||||
// Late arrival after cancel must not invoke onTranscript or onError
|
||||
assert.match(source, /if \(!cancelledRef\.current\) \{\s*\n\s*onTranscript\(data\.text\.trim\(\)\);/);
|
||||
assert.match(source, /if \(!cancelledRef\.current\) \{\s*\n\s*pendingAudioRef\.current = null;\s*\n\s*onTranscript\(data\.text\.trim\(\)\);/);
|
||||
});
|
||||
|
||||
test("ChatInput cancels dictation on Escape during both recording and transcribing", async () => {
|
||||
test("useDictation supports pause and resume with active-time accounting", async () => {
|
||||
const source = await readFile(new URL("./useDictation.ts", import.meta.url), "utf8");
|
||||
|
||||
// Native MediaRecorder pause/resume
|
||||
assert.match(source, /recorder\.pause\(\)/);
|
||||
assert.match(source, /recorder\.resume\(\)/);
|
||||
|
||||
// Paused time is accumulated so the elapsed clock only counts active time
|
||||
assert.match(source, /captureRef\.current\.pausedAt = performance\.now\(\)/);
|
||||
assert.match(source, /captureRef\.current\.pausedAccum = pausedAccum \+ \(performance\.now\(\) - pausedAt\)/);
|
||||
|
||||
// Pause toggle is a no-op once transcription or an error takes over
|
||||
assert.match(source, /if \(!recorder \|\| isTranscribing \|\| transcribeError\) return;/);
|
||||
});
|
||||
|
||||
test("useDictation exposes a live analyser for the waveform", async () => {
|
||||
const source = await readFile(new URL("./useDictation.ts", import.meta.url), "utf8");
|
||||
|
||||
assert.match(source, /createMediaStreamSource\(stream\)/);
|
||||
assert.match(source, /createAnalyser\(\)/);
|
||||
assert.match(source, /captureRef\.current\.analyser = analyser/);
|
||||
|
||||
// The AudioContext must be closed on cleanup
|
||||
assert.match(source, /audioContextRef\.current\.close\(\)/);
|
||||
});
|
||||
|
||||
test("useDictation retains audio after failure and supports retry", async () => {
|
||||
const source = await readFile(new URL("./useDictation.ts", import.meta.url), "utf8");
|
||||
|
||||
// The recorded blob is kept for retry and only dropped on success or discard
|
||||
assert.match(source, /pendingAudioRef\.current = \{ blob, ext \}/);
|
||||
assert.match(source, /pendingAudioRef\.current = null;\s*\n\s*onTranscript/);
|
||||
|
||||
// Retry re-runs transcription with the retained audio
|
||||
assert.match(source, /const retry = useCallback\(\(\) => \{\s*\n\s*const pending = pendingAudioRef\.current;/);
|
||||
assert.match(source, /void runTranscription\(pending\.blob, pending\.ext\)/);
|
||||
});
|
||||
|
||||
test("useDictation surfaces timeout as an error instead of swallowing it", async () => {
|
||||
const source = await readFile(new URL("./useDictation.ts", import.meta.url), "utf8");
|
||||
|
||||
// The fetch timeout aborts with a TimeoutError reason
|
||||
assert.match(source, /abort\(new DOMException\("Transcription timed out", "TimeoutError"\)\)/);
|
||||
|
||||
// TimeoutError is handled distinctly from the user-cancel AbortError
|
||||
assert.match(source, /err\.name === "TimeoutError"/);
|
||||
assert.match(source, /err\.name === "AbortError"\) return;/);
|
||||
});
|
||||
|
||||
test("ChatInput replaces the composer with the deck and routes dictation keys at window level", async () => {
|
||||
const source = await readFile(new URL("../components/ChatInput.tsx", import.meta.url), "utf8");
|
||||
|
||||
// Escape must guard both isRecording and isTranscribing
|
||||
assert.match(source, /if \(isRecording \|\| isTranscribing\) \{\s*\n\s*if \(e\.key === "Escape"\) \{\s*\n\s*e\.preventDefault\(\);\s*\n\s*cancelDictation\(\);/);
|
||||
// The deck covers every dictation state (recording, paused, transcribing, error)
|
||||
assert.match(source, /isRecording \|\| isPaused \|\| (?:isReviewing \|\| )?isTranscribing \|\| transcribeError \? \(/);
|
||||
|
||||
// Window-level listener because the textarea is unmounted while the deck shows
|
||||
assert.match(source, /window\.addEventListener\("keydown", onKeyDown\)/);
|
||||
assert.match(source, /if \(e\.key === "Escape"\) \{\s*\n\s*e\.preventDefault\(\);\s*\n\s*cancelDictationAndReset\(\);/);
|
||||
});
|
||||
|
||||
test("ChatInput supports transcribe-only and transcribe-and-send endings", async () => {
|
||||
const source = await readFile(new URL("../components/ChatInput.tsx", import.meta.url), "utf8");
|
||||
|
||||
// Send mode flags the transcript to be sent once transcription succeeds,
|
||||
// and queue modes route through sendQueued — asserted as one wired block
|
||||
// so a regression in the branch cannot pass via unrelated string matches.
|
||||
assert.match(
|
||||
source,
|
||||
/const after = dictationAfterRef\.current;[\s\S]*?if \(after === "send"\) \{\s*\n\s*void handleSend\(finalText\);\s*\n\s*\} else if \(after === "steer" \|\| after === "followup"\) \{\s*\n\s*sendQueued\(after, finalText\);/,
|
||||
);
|
||||
|
||||
// handleSend accepts the composed dictation text override
|
||||
assert.match(source, /async \(overrideText\?: string\) =>/);
|
||||
});
|
||||
|
||||
test("useDictation supports playback preview while paused and review mode upon stop", async () => {
|
||||
const source = await readFile(new URL("./useDictation.ts", import.meta.url), "utf8");
|
||||
|
||||
// Request data on pause to accumulate chunks for preview
|
||||
assert.match(source, /recorder\.requestData\(\)/);
|
||||
|
||||
// Exposes preview playback and review states
|
||||
assert.match(source, /isReviewing/);
|
||||
assert.match(source, /isPlayingPreview/);
|
||||
assert.match(source, /playPreview/);
|
||||
assert.match(source, /pausePreview/);
|
||||
assert.match(source, /seekPreview/);
|
||||
assert.match(source, /confirmTranscribe/);
|
||||
|
||||
// Stopping capture transitions to review state instead of immediately transcribing
|
||||
assert.match(source, /setIsReviewing\(true\)/);
|
||||
});
|
||||
|
||||
test("RecordingDeck renders left preview play button when paused and in review mode", async () => {
|
||||
const deckSource = await readFile(new URL("../components/RecordingDeck.tsx", import.meta.url), "utf8");
|
||||
|
||||
// Left play preview button when paused or in review mode
|
||||
assert.match(deckSource, /onPlayPreview/);
|
||||
assert.match(deckSource, /isPlayingPreview/);
|
||||
|
||||
// Review mode renders discard and confirm buttons
|
||||
assert.match(deckSource, /isReviewing/);
|
||||
assert.match(deckSource, /onConfirmTranscribe/);
|
||||
});
|
||||
+373
-86
@@ -7,9 +7,21 @@ export interface UseDictationOptions {
|
||||
onError?: (error: string) => void;
|
||||
}
|
||||
|
||||
const MAX_RECORDING_MS = 300_000;
|
||||
export const MAX_RECORDING_MS = 300_000;
|
||||
const STT_TIMEOUT_MS = 60_000;
|
||||
|
||||
/**
|
||||
* Live capture state polled by RecordingDeck (timer + waveform) without
|
||||
* re-rendering the hook's consumer. Mutated in place; identity is stable.
|
||||
*/
|
||||
export interface DictationCapture {
|
||||
analyser: AnalyserNode | null;
|
||||
startedAt: number;
|
||||
pausedAccum: number;
|
||||
pausedAt: number | null;
|
||||
finalDurationMs?: number | null;
|
||||
}
|
||||
|
||||
function normalizeErrorMessage(error: unknown, fallback: string): string {
|
||||
if (typeof error === "string" && error.trim()) return error;
|
||||
if (error && typeof error === "object" && "message" in error) {
|
||||
@@ -21,13 +33,54 @@ function normalizeErrorMessage(error: unknown, fallback: string): string {
|
||||
|
||||
export function useDictation({ onTranscript, onError }: UseDictationOptions) {
|
||||
const [isRecording, setIsRecording] = useState(false);
|
||||
const [isPaused, setIsPaused] = useState(false);
|
||||
const [isReviewing, setIsReviewing] = useState(false);
|
||||
const [isPlayingPreview, setIsPlayingPreview] = useState(false);
|
||||
const [previewCurrentTime, setPreviewCurrentTime] = useState(0);
|
||||
const [previewDuration, setPreviewDuration] = useState(0);
|
||||
const [isTranscribing, setIsTranscribing] = useState(false);
|
||||
const [transcribeError, setTranscribeError] = useState<string | null>(null);
|
||||
const mediaRecorderRef = useRef<MediaRecorder | null>(null);
|
||||
const streamRef = useRef<MediaStream | null>(null);
|
||||
const abortControllerRef = useRef<AbortController | null>(null);
|
||||
const cancelledRef = useRef(false);
|
||||
const isStartingRef = useRef(false);
|
||||
const maxTimeoutRef = useRef<number | null>(null);
|
||||
const pausePreviewTimeoutRef = useRef<number | null>(null);
|
||||
const audioContextRef = useRef<AudioContext | null>(null);
|
||||
const captureRef = useRef<DictationCapture>({ analyser: null, startedAt: 0, pausedAccum: 0, pausedAt: null, finalDurationMs: null });
|
||||
const previewAudioRef = useRef<HTMLAudioElement | null>(null);
|
||||
const previewUrlRef = useRef<string | null>(null);
|
||||
const chunksRef = useRef<Blob[]>([]);
|
||||
const immediateSendRef = useRef(false);
|
||||
// Audio kept after a failed/timed-out transcription so the user can retry
|
||||
// instead of losing the recording.
|
||||
const pendingAudioRef = useRef<{ blob: Blob; ext: string } | null>(null);
|
||||
|
||||
const teardownPreviewAudio = useCallback(() => {
|
||||
if (pausePreviewTimeoutRef.current !== null) {
|
||||
window.clearTimeout(pausePreviewTimeoutRef.current);
|
||||
pausePreviewTimeoutRef.current = null;
|
||||
}
|
||||
if (previewAudioRef.current) {
|
||||
try {
|
||||
previewAudioRef.current.pause();
|
||||
previewAudioRef.current.src = "";
|
||||
} catch {}
|
||||
previewAudioRef.current = null;
|
||||
}
|
||||
if (previewUrlRef.current) {
|
||||
URL.revokeObjectURL(previewUrlRef.current);
|
||||
previewUrlRef.current = null;
|
||||
}
|
||||
setIsPlayingPreview(false);
|
||||
setPreviewCurrentTime(0);
|
||||
setPreviewDuration(0);
|
||||
}, []);
|
||||
|
||||
const resetCaptureState = useCallback(() => {
|
||||
captureRef.current = { analyser: null, startedAt: 0, pausedAccum: 0, pausedAt: null, finalDurationMs: null };
|
||||
}, []);
|
||||
const clearMaxTimeout = useCallback(() => {
|
||||
if (maxTimeoutRef.current !== null) {
|
||||
window.clearTimeout(maxTimeoutRef.current);
|
||||
@@ -47,8 +100,17 @@ export function useDictation({ onTranscript, onError }: UseDictationOptions) {
|
||||
streamRef.current.getTracks().forEach((track) => track.stop());
|
||||
streamRef.current = null;
|
||||
}
|
||||
if (audioContextRef.current) {
|
||||
void audioContextRef.current.close().catch(() => {});
|
||||
audioContextRef.current = null;
|
||||
}
|
||||
teardownPreviewAudio();
|
||||
resetCaptureState();
|
||||
setIsRecording(false);
|
||||
}, [clearMaxTimeout]);
|
||||
setIsPaused(false);
|
||||
setIsReviewing(false);
|
||||
setIsTranscribing(false);
|
||||
}, [clearMaxTimeout, teardownPreviewAudio, resetCaptureState]);
|
||||
|
||||
useEffect(() => {
|
||||
return () => {
|
||||
@@ -57,9 +119,184 @@ export function useDictation({ onTranscript, onError }: UseDictationOptions) {
|
||||
abortControllerRef.current.abort();
|
||||
abortControllerRef.current = null;
|
||||
}
|
||||
pendingAudioRef.current = null;
|
||||
teardownPreviewAudio();
|
||||
cleanup();
|
||||
};
|
||||
}, [cleanup]);
|
||||
}, [cleanup, teardownPreviewAudio]);
|
||||
|
||||
const runTranscription = useCallback(async (blob: Blob, ext: string) => {
|
||||
setIsTranscribing(true);
|
||||
const abortController = new AbortController();
|
||||
abortControllerRef.current = abortController;
|
||||
const timeoutId = window.setTimeout(() => {
|
||||
abortController.abort(new DOMException("Transcription timed out", "TimeoutError"));
|
||||
}, STT_TIMEOUT_MS);
|
||||
|
||||
try {
|
||||
const body = new FormData();
|
||||
body.append("file", blob, ext);
|
||||
|
||||
const res = await fetch("/api/stt", {
|
||||
method: "POST",
|
||||
body,
|
||||
signal: abortController.signal,
|
||||
});
|
||||
if (cancelledRef.current) return;
|
||||
const data = await res.json();
|
||||
if (cancelledRef.current) return;
|
||||
if (!res.ok) {
|
||||
const message = normalizeErrorMessage(data?.error, "Transcription failed");
|
||||
setTranscribeError(message);
|
||||
onError?.(message);
|
||||
return;
|
||||
}
|
||||
if (typeof data.text === "string" && data.text.trim()) {
|
||||
if (!cancelledRef.current) {
|
||||
pendingAudioRef.current = null;
|
||||
onTranscript(data.text.trim());
|
||||
}
|
||||
} else if (!cancelledRef.current) {
|
||||
const message = "No speech detected";
|
||||
setTranscribeError(message);
|
||||
onError?.(message);
|
||||
}
|
||||
} catch (err) {
|
||||
if (cancelledRef.current) return;
|
||||
if (err instanceof DOMException && err.name === "TimeoutError") {
|
||||
const message = "Transcription timed out";
|
||||
setTranscribeError(message);
|
||||
onError?.(message);
|
||||
return;
|
||||
}
|
||||
if (err instanceof DOMException && err.name === "AbortError") return;
|
||||
const message = err instanceof Error ? err.message : normalizeErrorMessage(err, "Transcription failed");
|
||||
setTranscribeError(message);
|
||||
onError?.(message);
|
||||
} finally {
|
||||
window.clearTimeout(timeoutId);
|
||||
if (abortControllerRef.current === abortController) {
|
||||
abortControllerRef.current = null;
|
||||
}
|
||||
setIsTranscribing(false);
|
||||
}
|
||||
}, [onTranscript, onError]);
|
||||
|
||||
// Ends capture and moves the session into transcription. Gates on the live
|
||||
// MediaRecorder state (not the isRecording closure) because the 5-minute
|
||||
// cap timeout captures this callback from the render that started the
|
||||
// recording. Sets isTranscribing eagerly so the deck never flashes back to
|
||||
// the text composer between recorder.stop() and the async onstop event.
|
||||
const getCapturedBlob = useCallback(() => {
|
||||
if (pendingAudioRef.current) return pendingAudioRef.current;
|
||||
if (chunksRef.current.length === 0) return null;
|
||||
const recorder = mediaRecorderRef.current;
|
||||
const mimeType = recorder?.mimeType || "audio/webm";
|
||||
const blob = new Blob(chunksRef.current, { type: mimeType });
|
||||
const ext = mimeType.includes("mp4") ? "audio.mp4" : mimeType.includes("ogg") ? "audio.ogg" : "audio.webm";
|
||||
return { blob, ext };
|
||||
}, []);
|
||||
|
||||
const setupPreviewAudio = useCallback((blob: Blob) => {
|
||||
// Pause/detach any previous preview element before replacing it so a
|
||||
// still-playing preview cannot outlive the controls that reference it.
|
||||
teardownPreviewAudio();
|
||||
try {
|
||||
const url = URL.createObjectURL(blob);
|
||||
previewUrlRef.current = url;
|
||||
const audio = new Audio(url);
|
||||
audio.preload = "auto";
|
||||
previewAudioRef.current = audio;
|
||||
audio.ontimeupdate = () => {
|
||||
setPreviewCurrentTime(audio.currentTime);
|
||||
};
|
||||
audio.onloadedmetadata = () => {
|
||||
if (Number.isFinite(audio.duration)) {
|
||||
setPreviewDuration(audio.duration);
|
||||
}
|
||||
};
|
||||
audio.onended = () => {
|
||||
setIsPlayingPreview(false);
|
||||
setPreviewCurrentTime(0);
|
||||
};
|
||||
audio.onpause = () => {
|
||||
setIsPlayingPreview(false);
|
||||
};
|
||||
audio.onplay = () => {
|
||||
setIsPlayingPreview(true);
|
||||
};
|
||||
audio.onerror = () => {
|
||||
setIsPlayingPreview(false);
|
||||
};
|
||||
return audio;
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
}, [teardownPreviewAudio]);
|
||||
|
||||
const playPreview = useCallback(() => {
|
||||
let audio = previewAudioRef.current;
|
||||
if (!audio) {
|
||||
const captured = getCapturedBlob();
|
||||
if (!captured) return;
|
||||
audio = setupPreviewAudio(captured.blob);
|
||||
}
|
||||
if (audio) {
|
||||
if (audio.ended || (Number.isFinite(audio.duration) && audio.currentTime >= audio.duration)) {
|
||||
audio.currentTime = 0;
|
||||
}
|
||||
audio.play().catch(() => {
|
||||
setIsPlayingPreview(false);
|
||||
});
|
||||
}
|
||||
}, [getCapturedBlob, setupPreviewAudio]);
|
||||
|
||||
const pausePreview = useCallback(() => {
|
||||
if (previewAudioRef.current) {
|
||||
previewAudioRef.current.pause();
|
||||
}
|
||||
setIsPlayingPreview(false);
|
||||
}, []);
|
||||
|
||||
const seekPreview = useCallback((seconds: number) => {
|
||||
if (previewAudioRef.current) {
|
||||
previewAudioRef.current.currentTime = Math.max(0, seconds);
|
||||
setPreviewCurrentTime(Math.max(0, seconds));
|
||||
}
|
||||
}, []);
|
||||
|
||||
// Ends capture and moves the session into transcription or review. Gates on the live
|
||||
// MediaRecorder state (not the isRecording closure) because the 5-minute
|
||||
// cap timeout captures this callback from the render that started the
|
||||
// recording.
|
||||
const finishCapture = useCallback((options?: { immediateSend?: boolean }) => {
|
||||
clearMaxTimeout();
|
||||
immediateSendRef.current = options?.immediateSend ?? false;
|
||||
const recorder = mediaRecorderRef.current;
|
||||
if (!recorder || recorder.state === "inactive") return;
|
||||
const capture = captureRef.current;
|
||||
const finalMs = Math.max(
|
||||
0,
|
||||
performance.now() - capture.startedAt - (capture.pausedAccum + (capture.pausedAt !== null ? performance.now() - capture.pausedAt : 0)),
|
||||
);
|
||||
capture.finalDurationMs = finalMs;
|
||||
try {
|
||||
recorder.stop();
|
||||
} catch {}
|
||||
if (audioContextRef.current) {
|
||||
void audioContextRef.current.close().catch(() => {});
|
||||
audioContextRef.current = null;
|
||||
}
|
||||
captureRef.current.analyser = null;
|
||||
setIsRecording(false);
|
||||
setIsPaused(false);
|
||||
if (options?.immediateSend) {
|
||||
setIsTranscribing(true);
|
||||
} else {
|
||||
setIsReviewing(true);
|
||||
}
|
||||
}, [clearMaxTimeout]);
|
||||
|
||||
const start = useCallback(async () => {
|
||||
if (isStartingRef.current || isRecording || isTranscribing) return;
|
||||
isStartingRef.current = true;
|
||||
@@ -69,6 +306,13 @@ export function useDictation({ onTranscript, onError }: UseDictationOptions) {
|
||||
abortControllerRef.current = null;
|
||||
}
|
||||
cancelledRef.current = false;
|
||||
chunksRef.current = [];
|
||||
immediateSendRef.current = false;
|
||||
pendingAudioRef.current = null;
|
||||
teardownPreviewAudio();
|
||||
setTranscribeError(null);
|
||||
setIsTranscribing(false);
|
||||
setIsReviewing(false);
|
||||
if (
|
||||
typeof navigator?.mediaDevices?.getUserMedia !== "function" ||
|
||||
typeof window === "undefined" ||
|
||||
@@ -82,83 +326,64 @@ export function useDictation({ onTranscript, onError }: UseDictationOptions) {
|
||||
const stream = await navigator.mediaDevices.getUserMedia({ audio: true });
|
||||
streamRef.current = stream;
|
||||
|
||||
captureRef.current = { analyser: null, startedAt: performance.now(), pausedAccum: 0, pausedAt: null };
|
||||
if (typeof window.AudioContext === "function") {
|
||||
let ctx: AudioContext | null = null;
|
||||
try {
|
||||
ctx = new AudioContext();
|
||||
audioContextRef.current = ctx;
|
||||
if (ctx.state === "suspended") void ctx.resume();
|
||||
const source = ctx.createMediaStreamSource(stream);
|
||||
const analyser = ctx.createAnalyser();
|
||||
analyser.fftSize = 256;
|
||||
source.connect(analyser);
|
||||
captureRef.current.analyser = analyser;
|
||||
} catch {
|
||||
// Own the failure: never leave a live context behind untracked.
|
||||
void ctx?.close().catch(() => {});
|
||||
audioContextRef.current = null;
|
||||
}
|
||||
}
|
||||
|
||||
const recorder = new MediaRecorder(stream);
|
||||
mediaRecorderRef.current = recorder;
|
||||
const chunks: Blob[] = [];
|
||||
chunksRef.current = [];
|
||||
|
||||
recorder.ondataavailable = (event) => {
|
||||
if (event.data.size > 0) chunks.push(event.data);
|
||||
if (event.data.size > 0) chunksRef.current.push(event.data);
|
||||
};
|
||||
|
||||
recorder.onstop = async () => {
|
||||
if (cancelledRef.current) return;
|
||||
clearMaxTimeout();
|
||||
if (chunks.length === 0) return;
|
||||
setIsTranscribing(true);
|
||||
const abortController = new AbortController();
|
||||
abortControllerRef.current = abortController;
|
||||
const timeoutId = window.setTimeout(() => {
|
||||
abortController.abort(new DOMException("Transcription timed out", "TimeoutError"));
|
||||
}, STT_TIMEOUT_MS);
|
||||
|
||||
try {
|
||||
const mimeType = recorder.mimeType || "audio/webm";
|
||||
const blob = new Blob(chunks, { type: mimeType });
|
||||
const ext = mimeType.includes("mp4") ? "audio.mp4" : mimeType.includes("ogg") ? "audio.ogg" : "audio.webm";
|
||||
const body = new FormData();
|
||||
body.append("file", blob, ext);
|
||||
|
||||
const res = await fetch("/api/stt", {
|
||||
method: "POST",
|
||||
body,
|
||||
signal: abortController.signal,
|
||||
});
|
||||
if (cancelledRef.current) return;
|
||||
const data = await res.json();
|
||||
if (cancelledRef.current) return;
|
||||
if (!res.ok) {
|
||||
onError?.(normalizeErrorMessage(data?.error, "Transcription failed"));
|
||||
return;
|
||||
}
|
||||
if (typeof data.text === "string" && data.text.trim()) {
|
||||
if (!cancelledRef.current) {
|
||||
onTranscript(data.text.trim());
|
||||
}
|
||||
} else {
|
||||
if (!cancelledRef.current) {
|
||||
onError?.("No speech detected");
|
||||
}
|
||||
}
|
||||
} catch (err) {
|
||||
if (cancelledRef.current) return;
|
||||
if (err instanceof DOMException && err.name === "AbortError") return;
|
||||
onError?.(
|
||||
err instanceof Error
|
||||
? err.message
|
||||
: normalizeErrorMessage(err, "Transcription failed"),
|
||||
);
|
||||
} finally {
|
||||
window.clearTimeout(timeoutId);
|
||||
if (abortControllerRef.current === abortController) {
|
||||
abortControllerRef.current = null;
|
||||
}
|
||||
setIsTranscribing(false);
|
||||
}
|
||||
};
|
||||
|
||||
recorder.start();
|
||||
setIsRecording(true);
|
||||
maxTimeoutRef.current = window.setTimeout(() => {
|
||||
if (mediaRecorderRef.current?.state === "recording") {
|
||||
try {
|
||||
mediaRecorderRef.current.stop();
|
||||
} catch {}
|
||||
}
|
||||
recorder.onstop = () => {
|
||||
if (streamRef.current) {
|
||||
streamRef.current.getTracks().forEach((track) => track.stop());
|
||||
streamRef.current = null;
|
||||
}
|
||||
setIsRecording(false);
|
||||
if (cancelledRef.current) return;
|
||||
clearMaxTimeout();
|
||||
if (chunksRef.current.length === 0) {
|
||||
const message = "No speech detected";
|
||||
setTranscribeError(message);
|
||||
setIsTranscribing(false);
|
||||
setIsReviewing(false);
|
||||
onError?.(message);
|
||||
return;
|
||||
}
|
||||
const mimeType = recorder.mimeType || "audio/webm";
|
||||
const blob = new Blob(chunksRef.current, { type: mimeType });
|
||||
const ext = mimeType.includes("mp4") ? "audio.mp4" : mimeType.includes("ogg") ? "audio.ogg" : "audio.webm";
|
||||
pendingAudioRef.current = { blob, ext };
|
||||
if (immediateSendRef.current) {
|
||||
void runTranscription(blob, ext);
|
||||
} else {
|
||||
setIsReviewing(true);
|
||||
setupPreviewAudio(blob);
|
||||
}
|
||||
};
|
||||
|
||||
recorder.start(100);
|
||||
setIsRecording(true);
|
||||
maxTimeoutRef.current = window.setTimeout(() => {
|
||||
finishCapture();
|
||||
}, MAX_RECORDING_MS);
|
||||
} catch (err) {
|
||||
cleanup();
|
||||
@@ -166,37 +391,99 @@ export function useDictation({ onTranscript, onError }: UseDictationOptions) {
|
||||
} finally {
|
||||
isStartingRef.current = false;
|
||||
}
|
||||
}, [isRecording, isTranscribing, cleanup, clearMaxTimeout, onTranscript, onError]);
|
||||
}, [isRecording, isTranscribing, cleanup, clearMaxTimeout, runTranscription, onError, finishCapture]);
|
||||
|
||||
const stop = useCallback(() => {
|
||||
if (!isRecording) return;
|
||||
clearMaxTimeout();
|
||||
if (mediaRecorderRef.current && mediaRecorderRef.current.state !== "inactive") {
|
||||
const togglePause = useCallback(() => {
|
||||
const recorder = mediaRecorderRef.current;
|
||||
if (!recorder || isTranscribing || transcribeError) return;
|
||||
if (recorder.state === "recording") {
|
||||
try {
|
||||
mediaRecorderRef.current.stop();
|
||||
recorder.requestData();
|
||||
} catch {}
|
||||
recorder.pause();
|
||||
captureRef.current.pausedAt = performance.now();
|
||||
setIsPaused(true);
|
||||
if (pausePreviewTimeoutRef.current !== null) {
|
||||
window.clearTimeout(pausePreviewTimeoutRef.current);
|
||||
}
|
||||
pausePreviewTimeoutRef.current = window.setTimeout(() => {
|
||||
pausePreviewTimeoutRef.current = null;
|
||||
const captured = getCapturedBlob();
|
||||
if (captured) setupPreviewAudio(captured.blob);
|
||||
}, 0);
|
||||
} else if (recorder.state === "paused") {
|
||||
if (pausePreviewTimeoutRef.current !== null) {
|
||||
window.clearTimeout(pausePreviewTimeoutRef.current);
|
||||
pausePreviewTimeoutRef.current = null;
|
||||
}
|
||||
pausePreview();
|
||||
teardownPreviewAudio();
|
||||
const { pausedAt, pausedAccum } = captureRef.current;
|
||||
if (pausedAt !== null) {
|
||||
captureRef.current.pausedAccum = pausedAccum + (performance.now() - pausedAt);
|
||||
}
|
||||
captureRef.current.pausedAt = null;
|
||||
recorder.resume();
|
||||
setIsPaused(false);
|
||||
}
|
||||
if (streamRef.current) {
|
||||
streamRef.current.getTracks().forEach((track) => track.stop());
|
||||
streamRef.current = null;
|
||||
}
|
||||
setIsRecording(false);
|
||||
}, [isRecording, clearMaxTimeout]);
|
||||
}, [isTranscribing, transcribeError, getCapturedBlob, setupPreviewAudio, pausePreview, teardownPreviewAudio]);
|
||||
|
||||
const confirmTranscribe = useCallback(() => {
|
||||
const pending = pendingAudioRef.current || getCapturedBlob();
|
||||
if (!pending || isTranscribing) return;
|
||||
pausePreview();
|
||||
teardownPreviewAudio();
|
||||
setIsReviewing(false);
|
||||
setTranscribeError(null);
|
||||
void runTranscription(pending.blob, pending.ext);
|
||||
}, [getCapturedBlob, isTranscribing, pausePreview, teardownPreviewAudio, runTranscription]);
|
||||
const cancel = useCallback(() => {
|
||||
cancelledRef.current = true;
|
||||
if (abortControllerRef.current) {
|
||||
abortControllerRef.current.abort();
|
||||
abortControllerRef.current = null;
|
||||
}
|
||||
pendingAudioRef.current = null;
|
||||
setTranscribeError(null);
|
||||
setIsTranscribing(false);
|
||||
setIsReviewing(false);
|
||||
teardownPreviewAudio();
|
||||
resetCaptureState();
|
||||
cleanup();
|
||||
}, [cleanup]);
|
||||
}, [cleanup, teardownPreviewAudio, resetCaptureState]);
|
||||
|
||||
const retry = useCallback(() => {
|
||||
const pending = pendingAudioRef.current;
|
||||
if (!pending || isTranscribing) return;
|
||||
pausePreview();
|
||||
teardownPreviewAudio();
|
||||
setTranscribeError(null);
|
||||
void runTranscription(pending.blob, pending.ext);
|
||||
}, [isTranscribing, pausePreview, teardownPreviewAudio, runTranscription]);
|
||||
const toggle = useCallback(() => {
|
||||
if (isRecording) stop();
|
||||
if (isRecording) finishCapture();
|
||||
else void start();
|
||||
}, [isRecording, start, stop]);
|
||||
}, [isRecording, start, finishCapture]);
|
||||
|
||||
return { isRecording, isTranscribing, start, stop, cancel, toggle };
|
||||
return {
|
||||
isRecording,
|
||||
isPaused,
|
||||
isReviewing,
|
||||
isTranscribing,
|
||||
isPlayingPreview,
|
||||
previewCurrentTime,
|
||||
previewDuration,
|
||||
transcribeError,
|
||||
captureRef,
|
||||
start,
|
||||
stop: finishCapture,
|
||||
cancel,
|
||||
toggle,
|
||||
togglePause,
|
||||
retry,
|
||||
playPreview,
|
||||
pausePreview,
|
||||
seekPreview,
|
||||
confirmTranscribe,
|
||||
};
|
||||
}
|
||||
@@ -260,12 +260,24 @@
|
||||
"chatInput.plusOff": "Off",
|
||||
"chatInput.attachImage": "Attach image",
|
||||
"chatInput.startDictation": "Start dictation",
|
||||
"chatInput.stopDictation": "Stop dictation",
|
||||
"chatInput.transcribing": "Transcribing...",
|
||||
"chatInput.dictationFailed": "Transcription failed",
|
||||
"chatInput.dictationNoSpeech": "No speech detected",
|
||||
"chatInput.dictationNotSupported": "Microphone not supported in this browser or context",
|
||||
"chatInput.dictationPermissionDenied": "Microphone access denied",
|
||||
"chatInput.dictationTimedOut": "Transcription timed out",
|
||||
"chatInput.dictationSuccess": "Transcribed",
|
||||
"chatInput.dictationRecording": "Recording",
|
||||
"chatInput.pauseDictation": "Pause recording",
|
||||
"chatInput.resumeDictation": "Resume recording",
|
||||
"chatInput.sendDictation": "Transcribe and send",
|
||||
"chatInput.retryDictation": "Retry transcription",
|
||||
"chatInput.cancelDictation": "Cancel and discard",
|
||||
"chatInput.discardDictation": "Discard recording",
|
||||
"chatInput.playPreview": "Play preview",
|
||||
"chatInput.pausePreview": "Pause preview",
|
||||
"chatInput.stopDictation": "Stop recording",
|
||||
"chatInput.transcribeDictation": "Transcribe",
|
||||
"chatInput.fastLabel": "Fast",
|
||||
"chatInput.queuedDelete": "Delete",
|
||||
"chatInput.queuedDeleteTitle": "Remove this queued message",
|
||||
|
||||
@@ -247,12 +247,24 @@
|
||||
"chatInput.plusOff": "オフ",
|
||||
"chatInput.attachImage": "画像を添付",
|
||||
"chatInput.startDictation": "音声入力を開始",
|
||||
"chatInput.stopDictation": "音声入力を停止",
|
||||
"chatInput.transcribing": "文字起こし中...",
|
||||
"chatInput.dictationFailed": "文字起こしに失敗しました",
|
||||
"chatInput.dictationNoSpeech": "音声が検出されませんでした",
|
||||
"chatInput.dictationNotSupported": "お使いのブラウザまたは環境ではマイクがサポートされていません",
|
||||
"chatInput.dictationPermissionDenied": "マイクへのアクセスが拒否されました",
|
||||
"chatInput.dictationTimedOut": "文字起こしがタイムアウトしました",
|
||||
"chatInput.dictationSuccess": "文字起こしが完了しました",
|
||||
"chatInput.dictationRecording": "録音中",
|
||||
"chatInput.pauseDictation": "録音を一時停止",
|
||||
"chatInput.resumeDictation": "録音を再開",
|
||||
"chatInput.sendDictation": "文字起こしして送信",
|
||||
"chatInput.retryDictation": "文字起こしを再試行",
|
||||
"chatInput.cancelDictation": "キャンセルして破棄",
|
||||
"chatInput.discardDictation": "録音を破棄",
|
||||
"chatInput.playPreview": "プレビューを再生",
|
||||
"chatInput.pausePreview": "プレビューを一時停止",
|
||||
"chatInput.stopDictation": "録音を停止",
|
||||
"chatInput.transcribeDictation": "文字起こし",
|
||||
"chatInput.fastLabel": "高速",
|
||||
"chatInput.queuedDelete": "削除",
|
||||
"chatInput.queuedDeleteTitle": "このキューされたメッセージを削除",
|
||||
|
||||
@@ -247,12 +247,24 @@
|
||||
"chatInput.plusOff": "关",
|
||||
"chatInput.attachImage": "附加图片",
|
||||
"chatInput.startDictation": "开始语音输入",
|
||||
"chatInput.stopDictation": "停止语音输入",
|
||||
"chatInput.transcribing": "正在转录...",
|
||||
"chatInput.dictationFailed": "转录失败",
|
||||
"chatInput.dictationNoSpeech": "未检测到语音",
|
||||
"chatInput.dictationNotSupported": "当前浏览器或环境不支持麦克风",
|
||||
"chatInput.dictationPermissionDenied": "麦克风访问被拒绝",
|
||||
"chatInput.dictationTimedOut": "转写超时",
|
||||
"chatInput.dictationSuccess": "转写完成",
|
||||
"chatInput.dictationRecording": "录音中",
|
||||
"chatInput.pauseDictation": "暂停录音",
|
||||
"chatInput.resumeDictation": "恢复录音",
|
||||
"chatInput.sendDictation": "转写并发送",
|
||||
"chatInput.retryDictation": "重试转写",
|
||||
"chatInput.cancelDictation": "取消并丢弃",
|
||||
"chatInput.discardDictation": "丢弃录音",
|
||||
"chatInput.playPreview": "播放预览",
|
||||
"chatInput.pausePreview": "暂停预览",
|
||||
"chatInput.stopDictation": "停止录音",
|
||||
"chatInput.transcribeDictation": "转写",
|
||||
"chatInput.fastLabel": "快速",
|
||||
"chatInput.queuedDelete": "删除",
|
||||
"chatInput.queuedDeleteTitle": "移除这条已排队消息",
|
||||
|
||||
+1
-1
@@ -58,7 +58,7 @@ const nextConfig = (phase: string): NextConfig => {
|
||||
{ key: "X-Frame-Options", value: "DENY" },
|
||||
{ key: "Referrer-Policy", value: "no-referrer" },
|
||||
{ key: "Permissions-Policy", value: "camera=(), microphone=(self), geolocation=()" },
|
||||
{ key: "Content-Security-Policy", value: "default-src 'self'; base-uri 'self'; form-action 'self'; frame-ancestors 'none'; object-src 'none'; img-src 'self' data: blob: https:; style-src 'self' 'unsafe-inline'; script-src 'self' 'unsafe-inline' 'unsafe-eval'; connect-src 'self' ws: wss:; font-src 'self' data:" },
|
||||
{ key: "Content-Security-Policy", value: "default-src 'self'; base-uri 'self'; form-action 'self'; frame-ancestors 'none'; object-src 'none'; img-src 'self' data: blob: https:; style-src 'self' 'unsafe-inline'; script-src 'self' 'unsafe-inline' 'unsafe-eval'; connect-src 'self' ws: wss:; font-src 'self' data:; media-src 'self' blob:" },
|
||||
];
|
||||
// /api/files streams workspace files whose document policy depends on the
|
||||
// content type (strict CSP for SVG, the DOCX preview policy, none for
|
||||
|
||||
Reference in new issue
Block a user