Add voice input with audio recording and transcription (#85)
* Add voice transcription to input box Add a microphone button to the task input that records audio and transcribes it using ElevenLabs via the AI SDK. The button shows a pulsing animation while recording and a spinner while processing. Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com> * Make mic button round and focus input after transcription Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com> * Use subtler opacity pulse for mic recording indicator Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com> * Add voice transcription and multiline input to chat view Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com> * Fix file suggestions dropdown being clipped by overflow-hidden The FileSuggestionsDropdown was rendered inside a form with overflow-hidden, causing the dropdown (positioned with absolute bottom-full) to be clipped. Moved the dropdown outside the form into a new relative wrapper div. Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com> * Fix voice transcription issues: Safari compat, error display, and size limits - Add browser compatibility for MediaRecorder by detecting supported MIME types - Display recording errors to users in both input components - Make mimeType optional in transcribe API since it's not used - Add 7.5MB size limit for audio uploads - Fix textarea auto-resize flicker in chat view - Use consistent Button component for mic button Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com> * Add grep and xargs cat to allowed bash commands Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com> --------- Co-authored-by: Claude Opus 4.5 <noreply@anthropic.com>
This commit is contained in:
parent
7ec78fb6c6
commit
49c09062ee
@ -16,7 +16,9 @@
|
||||
"Bash(cat:*)",
|
||||
"Bash(jq:*)",
|
||||
"Bash(python3 -c:*)",
|
||||
"Bash(git commit:*)"
|
||||
"Bash(git commit:*)",
|
||||
"Bash(grep:*)",
|
||||
"Bash(xargs cat:*)"
|
||||
]
|
||||
},
|
||||
"hooks": {
|
||||
|
||||
56
apps/web/app/api/transcribe/route.ts
Normal file
56
apps/web/app/api/transcribe/route.ts
Normal file
@ -0,0 +1,56 @@
|
||||
import { experimental_transcribe as transcribe } from "ai";
|
||||
import { elevenlabs } from "@ai-sdk/elevenlabs";
|
||||
import { getServerSession } from "@/lib/session/get-server-session";
|
||||
|
||||
interface TranscribeRequestBody {
|
||||
audio: string; // base64-encoded audio data
|
||||
mimeType?: string; // e.g., "audio/webm" - accepted but not currently used
|
||||
}
|
||||
|
||||
export async function POST(req: Request) {
|
||||
const session = await getServerSession();
|
||||
if (!session?.user) {
|
||||
return Response.json({ error: "Not authenticated" }, { status: 401 });
|
||||
}
|
||||
|
||||
let body: TranscribeRequestBody;
|
||||
try {
|
||||
body = (await req.json()) as TranscribeRequestBody;
|
||||
} catch {
|
||||
return Response.json({ error: "Invalid JSON body" }, { status: 400 });
|
||||
}
|
||||
|
||||
const { audio } = body;
|
||||
|
||||
if (!audio) {
|
||||
return Response.json(
|
||||
{ error: "Missing required field: audio" },
|
||||
{ status: 400 },
|
||||
);
|
||||
}
|
||||
|
||||
// Limit audio size to ~7.5MB of raw audio (10MB base64)
|
||||
const maxBase64Length = 10 * 1024 * 1024;
|
||||
if (audio.length > maxBase64Length) {
|
||||
return Response.json(
|
||||
{ error: "Audio file too large. Maximum size is approximately 7.5MB." },
|
||||
{ status: 413 },
|
||||
);
|
||||
}
|
||||
|
||||
try {
|
||||
const result = await transcribe({
|
||||
model: elevenlabs.transcription("scribe_v1"),
|
||||
audio: audio, // base64 string is accepted directly
|
||||
});
|
||||
|
||||
return Response.json({ text: result.text });
|
||||
} catch (error) {
|
||||
const message = error instanceof Error ? error.message : String(error);
|
||||
console.error("Transcription failed:", message);
|
||||
return Response.json(
|
||||
{ error: "Transcription failed", details: message },
|
||||
{ status: 500 },
|
||||
);
|
||||
}
|
||||
}
|
||||
@ -20,6 +20,7 @@ import {
|
||||
GitCompare,
|
||||
Paperclip,
|
||||
Loader2,
|
||||
Mic,
|
||||
} from "lucide-react";
|
||||
|
||||
import { Button } from "@/components/ui/button";
|
||||
@ -35,6 +36,7 @@ import { CreateRepoDialog } from "@/components/create-repo-dialog";
|
||||
import { ImageAttachmentsPreview } from "@/components/image-attachments-preview";
|
||||
import { useScrollToBottom } from "@/hooks/use-scroll-to-bottom";
|
||||
import { useImageAttachments } from "@/hooks/use-image-attachments";
|
||||
import { useAudioRecording } from "@/hooks/use-audio-recording";
|
||||
import { ACCEPT_IMAGE_TYPES, isValidImageType } from "@/lib/image-utils";
|
||||
import type { WebAgentUIToolPart, WebAgentUIMessagePart } from "@/app/types";
|
||||
import type { TaskToolUIPart } from "@open-harness/agent";
|
||||
@ -372,7 +374,52 @@ export function TaskDetailContent() {
|
||||
const [showDiffPanel, setShowDiffPanel] = useState(false);
|
||||
const [cursorPosition, setCursorPosition] = useState(0);
|
||||
const [isDragging, setIsDragging] = useState(false);
|
||||
const inputRef = useRef<HTMLInputElement>(null);
|
||||
const inputRef = useRef<HTMLTextAreaElement>(null);
|
||||
const {
|
||||
state: recordingState,
|
||||
error: recordingError,
|
||||
clearError: clearRecordingError,
|
||||
toggleRecording,
|
||||
} = useAudioRecording();
|
||||
|
||||
const handleMicClick = async () => {
|
||||
clearRecordingError();
|
||||
const transcribedText = await toggleRecording();
|
||||
if (transcribedText) {
|
||||
setInput((prev) =>
|
||||
prev ? `${prev} ${transcribedText}` : transcribedText,
|
||||
);
|
||||
inputRef.current?.focus();
|
||||
}
|
||||
};
|
||||
|
||||
// Auto-resize textarea up to 3 lines
|
||||
useEffect(() => {
|
||||
const textarea = inputRef.current;
|
||||
if (!textarea) return;
|
||||
|
||||
const computedStyle = getComputedStyle(textarea);
|
||||
const lineHeight = parseFloat(computedStyle.lineHeight) || 24;
|
||||
const maxLines = 3;
|
||||
const maxHeight = lineHeight * maxLines;
|
||||
|
||||
// Store current height to avoid flicker
|
||||
const currentHeight = textarea.offsetHeight;
|
||||
|
||||
// Temporarily set height to 0 to measure scrollHeight accurately
|
||||
textarea.style.height = "0";
|
||||
const scrollHeight = textarea.scrollHeight;
|
||||
|
||||
// Set new height, capped at max
|
||||
const newHeight = Math.min(scrollHeight, maxHeight);
|
||||
|
||||
// Only update if height actually changed to minimize reflows
|
||||
if (Math.abs(newHeight - currentHeight) > 1) {
|
||||
textarea.style.height = `${newHeight}px`;
|
||||
} else {
|
||||
textarea.style.height = `${currentHeight}px`;
|
||||
}
|
||||
}, [input]);
|
||||
|
||||
const {
|
||||
images,
|
||||
@ -1155,54 +1202,7 @@ export function TaskDetailContent() {
|
||||
}}
|
||||
className="hidden"
|
||||
/>
|
||||
<form
|
||||
onSubmit={(e) => {
|
||||
e.preventDefault();
|
||||
const hasContent = input.trim() || images.length > 0;
|
||||
if (!hasContent || !isSandboxValid(sandboxInfo)) return;
|
||||
|
||||
const messageText = input;
|
||||
const files = getFileParts();
|
||||
setInput("");
|
||||
clearImages();
|
||||
|
||||
sendMessage({ text: messageText, files });
|
||||
}}
|
||||
onDragOver={(e) => {
|
||||
e.preventDefault();
|
||||
setIsDragging(true);
|
||||
}}
|
||||
onDragLeave={(e) => {
|
||||
e.preventDefault();
|
||||
// Only set isDragging to false if we're leaving the form entirely
|
||||
// (not just moving to a child element)
|
||||
if (!e.currentTarget.contains(e.relatedTarget as Node)) {
|
||||
setIsDragging(false);
|
||||
}
|
||||
}}
|
||||
onDrop={(e) => {
|
||||
e.preventDefault();
|
||||
setIsDragging(false);
|
||||
const files = e.dataTransfer.files;
|
||||
if (files.length > 0) {
|
||||
addImages(files);
|
||||
}
|
||||
}}
|
||||
className={`relative overflow-hidden rounded-2xl bg-muted transition-colors ${isDragging ? "ring-2 ring-blue-500/50" : ""}`}
|
||||
>
|
||||
{/* Sandbox overlay when inactive */}
|
||||
<SandboxInputOverlay
|
||||
sandboxInfo={sandboxInfo}
|
||||
isCreating={isCreatingSandbox}
|
||||
isRestoring={isRestoringSnapshot}
|
||||
hasSnapshot={!!task.snapshotUrl}
|
||||
onRestore={handleRestoreSnapshot}
|
||||
onCreateNew={handleCreateNewSandbox}
|
||||
/>
|
||||
|
||||
{/* Image attachments preview */}
|
||||
<ImageAttachmentsPreview images={images} onRemove={removeImage} />
|
||||
|
||||
<div className="relative">
|
||||
{showSuggestions && (
|
||||
<FileSuggestionsDropdown
|
||||
suggestions={suggestions}
|
||||
@ -1219,81 +1219,171 @@ export function TaskDetailContent() {
|
||||
isLoading={fileCache.isLoading}
|
||||
/>
|
||||
)}
|
||||
<form
|
||||
onSubmit={(e) => {
|
||||
e.preventDefault();
|
||||
const hasContent = input.trim() || images.length > 0;
|
||||
if (!hasContent || !isSandboxValid(sandboxInfo)) return;
|
||||
|
||||
<div className="flex items-center gap-2 px-4 py-2">
|
||||
<Button
|
||||
type="button"
|
||||
variant="ghost"
|
||||
size="icon"
|
||||
onClick={openFilePicker}
|
||||
className="h-8 w-8 rounded-full text-muted-foreground hover:text-foreground"
|
||||
>
|
||||
<Paperclip className="h-4 w-4" />
|
||||
</Button>
|
||||
<input
|
||||
ref={inputRef}
|
||||
value={input}
|
||||
placeholder="Request changes or ask a ..."
|
||||
onChange={(e) => {
|
||||
setInput(e.currentTarget.value);
|
||||
setCursorPosition(e.currentTarget.selectionStart ?? 0);
|
||||
}}
|
||||
onKeyDown={(e) => {
|
||||
// Let suggestions handle keyboard events first
|
||||
if (handleSuggestionsKeyDown(e)) {
|
||||
return;
|
||||
}
|
||||
// Handle form submission
|
||||
if (e.key === "Enter" && !e.shiftKey) {
|
||||
e.preventDefault();
|
||||
e.currentTarget.form?.requestSubmit();
|
||||
}
|
||||
}}
|
||||
onKeyUp={(e) => {
|
||||
setCursorPosition(e.currentTarget.selectionStart ?? 0);
|
||||
}}
|
||||
onClick={(e) => {
|
||||
setCursorPosition(e.currentTarget.selectionStart ?? 0);
|
||||
}}
|
||||
onPaste={(e) => {
|
||||
const items = e.clipboardData?.items;
|
||||
if (!items) return;
|
||||
for (const item of items) {
|
||||
if (isValidImageType(item.type)) {
|
||||
const file = item.getAsFile();
|
||||
if (file) {
|
||||
e.preventDefault();
|
||||
addImage(file).catch(() => {
|
||||
// Silently ignore paste errors - rare edge case
|
||||
});
|
||||
const messageText = input;
|
||||
const files = getFileParts();
|
||||
setInput("");
|
||||
clearImages();
|
||||
|
||||
sendMessage({ text: messageText, files });
|
||||
}}
|
||||
onDragOver={(e) => {
|
||||
e.preventDefault();
|
||||
setIsDragging(true);
|
||||
}}
|
||||
onDragLeave={(e) => {
|
||||
e.preventDefault();
|
||||
// Only set isDragging to false if we're leaving the form entirely
|
||||
// (not just moving to a child element)
|
||||
if (!e.currentTarget.contains(e.relatedTarget as Node)) {
|
||||
setIsDragging(false);
|
||||
}
|
||||
}}
|
||||
onDrop={(e) => {
|
||||
e.preventDefault();
|
||||
setIsDragging(false);
|
||||
const files = e.dataTransfer.files;
|
||||
if (files.length > 0) {
|
||||
addImages(files);
|
||||
}
|
||||
}}
|
||||
className={`overflow-hidden rounded-2xl bg-muted transition-colors ${isDragging ? "ring-2 ring-blue-500/50" : ""}`}
|
||||
>
|
||||
{/* Sandbox overlay when inactive */}
|
||||
<SandboxInputOverlay
|
||||
sandboxInfo={sandboxInfo}
|
||||
isCreating={isCreatingSandbox}
|
||||
isRestoring={isRestoringSnapshot}
|
||||
hasSnapshot={!!task.snapshotUrl}
|
||||
onRestore={handleRestoreSnapshot}
|
||||
onCreateNew={handleCreateNewSandbox}
|
||||
/>
|
||||
|
||||
{/* Image attachments preview */}
|
||||
<ImageAttachmentsPreview
|
||||
images={images}
|
||||
onRemove={removeImage}
|
||||
/>
|
||||
|
||||
{/* Textarea area */}
|
||||
<div className="px-4 pb-2 pt-3">
|
||||
<textarea
|
||||
ref={inputRef}
|
||||
value={input}
|
||||
placeholder="Request changes or ask a question..."
|
||||
rows={1}
|
||||
onChange={(e) => {
|
||||
setInput(e.currentTarget.value);
|
||||
setCursorPosition(e.currentTarget.selectionStart ?? 0);
|
||||
}}
|
||||
onKeyDown={(e) => {
|
||||
// Let suggestions handle keyboard events first
|
||||
if (handleSuggestionsKeyDown(e)) {
|
||||
return;
|
||||
}
|
||||
// Handle form submission
|
||||
if (e.key === "Enter" && !e.shiftKey) {
|
||||
e.preventDefault();
|
||||
e.currentTarget.form?.requestSubmit();
|
||||
}
|
||||
}}
|
||||
onKeyUp={(e) => {
|
||||
setCursorPosition(e.currentTarget.selectionStart ?? 0);
|
||||
}}
|
||||
onClick={(e) => {
|
||||
setCursorPosition(e.currentTarget.selectionStart ?? 0);
|
||||
}}
|
||||
onPaste={(e) => {
|
||||
const items = e.clipboardData?.items;
|
||||
if (!items) return;
|
||||
for (const item of items) {
|
||||
if (isValidImageType(item.type)) {
|
||||
const file = item.getAsFile();
|
||||
if (file) {
|
||||
e.preventDefault();
|
||||
addImage(file).catch(() => {
|
||||
// Silently ignore paste errors - rare edge case
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}}
|
||||
disabled={status === "streaming"}
|
||||
className="flex-1 bg-transparent text-foreground placeholder:text-muted-foreground focus:outline-none"
|
||||
/>
|
||||
{status === "streaming" ? (
|
||||
}}
|
||||
disabled={status === "streaming"}
|
||||
className="w-full resize-none overflow-y-auto bg-transparent text-foreground placeholder:text-muted-foreground focus:outline-none"
|
||||
style={{ minHeight: "24px" }}
|
||||
/>
|
||||
</div>
|
||||
|
||||
{/* Bottom toolbar */}
|
||||
<div className="flex items-center justify-between px-3 pb-2">
|
||||
<Button
|
||||
type="button"
|
||||
variant="ghost"
|
||||
size="icon"
|
||||
onClick={stop}
|
||||
className="h-8 w-8 rounded-full bg-destructive text-destructive-foreground hover:bg-destructive/90"
|
||||
onClick={openFilePicker}
|
||||
className="h-8 w-8 rounded-full text-muted-foreground hover:text-foreground"
|
||||
>
|
||||
<Square className="h-3 w-3 fill-current" />
|
||||
<Paperclip className="h-4 w-4" />
|
||||
</Button>
|
||||
) : (
|
||||
<Button
|
||||
type="submit"
|
||||
size="icon"
|
||||
disabled={!input.trim() && images.length === 0}
|
||||
className="h-8 w-8 rounded-full bg-primary text-primary-foreground hover:bg-primary/90 disabled:opacity-30"
|
||||
>
|
||||
<ArrowUp className="h-4 w-4" />
|
||||
</Button>
|
||||
)}
|
||||
</div>
|
||||
</form>
|
||||
|
||||
<div className="flex items-center gap-1">
|
||||
<Button
|
||||
type="button"
|
||||
variant="ghost"
|
||||
size="icon"
|
||||
onClick={handleMicClick}
|
||||
disabled={recordingState === "processing"}
|
||||
className={`relative h-8 w-8 rounded-full ${
|
||||
recordingState === "recording"
|
||||
? "text-red-500"
|
||||
: "text-muted-foreground hover:text-foreground"
|
||||
}`}
|
||||
>
|
||||
{recordingState === "recording" && (
|
||||
<span className="absolute inset-0 animate-pulse rounded-full bg-red-500/30" />
|
||||
)}
|
||||
{recordingState === "processing" ? (
|
||||
<Loader2 className="h-5 w-5 animate-spin" />
|
||||
) : (
|
||||
<Mic className="h-5 w-5" />
|
||||
)}
|
||||
</Button>
|
||||
|
||||
{status === "streaming" ? (
|
||||
<Button
|
||||
type="button"
|
||||
size="icon"
|
||||
onClick={stop}
|
||||
className="h-8 w-8 rounded-full bg-destructive text-destructive-foreground hover:bg-destructive/90"
|
||||
>
|
||||
<Square className="h-3 w-3 fill-current" />
|
||||
</Button>
|
||||
) : (
|
||||
<Button
|
||||
type="submit"
|
||||
size="icon"
|
||||
disabled={!input.trim() && images.length === 0}
|
||||
className="h-8 w-8 rounded-full bg-primary text-primary-foreground hover:bg-primary/90 disabled:opacity-30"
|
||||
>
|
||||
<ArrowUp className="h-4 w-4" />
|
||||
</Button>
|
||||
)}
|
||||
</div>
|
||||
</div>
|
||||
</form>
|
||||
|
||||
{/* Recording error message */}
|
||||
{recordingError && (
|
||||
<p className="mt-2 text-sm text-destructive">
|
||||
{recordingError}
|
||||
</p>
|
||||
)}
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
@ -2,10 +2,11 @@
|
||||
|
||||
import { useState, useRef, useEffect } from "react";
|
||||
import type { FileUIPart } from "ai";
|
||||
import { Plus, Mic, ArrowUp } from "lucide-react";
|
||||
import { Plus, Mic, ArrowUp, Loader2 } from "lucide-react";
|
||||
import { cn } from "@/lib/utils";
|
||||
import { ACCEPT_IMAGE_TYPES, isValidImageType } from "@/lib/image-utils";
|
||||
import { useImageAttachments } from "@/hooks/use-image-attachments";
|
||||
import { useAudioRecording } from "@/hooks/use-audio-recording";
|
||||
import { RepoSelectorCompact } from "./repo-selector-compact";
|
||||
import { BranchSelectorCompact } from "./branch-selector-compact";
|
||||
import { ImageAttachmentsPreview } from "./image-attachments-preview";
|
||||
@ -45,6 +46,24 @@ export function TaskInput({ onSubmit, isLoading }: TaskInputProps) {
|
||||
openFilePicker,
|
||||
} = useImageAttachments();
|
||||
|
||||
const {
|
||||
state: recordingState,
|
||||
error: recordingError,
|
||||
clearError: clearRecordingError,
|
||||
toggleRecording,
|
||||
} = useAudioRecording();
|
||||
|
||||
const handleMicClick = async () => {
|
||||
clearRecordingError();
|
||||
const transcribedText = await toggleRecording();
|
||||
if (transcribedText) {
|
||||
setPrompt((prev) =>
|
||||
prev ? `${prev} ${transcribedText}` : transcribedText,
|
||||
);
|
||||
inputRef.current?.focus();
|
||||
}
|
||||
};
|
||||
|
||||
// Handle click outside to collapse
|
||||
useEffect(() => {
|
||||
const handleClickOutside = (e: MouseEvent) => {
|
||||
@ -259,9 +278,25 @@ export function TaskInput({ onSubmit, isLoading }: TaskInputProps) {
|
||||
<div className="flex items-center gap-1">
|
||||
<button
|
||||
type="button"
|
||||
className="flex h-8 w-8 items-center justify-center rounded-md text-neutral-500 transition-colors hover:bg-white/5 hover:text-neutral-300"
|
||||
onClick={handleMicClick}
|
||||
disabled={recordingState === "processing"}
|
||||
className={cn(
|
||||
"relative flex h-8 w-8 items-center justify-center rounded-full transition-colors",
|
||||
recordingState === "recording"
|
||||
? "text-red-500"
|
||||
: "text-neutral-500 hover:bg-white/5 hover:text-neutral-300",
|
||||
recordingState === "processing" &&
|
||||
"cursor-not-allowed opacity-50",
|
||||
)}
|
||||
>
|
||||
<Mic className="h-5 w-5" />
|
||||
{recordingState === "recording" && (
|
||||
<span className="absolute inset-0 animate-pulse rounded-full bg-red-500/30" />
|
||||
)}
|
||||
{recordingState === "processing" ? (
|
||||
<Loader2 className="h-5 w-5 animate-spin" />
|
||||
) : (
|
||||
<Mic className="h-5 w-5" />
|
||||
)}
|
||||
</button>
|
||||
|
||||
<button
|
||||
@ -279,6 +314,13 @@ export function TaskInput({ onSubmit, isLoading }: TaskInputProps) {
|
||||
</button>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
{/* Recording error message */}
|
||||
{recordingError && (
|
||||
<div className="px-5 pb-3">
|
||||
<p className="text-sm text-red-400">{recordingError}</p>
|
||||
</div>
|
||||
)}
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
||||
170
apps/web/hooks/use-audio-recording.ts
Normal file
170
apps/web/hooks/use-audio-recording.ts
Normal file
@ -0,0 +1,170 @@
|
||||
"use client";
|
||||
|
||||
import { useState, useCallback, useRef } from "react";
|
||||
|
||||
type RecordingState = "idle" | "recording" | "processing";
|
||||
|
||||
interface TranscribeResponse {
|
||||
text?: string;
|
||||
error?: string;
|
||||
details?: string;
|
||||
}
|
||||
|
||||
function blobToBase64(blob: Blob): Promise<string> {
|
||||
return new Promise((resolve, reject) => {
|
||||
const reader = new FileReader();
|
||||
reader.addEventListener("loadend", () => {
|
||||
const result = reader.result as string;
|
||||
// Remove the data URL prefix (e.g., "data:audio/webm;base64,")
|
||||
const base64 = result.split(",")[1];
|
||||
if (base64) {
|
||||
resolve(base64);
|
||||
} else {
|
||||
reject(new Error("Failed to convert blob to base64"));
|
||||
}
|
||||
});
|
||||
reader.addEventListener("error", () => reject(reader.error));
|
||||
reader.readAsDataURL(blob);
|
||||
});
|
||||
}
|
||||
|
||||
function getSupportedMimeType(): string {
|
||||
const mimeTypes = ["audio/webm", "audio/mp4", "audio/ogg", "audio/wav"];
|
||||
for (const mimeType of mimeTypes) {
|
||||
if (MediaRecorder.isTypeSupported(mimeType)) {
|
||||
return mimeType;
|
||||
}
|
||||
}
|
||||
// Fallback - let the browser choose
|
||||
return "";
|
||||
}
|
||||
|
||||
export function useAudioRecording() {
|
||||
const [state, setState] = useState<RecordingState>("idle");
|
||||
const [error, setError] = useState<string | null>(null);
|
||||
|
||||
const mediaRecorderRef = useRef<MediaRecorder | null>(null);
|
||||
const audioChunksRef = useRef<Blob[]>([]);
|
||||
const streamRef = useRef<MediaStream | null>(null);
|
||||
const mimeTypeRef = useRef<string>("");
|
||||
|
||||
const startRecording = useCallback(async () => {
|
||||
setError(null);
|
||||
|
||||
try {
|
||||
const stream = await navigator.mediaDevices.getUserMedia({ audio: true });
|
||||
streamRef.current = stream;
|
||||
|
||||
const mimeType = getSupportedMimeType();
|
||||
mimeTypeRef.current = mimeType;
|
||||
|
||||
const mediaRecorder = new MediaRecorder(
|
||||
stream,
|
||||
mimeType ? { mimeType } : undefined,
|
||||
);
|
||||
mediaRecorderRef.current = mediaRecorder;
|
||||
audioChunksRef.current = [];
|
||||
|
||||
mediaRecorder.ondataavailable = (event) => {
|
||||
if (event.data.size > 0) {
|
||||
audioChunksRef.current.push(event.data);
|
||||
}
|
||||
};
|
||||
|
||||
mediaRecorder.start();
|
||||
setState("recording");
|
||||
} catch (err) {
|
||||
const message = err instanceof Error ? err.message : String(err);
|
||||
if (
|
||||
message.includes("Permission denied") ||
|
||||
message.includes("NotAllowedError")
|
||||
) {
|
||||
setError(
|
||||
"Microphone access denied. Please allow microphone access to use voice input.",
|
||||
);
|
||||
} else {
|
||||
setError(`Failed to start recording: ${message}`);
|
||||
}
|
||||
setState("idle");
|
||||
}
|
||||
}, []);
|
||||
|
||||
const stopRecording = useCallback(async (): Promise<string | null> => {
|
||||
const mediaRecorder = mediaRecorderRef.current;
|
||||
if (!mediaRecorder || state !== "recording") {
|
||||
return null;
|
||||
}
|
||||
|
||||
return new Promise((resolve) => {
|
||||
mediaRecorder.onstop = async () => {
|
||||
// Stop all tracks to release the microphone
|
||||
streamRef.current?.getTracks().forEach((track) => track.stop());
|
||||
streamRef.current = null;
|
||||
|
||||
setState("processing");
|
||||
|
||||
const mimeType = mimeTypeRef.current || "audio/webm";
|
||||
const audioBlob = new Blob(audioChunksRef.current, {
|
||||
type: mimeType,
|
||||
});
|
||||
|
||||
try {
|
||||
const base64Audio = await blobToBase64(audioBlob);
|
||||
|
||||
const response = await fetch("/api/transcribe", {
|
||||
method: "POST",
|
||||
headers: { "Content-Type": "application/json" },
|
||||
body: JSON.stringify({
|
||||
audio: base64Audio,
|
||||
mimeType,
|
||||
}),
|
||||
});
|
||||
|
||||
const data = (await response.json()) as TranscribeResponse;
|
||||
|
||||
if (!response.ok) {
|
||||
setError(data.error ?? "Transcription failed");
|
||||
setState("idle");
|
||||
resolve(null);
|
||||
return;
|
||||
}
|
||||
|
||||
setState("idle");
|
||||
resolve(data.text ?? null);
|
||||
} catch (err) {
|
||||
const message = err instanceof Error ? err.message : String(err);
|
||||
setError(`Transcription failed: ${message}`);
|
||||
setState("idle");
|
||||
resolve(null);
|
||||
}
|
||||
};
|
||||
|
||||
mediaRecorder.stop();
|
||||
});
|
||||
}, [state]);
|
||||
|
||||
const toggleRecording = useCallback(async (): Promise<string | null> => {
|
||||
if (state === "recording") {
|
||||
return stopRecording();
|
||||
} else if (state === "idle") {
|
||||
await startRecording();
|
||||
return null;
|
||||
}
|
||||
return null;
|
||||
}, [state, startRecording, stopRecording]);
|
||||
|
||||
const clearError = useCallback(() => {
|
||||
setError(null);
|
||||
}, []);
|
||||
|
||||
return {
|
||||
state,
|
||||
error,
|
||||
clearError,
|
||||
startRecording,
|
||||
stopRecording,
|
||||
toggleRecording,
|
||||
isRecording: state === "recording",
|
||||
isProcessing: state === "processing",
|
||||
};
|
||||
}
|
||||
@ -15,6 +15,7 @@
|
||||
"db:studio": "drizzle-kit studio"
|
||||
},
|
||||
"dependencies": {
|
||||
"@ai-sdk/elevenlabs": "^2.0.5",
|
||||
"@ai-sdk/react": "catalog:",
|
||||
"@hookform/resolvers": "^5.2.2",
|
||||
"@octokit/rest": "^22.0.1",
|
||||
|
||||
7
bun.lock
7
bun.lock
@ -41,6 +41,7 @@
|
||||
"name": "web",
|
||||
"version": "0.1.0",
|
||||
"dependencies": {
|
||||
"@ai-sdk/elevenlabs": "^2.0.5",
|
||||
"@ai-sdk/react": "catalog:",
|
||||
"@hookform/resolvers": "^5.2.2",
|
||||
"@octokit/rest": "^22.0.1",
|
||||
@ -188,6 +189,8 @@
|
||||
|
||||
"@ai-sdk/devtools": ["@ai-sdk/devtools@0.0.2", "", { "dependencies": { "@ai-sdk/provider": "3.0.0", "@hono/node-server": "^1.13.7", "hono": "^4.6.14" }, "bin": { "devtools": "bin/cli.js" } }, "sha512-iU/3gpyPPwUM2+G2HkdUJ5eQI5ti6/tplUoZL4EUUIJi2EBKHcvl03Boe72wlgpmMmk9HlpnsUjapHKUto7MgQ=="],
|
||||
|
||||
"@ai-sdk/elevenlabs": ["@ai-sdk/elevenlabs@2.0.5", "", { "dependencies": { "@ai-sdk/provider": "3.0.2", "@ai-sdk/provider-utils": "4.0.5" }, "peerDependencies": { "zod": "^3.25.76 || ^4.1.8" } }, "sha512-xhwcgNlalozRxEnp73QTYsC9Z0Wg6LlDGXyymEuRbvOjVgc0G5YYjSC1V4E3RogdOev7EhmNGc6/2G4HOGE+PQ=="],
|
||||
|
||||
"@ai-sdk/gateway": ["@ai-sdk/gateway@3.0.11", "", { "dependencies": { "@ai-sdk/provider": "3.0.2", "@ai-sdk/provider-utils": "4.0.4", "@vercel/oidc": "3.1.0" }, "peerDependencies": { "zod": "^3.25.76 || ^4.1.8" } }, "sha512-gLrgNXA95wdo/zAlA0miX/SJEYKSYCHg+e0Y/uQeABLScZAMjPw3jWaeANta/Db1T4xfi8cBvY3nnV8Pa27z+w=="],
|
||||
|
||||
"@ai-sdk/provider": ["@ai-sdk/provider@3.0.0", "", { "dependencies": { "json-schema": "^0.4.0" } }, "sha512-m9ka3ptkPQbaHHZHqDXDF9C9B5/Mav0KTdky1k2HZ3/nrW2t1AgObxIVPyGDWQNS9FXT/FS6PIoSjpcP/No8rQ=="],
|
||||
@ -1440,6 +1443,10 @@
|
||||
|
||||
"zwitch": ["zwitch@2.0.4", "", {}, "sha512-bXE4cR/kVZhKZX/RjPEflHaKVhUVl85noU3v6b8apfQEc1x4A+zBxjZ4lN8LqGd6WZ3dl98pY4o717VFmoPp+A=="],
|
||||
|
||||
"@ai-sdk/elevenlabs/@ai-sdk/provider": ["@ai-sdk/provider@3.0.2", "", { "dependencies": { "json-schema": "^0.4.0" } }, "sha512-HrEmNt/BH/hkQ7zpi2o6N3k1ZR1QTb7z85WYhYygiTxOQuaml4CMtHCWRbric5WPU+RNsYI7r1EpyVQMKO1pYw=="],
|
||||
|
||||
"@ai-sdk/elevenlabs/@ai-sdk/provider-utils": ["@ai-sdk/provider-utils@4.0.5", "", { "dependencies": { "@ai-sdk/provider": "3.0.2", "@standard-schema/spec": "^1.1.0", "eventsource-parser": "^3.0.6" }, "peerDependencies": { "zod": "^3.25.76 || ^4.1.8" } }, "sha512-Ow/X/SEkeExTTc1x+nYLB9ZHK2WUId8+9TlkamAx7Tl9vxU+cKzWx2dwjgMHeCN6twrgwkLrrtqckQeO4mxgVA=="],
|
||||
|
||||
"@ai-sdk/gateway/@ai-sdk/provider": ["@ai-sdk/provider@3.0.2", "", { "dependencies": { "json-schema": "^0.4.0" } }, "sha512-HrEmNt/BH/hkQ7zpi2o6N3k1ZR1QTb7z85WYhYygiTxOQuaml4CMtHCWRbric5WPU+RNsYI7r1EpyVQMKO1pYw=="],
|
||||
|
||||
"@ai-sdk/gateway/@ai-sdk/provider-utils": ["@ai-sdk/provider-utils@4.0.4", "", { "dependencies": { "@ai-sdk/provider": "3.0.2", "@standard-schema/spec": "^1.1.0", "eventsource-parser": "^3.0.6" }, "peerDependencies": { "zod": "^3.25.76 || ^4.1.8" } }, "sha512-VxhX0B/dWGbpNHxrKCWUAJKXIXV015J4e7qYjdIU9lLWeptk0KMLGcqkB4wFxff5Njqur8dt8wRi1MN9lZtDqg=="],
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user