feat: improve streaming mic and deepgram token handling

This commit is contained in:
colinislit
2025-11-25 22:53:04 +01:00
parent 779ab85e5d
commit 56b66ae6ff
7 changed files with 430 additions and 90 deletions

View File

@@ -3,13 +3,13 @@
import { useMemo, useState, useEffect, useCallback } from 'react';
import { useRouter } from 'next/navigation';
import dynamic from 'next/dynamic';
import { Mic, Sparkles } from 'lucide-react';
import { Sparkles } from 'lucide-react';
import { Button } from '@/components/ui/button';
import { toast } from '@/hooks/use-toast';
import type { ClassificationResult, Report, ReportType } from '@/lib/types/report';
import { createReport } from '../actions';
import { cn } from '@/lib/utils';
import type { Editor } from '@/components/rich-text-editor';
import { ToolbarMicButton } from './toolbar-mic-button';
// Dynamic imports
const RichTextEditor = dynamic(
@@ -17,11 +17,6 @@ const RichTextEditor = dynamic(
{ ssr: false, loading: () => <EditorSkeleton /> }
);
const SpeechRecorderStreaming = dynamic(
() => import('@/components/speech-recorder-streaming').then((m) => m.SpeechRecorderStreaming),
{ ssr: false, loading: () => <RecorderSkeleton /> }
);
function EditorSkeleton() {
return (
<div className="rounded-lg border border-slate-200 animate-pulse">
@@ -31,15 +26,6 @@ function EditorSkeleton() {
);
}
function RecorderSkeleton() {
return (
<div className="rounded-lg border border-slate-200 bg-white p-4 animate-pulse">
<div className="h-4 w-1/2 rounded bg-slate-100 mb-2" />
<div className="h-10 rounded bg-slate-100" />
</div>
);
}
// ─────────────────────────────────────────────────────────────────────────────
// Types
// ─────────────────────────────────────────────────────────────────────────────
@@ -81,8 +67,8 @@ export function ReportComposer({
const [isSaving, setIsSaving] = useState(false);
const [error, setError] = useState<string | null>(null);
const [lastAutosave, setLastAutosave] = useState<Date | null>(null);
const [isStreaming, setIsStreaming] = useState(false);
const [showRecorder, setShowRecorder] = useState(false);
const [isRecording, setIsRecording] = useState(false);
const [interimText, setInterimText] = useState('');
const draftStorageKey = useMemo(() => `rapportage-draft-${patientId}`, [patientId]);
// Calculate plain text length from HTML content
@@ -155,14 +141,16 @@ export function ReportComposer({
// Recording handlers
const handleRecordingStart = useCallback(() => {
setIsStreaming(true);
setIsRecording(true);
setInterimText('');
if (editorRef) {
editorRef.chain().focus().run();
}
}, [editorRef]);
const handleRecordingStop = useCallback(() => {
setIsStreaming(false);
setIsRecording(false);
setInterimText('');
}, []);
const handleTranscript = useCallback((text: string) => {
@@ -176,6 +164,10 @@ export function ReportComposer({
});
}, []);
const handleInterimTranscript = useCallback((text: string) => {
setInterimText(text);
}, []);
// Reference snippet for context
const referenceSnippet = useMemo(() => {
if (!selectedReport) return null;
@@ -257,22 +249,16 @@ export function ReportComposer({
setContent((prev) => (prev ? `${prev}${block}` : block));
};
// Mic button for toolbar
// Mic button for toolbar - directly starts/stops recording
const MicToolbarButton = (
<button
type="button"
onClick={() => setShowRecorder((prev) => !prev)}
className={cn(
'inline-flex h-7 items-center gap-1 px-2 rounded-md text-xs font-medium transition-colors',
showRecorder || isStreaming
? 'bg-emerald-100 text-emerald-700'
: 'text-slate-600 hover:bg-white hover:text-slate-900'
)}
title="Spraakopname"
>
<Mic className={cn('h-4 w-4', isStreaming && 'text-emerald-600 animate-pulse')} />
{isStreaming && <span className="text-emerald-600"></span>}
</button>
<ToolbarMicButton
onTranscript={handleTranscript}
onInterimTranscript={handleInterimTranscript}
onRecordingStart={handleRecordingStart}
onRecordingStop={handleRecordingStop}
disabled={isSaving || isAnalyzing}
patientId={patientId}
/>
);
return (
@@ -301,29 +287,23 @@ export function ReportComposer({
placeholder="Begin met typen of gebruik spraakopname..."
toolbarExtra={MicToolbarButton}
onEditorReady={setEditorRef}
isStreaming={isStreaming}
isStreaming={isRecording}
minHeight="180px"
/>
{/* Interim transcript preview during recording */}
{interimText && (
<div className="text-sm text-slate-500 italic bg-orange-50 border border-orange-200 rounded-lg px-3 py-2 -mt-2 animate-pulse">
{interimText}
</div>
)}
{/* Character count */}
<div className="flex justify-between text-xs text-slate-500 -mt-2">
<span>{characterCount} / 5000 karakters</span>
{characterCount > 0 && characterCount < 20 && <span className="text-amber-600">Minimaal 20 karakters</span>}
</div>
{/* Speech recorder (collapsible) */}
{showRecorder && (
<div className="animate-in slide-in-from-top-2 duration-200">
<SpeechRecorderStreaming
disabled={isSaving || isAnalyzing}
onTranscript={handleTranscript}
onRecordingStart={handleRecordingStart}
onRecordingStop={handleRecordingStop}
telemetryContext={{ context: 'report_composer', patientId }}
/>
</div>
)}
{/* AI Classification result (inline) */}
{classification && (
<div className="flex items-center gap-2 text-sm text-slate-600 bg-slate-50 rounded-lg px-3 py-2">

View File

@@ -0,0 +1,241 @@
'use client';
import { useCallback, useEffect, useRef, useState } from 'react';
import { Mic, Square, Loader2 } from 'lucide-react';
import { cn } from '@/lib/utils';
import {
useDeepgramStreaming,
type TranscriptResult,
} from '@/hooks/use-deepgram-streaming';
// ─────────────────────────────────────────────────────────────────────────────
// Types
// ─────────────────────────────────────────────────────────────────────────────
interface ToolbarMicButtonProps {
/** Callback voor final transcripts */
onTranscript: (text: string) => void;
/** Callback voor interim transcripts (live preview) */
onInterimTranscript?: (text: string) => void;
/** Callback wanneer recording start */
onRecordingStart?: () => void;
/** Callback wanneer recording stopt */
onRecordingStop?: () => void;
/** Disabled state */
disabled?: boolean;
/** Patient ID voor telemetry */
patientId?: string;
}
// ─────────────────────────────────────────────────────────────────────────────
// Mini Waveform Component
// ─────────────────────────────────────────────────────────────────────────────
function MiniWaveform({ analyserNode }: { analyserNode: AnalyserNode | null }) {
const canvasRef = useRef<HTMLCanvasElement>(null);
const animationRef = useRef<number | null>(null);
useEffect(() => {
if (!analyserNode || !canvasRef.current) {
if (animationRef.current) {
cancelAnimationFrame(animationRef.current);
animationRef.current = null;
}
return;
}
const canvas = canvasRef.current;
const ctx = canvas.getContext('2d');
if (!ctx) return;
const bufferLength = analyserNode.frequencyBinCount;
const dataArray = new Uint8Array(bufferLength);
const draw = () => {
animationRef.current = requestAnimationFrame(draw);
analyserNode.getByteFrequencyData(dataArray);
// Clear canvas
ctx.clearRect(0, 0, canvas.width, canvas.height);
// Draw bars
const barCount = 5;
const barWidth = 3;
const gap = 2;
const maxHeight = canvas.height - 4;
const startX = (canvas.width - (barCount * barWidth + (barCount - 1) * gap)) / 2;
for (let i = 0; i < barCount; i++) {
// Sample from different parts of frequency spectrum
const dataIndex = Math.floor((i / barCount) * bufferLength * 0.5);
const value = dataArray[dataIndex] / 255;
const barHeight = Math.max(4, value * maxHeight);
const x = startX + i * (barWidth + gap);
const y = (canvas.height - barHeight) / 2;
ctx.fillStyle = '#fff';
ctx.beginPath();
ctx.roundRect(x, y, barWidth, barHeight, 1);
ctx.fill();
}
};
draw();
return () => {
if (animationRef.current) {
cancelAnimationFrame(animationRef.current);
}
};
}, [analyserNode]);
return (
<canvas
ref={canvasRef}
width={32}
height={20}
className="inline-block"
/>
);
}
// ─────────────────────────────────────────────────────────────────────────────
// Main Component
// ─────────────────────────────────────────────────────────────────────────────
export function ToolbarMicButton({
onTranscript,
onInterimTranscript,
onRecordingStart,
onRecordingStop,
disabled = false,
patientId,
}: ToolbarMicButtonProps) {
const [interimText, setInterimText] = useState('');
// Completed utterances (after speechFinal)
const completedUtterancesRef = useRef<string[]>([]);
// Current utterance being built (before speechFinal)
const currentUtteranceRef = useRef<string>('');
const handleTranscript = useCallback(
(result: TranscriptResult) => {
if (result.isFinal) {
// Update current utterance with latest final text
currentUtteranceRef.current = result.transcript;
setInterimText('');
if (result.speechFinal) {
// Utterance complete - commit to completed list
if (currentUtteranceRef.current.trim()) {
completedUtterancesRef.current.push(currentUtteranceRef.current);
}
currentUtteranceRef.current = '';
}
// Build full text: completed utterances + current utterance
const parts = [...completedUtterancesRef.current];
if (currentUtteranceRef.current.trim()) {
parts.push(currentUtteranceRef.current);
}
onTranscript(parts.join(' '));
} else {
// Interim result - show as preview
setInterimText(result.transcript);
onInterimTranscript?.(result.transcript);
}
},
[onTranscript, onInterimTranscript]
);
const {
status,
isRecording,
startRecording,
stopRecording,
analyserNode,
isBrowserSupported,
} = useDeepgramStreaming({
onTranscript: handleTranscript,
});
const handleClick = async () => {
console.log('[ToolbarMicButton] Click - isRecording:', isRecording, 'status:', status);
if (isRecording) {
console.log('[ToolbarMicButton] Stopping recording...');
stopRecording();
onRecordingStop?.();
completedUtterancesRef.current = [];
currentUtteranceRef.current = '';
setInterimText('');
} else {
console.log('[ToolbarMicButton] Starting recording...');
completedUtterancesRef.current = [];
currentUtteranceRef.current = '';
setInterimText('');
onRecordingStart?.();
try {
await startRecording();
console.log('[ToolbarMicButton] startRecording() completed');
} catch (err) {
console.error('[ToolbarMicButton] startRecording() failed:', err);
}
}
};
const isConnecting = status === 'connecting';
const isActive = isRecording && status === 'connected';
// Determine button state and colors
const getButtonClasses = () => {
if (isActive) {
// Recording - orange background
return 'bg-orange-500 text-white hover:bg-orange-600 shadow-sm';
}
if (isConnecting) {
// Connecting - subtle loading state
return 'bg-amber-100 text-amber-700';
}
// Default - green background
return 'bg-emerald-500 text-white hover:bg-emerald-600 shadow-sm';
};
if (!isBrowserSupported) {
return (
<button
type="button"
disabled
className="inline-flex h-7 items-center gap-1 px-2 rounded-md text-xs font-medium bg-slate-200 text-slate-400 cursor-not-allowed"
title="Spraakopname niet ondersteund in deze browser"
>
<Mic className="h-4 w-4" />
</button>
);
}
return (
<button
type="button"
onClick={handleClick}
disabled={disabled || isConnecting}
className={cn(
'inline-flex h-7 items-center gap-1.5 px-2 rounded-md text-xs font-medium transition-all duration-200',
getButtonClasses(),
disabled && 'opacity-50 cursor-not-allowed'
)}
title={isRecording ? 'Stop opname' : 'Start opname'}
>
{isConnecting ? (
<Loader2 className="h-4 w-4 animate-spin" />
) : isActive ? (
<>
<MiniWaveform analyserNode={analyserNode} />
<Square className="h-3 w-3" />
</>
) : (
<Mic className="h-4 w-4" />
)}
</button>
);
}