colin/triqura-ecd
2
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat(cortex): update chat/nudge flow, deepgram streaming and agenda form

Browse Source 
Refines chat panel, nudge messages and artifact rendering, reworks
deepgram token/streaming handling, and adds test tooling deps
(playwright, cypress) to package.json.

Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
This commit is contained in:
colinislit
2026-07-09 22:37:46 +02:00
parent 6c6dae5eb7
commit cdeea793bd
20 changed files with 1634 additions and 283 deletions
Download Patch File Download Diff File Expand all files Collapse all files

263
hooks/use-deepgram-streaming.ts
View File

@@ -72,6 +72,20 @@ export interface UseDeepgramStreamingReturn {
isBrowserSupported: boolean
}
type DeepgramAuthMode = 'accessToken' | 'apiKey'
interface DeepgramTokenResponse {
token: string
expiresIn: number
authMode?: DeepgramAuthMode
}
function createTokenError(message: string) {
const error = new Error(message)
error.name = 'DeepgramTokenError'
return error
}
// ─────────────────────────────────────────────────────────────────────────────
// Browser Support Check
// ─────────────────────────────────────────────────────────────────────────────
@@ -94,6 +108,7 @@ function checkBrowserSupport(): boolean {
const MAX_RECONNECT_ATTEMPTS = 3
const RECONNECT_BASE_DELAY_MS = 1000
const AUDIO_CHUNK_INTERVAL_MS = 250
const CONNECTION_TIMEOUT_MS = 5000
// ─────────────────────────────────────────────────────────────────────────────
// Hook Implementation
@@ -127,6 +142,7 @@ export function useDeepgramStreaming({
const audioContextRef = useRef<AudioContext | null>(null)
const reconnectAttemptsRef = useRef(0)
const shouldReconnectRef = useRef(false)
const handleReconnectRef = useRef<(() => Promise<void>) | null>(null)
const isPausedRef = useRef(isPaused)
// Callback refs voor stabiele referenties
@@ -144,6 +160,29 @@ export function useDeepgramStreaming({
isPausedRef.current = isPaused
}, [isPaused])
const cleanupRecordingResources = useCallback(() => {
if (mediaRecorderRef.current && mediaRecorderRef.current.state !== 'inactive') {
mediaRecorderRef.current.stop()
}
mediaRecorderRef.current = null
if (mediaStreamRef.current) {
mediaStreamRef.current.getTracks().forEach((track) => track.stop())
}
mediaStreamRef.current = null
if (audioContextRef.current) {
audioContextRef.current.close()
}
audioContextRef.current = null
setAnalyserNode(null)
if (liveClientRef.current) {
liveClientRef.current.requestClose()
}
liveClientRef.current = null
}, [])
// Status update helper
const updateStatus = useCallback((newStatus: ConnectionStatus) => {
setStatus(newStatus)
@@ -151,7 +190,7 @@ export function useDeepgramStreaming({
}, [])
// Fetch token van onze proxy endpoint
const fetchToken = useCallback(async (): Promise<string> => {
const fetchToken = useCallback(async (): Promise<DeepgramTokenResponse> => {
console.log('[Deepgram] Fetching token from /api/deepgram/token...')
const response = await fetch('/api/deepgram/token', { method: 'POST' })
console.log('[Deepgram] Token response status:', response.status)
@@ -160,17 +199,17 @@ export function useDeepgramStreaming({
const data = await response.json().catch(() => ({}))
console.error('[Deepgram] Token fetch failed:', response.status, data)
if (response.status === 429) {
throw new Error('Rate limit bereikt. Probeer over een uur opnieuw.')
throw createTokenError('Rate limit bereikt. Probeer over een uur opnieuw.')
}
if (response.status === 401) {
throw new Error('Niet ingelogd. Log opnieuw in.')
throw createTokenError('Niet ingelogd. Log opnieuw in.')
}
throw new Error(data.error || 'Kon geen token ophalen')
throw createTokenError(data.error || 'Kon geen token ophalen')
}
const data = await response.json()
console.log('[Deepgram] Token received, expiresIn:', data.expiresIn)
return data.token
const data = (await response.json()) as DeepgramTokenResponse
console.log('[Deepgram] Token received, expiresIn:', data.expiresIn, 'authMode:', data.authMode || 'accessToken')
return data
}, [])
// Setup WebSocket verbinding met Deepgram
@@ -180,13 +219,14 @@ export function useDeepgramStreaming({
updateStatus('connecting')
setError(null)
const token = await fetchToken()
const tokenResponse = await fetchToken()
const authMode = tokenResponse.authMode || 'accessToken'
// IMPORTANT: Use accessToken option for JWT tokens from grantToken()
// This uses Bearer scheme authentication instead of Token scheme
// See: https://github.com/deepgram/deepgram-js-sdk
console.log('[Deepgram] Creating Deepgram client with accessToken...')
const deepgram = createClient({ accessToken: token })
console.log('[Deepgram] Creating Deepgram client with authMode:', authMode)
const deepgram =
authMode === 'apiKey'
? createClient(tokenResponse.token)
: createClient({ accessToken: tokenResponse.token })
console.log('[Deepgram] Creating live connection with options:', { model, language, endpointingMs })
const connection = deepgram.listen.live({
@@ -200,66 +240,97 @@ export function useDeepgramStreaming({
})
console.log('[Deepgram] Live connection created, setting up event handlers...')
// Event handlers
connection.on(LiveTranscriptionEvents.Open, () => {
console.log('[Deepgram] WebSocket OPEN')
updateStatus('connected')
reconnectAttemptsRef.current = 0
setError(null)
})
liveClientRef.current = connection
connection.on(
LiveTranscriptionEvents.Transcript,
(data: LiveTranscriptionEvent) => {
const alternative = data.channel?.alternatives?.[0]
if (!alternative) return
await new Promise<void>((resolve, reject) => {
let opened = false
const timeout = window.setTimeout(() => {
if (liveClientRef.current === connection) {
liveClientRef.current = null
}
connection.requestClose()
reject(new Error('Timeout bij verbinden met Deepgram'))
}, CONNECTION_TIMEOUT_MS)
const transcript = alternative.transcript || ''
if (!transcript.trim()) return
// Event handlers
connection.on(LiveTranscriptionEvents.Open, () => {
console.log('[Deepgram] WebSocket OPEN')
opened = true
window.clearTimeout(timeout)
updateStatus('connected')
setError(null)
resolve()
})
const words: TranscriptWord[] = (alternative.words || []).map(
(w: { word: string; confidence: number; start: number; end: number }) => ({
word: w.word,
confidence: w.confidence,
start: w.start,
end: w.end,
})
)
connection.on(
LiveTranscriptionEvents.Transcript,
(data: LiveTranscriptionEvent) => {
const alternative = data.channel?.alternatives?.[0]
if (!alternative) return
const result: TranscriptResult = {
transcript,
isFinal: data.is_final ?? false,
confidence: alternative.confidence ?? 1,
words,
speechFinal: data.speech_final ?? false,
const transcript = alternative.transcript || ''
if (!transcript.trim()) return
const words: TranscriptWord[] = (alternative.words || []).map(
(w: { word: string; confidence: number; start: number; end: number }) => ({
word: w.word,
confidence: w.confidence,
start: w.start,
end: w.end,
})
)
const result: TranscriptResult = {
transcript,
isFinal: data.is_final ?? false,
confidence: alternative.confidence ?? 1,
words,
speechFinal: data.speech_final ?? false,
}
onTranscriptRef.current(result)
}
)
connection.on(LiveTranscriptionEvents.Error, (err: Error) => {
console.error('[Deepgram] WebSocket ERROR:', err)
window.clearTimeout(timeout)
setError(err.message || 'WebSocket fout')
updateStatus('error')
onErrorRef.current?.(err)
if (!opened) {
reject(err)
return
}
onTranscriptRef.current(result)
}
)
// Probeer te reconnecten als we nog aan het opnemen waren
if (shouldReconnectRef.current) {
handleReconnectRef.current?.()
}
})
connection.on(LiveTranscriptionEvents.Error, (err: Error) => {
console.error('[Deepgram] WebSocket ERROR:', err)
setError(err.message || 'WebSocket fout')
updateStatus('error')
onErrorRef.current?.(err)
connection.on(LiveTranscriptionEvents.Close, () => {
console.log('[Deepgram] WebSocket CLOSE')
window.clearTimeout(timeout)
// Probeer te reconnecten als we nog aan het opnemen waren
if (shouldReconnectRef.current) {
handleReconnect()
}
if (!opened) {
reject(new Error('Deepgram verbinding sloot voordat deze klaar was'))
return
}
if (shouldReconnectRef.current && reconnectAttemptsRef.current < MAX_RECONNECT_ATTEMPTS) {
handleReconnectRef.current?.()
} else {
shouldReconnectRef.current = false
cleanupRecordingResources()
setIsRecording(false)
setIsPaused(false)
updateStatus('disconnected')
}
})
})
connection.on(LiveTranscriptionEvents.Close, () => {
console.log('[Deepgram] WebSocket CLOSE')
if (shouldReconnectRef.current && reconnectAttemptsRef.current < MAX_RECONNECT_ATTEMPTS) {
handleReconnect()
} else {
updateStatus('disconnected')
}
})
liveClientRef.current = connection
console.log('[Deepgram] Connection setup complete')
} catch (err) {
console.error('[Deepgram] Connection error:', err)
@@ -268,11 +339,13 @@ export function useDeepgramStreaming({
updateStatus('error')
onErrorRef.current?.(err instanceof Error ? err : new Error(errorMessage))
if (shouldReconnectRef.current) {
handleReconnect()
}
shouldReconnectRef.current = false
cleanupRecordingResources()
setIsRecording(false)
setIsPaused(false)
throw err
}
}, [fetchToken, model, language, endpointingMs, updateStatus])
}, [cleanupRecordingResources, fetchToken, model, language, endpointingMs, updateStatus])
// Reconnect met exponential backoff
const handleReconnect = useCallback(async () => {
@@ -280,6 +353,9 @@ export function useDeepgramStreaming({
setError('Kon niet herverbinden na 3 pogingen')
updateStatus('error')
shouldReconnectRef.current = false
cleanupRecordingResources()
setIsRecording(false)
setIsPaused(false)
return
}
@@ -290,9 +366,19 @@ export function useDeepgramStreaming({
await new Promise((resolve) => setTimeout(resolve, delay))
if (shouldReconnectRef.current) {
await connect()
try {
await connect()
} catch (err) {
console.error('[Deepgram] Reconnect failed:', err)
shouldReconnectRef.current = false
cleanupRecordingResources()
setIsRecording(false)
setIsPaused(false)
}
}
}, [connect, updateStatus])
}, [cleanupRecordingResources, connect, updateStatus])
handleReconnectRef.current = handleReconnect
// Setup audio stream en analyser
const setupAudio = useCallback(async () => {
@@ -344,12 +430,13 @@ export function useDeepgramStreaming({
try {
setError(null)
shouldReconnectRef.current = true
reconnectAttemptsRef.current = 0
// Eerst WebSocket verbinding opzetten
// Eerst verbinden en wachten op WebSocket OPEN. Daarna pas de MediaRecorder starten,
// zodat het eerste chunk (met WebM/EBML-header) bij Deepgram aankomt.
console.log('[Deepgram] Step 1: Connecting to Deepgram...')
await connect()
// Dan audio stream starten
console.log('[Deepgram] Step 2: Setting up audio...')
await setupAudio()
@@ -370,44 +457,24 @@ export function useDeepgramStreaming({
setError(errorMessage)
}
shouldReconnectRef.current = false
cleanupRecordingResources()
setIsRecording(false)
setIsPaused(false)
updateStatus('error')
onErrorRef.current?.(err instanceof Error ? err : new Error(errorMessage))
}
}, [connect, setupAudio, updateStatus])
}, [cleanupRecordingResources, connect, setupAudio, updateStatus])
// Stop opname
const stopRecording = useCallback(() => {
shouldReconnectRef.current = false
// Stop MediaRecorder
if (mediaRecorderRef.current && mediaRecorderRef.current.state !== 'inactive') {
mediaRecorderRef.current.stop()
}
mediaRecorderRef.current = null
// Stop alle audio tracks
if (mediaStreamRef.current) {
mediaStreamRef.current.getTracks().forEach((track) => track.stop())
}
mediaStreamRef.current = null
// Close AudioContext
if (audioContextRef.current) {
audioContextRef.current.close()
}
audioContextRef.current = null
setAnalyserNode(null)
// Close WebSocket
if (liveClientRef.current) {
liveClientRef.current.requestClose()
}
liveClientRef.current = null
cleanupRecordingResources()
setIsRecording(false)
setIsPaused(false)
updateStatus('disconnected')
}, [updateStatus])
}, [cleanupRecordingResources, updateStatus])
// Pauzeer opname
const pauseRecording = useCallback(() => {