From c139d8277e745d82fe7a920d0bf2a2e8cf0abf9d Mon Sep 17 00:00:00 2001 From: Cale Shapera <25466659+cshape@users.noreply.github.com> Date: Mon, 29 Dec 2025 15:42:28 -0800 Subject: [PATCH 01/49] Feat: add multi language support --- backend/config/languages.ts | 210 ++++++++++++++ backend/graphs/conversation-graph.ts | 260 ++++++++++++------ backend/graphs/flashcard-graph.ts | 51 +++- backend/graphs/introduction-state-graph.ts | 78 +++++- backend/helpers/anki-exporter.ts | 31 ++- backend/helpers/audio-processor.ts | 77 +++++- backend/helpers/flashcard-processor.ts | 53 +++- .../helpers/introduction-state-processor.ts | 37 ++- backend/helpers/prompt-templates.ts | 50 ++-- backend/server.ts | 133 +++++++-- flashcard-graph.json | 12 +- frontend/index.html | 15 +- frontend/js/flashcard-ui.js | 36 ++- frontend/js/main.js | 148 +++++++++- frontend/js/storage.js | 101 +++++-- frontend/styles/main.css | 50 +++- package-lock.json | 8 +- 17 files changed, 1138 insertions(+), 212 deletions(-) create mode 100644 backend/config/languages.ts diff --git a/backend/config/languages.ts b/backend/config/languages.ts new file mode 100644 index 0000000..0ce9e97 --- /dev/null +++ b/backend/config/languages.ts @@ -0,0 +1,210 @@ +/** + * Language Configuration System + * + * This module provides a centralized configuration for all supported languages. + * To add a new language: + * 1. Add a new entry to SUPPORTED_LANGUAGES with all required fields + * 2. The rest of the app will automatically support the new language + */ + +export interface TeacherPersona { + name: string; + age: number; + nationality: string; + description: string; +} + +export interface TTSConfig { + speakerId: string; + modelId: string; + speakingRate: number; + temperature: number; + languageCode?: string; // Optional TTS language code (e.g., 'ja-JP') +} + +export interface LanguageConfig { + // Identifier + code: string; // e.g., 'es', 'ja', 'fr' + + // Display names + name: string; // English name: "Spanish" + nativeName: string; // Native name: "Español" + flag: string; // Emoji flag + + // STT configuration + sttLanguageCode: string; // Language code for speech-to-text + + // TTS configuration + ttsConfig: TTSConfig; + + // Teacher persona for this language + teacherPersona: TeacherPersona; + + // Example conversation topics specific to this language's culture + exampleTopics: string[]; + + // Language-specific instructions for the LLM (written in English, about teaching this language) + promptInstructions: string; +} + +/** + * Supported Languages Configuration + * + * Each language defines everything needed for: + * - Speech recognition (STT) + * - Text-to-speech (TTS) + * - Teacher persona and conversation style + * - Cultural context and example topics + */ +export const SUPPORTED_LANGUAGES: Record = { + es: { + code: 'es', + name: 'Spanish', + nativeName: 'Español', + flag: '🇲🇽', + sttLanguageCode: 'es-MX', // Mexican Spanish + ttsConfig: { + speakerId: 'Diego', + modelId: 'inworld-tts-1', + speakingRate: 1, + temperature: 0.7, + languageCode: 'es-MX', + }, + teacherPersona: { + name: 'Señor Gael Herrera', + age: 35, + nationality: 'Mexican (Chilango)', + description: + "a 35 year old 'Chilango' (from Mexico City) who has loaned their brain to AI", + }, + exampleTopics: [ + 'Mexico City', + 'the Dunedin sound rock scene', + 'gardening', + 'the concept of brunch across cultures', + 'Balkan travel', + ], + promptInstructions: ` +- Gently correct the user if they make mistakes in Spanish +- Use natural Mexican Spanish expressions when appropriate +- Vary complexity based on the user's level`, + }, + + ja: { + code: 'ja', + name: 'Japanese', + nativeName: '日本語', + flag: '🇯🇵', + sttLanguageCode: 'ja-JP', // Japanese + ttsConfig: { + speakerId: 'Asuka', + modelId: 'inworld-tts-1', + speakingRate: 0.95, + temperature: 0.7, + languageCode: 'ja-JP', + }, + teacherPersona: { + name: '田中先生 (Tanaka-sensei)', + age: 42, + nationality: 'Japanese (Tokyo)', + description: + 'a 42 year old Japanese teacher from Tokyo who loves sharing Japanese culture and language', + }, + exampleTopics: [ + 'Tokyo neighborhoods', + 'Japanese cuisine and izakaya culture', + 'anime and manga', + 'traditional arts like calligraphy and tea ceremony', + 'Japanese music from enka to J-pop', + 'seasonal festivals (matsuri)', + ], + promptInstructions: ` +- Gently correct the user if they make mistakes in Japanese +- Explain the difference between casual and polite forms when relevant +- Introduce kanji gradually with furigana explanations when helpful +- Mention cultural context behind expressions (e.g., why certain phrases are used) +- Use romanji in parentheses when introducing new vocabulary`, + }, + + fr: { + code: 'fr', + name: 'French', + nativeName: 'Français', + flag: '🇫🇷', + sttLanguageCode: 'fr-FR', // French + ttsConfig: { + speakerId: 'Alain', + modelId: 'inworld-tts-1', + speakingRate: 1, + temperature: 0.7, + languageCode: 'fr-FR', + }, + teacherPersona: { + name: 'Monsieur Lucien Dubois', + age: 38, + nationality: 'French (Parisian)', + description: + 'a 38 year old Parisian who is passionate about French language, literature, and gastronomy', + }, + exampleTopics: [ + 'Parisian cafés and culture', + 'French cinema (nouvelle vague)', + 'wine regions and gastronomy', + 'French literature and philosophy', + 'travel in Provence and the French Riviera', + 'French music from Édith Piaf to modern artists', + ], + promptInstructions: ` +- Gently correct the user if they make mistakes in French +- Pay attention to gender agreement and verb conjugation corrections +- Explain the nuances between formal (vous) and informal (tu) when relevant +- Share cultural context about French expressions and idioms +- Mention pronunciation tips for tricky French sounds`, + }, +}; + +/** + * Get configuration for a specific language + * @param code - Language code (e.g., 'es', 'ja', 'fr') + * @returns Language configuration or default (Spanish) if not found + */ +export function getLanguageConfig(code: string): LanguageConfig { + const config = SUPPORTED_LANGUAGES[code]; + if (!config) { + console.warn( + `Language '${code}' not found, falling back to Spanish (es)` + ); + return SUPPORTED_LANGUAGES['es']; + } + return config; +} + +/** + * Get all supported language codes + */ +export function getSupportedLanguageCodes(): string[] { + return Object.keys(SUPPORTED_LANGUAGES); +} + +/** + * Get language options for frontend dropdown + */ +export function getLanguageOptions(): Array<{ + code: string; + name: string; + nativeName: string; + flag: string; +}> { + return Object.values(SUPPORTED_LANGUAGES).map((lang) => ({ + code: lang.code, + name: lang.name, + nativeName: lang.nativeName, + flag: lang.flag, + })); +} + +/** + * Default language code + */ +export const DEFAULT_LANGUAGE_CODE = 'es'; + diff --git a/backend/graphs/conversation-graph.ts b/backend/graphs/conversation-graph.ts index 9bface7..14afe67 100644 --- a/backend/graphs/conversation-graph.ts +++ b/backend/graphs/conversation-graph.ts @@ -7,12 +7,18 @@ import { RemoteSTTNode, RemoteTTSNode, TextChunkingNode, + Graph, } from '@inworld/runtime/graph'; import { GraphTypes } from '@inworld/runtime/common'; import { renderJinja } from '@inworld/runtime/primitives/llm'; import { AsyncLocalStorage } from 'async_hooks'; import { conversationTemplate } from '../helpers/prompt-templates.js'; import type { IntroductionState } from '../helpers/introduction-state-processor.js'; +import { + LanguageConfig, + getLanguageConfig, + DEFAULT_LANGUAGE_CODE, +} from '../config/languages.js'; export interface ConversationGraphConfig { apiKey: string; @@ -25,90 +31,146 @@ export const stateStorage = new AsyncLocalStorage<{ messages: Array<{ role: string; content: string; timestamp: string }>; }; getIntroductionState: () => IntroductionState; + getLanguageConfig: () => LanguageConfig; }>(); -export function createConversationGraph(_config: ConversationGraphConfig) { - // Create the custom node class that gets state from AsyncLocalStorage - class EnhancedPromptBuilderNode extends CustomNode { - async process(_context: ProcessContext, currentInput: string) { - // Get state accessors directly from AsyncLocalStorage (set before graph execution) - const stateAccessors = stateStorage.getStore(); - - if (!stateAccessors) { - // Fallback to empty state if not available - const conversationState = { messages: [] }; - const introductionState = { - name: '', - level: '', - goal: '', - timestamp: '', - }; - const templateData = { - messages: conversationState.messages || [], - current_input: currentInput, - introduction_state: introductionState, - }; - - const renderedPrompt = await renderJinja( - conversationTemplate, - JSON.stringify(templateData) - ); - - return new GraphTypes.LLMChatRequest({ - messages: [{ role: 'user', content: renderedPrompt }], - }); - } - - // Get state directly from accessors - const conversationState = stateAccessors.getConversationState(); - const introductionState = stateAccessors.getIntroductionState(); - - const templateData = { - messages: conversationState.messages || [], - current_input: currentInput, - introduction_state: introductionState || { - name: '', - level: '', - goal: '', - }, - }; - - console.log( - 'ConversationGraph - Introduction state being used:', - JSON.stringify(introductionState, null, 2) - ); - console.log( - 'ConversationGraph - Number of messages in history:', - conversationState.messages?.length || 0 - ); - - const renderedPrompt = await renderJinja( - conversationTemplate, - JSON.stringify(templateData) - ); - - // Return LLMChatRequest for the LLM node - return new GraphTypes.LLMChatRequest({ - messages: [{ role: 'user', content: renderedPrompt }], - }); - } +// Store the current execution context as module-level variables +// This is set by AudioProcessor before starting graph execution +// and read by the PromptBuilder during execution. +// This works because Node.js is single-threaded for synchronous execution. +let currentExecutionLanguageCode: string = DEFAULT_LANGUAGE_CODE; +let currentGetConversationState: (() => { messages: Array<{ role: string; content: string; timestamp: string }> }) | null = null; +let currentGetIntroductionState: (() => IntroductionState) | null = null; + +/** + * Set the execution context for the current graph execution. + * Must be called before starting graph execution. + */ +export function setCurrentExecutionContext(context: { + languageCode: string; + getConversationState: () => { messages: Array<{ role: string; content: string; timestamp: string }> }; + getIntroductionState: () => IntroductionState; +}): void { + currentExecutionLanguageCode = context.languageCode; + currentGetConversationState = context.getConversationState; + currentGetIntroductionState = context.getIntroductionState; + console.log(`[ConversationGraph] Set execution context for language: ${context.languageCode}`); +} + +/** + * Set the language code for the current graph execution. + * @deprecated Use setCurrentExecutionContext instead + */ +export function setCurrentExecutionLanguage(languageCode: string): void { + currentExecutionLanguageCode = languageCode; + console.log(`[ConversationGraph] Set execution language to: ${languageCode}`); +} + +/** + * EnhancedPromptBuilderNode - defined once at module level to avoid + * component registry collisions. State and language config are retrieved from + * module-level variables (set by AudioProcessor before graph execution). + */ +class EnhancedPromptBuilderNode extends CustomNode { + async process(_context: ProcessContext, currentInput: string) { + // Get language config using the current execution language code + const langConfig = getLanguageConfig(currentExecutionLanguageCode); + const nodeId = (this as unknown as { id: string }).id; + + console.log( + `[PromptBuilder] Node ${nodeId} using execution language: ${langConfig.name} (code: ${currentExecutionLanguageCode})` + ); + + // Build template variables from language config + const templateVars = { + target_language: langConfig.name, + target_language_native: langConfig.nativeName, + teacher_name: langConfig.teacherPersona.name, + teacher_description: langConfig.teacherPersona.description, + example_topics: langConfig.exampleTopics.join(', '), + language_instructions: langConfig.promptInstructions, + }; + + // Get state from module-level accessors (set by AudioProcessor before execution) + // These bypass AsyncLocalStorage which gets broken by Inworld runtime + const conversationState = currentGetConversationState + ? currentGetConversationState() + : { messages: [] }; + const introductionState = currentGetIntroductionState + ? currentGetIntroductionState() + : { name: '', level: '', goal: '', timestamp: '' }; + + console.log( + '[PromptBuilder] Introduction state:', + JSON.stringify(introductionState, null, 2) + ); + console.log('[PromptBuilder] Language:', langConfig.name); + console.log( + '[PromptBuilder] Messages in history:', + conversationState.messages?.length || 0 + ); + + const templateData = { + messages: conversationState.messages || [], + current_input: currentInput, + introduction_state: introductionState, + ...templateVars, + }; + + const renderedPrompt = await renderJinja( + conversationTemplate, + JSON.stringify(templateData) + ); + + // Debug: Log a snippet of the rendered prompt to verify content + const promptSnippet = renderedPrompt.substring(0, 400); + console.log( + `[PromptBuilder] Rendered prompt (first 400 chars): ${promptSnippet}...` + ); + + // Return LLMChatRequest for the LLM node + return new GraphTypes.LLMChatRequest({ + messages: [{ role: 'user', content: renderedPrompt }], + }); } +} + +/** + * Creates a conversation graph configured for a specific language + */ +function createConversationGraphForLanguage( + _config: ConversationGraphConfig, + languageConfig: LanguageConfig +): Graph { + // Use language code as suffix to make node IDs unique per language + // This prevents edge condition name collisions in the global callback registry + const langSuffix = `_${languageConfig.code}`; + const promptBuilderNodeId = `enhanced_prompt_builder_node${langSuffix}`; + + console.log( + `[ConversationGraph] Creating graph with prompt builder node: ${promptBuilderNodeId}` + ); + + // Configure STT for the specific language const sttNode = new RemoteSTTNode({ - id: 'stt_node', + id: `stt_node${langSuffix}`, sttConfig: { - languageCode: 'es', + languageCode: languageConfig.sttLanguageCode, }, }); + const sttOutputNode = new ProxyNode({ - id: 'proxy_node', + id: `proxy_node${langSuffix}`, reportToClient: true, }); + const promptBuilderNode = new EnhancedPromptBuilderNode({ - id: 'enhanced_prompt_builder_node', + id: promptBuilderNodeId, }); + const llmNode = new RemoteLLMChatNode({ - id: 'llm_node', + id: `llm_node${langSuffix}`, provider: 'openai', modelName: 'gpt-4o-mini', stream: true, @@ -123,18 +185,22 @@ export function createConversationGraph(_config: ConversationGraphConfig) { presencePenalty: 0, }, }); - const chunkerNode = new TextChunkingNode({ id: 'chunker_node' }); + + const chunkerNode = new TextChunkingNode({ id: `chunker_node${langSuffix}` }); + + // Configure TTS for the specific language const ttsNode = new RemoteTTSNode({ - id: 'tts_node', - speakerId: 'Diego', - modelId: 'inworld-tts-1', + id: `tts_node${langSuffix}`, + speakerId: languageConfig.ttsConfig.speakerId, + modelId: languageConfig.ttsConfig.modelId, sampleRate: 16000, - speakingRate: 1, - temperature: 0.7, + speakingRate: languageConfig.ttsConfig.speakingRate, + temperature: languageConfig.ttsConfig.temperature, + languageCode: languageConfig.ttsConfig.languageCode, }); const executor = new GraphBuilder({ - id: 'conversation_graph', + id: `conversation_graph_${languageConfig.code}`, enableRemoteConfig: false, }) .addNode(sttNode) @@ -159,3 +225,43 @@ export function createConversationGraph(_config: ConversationGraphConfig) { return executor; } + +// Cache for language-specific graphs +const graphCache = new Map(); + +/** + * Get or create a conversation graph for a specific language + * Graphs are cached to avoid recreation overhead + */ +export function getConversationGraph( + config: ConversationGraphConfig, + languageCode: string = DEFAULT_LANGUAGE_CODE +): Graph { + const cacheKey = languageCode; + + if (!graphCache.has(cacheKey)) { + const languageConfig = getLanguageConfig(languageCode); + console.log( + `Creating conversation graph for language: ${languageConfig.name} (${languageCode})` + ); + const graph = createConversationGraphForLanguage(config, languageConfig); + graphCache.set(cacheKey, graph); + } + + return graphCache.get(cacheKey)!; +} + +/** + * Legacy function for backwards compatibility + * Creates or returns the default (Spanish) graph + */ +export function createConversationGraph(config: ConversationGraphConfig): Graph { + return getConversationGraph(config, DEFAULT_LANGUAGE_CODE); +} + +/** + * Clear the graph cache (useful for testing or reconfiguration) + */ +export function clearGraphCache(): void { + graphCache.clear(); +} diff --git a/backend/graphs/flashcard-graph.ts b/backend/graphs/flashcard-graph.ts index 8452137..3990d7e 100644 --- a/backend/graphs/flashcard-graph.ts +++ b/backend/graphs/flashcard-graph.ts @@ -6,12 +6,18 @@ import { CustomNode, ProcessContext, RemoteLLMChatNode, + Graph, } from '@inworld/runtime/graph'; import { GraphTypes } from '@inworld/runtime/common'; import { renderJinja } from '@inworld/runtime/primitives/llm'; import { flashcardPromptTemplate } from '../helpers/prompt-templates.js'; import { v4 } from 'uuid'; import { Flashcard } from '../helpers/flashcard-processor.js'; +import { + LanguageConfig, + getLanguageConfig, + DEFAULT_LANGUAGE_CODE, +} from '../config/languages.js'; class FlashcardPromptBuilderNode extends CustomNode { async process( @@ -51,7 +57,8 @@ class FlashcardParserNode extends CustomNode { const parsed = JSON.parse(jsonMatch[0]); return { id: v4(), - spanish: parsed.spanish ?? '', + // Support both new 'targetWord' format and legacy 'spanish' format + targetWord: parsed.targetWord ?? parsed.spanish ?? '', english: parsed.english ?? '', example: parsed.example ?? '', mnemonic: parsed.mnemonic ?? '', @@ -64,7 +71,7 @@ class FlashcardParserNode extends CustomNode { return { id: v4(), - spanish: '', + targetWord: '', english: '', example: '', mnemonic: '', @@ -74,7 +81,10 @@ class FlashcardParserNode extends CustomNode { } } -export function createFlashcardGraph() { +/** + * Creates a flashcard generation graph for a specific language + */ +function createFlashcardGraphForLanguage(languageConfig: LanguageConfig): Graph { const apiKey = process.env.INWORLD_API_KEY; if (!apiKey) { throw new Error('INWORLD_API_KEY environment variable is required'); @@ -104,7 +114,7 @@ export function createFlashcardGraph() { const parserNode = new FlashcardParserNode({ id: 'flashcard-parser' }); const executor = new GraphBuilder({ - id: 'flashcard-generation-graph', + id: `flashcard-generation-graph-${languageConfig.code}`, enableRemoteConfig: true, }) .addNode(promptBuilderNode) @@ -118,7 +128,38 @@ export function createFlashcardGraph() { .setEndNode(parserNode) .build(); - fs.writeFileSync('flashcard-graph.json', executor.toJSON()); + // Only write debug file for default language to avoid cluttering + if (languageConfig.code === DEFAULT_LANGUAGE_CODE) { + fs.writeFileSync('flashcard-graph.json', executor.toJSON()); + } return executor; } + +// Cache for language-specific flashcard graphs +const flashcardGraphCache = new Map(); + +/** + * Get or create a flashcard graph for a specific language + */ +export function getFlashcardGraph( + languageCode: string = DEFAULT_LANGUAGE_CODE +): Graph { + if (!flashcardGraphCache.has(languageCode)) { + const languageConfig = getLanguageConfig(languageCode); + console.log( + `Creating flashcard graph for language: ${languageConfig.name} (${languageCode})` + ); + const graph = createFlashcardGraphForLanguage(languageConfig); + flashcardGraphCache.set(languageCode, graph); + } + + return flashcardGraphCache.get(languageCode)!; +} + +/** + * Legacy function for backwards compatibility + */ +export function createFlashcardGraph(): Graph { + return getFlashcardGraph(DEFAULT_LANGUAGE_CODE); +} diff --git a/backend/graphs/introduction-state-graph.ts b/backend/graphs/introduction-state-graph.ts index ae078a9..fb929a9 100644 --- a/backend/graphs/introduction-state-graph.ts +++ b/backend/graphs/introduction-state-graph.ts @@ -4,10 +4,16 @@ import { CustomNode, ProcessContext, RemoteLLMChatNode, + Graph, } from '@inworld/runtime/graph'; import { GraphTypes } from '@inworld/runtime/common'; import { renderJinja } from '@inworld/runtime/primitives/llm'; import { introductionStatePromptTemplate } from '../helpers/prompt-templates.js'; +import { + LanguageConfig, + getLanguageConfig, + DEFAULT_LANGUAGE_CODE, +} from '../config/languages.js'; type IntroductionStateLevel = 'beginner' | 'intermediate' | 'advanced' | ''; @@ -39,20 +45,53 @@ class TextToChatRequestNode extends CustomNode { } } +/** + * Normalize level strings from various languages to standard values + * Supports: English, Spanish, Japanese, French (and more can be added) + */ function normalizeLevel(level: unknown): IntroductionStateLevel { if (typeof level !== 'string') return ''; const lower = level .trim() .toLowerCase() .replace(/[.!?,;:]+$/g, ''); + const mapping: Record = { + // English beginner: 'beginner', intermediate: 'intermediate', advanced: 'advanced', + // Spanish principiante: 'beginner', intermedio: 'intermediate', avanzado: 'advanced', + // French + débutant: 'beginner', + debutant: 'beginner', + intermédiaire: 'intermediate', + intermediaire: 'intermediate', + avancé: 'advanced', + avance: 'advanced', + // Japanese (romanji) + shoshinsha: 'beginner', + chūkyū: 'intermediate', + chuukyuu: 'intermediate', + jōkyū: 'advanced', + joukyuu: 'advanced', + // Japanese (hiragana/katakana - common responses) + 初心者: 'beginner', + 中級: 'intermediate', + 上級: 'advanced', + // Additional variations + basic: 'beginner', + elementary: 'beginner', + beginning: 'beginner', + middle: 'intermediate', + medium: 'intermediate', + expert: 'advanced', + fluent: 'advanced', }; + return mapping[lower] || ''; } @@ -104,7 +143,12 @@ class IntroductionStateParserNode extends CustomNode { } } -export function createIntroductionStateGraph() { +/** + * Create an introduction state extraction graph for a specific language + */ +function createIntroductionStateGraphForLanguage( + languageConfig: LanguageConfig +): Graph { const apiKey = process.env.INWORLD_API_KEY; if (!apiKey) { throw new Error('INWORLD_API_KEY environment variable is required'); @@ -126,7 +170,9 @@ export function createIntroductionStateGraph() { id: 'introduction-state-parser', }); - const executor = new GraphBuilder('introduction-state-graph') + const executor = new GraphBuilder({ + id: `introduction-state-graph-${languageConfig.code}`, + }) .addNode(promptBuilderNode) .addNode(textToChatRequestNode) .addNode(llmNode) @@ -140,3 +186,31 @@ export function createIntroductionStateGraph() { return executor; } + +// Cache for language-specific introduction state graphs +const introductionStateGraphCache = new Map(); + +/** + * Get or create an introduction state graph for a specific language + */ +export function getIntroductionStateGraph( + languageCode: string = DEFAULT_LANGUAGE_CODE +): Graph { + if (!introductionStateGraphCache.has(languageCode)) { + const languageConfig = getLanguageConfig(languageCode); + console.log( + `Creating introduction state graph for language: ${languageConfig.name} (${languageCode})` + ); + const graph = createIntroductionStateGraphForLanguage(languageConfig); + introductionStateGraphCache.set(languageCode, graph); + } + + return introductionStateGraphCache.get(languageCode)!; +} + +/** + * Legacy function for backwards compatibility + */ +export function createIntroductionStateGraph(): Graph { + return getIntroductionStateGraph(DEFAULT_LANGUAGE_CODE); +} diff --git a/backend/helpers/anki-exporter.ts b/backend/helpers/anki-exporter.ts index bc78c88..16f47ba 100644 --- a/backend/helpers/anki-exporter.ts +++ b/backend/helpers/anki-exporter.ts @@ -8,28 +8,37 @@ export class AnkiExporter { */ async exportFlashcards( flashcards: Flashcard[], - deckName: string = 'Aprendemo Spanish Cards' + deckName: string = 'Aprendemo Language Cards' ): Promise { // eslint-disable-next-line @typescript-eslint/no-explicit-any const apkg = new (AnkiExport as any).default(deckName); // Add each flashcard as a card flashcards.forEach((flashcard) => { + // Support both new 'targetWord' and legacy 'spanish' field + const targetWord = flashcard.targetWord || (flashcard as { spanish?: string }).spanish; + // Skip empty or error flashcards if ( - !flashcard.spanish || + !targetWord || !flashcard.english || - flashcard.spanish.trim() === '' || + targetWord.trim() === '' || flashcard.english.trim() === '' ) { return; } - const front = flashcard.spanish.trim(); + const front = targetWord.trim(); const back = this.formatCardBack(flashcard); // Add tags for organization - const tags = ['aprendemo', 'spanish-learning']; + const tags = ['aprendemo']; + + // Add language tag if available + if (flashcard.languageCode) { + tags.push(`language-${flashcard.languageCode}`); + } + if (flashcard.timestamp) { const date = new Date(flashcard.timestamp).toISOString().split('T')[0]; tags.push(`created-${date}`); @@ -76,12 +85,14 @@ export class AnkiExporter { * Count valid flashcards (ones that can be exported) */ countValidFlashcards(flashcards: Flashcard[]): number { - return flashcards.filter( - (flashcard) => - flashcard.spanish && + return flashcards.filter((flashcard) => { + const targetWord = flashcard.targetWord || (flashcard as { spanish?: string }).spanish; + return ( + targetWord && flashcard.english && - flashcard.spanish.trim() !== '' && + targetWord.trim() !== '' && flashcard.english.trim() !== '' - ).length; + ); + }).length; } } diff --git a/backend/helpers/audio-processor.ts b/backend/helpers/audio-processor.ts index 59e2c51..1a122c7 100644 --- a/backend/helpers/audio-processor.ts +++ b/backend/helpers/audio-processor.ts @@ -4,8 +4,13 @@ import { UserContextInterface } from '@inworld/runtime/graph'; import { Graph } from '@inworld/runtime/graph'; import { WebSocket } from 'ws'; import { SileroVAD, VADConfig } from './silero-vad.js'; -import { stateStorage } from '../graphs/conversation-graph.js'; +import { stateStorage, setCurrentExecutionContext } from '../graphs/conversation-graph.js'; import type { IntroductionState } from './introduction-state-processor.js'; +import { + LanguageConfig, + getLanguageConfig, + DEFAULT_LANGUAGE_CODE, +} from '../config/languages.js'; export class AudioProcessor { private executor: Graph; @@ -37,13 +42,57 @@ export class AudioProcessor { private targetingKey: string | null = null; private clientTimezone: string | null = null; private graphStartTime: number = 0; - constructor(executor: Graph, websocket?: WebSocket) { + private languageCode: string = DEFAULT_LANGUAGE_CODE; + private languageConfig: LanguageConfig; + + constructor( + executor: Graph, + websocket?: WebSocket, + languageCode: string = DEFAULT_LANGUAGE_CODE + ) { this.executor = executor; this.websocket = websocket ?? null; + this.languageCode = languageCode; + this.languageConfig = getLanguageConfig(languageCode); this.setupWebSocketMessageHandler(); setTimeout(() => this.initialize(), 100); } + /** + * Update the language and graph for this processor + */ + setLanguage(languageCode: string, newGraph: Graph): void { + if (this.languageCode !== languageCode) { + console.log( + `AudioProcessor: Changing language from ${this.languageCode} to ${languageCode}` + ); + this.languageCode = languageCode; + this.languageConfig = getLanguageConfig(languageCode); + this.executor = newGraph; + + // Reset conversation state when language changes + this.reset(); + + console.log( + `AudioProcessor: Language changed to ${this.languageConfig.name}` + ); + } + } + + /** + * Get current language code + */ + getLanguageCode(): string { + return this.languageCode; + } + + /** + * Get current language config + */ + getLanguageConfig(): LanguageConfig { + return this.languageConfig; + } + // Public methods for state registry access getConversationState() { return this.conversationState; @@ -254,7 +303,7 @@ export class AudioProcessor { // Just mark as ready this.isReady = true; console.log( - 'AudioProcessor: Using shared conversation graph, ready for audio processing' + `AudioProcessor: Using ${this.languageConfig.name} conversation graph, ready for audio processing` ); } catch (error) { console.error( @@ -330,6 +379,7 @@ export class AudioProcessor { // Build user context for experiments const attributes: Record = { timezone: this.clientTimezone || '', + language: this.languageCode, }; attributes.name = (this.introductionState?.name && this.introductionState.name.trim()) || @@ -351,12 +401,21 @@ export class AudioProcessor { this.graphStartTime = Date.now(); - // Use AsyncLocalStorage to set state accessors for this execution context - // This allows the graph nodes to access state directly without needing connectionId + // Set the current execution context BEFORE starting the graph + // This ensures the PromptBuilder node has access to state and language config + // We use module-level variables because Inworld runtime breaks AsyncLocalStorage context + setCurrentExecutionContext({ + languageCode: this.languageCode, + getConversationState: () => this.getConversationState(), + getIntroductionState: () => this.getIntroductionState(), + }); + + // Also use AsyncLocalStorage as a fallback (may work in some contexts) const executionResult = await stateStorage.run( { getConversationState: () => this.getConversationState(), getIntroductionState: () => this.getIntroductionState(), + getLanguageConfig: () => this.getLanguageConfig(), }, async () => { try { @@ -400,7 +459,15 @@ export class AudioProcessor { string: (data: string) => { transcription = data; + // Debug: always log raw STT result for troubleshooting + console.log( + `VAD STT Raw Result [${this.languageCode}]: "${data}" (length: ${data.length}, trimmed: ${data.trim().length})` + ); + if (transcription.trim() === '') { + console.log( + `VAD STT: Empty transcription for ${this.languageCode}, skipping LLM` + ); return; } diff --git a/backend/helpers/flashcard-processor.ts b/backend/helpers/flashcard-processor.ts index ab1b028..ef82b3c 100644 --- a/backend/helpers/flashcard-processor.ts +++ b/backend/helpers/flashcard-processor.ts @@ -2,15 +2,21 @@ import { v4 } from 'uuid'; import { Graph } from '@inworld/runtime/graph'; import { GraphTypes } from '@inworld/runtime/common'; import { UserContextInterface } from '@inworld/runtime/graph'; -import { createFlashcardGraph } from '../graphs/flashcard-graph.js'; +import { getFlashcardGraph } from '../graphs/flashcard-graph.js'; +import { + LanguageConfig, + getLanguageConfig, + DEFAULT_LANGUAGE_CODE, +} from '../config/languages.js'; export interface Flashcard { id: string; - spanish: string; + targetWord: string; // The word in the target language (was 'spanish') english: string; example: string; mnemonic: string; timestamp: string; + languageCode?: string; // Track which language this card belongs to } export interface ConversationMessage { @@ -20,9 +26,30 @@ export interface ConversationMessage { export class FlashcardProcessor { private existingFlashcards: Flashcard[] = []; + private languageCode: string = DEFAULT_LANGUAGE_CODE; + private languageConfig: LanguageConfig; - constructor() { - // Initialize with empty flashcard array + constructor(languageCode: string = DEFAULT_LANGUAGE_CODE) { + this.languageCode = languageCode; + this.languageConfig = getLanguageConfig(languageCode); + } + + /** + * Update the language for this processor + */ + setLanguage(languageCode: string): void { + if (this.languageCode !== languageCode) { + this.languageCode = languageCode; + this.languageConfig = getLanguageConfig(languageCode); + console.log(`FlashcardProcessor: Language changed to ${this.languageConfig.name}`); + } + } + + /** + * Get current language code + */ + getLanguageCode(): string { + return this.languageCode; } async generateFlashcards( @@ -30,7 +57,7 @@ export class FlashcardProcessor { count: number = 1, userContext?: UserContextInterface ): Promise { - const executor = createFlashcardGraph(); + const executor = getFlashcardGraph(this.languageCode); // Generate flashcards in parallel const promises: Promise[] = []; @@ -46,7 +73,7 @@ export class FlashcardProcessor { // Filter out any failed generations and duplicates const validFlashcards = flashcards.filter( - (card) => card.spanish && card.english + (card) => card.targetWord && card.english ); // Add to existing flashcards to track for future duplicates @@ -67,7 +94,8 @@ export class FlashcardProcessor { try { const input = { studentName: 'Student', - teacherName: 'Señor Rosales', + teacherName: this.languageConfig.teacherPersona.name, + target_language: this.languageConfig.name, messages: messages, flashcards: this.existingFlashcards, }; @@ -92,10 +120,13 @@ export class FlashcardProcessor { } const flashcard = finalData as unknown as Flashcard; + // Add language code to the flashcard + flashcard.languageCode = this.languageCode; + // Check if this is a duplicate const isDuplicate = this.existingFlashcards.some( (existing) => - existing.spanish?.toLowerCase() === flashcard.spanish?.toLowerCase() + existing.targetWord?.toLowerCase() === flashcard.targetWord?.toLowerCase() ); if (isDuplicate) { @@ -103,11 +134,12 @@ export class FlashcardProcessor { // For simplicity, we'll just return an empty flashcard if duplicate return { id: v4(), - spanish: '', + targetWord: '', english: '', example: '', mnemonic: '', timestamp: new Date().toISOString(), + languageCode: this.languageCode, } as Flashcard & { error?: string }; } @@ -116,11 +148,12 @@ export class FlashcardProcessor { console.error('Error generating single flashcard:', error); return { id: v4(), - spanish: '', + targetWord: '', english: '', example: '', mnemonic: '', timestamp: new Date().toISOString(), + languageCode: this.languageCode, } as Flashcard & { error?: string }; } } diff --git a/backend/helpers/introduction-state-processor.ts b/backend/helpers/introduction-state-processor.ts index 678cfb3..ea465fe 100644 --- a/backend/helpers/introduction-state-processor.ts +++ b/backend/helpers/introduction-state-processor.ts @@ -1,7 +1,12 @@ import { v4 as uuidv4 } from 'uuid'; import { Graph } from '@inworld/runtime/graph'; import { GraphTypes } from '@inworld/runtime/common'; -import { createIntroductionStateGraph } from '../graphs/introduction-state-graph.js'; +import { getIntroductionStateGraph } from '../graphs/introduction-state-graph.js'; +import { + LanguageConfig, + getLanguageConfig, + DEFAULT_LANGUAGE_CODE, +} from '../config/languages.js'; export type IntroductionStateLevel = | 'beginner' @@ -29,9 +34,34 @@ export class IntroductionStateProcessor { timestamp: '', }; private executor: Graph; + private languageCode: string = DEFAULT_LANGUAGE_CODE; + private languageConfig: LanguageConfig; - constructor() { - this.executor = createIntroductionStateGraph(); + constructor(languageCode: string = DEFAULT_LANGUAGE_CODE) { + this.languageCode = languageCode; + this.languageConfig = getLanguageConfig(languageCode); + this.executor = getIntroductionStateGraph(languageCode); + } + + /** + * Update the language for this processor + */ + setLanguage(languageCode: string): void { + if (this.languageCode !== languageCode) { + this.languageCode = languageCode; + this.languageConfig = getLanguageConfig(languageCode); + this.executor = getIntroductionStateGraph(languageCode); + console.log( + `IntroductionStateProcessor: Language changed to ${this.languageConfig.name}` + ); + } + } + + /** + * Get current language code + */ + getLanguageCode(): string { + return this.languageCode; } isComplete(): boolean { @@ -68,6 +98,7 @@ export class IntroductionStateProcessor { const input = { messages, existingState: this.state, + target_language: this.languageConfig.name, }; console.log( diff --git a/backend/helpers/prompt-templates.ts b/backend/helpers/prompt-templates.ts index 4a2c64d..b07e1ef 100644 --- a/backend/helpers/prompt-templates.ts +++ b/backend/helpers/prompt-templates.ts @@ -1,39 +1,51 @@ +/** + * Prompt Templates for Multi-Language Support + * + * All templates use Jinja2-style variables that are injected at runtime: + * - {{target_language}} - English name of target language (e.g., "Spanish") + * - {{target_language_native}} - Native name (e.g., "Español") + * - {{teacher_name}} - Teacher persona name + * - {{teacher_description}} - Full teacher persona description + * - {{example_topics}} - Comma-separated list of conversation topics + * - {{language_instructions}} - Language-specific teaching instructions + */ + export const conversationTemplate = ` # Context -- You are a Señor Gael Herrera, 35 year old 'Chilango' who has loaned their brain to AI. -- You are embedded in a Spanish learning app called 'Aprendemo', which is a demonstration of the Inworld AI Runtime. -- You can help the user learn Spanish by having natural (verbalized) conversations with them. +- You are {{teacher_name}}, {{teacher_description}}. +- You are embedded in a {{target_language}} learning app called 'Aprendemo', which is a demonstration of the Inworld AI Runtime. +- You can help the user learn {{target_language}} by having natural (verbalized) conversations with them. - The app generates flashcards for the user during the conversation. They are ANKI formatted and can be exported by the user. # Instructions {% if introduction_state and (not introduction_state.name or not introduction_state.level or not introduction_state.goal) %} -- Your first priority is to collect missing onboarding info. Ask for exactly one missing item at a time, in Spanish, and keep it short and natural. +- Your first priority is to collect missing onboarding info. Ask for exactly one missing item at a time, in {{target_language}}, and keep it short and natural. - Missing items to collect: {% if not introduction_state.name %}- Ask their name. {% endif %} - {% if not introduction_state.level %}- Ask their Spanish level (beginner, intermediate, or advanced). + {% if not introduction_state.level %}- Ask their {{target_language}} level (beginner, intermediate, or advanced). {% endif %} - {% if not introduction_state.goal %}- Ask their goal for learning Spanish. + {% if not introduction_state.goal %}- Ask their goal for learning {{target_language}}. {% endif %} - Do not assume or guess values. If a value was already collected, do not ask for it again. -- If the user's latest message appears to answer one of these (e.g., they state their name, level like "principiante/intermedio/avanzado" or "beginner/intermediate/advanced", or share a goal), acknowledge it and immediately move to the next missing item instead of repeating the same question. +- If the user's latest message appears to answer one of these (e.g., they state their name, level like "beginner/intermediate/advanced" or equivalent in {{target_language}}, or share a goal), acknowledge it and immediately move to the next missing item instead of repeating the same question. - Use the name naturally as soon as they provide it. {% else %} -- Greet the user and introduce yourself in Spanish +- Greet the user and introduce yourself in {{target_language}} - Ask the user if they want a lesson or conversation on a specific topic, then proceed -- If they don't want anything in particular, lead them in a conversation or lesson about Mexico City, the Dunedin sound rock scene, gardening, the concept of brunch across cultures, Balkan travel, or any other topic which comes to mind +- If they don't want anything in particular, lead them in a conversation or lesson about {{example_topics}}, or any other topic which comes to mind - You can advise the user that if they want specific flashcards, they should just ask -- Gently correct the user if they make mistakes +{{language_instructions}} - Don't always ask the user questions, you can talk about yourself as well. Be natural! {% endif %} # Communication Style -- Vary your conversation starters - don't always begin with "¡Hola!" or exclamations +- Vary your conversation starters - don't always begin with greetings or exclamations - Respond naturally as if you're in the middle of an ongoing conversation - Use varied sentence structures and beginnings -- Sometimes start with: direct responses, "Ah", "Bueno", "Claro", "Pues", "Sí", or simply dive into your response -- Only use "¡Hola!" when it's actually a greeting at the start of a new conversation +- Sometimes start with: direct responses, interjections, or simply dive into your response +- Only use greetings when it's actually the start of a new conversation - Be conversational and natural, not overly enthusiastic with every response - When available, naturally use the user's name. Adjust complexity to their level (beginner, intermediate, advanced) and align topics with their goal. @@ -50,13 +62,13 @@ Please respond naturally and clearly in 1-2 sentences. Vary your response style export const flashcardPromptTemplate = ` -You are a system that generates flashcards for interesting new vocabulary for a Spanish learning app. +You are a system that generates flashcards for interesting new vocabulary for a {{target_language}} learning app. Based on the ongoing conversation between {{studentName}} and {{teacherName}}, generate one flashcard with the following things: -- The word in Spanish +- The word in {{target_language}} - The translation in English -- An example sentence in Spanish +- An example sentence in {{target_language}} - A mnemonic to help the student remember the word ## Conversation @@ -67,7 +79,7 @@ Based on the ongoing conversation between {{studentName}} and {{teacherName}}, g ## Already Created Flashcards {% for flashcard in flashcards %} -- Word: {{flashcard.spanish}} +- Word: {{flashcard.targetWord}} {% endfor %} ## Guidelines @@ -80,7 +92,7 @@ Based on the ongoing conversation between {{studentName}} and {{teacherName}}, g Now, return JSON with the following format: { - "spanish": "string", + "targetWord": "string", "english": "string", "example": "string", "mnemonic": "string" @@ -88,7 +100,7 @@ Now, return JSON with the following format: export const introductionStatePromptTemplate = ` -You extract onboarding information for a Spanish learning app. +You extract onboarding information for a {{target_language}} learning app. Your job is to collect the learner's name, level, and goal ONLY if they have been explicitly provided. Do not guess or infer. diff --git a/backend/server.ts b/backend/server.ts index 30b441f..23af9a7 100644 --- a/backend/server.ts +++ b/backend/server.ts @@ -21,7 +21,12 @@ import { AudioProcessor } from './helpers/audio-processor.js'; import { FlashcardProcessor } from './helpers/flashcard-processor.js'; import { AnkiExporter } from './helpers/anki-exporter.js'; import { IntroductionStateProcessor } from './helpers/introduction-state-processor.js'; -import { createConversationGraph } from './graphs/conversation-graph.js'; +import { getConversationGraph } from './graphs/conversation-graph.js'; +import { + getLanguageConfig, + getLanguageOptions, + DEFAULT_LANGUAGE_CODE, +} from './config/languages.js'; const app = express(); const server = createServer(app); @@ -57,11 +62,6 @@ try { console.error('[Telemetry] Initialization failed:', err); } -// Create a single shared conversation graph instance -// State is passed directly through AsyncLocalStorage, no registry needed -let sharedConversationGraph: ReturnType | null = - null; - // Store audio processors per connection const audioProcessors = new Map(); const flashcardProcessors = new Map(); @@ -69,20 +69,18 @@ const introductionStateProcessors = new Map< string, IntroductionStateProcessor >(); -// Store lightweight per-connection attributes provided by the client (e.g., timezone, userId) +// Store lightweight per-connection attributes provided by the client (e.g., timezone, userId, languageCode) const connectionAttributes = new Map< string, - { timezone?: string; userId?: string } + { timezone?: string; userId?: string; languageCode?: string } >(); -// Initialize shared graph once at startup -function initializeSharedGraph() { - if (!sharedConversationGraph) { - const apiKey = process.env.INWORLD_API_KEY || ''; - sharedConversationGraph = createConversationGraph({ apiKey }); - console.log('Shared conversation graph initialized'); - } - return sharedConversationGraph; +/** + * Get or create a conversation graph for the specified language + */ +function getGraphForLanguage(languageCode: string) { + const apiKey = process.env.INWORLD_API_KEY || ''; + return getConversationGraph({ apiKey }, languageCode); } // WebSocket handling with audio processing @@ -90,17 +88,22 @@ wss.on('connection', (ws) => { const connectionId = `conn_${Date.now()}_${Math.random().toString(36).substr(2, 9)}`; console.log(`WebSocket connection established: ${connectionId}`); - // Create audio processor for this connection - // Use the shared graph instance - state is passed via AsyncLocalStorage - const sharedGraph = initializeSharedGraph(); - const audioProcessor = new AudioProcessor(sharedGraph, ws); - const flashcardProcessor = new FlashcardProcessor(); - const introductionStateProcessor = new IntroductionStateProcessor(); + // Default language is Spanish + const defaultLanguageCode = DEFAULT_LANGUAGE_CODE; + + // Create processors with default language + const graph = getGraphForLanguage(defaultLanguageCode); + const audioProcessor = new AudioProcessor(graph, ws, defaultLanguageCode); + const flashcardProcessor = new FlashcardProcessor(defaultLanguageCode); + const introductionStateProcessor = new IntroductionStateProcessor( + defaultLanguageCode + ); - // Register audio processor + // Register processors audioProcessors.set(connectionId, audioProcessor); flashcardProcessors.set(connectionId, flashcardProcessor); introductionStateProcessors.set(connectionId, introductionStateProcessor); + connectionAttributes.set(connectionId, { languageCode: defaultLanguageCode }); // Set up flashcard generation callback audioProcessor.setFlashcardCallback(async (messages) => { @@ -215,6 +218,53 @@ wss.on('connection', (ws) => { } console.log(`Conversation restarted for connection: ${connectionId}`); + } else if (message.type === 'set_language') { + // Handle language change + const newLanguageCode = message.languageCode || DEFAULT_LANGUAGE_CODE; + const attrs = connectionAttributes.get(connectionId) || {}; + + // Only process if language actually changed + if (attrs.languageCode !== newLanguageCode) { + console.log( + `Language change requested for ${connectionId}: ${attrs.languageCode} -> ${newLanguageCode}` + ); + + // Update stored language + attrs.languageCode = newLanguageCode; + connectionAttributes.set(connectionId, attrs); + + // Get the new language config + const languageConfig = getLanguageConfig(newLanguageCode); + + // Update all processors with new language + const audioProc = audioProcessors.get(connectionId); + const flashcardProc = flashcardProcessors.get(connectionId); + const introStateProc = introductionStateProcessors.get(connectionId); + + if (flashcardProc) { + flashcardProc.setLanguage(newLanguageCode); + } + if (introStateProc) { + introStateProc.setLanguage(newLanguageCode); + } + if (audioProc) { + // Audio processor needs a new graph for the new language + const newGraph = getGraphForLanguage(newLanguageCode); + audioProc.setLanguage(newLanguageCode, newGraph); + } + + // Send confirmation to frontend + ws.send( + JSON.stringify({ + type: 'language_changed', + languageCode: newLanguageCode, + languageName: languageConfig.name, + teacherName: languageConfig.teacherPersona.name, + }) + ); + + console.log(`Language changed to ${languageConfig.name} for ${connectionId}`); + } } else if (message.type === 'user_context') { const timezone = message.timezone || @@ -222,7 +272,17 @@ wss.on('connection', (ws) => { undefined; const userId = message.userId || (message.data && message.data.userId) || undefined; - connectionAttributes.set(connectionId, { timezone, userId }); + const languageCode = + message.languageCode || + (message.data && message.data.languageCode) || + undefined; + const currentAttrs = connectionAttributes.get(connectionId) || {}; + connectionAttributes.set(connectionId, { + ...currentAttrs, + timezone: timezone || currentAttrs.timezone, + userId: userId || currentAttrs.userId, + languageCode: languageCode || currentAttrs.languageCode, + }); } else if (message.type === 'flashcard_clicked') { try { const card = message.card || {}; @@ -233,10 +293,11 @@ wss.on('connection', (ws) => { telemetry.metric.recordCounterUInt('flashcard_clicks_total', 1, { connectionId, cardId: card.id || '', - spanish: card.spanish || card.word || '', + targetWord: card.targetWord || card.spanish || card.word || '', english: card.english || card.translation || '', source: 'ui', timezone: attrs.timezone || '', + languageCode: attrs.languageCode || DEFAULT_LANGUAGE_CODE, name: (introState?.name && introState.name.trim()) || 'unknown', level: (introState?.level && (introState.level as string)) || 'unknown', @@ -286,20 +347,25 @@ wss.on('connection', (ws) => { // API endpoint for ANKI export app.post('/api/export-anki', async (req, res) => { try { - const { flashcards, deckName } = req.body; + const { flashcards, deckName, languageCode } = req.body; if (!flashcards || !Array.isArray(flashcards)) { return res.status(400).json({ error: 'Invalid flashcards data' }); } + // Get language config for deck naming + const langCode = languageCode || DEFAULT_LANGUAGE_CODE; + const languageConfig = getLanguageConfig(langCode); + const defaultDeckName = `Aprendemo ${languageConfig.name} Cards`; + const exporter = new AnkiExporter(); const ankiBuffer = await exporter.exportFlashcards( flashcards, - deckName || 'Aprendemo Spanish Cards' + deckName || defaultDeckName ); // Set headers for file download - const filename = `${deckName || 'aprendemo_cards'}.apkg`; + const filename = `${deckName || `aprendemo_${langCode}_cards`}.apkg`; res.setHeader('Content-Type', 'application/octet-stream'); res.setHeader('Content-Disposition', `attachment; filename="${filename}"`); res.setHeader('Content-Length', ankiBuffer.length); @@ -314,6 +380,17 @@ app.post('/api/export-anki', async (req, res) => { } }); +// API endpoint to get supported languages +app.get('/api/languages', (_req, res) => { + try { + const languages = getLanguageOptions(); + res.json({ languages, defaultLanguage: DEFAULT_LANGUAGE_CODE }); + } catch (error) { + console.error('Error getting languages:', error); + res.status(500).json({ error: 'Failed to get languages' }); + } +}); + // Serve static frontend files // When running from dist/backend/server.js, go up two levels to project root // When running from backend/server.ts (dev mode), go up one level to project root diff --git a/flashcard-graph.json b/flashcard-graph.json index 0f7f048..80be8e4 100644 --- a/flashcard-graph.json +++ b/flashcard-graph.json @@ -1,7 +1,7 @@ { "schema_version": "1.2.0", "main": { - "id": "flashcard-generation-graph", + "id": "flashcard-generation-graph-es", "nodes": [ { "type": "FlashcardPromptBuilderNodeType", @@ -70,8 +70,12 @@ "to_node": "flashcard-parser" } ], - "end_nodes": ["flashcard-parser"], - "start_nodes": ["flashcard-prompt-builder"] + "end_nodes": [ + "flashcard-parser" + ], + "start_nodes": [ + "flashcard-prompt-builder" + ] }, "components": [ { @@ -88,4 +92,4 @@ } } ] -} +} \ No newline at end of file diff --git a/frontend/index.html b/frontend/index.html index e84a55b..7eb9aab 100644 --- a/frontend/index.html +++ b/frontend/index.html @@ -13,9 +13,16 @@

Aprendemo

-
- - Connecting... +
+
+ +
+
+ + Connecting... +
@@ -90,4 +97,4 @@

Flashcards

- \ No newline at end of file + diff --git a/frontend/js/flashcard-ui.js b/frontend/js/flashcard-ui.js index fa236a4..1d887da 100644 --- a/frontend/js/flashcard-ui.js +++ b/frontend/js/flashcard-ui.js @@ -3,10 +3,12 @@ export class FlashcardUI { this.flashcardsGrid = document.getElementById('flashcardsGrid'); this.cardCount = document.getElementById('cardCount'); this.flashcards = []; + this.currentLanguage = 'es'; } - render(flashcards) { + render(flashcards, languageCode = 'es') { this.flashcards = flashcards; + this.currentLanguage = languageCode; this.updateCardCount(flashcards.length); this.renderFlashcards(flashcards); } @@ -14,7 +16,7 @@ export class FlashcardUI { addFlashcards(newFlashcards) { // Add new flashcards to the existing collection this.flashcards = [...this.flashcards, ...newFlashcards]; - this.render(this.flashcards); + this.render(this.flashcards, this.currentLanguage); } updateCardCount(count) { @@ -65,10 +67,13 @@ export class FlashcardUI { const card = document.createElement('div'); card.className = 'flashcard'; + // Support both new 'targetWord' and legacy 'spanish' field + const targetWord = flashcard.targetWord || flashcard.spanish || flashcard.word || ''; + card.innerHTML = `
-
${this.escapeHtml(flashcard.spanish || flashcard.word || '')}
+
${this.escapeHtml(targetWord)}
${this.escapeHtml(flashcard.english || flashcard.translation || '')}
@@ -104,13 +109,15 @@ export class FlashcardUI { async exportToAnki() { try { // Filter out invalid flashcards - const validFlashcards = this.flashcards.filter( - (flashcard) => - flashcard.spanish && + const validFlashcards = this.flashcards.filter((flashcard) => { + const targetWord = flashcard.targetWord || flashcard.spanish; + return ( + targetWord && flashcard.english && - flashcard.spanish.trim() !== '' && + targetWord.trim() !== '' && flashcard.english.trim() !== '' - ); + ); + }); if (validFlashcards.length === 0) { alert('No valid flashcards to export'); @@ -122,6 +129,14 @@ export class FlashcardUI { this.cardCount.textContent = 'Exporting...'; this.cardCount.style.cursor = 'wait'; + // Get language name for deck naming + const languageNames = { + es: 'Spanish', + ja: 'Japanese', + fr: 'French', + }; + const languageName = languageNames[this.currentLanguage] || 'Language'; + const response = await fetch('/api/export-anki', { method: 'POST', headers: { @@ -129,7 +144,8 @@ export class FlashcardUI { }, body: JSON.stringify({ flashcards: validFlashcards, - deckName: 'Aprendemo Spanish Cards', + deckName: `Aprendemo ${languageName} Cards`, + languageCode: this.currentLanguage, }), }); @@ -142,7 +158,7 @@ export class FlashcardUI { const url = window.URL.createObjectURL(blob); const a = document.createElement('a'); a.href = url; - a.download = 'aprendemo_cards.apkg'; + a.download = `aprendemo_${this.currentLanguage}_cards.apkg`; document.body.appendChild(a); a.click(); document.body.removeChild(a); diff --git a/frontend/js/main.js b/frontend/js/main.js index 0b60c4f..cf3f1d8 100644 --- a/frontend/js/main.js +++ b/frontend/js/main.js @@ -19,6 +19,10 @@ class App { this.wsClient.send({ type: 'flashcard_clicked', card }); }; + // Language state + this.currentLanguage = this.storage.getLanguage() || 'es'; + this.availableLanguages = []; + this.state = { chatHistory: [], flashcards: [], @@ -41,11 +45,59 @@ class App { async init() { this.loadState(); this.setupEventListeners(); + await this.fetchLanguages(); await this.connectWebSocket(); await this.initializeAudioPlayer(); this.render(); } + async fetchLanguages() { + try { + const response = await fetch('/api/languages'); + if (response.ok) { + const data = await response.json(); + this.availableLanguages = data.languages; + + // If current language is not in available languages, use default + const isValidLanguage = this.availableLanguages.some( + (lang) => lang.code === this.currentLanguage + ); + if (!isValidLanguage) { + this.currentLanguage = data.defaultLanguage || 'es'; + } + + this.populateLanguageDropdown(); + } + } catch (error) { + console.error('Failed to fetch languages:', error); + // Fallback to default language options + this.availableLanguages = [ + { code: 'es', name: 'Spanish', nativeName: 'Español', flag: '🇲🇽' }, + { code: 'ja', name: 'Japanese', nativeName: '日本語', flag: '🇯🇵' }, + { code: 'fr', name: 'French', nativeName: 'Français', flag: '🇫🇷' }, + ]; + this.populateLanguageDropdown(); + } + } + + populateLanguageDropdown() { + const languageSelect = document.getElementById('languageSelect'); + if (!languageSelect) return; + + languageSelect.innerHTML = ''; + this.availableLanguages.forEach((lang) => { + const option = document.createElement('option'); + option.value = lang.code; + option.textContent = `${lang.flag} ${lang.name}`; + if (lang.code === this.currentLanguage) { + option.selected = true; + } + languageSelect.appendChild(option); + }); + + languageSelect.disabled = false; + } + async initializeAudioPlayer() { try { await this.audioPlayer.initialize(); @@ -61,8 +113,8 @@ class App { this.state.chatHistory = savedState.chatHistory || []; } - // Load flashcards from storage - this.state.flashcards = this.storage.getFlashcards(); + // Load flashcards from storage (for current language) + this.state.flashcards = this.storage.getFlashcards(this.currentLanguage); // Load existing conversation history const existingConversation = this.storage.getConversationHistory(); @@ -88,6 +140,7 @@ class App { setupEventListeners() { const micButton = document.getElementById('micButton'); const restartButton = document.getElementById('restartButton'); + const languageSelect = document.getElementById('languageSelect'); // Check for iOS const isIOS = @@ -143,11 +196,25 @@ class App { ); } + // Language selector event listener + languageSelect.addEventListener('change', (e) => { + const newLanguage = e.target.value; + if (newLanguage !== this.currentLanguage) { + this.changeLanguage(newLanguage); + } + }); + this.wsClient.on('connection', (status) => { this.state.connectionStatus = status; // Send existing conversation history to backend when connected if (status === 'connected') { + // Send language preference first + this.wsClient.send({ + type: 'set_language', + languageCode: this.currentLanguage, + }); + const existingConversation = this.storage.getConversationHistory(); if (existingConversation.messages.length > 0) { console.log( @@ -341,6 +408,12 @@ class App { this.handleInterrupt(); }); + // Handle language change confirmation from backend + this.wsClient.on('language_changed', (data) => { + console.log(`[Main] Language changed to ${data.languageName}`); + // Optionally show a notification or update UI + }); + this.audioHandler.on('audioChunk', (audioData) => { this.wsClient.sendAudioChunk(audioData); }); @@ -354,6 +427,57 @@ class App { }); } + async changeLanguage(newLanguage) { + console.log(`[Main] Changing language from ${this.currentLanguage} to ${newLanguage}`); + + // Stop any ongoing recording + if (this.state.isRecording) { + this.audioHandler.stopStreaming(); + this.state.isRecording = false; + } + + // Stop audio playback + try { + this.audioPlayer.stop(); + } catch (error) { + console.error('Error stopping audio:', error); + } + + // Update language + this.currentLanguage = newLanguage; + this.storage.saveLanguage(newLanguage); + + // Clear conversation for new language + this.storage.clearConversation(); + this.state.chatHistory = []; + this.state.currentTranscript = ''; + this.state.currentLLMResponse = ''; + this.state.pendingTranscription = null; + this.state.pendingLLMResponse = null; + this.state.streamingLLMResponse = ''; + this.state.lastPendingTranscription = null; + this.state.speechDetected = false; + this.state.llmResponseComplete = false; + this.state.currentResponseId = null; + + // Clear typewriters + this.chatUI.clearAllTypewriters(); + + // Load flashcards for new language + this.state.flashcards = this.storage.getFlashcards(newLanguage); + + // Send language change to backend + if (this.wsClient && this.state.connectionStatus === 'connected') { + this.wsClient.send({ + type: 'set_language', + languageCode: newLanguage, + }); + } + + // Re-render UI + this.render(); + } + async connectWebSocket() { try { await this.wsClient.connect(); @@ -364,6 +488,7 @@ class App { type: 'user_context', timezone: tz, userId: this.userId, + languageCode: this.currentLanguage, }); } catch (e) { // ignore @@ -596,14 +721,16 @@ class App { } addFlashcard(flashcard) { - const exists = this.state.flashcards.some( - (card) => - card.spanish === flashcard.spanish || card.word === flashcard.word - ); + // Use targetWord for deduplication (backwards compatible with spanish) + const targetWord = flashcard.targetWord || flashcard.spanish || flashcard.word; + const exists = this.state.flashcards.some((card) => { + const cardWord = card.targetWord || card.spanish || card.word; + return cardWord === targetWord; + }); if (!exists) { this.state.flashcards.push(flashcard); - this.storage.addFlashcards([flashcard]); + this.storage.addFlashcards([flashcard], this.currentLanguage); this.saveState(); this.render(); } @@ -611,7 +738,7 @@ class App { addMultipleFlashcards(flashcards) { // Use storage method which handles deduplication and persistence - const updatedFlashcards = this.storage.addFlashcards(flashcards); + const updatedFlashcards = this.storage.addFlashcards(flashcards, this.currentLanguage); this.state.flashcards = updatedFlashcards; this.saveState(); this.render(); @@ -755,13 +882,16 @@ class App { this.state.isRecording, this.state.speechDetected ); - this.flashcardUI.render(this.state.flashcards); + this.flashcardUI.render(this.state.flashcards, this.currentLanguage); const micButton = document.getElementById('micButton'); const restartButton = document.getElementById('restartButton'); + const languageSelect = document.getElementById('languageSelect'); + micButton.disabled = this.state.connectionStatus !== 'connected'; micButton.classList.toggle('recording', this.state.isRecording); restartButton.disabled = this.state.connectionStatus !== 'connected'; + languageSelect.disabled = this.state.connectionStatus !== 'connected'; } updateConnectionStatus() { diff --git a/frontend/js/storage.js b/frontend/js/storage.js index 8ad4e7a..de011a8 100644 --- a/frontend/js/storage.js +++ b/frontend/js/storage.js @@ -3,6 +3,25 @@ export class Storage { this.storageKey = 'aprende-app-state'; this.conversationKey = 'aprende-conversation-history'; this.flashcardsKey = 'aprende-flashcards'; + this.languageKey = 'aprende-language'; + } + + // Language preference methods + getLanguage() { + try { + return localStorage.getItem(this.languageKey) || 'es'; + } catch (error) { + console.error('Failed to load language from localStorage:', error); + return 'es'; + } + } + + saveLanguage(languageCode) { + try { + localStorage.setItem(this.languageKey, languageCode); + } catch (error) { + console.error('Failed to save language to localStorage:', error); + } } saveState(state) { @@ -94,11 +113,34 @@ export class Storage { } } - // Flashcard methods - getFlashcards() { + // Flashcard methods - now support per-language storage + _getFlashcardsKey(languageCode) { + if (!languageCode) { + return this.flashcardsKey; + } + return `${this.flashcardsKey}-${languageCode}`; + } + + getFlashcards(languageCode) { try { - const serializedFlashcards = localStorage.getItem(this.flashcardsKey); + const key = this._getFlashcardsKey(languageCode); + const serializedFlashcards = localStorage.getItem(key); if (serializedFlashcards === null) { + // Try to migrate from old format if no language-specific data exists + if (languageCode === 'es') { + const oldFlashcards = localStorage.getItem(this.flashcardsKey); + if (oldFlashcards) { + const parsed = JSON.parse(oldFlashcards); + // Migrate old flashcards to new format + const migrated = parsed.map((card) => ({ + ...card, + targetWord: card.targetWord || card.spanish || '', + languageCode: 'es', + })); + this.saveFlashcards(migrated, 'es'); + return migrated; + } + } return []; } return JSON.parse(serializedFlashcards); @@ -108,42 +150,67 @@ export class Storage { } } - saveFlashcards(flashcards) { + saveFlashcards(flashcards, languageCode) { try { + const key = this._getFlashcardsKey(languageCode); const serializedFlashcards = JSON.stringify(flashcards); - localStorage.setItem(this.flashcardsKey, serializedFlashcards); + localStorage.setItem(key, serializedFlashcards); } catch (error) { console.error('Failed to save flashcards to localStorage:', error); } } - addFlashcards(newFlashcards) { - const existingFlashcards = this.getFlashcards(); + addFlashcards(newFlashcards, languageCode) { + const existingFlashcards = this.getFlashcards(languageCode); - // Filter out duplicates based on spanish word + // Filter out duplicates based on targetWord (backwards compatible with spanish) const uniqueNewFlashcards = newFlashcards.filter((newCard) => { - return !existingFlashcards.some( - (existing) => - existing.spanish?.toLowerCase() === newCard.spanish?.toLowerCase() - ); + const newWord = newCard.targetWord || newCard.spanish || ''; + return !existingFlashcards.some((existing) => { + const existingWord = existing.targetWord || existing.spanish || ''; + return existingWord.toLowerCase() === newWord.toLowerCase(); + }); }); - const updatedFlashcards = [...existingFlashcards, ...uniqueNewFlashcards]; + // Add language code to new flashcards + const flashcardsWithLanguage = uniqueNewFlashcards.map((card) => ({ + ...card, + targetWord: card.targetWord || card.spanish || '', + languageCode: languageCode, + })); - // Keep only the last 100 flashcards + const updatedFlashcards = [...existingFlashcards, ...flashcardsWithLanguage]; + + // Keep only the last 100 flashcards per language if (updatedFlashcards.length > 100) { updatedFlashcards.splice(0, updatedFlashcards.length - 100); } - this.saveFlashcards(updatedFlashcards); + this.saveFlashcards(updatedFlashcards, languageCode); return updatedFlashcards; } - clearFlashcards() { + clearFlashcards(languageCode) { try { - localStorage.removeItem(this.flashcardsKey); + const key = this._getFlashcardsKey(languageCode); + localStorage.removeItem(key); } catch (error) { console.error('Failed to clear flashcards from localStorage:', error); } } + + // Clear all flashcards for all languages + clearAllFlashcards() { + try { + // Clear the base key + localStorage.removeItem(this.flashcardsKey); + // Clear language-specific keys + const languages = ['es', 'ja', 'fr']; + languages.forEach((lang) => { + localStorage.removeItem(this._getFlashcardsKey(lang)); + }); + } catch (error) { + console.error('Failed to clear all flashcards from localStorage:', error); + } + } } diff --git a/frontend/styles/main.css b/frontend/styles/main.css index 4f2e0b3..00c0f6e 100644 --- a/frontend/styles/main.css +++ b/frontend/styles/main.css @@ -43,6 +43,50 @@ body { color: #1a1a1a; } +.header-controls { + display: flex; + align-items: center; + gap: 16px; +} + +.language-selector { + display: flex; + align-items: center; +} + +.language-dropdown { + font-family: 'Roboto Serif', Georgia, Cambria, 'Times New Roman', serif; + font-size: 14px; + padding: 8px 12px; + border: 1px solid #e5e5e5; + border-radius: 8px; + background: #ffffff; + color: #1a1a1a; + cursor: pointer; + transition: all 0.2s ease; + min-width: 140px; + appearance: none; + background-image: url("data:image/svg+xml,%3Csvg xmlns='http://www.w3.org/2000/svg' width='12' height='12' viewBox='0 0 24 24' fill='none' stroke='%236b7280' stroke-width='2' stroke-linecap='round' stroke-linejoin='round'%3E%3Cpath d='M6 9l6 6 6-6'/%3E%3C/svg%3E"); + background-repeat: no-repeat; + background-position: right 12px center; + padding-right: 32px; +} + +.language-dropdown:hover:not(:disabled) { + border-color: #1a1a1a; +} + +.language-dropdown:focus { + outline: none; + border-color: #1a1a1a; + box-shadow: 0 0 0 2px rgba(26, 26, 26, 0.1); +} + +.language-dropdown:disabled { + opacity: 0.5; + cursor: not-allowed; +} + .status-indicator { display: flex; align-items: center; @@ -383,7 +427,8 @@ body { transform: rotateY(180deg); } -.flashcard-spanish { +.flashcard-spanish, +.flashcard-target-word { font-size: 32px; font-weight: 700; text-align: center; @@ -441,7 +486,8 @@ body { scroll-snap-align: start; } - .flashcard-spanish { + .flashcard-spanish, + .flashcard-target-word { font-size: 24px; } diff --git a/package-lock.json b/package-lock.json index 626100b..7bdb6aa 100644 --- a/package-lock.json +++ b/package-lock.json @@ -32,7 +32,7 @@ "typescript-eslint": "^8.0.0" }, "engines": { - "node": ">=18.0.0", + "node": ">=20.0.0", "npm": ">=9.0.0" } }, @@ -1513,7 +1513,6 @@ "integrity": "sha512-tK3GPFWbirvNgsNKto+UmB/cRtn6TZfyw0D6IKrW55n6Vbs7KJoZtI//kpTKzE/DUmmnAFD8/Ca46s7Obs92/w==", "dev": true, "license": "MIT", - "peer": true, "dependencies": { "@typescript-eslint/scope-manager": "8.46.4", "@typescript-eslint/types": "8.46.4", @@ -1863,7 +1862,6 @@ "integrity": "sha512-NZyJarBfL7nWwIq+FDL6Zp/yHEhePMNnnJ0y3qfieCrmNvYct8uvtiV41UvlSe6apAfk0fY1FbWx+NwfmpvtTg==", "dev": true, "license": "MIT", - "peer": true, "bin": { "acorn": "bin/acorn" }, @@ -2893,7 +2891,6 @@ "integrity": "sha512-BhHmn2yNOFA9H9JmmIVKJmd288g9hrVRDkdoIgRCRuSySRUHH7r/DI6aAXW9T1WwUuY3DFgrcaqB+deURBLR5g==", "dev": true, "license": "MIT", - "peer": true, "dependencies": { "@eslint-community/eslint-utils": "^4.8.0", "@eslint-community/regexpp": "^4.12.1", @@ -2954,7 +2951,6 @@ "integrity": "sha512-iI1f+D2ViGn+uvv5HuHVUamg8ll4tN+JRHGc6IJi4TP9Kl976C57fzPXgseXNs8v0iA8aSJpHsTWjDb9QJamGQ==", "dev": true, "license": "MIT", - "peer": true, "bin": { "eslint-config-prettier": "bin/cli.js" }, @@ -4915,7 +4911,6 @@ "integrity": "sha512-I7AIg5boAr5R0FFtJ6rCfD+LFsWHp81dolrFD8S79U9tb8Az2nGrJncnMSnys+bpQJfRUzqs9hnA81OAA3hCuQ==", "dev": true, "license": "MIT", - "peer": true, "bin": { "prettier": "bin/prettier.cjs" }, @@ -5900,7 +5895,6 @@ "integrity": "sha512-CWBzXQrc/qOkhidw1OzBTQuYRbfyxDXJMVJ1XNwUHGROVmuaeiEm3OslpZ1RV96d7SKKjZKrSJu3+t/xlw3R9A==", "dev": true, "license": "Apache-2.0", - "peer": true, "bin": { "tsc": "bin/tsc", "tsserver": "bin/tsserver" From b756438b9f8eef21433c7003a1c39ddc5bcb4890 Mon Sep 17 00:00:00 2001 From: Cale Shapera <25466659+cshape@users.noreply.github.com> Date: Mon, 29 Dec 2025 17:58:17 -0800 Subject: [PATCH 02/49] Feat: port to long-running 0.9 graph --- backend/components/audio/audio_utils.ts | 71 + .../audio/multimodal_stream_manager.ts | 135 + .../components/graphs/conversation-graph.ts | 305 ++ .../graphs/nodes/assembly_ai_stt_ws_node.ts | 724 ++++ .../nodes/dialog_prompt_builder_node.ts | 94 + .../graphs/nodes/interaction_queue_node.ts | 167 + .../graphs/nodes/state_update_node.ts | 68 + .../graphs/nodes/text_input_node.ts | 61 + .../graphs/nodes/transcript_extractor_node.ts | 51 + .../graphs/nodes/tts_request_builder_node.ts | 64 + backend/graphs/conversation-graph.ts | 267 -- backend/graphs/flashcard-graph.ts | 18 +- backend/graphs/introduction-state-graph.ts | 11 +- backend/helpers/audio-processor.ts | 790 ---- backend/helpers/connection-manager.ts | 580 +++ backend/helpers/silero-vad.ts | 287 -- backend/server.ts | 414 +-- backend/types/index.ts | 81 + backend/types/settings.ts | 47 + flashcard-graph.json | 95 - frontend/js/audio-player.js | 62 +- frontend/js/chat-ui.js | 116 + frontend/js/main.js | 13 +- frontend/js/translator.js | 90 + frontend/js/websocket-client.js | 9 + frontend/styles/main.css | 104 + package-lock.json | 1499 +------- package.json | 2 +- realtime-service/.env-sample | 39 + realtime-service/Makefile | 53 + realtime-service/README.md | 91 + realtime-service/__tests__/README.md | 469 +++ realtime-service/__tests__/api/README.md | 312 ++ .../__tests__/api/realtime-api.spec.ts | 885 +++++ .../__tests__/api/websocket-server-helper.ts | 133 + .../__tests__/api/websocket-test-helper.ts | 311 ++ realtime-service/__tests__/config.ts | 8 + realtime-service/__tests__/setup.ts | 9 + realtime-service/__tests__/test-setup.ts | 32 + .../__tests__/utils/graph-test-helpers.ts | 71 + .../__tests__/utils/mock-helpers.ts | 93 + realtime-service/jest.config.js | 34 + realtime-service/load-tests/README.md | 61 + realtime-service/load-tests/load_test.js | 425 +++ realtime-service/load-tests/package.json | 30 + realtime-service/mock/echo_ws_server.go | 50 + realtime-service/package-lock.json | 33 + realtime-service/package.json | 26 + realtime-service/src/.gitignore | 32 + realtime-service/src/REALTIME_API.md | 313 ++ realtime-service/src/components/app.ts | 78 + .../src/components/audio/audio_utils.ts | 91 + .../audio/multimodal_stream_manager.ts | 125 + .../audio/realtime_audio_handler.ts | 213 ++ .../src/components/graphs/graph.ts | 308 ++ .../graphs/nodes/assembly_ai_stt_ws_node.ts | 855 +++++ .../nodes/dialog_prompt_builder_node.ts | 87 + .../graphs/nodes/interaction_queue_node.ts | 178 + .../graphs/nodes/state_update_node.ts | 67 + .../graphs/nodes/text_input_node.ts | 75 + .../graphs/nodes/text_output_stream_node.ts | 29 + .../graphs/nodes/transcript_extractor_node.ts | 63 + .../graphs/nodes/tts_request_builder_node.ts | 64 + .../graphs/realtime_graph_executor.ts | 1180 ++++++ .../realtime/realtime_event_factory.ts | 499 +++ .../realtime/realtime_message_handler.ts | 168 + .../realtime/realtime_session_manager.ts | 508 +++ .../src/components/runtime_app_manager.ts | 140 + realtime-service/src/config.ts | 15 + realtime-service/src/helpers.ts | 65 + realtime-service/src/index.ts | 319 ++ realtime-service/src/log-helpers.ts | 88 + realtime-service/src/logger.ts | 66 + realtime-service/src/package-lock.json | 3273 +++++++++++++++++ realtime-service/src/package.json | 36 + realtime-service/src/tsconfig.json | 23 + realtime-service/src/types/index.ts | 108 + realtime-service/src/types/realtime.ts | 526 +++ realtime-service/src/types/settings.ts | 45 + realtime-service/tsconfig.test.json | 21 + realtime-service/yarn.lock | 8 + 81 files changed, 15924 insertions(+), 3132 deletions(-) create mode 100644 backend/components/audio/audio_utils.ts create mode 100644 backend/components/audio/multimodal_stream_manager.ts create mode 100644 backend/components/graphs/conversation-graph.ts create mode 100644 backend/components/graphs/nodes/assembly_ai_stt_ws_node.ts create mode 100644 backend/components/graphs/nodes/dialog_prompt_builder_node.ts create mode 100644 backend/components/graphs/nodes/interaction_queue_node.ts create mode 100644 backend/components/graphs/nodes/state_update_node.ts create mode 100644 backend/components/graphs/nodes/text_input_node.ts create mode 100644 backend/components/graphs/nodes/transcript_extractor_node.ts create mode 100644 backend/components/graphs/nodes/tts_request_builder_node.ts delete mode 100644 backend/graphs/conversation-graph.ts delete mode 100644 backend/helpers/audio-processor.ts create mode 100644 backend/helpers/connection-manager.ts delete mode 100644 backend/helpers/silero-vad.ts create mode 100644 backend/types/index.ts create mode 100644 backend/types/settings.ts delete mode 100644 flashcard-graph.json create mode 100644 frontend/js/translator.js create mode 100644 realtime-service/.env-sample create mode 100644 realtime-service/Makefile create mode 100644 realtime-service/README.md create mode 100644 realtime-service/__tests__/README.md create mode 100644 realtime-service/__tests__/api/README.md create mode 100644 realtime-service/__tests__/api/realtime-api.spec.ts create mode 100644 realtime-service/__tests__/api/websocket-server-helper.ts create mode 100644 realtime-service/__tests__/api/websocket-test-helper.ts create mode 100644 realtime-service/__tests__/config.ts create mode 100644 realtime-service/__tests__/setup.ts create mode 100644 realtime-service/__tests__/test-setup.ts create mode 100644 realtime-service/__tests__/utils/graph-test-helpers.ts create mode 100644 realtime-service/__tests__/utils/mock-helpers.ts create mode 100644 realtime-service/jest.config.js create mode 100644 realtime-service/load-tests/README.md create mode 100644 realtime-service/load-tests/load_test.js create mode 100644 realtime-service/load-tests/package.json create mode 100644 realtime-service/mock/echo_ws_server.go create mode 100644 realtime-service/package-lock.json create mode 100644 realtime-service/package.json create mode 100644 realtime-service/src/.gitignore create mode 100644 realtime-service/src/REALTIME_API.md create mode 100644 realtime-service/src/components/app.ts create mode 100644 realtime-service/src/components/audio/audio_utils.ts create mode 100644 realtime-service/src/components/audio/multimodal_stream_manager.ts create mode 100644 realtime-service/src/components/audio/realtime_audio_handler.ts create mode 100644 realtime-service/src/components/graphs/graph.ts create mode 100644 realtime-service/src/components/graphs/nodes/assembly_ai_stt_ws_node.ts create mode 100644 realtime-service/src/components/graphs/nodes/dialog_prompt_builder_node.ts create mode 100644 realtime-service/src/components/graphs/nodes/interaction_queue_node.ts create mode 100644 realtime-service/src/components/graphs/nodes/state_update_node.ts create mode 100644 realtime-service/src/components/graphs/nodes/text_input_node.ts create mode 100644 realtime-service/src/components/graphs/nodes/text_output_stream_node.ts create mode 100644 realtime-service/src/components/graphs/nodes/transcript_extractor_node.ts create mode 100644 realtime-service/src/components/graphs/nodes/tts_request_builder_node.ts create mode 100644 realtime-service/src/components/graphs/realtime_graph_executor.ts create mode 100644 realtime-service/src/components/realtime/realtime_event_factory.ts create mode 100644 realtime-service/src/components/realtime/realtime_message_handler.ts create mode 100644 realtime-service/src/components/realtime/realtime_session_manager.ts create mode 100644 realtime-service/src/components/runtime_app_manager.ts create mode 100644 realtime-service/src/config.ts create mode 100644 realtime-service/src/helpers.ts create mode 100644 realtime-service/src/index.ts create mode 100644 realtime-service/src/log-helpers.ts create mode 100644 realtime-service/src/logger.ts create mode 100644 realtime-service/src/package-lock.json create mode 100644 realtime-service/src/package.json create mode 100644 realtime-service/src/tsconfig.json create mode 100644 realtime-service/src/types/index.ts create mode 100644 realtime-service/src/types/realtime.ts create mode 100644 realtime-service/src/types/settings.ts create mode 100644 realtime-service/tsconfig.test.json create mode 100644 realtime-service/yarn.lock diff --git a/backend/components/audio/audio_utils.ts b/backend/components/audio/audio_utils.ts new file mode 100644 index 0000000..f404a38 --- /dev/null +++ b/backend/components/audio/audio_utils.ts @@ -0,0 +1,71 @@ +/** + * Audio utility functions for format conversion + */ + +/** + * Convert Float32Array audio data to Int16Array (PCM16) + */ +export function float32ToPCM16(float32Data: Float32Array): Int16Array { + const pcm16 = new Int16Array(float32Data.length); + for (let i = 0; i < float32Data.length; i++) { + // Clamp to [-1, 1] range and convert to Int16 range [-32768, 32767] + const s = Math.max(-1, Math.min(1, float32Data[i])); + pcm16[i] = s < 0 ? s * 0x8000 : s * 0x7fff; + } + return pcm16; +} + +/** + * Convert Int16Array (PCM16) to Float32Array + */ +export function pcm16ToFloat32(pcm16Data: Int16Array): Float32Array { + const float32 = new Float32Array(pcm16Data.length); + for (let i = 0; i < pcm16Data.length; i++) { + float32[i] = pcm16Data[i] / 32768.0; + } + return float32; +} + +/** + * Convert audio data to PCM16 base64 string for WebSocket transmission + */ +export function convertToPCM16Base64( + audioData: number[] | Float32Array | string | undefined, + _sampleRate: number | undefined, + _logPrefix: string = 'Audio' +): string | null { + if (!audioData) { + return null; + } + + let base64Data: string; + + if (typeof audioData === 'string') { + // Already base64 encoded + base64Data = audioData; + } else { + // Convert Float32 array to PCM16 base64 + const float32Data = Array.isArray(audioData) + ? new Float32Array(audioData) + : audioData; + const pcm16Data = float32ToPCM16(float32Data); + base64Data = Buffer.from(pcm16Data.buffer).toString('base64'); + } + + return base64Data; +} + +/** + * Decode base64 audio to Float32Array + * Note: Node.js Buffer objects share ArrayBuffers with offsets, so we need to copy + */ +export function decodeBase64ToFloat32(base64Audio: string): Float32Array { + const buffer = Buffer.from(base64Audio, 'base64'); + // Create a clean copy to avoid Node.js Buffer's internal ArrayBuffer sharing + const cleanArray = new Uint8Array(buffer.length); + for (let i = 0; i < buffer.length; i++) { + cleanArray[i] = buffer[i]; + } + const int16Array = new Int16Array(cleanArray.buffer); + return pcm16ToFloat32(int16Array); +} diff --git a/backend/components/audio/multimodal_stream_manager.ts b/backend/components/audio/multimodal_stream_manager.ts new file mode 100644 index 0000000..212fe55 --- /dev/null +++ b/backend/components/audio/multimodal_stream_manager.ts @@ -0,0 +1,135 @@ +/** + * Manages a stream of multimodal content (audio and/or text) that can be fed + * asynchronously as data arrives from websocket connections. + * + * This unifies audio and text streaming into a single interface that always + * yields MultimodalContent, making it compatible with the 0.9 graph architecture. + */ + +import { GraphTypes } from '@inworld/runtime/graph'; + +export interface AudioChunkInterface { + data: number[] | Float32Array; + sampleRate: number; +} + +export class MultimodalStreamManager { + private queue: GraphTypes.MultimodalContent[] = []; + private waitingResolvers: Array< + (value: IteratorResult) => void + > = []; + private ended = false; + + /** + * Add an audio chunk to the stream (wrapped in MultimodalContent) + */ + pushAudio(chunk: AudioChunkInterface): void { + if (this.ended) { + return; + } + + // Create GraphTypes.Audio object and wrap in MultimodalContent + const audioData = new GraphTypes.Audio({ + data: Array.isArray(chunk.data) ? chunk.data : Array.from(chunk.data), + sampleRate: chunk.sampleRate, + }); + const multimodalContent = new GraphTypes.MultimodalContent({ + audio: audioData, + }); + + this.pushContent(multimodalContent); + } + + /** + * Add text to the stream (wrapped in MultimodalContent) + */ + pushText(text: string): void { + if (this.ended) { + return; + } + + const multimodalContent = new GraphTypes.MultimodalContent({ text }); + this.pushContent(multimodalContent); + } + + /** + * Internal method to push MultimodalContent to the stream + */ + private pushContent(content: GraphTypes.MultimodalContent): void { + // If there are waiting consumers, resolve immediately + if (this.waitingResolvers.length > 0) { + const resolve = this.waitingResolvers.shift()!; + resolve({ value: content, done: false }); + } else { + // Otherwise, queue the content + this.queue.push(content); + } + } + + /** + * Mark the stream as ended + */ + end(): void { + console.log('[MultimodalStreamManager] Ending stream'); + this.ended = true; + + // Resolve all waiting consumers with done signal + while (this.waitingResolvers.length > 0) { + const resolve = this.waitingResolvers.shift()!; + resolve({ value: undefined as unknown as GraphTypes.MultimodalContent, done: true }); + } + } + + /** + * Create an async iterator for the stream + */ + async *createStream(): AsyncIterableIterator { + while (true) { + // If stream ended and queue is empty, we're done + if (this.ended && this.queue.length === 0) { + console.log('[MultimodalStreamManager] Stream iteration complete'); + return; + } + + // If we have queued content, yield it immediately + if (this.queue.length > 0) { + const content = this.queue.shift()!; + yield content; + continue; + } + + // If stream ended but we just exhausted the queue, we're done + if (this.ended) { + console.log('[MultimodalStreamManager] Stream iteration complete'); + return; + } + + // Otherwise, wait for new content + const result = await new Promise< + IteratorResult + >((resolve) => { + this.waitingResolvers.push(resolve); + }); + + if (result.done) { + return; + } + + yield result.value; + } + } + + /** + * Check if the stream has ended + */ + isEnded(): boolean { + return this.ended; + } + + /** + * Get the current queue length (for debugging) + */ + getQueueLength(): number { + return this.queue.length; + } +} diff --git a/backend/components/graphs/conversation-graph.ts b/backend/components/graphs/conversation-graph.ts new file mode 100644 index 0000000..b873b7d --- /dev/null +++ b/backend/components/graphs/conversation-graph.ts @@ -0,0 +1,305 @@ +/** + * Conversation Graph for Language Learning App - Inworld Runtime 0.9 + * + * This is a long-running circular graph that: + * - Processes continuous audio streams via AssemblyAI STT with built-in VAD + * - Queues interactions for sequential processing + * - Uses language-specific prompts and TTS voices + * - Loops back for the next interaction automatically + * + * Graph Flow: + * AudioInput → AssemblyAI STT (loop) → TranscriptExtractor → InteractionQueue + * → TextInput → DialogPromptBuilder → LLM → TextChunking → TTSRequestBuilder → TTS + * → TextAggregator → StateUpdate → (loop back to InteractionQueue) + */ + +import { + Graph, + GraphBuilder, + ProxyNode, + RemoteLLMChatNode, + RemoteTTSNode, + TextChunkingNode, + TextAggregatorNode, +} from '@inworld/runtime/graph'; + +import { AssemblyAISTTWebSocketNode } from './nodes/assembly_ai_stt_ws_node.js'; +import { DialogPromptBuilderNode } from './nodes/dialog_prompt_builder_node.js'; +import { InteractionQueueNode } from './nodes/interaction_queue_node.js'; +import { StateUpdateNode } from './nodes/state_update_node.js'; +import { TextInputNode } from './nodes/text_input_node.js'; +import { TranscriptExtractorNode } from './nodes/transcript_extractor_node.js'; +import { TTSRequestBuilderNode } from './nodes/tts_request_builder_node.js'; +import { + ConnectionsMap, + TextInput, + INPUT_SAMPLE_RATE, + TTS_SAMPLE_RATE, +} from '../../types/index.js'; +import { + getLanguageConfig, + DEFAULT_LANGUAGE_CODE, +} from '../../config/languages.js'; +import { getAssemblyAISettingsForEagerness } from '../../types/settings.js'; + +export interface ConversationGraphConfig { + assemblyAIApiKey: string; + connections: ConnectionsMap; + defaultLanguageCode?: string; +} + +/** + * Wrapper class for the conversation graph + * Provides access to the graph and the AssemblyAI node for session management + */ +export class ConversationGraphWrapper { + graph: Graph; + assemblyAINode: AssemblyAISTTWebSocketNode; + + private constructor(params: { + graph: Graph; + assemblyAINode: AssemblyAISTTWebSocketNode; + }) { + this.graph = params.graph; + this.assemblyAINode = params.assemblyAINode; + } + + async destroy(): Promise { + await this.assemblyAINode.destroy(); + await this.graph.stop(); + } + + /** + * Create the conversation graph + */ + static create(config: ConversationGraphConfig): ConversationGraphWrapper { + const { + connections, + assemblyAIApiKey, + defaultLanguageCode = DEFAULT_LANGUAGE_CODE, + } = config; + const langConfig = getLanguageConfig(defaultLanguageCode); + const postfix = `-lang-learning`; + + console.log( + `[ConversationGraph] Creating graph for ${langConfig.name} (${defaultLanguageCode})` + ); + + // ============================================================ + // Create Nodes + // ============================================================ + + // Start node (audio input proxy) + const audioInputNode = new ProxyNode({ id: `audio-input-proxy${postfix}` }); + + // AssemblyAI STT with built-in VAD + const turnDetectionSettings = getAssemblyAISettingsForEagerness('medium'); + const assemblyAISTTNode = new AssemblyAISTTWebSocketNode({ + id: `assembly-ai-stt-ws-node${postfix}`, + config: { + apiKey: assemblyAIApiKey, + connections: connections, + sampleRate: INPUT_SAMPLE_RATE, + formatTurns: true, + endOfTurnConfidenceThreshold: + turnDetectionSettings.endOfTurnConfidenceThreshold, + minEndOfTurnSilenceWhenConfident: + turnDetectionSettings.minEndOfTurnSilenceWhenConfident, + maxTurnSilence: turnDetectionSettings.maxTurnSilence, + language: defaultLanguageCode.split('-')[0], // 'es' from 'es-MX' + }, + }); + + const transcriptExtractorNode = new TranscriptExtractorNode({ + id: `transcript-extractor-node${postfix}`, + reportToClient: true, + }); + + const interactionQueueNode = new InteractionQueueNode({ + id: `interaction-queue-node${postfix}`, + connections, + reportToClient: false, + }); + + const textInputNode = new TextInputNode({ + id: `text-input-node${postfix}`, + connections, + reportToClient: true, + }); + + const dialogPromptBuilderNode = new DialogPromptBuilderNode({ + id: `dialog-prompt-builder-node${postfix}`, + connections, + }); + + // LLM Node - uses Inworld Runtime's remote LLM + const llmNode = new RemoteLLMChatNode({ + id: `llm-node${postfix}`, + provider: 'openai', + modelName: 'gpt-4o-mini', + stream: true, + textGenerationConfig: { + maxNewTokens: 250, + maxPromptLength: 2000, + temperature: 1, + topP: 1, + repetitionPenalty: 1, + frequencyPenalty: 0, + presencePenalty: 0, + }, + reportToClient: true, + }); + + const textChunkingNode = new TextChunkingNode({ + id: `text-chunking-node${postfix}`, + }); + + const textAggregatorNode = new TextAggregatorNode({ + id: `text-aggregator-node${postfix}`, + }); + + const stateUpdateNode = new StateUpdateNode({ + id: `state-update-node${postfix}`, + connections, + reportToClient: true, + }); + + const ttsRequestBuilderNode = new TTSRequestBuilderNode({ + id: `tts-request-builder-node${postfix}`, + connections, + defaultVoiceId: langConfig.ttsConfig.speakerId, + reportToClient: false, + }); + + const ttsNode = new RemoteTTSNode({ + id: `tts-node${postfix}`, + speakerId: langConfig.ttsConfig.speakerId, + modelId: langConfig.ttsConfig.modelId, + sampleRate: TTS_SAMPLE_RATE, + temperature: langConfig.ttsConfig.temperature, + speakingRate: langConfig.ttsConfig.speakingRate, + languageCode: langConfig.ttsConfig.languageCode, + reportToClient: true, + }); + + // ============================================================ + // Build the Graph + // ============================================================ + + const graphBuilder = new GraphBuilder({ + id: `lang-learning-conversation-graph`, + enableRemoteConfig: false, + }); + + graphBuilder + // Add all nodes + .addNode(audioInputNode) + .addNode(assemblyAISTTNode) + .addNode(transcriptExtractorNode) + .addNode(interactionQueueNode) + .addNode(textInputNode) + .addNode(dialogPromptBuilderNode) + .addNode(llmNode) + .addNode(textChunkingNode) + .addNode(textAggregatorNode) + .addNode(ttsRequestBuilderNode) + .addNode(ttsNode) + .addNode(stateUpdateNode) + + // ============================================================ + // Audio Input Flow (STT with VAD) + // ============================================================ + .addEdge(audioInputNode, assemblyAISTTNode) + + // AssemblyAI loops back to itself while stream is active + .addEdge(assemblyAISTTNode, assemblyAISTTNode, { + condition: async (input: unknown) => { + const data = input as { stream_exhausted?: boolean }; + return data?.stream_exhausted !== true; + }, + loop: true, + optional: true, + }) + + // When interaction is complete, extract transcript + .addEdge(assemblyAISTTNode, transcriptExtractorNode, { + condition: async (input: unknown) => { + const data = input as { interaction_complete?: boolean }; + return data?.interaction_complete === true; + }, + }) + + // Transcript goes to interaction queue + .addEdge(transcriptExtractorNode, interactionQueueNode) + + // ============================================================ + // Processing Flow + // ============================================================ + + // InteractionQueue → TextInput only when there's text to process + .addEdge(interactionQueueNode, textInputNode, { + condition: (input: TextInput) => { + return Boolean(input.text && input.text.trim().length > 0); + }, + }) + + // TextInput updates state and passes to prompt builder + .addEdge(textInputNode, dialogPromptBuilderNode) + + // Also pass state to TTS builder for voice selection + .addEdge(textInputNode, ttsRequestBuilderNode) + + // Prompt builder → LLM + .addEdge(dialogPromptBuilderNode, llmNode) + + // LLM output splits: one for TTS, one for state update + .addEdge(llmNode, textChunkingNode) + .addEdge(llmNode, textAggregatorNode) + + // Text chunking → TTS request builder → TTS + .addEdge(textChunkingNode, ttsRequestBuilderNode) + .addEdge(ttsRequestBuilderNode, ttsNode) + + // Text aggregator → state update + .addEdge(textAggregatorNode, stateUpdateNode) + + // ============================================================ + // Loop Back + // ============================================================ + + // State update loops back to interaction queue for next turn + .addEdge(stateUpdateNode, interactionQueueNode, { + loop: true, + optional: true, + }) + + // ============================================================ + // Start and End Nodes + // ============================================================ + .setStartNode(audioInputNode) + .setEndNode(ttsNode); + + const graph = graphBuilder.build(); + + console.log('[ConversationGraph] Graph built successfully'); + + return new ConversationGraphWrapper({ + graph, + assemblyAINode: assemblyAISTTNode, + }); + } +} + +/** + * Legacy export for backwards compatibility during migration + */ +export function getConversationGraph( + _config: { apiKey: string }, + _languageCode: string = DEFAULT_LANGUAGE_CODE +): Graph { + console.warn( + '[ConversationGraph] getConversationGraph is deprecated. Use ConversationGraphWrapper.create() instead.' + ); + // This won't work properly without connections, but maintains API compatibility + return null as unknown as Graph; +} diff --git a/backend/components/graphs/nodes/assembly_ai_stt_ws_node.ts b/backend/components/graphs/nodes/assembly_ai_stt_ws_node.ts new file mode 100644 index 0000000..ec4c7c5 --- /dev/null +++ b/backend/components/graphs/nodes/assembly_ai_stt_ws_node.ts @@ -0,0 +1,724 @@ +/** + * AssemblyAI Speech-to-Text WebSocket Node + * + * Processes continuous multimodal streams (audio and/or text) using Assembly.AI's + * streaming Speech-to-Text service via direct WebSocket connection. + * + * Features: + * - Persistent WebSocket connection per session + * - Automatic turn detection (VAD built-in) + * - Multi-language support + * - Partial transcript updates + */ + +import { DataStreamWithMetadata } from '@inworld/runtime'; +import { CustomNode, GraphTypes, ProcessContext } from '@inworld/runtime/graph'; +import WebSocket from 'ws'; +import { v4 as uuidv4 } from 'uuid'; + +import { Connection } from '../../../types/index.js'; +// Settings imported but used via constructor config +// import { getAssemblyAISettingsForEagerness } from '../../../types/settings.js'; +import { float32ToPCM16 } from '../../audio/audio_utils.js'; + +/** + * Configuration interface for AssemblyAISTTWebSocketNode + */ +export interface AssemblyAISTTWebSocketNodeConfig { + /** Assembly.AI API key */ + apiKey: string; + /** Connections map to access session state */ + connections: { [sessionId: string]: Connection }; + /** Sample rate of the audio stream in Hz */ + sampleRate?: number; + /** Enable turn formatting from Assembly.AI */ + formatTurns?: boolean; + /** End of turn confidence threshold (0-1) */ + endOfTurnConfidenceThreshold?: number; + /** Minimum silence duration when confident (in milliseconds) */ + minEndOfTurnSilenceWhenConfident?: number; + /** Maximum turn silence (in milliseconds) */ + maxTurnSilence?: number; + /** Default language code (e.g., 'en', 'es') */ + language?: string; +} + +/** + * Manages a persistent WebSocket connection to Assembly.AI for a single session. + */ +class AssemblyAISession { + private ws: WebSocket | null = null; + private wsReady: boolean = false; + private wsConnectionPromise: Promise | null = null; + + public assemblySessionId: string = ''; + public sessionExpiresAt: number = 0; + public shouldStopProcessing: boolean = false; + + private inactivityTimeout: NodeJS.Timeout | null = null; + private lastActivityTime: number = Date.now(); + private readonly INACTIVITY_TIMEOUT_MS = 60000; // 60 seconds + + // Audio buffering for AssemblyAI's chunk size requirements (50-1000ms) + private audioBuffer: Int16Array = new Int16Array(0); + // Buffer to ~100ms worth of audio before sending (1600 samples at 16kHz) + private readonly MIN_SAMPLES_TO_SEND = 1600; + + constructor( + public readonly sessionId: string, + private apiKey: string, + private url: string, + private onCleanup: (sessionId: string) => void + ) {} + + /** + * Ensure WebSocket connection is ready, reconnecting if needed + */ + public async ensureConnection(): Promise { + const now = Math.floor(Date.now() / 1000); + const isExpired = + this.sessionExpiresAt > 0 && now >= this.sessionExpiresAt; + + if ( + !this.ws || + !this.wsReady || + this.ws.readyState !== WebSocket.OPEN || + isExpired + ) { + if (isExpired) { + console.log( + `[AssemblyAI] Session ${this.sessionId} expired, reconnecting` + ); + } + this.closeWebSocket(); + this.initializeWebSocket(); + } + + if (this.wsConnectionPromise) { + await this.wsConnectionPromise; + } + + this.shouldStopProcessing = false; + this.resetInactivityTimer(); + } + + private initializeWebSocket(): void { + console.log(`[AssemblyAI] Initializing WebSocket for session ${this.sessionId}`); + + this.wsConnectionPromise = new Promise((resolve, reject) => { + this.ws = new WebSocket(this.url, { + headers: { Authorization: this.apiKey }, + }); + + this.ws.on('open', () => { + console.log(`[AssemblyAI] WebSocket opened for session ${this.sessionId}`); + this.wsReady = true; + resolve(); + }); + + this.ws.on('message', (data: WebSocket.Data) => { + try { + const message = JSON.parse(data.toString()); + if (message.type === 'Begin') { + this.assemblySessionId = message.id || message.session_id || ''; + this.sessionExpiresAt = message.expires_at || 0; + console.log( + `[AssemblyAI] Session began: ${this.assemblySessionId}` + ); + } + } catch { + // Ignore parsing errors + } + }); + + this.ws.on('error', (error: Error) => { + console.error(`[AssemblyAI] WebSocket error:`, error); + this.wsReady = false; + reject(error); + }); + + this.ws.on('close', (code: number, reason: Buffer) => { + console.log( + `[AssemblyAI] WebSocket closed [code:${code}] [reason:${reason.toString()}]` + ); + this.wsReady = false; + }); + }); + } + + public onMessage(listener: (data: WebSocket.Data) => void): void { + if (this.ws) { + this.ws.on('message', listener); + } + } + + public offMessage(listener: (data: WebSocket.Data) => void): void { + if (this.ws) { + this.ws.off('message', listener); + } + } + + public sendAudio(pcm16Data: Int16Array): void { + if (!this.ws || this.ws.readyState !== WebSocket.OPEN) { + return; + } + + // Append new audio to buffer + const newBuffer = new Int16Array(this.audioBuffer.length + pcm16Data.length); + newBuffer.set(this.audioBuffer, 0); + newBuffer.set(pcm16Data, this.audioBuffer.length); + this.audioBuffer = newBuffer; + + // Send if we have enough samples (100ms worth at 16kHz = 1600 samples) + if (this.audioBuffer.length >= this.MIN_SAMPLES_TO_SEND) { + this.ws.send(Buffer.from(this.audioBuffer.buffer)); + this.audioBuffer = new Int16Array(0); + this.resetInactivityTimer(); + } + } + + /** + * Flush any remaining buffered audio (call before closing) + */ + public flushAudio(): void { + if (this.ws && this.ws.readyState === WebSocket.OPEN && this.audioBuffer.length > 0) { + // Pad to minimum size if needed + if (this.audioBuffer.length < 800) { + // Pad to at least 50ms (800 samples) + const paddedBuffer = new Int16Array(800); + paddedBuffer.set(this.audioBuffer, 0); + this.ws.send(Buffer.from(paddedBuffer.buffer)); + } else { + this.ws.send(Buffer.from(this.audioBuffer.buffer)); + } + this.audioBuffer = new Int16Array(0); + } + } + + private resetInactivityTimer(): void { + if (this.inactivityTimeout) { + clearTimeout(this.inactivityTimeout); + } + this.lastActivityTime = Date.now(); + this.inactivityTimeout = setTimeout(() => { + this.closeDueToInactivity(); + }, this.INACTIVITY_TIMEOUT_MS); + } + + private closeDueToInactivity(): void { + const inactiveFor = Date.now() - this.lastActivityTime; + console.log( + `[AssemblyAI] Closing session ${this.sessionId} due to inactivity (${inactiveFor}ms)` + ); + this.shouldStopProcessing = true; + this.close(); + this.onCleanup(this.sessionId); + } + + private closeWebSocket(): void { + if (this.ws) { + try { + this.ws.removeAllListeners(); + if (this.ws.readyState === WebSocket.OPEN) { + this.ws.close(); + } + } catch (e) { + console.warn('[AssemblyAI] Error closing socket:', e); + } + this.ws = null; + this.wsReady = false; + } + } + + public async close(): Promise { + if (this.inactivityTimeout) { + clearTimeout(this.inactivityTimeout); + } + + if (this.ws && this.ws.readyState === WebSocket.OPEN) { + try { + this.ws.send(JSON.stringify({ type: 'Terminate' })); + await new Promise((resolve) => setTimeout(resolve, 100)); + } catch { + // Ignore + } + } + + this.closeWebSocket(); + } +} + +/** + * AssemblyAI STT WebSocket Node for the Inworld runtime graph + */ +export class AssemblyAISTTWebSocketNode extends CustomNode { + private apiKey: string; + private connections: { [sessionId: string]: Connection }; + private sampleRate: number; + private formatTurns: boolean; + private endOfTurnConfidenceThreshold: number; + private minEndOfTurnSilenceWhenConfident: number; + private maxTurnSilence: number; + private defaultLanguage: string; + private wsEndpointBaseUrl: string = 'wss://streaming.assemblyai.com/v3/ws'; + + private sessions: Map = new Map(); + private readonly TURN_COMPLETION_TIMEOUT_MS = 2000; + private readonly MAX_TRANSCRIPTION_DURATION_MS = 40000; + + constructor(props: { id?: string; config: AssemblyAISTTWebSocketNodeConfig }) { + const { config, ...nodeProps } = props; + + if (!config.apiKey) { + throw new Error('AssemblyAISTTWebSocketNode requires an API key.'); + } + if (!config.connections) { + throw new Error('AssemblyAISTTWebSocketNode requires a connections object.'); + } + + super({ + id: nodeProps.id || 'assembly-ai-stt-ws-node', + executionConfig: { + sampleRate: config.sampleRate || 16000, + formatTurns: config.formatTurns !== false, + endOfTurnConfidenceThreshold: config.endOfTurnConfidenceThreshold || 0.4, + minEndOfTurnSilenceWhenConfident: + config.minEndOfTurnSilenceWhenConfident || 400, + maxTurnSilence: config.maxTurnSilence || 1280, + language: config.language || 'es', + }, + }); + + this.apiKey = config.apiKey; + this.connections = config.connections; + this.sampleRate = config.sampleRate || 16000; + this.formatTurns = config.formatTurns !== false; + this.endOfTurnConfidenceThreshold = + config.endOfTurnConfidenceThreshold || 0.4; + this.minEndOfTurnSilenceWhenConfident = + config.minEndOfTurnSilenceWhenConfident || 400; + this.maxTurnSilence = config.maxTurnSilence || 1280; + this.defaultLanguage = config.language || 'es'; + + console.log( + `[AssemblyAI] Configured with [threshold:${this.endOfTurnConfidenceThreshold}] [silence:${this.minEndOfTurnSilenceWhenConfident}ms] [lang:${this.defaultLanguage}]` + ); + } + + /** + * Build WebSocket URL with query parameters + * Dynamically uses connection.state.languageCode if available + */ + private buildWebSocketUrl(sessionId?: string): string { + let endOfTurnThreshold = this.endOfTurnConfidenceThreshold; + let minSilenceWhenConfident = this.minEndOfTurnSilenceWhenConfident; + let maxSilence = this.maxTurnSilence; + let language = this.defaultLanguage; + + if (sessionId) { + const connection = this.connections[sessionId]; + // Get language from connection state (supports dynamic language switching) + if (connection?.state?.languageCode) { + // Extract base language code (e.g., 'es' from 'es-MX') + language = connection.state.languageCode.split('-')[0]; + } + } + + const params = new URLSearchParams({ + sample_rate: this.sampleRate.toString(), + format_turns: this.formatTurns.toString(), + end_of_turn_confidence_threshold: endOfTurnThreshold.toString(), + min_end_of_turn_silence_when_confident: minSilenceWhenConfident.toString(), + max_turn_silence: maxSilence.toString(), + language: language, + }); + + const url = `${this.wsEndpointBaseUrl}?${params.toString()}`; + console.log( + `[AssemblyAI] Connecting with [lang:${language}] [threshold:${endOfTurnThreshold}]` + ); + + return url; + } + + /** + * Process multimodal stream and transcribe using Assembly.AI WebSocket + */ + async process( + context: ProcessContext, + input0: AsyncIterableIterator, + input: DataStreamWithMetadata + ): Promise { + const multimodalStream = + input !== undefined && + input !== null && + input instanceof DataStreamWithMetadata + ? (input.toStream() as unknown as AsyncIterableIterator) + : input0; + + const sessionId = context.getDatastore().get('sessionId') as string; + const connection = this.connections[sessionId]; + + if (connection?.unloaded) { + throw Error(`Session unloaded for sessionId: ${sessionId}`); + } + if (!connection) { + throw Error(`Failed to read connection for sessionId: ${sessionId}`); + } + + // Get or parse iteration from metadata + const metadata = input?.getMetadata?.() || {}; + let previousIteration = (metadata.iteration as number) || 0; + + if ( + !connection.state.interactionId || + connection.state.interactionId === '' + ) { + connection.state.interactionId = uuidv4(); + } + + const currentId = connection.state.interactionId; + const delimiterIndex = currentId.indexOf('#'); + + if (previousIteration === 0 && delimiterIndex !== -1) { + const iterationStr = currentId.substring(delimiterIndex + 1); + const parsedIteration = parseInt(iterationStr, 10); + if (!isNaN(parsedIteration) && /^\d+$/.test(iterationStr)) { + previousIteration = parsedIteration; + } + } + + const iteration = previousIteration + 1; + const baseId = + delimiterIndex !== -1 + ? currentId.substring(0, delimiterIndex) + : currentId; + const nextInteractionId = `${baseId}#${iteration}`; + + console.log(`[AssemblyAI] Starting transcription [iteration:${iteration}]`); + + // State tracking + let transcriptText = ''; + let turnDetected = false; + let speechDetected = false; + let audioChunkCount = 0; + let totalAudioSamples = 0; + let isStreamExhausted = false; + let errorOccurred = false; + let errorMessage = ''; + let maxDurationReached = false; + let isTextInput = false; + let textContent: string | undefined; + + // Get or create session + let session = this.sessions.get(sessionId); + if (!session) { + session = new AssemblyAISession( + sessionId, + this.apiKey, + this.buildWebSocketUrl(sessionId), + (id) => this.sessions.delete(id) + ); + this.sessions.set(sessionId, session); + } + + // Promise to capture turn result + let turnResolve: (value: string) => void = () => {}; + let turnReject: (error: Error) => void = () => {}; + let turnCompleted = false; + const turnPromise = new Promise((resolve, reject) => { + turnResolve = resolve; + turnReject = reject; + }); + const turnPromiseWithState = turnPromise.then((value) => { + turnCompleted = true; + return value; + }); + + // AssemblyAI message handler + const messageHandler = (data: WebSocket.Data) => { + try { + const message = JSON.parse(data.toString()); + const msgType = message.type; + + if (msgType === 'Turn') { + if (session?.shouldStopProcessing) { + return; + } + + const transcript = message.transcript || ''; + const utterance = message.utterance || ''; + const isFinal = message.end_of_turn; + + if (!transcript) return; + + if (!isFinal) { + // Partial transcript + const textToSend = utterance || transcript; + if (textToSend) { + this.sendPartialTranscript(sessionId, nextInteractionId, textToSend); + + if (connection?.onSpeechDetected && !speechDetected) { + console.log(`[AssemblyAI] Speech detected [iteration:${iteration}]`); + speechDetected = true; + connection.onSpeechDetected(nextInteractionId); + } + } + return; + } + + // Final transcript + console.log( + `[AssemblyAI] Turn detected [iteration:${iteration}]: "${transcript.substring(0, 50)}..."` + ); + + transcriptText = transcript; + turnDetected = true; + if (session) session.shouldStopProcessing = true; + turnResolve(transcript); + } else if (msgType === 'Termination') { + console.log(`[AssemblyAI] Session terminated [iteration:${iteration}]`); + } + } catch (error) { + console.error(`[AssemblyAI] Error handling message:`, error); + } + }; + + try { + await session.ensureConnection(); + session.onMessage(messageHandler); + + // Process multimodal content (audio chunks) + const audioProcessingPromise = (async () => { + let maxDurationTimeout: NodeJS.Timeout | null = null; + try { + maxDurationTimeout = setTimeout(() => { + maxDurationReached = true; + }, this.MAX_TRANSCRIPTION_DURATION_MS); + + while (true) { + if (session?.shouldStopProcessing) break; + if (maxDurationReached && !transcriptText) { + console.warn( + `[AssemblyAI] Max transcription duration reached [${this.MAX_TRANSCRIPTION_DURATION_MS}ms]` + ); + break; + } + + const result = await multimodalStream.next(); + + if (result.done) { + console.log( + `[AssemblyAI] Multimodal stream exhausted [iteration:${iteration}] [chunks:${audioChunkCount}]` + ); + isStreamExhausted = true; + break; + } + + if (session?.shouldStopProcessing) break; + + const content = result.value as GraphTypes.MultimodalContent; + + // Handle text input + if (content.text !== undefined && content.text !== null) { + console.log( + `[AssemblyAI] Text input detected [iteration:${iteration}]: "${content.text.substring(0, 50)}..."` + ); + isTextInput = true; + textContent = content.text; + transcriptText = content.text; + turnDetected = true; + if (session) session.shouldStopProcessing = true; + turnResolve(transcriptText); + break; + } + + // Extract audio from MultimodalContent + if (content.audio === undefined || content.audio === null) { + continue; + } + + const audioData = content.audio.data; + if (!audioData || audioData.length === 0) { + continue; + } + + const float32Data = Array.isArray(audioData) + ? new Float32Array(audioData) + : audioData; + + audioChunkCount++; + totalAudioSamples += float32Data.length; + + const pcm16Data = float32ToPCM16(float32Data); + session?.sendAudio(pcm16Data); + } + } catch (error) { + console.error(`[AssemblyAI] Error processing audio:`, error); + errorOccurred = true; + errorMessage = error instanceof Error ? error.message : String(error); + throw error; + } finally { + if (maxDurationTimeout) { + clearTimeout(maxDurationTimeout); + } + // Flush any remaining buffered audio + session?.flushAudio(); + } + })(); + + const raceResult = await Promise.race([ + turnPromiseWithState.then(() => ({ winner: 'turn' as const })), + audioProcessingPromise.then(() => ({ winner: 'audio' as const })), + ]); + + if ( + raceResult.winner === 'audio' && + !turnCompleted && + !maxDurationReached + ) { + console.log( + `[AssemblyAI] Audio ended before turn, waiting ${this.TURN_COMPLETION_TIMEOUT_MS}ms` + ); + + // Send silence to keep connection alive + const silenceIntervalMs = 100; + const silenceSamples = Math.floor( + (silenceIntervalMs / 1000) * this.sampleRate + ); + const silenceFrame = new Int16Array(silenceSamples); + const silenceTimer = setInterval(() => { + if (session && !session.shouldStopProcessing) { + session.sendAudio(silenceFrame); + } + }, silenceIntervalMs); + + const timeoutPromise = new Promise<{ winner: 'timeout' }>((resolve) => + setTimeout( + () => resolve({ winner: 'timeout' }), + this.TURN_COMPLETION_TIMEOUT_MS + ) + ); + + const waitResult = await Promise.race([ + turnPromiseWithState.then(() => ({ winner: 'turn' as const })), + timeoutPromise, + ]); + + clearInterval(silenceTimer); + + if (waitResult.winner === 'timeout' && !turnCompleted) { + console.warn(`[AssemblyAI] Timed out waiting for turn`); + turnReject?.(new Error('Timed out waiting for turn completion')); + } + } + + await audioProcessingPromise.catch(() => {}); + + console.log( + `[AssemblyAI] Transcription complete [iteration:${iteration}]: "${transcriptText?.substring(0, 50)}..."` + ); + + if (turnDetected) { + connection.state.interactionId = ''; + } + + // Tag the stream for runtime + const taggedStream = Object.assign(multimodalStream, { + type: 'MultimodalContent', + abort: () => {}, + getMetadata: () => ({}), + }); + + return new DataStreamWithMetadata(taggedStream, { + elementType: 'MultimodalContent', + iteration: iteration, + interactionId: nextInteractionId, + session_id: sessionId, + assembly_session_id: session.assemblySessionId, + transcript: transcriptText, + turn_detected: turnDetected, + audio_chunk_count: audioChunkCount, + total_audio_samples: totalAudioSamples, + sample_rate: this.sampleRate, + stream_exhausted: isStreamExhausted, + interaction_complete: turnDetected && transcriptText.length > 0, + error_occurred: errorOccurred, + error_message: errorMessage, + is_text_input: isTextInput, + text_content: textContent, + }); + } catch (error) { + console.error(`[AssemblyAI] Transcription failed [iteration:${iteration}]:`, error); + + const taggedStream = Object.assign(multimodalStream, { + type: 'MultimodalContent', + abort: () => {}, + getMetadata: () => ({}), + }); + + return new DataStreamWithMetadata(taggedStream, { + elementType: 'MultimodalContent', + iteration: iteration, + interactionId: nextInteractionId, + session_id: sessionId, + transcript: '', + turn_detected: false, + stream_exhausted: isStreamExhausted, + interaction_complete: false, + error_occurred: true, + error_message: error instanceof Error ? error.message : String(error), + is_text_input: isTextInput, + text_content: textContent, + }); + } finally { + if (session) { + session.offMessage(messageHandler); + } + } + } + + private sendPartialTranscript( + sessionId: string, + interactionId: string, + text: string + ): void { + const connection = this.connections[sessionId]; + if (!connection?.onPartialTranscript) { + return; + } + + try { + connection.onPartialTranscript(text, interactionId); + } catch (error) { + console.error('[AssemblyAI] Error sending partial transcript:', error); + } + } + + /** + * Close a specific session by sessionId + */ + async closeSession(sessionId: string): Promise { + const session = this.sessions.get(sessionId); + if (session) { + console.log(`[AssemblyAI] Closing session: ${sessionId}`); + await session.close(); + this.sessions.delete(sessionId); + } + } + + /** + * Clean up all resources + */ + async destroy(): Promise { + console.log(`[AssemblyAI] Destroying node: closing ${this.sessions.size} sessions`); + + const promises: Promise[] = []; + for (const session of this.sessions.values()) { + promises.push(session.close()); + } + + await Promise.all(promises); + this.sessions.clear(); + } +} diff --git a/backend/components/graphs/nodes/dialog_prompt_builder_node.ts b/backend/components/graphs/nodes/dialog_prompt_builder_node.ts new file mode 100644 index 0000000..73d05f7 --- /dev/null +++ b/backend/components/graphs/nodes/dialog_prompt_builder_node.ts @@ -0,0 +1,94 @@ +/** + * DialogPromptBuilderNode builds a LLM chat request from the state. + * + * This node is specifically designed for the language learning app and: + * - Receives the current conversation state + * - Applies the language-specific prompt template + * - Uses Jinja to render the prompt with conversation history and introduction state + * - Returns a formatted LLMChatRequest for the LLM node + */ + +import { CustomNode, GraphTypes, ProcessContext } from '@inworld/runtime/graph'; +import { PromptBuilder } from '@inworld/runtime/primitives/llm'; +import { ConnectionsMap, State } from '../../../types/index.js'; +import { getLanguageConfig } from '../../../config/languages.js'; +import { conversationTemplate } from '../../../helpers/prompt-templates.js'; + +export class DialogPromptBuilderNode extends CustomNode { + constructor(props: { + id: string; + connections: ConnectionsMap; + reportToClient?: boolean; + }) { + super({ + id: props.id, + reportToClient: props.reportToClient, + }); + // connections passed for future use (e.g., accessing global state) + } + + async process( + _context: ProcessContext, + state: State + ): Promise { + console.log( + `[DialogPromptBuilder] Building prompt for ${state.languageCode || 'es'}, ${state.messages?.length || 0} messages` + ); + + // Get language config from state + const langConfig = getLanguageConfig(state.languageCode || 'es'); + + // Build template variables from language config + const templateVars = { + target_language: langConfig.name, + target_language_native: langConfig.nativeName, + teacher_name: langConfig.teacherPersona.name, + teacher_description: langConfig.teacherPersona.description, + example_topics: langConfig.exampleTopics.join(', '), + language_instructions: langConfig.promptInstructions, + }; + + // Get all messages except the last user message (that's our current_input) + const messages = state.messages || []; + const historyMessages = messages.slice(0, -1); // All except last + const lastMessage = messages[messages.length - 1]; + const currentInput = lastMessage?.role === 'user' ? lastMessage.content : ''; + + // Get introduction state + const introductionState = state.introductionState || { + name: '', + level: '', + goal: '', + timestamp: '', + }; + + console.log('[DialogPromptBuilder] Introduction state:', JSON.stringify(introductionState)); + console.log('[DialogPromptBuilder] Language:', langConfig.name); + console.log('[DialogPromptBuilder] Messages in history:', historyMessages.length); + + const templateData = { + messages: historyMessages.map((m) => ({ + role: m.role, + content: m.content, + })), + current_input: currentInput, + introduction_state: introductionState, + ...templateVars, + }; + + // Render the prompt using PromptBuilder (Jinja2) + const builder = await PromptBuilder.create(conversationTemplate); + const renderedPrompt = await builder.build(templateData); + + // Debug: Log a snippet of the rendered prompt + const promptSnippet = renderedPrompt.substring(0, 400); + console.log( + `[DialogPromptBuilder] Rendered prompt (first 400 chars): ${promptSnippet}...` + ); + + // Return LLMChatRequest for the LLM node + return new GraphTypes.LLMChatRequest({ + messages: [{ role: 'user', content: renderedPrompt }], + }); + } +} diff --git a/backend/components/graphs/nodes/interaction_queue_node.ts b/backend/components/graphs/nodes/interaction_queue_node.ts new file mode 100644 index 0000000..5c129ae --- /dev/null +++ b/backend/components/graphs/nodes/interaction_queue_node.ts @@ -0,0 +1,167 @@ +/** + * InteractionQueueNode manages the queue of user interactions. + * + * This node: + * - Receives interaction info from STT processing + * - Manages a queue of interactions to ensure sequential processing + * - Prevents race conditions when multiple interactions arrive + * - Returns TextInput when ready to process, or empty when waiting + */ + +import { CustomNode, ProcessContext } from '@inworld/runtime/graph'; +import { ConnectionsMap, InteractionInfo, State, TextInput } from '../../../types/index.js'; + +export class InteractionQueueNode extends CustomNode { + private connections: ConnectionsMap; + + constructor(props?: { + id?: string; + connections?: ConnectionsMap; + reportToClient?: boolean; + }) { + super({ + id: props?.id || 'interaction-queue-node', + reportToClient: props?.reportToClient, + }); + this.connections = props?.connections || {}; + } + + process( + context: ProcessContext, + interactionInfo: InteractionInfo, + state: State + ): TextInput { + const sessionId = interactionInfo.sessionId; + console.log( + `[InteractionQueue] Processing interaction ${interactionInfo.interactionId}` + ); + + // Get current voiceId from connection state + const connection = this.connections[sessionId]; + const currentVoiceId = connection?.state?.voiceId || state?.voiceId; + + const dataStore = context.getDatastore(); + const QUEUED_PREFIX = 'q'; + const RUNNING_PREFIX = 'r'; + const COMPLETED_PREFIX = 'c'; + + // Register interaction in the queue + if (!dataStore.has(QUEUED_PREFIX + interactionInfo.interactionId)) { + dataStore.add( + QUEUED_PREFIX + interactionInfo.interactionId, + interactionInfo.text + ); + console.log( + `[InteractionQueue] New interaction queued: ${interactionInfo.interactionId}` + ); + } + + // Get all keys and categorize them + const allKeys = dataStore.keys(); + const queuedIds: string[] = []; + let completedCount = 0; + let runningCount = 0; + + for (const key of allKeys) { + if (key.startsWith(QUEUED_PREFIX)) { + const idStr = key.substring(QUEUED_PREFIX.length); + queuedIds.push(idStr); + } else if (key.startsWith(COMPLETED_PREFIX)) { + completedCount++; + } else if (key.startsWith(RUNNING_PREFIX)) { + runningCount++; + } + } + + // Sort queued IDs by iteration number + queuedIds.sort((a, b) => { + const getIteration = (id: string): number => { + const hashIndex = id.indexOf('#'); + if (hashIndex === -1) return 0; + const iter = parseInt(id.substring(hashIndex + 1), 10); + return isNaN(iter) ? 0 : iter; + }; + return getIteration(a) - getIteration(b); + }); + + console.log( + `[InteractionQueue] State: queued=${queuedIds.length}, completed=${completedCount}, running=${runningCount}` + ); + + // Decide if we should start processing the next interaction + if (queuedIds.length === 0) { + console.log('[InteractionQueue] No interactions to process'); + return { + text: '', + sessionId: sessionId, + interactionId: '', + voiceId: currentVoiceId, + }; + } + + if (queuedIds.length === completedCount) { + console.log('[InteractionQueue] All interactions completed'); + return { + text: '', + sessionId: sessionId, + interactionId: '', + voiceId: currentVoiceId, + }; + } + + // There are unprocessed interactions + if (runningCount === completedCount) { + // No interaction is currently running, start the next one + const nextId = queuedIds[completedCount]; + const runningKey = RUNNING_PREFIX + nextId; + + // Try to mark as running + if (dataStore.has(runningKey) || !dataStore.add(runningKey, '')) { + console.log( + `[InteractionQueue] Interaction ${nextId} already started` + ); + return { + text: '', + sessionId: sessionId, + interactionId: '', + voiceId: currentVoiceId, + }; + } + + const queuedText = dataStore.get(QUEUED_PREFIX + nextId) as string; + if (!queuedText) { + console.error( + `[InteractionQueue] Failed to retrieve text for ${nextId}` + ); + return { + text: '', + sessionId: sessionId, + interactionId: '', + voiceId: currentVoiceId, + }; + } + + console.log( + `[InteractionQueue] Starting LLM processing: "${queuedText.substring(0, 50)}..."` + ); + + return { + text: queuedText, + sessionId: sessionId, + interactionId: nextId, + voiceId: currentVoiceId, + }; + } else { + // An interaction is currently running, wait + console.log( + `[InteractionQueue] Waiting for interaction ${queuedIds[completedCount]}` + ); + return { + text: '', + sessionId: sessionId, + interactionId: '', + voiceId: currentVoiceId, + }; + } + } +} diff --git a/backend/components/graphs/nodes/state_update_node.ts b/backend/components/graphs/nodes/state_update_node.ts new file mode 100644 index 0000000..fb013c7 --- /dev/null +++ b/backend/components/graphs/nodes/state_update_node.ts @@ -0,0 +1,68 @@ +/** + * StateUpdateNode updates the state with the LLM's response. + * + * This node: + * - Receives the LLM output text + * - Updates the connection state with the assistant message + * - Marks the interaction as completed in the datastore + * - Returns the updated state + */ + +import { CustomNode, ProcessContext } from '@inworld/runtime/graph'; +import { v4 as uuidv4 } from 'uuid'; +import { ConnectionsMap, State } from '../../../types/index.js'; + +export class StateUpdateNode extends CustomNode { + private connections: ConnectionsMap; + + constructor(props: { + id: string; + connections: ConnectionsMap; + reportToClient?: boolean; + }) { + super({ + id: props.id, + reportToClient: props.reportToClient, + }); + this.connections = props.connections; + } + + process(context: ProcessContext, llmOutput: string): State { + const sessionId = context.getDatastore().get('sessionId') as string; + console.log( + `[StateUpdateNode] Processing [length:${llmOutput?.length || 0}]` + ); + + const connection = this.connections[sessionId]; + if (connection?.unloaded) { + throw Error(`Session unloaded for sessionId:${sessionId}`); + } + if (!connection) { + throw Error(`Failed to read connection for sessionId:${sessionId}`); + } + + // Only add assistant message if there's actual content + if (llmOutput && llmOutput.trim().length > 0) { + console.log( + `[StateUpdateNode] Adding assistant message: "${llmOutput.substring(0, 50)}..."` + ); + connection.state.messages.push({ + role: 'assistant', + content: llmOutput, + id: connection.state.interactionId || uuidv4(), + timestamp: new Date().toISOString(), + }); + } else { + console.log('[StateUpdateNode] Skipping empty message'); + } + + // Mark interaction as completed + const dataStore = context.getDatastore(); + dataStore.add('c' + connection.state.interactionId, ''); + console.log( + `[StateUpdateNode] Marked interaction ${connection.state.interactionId} as completed` + ); + + return connection.state; + } +} diff --git a/backend/components/graphs/nodes/text_input_node.ts b/backend/components/graphs/nodes/text_input_node.ts new file mode 100644 index 0000000..ba3111d --- /dev/null +++ b/backend/components/graphs/nodes/text_input_node.ts @@ -0,0 +1,61 @@ +/** + * TextInputNode updates the state with the user's input this turn. + * + * This node: + * - Receives user text input with interaction and session IDs + * - Updates the connection state with the user message + * - Returns the updated state for downstream processing + */ + +import { CustomNode, ProcessContext } from '@inworld/runtime/graph'; +import { v4 as uuidv4 } from 'uuid'; +import { ConnectionsMap, State, TextInput } from '../../../types/index.js'; + +export class TextInputNode extends CustomNode { + private connections: ConnectionsMap; + + constructor(props: { + id: string; + connections: ConnectionsMap; + reportToClient?: boolean; + }) { + super({ + id: props.id, + reportToClient: props.reportToClient, + }); + this.connections = props.connections; + } + + process(_context: ProcessContext, input: TextInput): State { + console.log( + `[TextInputNode] Processing: "${input.text?.substring(0, 50)}..."` + ); + + const { text, interactionId, sessionId } = input; + + const connection = this.connections[sessionId]; + if (connection?.unloaded) { + throw Error(`Session unloaded for sessionId:${sessionId}`); + } + if (!connection) { + throw Error(`Failed to read connection for sessionId:${sessionId}`); + } + const state = connection.state; + if (!state) { + throw Error(`Failed to read state from connection for sessionId:${sessionId}`); + } + + // Update interactionId + connection.state.interactionId = interactionId; + + // Add user message to conversation + connection.state.messages.push({ + role: 'user', + content: text, + id: interactionId || uuidv4(), + timestamp: new Date().toISOString(), + }); + + return connection.state; + } +} diff --git a/backend/components/graphs/nodes/transcript_extractor_node.ts b/backend/components/graphs/nodes/transcript_extractor_node.ts new file mode 100644 index 0000000..c4b33d4 --- /dev/null +++ b/backend/components/graphs/nodes/transcript_extractor_node.ts @@ -0,0 +1,51 @@ +/** + * TranscriptExtractorNode extracts transcript information from + * DataStreamWithMetadata (output from AssemblyAISTTNode) and converts + * it to InteractionInfo for downstream processing. + */ + +import { DataStreamWithMetadata } from '@inworld/runtime'; +import { CustomNode, ProcessContext } from '@inworld/runtime/graph'; +import { InteractionInfo } from '../../../types/index.js'; + +export class TranscriptExtractorNode extends CustomNode { + constructor(props?: { id?: string; reportToClient?: boolean }) { + super({ + id: props?.id || 'transcript-extractor-node', + reportToClient: props?.reportToClient, + }); + } + + /** + * Extract transcript from metadata and return as InteractionInfo + */ + process( + context: ProcessContext, + streamWithMetadata: DataStreamWithMetadata + ): InteractionInfo { + const metadata = streamWithMetadata.getMetadata(); + const sessionId = context.getDatastore().get('sessionId') as string; + + // Extract transcript and related info from metadata + const transcript = (metadata.transcript as string) || ''; + const interactionComplete = + (metadata.interaction_complete as boolean) || false; + const iteration = (metadata.iteration as number) || 1; + const interactionId = String(metadata.interactionId || iteration); + + console.log( + `[TranscriptExtractor] Processing [iteration:${iteration}]: "${transcript?.substring(0, 50)}..." [complete:${interactionComplete}]` + ); + + return { + sessionId, + interactionId: interactionId, + text: transcript, + interactionComplete, + }; + } + + async destroy(): Promise { + // No cleanup needed + } +} diff --git a/backend/components/graphs/nodes/tts_request_builder_node.ts b/backend/components/graphs/nodes/tts_request_builder_node.ts new file mode 100644 index 0000000..1af2ca8 --- /dev/null +++ b/backend/components/graphs/nodes/tts_request_builder_node.ts @@ -0,0 +1,64 @@ +/** + * TTSRequestBuilderNode builds a TTSRequest with dynamic voiceId. + * + * For long-running graphs, it reads voiceId from connection state at processing time + * to ensure voice changes via language switching are reflected immediately. + */ + +import { CustomNode, GraphTypes, ProcessContext } from '@inworld/runtime/graph'; +import { ConnectionsMap } from '../../../types/index.js'; +import { getLanguageConfig } from '../../../config/languages.js'; + +export class TTSRequestBuilderNode extends CustomNode { + private connections: ConnectionsMap; + private defaultVoiceId: string; + + constructor(props: { + id: string; + connections: ConnectionsMap; + defaultVoiceId: string; + reportToClient?: boolean; + }) { + super({ + id: props.id, + reportToClient: props.reportToClient, + }); + this.connections = props.connections; + this.defaultVoiceId = props.defaultVoiceId; + } + + /** + * Build a TTSRequest with the current voiceId from connection state + * Receives two inputs: + * 1. input - Graph input with sessionId (TextInput or State) + * 2. textStream - The text stream from TextChunkingNode + */ + process( + context: ProcessContext, + _input: unknown, + textStream: GraphTypes.TextStream + ): GraphTypes.TTSRequest { + const sessionId = context.getDatastore().get('sessionId') as string; + + // For long-running graphs, read voiceId from connection state at processing time + const connection = this.connections[sessionId]; + + // Get voice from state, or fall back to language config, or default + let voiceId = connection?.state?.voiceId; + if (!voiceId && connection?.state?.languageCode) { + const langConfig = getLanguageConfig(connection.state.languageCode); + voiceId = langConfig.ttsConfig.speakerId; + } + voiceId = voiceId || this.defaultVoiceId; + + console.log(`[TTSRequestBuilder] Building TTS request [voice:${voiceId}]`); + + return GraphTypes.TTSRequest.withStream(textStream, { + id: voiceId, + }); + } + + async destroy(): Promise { + // No cleanup needed + } +} diff --git a/backend/graphs/conversation-graph.ts b/backend/graphs/conversation-graph.ts deleted file mode 100644 index 14afe67..0000000 --- a/backend/graphs/conversation-graph.ts +++ /dev/null @@ -1,267 +0,0 @@ -import { - GraphBuilder, - CustomNode, - ProcessContext, - ProxyNode, - RemoteLLMChatNode, - RemoteSTTNode, - RemoteTTSNode, - TextChunkingNode, - Graph, -} from '@inworld/runtime/graph'; -import { GraphTypes } from '@inworld/runtime/common'; -import { renderJinja } from '@inworld/runtime/primitives/llm'; -import { AsyncLocalStorage } from 'async_hooks'; -import { conversationTemplate } from '../helpers/prompt-templates.js'; -import type { IntroductionState } from '../helpers/introduction-state-processor.js'; -import { - LanguageConfig, - getLanguageConfig, - DEFAULT_LANGUAGE_CODE, -} from '../config/languages.js'; - -export interface ConversationGraphConfig { - apiKey: string; -} - -// Use AsyncLocalStorage to store state accessors directly per async execution context -// This eliminates the need for a registry and connectionId lookup -export const stateStorage = new AsyncLocalStorage<{ - getConversationState: () => { - messages: Array<{ role: string; content: string; timestamp: string }>; - }; - getIntroductionState: () => IntroductionState; - getLanguageConfig: () => LanguageConfig; -}>(); - -// Store the current execution context as module-level variables -// This is set by AudioProcessor before starting graph execution -// and read by the PromptBuilder during execution. -// This works because Node.js is single-threaded for synchronous execution. -let currentExecutionLanguageCode: string = DEFAULT_LANGUAGE_CODE; -let currentGetConversationState: (() => { messages: Array<{ role: string; content: string; timestamp: string }> }) | null = null; -let currentGetIntroductionState: (() => IntroductionState) | null = null; - -/** - * Set the execution context for the current graph execution. - * Must be called before starting graph execution. - */ -export function setCurrentExecutionContext(context: { - languageCode: string; - getConversationState: () => { messages: Array<{ role: string; content: string; timestamp: string }> }; - getIntroductionState: () => IntroductionState; -}): void { - currentExecutionLanguageCode = context.languageCode; - currentGetConversationState = context.getConversationState; - currentGetIntroductionState = context.getIntroductionState; - console.log(`[ConversationGraph] Set execution context for language: ${context.languageCode}`); -} - -/** - * Set the language code for the current graph execution. - * @deprecated Use setCurrentExecutionContext instead - */ -export function setCurrentExecutionLanguage(languageCode: string): void { - currentExecutionLanguageCode = languageCode; - console.log(`[ConversationGraph] Set execution language to: ${languageCode}`); -} - -/** - * EnhancedPromptBuilderNode - defined once at module level to avoid - * component registry collisions. State and language config are retrieved from - * module-level variables (set by AudioProcessor before graph execution). - */ -class EnhancedPromptBuilderNode extends CustomNode { - async process(_context: ProcessContext, currentInput: string) { - // Get language config using the current execution language code - const langConfig = getLanguageConfig(currentExecutionLanguageCode); - const nodeId = (this as unknown as { id: string }).id; - - console.log( - `[PromptBuilder] Node ${nodeId} using execution language: ${langConfig.name} (code: ${currentExecutionLanguageCode})` - ); - - // Build template variables from language config - const templateVars = { - target_language: langConfig.name, - target_language_native: langConfig.nativeName, - teacher_name: langConfig.teacherPersona.name, - teacher_description: langConfig.teacherPersona.description, - example_topics: langConfig.exampleTopics.join(', '), - language_instructions: langConfig.promptInstructions, - }; - - // Get state from module-level accessors (set by AudioProcessor before execution) - // These bypass AsyncLocalStorage which gets broken by Inworld runtime - const conversationState = currentGetConversationState - ? currentGetConversationState() - : { messages: [] }; - const introductionState = currentGetIntroductionState - ? currentGetIntroductionState() - : { name: '', level: '', goal: '', timestamp: '' }; - - console.log( - '[PromptBuilder] Introduction state:', - JSON.stringify(introductionState, null, 2) - ); - console.log('[PromptBuilder] Language:', langConfig.name); - console.log( - '[PromptBuilder] Messages in history:', - conversationState.messages?.length || 0 - ); - - const templateData = { - messages: conversationState.messages || [], - current_input: currentInput, - introduction_state: introductionState, - ...templateVars, - }; - - const renderedPrompt = await renderJinja( - conversationTemplate, - JSON.stringify(templateData) - ); - - // Debug: Log a snippet of the rendered prompt to verify content - const promptSnippet = renderedPrompt.substring(0, 400); - console.log( - `[PromptBuilder] Rendered prompt (first 400 chars): ${promptSnippet}...` - ); - - // Return LLMChatRequest for the LLM node - return new GraphTypes.LLMChatRequest({ - messages: [{ role: 'user', content: renderedPrompt }], - }); - } -} - -/** - * Creates a conversation graph configured for a specific language - */ -function createConversationGraphForLanguage( - _config: ConversationGraphConfig, - languageConfig: LanguageConfig -): Graph { - - // Use language code as suffix to make node IDs unique per language - // This prevents edge condition name collisions in the global callback registry - const langSuffix = `_${languageConfig.code}`; - const promptBuilderNodeId = `enhanced_prompt_builder_node${langSuffix}`; - - console.log( - `[ConversationGraph] Creating graph with prompt builder node: ${promptBuilderNodeId}` - ); - - // Configure STT for the specific language - const sttNode = new RemoteSTTNode({ - id: `stt_node${langSuffix}`, - sttConfig: { - languageCode: languageConfig.sttLanguageCode, - }, - }); - - const sttOutputNode = new ProxyNode({ - id: `proxy_node${langSuffix}`, - reportToClient: true, - }); - - const promptBuilderNode = new EnhancedPromptBuilderNode({ - id: promptBuilderNodeId, - }); - - const llmNode = new RemoteLLMChatNode({ - id: `llm_node${langSuffix}`, - provider: 'openai', - modelName: 'gpt-4o-mini', - stream: true, - reportToClient: true, - textGenerationConfig: { - maxNewTokens: 250, - maxPromptLength: 2000, - repetitionPenalty: 1, - topP: 1, - temperature: 1, - frequencyPenalty: 0, - presencePenalty: 0, - }, - }); - - const chunkerNode = new TextChunkingNode({ id: `chunker_node${langSuffix}` }); - - // Configure TTS for the specific language - const ttsNode = new RemoteTTSNode({ - id: `tts_node${langSuffix}`, - speakerId: languageConfig.ttsConfig.speakerId, - modelId: languageConfig.ttsConfig.modelId, - sampleRate: 16000, - speakingRate: languageConfig.ttsConfig.speakingRate, - temperature: languageConfig.ttsConfig.temperature, - languageCode: languageConfig.ttsConfig.languageCode, - }); - - const executor = new GraphBuilder({ - id: `conversation_graph_${languageConfig.code}`, - enableRemoteConfig: false, - }) - .addNode(sttNode) - .addNode(sttOutputNode) - .addNode(promptBuilderNode) - .addNode(llmNode) - .addNode(chunkerNode) - .addNode(ttsNode) - .setStartNode(sttNode) - .addEdge(sttNode, sttOutputNode) - .addEdge(sttNode, promptBuilderNode, { - condition: async (input: string) => { - return input.trim().length > 0; - }, - }) - .addEdge(promptBuilderNode, llmNode) - .addEdge(llmNode, chunkerNode) - .addEdge(chunkerNode, ttsNode) - .setEndNode(sttOutputNode) - .setEndNode(ttsNode) - .build(); - - return executor; -} - -// Cache for language-specific graphs -const graphCache = new Map(); - -/** - * Get or create a conversation graph for a specific language - * Graphs are cached to avoid recreation overhead - */ -export function getConversationGraph( - config: ConversationGraphConfig, - languageCode: string = DEFAULT_LANGUAGE_CODE -): Graph { - const cacheKey = languageCode; - - if (!graphCache.has(cacheKey)) { - const languageConfig = getLanguageConfig(languageCode); - console.log( - `Creating conversation graph for language: ${languageConfig.name} (${languageCode})` - ); - const graph = createConversationGraphForLanguage(config, languageConfig); - graphCache.set(cacheKey, graph); - } - - return graphCache.get(cacheKey)!; -} - -/** - * Legacy function for backwards compatibility - * Creates or returns the default (Spanish) graph - */ -export function createConversationGraph(config: ConversationGraphConfig): Graph { - return getConversationGraph(config, DEFAULT_LANGUAGE_CODE); -} - -/** - * Clear the graph cache (useful for testing or reconfiguration) - */ -export function clearGraphCache(): void { - graphCache.clear(); -} diff --git a/backend/graphs/flashcard-graph.ts b/backend/graphs/flashcard-graph.ts index 3990d7e..b9b87e0 100644 --- a/backend/graphs/flashcard-graph.ts +++ b/backend/graphs/flashcard-graph.ts @@ -1,5 +1,4 @@ import 'dotenv/config'; -import fs from 'fs'; import { GraphBuilder, @@ -9,7 +8,7 @@ import { Graph, } from '@inworld/runtime/graph'; import { GraphTypes } from '@inworld/runtime/common'; -import { renderJinja } from '@inworld/runtime/primitives/llm'; +import { PromptBuilder } from '@inworld/runtime/primitives/llm'; import { flashcardPromptTemplate } from '../helpers/prompt-templates.js'; import { v4 } from 'uuid'; import { Flashcard } from '../helpers/flashcard-processor.js'; @@ -24,10 +23,8 @@ class FlashcardPromptBuilderNode extends CustomNode { _context: ProcessContext, input: GraphTypes.Content | Record ) { - const renderedPrompt = await renderJinja( - flashcardPromptTemplate, - JSON.stringify(input) - ); + const builder = await PromptBuilder.create(flashcardPromptTemplate); + const renderedPrompt = await builder.build(input as Record); return renderedPrompt; } } @@ -99,7 +96,7 @@ function createFlashcardGraphForLanguage(languageConfig: LanguageConfig): Graph const llmNode = new RemoteLLMChatNode({ id: 'llm_node', provider: 'openai', - modelName: 'gpt-5', + modelName: 'gpt-4o-mini', stream: false, textGenerationConfig: { maxNewTokens: 2500, @@ -115,7 +112,7 @@ function createFlashcardGraphForLanguage(languageConfig: LanguageConfig): Graph const executor = new GraphBuilder({ id: `flashcard-generation-graph-${languageConfig.code}`, - enableRemoteConfig: true, + enableRemoteConfig: false, }) .addNode(promptBuilderNode) .addNode(textToChatRequestNode) @@ -128,11 +125,6 @@ function createFlashcardGraphForLanguage(languageConfig: LanguageConfig): Graph .setEndNode(parserNode) .build(); - // Only write debug file for default language to avoid cluttering - if (languageConfig.code === DEFAULT_LANGUAGE_CODE) { - fs.writeFileSync('flashcard-graph.json', executor.toJSON()); - } - return executor; } diff --git a/backend/graphs/introduction-state-graph.ts b/backend/graphs/introduction-state-graph.ts index fb929a9..bc1e817 100644 --- a/backend/graphs/introduction-state-graph.ts +++ b/backend/graphs/introduction-state-graph.ts @@ -7,7 +7,7 @@ import { Graph, } from '@inworld/runtime/graph'; import { GraphTypes } from '@inworld/runtime/common'; -import { renderJinja } from '@inworld/runtime/primitives/llm'; +import { PromptBuilder } from '@inworld/runtime/primitives/llm'; import { introductionStatePromptTemplate } from '../helpers/prompt-templates.js'; import { LanguageConfig, @@ -29,10 +29,8 @@ class IntroductionPromptBuilderNode extends CustomNode { _context: ProcessContext, input: GraphTypes.Content | Record ) { - const renderedPrompt = await renderJinja( - introductionStatePromptTemplate, - JSON.stringify(input) - ); + const builder = await PromptBuilder.create(introductionStatePromptTemplate); + const renderedPrompt = await builder.build(input as Record); return renderedPrompt; } } @@ -163,7 +161,7 @@ function createIntroductionStateGraphForLanguage( const llmNode = new RemoteLLMChatNode({ id: 'llm_node', provider: 'openai', - modelName: 'gpt-4.1', + modelName: 'gpt-4o-mini', stream: false, }); const parserNode = new IntroductionStateParserNode({ @@ -172,6 +170,7 @@ function createIntroductionStateGraphForLanguage( const executor = new GraphBuilder({ id: `introduction-state-graph-${languageConfig.code}`, + enableRemoteConfig: false, }) .addNode(promptBuilderNode) .addNode(textToChatRequestNode) diff --git a/backend/helpers/audio-processor.ts b/backend/helpers/audio-processor.ts deleted file mode 100644 index 1a122c7..0000000 --- a/backend/helpers/audio-processor.ts +++ /dev/null @@ -1,790 +0,0 @@ -import { v4 as uuidv4 } from 'uuid'; -import { GraphTypes } from '@inworld/runtime/common'; -import { UserContextInterface } from '@inworld/runtime/graph'; -import { Graph } from '@inworld/runtime/graph'; -import { WebSocket } from 'ws'; -import { SileroVAD, VADConfig } from './silero-vad.js'; -import { stateStorage, setCurrentExecutionContext } from '../graphs/conversation-graph.js'; -import type { IntroductionState } from './introduction-state-processor.js'; -import { - LanguageConfig, - getLanguageConfig, - DEFAULT_LANGUAGE_CODE, -} from '../config/languages.js'; - -export class AudioProcessor { - private executor: Graph; - private vad: SileroVAD | null = null; - private isProcessing = false; - private isProcessingCancelled = false; // Track if current processing should be cancelled - private isReady = false; - private websocket: WebSocket | null = null; - private pendingSpeechSegments: Float32Array[] = []; // Accumulate speech segments - private conversationState: { - messages: Array<{ role: string; content: string; timestamp: string }>; - } = { - messages: [], - }; - private flashcardCallback: - | ((messages: Array<{ role: string; content: string }>) => Promise) - | null = null; - private introductionState: IntroductionState = { - name: '', - level: '', - goal: '', - timestamp: '', - }; - private introductionStateCallback: - | (( - messages: Array<{ role: string; content: string }> - ) => Promise) - | null = null; - private targetingKey: string | null = null; - private clientTimezone: string | null = null; - private graphStartTime: number = 0; - private languageCode: string = DEFAULT_LANGUAGE_CODE; - private languageConfig: LanguageConfig; - - constructor( - executor: Graph, - websocket?: WebSocket, - languageCode: string = DEFAULT_LANGUAGE_CODE - ) { - this.executor = executor; - this.websocket = websocket ?? null; - this.languageCode = languageCode; - this.languageConfig = getLanguageConfig(languageCode); - this.setupWebSocketMessageHandler(); - setTimeout(() => this.initialize(), 100); - } - - /** - * Update the language and graph for this processor - */ - setLanguage(languageCode: string, newGraph: Graph): void { - if (this.languageCode !== languageCode) { - console.log( - `AudioProcessor: Changing language from ${this.languageCode} to ${languageCode}` - ); - this.languageCode = languageCode; - this.languageConfig = getLanguageConfig(languageCode); - this.executor = newGraph; - - // Reset conversation state when language changes - this.reset(); - - console.log( - `AudioProcessor: Language changed to ${this.languageConfig.name}` - ); - } - } - - /** - * Get current language code - */ - getLanguageCode(): string { - return this.languageCode; - } - - /** - * Get current language config - */ - getLanguageConfig(): LanguageConfig { - return this.languageConfig; - } - - // Public methods for state registry access - getConversationState() { - return this.conversationState; - } - - getIntroductionState(): IntroductionState { - return this.introductionState; - } - - private trimConversationHistory(maxTurns: number = 40) { - const maxMessages = maxTurns * 2; - if (this.conversationState.messages.length > maxMessages) { - this.conversationState.messages = - this.conversationState.messages.slice(-maxMessages); - } - } - - private setupWebSocketMessageHandler() { - if (this.websocket) { - this.websocket.on('message', (data: Buffer | string) => { - try { - const raw = - typeof data === 'string' ? data : data?.toString?.() || ''; - const message = JSON.parse(raw); - - if (message.type === 'conversation_update') { - const incoming = - message.data && message.data.messages - ? message.data.messages - : []; - const existing = this.conversationState.messages || []; - const combined = [...existing, ...incoming]; - const seen = new Set(); - const deduped: Array<{ - role: string; - content: string; - timestamp: string; - }> = []; - for (const m of combined) { - const key = `${m.timestamp}|${m.role}|${m.content}`; - if (!seen.has(key)) { - seen.add(key); - deduped.push(m); - } - } - deduped.sort( - (a, b) => - new Date(a.timestamp).getTime() - - new Date(b.timestamp).getTime() - ); - this.conversationState = { messages: deduped }; - this.trimConversationHistory(40); - } else if (message.type === 'user_context') { - // Persist minimal attributes for user context - const tz = - message.timezone || (message.data && message.data.timezone) || ''; - const uid = - message.userId || (message.data && message.data.userId) || ''; - this.clientTimezone = tz || this.clientTimezone; - if (uid) this.targetingKey = uid; - } - } catch { - // Not a JSON message or conversation update, ignore - } - }); - } - } - - setFlashcardCallback( - callback: ( - messages: Array<{ role: string; content: string }> - ) => Promise - ) { - this.flashcardCallback = callback; - } - - setIntroductionStateCallback( - callback: ( - messages: Array<{ role: string; content: string }> - ) => Promise - ) { - this.introductionStateCallback = callback; - } - - private async initialize() { - try { - console.log('AudioProcessor: Starting initialization...'); - - // Initialize VAD - try { - const vadConfig: VADConfig = { - modelPath: 'backend/models/silero_vad.onnx', - threshold: 0.5, // Following working example SPEECH_THRESHOLD - minSpeechDuration: 0.2, // MIN_SPEECH_DURATION_MS / 1000 - minSilenceDuration: 0.4, // Reduced from 0.65 for faster response - speechResetSilenceDuration: 1.0, - minVolume: 0.01, // Lower threshold to start - can adjust based on testing - sampleRate: 16000, - }; - - console.log( - 'AudioProcessor: Creating SileroVAD with config:', - vadConfig - ); - this.vad = new SileroVAD(vadConfig); - - console.log('AudioProcessor: Initializing VAD...'); - await this.vad.initialize(); - console.log('AudioProcessor: VAD initialized successfully'); - - this.vad.on('speechStart', () => { - console.log('🎤 Speech started'); - - // Always notify frontend to stop audio playback when user starts speaking - if (this.websocket) { - try { - this.websocket.send( - JSON.stringify({ type: 'interrupt', reason: 'speech_start' }) - ); - // Notify frontend that speech was detected so it can show listening indicator - this.websocket.send( - JSON.stringify({ - type: 'speech_detected', - data: { text: '' }, // Empty text initially, will be updated with transcript - }) - ); - } catch { - // ignore send errors - } - } - - // If we're currently processing, set cancellation flag - if (this.isProcessing) { - console.log( - 'Setting cancellation flag - user started speaking during processing' - ); - this.isProcessingCancelled = true; - // Don't clear segments - we want to accumulate them - } - }); - - this.vad.on('speechEnd', async (event) => { - console.log( - '🔇 Speech ended, duration:', - event.speechDuration.toFixed(2) + 's' - ); - - // Always notify frontend that VAD stopped detecting speech - // This allows frontend to clear the real-time transcript bubble - if (this.websocket) { - try { - this.websocket.send( - JSON.stringify({ - type: 'speech_ended', - data: { - speechDuration: event.speechDuration, - hasSegment: - event.speechSegment !== null && - event.speechSegment.length > 0, - }, - }) - ); - } catch { - // ignore send errors - } - } - - try { - if (event.speechSegment && event.speechSegment.length > 0) { - // Add this segment to pending segments - this.pendingSpeechSegments.push(event.speechSegment); - - // Reset cancellation flag for new processing - this.isProcessingCancelled = false; - - // Process immediately if not already processing - if (!this.isProcessing && this.pendingSpeechSegments.length > 0) { - // Combine all pending segments - const totalLength = this.pendingSpeechSegments.reduce( - (sum, seg) => sum + seg.length, - 0 - ); - const combinedSegment = new Float32Array(totalLength); - let offset = 0; - for (const seg of this.pendingSpeechSegments) { - combinedSegment.set(seg, offset); - offset += seg.length; - } - - // Clear pending segments - this.pendingSpeechSegments = []; - - // Process immediately - console.log('Processing speech segment immediately'); - await this.processVADSpeechSegment(combinedSegment); - } - } - } catch (error) { - console.error('Error handling speech segment:', error); - } - }); - } catch (error) { - console.error('AudioProcessor: VAD initialization failed:', error); - this.vad = null; - } - - // Graph is already provided in constructor (shared instance) - // Just mark as ready - this.isReady = true; - console.log( - `AudioProcessor: Using ${this.languageConfig.name} conversation graph, ready for audio processing` - ); - } catch (error) { - console.error( - 'AudioProcessor: Initialization failed with unexpected error:', - error - ); - this.isReady = false; - // Don't rethrow - prevent unhandled promise rejection - } - } - - addAudioChunk(base64Audio: string) { - try { - // Feed audio to VAD if available and ready - if (this.vad && this.isReady) { - this.vad.addAudioData(base64Audio); - } else { - // VAD not available - log error and ignore audio chunk - if (!this.vad) { - console.error('VAD not available - cannot process audio chunk'); - } else if (!this.isReady) { - console.warn('AudioProcessor not ready yet - ignoring audio chunk'); - } - } - } catch (error) { - console.error('Error processing audio chunk:', error); - } - } - - private amplifyAudio(audioData: Float32Array, gain: number): Float32Array { - const amplified = new Float32Array(audioData.length); - - let maxVal = 0; - for (let i = 0; i < audioData.length; i++) { - const absVal = Math.abs(audioData[i]); - if (absVal > maxVal) { - maxVal = absVal; - } - } - - // For normalized Float32 audio [-1, 1], prevent clipping at 1.0 - const safeGain = maxVal > 0 ? Math.min(gain, 0.95 / maxVal) : gain; - - for (let i = 0; i < audioData.length; i++) { - // Clamp to [-1, 1] range to prevent distortion - amplified[i] = Math.max(-1, Math.min(1, audioData[i] * safeGain)); - } - - return amplified; - } - - private async processVADSpeechSegment( - speechSegment: Float32Array - ): Promise { - if (this.isProcessing) { - return; - } - - this.isProcessing = true; - - try { - const amplifiedSegment = this.amplifyAudio(speechSegment, 2.0); - - // Save debug audio before sending to STT - // await this.saveAudioDebug(amplifiedSegment, 'vad-segment'); - - // Create Audio instance for STT node - const audioInput = new GraphTypes.Audio({ - data: Array.from(amplifiedSegment), - sampleRate: 16000, - }); - - // Build user context for experiments - const attributes: Record = { - timezone: this.clientTimezone || '', - language: this.languageCode, - }; - attributes.name = - (this.introductionState?.name && this.introductionState.name.trim()) || - 'unknown'; - attributes.level = - (this.introductionState?.level && - (this.introductionState.level as string)) || - 'unknown'; - attributes.goal = - (this.introductionState?.goal && this.introductionState.goal.trim()) || - 'unknown'; - - const targetingKey = this.targetingKey || uuidv4(); - const userContext: UserContextInterface = { - attributes: attributes, - targetingKey: targetingKey, - }; - console.log(userContext); - - this.graphStartTime = Date.now(); - - // Set the current execution context BEFORE starting the graph - // This ensures the PromptBuilder node has access to state and language config - // We use module-level variables because Inworld runtime breaks AsyncLocalStorage context - setCurrentExecutionContext({ - languageCode: this.languageCode, - getConversationState: () => this.getConversationState(), - getIntroductionState: () => this.getIntroductionState(), - }); - - // Also use AsyncLocalStorage as a fallback (may work in some contexts) - const executionResult = await stateStorage.run( - { - getConversationState: () => this.getConversationState(), - getIntroductionState: () => this.getIntroductionState(), - getLanguageConfig: () => this.getLanguageConfig(), - }, - async () => { - try { - const executionContext = { - executionId: uuidv4(), - userContext: userContext, - }; - return await this.executor.start(audioInput, executionContext); - } catch (err) { - console.warn( - 'Executor.start with ExecutionContext failed, falling back without context:', - err - ); - return await this.executor.start(audioInput); - } - } - ); - - let transcription = ''; - let llmResponse = ''; - - for await (const chunk of executionResult.outputStream) { - // Check if processing has been cancelled - if (this.isProcessingCancelled) { - console.log( - 'Processing cancelled by user speech, breaking from loop' - ); - break; - } - - console.log( - `Audio Processor:Chunk received - Type: ${chunk.typeName}, Has processResponse: ${typeof chunk.processResponse === 'function'}` - ); - console.log( - `Audio Processor:Time since graph started: ${Date.now() - this.graphStartTime}ms` - ); - - // Use processResponse for type-safe handling - await chunk.processResponse({ - // Handle string output (from ProxyNode with STT transcription) - string: (data: string) => { - transcription = data; - - // Debug: always log raw STT result for troubleshooting - console.log( - `VAD STT Raw Result [${this.languageCode}]: "${data}" (length: ${data.length}, trimmed: ${data.trim().length})` - ); - - if (transcription.trim() === '') { - console.log( - `VAD STT: Empty transcription for ${this.languageCode}, skipping LLM` - ); - return; - } - - console.log( - `VAD STT Transcription (via ProxyNode): "${transcription}"` - ); - - if (this.websocket) { - this.websocket.send( - JSON.stringify({ - type: 'transcription', - text: transcription.trim(), - timestamp: Date.now(), - }) - ); - } - // Don't add to conversation state yet - the prompt template will use it as current_input - // We'll add it after processing completes - - // Opportunistically run introduction-state extraction as soon as we have user input - const isIntroCompleteEarly = Boolean( - this.introductionState?.name && - this.introductionState?.level && - this.introductionState?.goal - ); - if (!isIntroCompleteEarly && this.introductionStateCallback) { - const recentMessages = this.conversationState.messages - .slice(-6) - .map((msg) => ({ - role: msg.role, - content: msg.content, - })); - this.introductionStateCallback(recentMessages) - .then((state) => { - if (state) { - this.introductionState = state; - if (this.websocket) { - this.websocket.send( - JSON.stringify({ - type: 'introduction_state_updated', - introduction_state: this.introductionState, - timestamp: Date.now(), - }) - ); - } - } - }) - .catch((error) => { - console.error( - 'Error in introduction-state callback (early):', - error - ); - }); - } - }, - - // Handle ContentStream (from LLM) - ContentStream: async (streamIterator: GraphTypes.ContentStream) => { - console.log('VAD Processing LLM ContentStream...'); - let currentLLMResponse = ''; - for await (const streamChunk of streamIterator) { - if (streamChunk.text) { - currentLLMResponse += streamChunk.text; - // console.log('VAD LLM chunk:', streamChunk.text); - if (this.websocket) { - this.websocket.send( - JSON.stringify({ - type: 'llm_response_chunk', - text: streamChunk.text, - timestamp: Date.now(), - }) - ); - } - } - } - if (currentLLMResponse.trim()) { - llmResponse = currentLLMResponse; - console.log(`VAD Complete LLM Response: "${llmResponse}"`); - if (this.websocket) { - this.websocket.send( - JSON.stringify({ - type: 'llm_response_complete', - text: llmResponse.trim(), - timestamp: Date.now(), - }) - ); - } - // Now update conversation state with both user and assistant messages - // Add user message first (it wasn't added earlier to avoid duplication) - this.conversationState.messages.push({ - role: 'user', - content: transcription.trim(), - timestamp: new Date().toISOString(), - }); - // Then add assistant message - this.conversationState.messages.push({ - role: 'assistant', - content: llmResponse.trim(), - timestamp: new Date().toISOString(), - }); - this.trimConversationHistory(40); - console.log('Updated conversation state with full exchange'); - - // Mark that we'll need to trigger flashcard generation after TTS completes - // We'll do this after all TTS chunks have been sent - } - }, - - // Handle TTS output stream - TTSOutputStream: async ( - ttsStreamIterator: GraphTypes.TTSOutputStream - ) => { - console.log('VAD Processing TTS audio stream...'); - let isFirstChunk = true; - for await (const ttsChunk of ttsStreamIterator) { - if (ttsChunk.audio && ttsChunk.audio.data) { - // Log first chunk for latency tracking - if (isFirstChunk) { - console.log('Sending first TTS chunk immediately'); - } - - // Decode base64 and create Float32Array - const decodedData = Buffer.from(ttsChunk.audio.data, 'base64'); - const float32Array = new Float32Array(decodedData.buffer); - - // Convert Float32 to Int16 for web audio playback - const int16Array = new Int16Array(float32Array.length); - for (let i = 0; i < float32Array.length; i++) { - int16Array[i] = Math.max( - -32768, - Math.min(32767, float32Array[i] * 32767) - ); - } - const base64Audio = Buffer.from(int16Array.buffer).toString( - 'base64' - ); - - // Send immediately without buffering - if (this.websocket) { - this.websocket.send( - JSON.stringify({ - type: 'audio_stream', - audio: base64Audio, - sampleRate: ttsChunk.audio.sampleRate || 16000, - timestamp: Date.now(), - text: ttsChunk.text || '', - isFirstChunk: isFirstChunk, - }) - ); - } - - // Mark that we've sent the first chunk - isFirstChunk = false; - } - } - // Send completion signal for iOS - console.log('VAD TTS stream complete, sending completion signal'); - if (this.websocket) { - this.websocket.send( - JSON.stringify({ - type: 'audio_stream_complete', - timestamp: Date.now(), - }) - ); - } - - // Now that TTS is complete, trigger flashcard generation and other post-processing - if (transcription && llmResponse) { - console.log( - 'Triggering flashcard generation after TTS completion' - ); - - // Send conversation update to frontend - if (this.websocket) { - this.websocket.send( - JSON.stringify({ - type: 'conversation_update', - messages: this.conversationState.messages, - timestamp: Date.now(), - }) - ); - } - - // Generate flashcards - fire and forget - if (this.flashcardCallback) { - const recentMessages = this.conversationState.messages - .slice(-6) - .map((msg) => ({ - role: msg.role, - content: msg.content, - })); - - this.flashcardCallback(recentMessages).catch((error) => { - console.error( - 'Error in flashcard generation callback:', - error - ); - }); - } - - // Run introduction-state extraction while incomplete - const isIntroComplete = Boolean( - this.introductionState?.name && - this.introductionState?.level && - this.introductionState?.goal - ); - if (!isIntroComplete && this.introductionStateCallback) { - const recentMessages = this.conversationState.messages - .slice(-6) - .map((msg) => ({ - role: msg.role, - content: msg.content, - })); - - this.introductionStateCallback(recentMessages) - .then((state) => { - if (state) { - this.introductionState = state; - if (this.websocket) { - this.websocket.send( - JSON.stringify({ - type: 'introduction_state_updated', - introduction_state: this.introductionState, - timestamp: Date.now(), - }) - ); - } - } - }) - .catch((error) => { - console.error( - 'Error in introduction-state callback:', - error - ); - }); - } - } - }, - - // Handle any other type - default: (data: unknown) => { - console.log( - `VAD Unknown/unhandled chunk type: ${chunk.typeName}`, - data - ); - }, - }); - - // Check again after processing each chunk - if (this.isProcessingCancelled) { - console.log('Processing cancelled after chunk processing'); - break; - } - } - } catch (error) { - console.error('Error processing VAD speech segment:', error); - } finally { - this.isProcessing = false; - - // If we have pending segments (user spoke while we were processing), process them now - if ( - this.pendingSpeechSegments.length > 0 && - !this.isProcessingCancelled - ) { - console.log( - 'Found pending segments after processing, processing them now' - ); - - // Combine all pending segments - const totalLength = this.pendingSpeechSegments.reduce( - (sum, seg) => sum + seg.length, - 0 - ); - const combinedSegment = new Float32Array(totalLength); - let offset = 0; - for (const seg of this.pendingSpeechSegments) { - combinedSegment.set(seg, offset); - offset += seg.length; - } - - // Clear pending segments - this.pendingSpeechSegments = []; - - // Process the combined segments recursively - await this.processVADSpeechSegment(combinedSegment); - } - } - } - - reset() { - // Clear conversation state - this.conversationState = { - messages: [], - }; - // Clear pending segments - this.pendingSpeechSegments = []; - // Reset introduction state - this.introductionState = { - name: '', - level: '', - goal: '', - timestamp: '', - }; - // Cancel any ongoing processing - this.isProcessingCancelled = true; - console.log('AudioProcessor: Conversation reset'); - } - - async destroy() { - if (this.vad) { - try { - this.vad.destroy(); - } catch (error) { - console.error('Error destroying VAD:', error); - // Continue with cleanup even if destroy fails - } - this.vad = null; - } - } -} diff --git a/backend/helpers/connection-manager.ts b/backend/helpers/connection-manager.ts new file mode 100644 index 0000000..f395862 --- /dev/null +++ b/backend/helpers/connection-manager.ts @@ -0,0 +1,580 @@ +/** + * ConnectionManager - Manages WebSocket connections and graph execution + * + * This replaces the AudioProcessor for Inworld Runtime 0.9. + * Key differences from AudioProcessor: + * - Uses MultimodalStreamManager to feed audio to a long-running graph + * - VAD is handled inside the graph by AssemblyAI (not external Silero) + * - Graph runs continuously for the session duration + */ + +import { WebSocket } from 'ws'; +import { GraphTypes } from '@inworld/runtime/graph'; + +import { ConversationGraphWrapper } from '../components/graphs/conversation-graph.js'; +import { MultimodalStreamManager } from '../components/audio/multimodal_stream_manager.js'; +import { decodeBase64ToFloat32 } from '../components/audio/audio_utils.js'; +import { ConnectionsMap, INPUT_SAMPLE_RATE, TTS_SAMPLE_RATE } from '../types/index.js'; +import { + getLanguageConfig, + DEFAULT_LANGUAGE_CODE, + LanguageConfig, +} from '../config/languages.js'; +import type { IntroductionState } from './introduction-state-processor.js'; + +export class ConnectionManager { + private sessionId: string; + private ws: WebSocket; + private graphWrapper: ConversationGraphWrapper; + private multimodalStreamManager: MultimodalStreamManager; + private connections: ConnectionsMap; + private graphExecution: Promise | null = null; + private isDestroyed = false; + private languageCode: string; + private languageConfig: LanguageConfig; + + // Callbacks for flashcard and introduction state processing + private flashcardCallback: + | ((messages: Array<{ role: string; content: string }>) => Promise) + | null = null; + private introductionStateCallback: + | (( + messages: Array<{ role: string; content: string }> + ) => Promise) + | null = null; + + constructor( + sessionId: string, + ws: WebSocket, + graphWrapper: ConversationGraphWrapper, + connections: ConnectionsMap, + languageCode: string = DEFAULT_LANGUAGE_CODE + ) { + this.sessionId = sessionId; + this.ws = ws; + this.graphWrapper = graphWrapper; + this.connections = connections; + this.languageCode = languageCode; + this.languageConfig = getLanguageConfig(languageCode); + this.multimodalStreamManager = new MultimodalStreamManager(); + + // Initialize connection state + this.connections[sessionId] = { + ws: ws, + state: { + interactionId: '', + messages: [], + userName: '', + targetLanguage: this.languageConfig.name, + languageCode: languageCode, + voiceId: this.languageConfig.ttsConfig.speakerId, + introductionState: { name: '', level: '', goal: '', timestamp: '' }, + output_modalities: ['audio', 'text'], + }, + multimodalStreamManager: this.multimodalStreamManager, + onSpeechDetected: (interactionId) => + this.handleSpeechDetected(interactionId), + onPartialTranscript: (text, interactionId) => + this.handlePartialTranscript(text, interactionId), + }; + + console.log( + `[ConnectionManager] Created for session ${sessionId} with language ${this.languageConfig.name}` + ); + } + + /** + * Start the long-running graph execution + */ + async start(): Promise { + console.log(`[ConnectionManager] Starting graph for session ${this.sessionId}`); + + // Create the multimodal stream generator + const multimodalStream = this.createMultimodalStreamGenerator(); + + // Start graph execution (runs in background) + this.graphExecution = this.executeGraph(multimodalStream); + + // Don't await - the graph runs continuously + this.graphExecution.catch((error) => { + if (!this.isDestroyed) { + console.error(`[ConnectionManager] Graph execution error:`, error); + } + }); + } + + /** + * Create an async generator that yields multimodal content from the stream manager + */ + private async *createMultimodalStreamGenerator(): AsyncGenerator { + for await (const content of this.multimodalStreamManager.createStream()) { + yield content; + } + } + + /** + * Execute the graph with the multimodal stream + */ + private async executeGraph( + stream: AsyncGenerator + ): Promise { + const connection = this.connections[this.sessionId]; + if (!connection) { + throw new Error(`No connection found for session ${this.sessionId}`); + } + + // Tag the stream for the runtime + const taggedStream = Object.assign(stream, { + type: 'MultimodalContent', + }); + + console.log(`[ConnectionManager] Starting graph execution for ${this.sessionId}`); + + const { outputStream } = await this.graphWrapper.graph.start(taggedStream, { + executionId: this.sessionId, + dataStoreContent: { + sessionId: this.sessionId, + state: connection.state, + }, + userContext: { + attributes: { + languageCode: this.languageCode, + language: this.languageConfig.name, + }, + targetingKey: this.sessionId, + }, + }); + + // Store the output stream for potential cancellation + connection.currentAudioExecutionStream = outputStream; + + // Process graph outputs + try { + for await (const result of outputStream) { + if (this.isDestroyed) break; + await this.processGraphOutput(result); + } + } catch (error) { + if (!this.isDestroyed) { + console.error(`[ConnectionManager] Error processing output:`, error); + } + } finally { + connection.currentAudioExecutionStream = undefined; + } + + console.log(`[ConnectionManager] Graph execution completed for ${this.sessionId}`); + } + + /** + * Process a single output from the graph + */ + private async processGraphOutput(result: unknown): Promise { + const connection = this.connections[this.sessionId]; + if (!connection) return; + + let transcription = ''; + let llmResponse = ''; + + try { + // Cast to any to work around strict typing issues with processResponse handlers + // The handlers receive typed data at runtime even though the type system says unknown + const resultWithProcess = result as { processResponse: (handlers: Record Promise | void>) => Promise }; + await resultWithProcess.processResponse({ + // Handle string output (transcription from proxy node) + string: (data: unknown) => { + transcription = String(data); + if (transcription.trim()) { + console.log(`[ConnectionManager] Transcription: "${transcription}"`); + this.sendToClient({ + type: 'transcription', + text: transcription.trim(), + timestamp: Date.now(), + }); + + // Trigger introduction state extraction + this.triggerIntroductionStateExtraction(); + } + }, + + // Handle Custom data (transcription from transcript extractor) + // InteractionInfo has: sessionId, interactionId, text, interactionComplete + Custom: async (customData: unknown) => { + const data = customData as { text?: string; interactionId?: string; interactionComplete?: boolean }; + // Only send final transcriptions (interactionComplete=true) to avoid duplicates + if (data.text && data.interactionComplete) { + transcription = data.text; + console.log(`[ConnectionManager] Transcription (final): "${transcription}"`); + this.sendToClient({ + type: 'transcription', + text: transcription.trim(), + timestamp: Date.now(), + }); + this.triggerIntroductionStateExtraction(); + } + }, + + // Handle LLM response stream + ContentStream: async (streamData: unknown) => { + const stream = streamData as GraphTypes.ContentStream; + console.log('[ConnectionManager] Processing LLM ContentStream...'); + let currentResponse = ''; + + for await (const chunk of stream) { + if (this.isDestroyed) break; + if (chunk.text) { + currentResponse += chunk.text; + this.sendToClient({ + type: 'llm_response_chunk', + text: chunk.text, + timestamp: Date.now(), + }); + } + } + + if (currentResponse.trim()) { + llmResponse = currentResponse; + console.log( + `[ConnectionManager] LLM Response complete: "${llmResponse.substring(0, 50)}..."` + ); + this.sendToClient({ + type: 'llm_response_complete', + text: llmResponse.trim(), + timestamp: Date.now(), + }); + } + }, + + // Handle TTS output stream + TTSOutputStream: async (ttsData: unknown) => { + const ttsStream = ttsData as GraphTypes.TTSOutputStream; + console.log('[ConnectionManager] Processing TTS stream...'); + let isFirstChunk = true; + + for await (const chunk of ttsStream) { + if (this.isDestroyed) break; + if (chunk.audio?.data) { + // Log sample rate on first chunk + if (isFirstChunk) { + console.log(`[ConnectionManager] TTS audio: sampleRate=${chunk.audio.sampleRate || TTS_SAMPLE_RATE}, bytes=${Array.isArray(chunk.audio.data) ? chunk.audio.data.length : 'N/A'}`); + } + + // Convert audio to base64 for WebSocket transmission + // Use TTS_SAMPLE_RATE as fallback (not INPUT_SAMPLE_RATE which is for microphone input) + const audioResult = this.convertAudioToBase64(chunk.audio); + if (audioResult) { + this.sendToClient({ + type: 'audio_stream', + audio: audioResult.base64, + audioFormat: audioResult.format, + sampleRate: chunk.audio.sampleRate || TTS_SAMPLE_RATE, + text: chunk.text || '', + isFirstChunk: isFirstChunk, + timestamp: Date.now(), + }); + isFirstChunk = false; + } + } + } + + // Send completion signal + console.log('[ConnectionManager] TTS stream complete'); + this.sendToClient({ + type: 'audio_stream_complete', + timestamp: Date.now(), + }); + + // Send conversation update + this.sendToClient({ + type: 'conversation_update', + messages: connection.state.messages, + timestamp: Date.now(), + }); + + // Trigger flashcard generation after TTS completes + this.triggerFlashcardGeneration(); + }, + + // Handle errors + error: async (error: unknown) => { + const err = error as { message?: string }; + console.error('[ConnectionManager] Graph error:', err); + if (!err.message?.includes('recognition produced no text')) { + this.sendToClient({ + type: 'error', + message: err.message || 'Unknown error', + timestamp: Date.now(), + }); + } + }, + + // Default handler for unknown types + default: (_data: unknown) => { + // console.log('[ConnectionManager] Unknown output type:', data); + }, + }); + } catch (error) { + console.error('[ConnectionManager] Error processing graph output:', error); + } + } + + /** + * Add an audio chunk from the WebSocket + */ + addAudioChunk(base64Audio: string): void { + if (this.isDestroyed) return; + + try { + // Decode base64 to Float32Array + const float32Data = decodeBase64ToFloat32(base64Audio); + + // Push to multimodal stream + this.multimodalStreamManager.pushAudio({ + data: Array.from(float32Data), + sampleRate: INPUT_SAMPLE_RATE, + }); + } catch (error) { + console.error('[ConnectionManager] Error adding audio chunk:', error); + } + } + + /** + * Handle speech detected event from AssemblyAI + */ + private handleSpeechDetected(interactionId: string): void { + console.log(`[ConnectionManager] Speech detected: ${interactionId}`); + this.sendToClient({ + type: 'speech_detected', + interactionId, + data: { text: '' }, + timestamp: Date.now(), + }); + + // Could also send interrupt signal here if needed + this.sendToClient({ + type: 'interrupt', + reason: 'speech_start', + }); + } + + /** + * Handle partial transcript from AssemblyAI + */ + private handlePartialTranscript(text: string, interactionId: string): void { + this.sendToClient({ + type: 'partial_transcript', + text, + interactionId, + timestamp: Date.now(), + }); + } + + /** + * Convert audio data to base64 string for WebSocket transmission + * Inworld TTS returns Float32 PCM in [-1.0, 1.0] range - send directly to preserve quality + */ + private convertAudioToBase64(audio: { + data?: string | number[] | Float32Array; + sampleRate?: number; + }): { base64: string; format: 'float32' | 'int16' } | null { + if (!audio.data) return null; + + if (typeof audio.data === 'string') { + // Already base64 - assume Int16 format for backwards compatibility + return { base64: audio.data, format: 'int16' }; + } + + // Inworld SDK returns audio.data as an array of raw bytes (0-255) + // These bytes ARE the Float32 PCM data in IEEE 754 format (4 bytes per sample) + // Simply pass them through as-is, and frontend interprets as Float32Array + const audioBuffer = Array.isArray(audio.data) + ? Buffer.from(audio.data) // Treat each array element as a byte + : Buffer.from(audio.data.buffer, audio.data.byteOffset, audio.data.byteLength); + + return { + base64: audioBuffer.toString('base64'), + format: 'float32', // Frontend will interpret bytes as Float32Array + }; + } + + /** + * Send message to WebSocket client + */ + private sendToClient(message: Record): void { + if (this.ws.readyState === WebSocket.OPEN) { + try { + this.ws.send(JSON.stringify(message)); + } catch (error) { + console.error('[ConnectionManager] Error sending to client:', error); + } + } + } + + /** + * Trigger flashcard generation + */ + private triggerFlashcardGeneration(): void { + if (!this.flashcardCallback) return; + + const connection = this.connections[this.sessionId]; + if (!connection) return; + + const recentMessages = connection.state.messages.slice(-6).map((m) => ({ + role: m.role, + content: m.content, + })); + + this.flashcardCallback(recentMessages).catch((error) => { + console.error('[ConnectionManager] Flashcard generation error:', error); + }); + } + + /** + * Trigger introduction state extraction + */ + private triggerIntroductionStateExtraction(): void { + if (!this.introductionStateCallback) return; + + const connection = this.connections[this.sessionId]; + if (!connection) return; + + // Skip if introduction is already complete + const intro = connection.state.introductionState; + if (intro.name && intro.level && intro.goal) { + return; + } + + const recentMessages = connection.state.messages.slice(-6).map((m) => ({ + role: m.role, + content: m.content, + })); + + this.introductionStateCallback(recentMessages) + .then((state) => { + if (state) { + connection.state.introductionState = state; + this.sendToClient({ + type: 'introduction_state_updated', + introduction_state: state, + timestamp: Date.now(), + }); + } + }) + .catch((error) => { + console.error('[ConnectionManager] Introduction state error:', error); + }); + } + + // ============================================================ + // Public API (compatible with AudioProcessor) + // ============================================================ + + setFlashcardCallback( + callback: ( + messages: Array<{ role: string; content: string }> + ) => Promise + ): void { + this.flashcardCallback = callback; + } + + setIntroductionStateCallback( + callback: ( + messages: Array<{ role: string; content: string }> + ) => Promise + ): void { + this.introductionStateCallback = callback; + } + + getConversationState(): { messages: Array<{ role: string; content: string; timestamp: string }> } { + const connection = this.connections[this.sessionId]; + return { + messages: + connection?.state.messages.map((m) => ({ + role: m.role, + content: m.content, + timestamp: m.timestamp || new Date().toISOString(), + })) || [], + }; + } + + getIntroductionState(): IntroductionState { + const connection = this.connections[this.sessionId]; + return ( + connection?.state.introductionState || { + name: '', + level: '', + goal: '', + timestamp: '', + } + ); + } + + getLanguageCode(): string { + return this.languageCode; + } + + getLanguageConfig(): LanguageConfig { + return this.languageConfig; + } + + /** + * Change language for this session + */ + setLanguage(newLanguageCode: string): void { + if (this.languageCode === newLanguageCode) return; + + console.log( + `[ConnectionManager] Changing language from ${this.languageCode} to ${newLanguageCode}` + ); + + this.languageCode = newLanguageCode; + this.languageConfig = getLanguageConfig(newLanguageCode); + + const connection = this.connections[this.sessionId]; + if (connection) { + connection.state.languageCode = newLanguageCode; + connection.state.targetLanguage = this.languageConfig.name; + connection.state.voiceId = this.languageConfig.ttsConfig.speakerId; + } + + console.log( + `[ConnectionManager] Language changed to ${this.languageConfig.name}` + ); + } + + /** + * Reset conversation state + */ + reset(): void { + const connection = this.connections[this.sessionId]; + if (connection) { + connection.state.messages = []; + connection.state.introductionState = { + name: '', + level: '', + goal: '', + timestamp: '', + }; + connection.state.interactionId = ''; + } + console.log('[ConnectionManager] Conversation reset'); + } + + /** + * Clean up resources + */ + async destroy(): Promise { + console.log(`[ConnectionManager] Destroying session ${this.sessionId}`); + this.isDestroyed = true; + + // End the multimodal stream + this.multimodalStreamManager.end(); + + // Close AssemblyAI session + await this.graphWrapper.assemblyAINode.closeSession(this.sessionId); + + // Remove from connections map + delete this.connections[this.sessionId]; + + console.log(`[ConnectionManager] Session ${this.sessionId} destroyed`); + } +} diff --git a/backend/helpers/silero-vad.ts b/backend/helpers/silero-vad.ts deleted file mode 100644 index 83c0eee..0000000 --- a/backend/helpers/silero-vad.ts +++ /dev/null @@ -1,287 +0,0 @@ -import { EventEmitter } from 'events'; -import { VADFactory } from '@inworld/runtime/primitives/vad'; -import { DeviceRegistry, DeviceType } from '@inworld/runtime/core'; -import { AudioBuffer, AudioChunk } from './audio-buffer.js'; - -export interface VADConfig { - modelPath: string; - threshold: number; - minSpeechDuration: number; // seconds - minSilenceDuration: number; // seconds - speechResetSilenceDuration: number; // grace period before resetting speech timer - minVolume: number; // minimum RMS volume for speech (0.0-1.0) - sampleRate: number; -} - -export interface VADResult { - isSpeech: boolean; - confidence: number; - timestamp: number; -} - -// VAD type from the runtime library - using unknown since the exact type isn't exported -type VADInstance = { - detectVoiceActivity: (config: { - data: number[]; - sampleRate: number; - }) => Promise; - destroy: () => void; -}; - -export class SileroVAD extends EventEmitter { - private vad: VADInstance | null = null; - private config: VADConfig; - private audioBuffer: AudioBuffer; - private accumulatedSamples: Float32Array[] = []; - private isInitialized = false; - private isProcessingVAD = false; // Prevent concurrent VAD calls - - // Simplified state tracking following working example pattern - private isCapturingSpeech = false; - private speechBuffer: number[] = []; - private pauseDuration = 0; - private readonly FRAME_PER_BUFFER = 1024; - private readonly INPUT_SAMPLE_RATE = 16000; - - constructor(config: VADConfig) { - super(); - this.config = config; - this.audioBuffer = new AudioBuffer(20, config.sampleRate); - - // Listen to audio chunks from buffer - this.audioBuffer.on('audioChunk', this.processAudioChunk.bind(this)); - } - - async initialize(): Promise { - if (this.isInitialized) { - console.log('SileroVAD: Already initialized'); - return; - } - - try { - console.log( - 'SileroVAD: Starting initialization with model path:', - this.config.modelPath - ); - - // Try to find CUDA device - const availableDevices = DeviceRegistry.getAvailableDevices(); - console.log( - 'SileroVAD: Available devices:', - availableDevices.map((d) => d.getType()) - ); - - const cudaDevice = availableDevices.find( - (device) => device.getType() === DeviceType.CUDA - ); - console.log('SileroVAD: Using CUDA device:', !!cudaDevice); - - // Create local VAD instance - console.log('SileroVAD: Creating VAD instance...'); - this.vad = await VADFactory.createLocal({ - modelPath: this.config.modelPath, - device: cudaDevice, - }); - - this.isInitialized = true; - console.log('SileroVAD: Initialization complete'); - } catch (error) { - console.error('SileroVAD: Failed to initialize:', error); - throw error; - } - } - - addAudioData(base64Data: string): void { - try { - // Convert base64 to buffer (same as working example needs) - const binaryString = Buffer.from(base64Data, 'base64').toString('binary'); - const bytes = new Uint8Array(binaryString.length); - - for (let i = 0; i < binaryString.length; i++) { - bytes[i] = binaryString.charCodeAt(i); - } - - // Convert to Int16Array then normalize to Float32 range [-1, 1] - const int16Array = new Int16Array(bytes.buffer); - - // Convert to normalized Float32Array for consistent audio processing - const normalizedArray = new Float32Array(int16Array.length); - for (let i = 0; i < int16Array.length; i++) { - normalizedArray[i] = int16Array[i] / 32768.0; // Normalize to [-1, 1] range - } - - // Add to audio buffer with normalized Float32Array - this.audioBuffer.addChunk(normalizedArray); - } catch (error) { - console.error('SileroVAD: Error processing audio data:', error); - } - } - - private calculateRMSVolume(audioData: Float32Array): number { - let sumSquares = 0; - for (let i = 0; i < audioData.length; i++) { - sumSquares += audioData[i] * audioData[i]; - } - return Math.sqrt(sumSquares / audioData.length); - } - - reset(): void { - // Reset all state when interruption occurs - this.accumulatedSamples = []; - this.isProcessingVAD = false; - this.isCapturingSpeech = false; - this.speechBuffer = []; - this.pauseDuration = 0; - } - - private async processAudioChunk(chunk: AudioChunk): Promise { - if (!this.isInitialized || !this.vad) { - return; - } - - // Skip if already processing to prevent concurrent VAD calls - if (this.isProcessingVAD) { - return; - } - - // Accumulate samples until we have enough for VAD processing - this.accumulatedSamples.push(chunk.data); - - // Process when we have enough samples (using FRAME_PER_BUFFER like working example) - const totalSamples = this.accumulatedSamples.reduce( - (sum, arr) => sum + arr.length, - 0 - ); - if (totalSamples >= this.FRAME_PER_BUFFER) { - this.isProcessingVAD = true; // Set flag before async operation - try { - // Combine accumulated samples - const combinedAudio = new Float32Array(totalSamples); - let offset = 0; - for (const samples of this.accumulatedSamples) { - combinedAudio.set(samples, offset); - offset += samples.length; - } - - // Calculate RMS volume for noise filtering - const rmsVolume = this.calculateRMSVolume(combinedAudio); - - // Convert normalized Float32Array back to integer array for VAD model - const integerArray: number[] = []; - for (let i = 0; i < combinedAudio.length; i++) { - // Convert back to Int16 range for VAD processing - integerArray.push(Math.round(combinedAudio[i] * 32768)); - } - - // Process with Silero VAD - const result = await this.vad.detectVoiceActivity({ - data: integerArray, - sampleRate: this.config.sampleRate, - }); - - // Following working example: -1 = no voice activity, anything else = voice activity - // Add volume filtering to the working pattern - const hasVoiceActivity = - result !== -1 && rmsVolume >= this.config.minVolume; - - // Process using simplified state machine from working example - await this.processVADResult(hasVoiceActivity, integerArray, rmsVolume); - - // Clear accumulated samples - this.accumulatedSamples = []; - } catch (error) { - console.error('VAD processing error:', error); - this.accumulatedSamples = []; // Clear on error - } finally { - this.isProcessingVAD = false; // Always reset flag - } - } - } - - private async processVADResult( - hasVoiceActivity: boolean, - audioChunk: number[], - volume: number - ): Promise { - // Following the working example pattern exactly - if (this.isCapturingSpeech) { - this.speechBuffer.push(...audioChunk); - if (!hasVoiceActivity) { - // Already capturing speech but new chunk has no voice activity - this.pauseDuration += - (audioChunk.length * 1000) / this.INPUT_SAMPLE_RATE; // ms - - if (this.pauseDuration > this.config.minSilenceDuration * 1000) { - // Convert to ms - this.isCapturingSpeech = false; - - const speechDuration = - (this.speechBuffer.length * 1000) / this.INPUT_SAMPLE_RATE; // ms - - if (speechDuration > this.config.minSpeechDuration * 1000) { - // Convert to ms - await this.processCapturedSpeech(); - } else { - // Speech was too short, but still emit speechEnd to notify that VAD stopped detecting - this.emit('speechEnd', { - timestamp: Date.now() / 1000, - speechSegment: null, // No segment to process - speechDuration: speechDuration / 1000, - samplesCount: this.speechBuffer.length, - }); - } - - // Reset for next speech capture - this.speechBuffer = []; - this.pauseDuration = 0; - } - } else { - // Already capturing speech and new chunk has voice activity - this.pauseDuration = 0; - } - } else { - if (hasVoiceActivity) { - // Not capturing speech but new chunk has voice activity - start capturing - this.isCapturingSpeech = true; - this.speechBuffer = [...audioChunk]; // Start fresh - this.pauseDuration = 0; - - this.emit('speechStart', { - timestamp: Date.now() / 1000, - volume, - }); - } - // Not capturing speech and new chunk has no voice activity - do nothing - } - } - - private async processCapturedSpeech(): Promise { - if (this.speechBuffer.length === 0) return; - - // Convert integer array back to Float32Array for processing - const speechSegment = new Float32Array(this.speechBuffer.length); - for (let i = 0; i < this.speechBuffer.length; i++) { - speechSegment[i] = this.speechBuffer[i] / 32768.0; // Normalize back to [-1, 1] - } - - const speechDuration = - (this.speechBuffer.length * 1000) / this.INPUT_SAMPLE_RATE; - - this.emit('speechEnd', { - timestamp: Date.now() / 1000, - speechSegment, - speechDuration: speechDuration / 1000, // Convert back to seconds - samplesCount: this.speechBuffer.length, - }); - } - - destroy(): void { - if (this.vad) { - this.vad.destroy(); - this.vad = null; - } - - this.audioBuffer.clear(); - this.isInitialized = false; - } -} diff --git a/backend/server.ts b/backend/server.ts index 23af9a7..110f7e2 100644 --- a/backend/server.ts +++ b/backend/server.ts @@ -1,3 +1,14 @@ +/** + * Language Learning Server - Inworld Runtime 0.9 + * + * This server uses a long-running circular graph with AssemblyAI for VAD/STT. + * Key components: + * - ConversationGraphWrapper: The main graph that processes audio → STT → LLM → TTS + * - ConnectionManager: Manages WebSocket connections and feeds audio to the graph + * - FlashcardProcessor: Generates flashcards from conversations + * - IntroductionStateProcessor: Extracts user info (name, level, goal) + */ + // Load environment variables FIRST import dotenv from 'dotenv'; dotenv.config(); @@ -14,14 +25,16 @@ import { createServer } from 'http'; import { WebSocketServer } from 'ws'; import { telemetry, stopInworldRuntime } from '@inworld/runtime'; import { MetricType } from '@inworld/runtime/telemetry'; -import { UserContextInterface } from '@inworld/runtime/graph'; -// Import our audio processor -import { AudioProcessor } from './helpers/audio-processor.js'; +// Import new 0.9 components +import { ConversationGraphWrapper } from './components/graphs/conversation-graph.js'; +import { ConnectionManager } from './helpers/connection-manager.js'; +import { ConnectionsMap } from './types/index.js'; + +// Import existing components (still compatible) import { FlashcardProcessor } from './helpers/flashcard-processor.js'; -import { AnkiExporter } from './helpers/anki-exporter.js'; +// import { AnkiExporter } from './helpers/anki-exporter.js'; import { IntroductionStateProcessor } from './helpers/introduction-state-processor.js'; -import { getConversationGraph } from './graphs/conversation-graph.js'; import { getLanguageConfig, getLanguageOptions, @@ -37,7 +50,32 @@ app.use(express.json()); const PORT = process.env.PORT || 3000; -// Initialize telemetry once at startup +// ============================================================ +// Global State +// ============================================================ + +// Shared connections map (used by graph nodes) +const connections: ConnectionsMap = {}; + +// Graph wrapper (created once, shared across all connections) +let graphWrapper: ConversationGraphWrapper | null = null; + +// Connection managers per WebSocket +const connectionManagers = new Map(); +const flashcardProcessors = new Map(); +const introductionStateProcessors = new Map(); +const connectionAttributes = new Map< + string, + { timezone?: string; userId?: string; languageCode?: string } +>(); + +// Shutdown flag +let isShuttingDown = false; + +// ============================================================ +// Initialize Telemetry +// ============================================================ + try { const telemetryApiKey = process.env.INWORLD_API_KEY; if (telemetryApiKey) { @@ -54,84 +92,88 @@ try { unit: 'clicks', }); } else { - console.warn( - '[Telemetry] INWORLD_API_KEY not set. Metrics will be disabled.' - ); + console.warn('[Telemetry] INWORLD_API_KEY not set. Metrics will be disabled.'); } } catch (err) { console.error('[Telemetry] Initialization failed:', err); } -// Store audio processors per connection -const audioProcessors = new Map(); -const flashcardProcessors = new Map(); -const introductionStateProcessors = new Map< - string, - IntroductionStateProcessor ->(); -// Store lightweight per-connection attributes provided by the client (e.g., timezone, userId, languageCode) -const connectionAttributes = new Map< - string, - { timezone?: string; userId?: string; languageCode?: string } ->(); +// ============================================================ +// Initialize Graph +// ============================================================ -/** - * Get or create a conversation graph for the specified language - */ -function getGraphForLanguage(languageCode: string) { - const apiKey = process.env.INWORLD_API_KEY || ''; - return getConversationGraph({ apiKey }, languageCode); +async function initializeGraph(): Promise { + const assemblyAIApiKey = process.env.ASSEMBLY_AI_API_KEY; + if (!assemblyAIApiKey) { + throw new Error('ASSEMBLY_AI_API_KEY environment variable is required'); + } + + console.log('[Server] Creating conversation graph...'); + graphWrapper = ConversationGraphWrapper.create({ + assemblyAIApiKey, + connections, + defaultLanguageCode: DEFAULT_LANGUAGE_CODE, + }); + console.log('[Server] Conversation graph created successfully'); } -// WebSocket handling with audio processing -wss.on('connection', (ws) => { +// ============================================================ +// WebSocket Connection Handler +// ============================================================ + +wss.on('connection', async (ws) => { + if (!graphWrapper) { + console.error('[Server] Graph not initialized, rejecting connection'); + ws.close(1011, 'Server not ready'); + return; + } + const connectionId = `conn_${Date.now()}_${Math.random().toString(36).substr(2, 9)}`; - console.log(`WebSocket connection established: ${connectionId}`); + console.log(`[Server] WebSocket connection established: ${connectionId}`); // Default language is Spanish const defaultLanguageCode = DEFAULT_LANGUAGE_CODE; - // Create processors with default language - const graph = getGraphForLanguage(defaultLanguageCode); - const audioProcessor = new AudioProcessor(graph, ws, defaultLanguageCode); - const flashcardProcessor = new FlashcardProcessor(defaultLanguageCode); - const introductionStateProcessor = new IntroductionStateProcessor( + // Create connection manager (replaces AudioProcessor) + const connectionManager = new ConnectionManager( + connectionId, + ws, + graphWrapper, + connections, defaultLanguageCode ); - // Register processors - audioProcessors.set(connectionId, audioProcessor); + // Create flashcard and introduction state processors + const flashcardProcessor = new FlashcardProcessor(defaultLanguageCode); + const introductionStateProcessor = new IntroductionStateProcessor(defaultLanguageCode); + + // Store processors + connectionManagers.set(connectionId, connectionManager); flashcardProcessors.set(connectionId, flashcardProcessor); introductionStateProcessors.set(connectionId, introductionStateProcessor); connectionAttributes.set(connectionId, { languageCode: defaultLanguageCode }); // Set up flashcard generation callback - audioProcessor.setFlashcardCallback(async (messages) => { - // Skip flashcard generation if we're shutting down + connectionManager.setFlashcardCallback(async (messages) => { if (isShuttingDown) { - console.log('Skipping flashcard generation - server is shutting down'); + console.log('[Server] Skipping flashcard generation - shutting down'); return; } try { - // Build UserContext for flashcard graph execution const introState = introductionStateProcessor.getState(); const attrs = connectionAttributes.get(connectionId) || {}; const userAttributes: Record = { timezone: attrs.timezone || '', }; - userAttributes.name = - (introState?.name && introState.name.trim()) || 'unknown'; - userAttributes.level = - (introState?.level && (introState.level as string)) || 'unknown'; - userAttributes.goal = - (introState?.goal && introState.goal.trim()) || 'unknown'; - - // Prefer a stable targeting key from client if available, fallback to connectionId + userAttributes.name = introState?.name?.trim() || 'unknown'; + userAttributes.level = (introState?.level as string) || 'unknown'; + userAttributes.goal = introState?.goal?.trim() || 'unknown'; + const targetingKey = attrs.userId || connectionId; - const userContext: UserContextInterface = { + const userContext = { attributes: userAttributes, - targetingKey: targetingKey, + targetingKey, }; const flashcards = await flashcardProcessor.generateFlashcards( @@ -143,117 +185,88 @@ wss.on('connection', (ws) => { ws.send( JSON.stringify({ type: 'flashcards_generated', - flashcards: flashcards, + flashcards, }) ); } - } catch (error: unknown) { - // Suppress "Environment closed" errors during shutdown - they're expected - const err = error as { context?: string }; - if (isShuttingDown && err?.context === 'Environment closed') { - console.log('Flashcard generation cancelled due to shutdown'); - } else { - console.error('Error generating flashcards:', error); + } catch (error) { + if (!isShuttingDown) { + console.error('[Server] Error generating flashcards:', error); } } }); - // Set up introduction-state extraction callback (runs until complete) - audioProcessor.setIntroductionStateCallback(async (messages) => { + // Set up introduction state extraction callback + connectionManager.setIntroductionStateCallback(async (messages) => { try { const currentState = introductionStateProcessor.getState(); - console.log( - 'Server - Current introduction state before update:', - currentState - ); - console.log( - 'Server - Is complete?', - introductionStateProcessor.isComplete() - ); - if (introductionStateProcessor.isComplete()) { - console.log( - 'Server - Introduction state is complete, returning:', - currentState - ); return currentState; } const state = await introductionStateProcessor.update(messages); - console.log('Server - Updated introduction state:', state); return state; } catch (error) { - console.error('Error generating introduction state:', error); + console.error('[Server] Error extracting introduction state:', error); return null; } }); + // Start the graph for this connection + try { + await connectionManager.start(); + console.log(`[Server] Graph started for connection ${connectionId}`); + } catch (error) { + console.error(`[Server] Failed to start graph for ${connectionId}:`, error); + ws.close(1011, 'Failed to start audio processing'); + return; + } + + // Handle incoming messages ws.on('message', (data) => { try { const message = JSON.parse(data.toString()); if (message.type === 'audio_chunk' && message.audio_data) { // Process audio chunk - audioProcessor.addAudioChunk(message.audio_data); + connectionManager.addAudioChunk(message.audio_data); } else if (message.type === 'reset_flashcards') { - // Reset flashcards for new conversation const processor = flashcardProcessors.get(connectionId); if (processor) { processor.reset(); } } else if (message.type === 'restart_conversation') { - // Reset conversation state, introduction state, and flashcards - const audioProc = audioProcessors.get(connectionId); - const flashcardProc = flashcardProcessors.get(connectionId); - const introStateProc = introductionStateProcessors.get(connectionId); - - if (audioProc) { - audioProc.reset(); - } - if (flashcardProc) { - flashcardProc.reset(); - } - if (introStateProc) { - introStateProc.reset(); - } - - console.log(`Conversation restarted for connection: ${connectionId}`); + // Reset all state + connectionManager.reset(); + flashcardProcessors.get(connectionId)?.reset(); + introductionStateProcessors.get(connectionId)?.reset(); + console.log(`[Server] Conversation restarted for ${connectionId}`); } else if (message.type === 'set_language') { // Handle language change const newLanguageCode = message.languageCode || DEFAULT_LANGUAGE_CODE; const attrs = connectionAttributes.get(connectionId) || {}; - // Only process if language actually changed if (attrs.languageCode !== newLanguageCode) { console.log( - `Language change requested for ${connectionId}: ${attrs.languageCode} -> ${newLanguageCode}` + `[Server] Language change: ${attrs.languageCode} -> ${newLanguageCode}` ); - // Update stored language attrs.languageCode = newLanguageCode; connectionAttributes.set(connectionId, attrs); - // Get the new language config const languageConfig = getLanguageConfig(newLanguageCode); - // Update all processors with new language - const audioProc = audioProcessors.get(connectionId); - const flashcardProc = flashcardProcessors.get(connectionId); - const introStateProc = introductionStateProcessors.get(connectionId); - - if (flashcardProc) { - flashcardProc.setLanguage(newLanguageCode); - } - if (introStateProc) { - introStateProc.setLanguage(newLanguageCode); - } - if (audioProc) { - // Audio processor needs a new graph for the new language - const newGraph = getGraphForLanguage(newLanguageCode); - audioProc.setLanguage(newLanguageCode, newGraph); - } - - // Send confirmation to frontend + // Update all processors + flashcardProcessors.get(connectionId)?.setLanguage(newLanguageCode); + introductionStateProcessors.get(connectionId)?.setLanguage(newLanguageCode); + connectionManager.setLanguage(newLanguageCode); + + // Reset conversation on language change + connectionManager.reset(); + flashcardProcessors.get(connectionId)?.reset(); + introductionStateProcessors.get(connectionId)?.reset(); + + // Send confirmation ws.send( JSON.stringify({ type: 'language_changed', @@ -263,19 +276,12 @@ wss.on('connection', (ws) => { }) ); - console.log(`Language changed to ${languageConfig.name} for ${connectionId}`); + console.log(`[Server] Language changed to ${languageConfig.name}`); } } else if (message.type === 'user_context') { - const timezone = - message.timezone || - (message.data && message.data.timezone) || - undefined; - const userId = - message.userId || (message.data && message.data.userId) || undefined; - const languageCode = - message.languageCode || - (message.data && message.data.languageCode) || - undefined; + const timezone = message.timezone || message.data?.timezone; + const userId = message.userId || message.data?.userId; + const languageCode = message.languageCode || message.data?.languageCode; const currentAttrs = connectionAttributes.get(connectionId) || {}; connectionAttributes.set(connectionId, { ...currentAttrs, @@ -298,162 +304,140 @@ wss.on('connection', (ws) => { source: 'ui', timezone: attrs.timezone || '', languageCode: attrs.languageCode || DEFAULT_LANGUAGE_CODE, - name: (introState?.name && introState.name.trim()) || 'unknown', - level: - (introState?.level && (introState.level as string)) || 'unknown', - goal: (introState?.goal && introState.goal.trim()) || 'unknown', + name: introState?.name?.trim() || 'unknown', + level: (introState?.level as string) || 'unknown', + goal: introState?.goal?.trim() || 'unknown', }); } catch (err) { - console.error('Error recording flashcard click metric:', err); + console.error('[Server] Error recording flashcard click:', err); } } else { - console.log('Received non-audio message:', message.type); + console.log('[Server] Received message type:', message.type); } } catch (error) { - console.error('Error processing message:', error); + console.error('[Server] Error processing message:', error); } }); ws.on('error', (error) => { - console.error(`WebSocket error for ${connectionId}:`, error); - // Don't crash - errors are handled by close event + console.error(`[Server] WebSocket error for ${connectionId}:`, error); }); ws.on('close', async () => { - console.log(`WebSocket connection closed: ${connectionId}`); + console.log(`[Server] WebSocket closed: ${connectionId}`); - // Clean up audio processor - const processor = audioProcessors.get(connectionId); - if (processor) { + // Clean up connection manager + const manager = connectionManagers.get(connectionId); + if (manager) { try { - await processor.destroy(); + await manager.destroy(); } catch (error) { - console.error( - `Error destroying audio processor for ${connectionId}:`, - error - ); - // Continue with cleanup even if destroy fails + console.error(`[Server] Error destroying connection manager:`, error); } - audioProcessors.delete(connectionId); + connectionManagers.delete(connectionId); } - // Clean up processors + // Clean up other processors flashcardProcessors.delete(connectionId); introductionStateProcessors.delete(connectionId); connectionAttributes.delete(connectionId); }); }); -// API endpoint for ANKI export -app.post('/api/export-anki', async (req, res) => { - try { - const { flashcards, deckName, languageCode } = req.body; - - if (!flashcards || !Array.isArray(flashcards)) { - return res.status(400).json({ error: 'Invalid flashcards data' }); - } +// ============================================================ +// API Endpoints +// ============================================================ - // Get language config for deck naming - const langCode = languageCode || DEFAULT_LANGUAGE_CODE; - const languageConfig = getLanguageConfig(langCode); - const defaultDeckName = `Aprendemo ${languageConfig.name} Cards`; - - const exporter = new AnkiExporter(); - const ankiBuffer = await exporter.exportFlashcards( - flashcards, - deckName || defaultDeckName - ); - - // Set headers for file download - const filename = `${deckName || `aprendemo_${langCode}_cards`}.apkg`; - res.setHeader('Content-Type', 'application/octet-stream'); - res.setHeader('Content-Disposition', `attachment; filename="${filename}"`); - res.setHeader('Content-Length', ankiBuffer.length); - - // Send the file - res.send(ankiBuffer); - return; - } catch (error) { - console.error('Error exporting to ANKI:', error); - res.status(500).json({ error: 'Failed to export flashcards' }); - return; - } -}); +// ANKI export endpoint - temporarily disabled +// app.post('/api/export-anki', async (req, res) => { +// ... +// }); -// API endpoint to get supported languages +// Languages endpoint app.get('/api/languages', (_req, res) => { try { const languages = getLanguageOptions(); res.json({ languages, defaultLanguage: DEFAULT_LANGUAGE_CODE }); } catch (error) { - console.error('Error getting languages:', error); + console.error('[Server] Error getting languages:', error); res.status(500).json({ error: 'Failed to get languages' }); } }); -// Serve static frontend files -// When running from dist/backend/server.js, go up two levels to project root -// When running from backend/server.ts (dev mode), go up one level to project root +// ============================================================ +// Static Files +// ============================================================ + const frontendPath = path.join(__dirname, '../../frontend'); const devFrontendPath = path.join(__dirname, '../frontend'); const staticPath = path.resolve(frontendPath); const devStaticPath = path.resolve(devFrontendPath); -// Use the path that exists const finalStaticPath = existsSync(devStaticPath) ? devStaticPath : staticPath; app.use(express.static(finalStaticPath)); -server.listen(PORT, () => { - console.log(`Server running on port ${PORT}`); -}); +// ============================================================ +// Server Startup +// ============================================================ -// Graceful shutdown - prevent multiple calls -let isShuttingDown = false; - -async function gracefulShutdown() { - if (isShuttingDown) { - return; +async function startServer(): Promise { + try { + await initializeGraph(); + server.listen(PORT, () => { + console.log(`[Server] Running on port ${PORT}`); + console.log(`[Server] Using Inworld Runtime 0.9 with AssemblyAI STT`); + }); + } catch (error) { + console.error('[Server] Failed to start:', error); + process.exit(1); } +} + +startServer(); + +// ============================================================ +// Graceful Shutdown +// ============================================================ + +async function gracefulShutdown(): Promise { + if (isShuttingDown) return; isShuttingDown = true; - console.log('Shutting down gracefully...'); + console.log('[Server] Shutting down gracefully...'); try { - // Close all WebSocket connections immediately - console.log(`Closing ${wss.clients.size} WebSocket connections...`); + // Close all WebSocket connections + console.log(`[Server] Closing ${wss.clients.size} WebSocket connections...`); wss.clients.forEach((ws) => { if (ws.readyState === ws.OPEN || ws.readyState === ws.CONNECTING) { ws.close(); } }); - // Close WebSocket server (non-blocking) wss.close(); - // Clean up processors (fire and forget - don't wait) - for (const processor of audioProcessors.values()) { - processor.destroy().catch(() => { - // Ignore errors during shutdown - }); + // Clean up connection managers + for (const manager of connectionManagers.values()) { + manager.destroy().catch(() => {}); + } + + // Clean up graph wrapper + if (graphWrapper) { + await graphWrapper.destroy(); } - // Close HTTP server (non-blocking) server.close(() => { - console.log('HTTP server closed'); + console.log('[Server] HTTP server closed'); }); - // Stop Inworld Runtime (fire and forget - don't wait) stopInworldRuntime() - .then(() => console.log('Inworld Runtime stopped')) - .catch(() => { - // Ignore errors during shutdown - }); + .then(() => console.log('[Server] Inworld Runtime stopped')) + .catch(() => {}); - console.log('Shutdown complete'); + console.log('[Server] Shutdown complete'); } catch { // Ignore errors during shutdown } - // Exit immediately - don't wait for anything process.exitCode = 0; process.exit(0); } diff --git a/backend/types/index.ts b/backend/types/index.ts new file mode 100644 index 0000000..90c7965 --- /dev/null +++ b/backend/types/index.ts @@ -0,0 +1,81 @@ +/** + * Types for the 0.9 long-running graph architecture + */ + +import { WebSocket } from 'ws'; +import type { MultimodalStreamManager } from '../components/audio/multimodal_stream_manager.js'; +import type { GraphOutputStream } from '@inworld/runtime/graph'; +import type { IntroductionState } from '../helpers/introduction-state-processor.js'; + +/** + * Chat message in conversation history + */ +export interface ChatMessage { + id: string; + role: 'user' | 'assistant' | 'system'; + content: string; + timestamp?: string; +} + +/** + * Connection state for a session + * This is the central state object that graph nodes read from + */ +export interface State { + interactionId: string; + messages: ChatMessage[]; + voiceId?: string; + // Language learning specific + userName: string; + targetLanguage: string; + languageCode: string; + introductionState: IntroductionState; + // Output modalities (for graph routing) + output_modalities: ('text' | 'audio')[]; +} + +/** + * Connection object for a WebSocket session + */ +export interface Connection { + ws: WebSocket; + state: State; + unloaded?: true; + multimodalStreamManager?: MultimodalStreamManager; + currentAudioGraphExecution?: Promise; + currentAudioExecutionStream?: GraphOutputStream; + onSpeechDetected?: (interactionId: string) => void; + onPartialTranscript?: (text: string, interactionId: string) => void; +} + +/** + * Map of session IDs to connections + */ +export type ConnectionsMap = { [sessionId: string]: Connection }; + +/** + * Text input passed between graph nodes + */ +export interface TextInput { + sessionId: string; + text: string; + interactionId: string; + voiceId?: string; +} + +/** + * Interaction info extracted from STT + */ +export interface InteractionInfo { + sessionId: string; + interactionId: string; + text: string; + interactionComplete: boolean; +} + +/** + * Config constants + */ +export const INPUT_SAMPLE_RATE = 16000; +// Inworld TTS typically outputs at 22050Hz +export const TTS_SAMPLE_RATE = 22050; diff --git a/backend/types/settings.ts b/backend/types/settings.ts new file mode 100644 index 0000000..26b0184 --- /dev/null +++ b/backend/types/settings.ts @@ -0,0 +1,47 @@ +/** + * Maps eagerness levels to AssemblyAI turn detection settings + * Based on AssemblyAI's recommended configurations for different use cases + */ + +export interface AssemblyAITurnDetectionSettings { + endOfTurnConfidenceThreshold: number; + minEndOfTurnSilenceWhenConfident: number; + maxTurnSilence: number; + description: string; +} + +/** + * Get AssemblyAI turn detection settings for a given eagerness level + * @param eagerness - The eagerness level ('low' | 'medium' | 'high') + * @returns AssemblyAI turn detection settings including threshold values and description + */ +export function getAssemblyAISettingsForEagerness( + eagerness: 'low' | 'medium' | 'high' = 'medium' +): AssemblyAITurnDetectionSettings { + switch (eagerness) { + case 'high': // Aggressive - VERY responsive + return { + endOfTurnConfidenceThreshold: 0.4, + minEndOfTurnSilenceWhenConfident: 160, + maxTurnSilence: 320, + description: + 'Aggressive - VERY quick responses, ideal for rapid Q&A (Agent Assist, IVR)', + }; + case 'medium': // Balanced (default) - good for language learning + return { + endOfTurnConfidenceThreshold: 0.4, + minEndOfTurnSilenceWhenConfident: 400, + maxTurnSilence: 1280, + description: + 'Balanced - Natural conversation flow (Customer Support, Tech Support)', + }; + case 'low': // Conservative - VERY patient + return { + endOfTurnConfidenceThreshold: 0.7, + minEndOfTurnSilenceWhenConfident: 800, + maxTurnSilence: 3000, + description: + 'Conservative - VERY patient, allows long thinking pauses (Complex inquiries)', + }; + } +} diff --git a/flashcard-graph.json b/flashcard-graph.json deleted file mode 100644 index 80be8e4..0000000 --- a/flashcard-graph.json +++ /dev/null @@ -1,95 +0,0 @@ -{ - "schema_version": "1.2.0", - "main": { - "id": "flashcard-generation-graph-es", - "nodes": [ - { - "type": "FlashcardPromptBuilderNodeType", - "id": "flashcard-prompt-builder", - "execution_config": { - "type": "NodeExecutionConfig", - "properties": { - "report_to_client": false - } - } - }, - { - "type": "TextToChatRequestNodeType", - "id": "text-to-chat-request", - "execution_config": { - "type": "NodeExecutionConfig", - "properties": { - "report_to_client": false - } - } - }, - { - "id": "llm_node", - "type": "LLMChatNode", - "execution_config": { - "type": "LLMChatNodeExecutionConfig", - "properties": { - "llm_component_id": "llm_node_llm_component", - "text_generation_config": { - "max_new_tokens": 2500, - "max_prompt_length": 100, - "repetition_penalty": 1, - "top_p": 1, - "temperature": 1, - "frequency_penalty": 0, - "presence_penalty": 0 - }, - "stream": false, - "report_to_client": false, - "response_format": "text" - } - } - }, - { - "type": "FlashcardParserNodeType", - "id": "flashcard-parser", - "execution_config": { - "type": "NodeExecutionConfig", - "properties": { - "report_to_client": false - } - } - } - ], - "edges": [ - { - "from_node": "flashcard-prompt-builder", - "to_node": "text-to-chat-request" - }, - { - "from_node": "text-to-chat-request", - "to_node": "llm_node" - }, - { - "from_node": "llm_node", - "to_node": "flashcard-parser" - } - ], - "end_nodes": [ - "flashcard-parser" - ], - "start_nodes": [ - "flashcard-prompt-builder" - ] - }, - "components": [ - { - "id": "llm_node_llm_component", - "type": "LLMInterface", - "creation_config": { - "type": "RemoteLLMConfig", - "properties": { - "provider": "openai", - "model_name": "gpt-5", - "default_config": {}, - "api_key": "{{INWORLD_API_KEY}}" - } - } - } - ] -} \ No newline at end of file diff --git a/frontend/js/audio-player.js b/frontend/js/audio-player.js index 8ee152e..fcd34b2 100644 --- a/frontend/js/audio-player.js +++ b/frontend/js/audio-player.js @@ -55,7 +55,7 @@ export class AudioPlayer { } } - async addAudioStream(base64Audio, sampleRate = 16000, isLastChunk = false) { + async addAudioStream(base64Audio, sampleRate = 16000, isLastChunk = false, audioFormat = 'int16') { if (!base64Audio || base64Audio.length === 0) { console.warn('Empty audio data received'); return; @@ -110,7 +110,8 @@ export class AudioPlayer { // Create audio buffer from the decoded data const audioBuffer = await this.createAudioBuffer( bytes.buffer, - sampleRate + sampleRate, + audioFormat ); // Queue the audio buffer for playback @@ -131,24 +132,51 @@ export class AudioPlayer { } } - async createAudioBuffer(arrayBuffer, sampleRate) { + async createAudioBuffer(arrayBuffer, sampleRate, audioFormat = 'int16') { try { - // We know the backend sends raw PCM Int16 data, so skip decode attempt - // and directly process as PCM for faster playback - const int16Array = new Int16Array(arrayBuffer); - const audioBuffer = this.audioContext.createBuffer( - 1, - int16Array.length, - sampleRate - ); - const channelData = audioBuffer.getChannelData(0); + let channelData; + let numSamples; + + console.log(`[AudioPlayer] createAudioBuffer: format=${audioFormat}, byteLength=${arrayBuffer.byteLength}, sampleRate=${sampleRate}`); + + if (audioFormat === 'float32') { + // Float32 PCM - bytes are IEEE 754 Float32 representation + // 4 bytes per sample, values already in [-1.0, 1.0] range + const float32Array = new Float32Array(arrayBuffer); + numSamples = float32Array.length; + console.log(`[AudioPlayer] Float32 samples: ${numSamples}, first 3 values: [${Array.from(float32Array.slice(0, 3)).map(v => v.toFixed(4)).join(', ')}]`); + + const audioBuffer = this.audioContext.createBuffer( + 1, + numSamples, + sampleRate + ); + channelData = audioBuffer.getChannelData(0); - // Convert Int16 to Float32 and normalize - for (let i = 0; i < int16Array.length; i++) { - channelData[i] = int16Array[i] / 32768.0; - } + for (let i = 0; i < numSamples; i++) { + channelData[i] = float32Array[i]; + } - return audioBuffer; + return audioBuffer; + } else { + // Int16 PCM format - convert to Float32 + const int16Array = new Int16Array(arrayBuffer); + numSamples = int16Array.length; + console.log(`[AudioPlayer] Int16 samples: ${numSamples}`); + + const audioBuffer = this.audioContext.createBuffer( + 1, + numSamples, + sampleRate + ); + channelData = audioBuffer.getChannelData(0); + + for (let i = 0; i < numSamples; i++) { + channelData[i] = int16Array[i] / 32768.0; + } + + return audioBuffer; + } } catch (error) { console.error('Error creating audio buffer:', error); throw error; diff --git a/frontend/js/chat-ui.js b/frontend/js/chat-ui.js index 369621b..3ebfd61 100644 --- a/frontend/js/chat-ui.js +++ b/frontend/js/chat-ui.js @@ -1,3 +1,5 @@ +import { translator } from './translator.js'; + export class ChatUI { constructor() { this.messagesContainer = document.getElementById('messages'); @@ -5,6 +7,111 @@ export class ChatUI { this.typewriterTimers = new Map(); // Track active typewriter effects this.typewriterSpeed = 25; // milliseconds per character this.llmTypewriterCallback = null; // Callback for when LLM typewriter completes + + // Translation tooltip + this.translationTooltip = this._createTranslationTooltip(); + this.activeHoverElement = null; + this.hoverTimeout = null; + this.hideTimeout = null; + } + + _createTranslationTooltip() { + const tooltip = document.createElement('div'); + tooltip.className = 'translation-tooltip'; + tooltip.innerHTML = ` +
+ +
+
+ +
+ `; + document.body.appendChild(tooltip); + + // Keep tooltip visible when hovering over it + tooltip.addEventListener('mouseenter', () => { + if (this.hideTimeout) { + clearTimeout(this.hideTimeout); + this.hideTimeout = null; + } + }); + + tooltip.addEventListener('mouseleave', () => { + this._hideTooltip(); + }); + + return tooltip; + } + + _showTooltip(element, text) { + const rect = element.getBoundingClientRect(); + const tooltip = this.translationTooltip; + + // Position tooltip above the message + tooltip.style.left = `${rect.left + window.scrollX}px`; + tooltip.style.top = `${rect.top + window.scrollY - 8}px`; + tooltip.style.maxWidth = `${Math.min(rect.width + 40, 400)}px`; + + // Show loading state + tooltip.classList.add('visible', 'loading'); + tooltip.querySelector('.translation-text').textContent = ''; + + // Fetch translation + translator.translate(text, 'en', 'auto') + .then(translation => { + if (this.activeHoverElement === element) { + tooltip.querySelector('.translation-text').textContent = translation; + tooltip.classList.remove('loading'); + + // Reposition after content loads (in case size changed) + requestAnimationFrame(() => { + const tooltipRect = tooltip.getBoundingClientRect(); + tooltip.style.top = `${rect.top + window.scrollY - tooltipRect.height - 8}px`; + }); + } + }) + .catch(error => { + console.error('[ChatUI] Translation failed:', error); + if (this.activeHoverElement === element) { + tooltip.querySelector('.translation-text').textContent = 'Translation unavailable'; + tooltip.classList.remove('loading'); + } + }); + } + + _hideTooltip() { + this.translationTooltip.classList.remove('visible', 'loading'); + this.activeHoverElement = null; + } + + _setupTranslationHover(element) { + element.addEventListener('mouseenter', () => { + if (this.hideTimeout) { + clearTimeout(this.hideTimeout); + this.hideTimeout = null; + } + + // Small delay before showing tooltip to avoid flickering + this.hoverTimeout = setTimeout(() => { + this.activeHoverElement = element; + const text = element.textContent.replace('▊', '').trim(); // Remove cursor + if (text) { + this._showTooltip(element, text); + } + }, 300); + }); + + element.addEventListener('mouseleave', () => { + if (this.hoverTimeout) { + clearTimeout(this.hoverTimeout); + this.hoverTimeout = null; + } + + // Delay hiding to allow moving to tooltip + this.hideTimeout = setTimeout(() => { + this._hideTooltip(); + }, 150); + }); } render( @@ -149,6 +256,12 @@ export class ChatUI { const div = document.createElement('div'); div.className = `message ${message.role}`; div.textContent = message.content; + + // Add translation hover for teacher (LLM) messages + if (message.role === 'teacher') { + this._setupTranslationHover(div); + } + return div; } @@ -166,6 +279,9 @@ export class ChatUI { cursor.textContent = '▊'; div.appendChild(cursor); + // Add translation hover for streaming teacher messages too + this._setupTranslationHover(div); + return div; } diff --git a/frontend/js/main.js b/frontend/js/main.js index cf3f1d8..4cfcc8b 100644 --- a/frontend/js/main.js +++ b/frontend/js/main.js @@ -274,6 +274,15 @@ class App { this.render(); }); + this.wsClient.on('partial_transcript', (data) => { + // Update the transcript in real-time as AssemblyAI processes speech + if (data.text) { + this.state.currentTranscript = data.text; + this.state.speechDetected = true; + this.render(); + } + }); + this.wsClient.on('speech_ended', (data) => { // VAD stopped detecting speech - clear the real-time transcript bubble console.log( @@ -828,9 +837,9 @@ class App { try { if (data.audio && data.audio.length > 0) { console.log( - `Received audio stream: ${data.audio.length} bytes at ${data.sampleRate}Hz${data.text ? ` with text: "${data.text}"` : ''}` + `Received audio stream: ${data.audio.length} bytes at ${data.sampleRate}Hz [format:${data.audioFormat}]${data.text ? ` with text: "${data.text}"` : ''}` ); - await this.audioPlayer.addAudioStream(data.audio, data.sampleRate); + await this.audioPlayer.addAudioStream(data.audio, data.sampleRate, false, data.audioFormat); } } catch (error) { console.error('Error handling audio stream:', error); diff --git a/frontend/js/translator.js b/frontend/js/translator.js new file mode 100644 index 0000000..5045f02 --- /dev/null +++ b/frontend/js/translator.js @@ -0,0 +1,90 @@ +/** + * Translation service using Google's free translation endpoint + * No authentication required + */ +export class Translator { + constructor() { + this.cache = new Map(); + this.pendingRequests = new Map(); + } + + /** + * Translate text to target language using Google's free endpoint + * @param {string} text - Text to translate + * @param {string} targetLang - Target language code (default: 'en') + * @param {string} sourceLang - Source language code (default: 'auto' for auto-detect) + * @returns {Promise} - Translated text + */ + async translate(text, targetLang = 'en', sourceLang = 'auto') { + if (!text || !text.trim()) { + return ''; + } + + const cacheKey = `${sourceLang}:${targetLang}:${text}`; + + // Check cache first + if (this.cache.has(cacheKey)) { + return this.cache.get(cacheKey); + } + + // Check if there's already a pending request for this text + if (this.pendingRequests.has(cacheKey)) { + return this.pendingRequests.get(cacheKey); + } + + // Create the translation request + const requestPromise = this._fetchTranslation(text, targetLang, sourceLang) + .then(translation => { + this.cache.set(cacheKey, translation); + this.pendingRequests.delete(cacheKey); + return translation; + }) + .catch(error => { + this.pendingRequests.delete(cacheKey); + throw error; + }); + + this.pendingRequests.set(cacheKey, requestPromise); + return requestPromise; + } + + async _fetchTranslation(text, targetLang, sourceLang) { + const encodedText = encodeURIComponent(text); + const url = `https://translate.googleapis.com/translate_a/single?client=gtx&sl=${sourceLang}&tl=${targetLang}&dt=t&q=${encodedText}`; + + try { + const response = await fetch(url); + + if (!response.ok) { + throw new Error(`Translation failed: ${response.status}`); + } + + const data = await response.json(); + + // Google's response format: [[["translated text", "original text", ...], ...], ...] + // We need to concatenate all translated segments + if (data && data[0]) { + const translatedParts = data[0] + .filter(part => part && part[0]) + .map(part => part[0]); + return translatedParts.join(''); + } + + throw new Error('Invalid translation response format'); + } catch (error) { + console.error('[Translator] Translation error:', error); + throw error; + } + } + + /** + * Clear the translation cache + */ + clearCache() { + this.cache.clear(); + } +} + +// Singleton instance +export const translator = new Translator(); + diff --git a/frontend/js/websocket-client.js b/frontend/js/websocket-client.js index a07ff1a..200d0da 100644 --- a/frontend/js/websocket-client.js +++ b/frontend/js/websocket-client.js @@ -154,6 +154,14 @@ export class WebSocketClient { this.emit('speech_ended', message.data); break; + case 'partial_transcript': + this.emit('partial_transcript', { + text: message.text, + interactionId: message.interactionId, + timestamp: message.timestamp, + }); + break; + case 'llm_response_chunk': this.emit('llm_response_chunk', { text: message.text, @@ -171,6 +179,7 @@ export class WebSocketClient { case 'audio_stream': this.emit('audio_stream', { audio: message.audio, + audioFormat: message.audioFormat || 'int16', sampleRate: message.sampleRate, timestamp: message.timestamp, }); diff --git a/frontend/styles/main.css b/frontend/styles/main.css index 00c0f6e..066e468 100644 --- a/frontend/styles/main.css +++ b/frontend/styles/main.css @@ -576,4 +576,108 @@ body { .messages::-webkit-scrollbar-thumb:hover, .flashcards-grid::-webkit-scrollbar-thumb:hover { background: #9ca3af; +} + +/* Translation Tooltip */ +.translation-tooltip { + position: absolute; + z-index: 1000; + background: #1a1a1a; + color: #ffffff; + padding: 10px 14px; + border-radius: 8px; + font-size: 14px; + line-height: 1.4; + max-width: 400px; + box-shadow: 0 4px 20px rgba(0, 0, 0, 0.25); + opacity: 0; + visibility: hidden; + transform: translateY(4px); + transition: opacity 0.2s ease, transform 0.2s ease, visibility 0.2s; + pointer-events: none; +} + +.translation-tooltip.visible { + opacity: 1; + visibility: visible; + transform: translateY(0); + pointer-events: auto; +} + +.translation-tooltip::after { + content: ''; + position: absolute; + bottom: -6px; + left: 20px; + width: 12px; + height: 12px; + background: #1a1a1a; + transform: rotate(45deg); + border-radius: 0 0 2px 0; +} + +.translation-content { + display: block; +} + +.translation-text { + display: block; + word-wrap: break-word; +} + +.translation-loading { + display: none; + align-items: center; + justify-content: center; + gap: 4px; + padding: 4px 0; +} + +.translation-tooltip.loading .translation-loading { + display: flex; +} + +.translation-tooltip.loading .translation-content { + display: none; +} + +.translation-loading span { + width: 6px; + height: 6px; + border-radius: 50%; + background: rgba(255, 255, 255, 0.7); + animation: translation-dot 1.2s ease-in-out infinite; +} + +.translation-loading span:nth-child(1) { + animation-delay: 0s; +} + +.translation-loading span:nth-child(2) { + animation-delay: 0.15s; +} + +.translation-loading span:nth-child(3) { + animation-delay: 0.3s; +} + +@keyframes translation-dot { + 0%, 60%, 100% { + opacity: 0.4; + transform: scale(0.8); + } + 30% { + opacity: 1; + transform: scale(1); + } +} + +/* Make teacher messages show a subtle cursor hint for translation */ +.message.teacher { + cursor: help; + transition: background-color 0.15s ease; +} + +.message.teacher:hover { + background: #e9eaec; } \ No newline at end of file diff --git a/package-lock.json b/package-lock.json index 7bdb6aa..d94e982 100644 --- a/package-lock.json +++ b/package-lock.json @@ -9,7 +9,7 @@ "version": "1.0.0", "license": "MIT", "dependencies": { - "@inworld/runtime": "^0.8.0", + "@inworld/runtime": "https://storage.googleapis.com/assets-inworld-ai/node-packages/inworld-runtime-0.9.0-rc.13.tgz", "anki-apkg-export": "^4.0.3", "dotenv": "^17.2.1", "express": "^4.19.2", @@ -696,329 +696,6 @@ "node": "^18.18.0 || ^20.9.0 || >=21.1.0" } }, - "node_modules/@fastify/accept-negotiator": { - "version": "2.0.1", - "resolved": "https://registry.npmjs.org/@fastify/accept-negotiator/-/accept-negotiator-2.0.1.tgz", - "integrity": "sha512-/c/TW2bO/v9JeEgoD/g1G5GxGeCF1Hafdf79WPmUlgYiBXummY0oX3VVq4yFkKKVBKDNlaDUYoab7g38RpPqCQ==", - "funding": [ - { - "type": "github", - "url": "https://github.com/sponsors/fastify" - }, - { - "type": "opencollective", - "url": "https://opencollective.com/fastify" - } - ], - "license": "MIT" - }, - "node_modules/@fastify/ajv-compiler": { - "version": "4.0.5", - "resolved": "https://registry.npmjs.org/@fastify/ajv-compiler/-/ajv-compiler-4.0.5.tgz", - "integrity": "sha512-KoWKW+MhvfTRWL4qrhUwAAZoaChluo0m0vbiJlGMt2GXvL4LVPQEjt8kSpHI3IBq5Rez8fg+XeH3cneztq+C7A==", - "funding": [ - { - "type": "github", - "url": "https://github.com/sponsors/fastify" - }, - { - "type": "opencollective", - "url": "https://opencollective.com/fastify" - } - ], - "license": "MIT", - "dependencies": { - "ajv": "^8.12.0", - "ajv-formats": "^3.0.1", - "fast-uri": "^3.0.0" - } - }, - "node_modules/@fastify/cors": { - "version": "10.1.0", - "resolved": "https://registry.npmjs.org/@fastify/cors/-/cors-10.1.0.tgz", - "integrity": "sha512-MZyBCBJtII60CU9Xme/iE4aEy8G7QpzGR8zkdXZkDFt7ElEMachbE61tfhAG/bvSaULlqlf0huMT12T7iqEmdQ==", - "funding": [ - { - "type": "github", - "url": "https://github.com/sponsors/fastify" - }, - { - "type": "opencollective", - "url": "https://opencollective.com/fastify" - } - ], - "license": "MIT", - "dependencies": { - "fastify-plugin": "^5.0.0", - "mnemonist": "0.40.0" - } - }, - "node_modules/@fastify/error": { - "version": "4.2.0", - "resolved": "https://registry.npmjs.org/@fastify/error/-/error-4.2.0.tgz", - "integrity": "sha512-RSo3sVDXfHskiBZKBPRgnQTtIqpi/7zhJOEmAxCiBcM7d0uwdGdxLlsCaLzGs8v8NnxIRlfG0N51p5yFaOentQ==", - "funding": [ - { - "type": "github", - "url": "https://github.com/sponsors/fastify" - }, - { - "type": "opencollective", - "url": "https://opencollective.com/fastify" - } - ], - "license": "MIT" - }, - "node_modules/@fastify/fast-json-stringify-compiler": { - "version": "5.0.3", - "resolved": "https://registry.npmjs.org/@fastify/fast-json-stringify-compiler/-/fast-json-stringify-compiler-5.0.3.tgz", - "integrity": "sha512-uik7yYHkLr6fxd8hJSZ8c+xF4WafPK+XzneQDPU+D10r5X19GW8lJcom2YijX2+qtFF1ENJlHXKFM9ouXNJYgQ==", - "funding": [ - { - "type": "github", - "url": "https://github.com/sponsors/fastify" - }, - { - "type": "opencollective", - "url": "https://opencollective.com/fastify" - } - ], - "license": "MIT", - "dependencies": { - "fast-json-stringify": "^6.0.0" - } - }, - "node_modules/@fastify/forwarded": { - "version": "3.0.1", - "resolved": "https://registry.npmjs.org/@fastify/forwarded/-/forwarded-3.0.1.tgz", - "integrity": "sha512-JqDochHFqXs3C3Ml3gOY58zM7OqO9ENqPo0UqAjAjH8L01fRZqwX9iLeX34//kiJubF7r2ZQHtBRU36vONbLlw==", - "funding": [ - { - "type": "github", - "url": "https://github.com/sponsors/fastify" - }, - { - "type": "opencollective", - "url": "https://opencollective.com/fastify" - } - ], - "license": "MIT" - }, - "node_modules/@fastify/merge-json-schemas": { - "version": "0.2.1", - "resolved": "https://registry.npmjs.org/@fastify/merge-json-schemas/-/merge-json-schemas-0.2.1.tgz", - "integrity": "sha512-OA3KGBCy6KtIvLf8DINC5880o5iBlDX4SxzLQS8HorJAbqluzLRn80UXU0bxZn7UOFhFgpRJDasfwn9nG4FG4A==", - "funding": [ - { - "type": "github", - "url": "https://github.com/sponsors/fastify" - }, - { - "type": "opencollective", - "url": "https://opencollective.com/fastify" - } - ], - "license": "MIT", - "dependencies": { - "dequal": "^2.0.3" - } - }, - "node_modules/@fastify/proxy-addr": { - "version": "5.1.0", - "resolved": "https://registry.npmjs.org/@fastify/proxy-addr/-/proxy-addr-5.1.0.tgz", - "integrity": "sha512-INS+6gh91cLUjB+PVHfu1UqcB76Sqtpyp7bnL+FYojhjygvOPA9ctiD/JDKsyD9Xgu4hUhCSJBPig/w7duNajw==", - "funding": [ - { - "type": "github", - "url": "https://github.com/sponsors/fastify" - }, - { - "type": "opencollective", - "url": "https://opencollective.com/fastify" - } - ], - "license": "MIT", - "dependencies": { - "@fastify/forwarded": "^3.0.0", - "ipaddr.js": "^2.1.0" - } - }, - "node_modules/@fastify/proxy-addr/node_modules/ipaddr.js": { - "version": "2.2.0", - "resolved": "https://registry.npmjs.org/ipaddr.js/-/ipaddr.js-2.2.0.tgz", - "integrity": "sha512-Ag3wB2o37wslZS19hZqorUnrnzSkpOVy+IiiDEiTqNubEYpYuHWIf6K4psgN2ZWKExS4xhVCrRVfb/wfW8fWJA==", - "license": "MIT", - "engines": { - "node": ">= 10" - } - }, - "node_modules/@fastify/send": { - "version": "4.1.0", - "resolved": "https://registry.npmjs.org/@fastify/send/-/send-4.1.0.tgz", - "integrity": "sha512-TMYeQLCBSy2TOFmV95hQWkiTYgC/SEx7vMdV+wnZVX4tt8VBLKzmH8vV9OzJehV0+XBfg+WxPMt5wp+JBUKsVw==", - "funding": [ - { - "type": "github", - "url": "https://github.com/sponsors/fastify" - }, - { - "type": "opencollective", - "url": "https://opencollective.com/fastify" - } - ], - "license": "MIT", - "dependencies": { - "@lukeed/ms": "^2.0.2", - "escape-html": "~1.0.3", - "fast-decode-uri-component": "^1.0.1", - "http-errors": "^2.0.0", - "mime": "^3" - } - }, - "node_modules/@fastify/send/node_modules/mime": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/mime/-/mime-3.0.0.tgz", - "integrity": "sha512-jSCU7/VB1loIWBZe14aEYHU/+1UMEHoaO7qxCOVJOw9GgH72VAWppxNcjU+x9a2k3GSIBXNKxXQFqRvvZ7vr3A==", - "license": "MIT", - "bin": { - "mime": "cli.js" - }, - "engines": { - "node": ">=10.0.0" - } - }, - "node_modules/@fastify/static": { - "version": "8.3.0", - "resolved": "https://registry.npmjs.org/@fastify/static/-/static-8.3.0.tgz", - "integrity": "sha512-yKxviR5PH1OKNnisIzZKmgZSus0r2OZb8qCSbqmw34aolT4g3UlzYfeBRym+HJ1J471CR8e2ldNub4PubD1coA==", - "funding": [ - { - "type": "github", - "url": "https://github.com/sponsors/fastify" - }, - { - "type": "opencollective", - "url": "https://opencollective.com/fastify" - } - ], - "license": "MIT", - "dependencies": { - "@fastify/accept-negotiator": "^2.0.0", - "@fastify/send": "^4.0.0", - "content-disposition": "^0.5.4", - "fastify-plugin": "^5.0.0", - "fastq": "^1.17.1", - "glob": "^11.0.0" - } - }, - "node_modules/@fastify/swagger": { - "version": "9.6.1", - "resolved": "https://registry.npmjs.org/@fastify/swagger/-/swagger-9.6.1.tgz", - "integrity": "sha512-fKlpJqFMWoi4H3EdUkDaMteEYRCfQMEkK0HJJ0eaf4aRlKd8cbq0pVkOfXDXmtvMTXYcnx3E+l023eFDBsA1HA==", - "funding": [ - { - "type": "github", - "url": "https://github.com/sponsors/fastify" - }, - { - "type": "opencollective", - "url": "https://opencollective.com/fastify" - } - ], - "license": "MIT", - "dependencies": { - "fastify-plugin": "^5.0.0", - "json-schema-resolver": "^3.0.0", - "openapi-types": "^12.1.3", - "rfdc": "^1.3.1", - "yaml": "^2.4.2" - } - }, - "node_modules/@fastify/swagger-ui": { - "version": "5.2.3", - "resolved": "https://registry.npmjs.org/@fastify/swagger-ui/-/swagger-ui-5.2.3.tgz", - "integrity": "sha512-e7ivEJi9EpFcxTONqICx4llbpB2jmlI+LI1NQ/mR7QGQnyDOqZybPK572zJtcdHZW4YyYTBHcP3a03f1pOh0SA==", - "funding": [ - { - "type": "github", - "url": "https://github.com/sponsors/fastify" - }, - { - "type": "opencollective", - "url": "https://opencollective.com/fastify" - } - ], - "license": "MIT", - "dependencies": { - "@fastify/static": "^8.0.0", - "fastify-plugin": "^5.0.0", - "openapi-types": "^12.1.3", - "rfdc": "^1.3.1", - "yaml": "^2.4.1" - } - }, - "node_modules/@grpc/grpc-js": { - "version": "1.14.1", - "resolved": "https://registry.npmjs.org/@grpc/grpc-js/-/grpc-js-1.14.1.tgz", - "integrity": "sha512-sPxgEWtPUR3EnRJCEtbGZG2iX8LQDUls2wUS3o27jg07KqJFMq6YDeWvMo1wfpmy3rqRdS0rivpLwhqQtEyCuQ==", - "license": "Apache-2.0", - "dependencies": { - "@grpc/proto-loader": "^0.8.0", - "@js-sdsl/ordered-map": "^4.4.2" - }, - "engines": { - "node": ">=12.10.0" - } - }, - "node_modules/@grpc/grpc-js/node_modules/@grpc/proto-loader": { - "version": "0.8.0", - "resolved": "https://registry.npmjs.org/@grpc/proto-loader/-/proto-loader-0.8.0.tgz", - "integrity": "sha512-rc1hOQtjIWGxcxpb9aHAfLpIctjEnsDehj0DAiVfBlmT84uvR0uUtN2hEi/ecvWVjXUGf5qPF4qEgiLOx1YIMQ==", - "license": "Apache-2.0", - "dependencies": { - "lodash.camelcase": "^4.3.0", - "long": "^5.0.0", - "protobufjs": "^7.5.3", - "yargs": "^17.7.2" - }, - "bin": { - "proto-loader-gen-types": "build/bin/proto-loader-gen-types.js" - }, - "engines": { - "node": ">=6" - } - }, - "node_modules/@grpc/proto-loader": { - "version": "0.7.15", - "resolved": "https://registry.npmjs.org/@grpc/proto-loader/-/proto-loader-0.7.15.tgz", - "integrity": "sha512-tMXdRCfYVixjuFK+Hk0Q1s38gV9zDiDJfWL3h1rv4Qc39oILCu1TRTDt7+fGUI8K4G1Fj125Hx/ru3azECWTyQ==", - "license": "Apache-2.0", - "dependencies": { - "lodash.camelcase": "^4.3.0", - "long": "^5.0.0", - "protobufjs": "^7.2.5", - "yargs": "^17.7.2" - }, - "bin": { - "proto-loader-gen-types": "build/bin/proto-loader-gen-types.js" - }, - "engines": { - "node": ">=6" - } - }, - "node_modules/@grpc/reflection": { - "version": "1.0.4", - "resolved": "https://registry.npmjs.org/@grpc/reflection/-/reflection-1.0.4.tgz", - "integrity": "sha512-znA8v4AviOD3OPOxy11pxrtP8k8DanpefeTymS8iGW1fVr1U2cHuzfhYqDPHnVNDf4qvF9E25KtSihPy2DBWfQ==", - "license": "Apache-2.0", - "dependencies": { - "@grpc/proto-loader": "^0.7.13", - "protobufjs": "^7.2.5" - }, - "peerDependencies": { - "@grpc/grpc-js": "^1.8.21" - } - }, "node_modules/@humanfs/core": { "version": "0.19.1", "resolved": "https://registry.npmjs.org/@humanfs/core/-/core-0.19.1.tgz", @@ -1071,118 +748,34 @@ "url": "https://github.com/sponsors/nzakas" } }, - "node_modules/@inworld/graph-server": { - "version": "0.8.0", - "resolved": "https://registry.npmjs.org/@inworld/graph-server/-/graph-server-0.8.0.tgz", - "integrity": "sha512-mwAwgIPF23RbWECNfiAcugtMFQ1ronkRu3LMoqJN7fw6EKq3npQ2lJrYdAS7WI0aJbEIWbJ3/vio17z6TP5WLQ==", - "license": "MIT", - "dependencies": { - "@fastify/cors": "^10.0.1", - "@fastify/static": "^8.0.2", - "@fastify/swagger": "^9.4.0", - "@fastify/swagger-ui": "^5.2.0", - "@grpc/grpc-js": "^1.12.4", - "@grpc/proto-loader": "^0.7.13", - "@grpc/reflection": "^1.0.4", - "fastify": "^5.2.0", - "protobufjs": "^7.4.0" - } - }, "node_modules/@inworld/runtime": { - "version": "0.8.0", - "resolved": "https://registry.npmjs.org/@inworld/runtime/-/runtime-0.8.0.tgz", - "integrity": "sha512-bSOydjXGiuSTm8BZ1t/dsUOK7l/WCseHjNRCMdHUFkevDzbGXHPaf7jQEPM3r4c4nyRiVabjHpi2zvLSCa+jWw==", + "version": "0.9.0-rc.13", + "resolved": "https://storage.googleapis.com/assets-inworld-ai/node-packages/inworld-runtime-0.9.0-rc.13.tgz", + "integrity": "sha512-6C/aND5PqOVzujwoS0rvUR963VxJ+MO0X+IZg9WpjrxBbuFag7fXay5yyamH0cKoTvmaTT2s8fca8pK0zQ0CYg==", "hasInstallScript": true, "license": "SEE LICENSE IN LICENSE.md and LICENSE-CPP-BINARIES.md", "dependencies": { - "@inworld/graph-server": "0.8.0", "@types/lodash.snakecase": "^4.1.9", + "@types/protobufjs": "^6.0.0", + "@zod/core": "^0.11.6", "ajv": "^8.17.1", "ajv-formats": "^3.0.1", "assemblyai": "^4.17.0", + "chalk": "^4.1.2", "decompress": "^4.2.1", "decompress-targz": "^4.1.1", "decompress-unzip": "^4.0.1", - "dotenv": "^16.4.7", "groq-sdk": "^0.33.0", - "koffi": "^2.10.1", "lodash.snakecase": "^4.1.1", "node-record-lpcm16": "^1.0.1", - "uuid": "^11.1.0" + "protobufjs": "^7.5.4", + "uuid": "^11.1.0", + "zod-to-json-schema": "^3.25.0" }, "engines": { "node": ">=18.0.0" } }, - "node_modules/@inworld/runtime/node_modules/dotenv": { - "version": "16.6.1", - "resolved": "https://registry.npmjs.org/dotenv/-/dotenv-16.6.1.tgz", - "integrity": "sha512-uBq4egWHTcTt33a72vpSG0z3HnPuIl6NqYcTrKEg2azoEyl2hpW0zqlxysq2pK9HlDIHyHyakeYaYnSAwd8bow==", - "license": "BSD-2-Clause", - "engines": { - "node": ">=12" - }, - "funding": { - "url": "https://dotenvx.com" - } - }, - "node_modules/@isaacs/balanced-match": { - "version": "4.0.1", - "resolved": "https://registry.npmjs.org/@isaacs/balanced-match/-/balanced-match-4.0.1.tgz", - "integrity": "sha512-yzMTt9lEb8Gv7zRioUilSglI0c0smZ9k5D65677DLWLtWJaXIS3CqcGyUFByYKlnUj6TkjLVs54fBl6+TiGQDQ==", - "license": "MIT", - "engines": { - "node": "20 || >=22" - } - }, - "node_modules/@isaacs/brace-expansion": { - "version": "5.0.0", - "resolved": "https://registry.npmjs.org/@isaacs/brace-expansion/-/brace-expansion-5.0.0.tgz", - "integrity": "sha512-ZT55BDLV0yv0RBm2czMiZ+SqCGO7AvmOM3G/w2xhVPH+te0aKgFjmBvGlL1dH+ql2tgGO3MVrbb3jCKyvpgnxA==", - "license": "MIT", - "dependencies": { - "@isaacs/balanced-match": "^4.0.1" - }, - "engines": { - "node": "20 || >=22" - } - }, - "node_modules/@isaacs/cliui": { - "version": "8.0.2", - "resolved": "https://registry.npmjs.org/@isaacs/cliui/-/cliui-8.0.2.tgz", - "integrity": "sha512-O8jcjabXaleOG9DQ0+ARXWZBTfnP4WNAqzuiJK7ll44AmxGKv/J2M4TPjxjY3znBCfvBXFzucm1twdyFybFqEA==", - "license": "ISC", - "dependencies": { - "string-width": "^5.1.2", - "string-width-cjs": "npm:string-width@^4.2.0", - "strip-ansi": "^7.0.1", - "strip-ansi-cjs": "npm:strip-ansi@^6.0.1", - "wrap-ansi": "^8.1.0", - "wrap-ansi-cjs": "npm:wrap-ansi@^7.0.0" - }, - "engines": { - "node": ">=12" - } - }, - "node_modules/@js-sdsl/ordered-map": { - "version": "4.4.2", - "resolved": "https://registry.npmjs.org/@js-sdsl/ordered-map/-/ordered-map-4.4.2.tgz", - "integrity": "sha512-iUKgm52T8HOE/makSxjqoWhe95ZJA1/G1sYsGev2JDKUSS14KAgg1LHb+Ba+IPow0xflbnSkOsZcO08C7w1gYw==", - "license": "MIT", - "funding": { - "type": "opencollective", - "url": "https://opencollective.com/js-sdsl" - } - }, - "node_modules/@lukeed/ms": { - "version": "2.0.2", - "resolved": "https://registry.npmjs.org/@lukeed/ms/-/ms-2.0.2.tgz", - "integrity": "sha512-9I2Zn6+NJLfaGoz9jN3lpwDgAYvfGeNYdbAIjJOqzs4Tpc+VU3Jqq4IofSUBKajiDS8k9fZIg18/z13mpk1bsA==", - "license": "MIT", - "engines": { - "node": ">=8" - } - }, "node_modules/@nodelib/fs.scandir": { "version": "2.1.5", "resolved": "https://registry.npmjs.org/@nodelib/fs.scandir/-/fs.scandir-2.1.5.tgz", @@ -1221,12 +814,6 @@ "node": ">= 8" } }, - "node_modules/@pinojs/redact": { - "version": "0.4.0", - "resolved": "https://registry.npmjs.org/@pinojs/redact/-/redact-0.4.0.tgz", - "integrity": "sha512-k2ENnmBugE/rzQfEcdWHcCY+/FM3VLzH9cYEsbdsoqrvzAKRhUZeRNhAZvB8OitQJ1TBed3yqWtdjzS6wJKBwg==", - "license": "MIT" - }, "node_modules/@pkgr/core": { "version": "0.2.9", "resolved": "https://registry.npmjs.org/@pkgr/core/-/core-0.2.9.tgz", @@ -1420,6 +1007,16 @@ "form-data": "^4.0.4" } }, + "node_modules/@types/protobufjs": { + "version": "6.0.0", + "resolved": "https://registry.npmjs.org/@types/protobufjs/-/protobufjs-6.0.0.tgz", + "integrity": "sha512-A27RDExpAf3rdDjIrHKiJK6x8kqqJ4CmoChwtipfhVAn1p7+wviQFFP7dppn8FslSbHtQeVPvi8wNKkDjSYjHw==", + "deprecated": "This is a stub types definition for protobufjs (https://github.com/dcodeIO/ProtoBuf.js). protobufjs provides its own type definitions, so you don't need @types/protobufjs installed!", + "license": "MIT", + "dependencies": { + "protobufjs": "*" + } + }, "node_modules/@types/qs": { "version": "6.14.0", "resolved": "https://registry.npmjs.org/@types/qs/-/qs-6.14.0.tgz", @@ -1825,6 +1422,15 @@ "url": "https://opencollective.com/typescript-eslint" } }, + "node_modules/@zod/core": { + "version": "0.11.6", + "resolved": "https://registry.npmjs.org/@zod/core/-/core-0.11.6.tgz", + "integrity": "sha512-03Bv82fFSfjDAvMfdHHdGSS6SOJs0iCcJlWJv1kJHRtoTT02hZpyip/2Lk6oo4l4FtjuwTrsEQTwg/LD8I7dJA==", + "license": "MIT", + "funding": { + "url": "https://github.com/sponsors/colinhacks" + } + }, "node_modules/abort-controller": { "version": "3.0.0", "resolved": "https://registry.npmjs.org/abort-controller/-/abort-controller-3.0.0.tgz", @@ -1837,12 +1443,6 @@ "node": ">=6.5" } }, - "node_modules/abstract-logging": { - "version": "2.0.1", - "resolved": "https://registry.npmjs.org/abstract-logging/-/abstract-logging-2.0.1.tgz", - "integrity": "sha512-2BjRTZxTPvheOvGbBslFSYOUkr+SjPtOnrLP33f+VIWLzezQpZcqVg7ja3L4dBXmzzgwT+a029jRx5PCi3JuiA==", - "license": "MIT" - }, "node_modules/accepts": { "version": "1.3.8", "resolved": "https://registry.npmjs.org/accepts/-/accepts-1.3.8.tgz", @@ -1935,30 +1535,6 @@ "sql.js": "^0.5.0" } }, - "node_modules/ansi-regex": { - "version": "6.2.2", - "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-6.2.2.tgz", - "integrity": "sha512-Bq3SmSpyFHaWjPk8If9yc6svM8c56dB5BAtW4Qbw5jHTwwXXcTLoRMkpDJp6VL0XzlWaCHTXrkFURMYmD0sLqg==", - "license": "MIT", - "engines": { - "node": ">=12" - }, - "funding": { - "url": "https://github.com/chalk/ansi-regex?sponsor=1" - } - }, - "node_modules/ansi-styles": { - "version": "6.2.3", - "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-6.2.3.tgz", - "integrity": "sha512-4Dj6M28JB+oAH8kFkTLUo+a2jwOFkuqb3yucU0CANcRRUbxS0cP0nZYCGjcc3BNXwRIsUVmDGgzawme7zvJHvg==", - "license": "MIT", - "engines": { - "node": ">=12" - }, - "funding": { - "url": "https://github.com/chalk/ansi-styles?sponsor=1" - } - }, "node_modules/anymatch": { "version": "3.1.3", "resolved": "https://registry.npmjs.org/anymatch/-/anymatch-3.1.3.tgz", @@ -2004,15 +1580,6 @@ "integrity": "sha512-Oei9OH4tRh0YqU3GxhX79dM/mwVgvbZJaSNaRk+bshkj0S5cfHcgYakreBjrHwatXKbz+IoIdYLxrKim2MjW0Q==", "license": "MIT" }, - "node_modules/atomic-sleep": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/atomic-sleep/-/atomic-sleep-1.0.0.tgz", - "integrity": "sha512-kNOjDqAh7px0XWNI+4QbzoiR/nTkHAWNud2uvnJquD1/x5a7EQZMJT0AczqK0Qn67oY/TTQ1LbUKajZpp3I9tQ==", - "license": "MIT", - "engines": { - "node": ">=8.0.0" - } - }, "node_modules/available-typed-arrays": { "version": "1.0.7", "resolved": "https://registry.npmjs.org/available-typed-arrays/-/available-typed-arrays-1.0.7.tgz", @@ -2028,16 +1595,6 @@ "url": "https://github.com/sponsors/ljharb" } }, - "node_modules/avvio": { - "version": "9.1.0", - "resolved": "https://registry.npmjs.org/avvio/-/avvio-9.1.0.tgz", - "integrity": "sha512-fYASnYi600CsH/j9EQov7lECAniYiBFiiAtBNuZYLA2leLe9qOvZzqYHFjtIj6gD2VMoMLP14834LFWvr4IfDw==", - "license": "MIT", - "dependencies": { - "@fastify/error": "^4.0.0", - "fastq": "^1.17.1" - } - }, "node_modules/balanced-match": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/balanced-match/-/balanced-match-1.0.2.tgz", @@ -2261,7 +1818,6 @@ "version": "4.1.2", "resolved": "https://registry.npmjs.org/chalk/-/chalk-4.1.2.tgz", "integrity": "sha512-oKnbhFyRIXpUuez8iBMmyEa4nbj4IOQyuhc/wy9kY7/WVPcwIO9VA668Pu8RkO7+0G76SLROeyw9CpQ061i4mA==", - "dev": true, "license": "MIT", "dependencies": { "ansi-styles": "^4.1.0", @@ -2278,7 +1834,6 @@ "version": "4.3.0", "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-4.3.0.tgz", "integrity": "sha512-zbB9rCJAT1rbjiVDb2hqKFHNYLxgtk8NURxZ3IZwD3F6NtxbXZQCnnSi1Lkx+IDohdPlFp222wVALIheZJQSEg==", - "dev": true, "license": "MIT", "dependencies": { "color-convert": "^2.0.1" @@ -2294,7 +1849,6 @@ "version": "4.0.0", "resolved": "https://registry.npmjs.org/has-flag/-/has-flag-4.0.0.tgz", "integrity": "sha512-EykJT/Q1KjTWctppgIAgfSO0tKVuZUjhgMr17kqTumMl6Afv3EISleU7qZUzoXDFTAHTDC4NOoG/ZxU3EvlMPQ==", - "dev": true, "license": "MIT", "engines": { "node": ">=8" @@ -2304,7 +1858,6 @@ "version": "7.2.0", "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-7.2.0.tgz", "integrity": "sha512-qpCAvRl9stuOHveKsn7HncJRvv501qIacKzQlO/+Lwxc9+0q2wLyv4Dfvt80/DPn2pqOBsJdDiogXGR9+OvwRw==", - "dev": true, "license": "MIT", "dependencies": { "has-flag": "^4.0.0" @@ -2347,93 +1900,6 @@ "fsevents": "~2.3.2" } }, - "node_modules/cliui": { - "version": "8.0.1", - "resolved": "https://registry.npmjs.org/cliui/-/cliui-8.0.1.tgz", - "integrity": "sha512-BSeNnyus75C4//NQ9gQt1/csTXyo/8Sb+afLAkzAptFuMsod9HFokGNudZpi/oQV73hnVK+sR+5PVRMd+Dr7YQ==", - "license": "ISC", - "dependencies": { - "string-width": "^4.2.0", - "strip-ansi": "^6.0.1", - "wrap-ansi": "^7.0.0" - }, - "engines": { - "node": ">=12" - } - }, - "node_modules/cliui/node_modules/ansi-regex": { - "version": "5.0.1", - "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-5.0.1.tgz", - "integrity": "sha512-quJQXlTSUGL2LH9SUXo8VwsY4soanhgo6LNSm84E1LBcE8s3O0wpdiRzyR9z/ZZJMlMWv37qOOb9pdJlMUEKFQ==", - "license": "MIT", - "engines": { - "node": ">=8" - } - }, - "node_modules/cliui/node_modules/ansi-styles": { - "version": "4.3.0", - "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-4.3.0.tgz", - "integrity": "sha512-zbB9rCJAT1rbjiVDb2hqKFHNYLxgtk8NURxZ3IZwD3F6NtxbXZQCnnSi1Lkx+IDohdPlFp222wVALIheZJQSEg==", - "license": "MIT", - "dependencies": { - "color-convert": "^2.0.1" - }, - "engines": { - "node": ">=8" - }, - "funding": { - "url": "https://github.com/chalk/ansi-styles?sponsor=1" - } - }, - "node_modules/cliui/node_modules/emoji-regex": { - "version": "8.0.0", - "resolved": "https://registry.npmjs.org/emoji-regex/-/emoji-regex-8.0.0.tgz", - "integrity": "sha512-MSjYzcWNOA0ewAHpz0MxpYFvwg6yjy1NG3xteoqz644VCo/RPgnr1/GGt+ic3iJTzQ8Eu3TdM14SawnVUmGE6A==", - "license": "MIT" - }, - "node_modules/cliui/node_modules/string-width": { - "version": "4.2.3", - "resolved": "https://registry.npmjs.org/string-width/-/string-width-4.2.3.tgz", - "integrity": "sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g==", - "license": "MIT", - "dependencies": { - "emoji-regex": "^8.0.0", - "is-fullwidth-code-point": "^3.0.0", - "strip-ansi": "^6.0.1" - }, - "engines": { - "node": ">=8" - } - }, - "node_modules/cliui/node_modules/strip-ansi": { - "version": "6.0.1", - "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-6.0.1.tgz", - "integrity": "sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A==", - "license": "MIT", - "dependencies": { - "ansi-regex": "^5.0.1" - }, - "engines": { - "node": ">=8" - } - }, - "node_modules/cliui/node_modules/wrap-ansi": { - "version": "7.0.0", - "resolved": "https://registry.npmjs.org/wrap-ansi/-/wrap-ansi-7.0.0.tgz", - "integrity": "sha512-YVGIj2kamLSTxw6NsZjoBxfSwsn0ycdesmc4p+Q21c5zPuZ1pl+NfxVdxPtdHvmNVOQ6XSYG4AUtyt/Fi7D16Q==", - "license": "MIT", - "dependencies": { - "ansi-styles": "^4.0.0", - "string-width": "^4.1.0", - "strip-ansi": "^6.0.0" - }, - "engines": { - "node": ">=10" - }, - "funding": { - "url": "https://github.com/chalk/wrap-ansi?sponsor=1" - } - }, "node_modules/color-convert": { "version": "2.0.1", "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-2.0.1.tgz", @@ -2523,6 +1989,7 @@ "version": "7.0.6", "resolved": "https://registry.npmjs.org/cross-spawn/-/cross-spawn-7.0.6.tgz", "integrity": "sha512-uV2QOWP2nWzsy2aMp8aRibhi9dlzF5Hgh5SHaB9OiTGEyDTiJJyx0uy51QXdyWbtAHNua4XJzUKca3OzKUd3vA==", + "dev": true, "license": "MIT", "dependencies": { "path-key": "^3.1.0", @@ -2689,15 +2156,6 @@ "node": ">= 0.8" } }, - "node_modules/dequal": { - "version": "2.0.3", - "resolved": "https://registry.npmjs.org/dequal/-/dequal-2.0.3.tgz", - "integrity": "sha512-0je+qPKHEMohvfRTCEo3CrPG6cAzAYgmzKyxRiYSSDkS6eGJdyVJm7WaYA5ECaAD9wLB2T4EEeymA5aFVcYXCA==", - "license": "MIT", - "engines": { - "node": ">=6" - } - }, "node_modules/destroy": { "version": "1.2.0", "resolved": "https://registry.npmjs.org/destroy/-/destroy-1.2.0.tgz", @@ -2734,24 +2192,12 @@ "node": ">= 0.4" } }, - "node_modules/eastasianwidth": { - "version": "0.2.0", - "resolved": "https://registry.npmjs.org/eastasianwidth/-/eastasianwidth-0.2.0.tgz", - "integrity": "sha512-I88TYZWc9XiYHRQ4/3c5rjjfgkjhLyW2luGIheGERbNQ6OY7yTybanSpDXZa8y7VUP9YmDcYa+eyq4ca7iLqWA==", - "license": "MIT" - }, "node_modules/ee-first": { "version": "1.1.1", "resolved": "https://registry.npmjs.org/ee-first/-/ee-first-1.1.1.tgz", "integrity": "sha512-WMwm9LhRUo+WUaRN+vRuETqG89IgZphVSNkdFgeb6sS/E4OrDIN7t48CAewSHXc6C8lefD8KKfr5vY61brQlow==", "license": "MIT" }, - "node_modules/emoji-regex": { - "version": "9.2.2", - "resolved": "https://registry.npmjs.org/emoji-regex/-/emoji-regex-9.2.2.tgz", - "integrity": "sha512-L18DaJsXSUk2+42pv8mLs5jJT2hqFkFE4j21wOmgbUqsZ2hL72NsUU785g9RXgo3s0ZNgVl42TiHp3ZtOv/Vyg==", - "license": "MIT" - }, "node_modules/encodeurl": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/encodeurl/-/encodeurl-2.0.0.tgz", @@ -2857,15 +2303,6 @@ "@esbuild/win32-x64": "0.25.12" } }, - "node_modules/escalade": { - "version": "3.2.0", - "resolved": "https://registry.npmjs.org/escalade/-/escalade-3.2.0.tgz", - "integrity": "sha512-WUj2qlxaQtO4g6Pq5c29GTcWGDyd8itL8zTlipgECz3JesAiiOKotd8JU6otB3PACgG6xkJUyVhboMS+bje/jA==", - "license": "MIT", - "engines": { - "node": ">=6" - } - }, "node_modules/escape-html": { "version": "1.0.3", "resolved": "https://registry.npmjs.org/escape-html/-/escape-html-1.0.3.tgz", @@ -3207,13 +2644,7 @@ "funding": { "type": "opencollective", "url": "https://opencollective.com/express" - } - }, - "node_modules/fast-decode-uri-component": { - "version": "1.0.1", - "resolved": "https://registry.npmjs.org/fast-decode-uri-component/-/fast-decode-uri-component-1.0.1.tgz", - "integrity": "sha512-WKgKWg5eUxvRZGwW8FvfbaH7AXSh2cL+3j5fMGzUMCxWBJ3dV3a7Wz8y2f/uQ0e3B6WmodD3oS54jTQ9HVTIIg==", - "license": "MIT" + } }, "node_modules/fast-deep-equal": { "version": "3.1.3", @@ -3252,30 +2683,6 @@ "dev": true, "license": "MIT" }, - "node_modules/fast-json-stringify": { - "version": "6.1.1", - "resolved": "https://registry.npmjs.org/fast-json-stringify/-/fast-json-stringify-6.1.1.tgz", - "integrity": "sha512-DbgptncYEXZqDUOEl4krff4mUiVrTZZVI7BBrQR/T3BqMj/eM1flTC1Uk2uUoLcWCxjT95xKulV/Lc6hhOZsBQ==", - "funding": [ - { - "type": "github", - "url": "https://github.com/sponsors/fastify" - }, - { - "type": "opencollective", - "url": "https://opencollective.com/fastify" - } - ], - "license": "MIT", - "dependencies": { - "@fastify/merge-json-schemas": "^0.2.0", - "ajv": "^8.12.0", - "ajv-formats": "^3.0.1", - "fast-uri": "^3.0.0", - "json-schema-ref-resolver": "^3.0.0", - "rfdc": "^1.2.0" - } - }, "node_modules/fast-levenshtein": { "version": "2.0.6", "resolved": "https://registry.npmjs.org/fast-levenshtein/-/fast-levenshtein-2.0.6.tgz", @@ -3283,15 +2690,6 @@ "dev": true, "license": "MIT" }, - "node_modules/fast-querystring": { - "version": "1.1.2", - "resolved": "https://registry.npmjs.org/fast-querystring/-/fast-querystring-1.1.2.tgz", - "integrity": "sha512-g6KuKWmFXc0fID8WWH0jit4g0AGBoJhCkJMb1RmbsSEUNvQ+ZC8D6CUZ+GtF8nMzSPXnhiePyyqqipzNNEnHjg==", - "license": "MIT", - "dependencies": { - "fast-decode-uri-component": "^1.0.1" - } - }, "node_modules/fast-uri": { "version": "3.1.0", "resolved": "https://registry.npmjs.org/fast-uri/-/fast-uri-3.1.0.tgz", @@ -3308,59 +2706,11 @@ ], "license": "BSD-3-Clause" }, - "node_modules/fastify": { - "version": "5.6.2", - "resolved": "https://registry.npmjs.org/fastify/-/fastify-5.6.2.tgz", - "integrity": "sha512-dPugdGnsvYkBlENLhCgX8yhyGCsCPrpA8lFWbTNU428l+YOnLgYHR69hzV8HWPC79n536EqzqQtvhtdaCE0dKg==", - "funding": [ - { - "type": "github", - "url": "https://github.com/sponsors/fastify" - }, - { - "type": "opencollective", - "url": "https://opencollective.com/fastify" - } - ], - "license": "MIT", - "dependencies": { - "@fastify/ajv-compiler": "^4.0.0", - "@fastify/error": "^4.0.0", - "@fastify/fast-json-stringify-compiler": "^5.0.0", - "@fastify/proxy-addr": "^5.0.0", - "abstract-logging": "^2.0.1", - "avvio": "^9.0.0", - "fast-json-stringify": "^6.0.0", - "find-my-way": "^9.0.0", - "light-my-request": "^6.0.0", - "pino": "^10.1.0", - "process-warning": "^5.0.0", - "rfdc": "^1.3.1", - "secure-json-parse": "^4.0.0", - "semver": "^7.6.0", - "toad-cache": "^3.7.0" - } - }, - "node_modules/fastify-plugin": { - "version": "5.1.0", - "resolved": "https://registry.npmjs.org/fastify-plugin/-/fastify-plugin-5.1.0.tgz", - "integrity": "sha512-FAIDA8eovSt5qcDgcBvDuX/v0Cjz0ohGhENZ/wpc3y+oZCY2afZ9Baqql3g/lC+OHRnciQol4ww7tuthOb9idw==", - "funding": [ - { - "type": "github", - "url": "https://github.com/sponsors/fastify" - }, - { - "type": "opencollective", - "url": "https://opencollective.com/fastify" - } - ], - "license": "MIT" - }, "node_modules/fastq": { "version": "1.19.1", "resolved": "https://registry.npmjs.org/fastq/-/fastq-1.19.1.tgz", "integrity": "sha512-GwLTyxkCXjXbxqIhTsMI2Nui8huMPtnxg7krajPJAjnEG/iiOS7i+zCtWGZR9G0NBKbXKh6X9m9UIsYX/N6vvQ==", + "dev": true, "license": "ISC", "dependencies": { "reusify": "^1.0.4" @@ -3428,20 +2778,6 @@ "node": ">= 0.8" } }, - "node_modules/find-my-way": { - "version": "9.3.0", - "resolved": "https://registry.npmjs.org/find-my-way/-/find-my-way-9.3.0.tgz", - "integrity": "sha512-eRoFWQw+Yv2tuYlK2pjFS2jGXSxSppAs3hSQjfxVKxM5amECzIgYYc1FEI8ZmhSh/Ig+FrKEz43NLRKJjYCZVg==", - "license": "MIT", - "dependencies": { - "fast-deep-equal": "^3.1.3", - "fast-querystring": "^1.0.0", - "safe-regex2": "^5.0.0" - }, - "engines": { - "node": ">=20" - } - }, "node_modules/find-up": { "version": "5.0.0", "resolved": "https://registry.npmjs.org/find-up/-/find-up-5.0.0.tgz", @@ -3495,22 +2831,6 @@ "url": "https://github.com/sponsors/ljharb" } }, - "node_modules/foreground-child": { - "version": "3.3.1", - "resolved": "https://registry.npmjs.org/foreground-child/-/foreground-child-3.3.1.tgz", - "integrity": "sha512-gIXjKqtFuWEgzFRJA9WCQeSJLZDjgJUOMCMzxtvFq/37KojM1BFGufqsCy0r4qSQmYLsZYMeyRqzIWOMup03sw==", - "license": "ISC", - "dependencies": { - "cross-spawn": "^7.0.6", - "signal-exit": "^4.0.1" - }, - "engines": { - "node": ">=14" - }, - "funding": { - "url": "https://github.com/sponsors/isaacs" - } - }, "node_modules/form-data": { "version": "4.0.5", "resolved": "https://registry.npmjs.org/form-data/-/form-data-4.0.5.tgz", @@ -3594,15 +2914,6 @@ "url": "https://github.com/sponsors/ljharb" } }, - "node_modules/get-caller-file": { - "version": "2.0.5", - "resolved": "https://registry.npmjs.org/get-caller-file/-/get-caller-file-2.0.5.tgz", - "integrity": "sha512-DyFP3BM/3YHTQOCUL/w0OZHR0lpKeGrxotcHWcqNEdnltqFwXVfhEBQ94eIo34AfQpo0rGki4cyIiftY06h2Fg==", - "license": "ISC", - "engines": { - "node": "6.* || 8.* || >= 10.*" - } - }, "node_modules/get-intrinsic": { "version": "1.3.0", "resolved": "https://registry.npmjs.org/get-intrinsic/-/get-intrinsic-1.3.0.tgz", @@ -3666,29 +2977,6 @@ "url": "https://github.com/privatenumber/get-tsconfig?sponsor=1" } }, - "node_modules/glob": { - "version": "11.1.0", - "resolved": "https://registry.npmjs.org/glob/-/glob-11.1.0.tgz", - "integrity": "sha512-vuNwKSaKiqm7g0THUBu2x7ckSs3XJLXE+2ssL7/MfTGPLLcrJQ/4Uq1CjPTtO5cCIiRxqvN6Twy1qOwhL0Xjcw==", - "license": "BlueOak-1.0.0", - "dependencies": { - "foreground-child": "^3.3.1", - "jackspeak": "^4.1.1", - "minimatch": "^10.1.1", - "minipass": "^7.1.2", - "package-json-from-dist": "^1.0.0", - "path-scurry": "^2.0.0" - }, - "bin": { - "glob": "dist/esm/bin.mjs" - }, - "engines": { - "node": "20 || >=22" - }, - "funding": { - "url": "https://github.com/sponsors/isaacs" - } - }, "node_modules/glob-parent": { "version": "5.1.2", "resolved": "https://registry.npmjs.org/glob-parent/-/glob-parent-5.1.2.tgz", @@ -3702,21 +2990,6 @@ "node": ">= 6" } }, - "node_modules/glob/node_modules/minimatch": { - "version": "10.1.1", - "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-10.1.1.tgz", - "integrity": "sha512-enIvLvRAFZYXJzkCYG5RKmPfrFArdLv+R+lbQ53BmIMLIry74bjKzX6iHAm8WYamJkhSSEabrWN5D97XnKObjQ==", - "license": "BlueOak-1.0.0", - "dependencies": { - "@isaacs/brace-expansion": "^5.0.0" - }, - "engines": { - "node": "20 || >=22" - }, - "funding": { - "url": "https://github.com/sponsors/isaacs" - } - }, "node_modules/globals": { "version": "14.0.0", "resolved": "https://registry.npmjs.org/globals/-/globals-14.0.0.tgz", @@ -4003,15 +3276,6 @@ "node": ">=0.10.0" } }, - "node_modules/is-fullwidth-code-point": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/is-fullwidth-code-point/-/is-fullwidth-code-point-3.0.0.tgz", - "integrity": "sha512-zymm5+u+sCsSWyD9qNaejV3DFvhCKclKdizYaJUuHA83RLjb7nSuGnddCHGv0hk+KY7BMAlsWeK4Ueg6EV6XQg==", - "license": "MIT", - "engines": { - "node": ">=8" - } - }, "node_modules/is-glob": { "version": "4.0.3", "resolved": "https://registry.npmjs.org/is-glob/-/is-glob-4.0.3.tgz", @@ -4075,23 +3339,9 @@ "version": "2.0.0", "resolved": "https://registry.npmjs.org/isexe/-/isexe-2.0.0.tgz", "integrity": "sha512-RHxMLp9lnKHGHRng9QFhRCMbYAcVpn69smSGcq3f36xjgVVWThj4qqLbTLlq7Ssj8B+fIQ1EuCEGI2lKsyQeIw==", + "dev": true, "license": "ISC" }, - "node_modules/jackspeak": { - "version": "4.1.1", - "resolved": "https://registry.npmjs.org/jackspeak/-/jackspeak-4.1.1.tgz", - "integrity": "sha512-zptv57P3GpL+O0I7VdMJNBZCu+BPHVQUk55Ft8/QCJjTVxrnJHuVuX/0Bl2A6/+2oyR/ZMEuFKwmzqqZ/U5nPQ==", - "license": "BlueOak-1.0.0", - "dependencies": { - "@isaacs/cliui": "^8.0.2" - }, - "engines": { - "node": "20 || >=22" - }, - "funding": { - "url": "https://github.com/sponsors/isaacs" - } - }, "node_modules/js-yaml": { "version": "4.1.1", "resolved": "https://registry.npmjs.org/js-yaml/-/js-yaml-4.1.1.tgz", @@ -4112,65 +3362,6 @@ "dev": true, "license": "MIT" }, - "node_modules/json-schema-ref-resolver": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/json-schema-ref-resolver/-/json-schema-ref-resolver-3.0.0.tgz", - "integrity": "sha512-hOrZIVL5jyYFjzk7+y7n5JDzGlU8rfWDuYyHwGa2WA8/pcmMHezp2xsVwxrebD/Q9t8Nc5DboieySDpCp4WG4A==", - "funding": [ - { - "type": "github", - "url": "https://github.com/sponsors/fastify" - }, - { - "type": "opencollective", - "url": "https://opencollective.com/fastify" - } - ], - "license": "MIT", - "dependencies": { - "dequal": "^2.0.3" - } - }, - "node_modules/json-schema-resolver": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/json-schema-resolver/-/json-schema-resolver-3.0.0.tgz", - "integrity": "sha512-HqMnbz0tz2DaEJ3ntsqtx3ezzZyDE7G56A/pPY/NGmrPu76UzsWquOpHFRAf5beTNXoH2LU5cQePVvRli1nchA==", - "license": "MIT", - "dependencies": { - "debug": "^4.1.1", - "fast-uri": "^3.0.5", - "rfdc": "^1.1.4" - }, - "engines": { - "node": ">=20" - }, - "funding": { - "url": "https://github.com/Eomm/json-schema-resolver?sponsor=1" - } - }, - "node_modules/json-schema-resolver/node_modules/debug": { - "version": "4.4.3", - "resolved": "https://registry.npmjs.org/debug/-/debug-4.4.3.tgz", - "integrity": "sha512-RGwwWnwQvkVfavKVt22FGLw+xYSdzARwm0ru6DhTVA3umU5hZc28V3kO4stgYryrTlLpuvgI9GiijltAjNbcqA==", - "license": "MIT", - "dependencies": { - "ms": "^2.1.3" - }, - "engines": { - "node": ">=6.0" - }, - "peerDependenciesMeta": { - "supports-color": { - "optional": true - } - } - }, - "node_modules/json-schema-resolver/node_modules/ms": { - "version": "2.1.3", - "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz", - "integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==", - "license": "MIT" - }, "node_modules/json-schema-traverse": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/json-schema-traverse/-/json-schema-traverse-1.0.0.tgz", @@ -4206,13 +3397,6 @@ "json-buffer": "3.0.1" } }, - "node_modules/koffi": { - "version": "2.13.0", - "resolved": "https://registry.npmjs.org/koffi/-/koffi-2.13.0.tgz", - "integrity": "sha512-VqQzBC7XBVJHXA4DkmY68HbH8VTYVaBKq3MFlziI+pdJXIpd/lO4LeXKo2YpIhuTkLgXYra+dDjJOo2+yT1Tsg==", - "hasInstallScript": true, - "license": "MIT" - }, "node_modules/levn": { "version": "0.4.1", "resolved": "https://registry.npmjs.org/levn/-/levn-0.4.1.tgz", @@ -4236,52 +3420,6 @@ "immediate": "~3.0.5" } }, - "node_modules/light-my-request": { - "version": "6.6.0", - "resolved": "https://registry.npmjs.org/light-my-request/-/light-my-request-6.6.0.tgz", - "integrity": "sha512-CHYbu8RtboSIoVsHZ6Ye4cj4Aw/yg2oAFimlF7mNvfDV192LR7nDiKtSIfCuLT7KokPSTn/9kfVLm5OGN0A28A==", - "funding": [ - { - "type": "github", - "url": "https://github.com/sponsors/fastify" - }, - { - "type": "opencollective", - "url": "https://opencollective.com/fastify" - } - ], - "license": "BSD-3-Clause", - "dependencies": { - "cookie": "^1.0.1", - "process-warning": "^4.0.0", - "set-cookie-parser": "^2.6.0" - } - }, - "node_modules/light-my-request/node_modules/cookie": { - "version": "1.0.2", - "resolved": "https://registry.npmjs.org/cookie/-/cookie-1.0.2.tgz", - "integrity": "sha512-9Kr/j4O16ISv8zBBhJoi4bXOYNTkFLOqSL3UDB0njXxCXNezjeyVrJyGOWtgfs/q2km1gwBcfH8q1yEGoMYunA==", - "license": "MIT", - "engines": { - "node": ">=18" - } - }, - "node_modules/light-my-request/node_modules/process-warning": { - "version": "4.0.1", - "resolved": "https://registry.npmjs.org/process-warning/-/process-warning-4.0.1.tgz", - "integrity": "sha512-3c2LzQ3rY9d0hc1emcsHhfT9Jwz0cChib/QN89oME2R451w5fy3f0afAhERFZAwrbDU43wk12d0ORBpDVME50Q==", - "funding": [ - { - "type": "github", - "url": "https://github.com/sponsors/fastify" - }, - { - "type": "opencollective", - "url": "https://opencollective.com/fastify" - } - ], - "license": "MIT" - }, "node_modules/locate-path": { "version": "6.0.0", "resolved": "https://registry.npmjs.org/locate-path/-/locate-path-6.0.0.tgz", @@ -4298,12 +3436,6 @@ "url": "https://github.com/sponsors/sindresorhus" } }, - "node_modules/lodash.camelcase": { - "version": "4.3.0", - "resolved": "https://registry.npmjs.org/lodash.camelcase/-/lodash.camelcase-4.3.0.tgz", - "integrity": "sha512-TwuEnCnxbc3rAvhf/LbG7tJUDzhqXyFnv3dtzLOPgCG/hODL7WFnsbwktkD7yUV0RrreP/l1PALq/YSg6VvjlA==", - "license": "MIT" - }, "node_modules/lodash.merge": { "version": "4.6.2", "resolved": "https://registry.npmjs.org/lodash.merge/-/lodash.merge-4.6.2.tgz", @@ -4323,15 +3455,6 @@ "integrity": "sha512-mNAgZ1GmyNhD7AuqnTG3/VQ26o760+ZYBPKjPvugO8+nLbYfX6TVpJPseBvopbdY+qpZ/lKUnmEc1LeZYS3QAA==", "license": "Apache-2.0" }, - "node_modules/lru-cache": { - "version": "11.2.2", - "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-11.2.2.tgz", - "integrity": "sha512-F9ODfyqML2coTIsQpSkRHnLSZMtkU8Q+mSfcaIyKwy58u+8k5nvAYeiNhsyMARvzNcXJ9QfWVrcPsC9e9rAxtg==", - "license": "ISC", - "engines": { - "node": "20 || >=22" - } - }, "node_modules/make-dir": { "version": "1.3.0", "resolved": "https://registry.npmjs.org/make-dir/-/make-dir-1.3.0.tgz", @@ -4459,24 +3582,6 @@ "node": "*" } }, - "node_modules/minipass": { - "version": "7.1.2", - "resolved": "https://registry.npmjs.org/minipass/-/minipass-7.1.2.tgz", - "integrity": "sha512-qOOzS1cBTWYF4BH8fVePDBOO9iptMnGUEZwNc/cMWnTV2nVLZ7VoNWEPHkYczZA0pdoA7dl6e7FL659nX9S2aw==", - "license": "ISC", - "engines": { - "node": ">=16 || 14 >=14.17" - } - }, - "node_modules/mnemonist": { - "version": "0.40.0", - "resolved": "https://registry.npmjs.org/mnemonist/-/mnemonist-0.40.0.tgz", - "integrity": "sha512-kdd8AFNig2AD5Rkih7EPCXhu/iMvwevQFX/uEiGhZyPZi7fHqOoF4V4kHLpCfysxXMgQ4B52kdPMCwARshKvEg==", - "license": "MIT", - "dependencies": { - "obliterator": "^2.0.4" - } - }, "node_modules/ms": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/ms/-/ms-2.0.0.tgz", @@ -4633,21 +3738,6 @@ "url": "https://github.com/sponsors/ljharb" } }, - "node_modules/obliterator": { - "version": "2.0.5", - "resolved": "https://registry.npmjs.org/obliterator/-/obliterator-2.0.5.tgz", - "integrity": "sha512-42CPE9AhahZRsMNslczq0ctAEtqk8Eka26QofnqC346BZdHDySk3LWka23LI7ULIw11NmltpiLagIq8gBozxTw==", - "license": "MIT" - }, - "node_modules/on-exit-leak-free": { - "version": "2.1.2", - "resolved": "https://registry.npmjs.org/on-exit-leak-free/-/on-exit-leak-free-2.1.2.tgz", - "integrity": "sha512-0eJJY6hXLGf1udHwfNftBqH+g73EU4B504nZeKpz1sYRKafAghwxEJunB2O7rDZkL4PGfsMVnTXZ2EjibbqcsA==", - "license": "MIT", - "engines": { - "node": ">=14.0.0" - } - }, "node_modules/on-finished": { "version": "2.4.1", "resolved": "https://registry.npmjs.org/on-finished/-/on-finished-2.4.1.tgz", @@ -4669,12 +3759,6 @@ "wrappy": "1" } }, - "node_modules/openapi-types": { - "version": "12.1.3", - "resolved": "https://registry.npmjs.org/openapi-types/-/openapi-types-12.1.3.tgz", - "integrity": "sha512-N4YtSYJqghVu4iek2ZUvcN/0aqH1kRDuNqzcycDxhOUpg7GdvLa2F3DgS6yBNhInhv2r/6I0Flkn7CqL8+nIcw==", - "license": "MIT" - }, "node_modules/optionator": { "version": "0.9.4", "resolved": "https://registry.npmjs.org/optionator/-/optionator-0.9.4.tgz", @@ -4725,12 +3809,6 @@ "url": "https://github.com/sponsors/sindresorhus" } }, - "node_modules/package-json-from-dist": { - "version": "1.0.1", - "resolved": "https://registry.npmjs.org/package-json-from-dist/-/package-json-from-dist-1.0.1.tgz", - "integrity": "sha512-UEZIS3/by4OC8vL3P2dTXRETpebLI2NiI5vIrjaD/5UtrkFX/tNbwjTSRAGC/+7CAo2pIcBaRgWmcBBHcsaCIw==", - "license": "BlueOak-1.0.0" - }, "node_modules/pako": { "version": "1.0.11", "resolved": "https://registry.npmjs.org/pako/-/pako-1.0.11.tgz", @@ -4773,27 +3851,12 @@ "version": "3.1.1", "resolved": "https://registry.npmjs.org/path-key/-/path-key-3.1.1.tgz", "integrity": "sha512-ojmeN0qd+y0jszEtoY48r0Peq5dwMEkIlCOu6Q5f41lfkswXuKtYrhgoTpLnyIcHm24Uhqx+5Tqm2InSwLhE6Q==", + "dev": true, "license": "MIT", "engines": { "node": ">=8" } }, - "node_modules/path-scurry": { - "version": "2.0.1", - "resolved": "https://registry.npmjs.org/path-scurry/-/path-scurry-2.0.1.tgz", - "integrity": "sha512-oWyT4gICAu+kaA7QWk/jvCHWarMKNs6pXOGWKDTr7cw4IGcUbW+PeTfbaQiLGheFRpjo6O9J0PmyMfQPjH71oA==", - "license": "BlueOak-1.0.0", - "dependencies": { - "lru-cache": "^11.0.0", - "minipass": "^7.1.2" - }, - "engines": { - "node": "20 || >=22" - }, - "funding": { - "url": "https://github.com/sponsors/isaacs" - } - }, "node_modules/path-to-regexp": { "version": "0.1.12", "resolved": "https://registry.npmjs.org/path-to-regexp/-/path-to-regexp-0.1.12.tgz", @@ -4849,43 +3912,6 @@ "node": ">=0.10.0" } }, - "node_modules/pino": { - "version": "10.1.0", - "resolved": "https://registry.npmjs.org/pino/-/pino-10.1.0.tgz", - "integrity": "sha512-0zZC2ygfdqvqK8zJIr1e+wT1T/L+LF6qvqvbzEQ6tiMAoTqEVK9a1K3YRu8HEUvGEvNqZyPJTtb2sNIoTkB83w==", - "license": "MIT", - "dependencies": { - "@pinojs/redact": "^0.4.0", - "atomic-sleep": "^1.0.0", - "on-exit-leak-free": "^2.1.0", - "pino-abstract-transport": "^2.0.0", - "pino-std-serializers": "^7.0.0", - "process-warning": "^5.0.0", - "quick-format-unescaped": "^4.0.3", - "real-require": "^0.2.0", - "safe-stable-stringify": "^2.3.1", - "sonic-boom": "^4.0.1", - "thread-stream": "^3.0.0" - }, - "bin": { - "pino": "bin.js" - } - }, - "node_modules/pino-abstract-transport": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/pino-abstract-transport/-/pino-abstract-transport-2.0.0.tgz", - "integrity": "sha512-F63x5tizV6WCh4R6RHyi2Ml+M70DNRXt/+HANowMflpgGFMAym/VKm6G7ZOQRjqN7XbGxK1Lg9t6ZrtzOaivMw==", - "license": "MIT", - "dependencies": { - "split2": "^4.0.0" - } - }, - "node_modules/pino-std-serializers": { - "version": "7.0.0", - "resolved": "https://registry.npmjs.org/pino-std-serializers/-/pino-std-serializers-7.0.0.tgz", - "integrity": "sha512-e906FRY0+tV27iq4juKzSYPbUj2do2X2JX4EzSca1631EB2QJQUqGbDuERal7LCtOpxl6x3+nvo9NPZcmjkiFA==", - "license": "MIT" - }, "node_modules/possible-typed-array-names": { "version": "1.1.0", "resolved": "https://registry.npmjs.org/possible-typed-array-names/-/possible-typed-array-names-1.1.0.tgz", @@ -4936,24 +3962,8 @@ }, "node_modules/process-nextick-args": { "version": "2.0.1", - "resolved": "https://registry.npmjs.org/process-nextick-args/-/process-nextick-args-2.0.1.tgz", - "integrity": "sha512-3ouUOpQhtgrbOa17J7+uxOTpITYWaGP7/AhoR3+A+/1e9skrzelGi/dXzEYyvbxubEF6Wn2ypscTKiKJFFn1ag==", - "license": "MIT" - }, - "node_modules/process-warning": { - "version": "5.0.0", - "resolved": "https://registry.npmjs.org/process-warning/-/process-warning-5.0.0.tgz", - "integrity": "sha512-a39t9ApHNx2L4+HBnQKqxxHNs1r7KF+Intd8Q/g1bUh6q0WIp9voPXJ/x0j+ZL45KF1pJd9+q2jLIRMfvEshkA==", - "funding": [ - { - "type": "github", - "url": "https://github.com/sponsors/fastify" - }, - { - "type": "opencollective", - "url": "https://opencollective.com/fastify" - } - ], + "resolved": "https://registry.npmjs.org/process-nextick-args/-/process-nextick-args-2.0.1.tgz", + "integrity": "sha512-3ouUOpQhtgrbOa17J7+uxOTpITYWaGP7/AhoR3+A+/1e9skrzelGi/dXzEYyvbxubEF6Wn2ypscTKiKJFFn1ag==", "license": "MIT" }, "node_modules/protobufjs": { @@ -5046,12 +4056,6 @@ ], "license": "MIT" }, - "node_modules/quick-format-unescaped": { - "version": "4.0.4", - "resolved": "https://registry.npmjs.org/quick-format-unescaped/-/quick-format-unescaped-4.0.4.tgz", - "integrity": "sha512-tYC1Q1hgyRuHgloV/YXs2w15unPVh8qfu/qCTfhTYamaw7fyhumKa2yGpdSo87vY32rIclj+4fWYQXUMs9EHvg==", - "license": "MIT" - }, "node_modules/range-parser": { "version": "1.2.1", "resolved": "https://registry.npmjs.org/range-parser/-/range-parser-1.2.1.tgz", @@ -5110,24 +4114,6 @@ "node": ">=8.10.0" } }, - "node_modules/real-require": { - "version": "0.2.0", - "resolved": "https://registry.npmjs.org/real-require/-/real-require-0.2.0.tgz", - "integrity": "sha512-57frrGM/OCTLqLOAh0mhVA9VBMHd+9U7Zb2THMGdBUoZVOtGbJzjxsYGDJ3A9AYYCP4hn6y1TVbaOfzWtm5GFg==", - "license": "MIT", - "engines": { - "node": ">= 12.13.0" - } - }, - "node_modules/require-directory": { - "version": "2.1.1", - "resolved": "https://registry.npmjs.org/require-directory/-/require-directory-2.1.1.tgz", - "integrity": "sha512-fGxEI7+wsG9xrvdjsrlmL22OMTTiHRwAMroiEeMgq8gzoLC/PQr7RsRDSTLUg/bZAZtF+TVIkHc6/4RIKrui+Q==", - "license": "MIT", - "engines": { - "node": ">=0.10.0" - } - }, "node_modules/require-from-string": { "version": "2.0.2", "resolved": "https://registry.npmjs.org/require-from-string/-/require-from-string-2.0.2.tgz", @@ -5157,31 +4143,17 @@ "url": "https://github.com/privatenumber/resolve-pkg-maps?sponsor=1" } }, - "node_modules/ret": { - "version": "0.5.0", - "resolved": "https://registry.npmjs.org/ret/-/ret-0.5.0.tgz", - "integrity": "sha512-I1XxrZSQ+oErkRR4jYbAyEEu2I0avBvvMM5JN+6EBprOGRCs63ENqZ3vjavq8fBw2+62G5LF5XelKwuJpcvcxw==", - "license": "MIT", - "engines": { - "node": ">=10" - } - }, "node_modules/reusify": { "version": "1.1.0", "resolved": "https://registry.npmjs.org/reusify/-/reusify-1.1.0.tgz", "integrity": "sha512-g6QUff04oZpHs0eG5p83rFLhHeV00ug/Yf9nZM6fLeUrPguBTkTQOdpAWWspMh55TZfVQDPaN3NQJfbVRAxdIw==", + "dev": true, "license": "MIT", "engines": { "iojs": ">=1.0.0", "node": ">=0.10.0" } }, - "node_modules/rfdc": { - "version": "1.4.1", - "resolved": "https://registry.npmjs.org/rfdc/-/rfdc-1.4.1.tgz", - "integrity": "sha512-q1b3N5QkRUWUl7iyylaaj3kOpIT0N2i9MqIEQXP73GVsN9cw3fdx8X63cEmWhJGi2PPCF23Ijp7ktmd39rawIA==", - "license": "MIT" - }, "node_modules/run-parallel": { "version": "1.2.0", "resolved": "https://registry.npmjs.org/run-parallel/-/run-parallel-1.2.0.tgz", @@ -5226,56 +4198,12 @@ ], "license": "MIT" }, - "node_modules/safe-regex2": { - "version": "5.0.0", - "resolved": "https://registry.npmjs.org/safe-regex2/-/safe-regex2-5.0.0.tgz", - "integrity": "sha512-YwJwe5a51WlK7KbOJREPdjNrpViQBI3p4T50lfwPuDhZnE3XGVTlGvi+aolc5+RvxDD6bnUmjVsU9n1eboLUYw==", - "funding": [ - { - "type": "github", - "url": "https://github.com/sponsors/fastify" - }, - { - "type": "opencollective", - "url": "https://opencollective.com/fastify" - } - ], - "license": "MIT", - "dependencies": { - "ret": "~0.5.0" - } - }, - "node_modules/safe-stable-stringify": { - "version": "2.5.0", - "resolved": "https://registry.npmjs.org/safe-stable-stringify/-/safe-stable-stringify-2.5.0.tgz", - "integrity": "sha512-b3rppTKm9T+PsVCBEOUR46GWI7fdOs00VKZ1+9c1EWDaDMvjQc6tUwuFyIprgGgTcWoVHSKrU8H31ZHA2e0RHA==", - "license": "MIT", - "engines": { - "node": ">=10" - } - }, "node_modules/safer-buffer": { "version": "2.1.2", "resolved": "https://registry.npmjs.org/safer-buffer/-/safer-buffer-2.1.2.tgz", "integrity": "sha512-YZo3K82SD7Riyi0E1EQPojLz7kpepnSQI9IyPbHHg1XXXevb5dJI7tpyN2ADxGcQbHG7vcyRHk0cbwqcQriUtg==", "license": "MIT" }, - "node_modules/secure-json-parse": { - "version": "4.1.0", - "resolved": "https://registry.npmjs.org/secure-json-parse/-/secure-json-parse-4.1.0.tgz", - "integrity": "sha512-l4KnYfEyqYJxDwlNVyRfO2E4NTHfMKAWdUuA8J0yve2Dz/E/PdBepY03RvyJpssIpRFwJoCD55wA+mEDs6ByWA==", - "funding": [ - { - "type": "github", - "url": "https://github.com/sponsors/fastify" - }, - { - "type": "opencollective", - "url": "https://opencollective.com/fastify" - } - ], - "license": "BSD-3-Clause" - }, "node_modules/seek-bzip": { "version": "1.0.6", "resolved": "https://registry.npmjs.org/seek-bzip/-/seek-bzip-1.0.6.tgz", @@ -5293,6 +4221,7 @@ "version": "7.7.2", "resolved": "https://registry.npmjs.org/semver/-/semver-7.7.2.tgz", "integrity": "sha512-RF0Fw+rO5AMf9MAyaRXI4AV0Ulj5lMHqVxxdSgiVbixSCXoEmmX/jk0CuJw4+3SqroYO9VoUh+HcuJivvtJemA==", + "dev": true, "license": "ISC", "bin": { "semver": "bin/semver.js" @@ -5355,12 +4284,6 @@ "node": ">= 0.8.0" } }, - "node_modules/set-cookie-parser": { - "version": "2.7.2", - "resolved": "https://registry.npmjs.org/set-cookie-parser/-/set-cookie-parser-2.7.2.tgz", - "integrity": "sha512-oeM1lpU/UvhTxw+g3cIfxXHyJRc/uidd3yK1P242gzHds0udQBYzs3y8j4gCCW+ZJ7ad0yctld8RYO+bdurlvw==", - "license": "MIT" - }, "node_modules/set-function-length": { "version": "1.2.2", "resolved": "https://registry.npmjs.org/set-function-length/-/set-function-length-1.2.2.tgz", @@ -5407,6 +4330,7 @@ "version": "2.0.0", "resolved": "https://registry.npmjs.org/shebang-command/-/shebang-command-2.0.0.tgz", "integrity": "sha512-kHxr2zZpYtdmrN1qDjrrX/Z1rR1kG8Dx+gkpK1G4eXmvXswmcE1hTWBWYUzlraYw1/yZp6YuDY77YtvbN0dmDA==", + "dev": true, "license": "MIT", "dependencies": { "shebang-regex": "^3.0.0" @@ -5419,6 +4343,7 @@ "version": "3.0.0", "resolved": "https://registry.npmjs.org/shebang-regex/-/shebang-regex-3.0.0.tgz", "integrity": "sha512-7++dFhtcx3353uBaq8DDR4NuxBetBzC7ZQOhmTQInHEd6bSrXdiEyzCvG07Z44UYdLShWUyXt5M/yhz8ekcb1A==", + "dev": true, "license": "MIT", "engines": { "node": ">=8" @@ -5496,18 +4421,6 @@ "url": "https://github.com/sponsors/ljharb" } }, - "node_modules/signal-exit": { - "version": "4.1.0", - "resolved": "https://registry.npmjs.org/signal-exit/-/signal-exit-4.1.0.tgz", - "integrity": "sha512-bzyZ1e88w9O1iNJbKnOlvYTrWPDl46O1bG0D3XInv+9tkPrxrN8jUUTiFlDkkmKWgn1M6CfIA13SuGqOa9Korw==", - "license": "ISC", - "engines": { - "node": ">=14" - }, - "funding": { - "url": "https://github.com/sponsors/isaacs" - } - }, "node_modules/simple-update-notifier": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/simple-update-notifier/-/simple-update-notifier-2.0.0.tgz", @@ -5521,24 +4434,6 @@ "node": ">=10" } }, - "node_modules/sonic-boom": { - "version": "4.2.0", - "resolved": "https://registry.npmjs.org/sonic-boom/-/sonic-boom-4.2.0.tgz", - "integrity": "sha512-INb7TM37/mAcsGmc9hyyI6+QR3rR1zVRu36B0NeGXKnOOLiZOfER5SA+N7X7k3yUYRzLWafduTDvJAfDswwEww==", - "license": "MIT", - "dependencies": { - "atomic-sleep": "^1.0.0" - } - }, - "node_modules/split2": { - "version": "4.2.0", - "resolved": "https://registry.npmjs.org/split2/-/split2-4.2.0.tgz", - "integrity": "sha512-UcjcJOWknrNkF6PLX83qcHM6KHgVKNkV62Y8a5uYDVv9ydGQVwAHMKqHdJje1VTWpljG0WYpCDhrCdAOYH4TWg==", - "license": "ISC", - "engines": { - "node": ">= 10.x" - } - }, "node_modules/sql.js": { "version": "0.5.0", "resolved": "https://registry.npmjs.org/sql.js/-/sql.js-0.5.0.tgz", @@ -5569,102 +4464,6 @@ "integrity": "sha512-Gd2UZBJDkXlY7GbJxfsE8/nvKkUEU1G38c1siN6QP6a9PT9MmHB8GnpscSmMJSoF8LOIrt8ud/wPtojys4G6+g==", "license": "MIT" }, - "node_modules/string-width": { - "version": "5.1.2", - "resolved": "https://registry.npmjs.org/string-width/-/string-width-5.1.2.tgz", - "integrity": "sha512-HnLOCR3vjcY8beoNLtcjZ5/nxn2afmME6lhrDrebokqMap+XbeW8n9TXpPDOqdGK5qcI3oT0GKTW6wC7EMiVqA==", - "license": "MIT", - "dependencies": { - "eastasianwidth": "^0.2.0", - "emoji-regex": "^9.2.2", - "strip-ansi": "^7.0.1" - }, - "engines": { - "node": ">=12" - }, - "funding": { - "url": "https://github.com/sponsors/sindresorhus" - } - }, - "node_modules/string-width-cjs": { - "name": "string-width", - "version": "4.2.3", - "resolved": "https://registry.npmjs.org/string-width/-/string-width-4.2.3.tgz", - "integrity": "sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g==", - "license": "MIT", - "dependencies": { - "emoji-regex": "^8.0.0", - "is-fullwidth-code-point": "^3.0.0", - "strip-ansi": "^6.0.1" - }, - "engines": { - "node": ">=8" - } - }, - "node_modules/string-width-cjs/node_modules/ansi-regex": { - "version": "5.0.1", - "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-5.0.1.tgz", - "integrity": "sha512-quJQXlTSUGL2LH9SUXo8VwsY4soanhgo6LNSm84E1LBcE8s3O0wpdiRzyR9z/ZZJMlMWv37qOOb9pdJlMUEKFQ==", - "license": "MIT", - "engines": { - "node": ">=8" - } - }, - "node_modules/string-width-cjs/node_modules/emoji-regex": { - "version": "8.0.0", - "resolved": "https://registry.npmjs.org/emoji-regex/-/emoji-regex-8.0.0.tgz", - "integrity": "sha512-MSjYzcWNOA0ewAHpz0MxpYFvwg6yjy1NG3xteoqz644VCo/RPgnr1/GGt+ic3iJTzQ8Eu3TdM14SawnVUmGE6A==", - "license": "MIT" - }, - "node_modules/string-width-cjs/node_modules/strip-ansi": { - "version": "6.0.1", - "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-6.0.1.tgz", - "integrity": "sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A==", - "license": "MIT", - "dependencies": { - "ansi-regex": "^5.0.1" - }, - "engines": { - "node": ">=8" - } - }, - "node_modules/strip-ansi": { - "version": "7.1.2", - "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-7.1.2.tgz", - "integrity": "sha512-gmBGslpoQJtgnMAvOVqGZpEz9dyoKTCzy2nfz/n8aIFhN/jCE/rCmcxabB6jOOHV+0WNnylOxaxBQPSvcWklhA==", - "license": "MIT", - "dependencies": { - "ansi-regex": "^6.0.1" - }, - "engines": { - "node": ">=12" - }, - "funding": { - "url": "https://github.com/chalk/strip-ansi?sponsor=1" - } - }, - "node_modules/strip-ansi-cjs": { - "name": "strip-ansi", - "version": "6.0.1", - "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-6.0.1.tgz", - "integrity": "sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A==", - "license": "MIT", - "dependencies": { - "ansi-regex": "^5.0.1" - }, - "engines": { - "node": ">=8" - } - }, - "node_modules/strip-ansi-cjs/node_modules/ansi-regex": { - "version": "5.0.1", - "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-5.0.1.tgz", - "integrity": "sha512-quJQXlTSUGL2LH9SUXo8VwsY4soanhgo6LNSm84E1LBcE8s3O0wpdiRzyR9z/ZZJMlMWv37qOOb9pdJlMUEKFQ==", - "license": "MIT", - "engines": { - "node": ">=8" - } - }, "node_modules/strip-dirs": { "version": "2.1.0", "resolved": "https://registry.npmjs.org/strip-dirs/-/strip-dirs-2.1.0.tgz", @@ -5734,15 +4533,6 @@ "node": ">= 0.8.0" } }, - "node_modules/thread-stream": { - "version": "3.1.0", - "resolved": "https://registry.npmjs.org/thread-stream/-/thread-stream-3.1.0.tgz", - "integrity": "sha512-OqyPZ9u96VohAyMfJykzmivOrY2wfMSf3C5TtFJVgN+Hm6aj+voFhlK+kZEIv2FBh1X6Xp3DlnCOfEQ3B2J86A==", - "license": "MIT", - "dependencies": { - "real-require": "^0.2.0" - } - }, "node_modules/through": { "version": "2.3.8", "resolved": "https://registry.npmjs.org/through/-/through-2.3.8.tgz", @@ -5782,15 +4572,6 @@ "node": ">=8.0" } }, - "node_modules/toad-cache": { - "version": "3.7.0", - "resolved": "https://registry.npmjs.org/toad-cache/-/toad-cache-3.7.0.tgz", - "integrity": "sha512-/m8M+2BJUpoJdgAHoG+baCwBT+tf2VraSfkBgl0Y00qIWt41DJ8R5B8nsEw0I58YwF5IZH6z24/2TobDKnqSWw==", - "license": "MIT", - "engines": { - "node": ">=12" - } - }, "node_modules/toidentifier": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/toidentifier/-/toidentifier-1.0.1.tgz", @@ -6035,6 +4816,7 @@ "version": "2.0.2", "resolved": "https://registry.npmjs.org/which/-/which-2.0.2.tgz", "integrity": "sha512-BLI3Tl1TW3Pvl70l3yq3Y64i+awpwXqsGBYWkkqMtnbXgrMD+yj7rhW0kuEDxzJaYXGjEW5ogapKNMEKNMjibA==", + "dev": true, "license": "ISC", "dependencies": { "isexe": "^2.0.0" @@ -6077,97 +4859,6 @@ "node": ">=0.10.0" } }, - "node_modules/wrap-ansi": { - "version": "8.1.0", - "resolved": "https://registry.npmjs.org/wrap-ansi/-/wrap-ansi-8.1.0.tgz", - "integrity": "sha512-si7QWI6zUMq56bESFvagtmzMdGOtoxfR+Sez11Mobfc7tm+VkUckk9bW2UeffTGVUbOksxmSw0AA2gs8g71NCQ==", - "license": "MIT", - "dependencies": { - "ansi-styles": "^6.1.0", - "string-width": "^5.0.1", - "strip-ansi": "^7.0.1" - }, - "engines": { - "node": ">=12" - }, - "funding": { - "url": "https://github.com/chalk/wrap-ansi?sponsor=1" - } - }, - "node_modules/wrap-ansi-cjs": { - "name": "wrap-ansi", - "version": "7.0.0", - "resolved": "https://registry.npmjs.org/wrap-ansi/-/wrap-ansi-7.0.0.tgz", - "integrity": "sha512-YVGIj2kamLSTxw6NsZjoBxfSwsn0ycdesmc4p+Q21c5zPuZ1pl+NfxVdxPtdHvmNVOQ6XSYG4AUtyt/Fi7D16Q==", - "license": "MIT", - "dependencies": { - "ansi-styles": "^4.0.0", - "string-width": "^4.1.0", - "strip-ansi": "^6.0.0" - }, - "engines": { - "node": ">=10" - }, - "funding": { - "url": "https://github.com/chalk/wrap-ansi?sponsor=1" - } - }, - "node_modules/wrap-ansi-cjs/node_modules/ansi-regex": { - "version": "5.0.1", - "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-5.0.1.tgz", - "integrity": "sha512-quJQXlTSUGL2LH9SUXo8VwsY4soanhgo6LNSm84E1LBcE8s3O0wpdiRzyR9z/ZZJMlMWv37qOOb9pdJlMUEKFQ==", - "license": "MIT", - "engines": { - "node": ">=8" - } - }, - "node_modules/wrap-ansi-cjs/node_modules/ansi-styles": { - "version": "4.3.0", - "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-4.3.0.tgz", - "integrity": "sha512-zbB9rCJAT1rbjiVDb2hqKFHNYLxgtk8NURxZ3IZwD3F6NtxbXZQCnnSi1Lkx+IDohdPlFp222wVALIheZJQSEg==", - "license": "MIT", - "dependencies": { - "color-convert": "^2.0.1" - }, - "engines": { - "node": ">=8" - }, - "funding": { - "url": "https://github.com/chalk/ansi-styles?sponsor=1" - } - }, - "node_modules/wrap-ansi-cjs/node_modules/emoji-regex": { - "version": "8.0.0", - "resolved": "https://registry.npmjs.org/emoji-regex/-/emoji-regex-8.0.0.tgz", - "integrity": "sha512-MSjYzcWNOA0ewAHpz0MxpYFvwg6yjy1NG3xteoqz644VCo/RPgnr1/GGt+ic3iJTzQ8Eu3TdM14SawnVUmGE6A==", - "license": "MIT" - }, - "node_modules/wrap-ansi-cjs/node_modules/string-width": { - "version": "4.2.3", - "resolved": "https://registry.npmjs.org/string-width/-/string-width-4.2.3.tgz", - "integrity": "sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g==", - "license": "MIT", - "dependencies": { - "emoji-regex": "^8.0.0", - "is-fullwidth-code-point": "^3.0.0", - "strip-ansi": "^6.0.1" - }, - "engines": { - "node": ">=8" - } - }, - "node_modules/wrap-ansi-cjs/node_modules/strip-ansi": { - "version": "6.0.1", - "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-6.0.1.tgz", - "integrity": "sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A==", - "license": "MIT", - "dependencies": { - "ansi-regex": "^5.0.1" - }, - "engines": { - "node": ">=8" - } - }, "node_modules/wrappy": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/wrappy/-/wrappy-1.0.2.tgz", @@ -6204,95 +4895,6 @@ "node": ">=0.4" } }, - "node_modules/y18n": { - "version": "5.0.8", - "resolved": "https://registry.npmjs.org/y18n/-/y18n-5.0.8.tgz", - "integrity": "sha512-0pfFzegeDWJHJIAmTLRP2DwHjdF5s7jo9tuztdQxAhINCdvS+3nGINqPd00AphqJR/0LhANUS6/+7SCb98YOfA==", - "license": "ISC", - "engines": { - "node": ">=10" - } - }, - "node_modules/yaml": { - "version": "2.8.1", - "resolved": "https://registry.npmjs.org/yaml/-/yaml-2.8.1.tgz", - "integrity": "sha512-lcYcMxX2PO9XMGvAJkJ3OsNMw+/7FKes7/hgerGUYWIoWu5j/+YQqcZr5JnPZWzOsEBgMbSbiSTn/dv/69Mkpw==", - "license": "ISC", - "bin": { - "yaml": "bin.mjs" - }, - "engines": { - "node": ">= 14.6" - } - }, - "node_modules/yargs": { - "version": "17.7.2", - "resolved": "https://registry.npmjs.org/yargs/-/yargs-17.7.2.tgz", - "integrity": "sha512-7dSzzRQ++CKnNI/krKnYRV7JKKPUXMEh61soaHKg9mrWEhzFWhFnxPxGl+69cD1Ou63C13NUPCnmIcrvqCuM6w==", - "license": "MIT", - "dependencies": { - "cliui": "^8.0.1", - "escalade": "^3.1.1", - "get-caller-file": "^2.0.5", - "require-directory": "^2.1.1", - "string-width": "^4.2.3", - "y18n": "^5.0.5", - "yargs-parser": "^21.1.1" - }, - "engines": { - "node": ">=12" - } - }, - "node_modules/yargs-parser": { - "version": "21.1.1", - "resolved": "https://registry.npmjs.org/yargs-parser/-/yargs-parser-21.1.1.tgz", - "integrity": "sha512-tVpsJW7DdjecAiFpbIB1e3qxIQsE6NoPc5/eTdrbbIC4h0LVsWhnoa3g+m2HclBIujHzsxZ4VJVA+GUuc2/LBw==", - "license": "ISC", - "engines": { - "node": ">=12" - } - }, - "node_modules/yargs/node_modules/ansi-regex": { - "version": "5.0.1", - "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-5.0.1.tgz", - "integrity": "sha512-quJQXlTSUGL2LH9SUXo8VwsY4soanhgo6LNSm84E1LBcE8s3O0wpdiRzyR9z/ZZJMlMWv37qOOb9pdJlMUEKFQ==", - "license": "MIT", - "engines": { - "node": ">=8" - } - }, - "node_modules/yargs/node_modules/emoji-regex": { - "version": "8.0.0", - "resolved": "https://registry.npmjs.org/emoji-regex/-/emoji-regex-8.0.0.tgz", - "integrity": "sha512-MSjYzcWNOA0ewAHpz0MxpYFvwg6yjy1NG3xteoqz644VCo/RPgnr1/GGt+ic3iJTzQ8Eu3TdM14SawnVUmGE6A==", - "license": "MIT" - }, - "node_modules/yargs/node_modules/string-width": { - "version": "4.2.3", - "resolved": "https://registry.npmjs.org/string-width/-/string-width-4.2.3.tgz", - "integrity": "sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g==", - "license": "MIT", - "dependencies": { - "emoji-regex": "^8.0.0", - "is-fullwidth-code-point": "^3.0.0", - "strip-ansi": "^6.0.1" - }, - "engines": { - "node": ">=8" - } - }, - "node_modules/yargs/node_modules/strip-ansi": { - "version": "6.0.1", - "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-6.0.1.tgz", - "integrity": "sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A==", - "license": "MIT", - "dependencies": { - "ansi-regex": "^5.0.1" - }, - "engines": { - "node": ">=8" - } - }, "node_modules/yauzl": { "version": "2.10.0", "resolved": "https://registry.npmjs.org/yauzl/-/yauzl-2.10.0.tgz", @@ -6315,6 +4917,25 @@ "funding": { "url": "https://github.com/sponsors/sindresorhus" } + }, + "node_modules/zod": { + "version": "4.2.1", + "resolved": "https://registry.npmjs.org/zod/-/zod-4.2.1.tgz", + "integrity": "sha512-0wZ1IRqGGhMP76gLqz8EyfBXKk0J2qo2+H3fi4mcUP/KtTocoX08nmIAHl1Z2kJIZbZee8KOpBCSNPRgauucjw==", + "license": "MIT", + "peer": true, + "funding": { + "url": "https://github.com/sponsors/colinhacks" + } + }, + "node_modules/zod-to-json-schema": { + "version": "3.25.1", + "resolved": "https://registry.npmjs.org/zod-to-json-schema/-/zod-to-json-schema-3.25.1.tgz", + "integrity": "sha512-pM/SU9d3YAggzi6MtR4h7ruuQlqKtad8e9S0fmxcMi+ueAK5Korys/aWcV9LIIHTVbj01NdzxcnXSN+O74ZIVA==", + "license": "ISC", + "peerDependencies": { + "zod": "^3.25 || ^4" + } } } } diff --git a/package.json b/package.json index 5a52a59..19fbda3 100644 --- a/package.json +++ b/package.json @@ -43,7 +43,7 @@ "typescript-eslint": "^8.0.0" }, "dependencies": { - "@inworld/runtime": "^0.8.0", + "@inworld/runtime": "https://storage.googleapis.com/assets-inworld-ai/node-packages/inworld-runtime-0.9.0-rc.13.tgz", "anki-apkg-export": "^4.0.3", "dotenv": "^17.2.1", "express": "^4.19.2", diff --git a/realtime-service/.env-sample b/realtime-service/.env-sample new file mode 100644 index 0000000..ffdcd2e --- /dev/null +++ b/realtime-service/.env-sample @@ -0,0 +1,39 @@ +# DO NOT EDIT THIS FILE DIRECTLY +# Copy this file to .env and enter your own values +# +# INWORLD_API_KEY is required +INWORLD_API_KEY= +# ASSEMBLYAI_API_KEY is required for audio transcription +ASSEMBLYAI_API_KEY= +# VAD_MODEL_PATH is optional, defaults to packaged https://github.com/snakers4/silero-vad +VAD_MODEL_PATH= +# LLM_MODEL_NAME is optional, defaults to `gpt-4o-mini` +LLM_MODEL_NAME= +# LLM_PROVIDER is optional, defaults to `openai` +LLM_PROVIDER= +# VOICE_ID is optional, defaults to `Dennis` +VOICE_ID= +# TTS_MODEL_ID is optional, defaults to `inworld-tts-1` +TTS_MODEL_ID= +# If enabled, it will be saved in system tmp folder. +# Path will be printed in CLI on application start. +# Default value is `false`, set `true` to enable this feature +GRAPH_VISUALIZATION_ENABLED= +# If enabled, the user can stop the agent's audio playback with +# text or speech input. For speech input use case, users could accidentally interrupt +# in the case of false postivie speech detection. You can tune the following parameters +# to achieve the right level of speech detection sensitivy for your application: +# MIN_SPEECH_DURATION_MS, PAUSE_DURATION_THRESHOLD_MS, SPEECH_THRESHOLD +# AUTH_TOKEN is optional, but recommended for basic security. +# If set, clients must provide this token in the Authorization header. +AUTH_TOKEN= + +# Logging configuration +# REALTIME_LOG_LEVEL: 'debug', 'info', 'warn', 'error' (default: 'info') +# REALTIME_LOG_PRETTY: '1' to enable pretty logs locally (default: '0' = JSON) +# Examples: +# npm start # JSON logs (default, cloud-ready) +# REALTIME_LOG_PRETTY=1 npm start # Pretty logs (local dev) +# REALTIME_LOG_LEVEL=debug npm start # JSON with debug level +REALTIME_LOG_LEVEL=info +REALTIME_LOG_PRETTY=1 \ No newline at end of file diff --git a/realtime-service/Makefile b/realtime-service/Makefile new file mode 100644 index 0000000..711dd59 --- /dev/null +++ b/realtime-service/Makefile @@ -0,0 +1,53 @@ +.PHONY: build docker-build docker-run local-install local-start local-build help + +# Docker image name +IMAGE_NAME ?= oai-realtime-api-voice-agent +IMAGE_TAG ?= latest + +GH_TOKEN ?= + +# Default target +help: + @echo "Available commands:" + @echo " make docker-build - Build Docker image" + @echo " make docker-run - Run Docker container" + @echo " make local-install - Install dependencies locally" + @echo " make local-start - Start server locally (development mode)" + @echo " make local-build - Build TypeScript project locally" + @echo " make help - Show this help message" + @echo "" + @echo "Note: For docker-build, you can set GITHUB_TOKEN environment variable" + @echo " to download @inworld/runtime binaries from GitHub:" + @echo " GH_TOKEN=your_token make docker-build" + +# Build Docker image +docker-build: + @echo "Building Docker image: $(IMAGE_NAME):$(IMAGE_TAG)" + @if [ -n "$(GH_TOKEN)" ]; then \ + echo "Using GH_TOKEN for downloading binaries..."; \ + docker build --platform linux/amd64 --build-arg GH_TOKEN=$(GH_TOKEN) --build-arg SERVICE_PATH=serving/realtime-service --build-arg SERVICE_PORT=4000 -t $(IMAGE_NAME):$(IMAGE_TAG) -f ../../.github/builds/Dockerfile.node-service.common ../..; \ + else \ + echo "Warning: GH_TOKEN not set. Build may fail if @inworld/runtime binaries require authentication."; \ + docker build --platform linux/amd64 --build-arg SERVICE_PATH=serving/realtime-service --build-arg SERVICE_PORT=4000 -t $(IMAGE_NAME):$(IMAGE_TAG) -f ../../.github/builds/Dockerfile.node-service.common ../..; \ + fi + +# Run Docker container +docker-run: + @echo "Running Docker container: $(IMAGE_NAME):$(IMAGE_TAG)" + docker run --platform linux/amd64 -p 4000:4000 -p 9000:9000 --env-file .env -e LLM_PROVIDER=groq -e LLM_MODEL_NAME=llama-3.3-70b-versatile $(IMAGE_NAME):$(IMAGE_TAG) + +# Install dependencies locally +local-install: + @echo "Installing dependencies..." + cd server && npm install + +# Build TypeScript project locally +local-build: + @echo "Building TypeScript project..." + cd server && npm run build + +# Start server locally (development mode with nodemon) +local-start: local-install + @echo "Starting server locally..." + cd server && npm start + diff --git a/realtime-service/README.md b/realtime-service/README.md new file mode 100644 index 0000000..7ba1c04 --- /dev/null +++ b/realtime-service/README.md @@ -0,0 +1,91 @@ +# Inworld Realtime Service + +This is an Inworld service that implements a web-sockets OpenAI Realtime API compatible API. + +## Building and Running with Docker + +### Build Docker Image + +```bash +make docker-build +``` + +Optionally, you can provide a GitHub token to download `@inworld/runtime` binaries: + +```bash +GH_TOKEN=your_token make docker-build +``` + +### Run Docker Container + +```bash +make docker-run +``` + +This will run the container on port 4000. Make sure you have a `.env` file in the root directory with the required environment variables. + +## Building and Running without Docker + +### Install Dependencies + +```bash +cd src +npm install +``` + +### Build Project + +```bash +cd src +npm run build +``` + +### Run the Server + +For development (with nodemon): + +```bash +cd src +npm start +``` + +The server will start on port 4000 by default (or the port specified in your environment variables). + +## Configuration + +### Logging + +The service outputs JSON logs by default (Google Cloud Logging compatible). + +**Environment Variables:** +- `REALTIME_LOG_LEVEL` - Log level: `debug`, `info`, `warn`, `error` (default: `info`) +- `REALTIME_LOG_PRETTY` - Set to `1` for human-readable logs during development (default: `0` = JSON) + +**Examples:** +```bash +npm start # JSON logs (production-ready) +REALTIME_LOG_PRETTY=1 npm start # Pretty logs (local dev) +REALTIME_LOG_LEVEL=debug npm start # JSON with debug level +REALTIME_LOG_PRETTY=1 REALTIME_LOG_LEVEL=debug npm start # Pretty with debug level +``` + +See [LOGGING.md](LOGGING.md) for detailed logging best practices. + +## Test websocket connection +### Locally running w-proxy and realtime service +``` +ws://localhost:8081/api/v1/realtime/session?key=&protocol=realtime +``` +with Authorization header + +### Dev: running w-proxy and realtime service +``` +wss://api.dev.inworld.ai:443/api/v1/realtime/session?key=&protocol=realtime +``` +with Authorization header + +### Dev: tailscale and directly to realtime service +``` +ws://realtime-service-dev.tail4c73a.ts.net:4000/session?key=&protocol=realtime +``` +no need to provide Authorization header (debug only) diff --git a/realtime-service/__tests__/README.md b/realtime-service/__tests__/README.md new file mode 100644 index 0000000..1ba3380 --- /dev/null +++ b/realtime-service/__tests__/README.md @@ -0,0 +1,469 @@ +# Realtime Service Tests + +Comprehensive API-level integration tests for the OpenAI Realtime API WebSocket service. + +## Quick Start + +### 1. Install Dependencies +```bash +npm install +``` + +### 2. Start Server (Terminal 1) +```bash +cd src && npm start +``` +Wait for: `Application Server listening on port 4000` + +### 3. Run Tests (Terminal 2) +```bash +npm test +``` + +That's it! ✅ + +> **TL;DR:** See [`SIMPLE_START.md`](./SIMPLE_START.md) for the absolute quickest guide. + +--- + +## Table of Contents + +- [What We Test](#what-we-test) +- [Running Tests](#running-tests) +- [Writing Tests](#writing-tests) +- [Test Utilities](#test-utilities) +- [Troubleshooting](#troubleshooting) +- [Why API Tests Only](#why-api-tests-only) +- [CI/CD Integration](#cicd-integration) + +--- + +## What We Test + +We use **API-level integration tests** that validate complete WebSocket behavior: + +✅ **Session Management** (3 tests) - Creation, configuration, turn detection +✅ **Conversation Management** (5 tests) - CRUD operations on items +✅ **Response Generation** (4 tests) - Complete response lifecycle +✅ **Function Calling** (1 test) - Tool execution +✅ **Audio Input** (1 test) - Buffer operations +✅ **Error Handling** (2 tests) - Invalid requests +✅ **Multi-Session** (1 test) - Concurrent sessions +✅ **Workspace Isolation** (1 test) - Multi-tenancy + +**Total: 18 passing tests, 1 skipped** (~6 seconds runtime) + +--- + +## Running Tests + +### Basic Commands + +```bash +# Run all tests (requires server running) +npm test + +# Run in watch mode +npm run test:watch + +# Run with coverage report +npm run test:coverage + +# Run specific test +npm test -- -t "should create a session" +``` + +### Environment Variables + +Optionally create `.env.test` in the project root: + +```bash +INWORLD_API_KEY=your-api-key +WORKSPACE_ID=your-workspace-id +``` + +**Note:** `WS_APP_PORT` is automatically managed by tests - they find an available port dynamically. + +--- + +## Writing Tests + +### Test Structure + +```typescript +import { v4 as uuidv4 } from 'uuid'; +import { createWebSocketTestClient, createTextMessage } from './websocket-test-helper'; + +it('should test some API behavior', async () => { + // 1. Create client with unique session ID + const sessionKey = `test-session-${uuidv4()}`; + const client = await createWebSocketTestClient(sessionKey, 4000); + + try { + // 2. Wait for session initialization + await client.waitForEvent('session.created'); + + // 3. Send client event + client.sendEvent({ + type: 'conversation.item.create', + item: createTextMessage('Hello, world!'), + }); + + // 4. Verify server response + const addedEvent = await client.waitForEvent('conversation.item.added'); + expect(addedEvent.item.content[0].text).toBe('Hello, world!'); + + } finally { + // 5. Always clean up + await client.close(); + } +}); +``` + +### Best Practices + +✅ **Use unique session IDs** - `uuidv4()` for each test +✅ **Always close connections** - Use try/finally blocks +✅ **Set appropriate timeouts** - Default is 5000ms, adjust if needed +✅ **Test both success and errors** - Don't just test happy paths +✅ **Keep tests independent** - No shared state between tests + +--- + +## Test Utilities + +### Creating a Test Client + +```typescript +const client = await createWebSocketTestClient( + 'session-123', // session key + 4000, // port (optional, defaults to 3001) + 'workspace-id' // workspace ID (optional) +); +``` + +### Waiting for Events + +```typescript +// Wait for single event +const event = await client.waitForEvent('session.created', 5000); + +// Wait for multiple events in sequence +const [event1, event2] = await client.waitForEvents([ + 'conversation.item.added', + 'conversation.item.done' +], 10000); + +// Wait for complete response cycle +const response = await waitForCompleteResponse(client); +const text = extractTextFromResponse(response.allEvents); +``` + +### Sending Events + +```typescript +// Send any client event +client.sendEvent({ + type: 'conversation.item.create', + item: createTextMessage('Hello'), +}); + +// Create different message types +const userMsg = createTextMessage('Hello', 'user', 'msg-123'); +const functionOutput = createFunctionCallOutput('call-123', '{"result": true}'); +``` + +### Assertions + +```typescript +// Assert event properties +assertEvent(event, 'session.created', (evt) => { + expect(evt.session.id).toBeDefined(); + expect(evt.session.model).toBe('expected-model'); +}); +``` + +### All Helper Functions + +- `createWebSocketTestClient()` - Create test client +- `waitForEvent()` - Wait for specific event +- `waitForEvents()` - Wait for multiple events +- `waitForCompleteResponse()` - Wait for full response +- `assertEvent()` - Assert event properties +- `createTextMessage()` - Create text message item +- `createFunctionCallOutput()` - Create function output +- `extractTextFromResponse()` - Extract text from events +- `extractAudioTranscriptFromResponse()` - Extract audio transcript + +See [`api/README.md`](./api/README.md) for detailed API documentation. + +--- + +## Troubleshooting + +### Connection Refused + +**Problem:** Tests fail with `ECONNREFUSED` +**Solution:** Start the server first: +```bash +cd src && npm start +``` +Wait for "Application Server listening on port 4000", then run tests. + +### Tests Timeout + +**Problem:** Tests timeout waiting for events +**Solutions:** +- Check server logs for errors +- Verify environment variables are set +- Increase timeout: `await client.waitForEvent('event', 30000)` +- Make sure server is running on correct port (4000) + +### Port Already in Use + +**Problem:** Can't start server - port 4000 in use +**Solution:** +```bash +# Kill process on port 4000 +lsof -ti:4000 | xargs kill + +# Or use different port +WS_APP_PORT=4001 npm start # Terminal 1 +WS_APP_PORT=4001 npm test # Terminal 2 +``` + +### Module Not Found + +**Problem:** `Cannot find module 'uuid'` +**Solution:** Run `npm install` + +### Debugging Tests + +```typescript +// View all received events +it('debug test', async () => { + const client = await createWebSocketTestClient('test'); + await client.waitForEvent('session.created'); + + console.log('Events:', JSON.stringify(client.events, null, 2)); +}); + +// Increase timeout for debugging +jest.setTimeout(300000); // 5 minutes +``` + +--- + +## Why API Tests Only? + +We chose **API integration tests** over traditional unit tests. + +### What We DON'T Do ❌ + +```typescript +// Unit testing internal nodes (we avoid this) +it('node should process input', () => { + const node = new TextInputNode(config); + const result = node.process(context, input); + expect(result.messages).toContainEqual(...); +}); +``` +**Problem:** Tests implementation details, breaks on refactoring + +### What We DO ✅ + +```typescript +// Testing API behavior (our approach) +it('should create conversation item', async () => { + const client = await createWebSocketTestClient('session-123'); + await client.waitForEvent('session.created'); + + client.sendEvent({ + type: 'conversation.item.create', + item: createTextMessage('Hello'), + }); + + const event = await client.waitForEvent('conversation.item.added'); + expect(event.item.content[0].text).toBe('Hello'); +}); +``` +**Benefit:** Tests actual behavior, survives refactoring + +### Benefits + +1. **Tests Real Behavior** - Validates what users actually experience +2. **OpenAI Compliance** - Ensures API contract adherence +3. **Refactor-Safe** - Tests survive internal code changes +4. **Living Documentation** - Tests show how to use the API +5. **Higher Confidence** - Tests complete integration paths +6. **Fewer Tests** - 18 comprehensive tests vs 100+ unit tests + +### Trade-offs + +We accept these trade-offs for better overall testing: + +- ⚠️ Requires running server (~1 command in separate terminal) +- ⚠️ Slower execution (~6 seconds vs milliseconds for unit tests) +- ⚠️ May need environment configuration for API keys +- ⚠️ Debugging spans multiple components + +**Result:** Better testing with less maintenance burden. Simple setup, comprehensive coverage! + +--- + +## CI/CD Integration + +### GitHub Actions Example + +```yaml +name: API Tests + +on: [push, pull_request] + +jobs: + api-tests: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + + - uses: actions/setup-node@v2 + with: + node-version: '18' + + - name: Install dependencies + run: npm install + + - name: Start server + run: cd src && npm start & + + - name: Wait for server + run: sleep 5 + + - name: Run tests + run: npm test + + - name: Upload coverage + uses: codecov/codecov-action@v2 + if: always() +``` + +--- + +## Project Structure + +``` +__tests__/ +├── api/ +│ ├── realtime-api.spec.ts # 18 test cases +│ ├── websocket-test-helper.ts # Test utilities +│ ├── websocket-server-helper.ts +│ └── README.md # Detailed API docs +│ +├── utils/ # Legacy (kept for reference) +│ ├── mock-helpers.ts +│ └── graph-test-helpers.ts +│ +├── config.ts # Test constants +├── setup.ts # Jest setup +├── test-setup.ts # Test environment +└── README.md # This file +``` + +--- + +## Test Statistics + +- **Test Files:** 1 (`realtime-api.spec.ts`) +- **Test Cases:** 18 passing, 1 skipped +- **Test Duration:** ~6 seconds +- **Test Categories:** 8 +- **Lines of Code:** ~1,100 test code + +### Latest Results + +``` +PASS __tests__/api/realtime-api.spec.ts (6.704 s) + ✓ Session Management (3/3) + ✓ Conversation Management (5/5) + ✓ Response Generation (4/4) + ✓ Function Calling (1/1) + ✓ Audio Input Management (1/2, 1 skipped) + ✓ Error Handling (2/2) + ✓ Multi-Session Support (1/1) + ✓ Workspace Isolation (1/1) + +Tests: 18 passed, 1 skipped, 19 total +Time: 6.826 s +``` + +--- + +## Configuration + +### Jest (`jest.config.js`) + +- Timeout: 30 seconds +- Environment: Node.js +- Uses: ts-jest with `tsconfig.test.json` + +### TypeScript (`tsconfig.test.json`) + +- Extends: `src/tsconfig.json` +- Includes: `__tests__/**/*.ts` +- CommonJS modules for Jest compatibility + +### Environment (`.env.test`) + +```bash +WS_APP_PORT=4000 +INWORLD_API_KEY=your-key +WORKSPACE_ID=your-workspace +``` + +--- + +## Contributing + +When adding new features: + +1. **Add API tests** for the new behavior +2. **Test success and error cases** +3. **Use existing tests as templates** +4. **Update this README** if patterns change +5. **Ensure all tests pass** before committing + +### Example: Adding a New Test + +```typescript +describe('New Feature', () => { + let client: WebSocketTestClient; + const sessionKey = `test-session-${uuidv4()}`; + + beforeEach(async () => { + client = await createWebSocketTestClient(sessionKey); + await client.waitForEvent('session.created'); + }); + + afterEach(async () => { + if (client) await client.close(); + }); + + it('should handle new feature', async () => { + // Test implementation + }); +}); +``` + +--- + +## Support + +- **Detailed API docs:** [`api/README.md`](./api/README.md) +- **Jest documentation:** https://jestjs.io/ +- **WebSocket API spec:** Check `src/REALTIME_API.md` + +--- + +**Last Updated:** November 2024 +**Status:** ✅ All tests passing +**Test Coverage:** Run `npm run test:coverage` to check diff --git a/realtime-service/__tests__/api/README.md b/realtime-service/__tests__/api/README.md new file mode 100644 index 0000000..c9a7f86 --- /dev/null +++ b/realtime-service/__tests__/api/README.md @@ -0,0 +1,312 @@ +# API Test Utilities Reference + +Helper functions and utilities for writing API integration tests. + +> **Main Documentation:** See [`../__tests__/README.md`](../README.md) for complete testing guide. + +--- + +## WebSocket Test Client + +### Creating a Client + +```typescript +import { createWebSocketTestClient } from './websocket-test-helper'; + +const client = await createWebSocketTestClient( + sessionKey, // string: unique session identifier + port?, // number: server port (default: 3001) + workspaceId? // string: optional workspace ID +); +``` + +### Client Interface + +```typescript +interface WebSocketTestClient { + ws: WebSocket; // Raw WebSocket instance + events: RT.ServerEvent[]; // All received events + + // Wait for specific event + waitForEvent(eventType: string, timeout?: number): Promise; + + // Wait for multiple events in sequence + waitForEvents(eventTypes: string[], timeout?: number): Promise; + + // Send client event + sendEvent(event: RT.ClientEvent): void; + + // Close connection + close(): Promise; +} +``` + +--- + +## Event Utilities + +### Wait for Event + +```typescript +// Wait for single event (default 5s timeout) +const event = await client.waitForEvent('session.created'); + +// With custom timeout +const event = await client.waitForEvent('response.done', 30000); +``` + +### Wait for Multiple Events + +```typescript +const [event1, event2] = await client.waitForEvents([ + 'conversation.item.added', + 'conversation.item.done' +], 10000); +``` + +### Wait for Complete Response + +```typescript +import { waitForCompleteResponse } from './websocket-test-helper'; + +const response = await waitForCompleteResponse(client, 30000); +// Returns: { created, done, allEvents } +``` + +### Collect Events Until + +```typescript +import { collectEventsUntil } from './websocket-test-helper'; + +const events = await collectEventsUntil(client, 'response.done', 10000); +``` + +--- + +## Message Creation + +### Create Text Message + +```typescript +import { createTextMessage } from './websocket-test-helper'; + +// Basic +const msg = createTextMessage('Hello, world!'); + +// With role +const userMsg = createTextMessage('Hello', 'user'); +const sysMsg = createTextMessage('Instructions', 'system'); + +// With ID +const msg = createTextMessage('Hello', 'user', 'msg-123'); +``` + +Returns: +```typescript +{ + type: 'message', + role: 'user' | 'assistant' | 'system', + content: [{ type: 'input_text', text: string }], + id?: string +} +``` + +### Create Function Call Output + +```typescript +import { createFunctionCallOutput } from './websocket-test-helper'; + +const output = createFunctionCallOutput( + 'call-123', // call_id + '{"temperature": 72}', // output (JSON string) + 'output-456' // optional id +); +``` + +Returns: +```typescript +{ + type: 'function_call_output', + call_id: string, + output: string, + id?: string +} +``` + +--- + +## Assertions + +### Assert Event Properties + +```typescript +import { assertEvent } from './websocket-test-helper'; + +// Basic assertion +assertEvent(event, 'session.created'); + +// With additional checks +assertEvent(event, 'session.created', (evt) => { + expect(evt.session.id).toBeDefined(); + expect(evt.session.model).toBe('expected-model'); + expect(evt.session.audio.output.voice).toBe('Dennis'); +}); +``` + +--- + +## Response Utilities + +### Extract Text from Response + +```typescript +import { extractTextFromResponse } from './websocket-test-helper'; + +const response = await waitForCompleteResponse(client); +const text = extractTextFromResponse(response.allEvents); + +console.log('Response text:', text); +``` + +### Extract Audio Transcript + +```typescript +import { extractAudioTranscriptFromResponse } from './websocket-test-helper'; + +const response = await waitForCompleteResponse(client); +const transcript = extractAudioTranscriptFromResponse(response.allEvents); + +console.log('Audio transcript:', transcript); +``` + +--- + +## Complete Example + +```typescript +import { v4 as uuidv4 } from 'uuid'; +import { + createWebSocketTestClient, + createTextMessage, + waitForCompleteResponse, + extractTextFromResponse, + assertEvent, +} from './websocket-test-helper'; + +it('should handle conversation flow', async () => { + // Create client + const sessionKey = `test-${uuidv4()}`; + const client = await createWebSocketTestClient(sessionKey, 4000); + + try { + // Wait for session + const sessionEvent = await client.waitForEvent('session.created'); + assertEvent(sessionEvent, 'session.created', (evt) => { + expect(evt.session.id).toBeDefined(); + }); + + // Create conversation item + client.sendEvent({ + type: 'conversation.item.create', + item: createTextMessage('Hello, how are you?', 'user'), + }); + + // Verify item was added + const [added, done] = await client.waitForEvents([ + 'conversation.item.added', + 'conversation.item.done' + ]); + + assertEvent(added, 'conversation.item.added'); + assertEvent(done, 'conversation.item.done'); + + // Generate response + client.sendEvent({ type: 'response.create' }); + + // Wait for complete response + const response = await waitForCompleteResponse(client, 30000); + const text = extractTextFromResponse(response.allEvents); + + expect(text.length).toBeGreaterThan(0); + + } finally { + await client.close(); + } +}); +``` + +--- + +## Server Management (Optional) + +For automated server lifecycle management: + +```typescript +import { + startTestServer, + stopTestServer, + waitForServerReady, + getServerPort, +} from './websocket-server-helper'; + +// Start server +const port = await startTestServer(4000); +await waitForServerReady(port); + +// Run tests... + +// Stop server +await stopTestServer(); +``` + +--- + +## Type Definitions + +All types are imported from `../../src/types/realtime`: + +```typescript +import * as RT from '../../src/types/realtime'; + +// Client events +RT.ClientEvent +RT.SessionUpdateEvent +RT.ConversationItemCreateEvent +RT.ResponseCreateEvent +// ... etc + +// Server events +RT.ServerEvent +RT.SessionCreatedEvent +RT.ConversationItemAddedEvent +RT.ResponseCreatedEvent +// ... etc + +// Data types +RT.MessageItem +RT.FunctionCallItem +RT.FunctionCallOutputItem +RT.Session +RT.Response +``` + +--- + +## Helper Function Summary + +| Function | Purpose | +|----------|---------| +| `createWebSocketTestClient()` | Create and connect test client | +| `waitForEvent()` | Wait for specific event | +| `waitForEvents()` | Wait for multiple events | +| `waitForCompleteResponse()` | Wait for full response cycle | +| `collectEventsUntil()` | Collect all events until target | +| `assertEvent()` | Assert event type and properties | +| `createTextMessage()` | Create text message item | +| `createFunctionCallOutput()` | Create function output item | +| `extractTextFromResponse()` | Extract text from response events | +| `extractAudioTranscriptFromResponse()` | Extract audio transcript | + +--- + +For complete testing documentation, examples, and troubleshooting, see the [main README](../README.md). diff --git a/realtime-service/__tests__/api/realtime-api.spec.ts b/realtime-service/__tests__/api/realtime-api.spec.ts new file mode 100644 index 0000000..85b8690 --- /dev/null +++ b/realtime-service/__tests__/api/realtime-api.spec.ts @@ -0,0 +1,885 @@ +/** + * OpenAI Realtime API Integration Tests + * Tests the WebSocket API behavior end-to-end + * + * IMPORTANT: Before running these tests, start the WebSocket server: + * 1. Open a new terminal + * 2. cd to the serving/realtime-service/src directory + * 3. npm start + * 4. Wait for "Application Server listening on port 3001" + * 5. Then run: npm run test:api + */ + +import { v4 as uuidv4 } from 'uuid'; +import * as RT from '../../src/types/realtime'; +import { + createWebSocketTestClient, + assertEvent, + createTextMessage, + createFunctionCallOutput, + waitForCompleteResponse, + extractTextFromResponse, + verifyContinuationAfterCancellation, + WebSocketTestClient, +} from './websocket-test-helper'; + +// Get port from environment variable +const TEST_PORT = process.env.WS_APP_PORT ? parseInt(process.env.WS_APP_PORT) : 4000; + +describe('OpenAI Realtime API - Integration Tests', () => { + jest.setTimeout(60000); // 60 second timeout for API tests + + describe('Session Management', () => { + let client: WebSocketTestClient; + const sessionKey = `test-session-${uuidv4()}`; + + afterEach(async () => { + if (client) { + await client.close(); + } + }); + + it('should create a session on connection and send session.created event', async () => { + client = await createWebSocketTestClient(sessionKey, TEST_PORT); + + const sessionCreatedEvent = await client.waitForEvent('session.created', 5000); + + assertEvent(sessionCreatedEvent, 'session.created', (event: RT.SessionCreatedEvent) => { + expect(event.session).toBeDefined(); + expect(event.session.id).toBeDefined(); + expect(event.session.type).toBe('realtime'); + expect(event.session.object).toBe('realtime.session'); + expect(event.session.model).toBeDefined(); + expect(event.session.output_modalities).toEqual(expect.arrayContaining(['audio', 'text'])); + expect(event.session.audio).toBeDefined(); + expect(event.session.audio.input).toBeDefined(); + expect(event.session.audio.output).toBeDefined(); + expect(event.session.audio.output.voice).toBeDefined(); + expect(event.session.instructions).toBeDefined(); + }); + }); + + it('should update session configuration via session.update', async () => { + client = await createWebSocketTestClient(sessionKey, TEST_PORT); + + await client.waitForEvent('session.created'); + + const newInstructions = 'You are a helpful assistant that speaks like a pirate.'; + const newVoice = 'TestVoice'; + + client.sendEvent({ + type: 'session.update', + session: { + instructions: newInstructions, + audio: { + output: { + voice: newVoice, + }, + }, + temperature: 0.9, + }, + }); + + const sessionUpdatedEvent = await client.waitForEvent('session.updated', 5000); + + assertEvent(sessionUpdatedEvent, 'session.updated', (event: RT.SessionUpdatedEvent) => { + expect(event.session.instructions).toBe(newInstructions); + expect(event.session.audio.output.voice).toBe(newVoice); + expect(event.session.temperature).toBe(0.9); + }); + }); + + it('should update turn detection settings', async () => { + client = await createWebSocketTestClient(sessionKey, TEST_PORT); + + await client.waitForEvent('session.created'); + + client.sendEvent({ + type: 'session.update', + session: { + audio: { + input: { + turn_detection: { + type: 'semantic_vad', + eagerness: 'high', + create_response: true, + interrupt_response: true, + }, + }, + }, + }, + }); + + const sessionUpdatedEvent = await client.waitForEvent('session.updated', 5000); + + assertEvent(sessionUpdatedEvent, 'session.updated', (event: RT.SessionUpdatedEvent) => { + expect(event.session.audio.input.turn_detection).toBeDefined(); + expect(event.session.audio.input.turn_detection?.type).toBe('semantic_vad'); + if (event.session.audio.input.turn_detection?.type === 'semantic_vad') { + expect(event.session.audio.input.turn_detection.eagerness).toBe('high'); + } + }); + }); + }); + + describe('Conversation Management', () => { + let client: WebSocketTestClient; + const sessionKey = `test-session-${uuidv4()}`; + + beforeEach(async () => { + client = await createWebSocketTestClient(sessionKey, TEST_PORT); + await client.waitForEvent('session.created'); + }); + + afterEach(async () => { + if (client) { + await client.close(); + } + }); + + it('should create a conversation item and receive confirmation events', async () => { + const messageId = `msg-${uuidv4()}`; + const messageText = 'Hello, this is a test message.'; + + client.sendEvent({ + type: 'conversation.item.create', + item: createTextMessage(messageText, 'user', messageId), + }); + + const events = await client.waitForEvents([ + 'conversation.item.added', + 'conversation.item.done', + ], 5000); + + const [addedEvent, doneEvent] = events; + + assertEvent(addedEvent, 'conversation.item.added', (event: RT.ConversationItemAddedEvent) => { + expect(event.item.id).toBe(messageId); + expect(event.item.type).toBe('message'); + expect(event.item.status).toBe('completed'); + }); + + assertEvent(doneEvent, 'conversation.item.done', (event: RT.ConversationItemDoneEvent) => { + expect(event.item.id).toBe(messageId); + }); + }); + + it('should retrieve a conversation item by ID', async () => { + const messageId = `msg-${uuidv4()}`; + const messageText = 'Test message for retrieval'; + + // Create item first + client.sendEvent({ + type: 'conversation.item.create', + item: createTextMessage(messageText, 'user', messageId), + }); + + await client.waitForEvent('conversation.item.done'); + + // Retrieve the item + client.sendEvent({ + type: 'conversation.item.retrieve', + item_id: messageId, + }); + + const retrievedEvent = await client.waitForEvent('conversation.item.retrieved', 5000); + + assertEvent(retrievedEvent, 'conversation.item.retrieved', (event: RT.ConversationItemRetrievedEvent) => { + expect(event.item.id).toBe(messageId); + expect(event.item.type).toBe('message'); + }); + }); + + it('should delete a conversation item', async () => { + const messageId = `msg-${uuidv4()}`; + + // Create item first + client.sendEvent({ + type: 'conversation.item.create', + item: createTextMessage('Message to delete', 'user', messageId), + }); + + await client.waitForEvent('conversation.item.done'); + + // Delete the item + client.sendEvent({ + type: 'conversation.item.delete', + item_id: messageId, + }); + + const deletedEvent = await client.waitForEvent('conversation.item.deleted', 5000); + + assertEvent(deletedEvent, 'conversation.item.deleted', (event: RT.ConversationItemDeletedEvent) => { + expect(event.item_id).toBe(messageId); + }); + + // Verify item is gone by trying to retrieve it + client.sendEvent({ + type: 'conversation.item.retrieve', + item_id: messageId, + }); + + const errorEvent = await client.waitForEvent('error', 5000); + + assertEvent(errorEvent, 'error', (event: RT.ErrorEvent) => { + expect(event.error.code).toBe('item_not_found'); + }); + }); + + it('should handle truncation of conversation items', async () => { + const messageId = `msg-${uuidv4()}`; + + // Create an assistant message with audio (in real scenario) + client.sendEvent({ + type: 'conversation.item.create', + item: { + type: 'message', + role: 'assistant', + id: messageId, + content: [ + { + type: 'audio', + audio: 'base64audiodata', + transcript: 'This is a long message that will be truncated', + }, + ], + } as RT.MessageItem, + }); + + await client.waitForEvent('conversation.item.done'); + + // Truncate at 1000ms + client.sendEvent({ + type: 'conversation.item.truncate', + item_id: messageId, + content_index: 0, + audio_end_ms: 1000, + }); + + const truncatedEvent = await client.waitForEvent('conversation.item.truncated', 5000); + + assertEvent(truncatedEvent, 'conversation.item.truncated', (event: RT.ConversationItemTruncatedEvent) => { + expect(event.item_id).toBe(messageId); + expect(event.content_index).toBe(0); + expect(event.audio_end_ms).toBe(1000); + }); + }); + + it('should return error when trying to retrieve non-existent item', async () => { + const nonExistentId = `msg-${uuidv4()}`; + + client.sendEvent({ + type: 'conversation.item.retrieve', + item_id: nonExistentId, + }); + + const errorEvent = await client.waitForEvent('error', 5000); + + assertEvent(errorEvent, 'error', (event: RT.ErrorEvent) => { + expect(event.error.type).toBe('invalid_request_error'); + expect(event.error.code).toBe('item_not_found'); + expect(event.error.message).toContain(nonExistentId); + }); + }); + }); + + describe('Response Generation', () => { + let client: WebSocketTestClient; + const sessionKey = `test-session-${uuidv4()}`; + + beforeEach(async () => { + client = await createWebSocketTestClient(sessionKey, TEST_PORT); + await client.waitForEvent('session.created'); + + // Set up session with text-only output for simpler testing + client.sendEvent({ + type: 'session.update', + session: { + output_modalities: ['text'], + instructions: 'You are a helpful assistant. Keep responses brief.', + }, + }); + + await client.waitForEvent('session.updated'); + }); + + afterEach(async () => { + if (client) { + await client.close(); + } + }); + + it('should generate a response to a user message', async () => { + // Create user message + client.sendEvent({ + type: 'conversation.item.create', + item: createTextMessage('Say hello', 'user'), + }); + + await client.waitForEvent('conversation.item.done'); + + // Request response + client.sendEvent({ + type: 'response.create', + }); + + const response = await waitForCompleteResponse(client, 30000); + + assertEvent(response.created, 'response.created', (event: RT.ResponseCreatedEvent) => { + expect(event.response.id).toBeDefined(); + expect(event.response.status).toBe('in_progress'); + }); + + assertEvent(response.done, 'response.done', (event: RT.ResponseDoneEvent) => { + expect(event.response.id).toBe(response.created.response.id); + expect(['completed', 'incomplete']).toContain(event.response.status); + }); + + // Verify we got text output + const text = extractTextFromResponse(response.allEvents); + expect(text.length).toBeGreaterThan(0); + }); + + it('should emit response events in correct order', async () => { + // Create user message + client.sendEvent({ + type: 'conversation.item.create', + item: createTextMessage('Count to three', 'user'), + }); + + await client.waitForEvent('conversation.item.done'); + + // Request response + client.sendEvent({ + type: 'response.create', + }); + + const response = await waitForCompleteResponse(client, 30000); + + const eventTypes = response.allEvents.map(e => e.type); + + // Verify event ordering + const createdIndex = eventTypes.indexOf('response.created'); + const doneIndex = eventTypes.indexOf('response.done'); + + expect(createdIndex).toBeGreaterThanOrEqual(0); + expect(doneIndex).toBeGreaterThan(createdIndex); + + // All response-related events should be between created and done + const betweenEvents = eventTypes.slice(createdIndex + 1, doneIndex); + betweenEvents.forEach(type => { + expect(type).toMatch(/^(response\.|conversation\.item)/); + }); + }); + + it('should cancel response immediately after creation', async () => { + // Create user message + client.sendEvent({ + type: 'conversation.item.create', + item: createTextMessage('Tell me a long story', 'user'), + }); + + await client.waitForEvent('conversation.item.done'); + + // Request response + client.sendEvent({ + type: 'response.create', + }); + + // Wait for response to start + const createdEvent = await client.waitForEvent('response.created', 5000); + const responseId = (createdEvent as RT.ResponseCreatedEvent).response.id; + + // Cancel immediately after creation + client.sendEvent({ + type: 'response.cancel', + response_id: responseId, + }); + + // Wait for response.done with cancelled status + const doneEvent = await client.waitForEvent('response.done', 10000); + + assertEvent(doneEvent, 'response.done', (event: RT.ResponseDoneEvent) => { + expect(event.response.id).toBe(responseId); + expect(event.response.status).toBe('cancelled'); + expect(event.response.status_details?.type).toBe('cancelled'); + expect(event.response.status_details?.reason).toBe('client_cancelled'); + }); + + // Verify conversation continues to work after cancellation + await verifyContinuationAfterCancellation(client, responseId, 'Say hello'); + }); + + it('should cancel response after first text delta', async () => { + // Create user message + client.sendEvent({ + type: 'conversation.item.create', + item: createTextMessage('Tell me a story', 'user'), + }); + + await client.waitForEvent('conversation.item.done'); + + // Request response (text-only mode already set in session) + client.sendEvent({ + type: 'response.create', + }); + + // Wait for response to start + const createdEvent = await client.waitForEvent('response.created', 5000); + const responseId = (createdEvent as RT.ResponseCreatedEvent).response.id; + + // Wait for first text delta (using correct event type: response.output_text.delta) + const firstDelta = await client.waitForEvent('response.output_text.delta', 10000); + expect(firstDelta).toBeDefined(); + + // Cancel after receiving first delta + client.sendEvent({ + type: 'response.cancel', + response_id: responseId, + }); + + // Wait for response.done with cancelled status + const doneEvent = await client.waitForEvent('response.done', 10000); + + assertEvent(doneEvent, 'response.done', (event: RT.ResponseDoneEvent) => { + expect(event.response.id).toBe(responseId); + expect(event.response.status).toBe('cancelled'); + expect(event.response.status_details?.type).toBe('cancelled'); + expect(event.response.status_details?.reason).toBe('client_cancelled'); + // Response should have at least one output item with some content + expect(event.response.output.length).toBeGreaterThan(0); + }); + + // Verify conversation continues to work after cancellation + await verifyContinuationAfterCancellation(client, responseId, 'Say goodbye'); + }); + + it('should cancel response during audio streaming', async () => { + // Update session to enable audio output + client.sendEvent({ + type: 'session.update', + session: { + output_modalities: ['audio', 'text'], + instructions: 'You are a helpful assistant. Provide detailed responses.', + }, + }); + + await client.waitForEvent('session.updated'); + + // Create user message with a prompt that will generate a longer response + client.sendEvent({ + type: 'conversation.item.create', + item: createTextMessage('Hello', 'user'), + }); + + await client.waitForEvent('conversation.item.done'); + + // Request response with audio output + client.sendEvent({ + type: 'response.create', + }); + + // Wait for response to start + const createdEvent = await client.waitForEvent('response.created', 5000); + const responseId = (createdEvent as RT.ResponseCreatedEvent).response.id; + + // Wait for audio to start streaming (using correct event type: response.output_audio.delta) + await client.waitForEvent('response.output_audio.delta', 10000); + + // Cancel during audio streaming + client.sendEvent({ + type: 'response.cancel', + response_id: responseId, + }); + + // Wait for response.done with cancelled status + const doneEvent = await client.waitForEvent('response.done', 10000); + + assertEvent(doneEvent, 'response.done', (event: RT.ResponseDoneEvent) => { + expect(event.response.id).toBe(responseId); + expect(event.response.status).toBe('cancelled'); + expect(event.response.status_details?.type).toBe('cancelled'); + expect(event.response.status_details?.reason).toBe('client_cancelled'); + }); + + // Verify conversation continues to work after cancellation + await verifyContinuationAfterCancellation(client, responseId, 'Hi again'); + }); + + it.skip('should cancel response during long audio/text streaming', async () => { + // Update session to enable audio output + client.sendEvent({ + type: 'session.update', + session: { + output_modalities: ['audio', 'text'], + instructions: 'You are a helpful assistant. Provide detailed responses.', + }, + }); + + await client.waitForEvent('session.updated'); + + // Create user message with a prompt that will generate a longer response + client.sendEvent({ + type: 'conversation.item.create', + item: createTextMessage('Tell me a detailed story about space exploration with multiple paragraphs', 'user'), + }); + + await client.waitForEvent('conversation.item.done'); + + // Request response with audio output + client.sendEvent({ + type: 'response.create', + }); + + // Wait for response to start + const createdEvent = await client.waitForEvent('response.created', 5000); + const responseId = (createdEvent as RT.ResponseCreatedEvent).response.id; + + // Wait for audio to start streaming (using correct event type: response.output_audio.delta) + await client.waitForEvent('response.output_audio.delta', 10000); + + // Wait just a moment to ensure we're mid-stream (but not so long that response completes) + await new Promise(resolve => setTimeout(resolve, 50)); + + // Cancel during audio streaming + client.sendEvent({ + type: 'response.cancel', + response_id: responseId, + }); + + // Wait for response.done with cancelled status + const doneEvent = await client.waitForEvent('response.done', 10000); + + assertEvent(doneEvent, 'response.done', (event: RT.ResponseDoneEvent) => { + expect(event.response.id).toBe(responseId); + expect(event.response.status).toBe('cancelled'); + expect(event.response.status_details?.type).toBe('cancelled'); + expect(event.response.status_details?.reason).toBe('client_cancelled'); + }); + + // Verify conversation continues to work after cancellation + await verifyContinuationAfterCancellation(client, responseId, 'What is 2+2?'); + }); + + it('should include usage information in response', async () => { + // Create user message + client.sendEvent({ + type: 'conversation.item.create', + item: createTextMessage('Hello', 'user'), + }); + + await client.waitForEvent('conversation.item.done'); + + // Request response + client.sendEvent({ + type: 'response.create', + }); + + const response = await waitForCompleteResponse(client, 30000); + + assertEvent(response.done, 'response.done', (event: RT.ResponseDoneEvent) => { + // Usage might be present (depending on implementation) + if (event.response.usage) { + expect(event.response.usage.total_tokens).toBeGreaterThan(0); + expect(event.response.usage.input_tokens).toBeGreaterThanOrEqual(0); + expect(event.response.usage.output_tokens).toBeGreaterThan(0); + } + }); + }); + }); + + describe('Function Calling', () => { + let client: WebSocketTestClient; + const sessionKey = `test-session-${uuidv4()}`; + + beforeEach(async () => { + client = await createWebSocketTestClient(sessionKey, TEST_PORT); + await client.waitForEvent('session.created'); + + // Configure session with function tools + client.sendEvent({ + type: 'session.update', + session: { + output_modalities: ['text'], + instructions: 'You are a helpful assistant with access to tools.', + tools: [ + { + type: 'function', + name: 'get_weather', + description: 'Get the current weather for a location', + parameters: { + type: 'object', + properties: { + location: { + type: 'string', + description: 'The city and state, e.g. San Francisco, CA', + }, + }, + required: ['location'], + }, + }, + ], + tool_choice: 'auto', + }, + }); + + await client.waitForEvent('session.updated'); + }); + + afterEach(async () => { + if (client) { + await client.close(); + } + }); + + it('should handle function call output', async () => { + const callId = `call-${uuidv4()}`; + const functionOutput = JSON.stringify({ + temperature: 72, + condition: 'sunny', + }); + + // Create function output item + client.sendEvent({ + type: 'conversation.item.create', + item: createFunctionCallOutput(callId, functionOutput), + }); + + const events = await client.waitForEvents([ + 'conversation.item.added', + 'conversation.item.done', + ], 5000); + + assertEvent(events[0], 'conversation.item.added', (event: RT.ConversationItemAddedEvent) => { + expect(event.item.type).toBe('function_call_output'); + const item = event.item as RT.FunctionCallOutputItem; + expect(item.call_id).toBe(callId); + expect(item.output).toBe(functionOutput); + }); + }); + }); + + describe('Audio Input Management', () => { + let client: WebSocketTestClient; + const sessionKey = `test-session-${uuidv4()}`; + + beforeEach(async () => { + client = await createWebSocketTestClient(sessionKey, TEST_PORT); + await client.waitForEvent('session.created'); + }); + + afterEach(async () => { + if (client) { + await client.close(); + } + }); + + it('should handle input audio buffer clear', async () => { + // Append some audio data first + client.sendEvent({ + type: 'input_audio_buffer.append', + audio: 'YmFzZTY0ZW5jb2RlZGF1ZGlv', // base64 encoded dummy data + }); + + // Give it a moment to process + await new Promise(resolve => setTimeout(resolve, 100)); + + // Clear the buffer + client.sendEvent({ + type: 'input_audio_buffer.clear', + }); + + const clearedEvent = await client.waitForEvent('input_audio_buffer.cleared', 5000); + + assertEvent(clearedEvent, 'input_audio_buffer.cleared'); + }); + + // TODO: This test times out - audio buffer commit may require actual audio data or STT setup + it.skip('should handle input audio buffer commit', async () => { + // NOTE: This test is currently skipped because it requires: + // - Proper audio data (not just base64 dummy data) + // - Speech-to-text service to be properly configured + // - AssemblyAI or other STT provider setup + + // Append audio data + client.sendEvent({ + type: 'input_audio_buffer.append', + audio: 'YmFzZTY0ZW5jb2RlZGF1ZGlv', + }); + + await new Promise(resolve => setTimeout(resolve, 100)); + + // Commit the buffer + client.sendEvent({ + type: 'input_audio_buffer.commit', + }); + + const committedEvent = await client.waitForEvent('input_audio_buffer.committed', 30000); + + assertEvent(committedEvent, 'input_audio_buffer.committed', (event: RT.InputAudioBufferCommittedEvent) => { + expect(event.item_id).toBeDefined(); + }); + }); + }); + + describe('Error Handling', () => { + let client: WebSocketTestClient; + const sessionKey = `test-session-${uuidv4()}`; + + beforeEach(async () => { + client = await createWebSocketTestClient(sessionKey, TEST_PORT); + await client.waitForEvent('session.created'); + }); + + afterEach(async () => { + if (client) { + await client.close(); + } + }); + + it('should return error for invalid event format', async () => { + // Send malformed JSON + client.ws.send('{ invalid json }'); + + const errorEvent = await client.waitForEvent('error', 5000); + + assertEvent(errorEvent, 'error', (event: RT.ErrorEvent) => { + expect(event.error.type).toBe('invalid_request_error'); + }); + }); + + it('should return error when truncating user message', async () => { + const messageId = `msg-${uuidv4()}`; + + // Create a user message + client.sendEvent({ + type: 'conversation.item.create', + item: createTextMessage('User message', 'user', messageId), + }); + + await client.waitForEvent('conversation.item.done'); + + // Try to truncate (only assistant messages can be truncated) + client.sendEvent({ + type: 'conversation.item.truncate', + item_id: messageId, + content_index: 0, + audio_end_ms: 1000, + }); + + const errorEvent = await client.waitForEvent('error', 5000); + + assertEvent(errorEvent, 'error', (event: RT.ErrorEvent) => { + expect(event.error.type).toBe('invalid_request_error'); + expect(event.error.code).toBe('invalid_item_type'); + }); + }); + }); + + describe('Multi-Session Support', () => { + it('should handle multiple concurrent sessions', async () => { + const session1Key = `test-session-${uuidv4()}`; + const session2Key = `test-session-${uuidv4()}`; + + const client1 = await createWebSocketTestClient(session1Key, TEST_PORT); + const client2 = await createWebSocketTestClient(session2Key, TEST_PORT); + + try { + // Wait for both sessions to be created + const [session1Created, session2Created] = await Promise.all([ + client1.waitForEvent('session.created'), + client2.waitForEvent('session.created'), + ]); + + // Verify they have different session IDs + const s1 = session1Created as RT.SessionCreatedEvent; + const s2 = session2Created as RT.SessionCreatedEvent; + expect(s1.session.id).not.toBe(s2.session.id); + + // Configure sessions differently + client1.sendEvent({ + type: 'session.update', + session: { + instructions: 'You are session 1', + temperature: 0.5, + }, + }); + + client2.sendEvent({ + type: 'session.update', + session: { + instructions: 'You are session 2', + temperature: 0.9, + }, + }); + + const [updated1, updated2] = await Promise.all([ + client1.waitForEvent('session.updated'), + client2.waitForEvent('session.updated'), + ]); + + // Verify each session kept its own configuration + const u1 = updated1 as RT.SessionUpdatedEvent; + const u2 = updated2 as RT.SessionUpdatedEvent; + expect(u1.session.instructions).toBe('You are session 1'); + expect(u1.session.temperature).toBe(0.5); + expect(u2.session.instructions).toBe('You are session 2'); + expect(u2.session.temperature).toBe(0.9); + } finally { + await client1.close(); + await client2.close(); + } + }); + }); + + describe('Workspace Isolation', () => { + it('should isolate sessions by workspace', async () => { + const sessionKey = `test-session-${uuidv4()}`; + const workspace1 = `workspace-${uuidv4()}`; + const workspace2 = `workspace-${uuidv4()}`; + + const client1 = await createWebSocketTestClient(sessionKey, TEST_PORT, workspace1); + const client2 = await createWebSocketTestClient(sessionKey, TEST_PORT, workspace2); + + try { + // Both sessions should be created successfully with same key but different workspaces + await Promise.all([ + client1.waitForEvent('session.created'), + client2.waitForEvent('session.created'), + ]); + + // They should operate independently + client1.sendEvent({ + type: 'session.update', + session: { + instructions: 'Workspace 1 instructions', + }, + }); + + const updated1 = await client1.waitForEvent('session.updated'); + + // Verify workspace 1 has its instructions + const u1 = updated1 as RT.SessionUpdatedEvent; + expect(u1.session.instructions).toBe('Workspace 1 instructions'); + + // Workspace 2 should still have default instructions + // (we'll verify by updating workspace 2 and checking it didn't get workspace 1's instructions) + client2.sendEvent({ + type: 'session.update', + session: { + instructions: 'Workspace 2 instructions', + }, + }); + + const updated2 = await client2.waitForEvent('session.updated'); + const u2 = updated2 as RT.SessionUpdatedEvent; + expect(u2.session.instructions).toBe('Workspace 2 instructions'); + expect(u2.session.instructions).not.toBe(u1.session.instructions); + } finally { + await client1.close(); + await client2.close(); + } + }); + }); +}); + diff --git a/realtime-service/__tests__/api/websocket-server-helper.ts b/realtime-service/__tests__/api/websocket-server-helper.ts new file mode 100644 index 0000000..c1daa62 --- /dev/null +++ b/realtime-service/__tests__/api/websocket-server-helper.ts @@ -0,0 +1,133 @@ +/** + * WebSocket Server Helper + * Manages the WebSocket server lifecycle for integration tests + */ + +import { ChildProcess, spawn } from 'child_process'; +import WebSocket from 'ws'; + +let serverProcess: ChildProcess | null = null; +let serverPort: number = 3001; + +/** + * Start the WebSocket server for testing + */ +export async function startTestServer(port: number = 3001): Promise { + if (serverProcess) { + return serverPort; + } + + serverPort = port; + + return new Promise((resolve, reject) => { + // Start the server process from the src directory + const srcDir = __dirname + '/../../src'; + serverProcess = spawn('npm', ['start'], { + cwd: srcDir, + env: { + ...process.env, + WS_APP_PORT: String(port), + AUTH_TOKEN: '', // Disable auth for tests + NODE_ENV: 'test', + }, + stdio: 'ignore', // Ignore stdio to prevent blocking + detached: true, // Detach so it doesn't block Jest + }); + + // Unref so the parent process can exit without waiting + serverProcess.unref(); + + // Wait for server to be ready by attempting to connect + const startTime = Date.now(); + const maxWait = 15000; // 15 seconds + + const checkServer = async (): Promise => { + while (Date.now() - startTime < maxWait) { + try { + await waitForServerReady(port, 1); + resolve(port); + return; + } catch (error) { + // Server not ready yet, wait a bit + await new Promise(r => setTimeout(r, 500)); + } + } + reject(new Error('Server startup timeout')); + }; + + checkServer().catch(reject); + }); +} + +/** + * Stop the WebSocket server + */ +export async function stopTestServer(): Promise { + if (!serverProcess) { + return; + } + + return new Promise((resolve) => { + const process = serverProcess!; + serverProcess = null; + + process.on('exit', () => { + resolve(); + }); + + process.kill('SIGTERM'); + + // Force kill after 5 seconds + setTimeout(() => { + if (!process.killed) { + process.kill('SIGKILL'); + resolve(); + } + }, 5000); + }); +} + +/** + * Wait for server to be ready by attempting to connect + */ +export async function waitForServerReady(port: number, maxAttempts: number = 20): Promise { + for (let i = 0; i < maxAttempts; i++) { + try { + await new Promise((resolve, reject) => { + const ws = new WebSocket(`ws://localhost:${port}/session?key=test-health-check`); + + const timeout = setTimeout(() => { + ws.close(); + reject(new Error('Connection timeout')); + }, 1000); + + ws.on('open', () => { + clearTimeout(timeout); + ws.close(); + resolve(); + }); + + ws.on('error', () => { + clearTimeout(timeout); + reject(new Error('Connection failed')); + }); + }); + + // Server is ready + return; + } catch (error) { + // Wait before retrying + await new Promise(resolve => setTimeout(resolve, 500)); + } + } + + throw new Error('Server failed to become ready'); +} + +/** + * Get the current server port + */ +export function getServerPort(): number { + return serverPort; +} + diff --git a/realtime-service/__tests__/api/websocket-test-helper.ts b/realtime-service/__tests__/api/websocket-test-helper.ts new file mode 100644 index 0000000..ce7b207 --- /dev/null +++ b/realtime-service/__tests__/api/websocket-test-helper.ts @@ -0,0 +1,311 @@ +/** + * WebSocket API Test Helper + * Provides utilities for testing the WebSocket Realtime API + */ + +import WebSocket from 'ws'; +import * as RT from '../../src/types/realtime'; + +export interface WebSocketTestClient { + ws: WebSocket; + events: RT.ServerEvent[]; + waitForEvent: (eventType: string, timeout?: number) => Promise; + waitForEvents: (eventTypes: string[], timeout?: number) => Promise; + sendEvent: (event: RT.ClientEvent) => void; + clearEvents: () => void; + close: () => Promise; +} + +/** + * Create a WebSocket test client that connects to the realtime service + */ +export async function createWebSocketTestClient( + sessionKey: string, + port: number = 3001, + workspaceId?: string +): Promise { + const options: any = {}; + if (workspaceId) { + options.headers = { + 'workspace-id': workspaceId, + }; + } + + const ws = new WebSocket(`ws://localhost:${port}/session?key=${sessionKey}`, options); + + const events: RT.ServerEvent[] = []; + const eventPromises = new Map void>>(); + + return new Promise((resolve, reject) => { + const timeout = setTimeout(() => { + ws.close(); + reject(new Error( + `WebSocket connection timeout. ` + + `Make sure the server is running: cd src && npm start` + )); + }, 5000); + + ws.on('open', () => { + clearTimeout(timeout); + + const client: WebSocketTestClient = { + ws, + events, + + waitForEvent: (eventType: string, timeout = 5000): Promise => { + // Check if event already received + const existingEvent = events.find(e => e.type === eventType); + if (existingEvent) { + return Promise.resolve(existingEvent); + } + + // Wait for new event + return new Promise((resolve, reject) => { + const timeoutId = setTimeout(() => { + reject(new Error(`Timeout waiting for event: ${eventType}`)); + }, timeout); + + if (!eventPromises.has(eventType)) { + eventPromises.set(eventType, []); + } + + eventPromises.get(eventType)!.push((event) => { + clearTimeout(timeoutId); + resolve(event); + }); + }); + }, + + waitForEvents: async (eventTypes: string[], timeout = 5000): Promise => { + const results: RT.ServerEvent[] = []; + const startTime = Date.now(); + + for (const eventType of eventTypes) { + const remainingTime = timeout - (Date.now() - startTime); + if (remainingTime <= 0) { + throw new Error(`Timeout waiting for events: ${eventTypes.join(', ')}`); + } + + const event = await client.waitForEvent(eventType, remainingTime); + results.push(event); + } + + return results; + }, + + sendEvent: (event: RT.ClientEvent): void => { + ws.send(JSON.stringify(event)); + }, + + clearEvents: (): void => { + events.length = 0; + }, + + close: (): Promise => { + return new Promise((resolve) => { + ws.once('close', () => resolve()); + ws.close(); + }); + }, + }; + + resolve(client); + }); + + ws.on('message', (data: any) => { + try { + const event = JSON.parse(data.toString()) as RT.ServerEvent; + events.push(event); + + // Resolve any waiting promises for this event type + const waiters = eventPromises.get(event.type); + if (waiters && waiters.length > 0) { + const waiter = waiters.shift()!; + waiter(event); + } + } catch (error) { + console.error('Failed to parse WebSocket message:', error); + } + }); + + ws.on('error', (error) => { + clearTimeout(timeout); + reject(new Error( + `WebSocket connection failed: ${error.message}\n` + + `\n` + + `Make sure the server is running on port ${port}:\n` + + ` 1. Open a new terminal\n` + + ` 2. cd /serving/realtime-service/src\n` + + ` 3. npm start\n` + + ` 4. Wait for "Application Server listening on port ${port}"\n` + + ` 5. Then run: npm run test:api` + )); + }); + }); +} + +/** + * Helper to collect all events until a specific event type is received + */ +export async function collectEventsUntil( + client: WebSocketTestClient, + untilEventType: string, + timeout: number = 5000 +): Promise { + const startLength = client.events.length; + await client.waitForEvent(untilEventType, timeout); + return client.events.slice(startLength); +} + +/** + * Assert that an event matches expected properties + */ +export function assertEvent( + event: RT.ServerEvent, + expectedType: string, + additionalChecks?: (event: any) => void +): void { + expect(event.type).toBe(expectedType); + expect(event.event_id).toBeDefined(); + + if (additionalChecks) { + additionalChecks(event); + } +} + +/** + * Create a text message conversation item + */ +export function createTextMessage( + text: string, + role: 'user' | 'assistant' | 'system' = 'user', + id?: string +): RT.MessageItem { + return { + type: 'message', + role, + content: [ + { + type: 'input_text', + text, + }, + ], + ...(id && { id }), + }; +} + +/** + * Create a function call output item + */ +export function createFunctionCallOutput( + callId: string, + output: string, + id?: string +): RT.FunctionCallOutputItem { + return { + type: 'function_call_output', + call_id: callId, + output, + ...(id && { id }), + }; +} + +/** + * Wait for a complete response (from response.created to response.done) + */ +export async function waitForCompleteResponse( + client: WebSocketTestClient, + timeout: number = 30000 +): Promise<{ + created: RT.ResponseCreatedEvent; + done: RT.ResponseDoneEvent; + allEvents: RT.ServerEvent[]; +}> { + const startLength = client.events.length; + + const created = await client.waitForEvent('response.created', timeout); + const done = await client.waitForEvent('response.done', timeout); + + const allEvents = client.events.slice(startLength); + + return { + created: created as RT.ResponseCreatedEvent, + done: done as RT.ResponseDoneEvent, + allEvents, + }; +} + +/** + * Extract text content from a response + */ +export function extractTextFromResponse(events: RT.ServerEvent[]): string { + let text = ''; + + for (const event of events) { + if (event.type === 'response.output_text.delta') { + const deltaEvent = event as RT.ResponseTextDeltaEvent; + text += deltaEvent.delta; + } + } + + return text; +} + +/** + * Extract audio transcript from a response + */ +export function extractAudioTranscriptFromResponse(events: RT.ServerEvent[]): string { + let transcript = ''; + + for (const event of events) { + if (event.type === 'response.output_audio_transcript.delta') { + const deltaEvent = event as RT.ResponseAudioTranscriptDeltaEvent; + transcript += deltaEvent.delta; + } + } + + return transcript; +} + +/** + * Verify that a new conversation request works after cancellation + */ +export async function verifyContinuationAfterCancellation( + client: WebSocketTestClient, + originalResponseId: string, + newMessageText: string, +): Promise { + // Clear events to ensure we wait for NEW response events + client.clearEvents(); + + // Verify we can send a new conversation request after cancellation + client.sendEvent({ + type: 'conversation.item.create', + item: createTextMessage(newMessageText, 'user'), + }); + + await client.waitForEvent('conversation.item.done'); + + client.sendEvent({ + type: 'response.create', + }); + + const newResponse = await waitForCompleteResponse(client, 30000); + + assertEvent(newResponse.created, 'response.created', (event: RT.ResponseCreatedEvent) => { + expect(event.response.id).toBeDefined(); + expect(event.response.id).not.toBe(originalResponseId); + expect(event.response.status).toBe('in_progress'); + }); + + assertEvent(newResponse.done, 'response.done', (event: RT.ResponseDoneEvent) => { + expect(['completed', 'incomplete']).toContain(event.response.status); + }); + + // Verify we got some output in the new response + const hasOutput = newResponse.allEvents.some( + e => e.type === 'response.output_text.delta' || e.type === 'response.output_audio.delta' + ); + expect(hasOutput).toBe(true); +} + diff --git a/realtime-service/__tests__/config.ts b/realtime-service/__tests__/config.ts new file mode 100644 index 0000000..47f9d44 --- /dev/null +++ b/realtime-service/__tests__/config.ts @@ -0,0 +1,8 @@ +/** + * Test configuration constants + */ + +export const TEST_SESSION_ID = 'test-session-123'; +export const TEST_INTERACTION_ID = 'test-interaction-456'; +export const TEST_WORKSPACE_ID = 'test-workspace-789'; + diff --git a/realtime-service/__tests__/setup.ts b/realtime-service/__tests__/setup.ts new file mode 100644 index 0000000..acc86f6 --- /dev/null +++ b/realtime-service/__tests__/setup.ts @@ -0,0 +1,9 @@ +/** + * Jest setup file + * This file runs before all tests and sets up the testing environment + */ + +import * as dotenv from 'dotenv'; + +// Load test environment variables from root .env.test +dotenv.config({ path: './.env.test' }); diff --git a/realtime-service/__tests__/test-setup.ts b/realtime-service/__tests__/test-setup.ts new file mode 100644 index 0000000..682ad56 --- /dev/null +++ b/realtime-service/__tests__/test-setup.ts @@ -0,0 +1,32 @@ +/** + * Jest setupFilesAfterEnv configuration + * This runs after the test framework is installed + */ + +// Mock the logger to prevent console output during tests +jest.mock('../src/logger', () => ({ + default: { + info: jest.fn(), + debug: jest.fn(), + warn: jest.fn(), + error: jest.fn(), + trace: jest.fn(), + fatal: jest.fn(), + }, +})); + +// Mock log-helpers +jest.mock('../src/log-helpers', () => ({ + formatContext: jest.fn((context: any) => JSON.stringify(context)), + formatSession: jest.fn((session: any) => JSON.stringify(session)), + formatWorkspace: jest.fn((workspace: any) => JSON.stringify(workspace)), +})); + +// Set up common test behaviors +beforeEach(() => { + jest.clearAllMocks(); +}); + +// Global test timeout +jest.setTimeout(30000); + diff --git a/realtime-service/__tests__/utils/graph-test-helpers.ts b/realtime-service/__tests__/utils/graph-test-helpers.ts new file mode 100644 index 0000000..efc24db --- /dev/null +++ b/realtime-service/__tests__/utils/graph-test-helpers.ts @@ -0,0 +1,71 @@ +/** + * Helper utilities for testing graph-related components + */ + +import { GraphBuilder } from '@inworld/runtime/graph'; + +/** + * Creates a mock GraphBuilder for testing + */ +export function createMockGraphBuilder(graphId: string = 'test-graph') { + const nodes: any[] = []; + const edges: any[] = []; + + const builder = { + addNode: jest.fn((node: any) => { + nodes.push(node); + return builder; + }), + addEdge: jest.fn((from: any, to: any, options?: any) => { + edges.push({ from, to, options }); + return builder; + }), + setStartNode: jest.fn((node: any) => { + return builder; + }), + setEndNode: jest.fn((node: any) => { + return builder; + }), + build: jest.fn(() => ({ + id: graphId, + nodes, + edges, + start: jest.fn(), + stop: jest.fn(), + execute: jest.fn(), + })), + }; + + return builder; +} + +/** + * Creates a mock Graph instance + */ +export function createMockGraph(graphId: string = 'test-graph') { + return { + id: graphId, + start: jest.fn().mockResolvedValue(undefined), + stop: jest.fn().mockResolvedValue(undefined), + execute: jest.fn().mockResolvedValue(undefined), + getNodeById: jest.fn(), + getAllNodes: jest.fn(() => []), + }; +} + +/** + * Helper to wait for async operations in tests + */ +export async function waitForAsync(ms: number = 0): Promise { + return new Promise(resolve => setTimeout(resolve, ms)); +} + +/** + * Helper to create a mock async iterator for streaming data + */ +export async function* createMockAsyncIterator(items: T[]): AsyncIterableIterator { + for (const item of items) { + yield item; + } +} + diff --git a/realtime-service/__tests__/utils/mock-helpers.ts b/realtime-service/__tests__/utils/mock-helpers.ts new file mode 100644 index 0000000..73fd89d --- /dev/null +++ b/realtime-service/__tests__/utils/mock-helpers.ts @@ -0,0 +1,93 @@ +/** + * Mock helpers for unit tests + * These utilities help create mock objects for testing + */ + +import { ProcessContext } from '@inworld/runtime/graph'; +import { ConnectionsMap, State, Connection } from '../../types/index'; + +/** + * Creates a mock ProcessContext for testing + */ +export function createMockProcessContext(datastoreData: Record = {}): ProcessContext { + const datastore = new Map(Object.entries(datastoreData)); + + return { + getDatastore: jest.fn(() => ({ + get: jest.fn((key: string) => datastore.get(key)), + set: jest.fn((key: string, value: any) => datastore.set(key, value)), + has: jest.fn((key: string) => datastore.has(key)), + delete: jest.fn((key: string) => datastore.delete(key)), + })), + getNodeId: jest.fn(() => 'test-node-id'), + getGraphId: jest.fn(() => 'test-graph-id'), + } as any; +} + +/** + * Creates a mock State object for testing + */ +export function createMockState(overrides: Partial = {}): State { + return { + sessionId: overrides.sessionId || 'test-session-id', + interactionId: overrides.interactionId || 'test-interaction-id', + messages: overrides.messages || [], + eagerness: overrides.eagerness || 'medium', + ...overrides, + } as State; +} + +/** + * Creates a mock Connection object for testing + */ +export function createMockConnection( + sessionId: string = 'test-session-id', + stateOverrides: Partial = {} +): Connection { + return { + sessionId, + state: createMockState({ sessionId, ...stateOverrides }), + unloaded: false, + } as Connection; +} + +/** + * Creates a mock ConnectionsMap for testing + */ +export function createMockConnectionsMap( + sessions: string[] = ['test-session-id'] +): ConnectionsMap { + const connections: ConnectionsMap = {}; + + sessions.forEach(sessionId => { + connections[sessionId] = createMockConnection(sessionId); + }); + + return connections; +} + +/** + * Creates a mock DataStreamWithMetadata for testing + */ +export function createMockDataStreamWithMetadata(metadata: Record = {}) { + return { + getMetadata: jest.fn(() => metadata), + getData: jest.fn(() => Buffer.from('test-data')), + setMetadata: jest.fn(), + }; +} + +/** + * Helper to create a spy on a logger + */ +export function createLoggerSpy() { + return { + info: jest.fn(), + debug: jest.fn(), + warn: jest.fn(), + error: jest.fn(), + trace: jest.fn(), + fatal: jest.fn(), + }; +} + diff --git a/realtime-service/jest.config.js b/realtime-service/jest.config.js new file mode 100644 index 0000000..6865b24 --- /dev/null +++ b/realtime-service/jest.config.js @@ -0,0 +1,34 @@ +module.exports = { + clearMocks: true, + preset: 'ts-jest', + testEnvironment: 'node', + testTimeout: 30000, + roots: ['/__tests__'], + testMatch: [ + '**/__tests__/**/*.spec.ts', + '**/?(*.)+(spec).ts' + ], + transform: { + '^.+\\.ts$': ['ts-jest', { + tsconfig: 'tsconfig.test.json' + }] + }, + collectCoverageFrom: [ + 'src/**/*.ts', + '!src/**/*.d.ts', + '!src/dist/**', + '!src/node_modules/**', + '!src/index.ts', + ], + coverageDirectory: 'coverage', + coverageReporters: ['text', 'lcov', 'html'], + coveragePathIgnorePatterns: ['node_modules', 'dist', '__tests__'], + moduleFileExtensions: ['ts', 'js', 'json'], + moduleNameMapper: { + '^@/(.*)$': '/src/$1', + }, + setupFiles: ['/__tests__/setup.ts'], + setupFilesAfterEnv: ['/__tests__/test-setup.ts'], + modulePathIgnorePatterns: ['dist/', 'node_modules/'], + verbose: true, +}; diff --git a/realtime-service/load-tests/README.md b/realtime-service/load-tests/README.md new file mode 100644 index 0000000..3f10757 --- /dev/null +++ b/realtime-service/load-tests/README.md @@ -0,0 +1,61 @@ +# Load Tests + +Load testing suite for the Inworld Realtime API service using k6. + +## Prerequisites + +- [k6](https://k6.io/docs/getting-started/installation/) installed + +## Available Test Scripts + +### Local Mock Environment +- `npm run test:local:mock:small` - Small load test +- `npm run test:local:mock:medium` - Medium load test +- `npm run test:local:mock:large` - Large load test +- `npm run test:local:mock:xlarge` - Extra large load test + +### Local Proxy Environment +- `npm run test:local:proxy:small` - Small load test +- `npm run test:local:proxy:medium` - Medium load test +- `npm run test:local:proxy:large` - Large load test +- `npm run test:local:proxy:xlarge` - Extra large load test + +### Local Realtime Service +- `npm run test:local:realtime:small` - Small load test +- `npm run test:local:realtime:medium` - Medium load test +- `npm run test:local:realtime:xmedium` - Extra medium load test +- `npm run test:local:realtime:x2medium` - 2x medium load test +- `npm run test:local:realtime:x2mediumLong` - 2x medium long duration test +- `npm run test:local:realtime:large` - Large load test +- `npm run test:local:realtime:xlarge` - Extra large load test + +### Dev Environment +- `npm run test:dev:realtime:small` - Small load test +- `npm run test:dev:realtime:medium` - Medium load test +- `npm run test:dev:realtime:large` - Large load test +- `npm run test:dev:realtime:xlarge` - Extra large load test + +## Environment Variables + +- `INWORLD_API_KEY` - Base64 encoded API key for authentication (optional) +- `ENV_NAME` - Target environment (local-mock, local-proxy, local-realtime, dev) +- `SCENARIO` - Load scenario (small, medium, large, xlarge, etc.) +- `USE_ITEM_DONE_FOR_LATENCY` - Use response.done for latency measurement for response.create (true/false) +- `WAIT_FOR_RESPONSE` - Wait for response before sending next message (true/false) +- `DEBUG` - Enable debug logging (true/false) +- `USE_WORKSPACE_PER_VU` - Enable propagate `workspace-id` unique header per VU + +## Usage + +Run a test script: +```bash +npm run test:local:realtime:medium +``` + +Or run k6 directly with custom parameters: +```bash +k6 run load_test.js -e ENV_NAME=local-realtime -e SCENARIO=medium +``` + +For local load testing or for direct Realtime service testing, you can use env variable `USE_WORKSPACE_PER_VU=true` +to emulate multiple workspace's requests. diff --git a/realtime-service/load-tests/load_test.js b/realtime-service/load-tests/load_test.js new file mode 100644 index 0000000..b249d6e --- /dev/null +++ b/realtime-service/load-tests/load_test.js @@ -0,0 +1,425 @@ +import ws from 'k6/ws'; +import {check} from 'k6'; +import {Rate, Trend, Counter} from 'k6/metrics'; + +const connectionLatency = new Trend('ws_connection_latency', true); +const responseLatency = new Trend('ws_response_latency', true); +const connectionSuccessRate = new Rate('ws_connection_success'); +const messageSuccessRate = new Rate('ws_message_success'); +const messagesSentCounter = new Counter('ws_messages_sent_total'); +const messagesReceivedCounter = new Counter('ws_messages_received_total'); +const websocketErrorCounter = new Counter('websocket_errors_total'); + +// Configuration from environment variables +const INWORLD_API_KEY = __ENV.INWORLD_API_KEY || ''; +const VU_ID = __VU; +const ENV_NAME = __ENV.ENV_NAME || 'local-mock'; +const IS_DEBUG = __ENV.DEBUG === 'true'; +const USE_WORKSPACE_PER_VU = __ENV.USE_WORKSPACE_PER_VU === 'true'; +const USE_ITEM_DONE_FOR_LATENCY = __ENV.USE_ITEM_DONE_FOR_LATENCY === 'true'; +const WAIT_FOR_RESPONSE = __ENV.WAIT_FOR_RESPONSE === 'true'; + +let baseWsUrl; +// Select the base URL based on the environment name +switch (ENV_NAME.toLowerCase()) { + case 'local-proxy': + // local w-proxy + baseWsUrl = 'ws://localhost:8081/api/v1/realtime/session'; + break; + case 'local-realtime': + // local realtime + baseWsUrl = 'ws://localhost:4000/session'; + break; + case 'dev': + // dev Realtime service + baseWsUrl = 'wss://api.dev.inworld.ai:443/api/v1/realtime/session'; + break; + case 'local-mock': + default: + // local mock web-socket server + baseWsUrl = 'ws://localhost:9002/api/v1/realtime/session'; + break; +} + +const WS_URL = `${baseWsUrl}?protocol=realtime`; +console.log(`[Config] Running test against environment: ${ENV_NAME}. URL: ${WS_URL}`); +console.log(`[Config] USE_ITEM_DONE_FOR_LATENCY: ${USE_ITEM_DONE_FOR_LATENCY} (latency for response.create will be measured from response.done)`); +console.log(`[Config] WAIT_FOR_RESPONSE: ${WAIT_FOR_RESPONSE} (send next message only after receiving response to previous)`); + +// Messages to send +const messages = [ + {type: 'session.update', session: {instructions: 'You are a helpful assistant that speaks in a friendly tone.'}}, + {type: 'session.update', session: {audio: {output: {voice: 'Alex'}}}}, + {type: 'session.update', session: {output_modalities: ["text"]}}, + { + type: 'conversation.item.create', + item: {type: 'message', role: 'user', content: [{type: 'input_text', text: 'What is 2+2?'}]} + }, + {type: 'response.create'} +]; + +const scenarios = { + small: [ + { duration: '5s', target: 1 }, + { duration: '5s', target: 3 }, + { duration: '5s', target: 5 }, + { duration: '5s', target: 5 }, + { duration: '5s', target: 0 }, + ], + + medium: [ + { duration: '5s', target: 5 }, + { duration: '5s', target: 8 }, + { duration: '5s', target: 15 }, + { duration: '10s', target: 15 }, + { duration: '5s', target: 0 }, + ], + + xmedium: [ + { duration: '5s', target: 10 }, + { duration: '5s', target: 15 }, + { duration: '5s', target: 25 }, + { duration: '10s', target: 25 }, + { duration: '5s', target: 0 }, + ], + + x2medium: [ + { duration: '5s', target: 20 }, + { duration: '5s', target: 30 }, + { duration: '5s', target: 50 }, + { duration: '10s', target: 50 }, + { duration: '5s', target: 0 }, + ], + + x2mediumLong: [ + { duration: '10s', target: 20 }, + { duration: '10s', target: 30 }, + { duration: '10s', target: 50 }, + { duration: '30s', target: 50 }, + { duration: '10s', target: 0 }, + ], + + large: [ + { duration: '10s', target: 20 }, + { duration: '10s', target: 50 }, + { duration: '10s', target: 80 }, + { duration: '10s', target: 100 }, + { duration: '30s', target: 100 }, + { duration: '10s', target: 0 }, + ], + + xlarge: [ + { duration: '10s', target: 20 }, + { duration: '10s', target: 50 }, + { duration: '10s', target: 80 }, + { duration: '10s', target: 100 }, + { duration: '10s', target: 150 }, + { duration: '10s', target: 200 }, + { duration: '30s', target: 200 }, + { duration: '10s', target: 0 }, + ], +}; + +const scenarioName = __ENV.SCENARIO || "small"; + +// Dynamically create metrics for each message +const messageLatencies = []; +const messageSuccessRates = []; + +for (let i = 0; i < messages.length; i++) { + const messageIndex = i + 1; // 1-based index for metric names + messageLatencies.push(new Trend(`message${messageIndex}_latency`, true)); + messageSuccessRates.push(new Rate(`message${messageIndex}_success`)); +} + +export const options = { + stages: scenarios[scenarioName], + thresholds: { + 'ws_connection_success': ['rate>0.95'], + 'ws_message_success': ['rate>0.95'], + 'ws_connection_latency': ['p(95)<1000'], + 'ws_response_latency': ['p(95)<5000'], + }, + summaryTrendStats: ["min", "max", "avg", "med", "p(90)", "p(95)", "p(99)", "p(99.9)"], +}; + +export default function () { + const sessionKey = `test-session-${VU_ID}-${Date.now()}`; + const url = `${WS_URL}&key=${sessionKey}`; + const params = { + tags: {name: 'WebSocket Connection'}, + headers: {}, + }; + if (INWORLD_API_KEY) { + params.headers['Authorization'] = `Basic ${INWORLD_API_KEY}`; + } + + if (USE_WORKSPACE_PER_VU) { + params.headers['workspace-id'] = `workspace-${VU_ID}}`; + console.log(`[VU ${VU_ID}] Using workspace-id: ${params.headers['workspace-id']}`); + } + + const connectionStart = Date.now(); + + const response = ws.connect(url, params, function (socket) { + const connectionEnd = Date.now(); + const connectionTime = connectionEnd - connectionStart; + connectionLatency.add(connectionTime); + connectionSuccessRate.add(1); + + // Track message sending and response times + const pendingMessages = new Map(); + const pendingResponseCreate = new Map(); + let receivedMessageCount = 0; + let sessionCreatedReceived = false; + let allMessagesSent = false; + let currentMessageIndex = 0; + + const sendMessage = function (index) { + if (index >= messages.length) { + allMessagesSent = true; + return; + } + + const messageStart = Date.now(); + const isResponseCreate = messages[index].type === 'response.create'; + + if (USE_ITEM_DONE_FOR_LATENCY && isResponseCreate) { + pendingResponseCreate.set(index, {sentTime: messageStart}); + } else { + pendingMessages.set(index, {sentTime: messageStart, responded: false}); + } + + messagesSentCounter.add(1); + socket.send(JSON.stringify(messages[index])); + + if (IS_DEBUG) { + console.log(`[VU ${VU_ID}] Sent message ${index + 1}/${messages.length}:`, messages[index].type); + } + + socket.setTimeout(() => { + if (USE_ITEM_DONE_FOR_LATENCY && isResponseCreate) { + if (pendingResponseCreate.has(index)) { + pendingResponseCreate.delete(index); + messageSuccessRate.add(0); + if (index >= 0 && index < messageSuccessRates.length) { + messageSuccessRates[index].add(0); + } + if (IS_DEBUG) { + console.log(`[VU ${VU_ID}] Timeout waiting for response.done for message ${index + 1}`); + } + } + } else { + if (pendingMessages.has(index) && !pendingMessages.get(index).responded) { + pendingMessages.delete(index); + messageSuccessRate.add(0); + if (index >= 0 && index < messageSuccessRates.length) { + messageSuccessRates[index].add(0); + } + if (IS_DEBUG) { + console.log(`[VU ${VU_ID}] Timeout waiting for response to message ${index + 1}`); + } + } + } + }, 10000); + }; + + socket.on('open', () => { + if (IS_DEBUG) { + console.log(`[VU ${VU_ID}] WebSocket connection opened to url ${url}`); + } + + // Send the first message immediately + if (messages.length > 0) { + sendMessage(0); + currentMessageIndex = 1; + } + + if (!WAIT_FOR_RESPONSE) { + // In timed mode, send messages with delays + for (let index = 1; index < messages.length; index++) { + socket.setTimeout(() => { + sendMessage(index); + }, index * 50); + } + + socket.setTimeout(() => { + allMessagesSent = true; + }, (messages.length - 1) * 50); + } + }); + + socket.on('message', (data) => { + const ts = Date.now(); + messagesReceivedCounter.add(1); + + let msg; + try { msg = JSON.parse(data); } + catch { if (IS_DEBUG) console.log("Non-JSON message"); return; } + + const type = msg.type || "unknown"; + if (IS_DEBUG) console.log(`[VU ${VU_ID}] Received: ${type}`); + + if (type === 'session.created') { + sessionCreatedReceived = true; + return; + } + + if (USE_ITEM_DONE_FOR_LATENCY && type === "response.done") { + handleResponseDone(ts); + } else { + handleRegularResponse(ts); + } + + attemptSendNextMessage(); + tryCloseSocket(); + }); + + + function handleResponseDone(ts) { + if (pendingResponseCreate.size === 0) { + if (IS_DEBUG) console.log("response.done received without a pending response.create"); + return; + } + + const [index, entry] = [...pendingResponseCreate.entries()].sort(([a],[b]) => a - b)[0]; + pendingResponseCreate.delete(index); + + const latency = Math.max(0, ts - entry.sentTime); + recordSuccess(index, latency); + + if (IS_DEBUG) { + console.log(`[VU ${VU_ID}] response.done matched message ${index+1}, latency: ${latency}ms`); + } + } + + function handleRegularResponse(ts) { + if (pendingMessages.size === 0) { + if (IS_DEBUG) console.log("Response w/o pendingMessages"); + return; + } + + const [index, entry] = [...pendingMessages.entries()].sort(([a],[b]) => a - b)[0]; + pendingMessages.delete(index); + + let latency = ts - entry.sentTime; + if (latency < 0) latency = 0; + if (latency === 0) latency = 0.5; + + recordSuccess(index, latency); + + if (IS_DEBUG) { + console.log(`[VU ${VU_ID}] matched response for message ${index+1}, latency: ${latency}ms`); + } + } + + function recordSuccess(index, latency) { + responseLatency.add(latency); + messageSuccessRate.add(1); + + if (index >= 0 && index < messageLatencies.length) { + messageLatencies[index].add(latency); + messageSuccessRates[index].add(1); + } + } + + function attemptSendNextMessage() { + if (!WAIT_FOR_RESPONSE) return; + if (currentMessageIndex >= messages.length) return; + + socket.setTimeout(() => { + sendMessage(currentMessageIndex); + currentMessageIndex++; + if (currentMessageIndex >= messages.length) allMessagesSent = true; + }, 50); + } + + function tryCloseSocket() { + const pendingCount = + pendingMessages.size + + (USE_ITEM_DONE_FOR_LATENCY ? pendingResponseCreate.size : 0); + + if (!allMessagesSent || pendingCount > 0) return; + + if (IS_DEBUG) { + console.log(`[VU ${VU_ID}] All responses received. Closing socket.`); + } + + socket.close(); + } + + socket.on('error', function (e) { + console.error(`[VU ${VU_ID}] WebSocket error:`, e.error()); + websocketErrorCounter.add(1); + connectionSuccessRate.add(0); + check(false, {'WebSocket connection must not have errors': e.error() == undefined}); + }); + + socket.on('close', () => { + if (IS_DEBUG) { + console.log(`[VU ${VU_ID}] Connection closed`); + } + + for (const [msgIndex, entry] of pendingMessages.entries()) { + if (!entry.responded) { + messageSuccessRate.add(0); + if (msgIndex >= 0 && msgIndex < messageSuccessRates.length) { + messageSuccessRates[msgIndex].add(0); + } + } + } + pendingMessages.clear(); + + if (USE_ITEM_DONE_FOR_LATENCY) { + for (const [msgIndex, entry] of pendingResponseCreate.entries()) { + messageSuccessRate.add(0); + if (msgIndex >= 0 && msgIndex < messageSuccessRates.length) { + messageSuccessRates[msgIndex].add(0); + } + } + pendingResponseCreate.clear(); + } + }); + + const safetyTimeout = (messages.length * 1000) + 1000; + + socket.setTimeout(() => { + const pendingCount = pendingMessages.size + (USE_ITEM_DONE_FOR_LATENCY ? pendingResponseCreate.size : 0); + if (IS_DEBUG) { + console.log(`[VU ${VU_ID}] Safety timeout - closing connection. Sent: ${messages.length}, Received: ${receivedMessageCount}, Pending: ${pendingCount}`); + } + + for (const [msgIndex, entry] of pendingMessages.entries()) { + if (!entry.responded) { + messageSuccessRate.add(0); + if (msgIndex >= 0 && msgIndex < messageSuccessRates.length) { + messageSuccessRates[msgIndex].add(0); + } + } + } + pendingMessages.clear(); + + if (USE_ITEM_DONE_FOR_LATENCY) { + for (const [msgIndex, entry] of pendingResponseCreate.entries()) { + messageSuccessRate.add(0); + if (msgIndex >= 0 && msgIndex < messageSuccessRates.length) { + messageSuccessRates[msgIndex].add(0); + } + } + pendingResponseCreate.clear(); + } + + socket.close(); + }, safetyTimeout); + }); + + check(response, { + 'WebSocket connection established': (r) => r && r.status === 101, + }, {name: 'Connection Check'}); + + if (!response || response.status !== 101) { + connectionSuccessRate.add(0); + const connectionTime = Date.now() - connectionStart; + connectionLatency.add(connectionTime); + } +} diff --git a/realtime-service/load-tests/package.json b/realtime-service/load-tests/package.json new file mode 100644 index 0000000..166852b --- /dev/null +++ b/realtime-service/load-tests/package.json @@ -0,0 +1,30 @@ +{ + "name": "load-tests", + "version": "1.0.0", + "description": "", + "scripts": { + "test:local:mock:small": "k6 run load_test.js -e ENV_NAME=local-mock -e SCENARIO=small -e USE_ITEM_DONE_FOR_LATENCY=false", + "test:local:mock:medium": "k6 run load_test.js -e ENV_NAME=local-mock -e SCENARIO=medium -e USE_ITEM_DONE_FOR_LATENCY=false", + "test:local:mock:large": "k6 run load_test.js -e ENV_NAME=local-mock -e SCENARIO=large -e USE_ITEM_DONE_FOR_LATENCY=false", + "test:local:mock:xlarge": "k6 run load_test.js -e ENV_NAME=local-mock -e SCENARIO=xlarge -e USE_ITEM_DONE_FOR_LATENCY=false", + + "test:local:proxy:small": "k6 run load_test.js -e ENV_NAME=local-proxy -e SCENARIO=small -e WAIT_FOR_RESPONSE=true -e USE_ITEM_DONE_FOR_LATENCY=true", + "test:local:proxy:medium": "k6 run load_test.js -e ENV_NAME=local-proxy -e SCENARIO=medium -e WAIT_FOR_RESPONSE=true -e USE_ITEM_DONE_FOR_LATENCY=true", + "test:local:proxy:large": "k6 run load_test.js -e ENV_NAME=local-proxy -e SCENARIO=large -e WAIT_FOR_RESPONSE=true -e USE_ITEM_DONE_FOR_LATENCY=true", + "test:local:proxy:xlarge": "k6 run load_test.js -e ENV_NAME=local-proxy -e SCENARIO=xlarge -e WAIT_FOR_RESPONSE=true -e USE_ITEM_DONE_FOR_LATENCY=true", + + "test:local:realtime:small": "k6 run load_test.js -e ENV_NAME=local-realtime -e SCENARIO=small -e WAIT_FOR_RESPONSE=true -e USE_ITEM_DONE_FOR_LATENCY=true", + "test:local:realtime:medium": "k6 run load_test.js -e ENV_NAME=local-realtime -e SCENARIO=medium -e WAIT_FOR_RESPONSE=true -e USE_ITEM_DONE_FOR_LATENCY=true", + "test:local:realtime:xmedium": "k6 run load_test.js -e ENV_NAME=local-realtime -e SCENARIO=xmedium -e WAIT_FOR_RESPONSE=true -e USE_ITEM_DONE_FOR_LATENCY=true", + "test:local:realtime:x2medium": "k6 run load_test.js -e ENV_NAME=local-realtime -e SCENARIO=x2medium -e WAIT_FOR_RESPONSE=true -e USE_ITEM_DONE_FOR_LATENCY=true", + "test:local:realtime:x2mediumLong": "k6 run load_test.js -e ENV_NAME=local-realtime -e SCENARIO=x2mediumLong -e WAIT_FOR_RESPONSE=true -e USE_ITEM_DONE_FOR_LATENCY=true", + "test:local:realtime:large": "k6 run load_test.js -e ENV_NAME=local-realtime -e SCENARIO=large -e WAIT_FOR_RESPONSE=true -e USE_ITEM_DONE_FOR_LATENCY=true", + "test:local:realtime:xlarge": "k6 run load_test.js -e ENV_NAME=local-realtime -e SCENARIO=xlarge -e WAIT_FOR_RESPONSE=true -e USE_ITEM_DONE_FOR_LATENCY=true", + + "test:dev:realtime:small": "k6 run load_test.js -e ENV_NAME=dev -e SCENARIO=small -e USE_ITEM_DONE_FOR_LATENCY=true", + "test:dev:realtime:medium": "k6 run load_test.js -e ENV_NAME=dev -e SCENARIO=medium -e USE_ITEM_DONE_FOR_LATENCY=true", + "test:dev:realtime:large": "k6 run load_test.js -e ENV_NAME=dev -e SCENARIO=large -e USE_ITEM_DONE_FOR_LATENCY=true", + "test:dev:realtime:xlarge": "k6 run load_test.js -e ENV_NAME=dev -e SCENARIO=xlarge -e USE_ITEM_DONE_FOR_LATENCY=true" + }, + "private": true +} diff --git a/realtime-service/mock/echo_ws_server.go b/realtime-service/mock/echo_ws_server.go new file mode 100644 index 0000000..09c95e8 --- /dev/null +++ b/realtime-service/mock/echo_ws_server.go @@ -0,0 +1,50 @@ +package main + +import ( + "log" + "net/http" + + "github.com/gorilla/websocket" +) + +var upgrader = websocket.Upgrader{ + CheckOrigin: func(r *http.Request) bool { + return true + }, +} + +func echoServer(w http.ResponseWriter, r *http.Request) { + conn, err := upgrader.Upgrade(w, r, nil) + if err != nil { + log.Println("Upgrade failed:", err) + return + } + defer conn.Close() + + log.Println("Client connected!") + + for { + messageType, message, err := conn.ReadMessage() + if err != nil { + log.Println("Read failed:", err) + break + } + + log.Printf("Received: %s", message) + + err = conn.WriteMessage(messageType, message) + if err != nil { + log.Println("Write failed:", err) + break + } + } +} + +func main() { + http.HandleFunc("/session", echoServer) + log.Println("WebSocket Mock Server starting on :4000/ws") + err := http.ListenAndServe(":4000", nil) + if err != nil { + log.Fatal("ListenAndServe: ", err) + } +} diff --git a/realtime-service/package-lock.json b/realtime-service/package-lock.json new file mode 100644 index 0000000..fa4ab77 --- /dev/null +++ b/realtime-service/package-lock.json @@ -0,0 +1,33 @@ +{ + "name": "animations-inc-voice-agent", + "lockfileVersion": 3, + "requires": true, + "packages": { + "": { + "dependencies": { + "ws": "^8.18.3" + } + }, + "node_modules/ws": { + "version": "8.18.3", + "resolved": "https://registry.npmjs.org/ws/-/ws-8.18.3.tgz", + "integrity": "sha512-PEIGCY5tSlUt50cqyMXfCzX+oOPqN0vuGqWzbcJ2xvnkzkq46oOpz7dQaTDBdfICb4N14+GARUDw2XV2N4tvzg==", + "license": "MIT", + "engines": { + "node": ">=10.0.0" + }, + "peerDependencies": { + "bufferutil": "^4.0.1", + "utf-8-validate": ">=5.0.2" + }, + "peerDependenciesMeta": { + "bufferutil": { + "optional": true + }, + "utf-8-validate": { + "optional": true + } + } + } + } +} diff --git a/realtime-service/package.json b/realtime-service/package.json new file mode 100644 index 0000000..2bb8f52 --- /dev/null +++ b/realtime-service/package.json @@ -0,0 +1,26 @@ +{ + "name": "realtime-service", + "version": "1.0.0", + "description": "Realtime service with unit testing support", + "scripts": { + "test": "WS_APP_PORT=4000 jest", + "test:api": "WS_APP_PORT=4000 jest __tests__/api", + "test:watch": "WS_APP_PORT=4000 jest --watch", + "test:coverage": "WS_APP_PORT=4000 jest --coverage", + "test:verbose": "WS_APP_PORT=4000 jest --verbose" + }, + "dependencies": { + "uuid": "^11.1.0", + "ws": "^8.18.3" + }, + "devDependencies": { + "@types/jest": "^29.5.11", + "@types/node": "^20.10.6", + "@types/uuid": "^9.0.0", + "@types/ws": "^8.5.10", + "dotenv": "^16.4.7", + "jest": "^29.7.0", + "ts-jest": "^29.1.1", + "typescript": "^5.3.3" + } +} diff --git a/realtime-service/src/.gitignore b/realtime-service/src/.gitignore new file mode 100644 index 0000000..67c86a3 --- /dev/null +++ b/realtime-service/src/.gitignore @@ -0,0 +1,32 @@ +# TypeScript build output +dist/ + +# Dependencies +node_modules/ + +# Environment variables +.env +.env.local +.env.*.local + +# Logs +*.log +npm-debug.log* +yarn-debug.log* +yarn-error.log* + +# OS +.DS_Store +Thumbs.db + +# IDE +.vscode/ +.idea/ +*.swp +*.swo +*~ + +# Testing +coverage/ +.nyc_output/ + diff --git a/realtime-service/src/REALTIME_API.md b/realtime-service/src/REALTIME_API.md new file mode 100644 index 0000000..2c7b371 --- /dev/null +++ b/realtime-service/src/REALTIME_API.md @@ -0,0 +1,313 @@ +# OpenAI Realtime API Implementation + +This server now supports the OpenAI Realtime API protocol for WebSocket-based real-time voice and audio interactions. + +## Protocol Selection + +The server previously supports two protocols: + +1. **Legacy Protocol** (default): The original custom protocol +2. **Realtime Protocol**: OpenAI Realtime API compatible protocol + +The legacy protocol is no longer supported. We now support serving realtime events through OpenAI compatible API events with a few exceptions. + +To use the Realtime protocol, add `protocol=realtime` as a query parameter when connecting to the WebSocket: + +``` +ws://YOUR_API_HOST:PORT/session?protocol=realtime&key=YOUR_SESSION_KEY +``` + +## Connection Flow + + +### 1. Connect via WebSocket + +Connect to the WebSocket endpoint with the Realtime protocol: + +```javascript +const ws = new WebSocket('ws://YOUR_API_HOST:PORT/session?protocol=realtime&key=YOUR_SESSION_KEY'); + +ws.onopen = () => { + console.log('Connected'); + handleOpen(); +}; + +ws.onmessage = (event) => this.handleMessage(event); +ws.onerror = (error) => this.handleError(error); +ws.onclose = () => this.handleClose(); + +``` + +### 2. Send session.update Event + +You should send a session.update Event when the websocket opens, just like when you are using OpenAI API. + +```javascript +const sessionUpdate = { + type: 'session.update', + session: { + type: 'realtime', // Required by OpenAI API + output_modalities: modalities, + instructions: instructions, + tools: tools, + tool_choice: toolChoice, + audio: { + input: { + turn_detection: { + type: 'semantic_vad', + eagerness: eagerness, + create_response: true, + interrupt_response: false, + }, + transcription: { + model: 'gpt-4o-mini-transcribe', + }, + }, + output: { + voice: voice, // Selected voice from dropdown + }, + }, + }, + }; +``` + +You will then be able to send Client Events and receive responses from the server. + +## Client Events + +### Update Session Configuration + +We support partial update to session such as: + +```javascript +ws.send(JSON.stringify({ + type: 'session.update', + session: { + instructions: 'You are a friendly assistant', + voice: 'Hades', + temperature: 0.7 + } +})); +``` + +### Send Audio Input + +The Realtime API uses PCM16 audio at 24kHz sample rate. + +```javascript +// Append audio to input buffer +ws.send(JSON.stringify({ + type: 'input_audio_buffer.append', + audio: base64AudioData // base64-encoded PCM16 audio at 24kHz +})); + +// Manually commit the audio buffer (not needed with server VAD) +ws.send(JSON.stringify({ + type: 'input_audio_buffer.commit' +})); +``` + +### Create a Text Message + +```javascript +ws.send(JSON.stringify({ + type: 'conversation.item.create', + item: { + type: 'message', + role: 'user', + content: [ + { + type: 'input_text', + text: 'Hello, how are you?' + } + ] + } +})); + +// Trigger a response +ws.send(JSON.stringify({ + type: 'response.create' +})); +``` + +### Cancel a Response + +```javascript +ws.send(JSON.stringify({ + type: 'response.cancel' +})); +``` + +### Delete / Retrieve a Response via Item ID + +```javascript +ws.send(JSON.stringify({ + type: 'conversation.item.delete', + item_id: itemId, +})); +``` + +```javascript +ws.send(JSON.stringify({ + type: 'conversation.item.retrieve', + item_id: itemId, +})); +``` + +The item id will be generated by the server in the corresponding event sent back (such as `response.created`). + +We have covered all client events available in OpenAI Websocket Protocol, except for `conversation.item.truncate`. + +## Server Events + +The server will send various events during the conversation: + +### Audio Response Flow + +Here's a sequence of events that will be sent back upon the server receiving audio buffer commits. + +1. `response.created` - Response generation started +2. `response.output_item.added` - Output item created +3. `response.content_part.added` - Content part (audio) added +4. `response.audio_transcript.delta` - Transcript chunks (streaming) +5. `response.audio.delta` - Audio chunks (streaming, base64-encoded WAV) +6. `response.audio_transcript.done` - Transcript complete +7. `response.audio.done` - Audio complete +8. `response.content_part.done` - Content part complete +9. `response.output_item.done` - Output item complete +10. `response.done` - Response complete + +Refer to our API Reference for contents of such events. + +### Voice Activity Detection (VAD) Events + +When server VAD is enabled (default), instead of buffer commits, the server will decide when to create the conversation item. + +As such, events that informs the client will be prepended to the above flow. + +1. `input_audio_buffer.speech_started` - Speech detected +2. `input_audio_buffer.speech_stopped` - Speech ended +3. `input_audio_buffer.committed` - Audio buffer committed +4. `conversation.item.created` - User message item created +5. Response flow (see above) + +## Audio Format + +- **Input**: PCM16, 24kHz, mono, base64-encoded +- **Output**: WAV format, base64-encoded (contains PCM16 at 24kHz) + +## Example: Simple Audio Conversation + +```javascript +const ws = new WebSocket('ws://localhost:4000/session?key=my-session&protocol=realtime'); + +ws.onmessage = (event) => { + const msg = JSON.parse(event.data); + + switch (msg.type) { + case 'session.created': + console.log('Session ready'); + // Start sending audio + break; + + case 'input_audio_buffer.speech_started': + console.log('User started speaking'); + break; + + case 'input_audio_buffer.speech_stopped': + console.log('User stopped speaking'); + break; + + case 'response.audio.delta': + // Decode and play audio + const audioBuffer = Buffer.from(msg.delta, 'base64'); + playAudio(audioBuffer); + break; + + case 'response.audio_transcript.delta': + console.log('Assistant:', msg.delta); + break; + + case 'response.done': + console.log('Response complete'); + break; + + case 'error': + console.error('Error:', msg.error); + break; + } +}; + +// Send audio chunks +function sendAudioChunk(pcm16Data) { + ws.send(JSON.stringify({ + type: 'input_audio_buffer.append', + audio: pcm16Data.toString('base64') + })); +} +``` + +## Configuration + +### Server VAD (Voice Activity Detection) + +By default, server VAD is enabled with these settings: + +- **threshold**: 0.5 (voice detection sensitivity) +- **prefix_padding_ms**: 300 (audio to include before speech) +- **silence_duration_ms**: 500 (silence duration to detect end of speech) +- **create_response**: true (automatically create response when speech ends) +- Alternatively, use eagerness for presets that fits different purposes. (`'low' | 'medium' | 'high'`) The higher eagerness, the more easily for the server to conclude your conversation round. + +You can update these settings using `session.update`: + +```javascript +ws.send(JSON.stringify({ + type: 'session.update', + session: { + turn_detection: { + type: 'server_vad', + threshold: 0.7, + silence_duration_ms: 800, + // eagerness: 'high', + } + } +})); +``` + +To disable server VAD: + +```javascript +ws.send(JSON.stringify({ + type: 'session.update', + session: { + turn_detection: null + } +})); +``` + +## Error Handling + +Errors are sent as `error` events: + +```json +{ + "event_id": "evt_789", + "type": "error", + "error": { + "type": "invalid_request_error", + "code": "invalid_event", + "message": "Input audio buffer is empty", + "param": null, + "event_id": "evt_123" + } +} +``` + +## Differences from OpenAI's Implementation + +This implementation provides compatibility with the OpenAI Realtime API protocol while using different backend services: + +- Uses custom LLM and TTS models (configured via environment variables) +- Audio processing handled by Inworld Runtime graphs +- Some advanced features (like function calling) may have different behavior diff --git a/realtime-service/src/components/app.ts b/realtime-service/src/components/app.ts new file mode 100644 index 0000000..d22dbd0 --- /dev/null +++ b/realtime-service/src/components/app.ts @@ -0,0 +1,78 @@ +import logger from '../logger'; +import { Connection } from '../types/index'; +import { InworldGraphWrapper } from './graphs/graph'; +import { InworldAppConfig } from './runtime_app_manager'; +import { DEFAULT_LLM_MODEL_NAME, DEFAULT_LLM_PROVIDER, DEFAULT_VOICE_ID, DEFAULT_TTS_MODEL_ID } from '../config'; + +export class InworldApp { + graphId?: string; + llmModelName: string; + llmProvider: string; + voiceId: string; + graphVisualizationEnabled: boolean; + ttsModelId: string; + assemblyAIApiKey: string; + useMocks: boolean; + connections: { + [sessionId: string]: Connection; + } = {}; + + graphWithAudioInput: InworldGraphWrapper; + + private constructor(config: InworldAppConfig) { + this.graphId = config.graphId; + this.llmModelName = config.llmModelName || DEFAULT_LLM_MODEL_NAME; + this.llmProvider = config.llmProvider || DEFAULT_LLM_PROVIDER; + this.voiceId = config.voiceId || DEFAULT_VOICE_ID; + this.ttsModelId = config.ttsModelId || DEFAULT_TTS_MODEL_ID; + this.graphVisualizationEnabled = config.graphVisualizationEnabled!; + this.assemblyAIApiKey = config.assemblyAIApiKey || ''; + this.useMocks = config.useMocks ?? false; + + // Use shared connections if provided, otherwise create own + this.connections = config.sharedConnections || {}; + } + + /** + * Factory method to create and initialize an InworldApp instance + * Use this instead of calling the constructor directly + */ + static async create(config: InworldAppConfig): Promise { + const app = new InworldApp(config); + await app.initialize(); + return app; + } + + private async initialize() { + // Create audio graph with Assembly.AI STT (handles both audio and text inputs) + this.graphWithAudioInput = await InworldGraphWrapper.create({ + llmModelName: this.llmModelName, + llmProvider: this.llmProvider, + voiceId: this.voiceId, + connections: this.connections, + graphVisualizationEnabled: this.graphVisualizationEnabled, + ttsModelId: this.ttsModelId, + useAssemblyAI: true, + assemblyAIApiKey: this.assemblyAIApiKey, + useMocks: this.useMocks, + }); + + logger.info('✓ Audio input graph initialized (Assembly.AI STT)'); + } + + /** + * Remove a session and clean up its connection + */ + removeSession(sessionId: string): void { + delete this.connections[sessionId]; + } + + async shutdown() { + // Clear all connections + Object.keys(this.connections).forEach(sessionId => { + delete this.connections[sessionId]; + }); + + await this.graphWithAudioInput.destroy(); + } +} diff --git a/realtime-service/src/components/audio/audio_utils.ts b/realtime-service/src/components/audio/audio_utils.ts new file mode 100644 index 0000000..c11dad3 --- /dev/null +++ b/realtime-service/src/components/audio/audio_utils.ts @@ -0,0 +1,91 @@ +/** + * Audio utility functions for converting between audio formats + */ +import logger from '../../logger'; + +/** + * Convert Float32Array (-1.0 to 1.0) to Int16Array PCM16 (-32768 to 32767) + * @param float32Data - Input audio data as Float32Array + * @returns Int16Array containing PCM16 audio data + */ +export function float32ToPCM16(float32Data: Float32Array): Int16Array { + const pcm16 = new Int16Array(float32Data.length); + for (let i = 0; i < float32Data.length; i++) { + // Clamp values to [-1, 1] range + const clamped = Math.max(-1, Math.min(1, float32Data[i])); + // Convert to 16-bit PCM + pcm16[i] = clamped < 0 ? clamped * 0x8000 : clamped * 0x7fff; + } + return pcm16; +} + +/** + * Convert audio data to PCM16 base64 format for OpenAI Realtime API + * @param audioData - Audio data in various formats (array, Buffer, Float32Array, base64 string) + * @param sampleRate - Sample rate of the audio (for logging purposes) + * @param debugLabel - Label for debug logging + * @returns Base64 encoded PCM16 audio data, or null if conversion fails + */ +export function convertToPCM16Base64( + audioData: any, + sampleRate?: number, + debugLabel: string = 'Audio', +): string | null { + if (!audioData) { + logger.error({ debugLabel }, 'Audio Utils - No audio data provided'); + return null; + } + + // Convert audio data to Float32Array based on its actual type + let floatSamples: Float32Array; + + if (Array.isArray(audioData)) { + // The array contains byte values from a Buffer, not float values + // Interpret these bytes as Float32 data (4 bytes per float) + // console.log(`[${debugLabel}] Converting byte array to Float32Array`); + const byteBuffer = Buffer.from(audioData); + floatSamples = new Float32Array( + byteBuffer.buffer, + byteBuffer.byteOffset, + byteBuffer.length / 4, + ); + } else if (Buffer.isBuffer(audioData)) { + // If it's already a Buffer + logger.debug({ debugLabel }, 'Audio Utils - Converting Buffer to Float32Array'); + floatSamples = new Float32Array( + audioData.buffer, + audioData.byteOffset, + audioData.length / 4, + ); + } else if (audioData instanceof Float32Array) { + // Already Float32Array + logger.debug({ debugLabel }, 'Audio Utils - Using existing Float32Array'); + floatSamples = audioData; + } else if (typeof audioData === 'string') { + // If it's a base64 string (legacy format) + logger.debug({ debugLabel }, 'Audio Utils - Decoding base64 string to Float32Array'); + const decodedData = Buffer.from(audioData, 'base64'); + floatSamples = new Float32Array( + decodedData.buffer, + decodedData.byteOffset, + decodedData.length / 4, + ); + } else { + logger.error({ debugLabel, dataType: typeof audioData }, `Audio Utils - Unsupported audio data type: ${typeof audioData}`); + return null; + } + + // Validate Float32Array has data + if (floatSamples.length === 0) { + logger.warn({ debugLabel }, 'Audio Utils - Skipping zero-length audio samples'); + return null; + } + + // Convert Float32 (-1.0 to 1.0) to PCM16 (-32768 to 32767) + const pcm16 = float32ToPCM16(floatSamples); + + const audioBase64 = Buffer.from(pcm16.buffer).toString('base64'); + logger.debug({ debugLabel, length: audioBase64.length }, 'Audio Utils - Sending base64 audio'); + + return audioBase64; +} diff --git a/realtime-service/src/components/audio/multimodal_stream_manager.ts b/realtime-service/src/components/audio/multimodal_stream_manager.ts new file mode 100644 index 0000000..8c5d5c6 --- /dev/null +++ b/realtime-service/src/components/audio/multimodal_stream_manager.ts @@ -0,0 +1,125 @@ +import { AudioChunkInterface } from '@inworld/runtime/common'; +import { GraphTypes } from '@inworld/runtime/graph'; + +import logger from '../../logger'; + +/** + * Manages a stream of multimodal content (audio and/or text) that can be fed + * asynchronously as data arrives from websocket connections. + * + * This unifies audio and text streaming into a single interface that always + * yields MultimodalContent, making it compatible with entry node routing. + */ +export class MultimodalStreamManager { + private queue: GraphTypes.MultimodalContent[] = []; + private waitingResolvers: Array< + (value: IteratorResult) => void + > = []; + private ended = false; + + /** + * Add an audio chunk to the stream (wrapped in MultimodalContent) + */ + pushAudio(chunk: AudioChunkInterface): void { + if (this.ended) { + return; + } + + // Create GraphTypes.Audio object and wrap in MultimodalContent + const audioData = new GraphTypes.Audio({ + data: Array.isArray(chunk.data) ? chunk.data : Array.from(chunk.data), + sampleRate: chunk.sampleRate, + }); + const multimodalContent = new GraphTypes.MultimodalContent({ + audio: audioData, + }); + + this.pushContent(multimodalContent); + } + + /** + * Add text to the stream (wrapped in MultimodalContent) + */ + pushText(text: string): void { + if (this.ended) { + return; + } + + const multimodalContent = new GraphTypes.MultimodalContent({ text }); + this.pushContent(multimodalContent); + } + + /** + * Internal method to push MultimodalContent to the stream + */ + private pushContent(content: GraphTypes.MultimodalContent): void { + // If there are waiting consumers, resolve immediately + if (this.waitingResolvers.length > 0) { + const resolve = this.waitingResolvers.shift()!; + resolve({ value: content, done: false }); + } else { + // Otherwise, queue the content + this.queue.push(content); + } + } + + /** + * Mark the stream as ended + */ + end(): void { + logger.info('[MultimodalStreamManager] Ending stream'); + this.ended = true; + + // Resolve all waiting consumers with done signal + while (this.waitingResolvers.length > 0) { + const resolve = this.waitingResolvers.shift()!; + resolve({ value: undefined as any, done: true }); + } + } + + /** + * Create an async iterator for the stream + */ + async *createStream(): AsyncIterableIterator { + while (true) { + // If stream ended and queue is empty, we're done + if (this.ended && this.queue.length === 0) { + logger.info('[MultimodalStreamManager] Stream iteration complete'); + return; + } + + // If we have queued content, yield it immediately + if (this.queue.length > 0) { + const content = this.queue.shift()!; + yield content; + continue; + } + + // If stream ended but we just exhausted the queue, we're done + if (this.ended) { + logger.info('[MultimodalStreamManager] Stream iteration complete'); + return; + } + + // Otherwise, wait for new content + const result = await new Promise< + IteratorResult + >((resolve) => { + this.waitingResolvers.push(resolve); + }); + + if (result.done) { + return; + } + + yield result.value; + } + } + + /** + * Check if the stream has ended + */ + isEnded(): boolean { + return this.ended; + } +} diff --git a/realtime-service/src/components/audio/realtime_audio_handler.ts b/realtime-service/src/components/audio/realtime_audio_handler.ts new file mode 100644 index 0000000..9615b64 --- /dev/null +++ b/realtime-service/src/components/audio/realtime_audio_handler.ts @@ -0,0 +1,213 @@ +import logger from '../../logger'; +import { formatSession, formatError } from '../../log-helpers'; +import { INPUT_SAMPLE_RATE } from '../../config'; +import * as RT from '../../types/realtime'; +import { InworldApp } from '../app'; +import { MultimodalStreamManager } from './multimodal_stream_manager'; +import { RealtimeEventFactory } from '../realtime/realtime_event_factory'; +import { RealtimeGraphExecutor } from '../graphs/realtime_graph_executor'; +import { RealtimeSessionManager } from '../realtime/realtime_session_manager'; + +export class RealtimeAudioHandler { + constructor( + private inworldApp: InworldApp, + private sessionKey: string, + private send: (data: RT.ServerEvent) => void, + private graphExecutor: RealtimeGraphExecutor, + private sessionManager: RealtimeSessionManager + ) {} + + /** + * Ensures that the audio graph execution is running. + * Creates the multimodal stream manager and starts the graph execution if not already running. + * @param connection - The connection object to initialize + * @param context - Context string for logging (e.g., 'Audio', 'Text Input') + */ + private ensureAudioGraphExecution( + connection: NonNullable, + context: string = 'Audio' + ): void { + if (connection.multimodalStreamManager) { + return; + } + + connection.multimodalStreamManager = new MultimodalStreamManager(); + + const session = this.sessionManager.getSession(); + + // Start the audio graph execution with the stream + const audioStreamInput = { + sessionId: this.sessionKey, + state: connection.state, + voiceId: connection.state.voiceId || session.session.audio.output.voice, + }; + + // Use the Assembly.AI audio graph + const graphWrapper = this.inworldApp.graphWithAudioInput; + + // Start graph execution in the background - it will consume from the stream + connection.currentAudioGraphExecution = + this.graphExecutor.executeAudioGraph({ + sessionId: this.sessionKey, + workspaceId: connection.workspaceId, + apiKey: connection.apiKey, + input: audioStreamInput, + graphWrapper, + multimodalStreamManager: connection.multimodalStreamManager, + }).catch((error) => { + logger.error({ err: error, sessionId: this.sessionKey }, `${context} - Error in audio graph execution`); + // Clean up on error + if (connection.multimodalStreamManager) { + connection.multimodalStreamManager.end(); + connection.multimodalStreamManager = undefined; + } + connection.currentAudioGraphExecution = undefined; + // Send error to websocket + this.send( + RealtimeEventFactory.error({ + type: 'server_error', + message: error instanceof Error ? error.message : 'Error in audio graph execution', + }), + ); + }); + } + + /** + * Handle input_audio_buffer.append event + * Stream audio directly to Inworld SDK 0.8 audio graph + */ + async handleInputAudioBufferAppend( + event: RT.InputAudioBufferAppendEvent, + ): Promise { + const connection = this.inworldApp.connections[this.sessionKey]; + if (!connection) { + logger.error({ sessionId: this.sessionKey }, 'Audio - No connection found'); + return; + } + + // Decode base64 audio (PCM16 at 24kHz from OpenAI) + const audioBuffer = Buffer.from(event.audio, 'base64'); + const int16Array = new Int16Array( + audioBuffer.buffer, + audioBuffer.byteOffset, + audioBuffer.length / 2, + ); + + // Convert PCM16 Int16 to Float32 for Inworld graph (normalize to -1.0 to 1.0) + const float32Array = new Float32Array(int16Array.length); + for (let i = 0; i < int16Array.length; i++) { + float32Array[i] = int16Array[i] / 32768.0; + } + + // Downsample from 24kHz to 16kHz (2:3 ratio) + // For every 3 samples at 24kHz, we output 2 samples at 16kHz + const targetLength = Math.floor(float32Array.length * 2 / 3); + const resampled = new Float32Array(targetLength); + + for (let i = 0; i < targetLength; i++) { + const sourceIndex = i * 1.5; // 24kHz/16kHz = 1.5 + const index0 = Math.floor(sourceIndex); + const index1 = Math.min(index0 + 1, float32Array.length - 1); + const frac = sourceIndex - index0; + + // Linear interpolation + resampled[i] = float32Array[index0] * (1 - frac) + float32Array[index1] * frac; + } + + // Convert to number array for MultimodalStreamManager + const audioData = Array.from(resampled); + + // Ensure the audio graph execution is running + this.ensureAudioGraphExecution(connection, 'Audio'); + + // Push the audio chunk to the stream (already resampled to 16kHz) + connection.multimodalStreamManager!.pushAudio({ + data: audioData, + sampleRate: INPUT_SAMPLE_RATE, // 16kHz for Inworld graph + }); + } + + /** + * Handle input_audio_buffer.commit event + * End the audio stream and wait for graph to complete + */ + async handleInputAudioBufferCommit( + event: RT.InputAudioBufferCommitEvent, + ): Promise { + const connection = this.inworldApp.connections[this.sessionKey]; + if (!connection) { + this.send( + RealtimeEventFactory.error({ + type: 'invalid_request_error', + message: 'No connection found', + event_id: event.event_id, + }), + ); + return; + } + + if (!connection.multimodalStreamManager) { + this.send( + RealtimeEventFactory.error({ + type: 'invalid_request_error', + message: 'No active audio stream', + event_id: event.event_id, + }), + ); + return; + } + + logger.info({ sessionId: this.sessionKey }, `Commit - Manual commit requested - ending audio stream [${this.sessionKey}]`); + connection.multimodalStreamManager.end(); + + // Wait for the graph execution to complete + if (connection.currentAudioGraphExecution) { + await connection.currentAudioGraphExecution; + } + + // Clear the buffer + this.sessionManager.getSession().inputAudioBuffer = []; + } + + /** + * Handle input_audio_buffer.clear event + */ + async handleInputAudioBufferClear( + event: RT.InputAudioBufferClearEvent, + ): Promise { + const connection = this.inworldApp.connections[this.sessionKey]; + if (connection?.multimodalStreamManager) { + connection.multimodalStreamManager.end(); + connection.multimodalStreamManager = undefined; + connection.currentAudioGraphExecution = undefined; + } + + this.sessionManager.getSession().inputAudioBuffer = []; + this.send(RealtimeEventFactory.inputAudioBufferCleared()); + } + + /** + * Handle text input by pushing it to the multimodal stream manager + * This unifies text and audio inputs through the same audio graph + */ + async handleTextInput(text: string): Promise { + const connection = this.inworldApp.connections[this.sessionKey]; + if (!connection) { + logger.error({ sessionId: this.sessionKey }, 'Text Input - No connection found'); + return; + } + + logger.info({ sessionId: this.sessionKey, text: text.substring(0, 100) }, `Text Input - Pushing text to audio graph: "${text.substring(0, 50)}..."`); + + // Ensure the audio graph execution is running + this.ensureAudioGraphExecution(connection, 'Text Input'); + + // Push the text to the stream + connection.multimodalStreamManager!.pushText(text); + + // Don't wait for completion here - let the graph run in the background + // The graph will automatically create a response when it processes the text + // The response.create event will wait for completion if needed + } +} + diff --git a/realtime-service/src/components/graphs/graph.ts b/realtime-service/src/components/graphs/graph.ts new file mode 100644 index 0000000..704fdb9 --- /dev/null +++ b/realtime-service/src/components/graphs/graph.ts @@ -0,0 +1,308 @@ +import { + Graph, + GraphBuilder, + ProxyNode, + RemoteLLMChatNode, + RemoteTTSNode, + FakeTTSComponent, + TextAggregatorNode, + TextChunkingNode, FakeRemoteLLMComponent, + RemoteLLMChatRoutingNode // TODO: Create GraphTypes.LLMChatRoutingRequest and use remoteLLMChatRoutingNode instead +} from '@inworld/runtime/graph'; +import * as os from 'os'; +import * as path from 'path'; +import logger from '../../logger'; + +import { + INPUT_SAMPLE_RATE, + TTS_SAMPLE_RATE, +} from '../../config'; +import { CreateGraphPropsInterface, State, TextInput } from '../../types/index'; +import { getAssemblyAISettingsForEagerness } from '../../types/settings'; +import { AssemblyAISTTWebSocketNode } from './nodes/assembly_ai_stt_ws_node'; +import { DialogPromptBuilderNode } from './nodes/dialog_prompt_builder_node'; +import { InteractionQueueNode } from './nodes/interaction_queue_node'; +import { StateUpdateNode } from './nodes/state_update_node'; +import { TextInputNode } from './nodes/text_input_node'; +import { TranscriptExtractorNode } from './nodes/transcript_extractor_node'; +import { TTSRequestBuilderNode } from './nodes/tts_request_builder_node'; + +export class InworldGraphWrapper { + graph: Graph; + assemblyAINode: AssemblyAISTTWebSocketNode; + + private constructor({ graph, assemblyAINode }: { graph: Graph; assemblyAINode: AssemblyAISTTWebSocketNode }) { + this.graph = graph; + this.assemblyAINode = assemblyAINode; + } + + async destroy() { + await this.graph.stop(); + } + + static async create(props: CreateGraphPropsInterface) { + const { + llmModelName, + llmProvider, + voiceId, + connections, + ttsModelId, + useMocks = false, + } = props; + + const postfix = `-multimodal`; + + const dialogPromptBuilderNode = new DialogPromptBuilderNode({ + id: `dialog-prompt-builder-node${postfix}`, + }); + + const textInputNode = new TextInputNode({ + id: `text-input-node${postfix}`, + connections, + reportToClient: true, + }); + + const llmNode = new RemoteLLMChatNode({ + id: `llm-node${postfix}`, + provider: llmProvider, + modelName: llmModelName, + stream: true, + textGenerationConfig: { + maxNewTokens: 320 + }, + reportToClient: true, + ...(useMocks && { + llmComponent: new FakeRemoteLLMComponent({ + id: `llm-component${postfix}`, + modelName: llmModelName, + provider: llmProvider, + }), + }), + }); + + const textChunkingNode = new TextChunkingNode({ + id: `text-chunking-node${postfix}`, + }); + + const textAggregatorNode = new TextAggregatorNode({ + id: `text-aggregator-node${postfix}`, + }); + + const stateUpdateNode = new StateUpdateNode({ + id: `state-update-node${postfix}`, + connections, + reportToClient: true, + }); + + const ttsRequestBuilderNode = new TTSRequestBuilderNode({ + id: `tts-request-builder-node${postfix}`, + connections, + defaultVoiceId: voiceId, + reportToClient: false, + }); + + const ttsNode = new RemoteTTSNode({ + id: `tts-node${postfix}`, + speakerId: voiceId, + modelId: ttsModelId, + sampleRate: TTS_SAMPLE_RATE, + temperature: 1.1, + speakingRate: 1, + reportToClient: true, + ...(useMocks && { + ttsComponent: new FakeTTSComponent({ + id: `tts-component-${postfix}`, + loadTestConfig: { + firstChunkDelay: 200, + sampleRate: 48000, + errorProbability: 0.0, + chunksPerRequest: 20, + interChunkDelay: 100, + collectMetrics: true, + }, + }), + }), + }); + + // A second branch that only executes text chunking - we will not execute TTS when output_modality doesn't contain audio + const dialogPromptBuilderNodeTextOnly = new DialogPromptBuilderNode({ + id: `dialog-prompt-builder-node-text-only${postfix}`, + }); + + const textChunkingNodeTextOnly = new TextChunkingNode({ + id: `text-chunking-node-text-only${postfix}`, + reportToClient: true, + }); + + const llmNodeTextOnly = new RemoteLLMChatNode({ + id: `llm-node-text-only${postfix}`, + provider: llmProvider, + modelName: llmModelName, + stream: true, + textGenerationConfig: { maxNewTokens: 320 }, + reportToClient: true, + ...(useMocks && { + llmComponent: new FakeRemoteLLMComponent({ + id: `llm-component-text-only${postfix}`, + modelName: llmModelName, + provider: llmProvider, + }), + }), + }); + + const textAggregatorNodeTextOnly = new TextAggregatorNode({ + id: `text-aggregator-node-text-only${postfix}`, + }); + + const stateUpdateNodeTextOnly = new StateUpdateNode({ + id: `state-update-node-text-only${postfix}`, + connections, + reportToClient: true, + }); + // End of the text only branch + + const graphName = `voice-agent${postfix}`; + const graphBuilder = new GraphBuilder({ + id: graphName, + enableRemoteConfig: false, + }); + + graphBuilder + .addNode(textInputNode) + .addNode(dialogPromptBuilderNode) + .addNode(llmNode) + .addNode(textChunkingNode) + .addNode(textAggregatorNode) + .addNode(ttsRequestBuilderNode) + .addNode(ttsNode) + .addNode(stateUpdateNode) + .addEdge(textInputNode, dialogPromptBuilderNode, { + condition: async (input: State) => { + return input?.output_modalities.includes("audio") + }, + }) + .addEdge(dialogPromptBuilderNode, llmNode) + .addEdge(llmNode, textChunkingNode) + .addEdge(textInputNode, ttsRequestBuilderNode, { + condition: async (input: State) => { + return input?.output_modalities.includes("audio") + }, + }) + .addEdge(textChunkingNode, ttsRequestBuilderNode) + .addEdge(ttsRequestBuilderNode, ttsNode) + .addEdge(llmNode, textAggregatorNode) + .addEdge(textAggregatorNode, stateUpdateNode) + // Text-only outputs nodes/edges + .addNode(dialogPromptBuilderNodeTextOnly) + .addNode(textChunkingNodeTextOnly) + .addNode(llmNodeTextOnly) + .addNode(textAggregatorNodeTextOnly) + .addNode(stateUpdateNodeTextOnly) + .addEdge(textInputNode, dialogPromptBuilderNodeTextOnly, { + condition: async (input: State) => { + return !input?.output_modalities.includes("audio") && input?.output_modalities.includes("text") + }, + }) + .addEdge(dialogPromptBuilderNodeTextOnly, llmNodeTextOnly) + .addEdge(llmNodeTextOnly, textChunkingNodeTextOnly) + .addEdge(llmNodeTextOnly, textAggregatorNodeTextOnly) + .addEdge(textAggregatorNodeTextOnly, stateUpdateNodeTextOnly) + ; + + + // Validate configuration + if (!props.assemblyAIApiKey) { + throw new Error('Assembly.AI API key is required for audio input'); + } + + logger.info('Building graph with Multimodal pipeline'); + + // Start node to pass the audio input to Assembly.AI STT + const audioInputNode = new ProxyNode(); + const interactionQueueNode = new InteractionQueueNode({ + id: `interaction-queue-node${postfix}`, + connections, + reportToClient: false, + }); + + // Get eagerness settings from connection state, default to 'medium' + const connection = connections[Object.keys(connections)[0]]; + const eagerness = connection?.state?.eagerness || 'medium'; + const turnDetectionSettings = getAssemblyAISettingsForEagerness(eagerness); + + logger.info({ + eagerness, + profile: turnDetectionSettings.description, + endOfTurnConfidenceThreshold: turnDetectionSettings.endOfTurnConfidenceThreshold, + minEndOfTurnSilenceWhenConfident: turnDetectionSettings.minEndOfTurnSilenceWhenConfident, + maxTurnSilence: turnDetectionSettings.maxTurnSilence, + }, `Configured eagerness: ${eagerness} (${turnDetectionSettings.description})`); + + const assemblyAISTTNode = new AssemblyAISTTWebSocketNode({ + id: `assembly-ai-stt-ws-node${postfix}`, + config: { + apiKey: props.assemblyAIApiKey!, + connections: connections, + sampleRate: INPUT_SAMPLE_RATE, + formatTurns: false, + endOfTurnConfidenceThreshold: turnDetectionSettings.endOfTurnConfidenceThreshold, + minEndOfTurnSilenceWhenConfident: turnDetectionSettings.minEndOfTurnSilenceWhenConfident, + maxTurnSilence: turnDetectionSettings.maxTurnSilence, + }, + }); + + const transcriptExtractorNode = new TranscriptExtractorNode({ + id: `transcript-extractor-node${postfix}`, + reportToClient: true, + }); + + graphBuilder + .addNode(audioInputNode) + .addNode(assemblyAISTTNode) + .addNode(transcriptExtractorNode) + .addNode(interactionQueueNode) + .addEdge(audioInputNode, assemblyAISTTNode) + .addEdge(assemblyAISTTNode, assemblyAISTTNode, { + condition: async (input: any) => { + return input?.stream_exhausted !== true; + }, + loop: true, + optional: true, + }) + // When interaction is complete, send to transcriptExtractorNode for processing + .addEdge(assemblyAISTTNode, transcriptExtractorNode, { + condition: async (input: any) => { + return input?.interaction_complete === true; + }, + }) + .addEdge(transcriptExtractorNode, interactionQueueNode) + .addEdge(interactionQueueNode, textInputNode, { + condition: (input: TextInput) => { + logger.debug({ text: input.text?.substring(0, 100) }, `InteractionQueueNode checking condition: "${input.text?.substring(0, 50)}..."`); + return input.text && input.text.trim().length > 0; + }, + }) + .addEdge(stateUpdateNode, interactionQueueNode, { + loop: true, + optional: true, + }) + .setStartNode(audioInputNode); + + graphBuilder.setEndNode(ttsNode); + + const graph = graphBuilder.build(); + if (props.graphVisualizationEnabled) { + const graphPath = path.join(os.tmpdir(), `${graphName}.png`); + logger.info( + { graphPath }, + 'The Graph visualization will be saved to this path. If you see any fatal error after this message, pls disable graph visualization' + ); + } + + // Return wrapper with assemblyAI node reference + return new InworldGraphWrapper({ + graph, + assemblyAINode: assemblyAISTTNode, + }); + } +} diff --git a/realtime-service/src/components/graphs/nodes/assembly_ai_stt_ws_node.ts b/realtime-service/src/components/graphs/nodes/assembly_ai_stt_ws_node.ts new file mode 100644 index 0000000..8b6a9c8 --- /dev/null +++ b/realtime-service/src/components/graphs/nodes/assembly_ai_stt_ws_node.ts @@ -0,0 +1,855 @@ +import { DataStreamWithMetadata } from '@inworld/runtime'; +import { CustomNode, GraphTypes, ProcessContext } from '@inworld/runtime/graph'; +import WebSocket from 'ws'; +import { v4 as uuidv4 } from 'uuid'; + +import { Connection } from '../../../types/index'; +import { getAssemblyAISettingsForEagerness } from '../../../types/settings'; +import logger from '../../../logger'; +import { formatSession, formatContext, formatError } from '../../../log-helpers'; +import { float32ToPCM16 } from '../../audio/audio_utils'; + +/** + * Configuration interface for AssemblyAISTTWebSocketNode + */ +export interface AssemblyAISTTWebSocketNodeConfig { + /** Assembly.AI API key */ + apiKey: string; + /** Connections map to access session state */ + connections: { [sessionId: string]: Connection }; + /** Sample rate of the audio stream in Hz */ + sampleRate?: number; + /** Enable turn formatting from Assembly.AI */ + formatTurns?: boolean; + /** End of turn confidence threshold (0-1) */ + endOfTurnConfidenceThreshold?: number; + /** Minimum silence duration when confident (in milliseconds) */ + minEndOfTurnSilenceWhenConfident?: number; + /** Maximum turn silence (in milliseconds) */ + maxTurnSilence?: number; + /** Language code (e.g., 'en', 'es') */ + language?: string; + /** Keywords/keyterms to boost recognition */ + keytermsPrompt?: string[]; +} + +/** + * Manages a persistent WebSocket connection to Assembly.AI for a single session. + * Encapsulates connection lifecycle, inactivity timeouts, and message sending. + */ +class AssemblyAISession { + private ws: WebSocket | null = null; + private wsReady: boolean = false; + private wsConnectionPromise: Promise | null = null; + + public assemblySessionId: string = ''; + public sessionExpiresAt: number = 0; + public shouldStopProcessing: boolean = false; + + private inactivityTimeout: NodeJS.Timeout | null = null; + private lastActivityTime: number = Date.now(); + private readonly INACTIVITY_TIMEOUT_MS = 60000; // 60 seconds + + constructor( + public readonly sessionId: string, + private apiKey: string, + private url: string, + private onCleanup: (sessionId: string) => void + ) {} + + /** + * Ensure WebSocket connection is ready, reconnecting if needed + */ + public async ensureConnection(): Promise { + // Check if connection is expired + const now = Math.floor(Date.now() / 1000); + const isExpired = this.sessionExpiresAt > 0 && now >= this.sessionExpiresAt; + + if ( + !this.ws || + !this.wsReady || + this.ws.readyState !== WebSocket.OPEN || + isExpired + ) { + if (isExpired) { + logger.info({ sessionId: this.sessionId }, 'AssemblyAI session expired, reconnecting'); + } else if (this.ws) { + logger.info({ sessionId: this.sessionId, readyState: this.ws.readyState }, `AssemblyAI connection not ready [state:${this.ws.readyState}], reconnecting`); + } else { + logger.info({ sessionId: this.sessionId }, 'AssemblyAI connecting'); + } + + // Close existing connection if any + this.closeWebSocket(); + + // Start new connection + this.initializeWebSocket(); + } + + if (this.wsConnectionPromise) { + await this.wsConnectionPromise; + } + + // Reset flags + this.shouldStopProcessing = false; + this.resetInactivityTimer(); + } + + /** + * Initialize the WebSocket connection + */ + private initializeWebSocket(): void { + logger.info({ sessionId: this.sessionId }, 'AssemblyAI initializing WebSocket'); + + this.wsConnectionPromise = new Promise((resolve, reject) => { + logger.info({ sessionId: this.sessionId, url: this.url }, 'AssemblyAI WS STT - Connecting'); + + this.ws = new WebSocket(this.url, { + headers: { Authorization: this.apiKey }, + }); + + this.ws.on('open', () => { + logger.info({ sessionId: this.sessionId }, `AssemblyAI WebSocket opened ${formatSession(this.sessionId)}`); + this.wsReady = true; + resolve(); + }); + + // Permanent message handler for session metadata + this.ws.on('message', (data: WebSocket.Data) => { + try { + const message = JSON.parse(data.toString()); + if (message.type === 'Begin') { + this.assemblySessionId = message.id || message.session_id || ''; + this.sessionExpiresAt = message.expires_at || 0; + logger.info({ + sessionId: this.sessionId, + assemblySessionId: this.assemblySessionId, + expiresAt: this.sessionExpiresAt ? new Date(this.sessionExpiresAt * 1000).toISOString() : 'unknown', + }, `AssemblyAI session began ${formatSession(this.sessionId)} [assembly:${this.assemblySessionId}]`); + } + } catch (error) { + // Ignore parsing errors here, they might be handled by other listeners + } + }); + + this.ws.on('error', (error: Error) => { + logger.error({ err: error, sessionId: this.sessionId }, 'AssemblyAI WebSocket error'); + this.wsReady = false; + reject(error); + }); + + this.ws.on('close', (code: number, reason: Buffer) => { + logger.info({ + sessionId: this.sessionId, + code, + reason: reason.toString(), + }, `AssemblyAI WebSocket closed ${formatSession(this.sessionId)} [code:${code}] [reason:${reason.toString()}]`); + this.wsReady = false; + }); + }); + } + + /** + * Add a message listener + */ + public onMessage(listener: (data: WebSocket.Data) => void): void { + if (this.ws) { + this.ws.on('message', listener); + } + } + + /** + * Remove a message listener + */ + public offMessage(listener: (data: WebSocket.Data) => void): void { + if (this.ws) { + this.ws.off('message', listener); + } + } + + /** + * Send audio data + */ + public sendAudio(pcm16Data: Int16Array): void { + if (this.ws && this.ws.readyState === WebSocket.OPEN) { + this.ws.send(Buffer.from(pcm16Data.buffer)); + this.resetInactivityTimer(); + } else { + logger.warn({ sessionId: this.sessionId }, 'AssemblyAI WebSocket not open, skipping audio chunk'); + } + } + + /** + * Update turn detection configuration on the active WebSocket connection + */ + public updateConfiguration(config: { + endOfTurnConfidenceThreshold?: number; + minEndOfTurnSilenceWhenConfident?: number; + maxTurnSilence?: number; + }): void { + if (this.ws && this.ws.readyState === WebSocket.OPEN) { + const updateMessage: any = { + type: 'UpdateConfiguration', + }; + + if (config.endOfTurnConfidenceThreshold !== undefined) { + updateMessage.end_of_turn_confidence_threshold = config.endOfTurnConfidenceThreshold; + } + if (config.minEndOfTurnSilenceWhenConfident !== undefined) { + updateMessage.min_end_of_turn_silence_when_confident = config.minEndOfTurnSilenceWhenConfident; + } + if (config.maxTurnSilence !== undefined) { + updateMessage.max_turn_silence = config.maxTurnSilence; + } + + this.ws.send(JSON.stringify(updateMessage)); + logger.info({ sessionId: this.sessionId, config: updateMessage }, `AssemblyAI configuration updated ${formatSession(this.sessionId)}`); + } else { + logger.warn({ sessionId: this.sessionId }, 'AssemblyAI cannot update config: WebSocket not open'); + } + } + + + /** + * Reset the inactivity timer + */ + private resetInactivityTimer(): void { + if (this.inactivityTimeout) { + clearTimeout(this.inactivityTimeout); + } + + this.lastActivityTime = Date.now(); + this.inactivityTimeout = setTimeout(() => { + this.closeDueToInactivity(); + }, this.INACTIVITY_TIMEOUT_MS); + } + + /** + * Close connection due to inactivity + */ + private closeDueToInactivity(): void { + const inactiveFor = Date.now() - this.lastActivityTime; + logger.info({ sessionId: this.sessionId, inactiveFor }, `AssemblyAI closing due to inactivity ${formatSession(this.sessionId)} [inactive:${inactiveFor}ms]`); + + this.shouldStopProcessing = true; + this.close(); + this.onCleanup(this.sessionId); + } + + /** + * Close the WebSocket connection + */ + private closeWebSocket(): void { + if (this.ws) { + try { + // Remove all listeners to prevent leaks + this.ws.removeAllListeners(); + if (this.ws.readyState === WebSocket.OPEN) { + this.ws.close(); + } + } catch (e) { + logger.warn({ err: e, sessionId: this.sessionId }, 'AssemblyAI error closing socket'); + } + this.ws = null; + this.wsReady = false; + } + } + + /** + * Gracefully close the session + */ + public async close(): Promise { + if (this.inactivityTimeout) { + clearTimeout(this.inactivityTimeout); + } + + if (this.ws && this.ws.readyState === WebSocket.OPEN) { + try { + this.ws.send(JSON.stringify({ type: 'Terminate' })); + await new Promise((resolve) => setTimeout(resolve, 100)); + } catch (e) { + // Ignore error on send + } + } + + this.closeWebSocket(); + } +} + +/** + * AssemblyAISTTWebSocketNode processes continuous multimodal streams (audio and/or text) using Assembly.AI's + * streaming Speech-to-Text service via direct WebSocket connection. + * + * This node: + * - Receives MultimodalContent stream (audio and/or text) + * - For audio: extracts audio from MultimodalContent and feeds to Assembly.AI streaming transcriber + * - For text: skips processing (text input should bypass STT) + * - Connects directly to Assembly.AI WebSocket endpoint + * - Detects turn endings automatically using Assembly.AI's turn detection + * - Returns DataStreamWithMetadata with transcribed text when a turn completes + */ +export class AssemblyAISTTWebSocketNode extends CustomNode { + private apiKey: string; + private connections: { [sessionId: string]: Connection }; + private sampleRate: number; + private formatTurns: boolean; + private endOfTurnConfidenceThreshold: number; + private minEndOfTurnSilenceWhenConfident: number; + private maxTurnSilence: number; + private language: string; + private keytermsPrompt: string[]; + private wsEndpointBaseUrl: string = 'wss://streaming.assemblyai.com/v3/ws'; + + private sessions: Map = new Map(); + private readonly TURN_COMPLETION_TIMEOUT_MS = 2000; + private readonly MAX_TRANSCRIPTION_DURATION_MS = 40000; + + constructor(props: { + id?: string; + config: AssemblyAISTTWebSocketNodeConfig; + }) { + const { config, ...nodeProps } = props; + + if (!config.apiKey) { + throw new Error('AssemblyAISTTWebSocketNode requires an API key.'); + } + + if (!config.connections) { + throw new Error( + 'AssemblyAISTTWebSocketNode requires a connections object.', + ); + } + + super({ + id: nodeProps.id || 'assembly-ai-stt-ws-node', + executionConfig: { + sampleRate: config.sampleRate || 16000, + formatTurns: config.formatTurns !== false, + endOfTurnConfidenceThreshold: + config.endOfTurnConfidenceThreshold || 0.4, + minEndOfTurnSilenceWhenConfident: + config.minEndOfTurnSilenceWhenConfident || 400, + maxTurnSilence: config.maxTurnSilence || 1280, + language: config.language || 'en', + }, + }); + + this.apiKey = config.apiKey; + this.connections = config.connections; + this.sampleRate = config.sampleRate || 16000; + this.formatTurns = config.formatTurns !== false; + this.endOfTurnConfidenceThreshold = + config.endOfTurnConfidenceThreshold || 0.4; + this.minEndOfTurnSilenceWhenConfident = + config.minEndOfTurnSilenceWhenConfident || 400; + this.maxTurnSilence = config.maxTurnSilence || 1280; + this.language = config.language || 'en'; + this.keytermsPrompt = config.keytermsPrompt || []; + + // Log the turn detection settings being used + logger.info({ + endOfTurnConfidenceThreshold: this.endOfTurnConfidenceThreshold, + minEndOfTurnSilenceWhenConfident: this.minEndOfTurnSilenceWhenConfident, + maxTurnSilence: this.maxTurnSilence, + sampleRate: this.sampleRate, + formatTurns: this.formatTurns, + language: this.language, + }, `AssemblyAI configured [threshold:${this.endOfTurnConfidenceThreshold}] [silence:${this.minEndOfTurnSilenceWhenConfident}ms] [lang:${this.language}]`); + } + + /** + * Build WebSocket URL with query parameters + * Dynamically uses connection.state.eagerness if available + */ + private buildWebSocketUrl(sessionId?: string): string { + // Get current settings - check connection state for eagerness updates + let endOfTurnThreshold = this.endOfTurnConfidenceThreshold; + let minSilenceWhenConfident = this.minEndOfTurnSilenceWhenConfident; + let maxSilence = this.maxTurnSilence; + + if (sessionId) { + const connection = this.connections[sessionId]; + const eagerness = connection?.state?.eagerness; + + if (eagerness) { + // Map eagerness to settings dynamically using shared settings function + const settings = getAssemblyAISettingsForEagerness(eagerness); + endOfTurnThreshold = settings.endOfTurnConfidenceThreshold; + minSilenceWhenConfident = settings.minEndOfTurnSilenceWhenConfident; + maxSilence = settings.maxTurnSilence; + + logger.info({ sessionId, eagerness }, `AssemblyAI using eagerness settings: ${eagerness}`); + } + } + + const params = new URLSearchParams({ + sample_rate: this.sampleRate.toString(), + format_turns: this.formatTurns.toString(), + end_of_turn_confidence_threshold: endOfTurnThreshold.toString(), + min_end_of_turn_silence_when_confident: minSilenceWhenConfident.toString(), + max_turn_silence: maxSilence.toString(), + language: this.language, + }); + + // Add keyterms if provided + if (this.keytermsPrompt.length > 0) { + this.keytermsPrompt.forEach((term) => { + params.append('keyterms_prompt', term); + }); + } + + const url = `${this.wsEndpointBaseUrl}?${params.toString()}`; + + logger.info({ + sessionId, + endOfTurnConfidenceThreshold: endOfTurnThreshold, + minEndOfTurnSilenceWhenConfident: minSilenceWhenConfident, + maxTurnSilence: maxSilence, + }, `AssemblyAI connecting ${formatSession(sessionId)} [threshold:${endOfTurnThreshold}] [silence:${minSilenceWhenConfident}ms]`); + + return url; + } + + + /** + * Process multimodal stream (audio and/or text) and transcribe using Assembly.AI WebSocket + * For audio: extracts audio from MultimodalContent and sends to Assembly.AI + * For text: currently not handled (text input should bypass STT) + */ + async process( + context: ProcessContext, + input0: AsyncIterableIterator, + input: DataStreamWithMetadata, + ): Promise { + // Extract MultimodalContent stream from either input type + const multimodalStream = + input !== undefined && + input !== null && + input instanceof DataStreamWithMetadata + ? (input.toStream() as any as AsyncIterableIterator) + : input0; + + const sessionId = context.getDatastore().get('sessionId') as string; + const connection = this.connections[sessionId]; + + // Check connection exists before accessing its properties + if (connection?.unloaded) { + throw Error(`Session unloaded for sessionId: ${sessionId}`); + } + if (!connection) { + throw Error(`Failed to read connection for sessionId: ${sessionId}`); + } + + // Get iteration number from metadata, or parse from interactionId + const metadata = input?.getMetadata?.() || {}; + let previousIteration = (metadata.iteration as number) || 0; + + // If interactionId is empty, assign a UUID + if (!connection.state.interactionId || connection.state.interactionId === '') { + connection.state.interactionId = uuidv4(); + logger.info({ sessionId, interactionId: connection.state.interactionId }, 'AssemblyAI assigned new UUID for empty interactionId'); + } + + const currentId = connection.state.interactionId; + const delimiterIndex = currentId.indexOf('#'); + + if (previousIteration === 0 && delimiterIndex !== -1) { + const iterationStr = currentId.substring(delimiterIndex + 1); + const parsedIteration = parseInt(iterationStr, 10); + if (!isNaN(parsedIteration) && /^\d+$/.test(iterationStr)) { + previousIteration = parsedIteration; + } + } + + const iteration = previousIteration + 1; + const baseId = + delimiterIndex !== -1 + ? currentId.substring(0, delimiterIndex) + : currentId; + const nextInteractionId = `${baseId}#${iteration}`; + + logger.info({ sessionId, iteration }, `AssemblyAI starting transcription [iteration:${iteration}]`); + + // State tracking + let transcriptText = ''; + let turnDetected = false; + let speechDetected = false; + let audioChunkCount = 0; + let totalAudioSamples = 0; + let isStreamExhausted = false; + let errorOccurred = false; + let errorMessage = ''; + let maxDurationReached = false; + // For text modality input + let isTextInput = false; + let textContent: string | undefined; + + // Get or create session + let session = this.sessions.get(sessionId); + if (!session) { + session = new AssemblyAISession( + sessionId, + this.apiKey, + this.buildWebSocketUrl(sessionId), // Pass sessionId to get dynamic eagerness settings + (id) => this.sessions.delete(id) + ); + this.sessions.set(sessionId, session); + } + + // Promise to capture the turn result + let turnResolve: (value: string) => void; + let turnReject: (error: any) => void; + let turnCompleted = false; + const turnPromise = new Promise((resolve, reject) => { + turnResolve = resolve; + turnReject = reject; + }); + const turnPromiseWithState = turnPromise.then((value) => { + turnCompleted = true; + return value; + }); + + // Assembly AI Callback handler + const messageHandler = (data: WebSocket.Data) => { + try { + const message = JSON.parse(data.toString()); + const msgType = message.type; + + if (msgType === 'Turn') { + // Ignore turn events if we've already decided to stop + if (session?.shouldStopProcessing) { + return; + } + + const transcript = message.transcript || ''; + const utterance = message.utterance || ''; + const isFinal = message.end_of_turn; + + if (!transcript) return; + + if (!isFinal) { + // Send partial transcript + const textToSend = utterance || transcript; + if (textToSend) { + this.sendPartialTranscript(sessionId, nextInteractionId, textToSend); + + if (connection?.onSpeechDetected && !speechDetected) { + logger.info({ sessionId, iteration, interactionId: nextInteractionId }, `AssemblyAI speech detected [iteration:${iteration}]`); + speechDetected = true; + connection.onSpeechDetected(nextInteractionId); + } + } + return; + } + + // Final transcript + logger.info({ sessionId, iteration, transcript }, `AssemblyAI turn detected ${formatSession(sessionId)} [iteration:${iteration}]: "${transcript.substring(0, 50)}..."`); + + transcriptText = transcript; + turnDetected = true; + if (session) session.shouldStopProcessing = true; + turnResolve(transcript); + + } else if (msgType === 'Termination') { + logger.info({ sessionId, iteration }, `AssemblyAI session terminated ${formatSession(sessionId)} [iteration:${iteration}]`); + } + } catch (error) { + logger.error({ err: error, sessionId, iteration }, `AssemblyAI error handling message [iteration:${iteration}]`); + } + }; + + try { + // Ensure WebSocket connection is ready + await session.ensureConnection(); + + // Attach message handler + session.onMessage(messageHandler); + + // Process multimodal content (audio chunks) + const audioProcessingPromise = (async () => { + let maxDurationTimeout: NodeJS.Timeout | null = null; + try { + logger.debug({ sessionId, iteration }, 'AssemblyAI WS STT - Starting multimodal processing loop'); + + + // Safety timer: prevent infinite loops if no turn is detected + maxDurationTimeout = setTimeout(() => { + maxDurationReached = true; // Ensure maximum process() execution length doesn't exceed 40. If the player with an active mic does not speak for 60s, the node executor will error out thinking it's a zombie node + // We'll loop back in the graph and continue after timing out + }, this.MAX_TRANSCRIPTION_DURATION_MS); + + while (true) { + if (session?.shouldStopProcessing) { + break; + } + + if (maxDurationReached) { + if (!transcriptText) { + logger.warn({ sessionId, iteration }, `AssemblyAI max transcription duration reached [limit:${this.MAX_TRANSCRIPTION_DURATION_MS}ms]`); + break; + } + } + + const result = await multimodalStream.next(); + + if (result.done) { + logger.info({ sessionId, iteration, audioChunkCount }, `AssemblyAI multimodal stream exhausted [iteration:${iteration}] [chunks:${audioChunkCount}]`); + isStreamExhausted = true; + break; + } + + if (session?.shouldStopProcessing) break; + + const content = result.value as GraphTypes.MultimodalContent; + + // Handle text input - immediately simulate turn detection + if (content.text !== undefined && content.text !== null) { + logger.info({ sessionId, iteration, text: content.text }, `AssemblyAI text input detected [iteration:${iteration}]: "${content.text.substring(0, 50)}..."`); + isTextInput = true; + textContent = content.text; + transcriptText = content.text; + turnDetected = true; + if (session) session.shouldStopProcessing = true; + turnResolve(transcriptText); + // For text input, we immediately complete and bypass STT + break; + } + + // Extract audio from MultimodalContent + if (content.audio === undefined || content.audio === null) { + continue; + } + + const audioData = content.audio.data; + if (!audioData || audioData.length === 0) { + continue; + } + + // Convert to Float32Array if needed + const float32Data = Array.isArray(audioData) + ? new Float32Array(audioData) + : audioData; + + audioChunkCount++; + totalAudioSamples += float32Data.length; + + const pcm16Data = float32ToPCM16(float32Data); + + session?.sendAudio(pcm16Data); + + if (audioChunkCount % 20 === 0) { + // Heartbeat log + } + } + } catch (error) { + logger.error({ err: error, sessionId, iteration }, `AssemblyAI error processing audio [iteration:${iteration}]`); + errorOccurred = true; + errorMessage = error instanceof Error ? error.message : String(error); + throw error; + } finally { + if (maxDurationTimeout) { + clearTimeout(maxDurationTimeout); + } + } + })(); + + const raceResult = await Promise.race([ + turnPromiseWithState.then(() => ({ winner: 'turn' as const })), + audioProcessingPromise.then(() => ({ winner: 'audio' as const })), // Audio will immediately win after the stream stops manually, + ]); + + if (raceResult.winner === 'audio' && !turnCompleted && !maxDurationReached) { // and if audio wins, we enter this race, as turnComplete is not here yet + logger.info({ sessionId, iteration, timeout: this.TURN_COMPLETION_TIMEOUT_MS }, `AssemblyAI audio ended before turn [iteration:${iteration}], waiting ${this.TURN_COMPLETION_TIMEOUT_MS}ms for transcript`); + + // Send 100ms of silence every 100ms to keep the connection alive/processing + const silenceIntervalMs = 100; + const silenceSamples = Math.floor((silenceIntervalMs / 1000) * this.sampleRate); + const silenceFrame = new Int16Array(silenceSamples); + const silenceTimer = setInterval(() => { + if (session && !session.shouldStopProcessing) { + session.sendAudio(silenceFrame); + } + }, silenceIntervalMs); // This is critical. Assembly AI Streaming API expects constant audio stream, or it will not emit any events. + // We need to continue streaming even if the user is not actively sending audio. + + const timeoutPromise = new Promise<{ winner: 'timeout' }>((resolve) => + setTimeout(() => resolve({ winner: 'timeout' }), this.TURN_COMPLETION_TIMEOUT_MS), + ); + + const waitResult = await Promise.race([ + turnPromiseWithState.then(() => ({ winner: 'turn' as const })), + timeoutPromise, + ]); + + // We either timed out here or received a final turn event + clearInterval(silenceTimer); + + if (waitResult.winner === 'timeout' && !turnCompleted) { + logger.warn({ sessionId, iteration }, `AssemblyAI timed out waiting for turn [iteration:${iteration}]`); + turnReject?.(new Error('Timed out waiting for turn completion')); + } + } + + // Ensure the audio processing loop fully exits before returning + await audioProcessingPromise.catch(() => {}); + + logger.info({ sessionId, iteration, transcript: transcriptText }, `AssemblyAI transcription complete [iteration:${iteration}]: "${transcriptText?.substring(0, 50)}..."`); + + // Clear interactionId on turn completion + if (turnDetected) { + logger.info({ sessionId, iteration, interactionId: nextInteractionId }, 'AssemblyAI clearing interactionId after turn completion'); + connection.state.interactionId = ''; + } + + // Tag the stream with type for runtime + const taggedStream = Object.assign(multimodalStream, { + type: 'MultimodalContent', + abort: () => { + // No-op abort handler + }, + }); + + return new DataStreamWithMetadata(taggedStream as any, { + elementType: 'MultimodalContent', + iteration: iteration, + interactionId: nextInteractionId, + session_id: sessionId, + assembly_session_id: session.assemblySessionId, + transcript: transcriptText, + turn_detected: turnDetected, + audio_chunk_count: audioChunkCount, + total_audio_samples: totalAudioSamples, + sample_rate: this.sampleRate, + stream_exhausted: isStreamExhausted, + interaction_complete: turnDetected && transcriptText.length > 0, + error_occurred: errorOccurred, + error_message: errorMessage, + endpointing_latency_ms: 0, + // Flags to match native graph structure + is_running: !isStreamExhausted, + is_text_input: isTextInput, + is_interruption: false, // Not currently handling interruptions in this node + text_content: textContent, + }); + + } catch (error) { + logger.error({ err: error, sessionId, iteration }, `AssemblyAI transcription failed [iteration:${iteration}]`); + + // Tag the stream with type for runtime + const taggedStream = Object.assign(multimodalStream, { + type: 'MultimodalContent', + abort: () => { + // No-op abort handler + }, + }); + + return new DataStreamWithMetadata(taggedStream as any, { + elementType: 'MultimodalContent', + iteration: iteration, + interactionId: nextInteractionId, + session_id: sessionId, + assembly_session_id: session?.assemblySessionId || '', + transcript: '', + turn_detected: false, + audio_chunk_count: audioChunkCount, + total_audio_samples: totalAudioSamples, + sample_rate: this.sampleRate, + stream_exhausted: isStreamExhausted, + interaction_complete: false, + error_occurred: true, + error_message: error instanceof Error ? error.message : String(error), + endpointing_latency_ms: 0, + // Flags to match native graph structure + is_running: !isStreamExhausted, + is_text_input: isTextInput, + is_interruption: false, // Not currently handling interruptions in this node + text_content: textContent, + }); + } + finally { + // Clean up message handler after execution ends + if (session) { + session.offMessage(messageHandler); + } + } + } + + /** + * Send partial transcript update to the client for real-time feedback + */ + private sendPartialTranscript( + sessionId: string, + interactionId: string, + text: string, + ): void { + const connection = this.connections[sessionId]; + if (!connection || !connection.ws) { + return; + } + + try { + if (connection.onPartialTranscript) { + connection.onPartialTranscript(text, interactionId); + } + } catch (error) { + logger.error({ err: error, sessionId }, 'AssemblyAI error sending partial transcript'); + } + } + + /** + * Update turn detection configuration for a specific session + */ + updateTurnDetectionSettings( + sessionId: string, + settings: { + endOfTurnConfidenceThreshold: number; + minEndOfTurnSilenceWhenConfident: number; + maxTurnSilence: number; + } + ): void { + const session = this.sessions.get(sessionId); + if (!session) { + logger.warn({ sessionId }, 'AssemblyAI cannot update settings: no active session'); + return; + } + + // Update the node's stored settings + this.endOfTurnConfidenceThreshold = settings.endOfTurnConfidenceThreshold; + this.minEndOfTurnSilenceWhenConfident = settings.minEndOfTurnSilenceWhenConfident; + this.maxTurnSilence = settings.maxTurnSilence; + + logger.info({ sessionId, settings }, 'AssemblyAI updating turn detection'); + + // Send UpdateConfiguration message to active AssemblyAI WebSocket + session.updateConfiguration(settings); + } + + /** + * Close a specific session by sessionId + */ + async closeSession(sessionId: string): Promise { + const session = this.sessions.get(sessionId); + if (session) { + logger.info(`[AssemblyAI WS STT] Closing session: ${sessionId}`); + await session.close(); + this.sessions.delete(sessionId); + logger.info(`[AssemblyAI WS STT] Session ${sessionId} closed and removed`); + } + } + + /** + * Clean up resources + */ + async destroy(): Promise { + logger.info({ sessionCount: this.sessions.size }, `AssemblyAI destroying node: closing ${this.sessions.size} WebSocket connections`); + + const promises: Promise[] = []; + for (const session of this.sessions.values()) { + promises.push(session.close()); + } + + await Promise.all(promises); + this.sessions.clear(); + logger.info('AssemblyAI all sessions cleaned up'); + } +} diff --git a/realtime-service/src/components/graphs/nodes/dialog_prompt_builder_node.ts b/realtime-service/src/components/graphs/nodes/dialog_prompt_builder_node.ts new file mode 100644 index 0000000..28030b1 --- /dev/null +++ b/realtime-service/src/components/graphs/nodes/dialog_prompt_builder_node.ts @@ -0,0 +1,87 @@ +import { CustomNode, GraphTypes, ProcessContext } from '@inworld/runtime/graph'; +import logger from '../../../logger'; + +import { State } from '../../../types/index'; + +/** + * DialogPromptBuilderNode builds a LLM chat request from the state. + * + * This node: + * - Receives the current conversation state + * - Converts state messages to LLM message format + * - Returns a formatted LLMChatRequest for the LLM node + */ +export class DialogPromptBuilderNode extends CustomNode { + process(_context: ProcessContext, state: State): GraphTypes.LLMChatRequest { + try { + logger.debug({ messageCount: state.messages?.length }, `DialogPromptBuilderNode start: ${state.messages?.length || 0} messages`); + + // Convert state messages to LLMMessageInterface format + // Filter out messages with empty content + const conversationMessages = state.messages + .filter((msg) => { + if (!msg) { + logger.warn('DialogPromptBuilderNode - Found undefined message in state'); + return false; + } + // Filter out messages with empty content + if (!msg.content || msg.content.trim() === '') { + logger.debug({ messageId: msg.id }, 'DialogPromptBuilderNode - Filtering out empty message'); + return false; + } + return true; + }) + .map((msg) => ({ + role: msg.role, + content: msg.content, + })); + + const request: any = { + messages: conversationMessages, + }; + + if (state.tools && Array.isArray(state.tools) && state.tools.length > 0) { + logger.debug({ toolCount: state.tools.length }, `DialogPromptBuilderNode processing ${state.tools.length} tools`); + + // Transform OpenAI Realtime API format to Inworld SDK format + // OpenAI Realtime API: { type: 'function', name, description, parameters } + // Inworld SDK: { name, description, properties } + // The key difference: 'parameters' -> 'properties' (which is now a stringified JSON) + request.tools = state.tools + .filter((t: any) => t != null && typeof t === 'object') + .map((tool: any) => { + if (tool.type === 'function' && tool.name) { + return { + name: tool.name, + description: tool.description, + properties: JSON.stringify(tool.parameters || {}), + }; + } + // If already in Inworld format or unknown, pass through + return tool; + }); + + logger.debug({ toolCount: request.tools.length }, `DialogPromptBuilderNode converted ${request.tools.length} tools to Inworld format`); + + // Handle toolChoice - ensure it's in the right format + if (state.toolChoice) { + if (typeof state.toolChoice === 'string') { + request.toolChoice = { type: state.toolChoice }; + } else { + request.toolChoice = state.toolChoice; + } + } else { + request.toolChoice = { type: 'auto' }; + } + + logger.debug({ toolChoice: request.toolChoice }, `DialogPromptBuilderNode tool choice: ${request.toolChoice}`); + } + + logger.debug({ messageCount: conversationMessages.length }, `DialogPromptBuilderNode final request: ${conversationMessages.length} messages`); + return new GraphTypes.LLMChatRequest(request); + } catch (error) { + logger.error({ err: error }, 'DialogPromptBuilderNode fatal error'); + throw error; + } + } +} diff --git a/realtime-service/src/components/graphs/nodes/interaction_queue_node.ts b/realtime-service/src/components/graphs/nodes/interaction_queue_node.ts new file mode 100644 index 0000000..635d73e --- /dev/null +++ b/realtime-service/src/components/graphs/nodes/interaction_queue_node.ts @@ -0,0 +1,178 @@ +import { CustomNode, ProcessContext } from '@inworld/runtime/graph'; +import logger from '../../../logger'; +import { formatSession, formatContext } from '../../../log-helpers'; + +import { ConnectionsMap, InteractionInfo, State, TextInput } from '../../../types/index'; + +/** + * InteractionQueueNode manages the queue of user interactions. + * + * This node: + * - Receives interaction info from STT processing + * - Manages a queue of interactions to ensure sequential processing + * - Prevents race conditions when multiple interactions arrive + * - Returns TextInput when ready to process, or empty when waiting + * + * Queue states tracked in datastore: + * - 'q{id}': Queued interactions waiting to be processed + * - 'r{id}': Running interactions currently being processed + * - 'c{id}': Completed interactions + */ +export class InteractionQueueNode extends CustomNode { + private connections: ConnectionsMap; + + constructor(props?: { + id?: string; + connections?: ConnectionsMap; + reportToClient?: boolean; + }) { + super({ + id: props?.id || 'interaction-queue-node', + reportToClient: props?.reportToClient, + }); + this.connections = props?.connections || {}; + } + + process( + context: ProcessContext, + interactionInfo: InteractionInfo, + state: State, + ): TextInput { + const sessionId = interactionInfo.sessionId; + logger.debug({ sessionId, interactionId: interactionInfo.interactionId }, 'InteractionQueueNode processing'); + + // Get current voiceId from connection state (in case it was updated via session.update) + const connection = this.connections[sessionId]; + const currentVoiceId = connection?.state?.voiceId || state?.voiceId; + + // ==================================================================== + // STEP 1-3: Store text and analyze queue state + // ==================================================================== + const dataStore = context.getDatastore(); + const QUEUED_PREFIX = 'q'; + const RUNNING_PREFIX = 'r'; + const COMPLETED_PREFIX = 'c'; + + // Register interaction in the queue + if (!dataStore.has(QUEUED_PREFIX + interactionInfo.interactionId)) { + // Store queued interaction + dataStore.add( + QUEUED_PREFIX + interactionInfo.interactionId, + interactionInfo.text, + ); + logger.info({ sessionId, interactionId: interactionInfo.interactionId }, 'InteractionQueue - New interaction queued'); + } + + // Get all keys and categorize them + const allKeys = dataStore.keys(); + const queuedIds: string[] = []; + let completedCount = 0; + let runningCount = 0; + + for (const key of allKeys) { + if (key.startsWith(QUEUED_PREFIX)) { + const idStr = key.substring(QUEUED_PREFIX.length); + queuedIds.push(idStr); + } else if (key.startsWith(COMPLETED_PREFIX)) { + completedCount++; + } else if (key.startsWith(RUNNING_PREFIX)) { + runningCount++; + } + } + + // Sort queued IDs - extract iteration number for sorting + queuedIds.sort((a, b) => { + const getIteration = (id: string): number => { + const hashIndex = id.indexOf('#'); + if (hashIndex === -1) return 0; + const iter = parseInt(id.substring(hashIndex + 1), 10); + return isNaN(iter) ? 0 : iter; + }; + return getIteration(a) - getIteration(b); + }); + + logger.debug({ + sessionId, + queuedCount: queuedIds.length, + completedCount, + runningCount, + }, 'InteractionQueue - State'); + + // ==================================================================== + // STEP 4: Decide if we should start processing the next interaction + // ==================================================================== + if (queuedIds.length === 0) { + // No interactions to process yet + logger.debug({ sessionId }, 'InteractionQueue - No interactions to process yet'); + return { + text: '', + sessionId: sessionId, + interactionId: '', + voiceId: currentVoiceId, + } as TextInput; + } + + if (queuedIds.length === completedCount) { + // All interactions have been processed + logger.debug({ sessionId }, 'InteractionQueue - All interactions completed'); + return { + text: '', + sessionId: sessionId, + interactionId: '', + voiceId: currentVoiceId, + } as TextInput; + } + + // There are unprocessed interactions + if (runningCount === completedCount) { + // No interaction is currently running, start the next one + const nextId = queuedIds[completedCount]; + const runningKey = RUNNING_PREFIX + nextId; + + // NOTE: We do NOT skip interactions marked as "interrupted" + // The "isInterrupted" flag means "this interaction caused an interruption of a previous response" + // But the user's speech is still a valid request that should be processed! + // All user speech should result in a response. + + // Try to mark as running (prevents race conditions) + if (dataStore.has(runningKey) || !dataStore.add(runningKey, '')) { + logger.debug({ sessionId, interactionId: nextId }, 'InteractionQueue - Interaction already started'); + return { + text: '', + sessionId: sessionId, + interactionId: '', + voiceId: currentVoiceId, + } as TextInput; + } + + const queuedText = dataStore.get(QUEUED_PREFIX + nextId) as string; + if (!queuedText) { + logger.error({ sessionId, interactionId: nextId }, 'InteractionQueue - Failed to retrieve text'); + return { + text: '', + sessionId: sessionId, + interactionId: '', + voiceId: currentVoiceId, + } as TextInput; + } + + logger.info({ sessionId, interactionId: nextId, text: queuedText.substring(0, 100) }, `InteractionQueue - Starting LLM processing: "${queuedText.substring(0, 50)}..."`); + + return { + text: queuedText, + sessionId: sessionId, + interactionId: nextId, + voiceId: currentVoiceId, + } as TextInput; + } else { + // An interaction is currently running, wait for it to complete + logger.debug({ sessionId, waitingForInteraction: queuedIds[completedCount] }, `InteractionQueue - Waiting for interaction [waiting for:${queuedIds[completedCount]}]`); + return { + text: '', + sessionId: sessionId, + interactionId: '', + voiceId: currentVoiceId, + } as TextInput; + } + } +} diff --git a/realtime-service/src/components/graphs/nodes/state_update_node.ts b/realtime-service/src/components/graphs/nodes/state_update_node.ts new file mode 100644 index 0000000..7f31c82 --- /dev/null +++ b/realtime-service/src/components/graphs/nodes/state_update_node.ts @@ -0,0 +1,67 @@ +import { CustomNode, ProcessContext } from '@inworld/runtime/graph'; +import logger from '../../../logger'; +import { formatSession, formatContext } from '../../../log-helpers'; + +import { ConnectionsMap, State } from '../../../types/index'; + +/** + * StateUpdateNode updates the state with the LLM's response. + * + * This node: + * - Receives the LLM output text + * - Updates the connection state with the assistant message + * - Marks the interaction as completed in the datastore + * - Returns the updated state + */ +export class StateUpdateNode extends CustomNode { + private connections: ConnectionsMap; + + constructor(props: { + id: string; + connections: ConnectionsMap; + reportToClient?: boolean; + }) { + super({ + id: props.id, + reportToClient: props.reportToClient, + }); + this.connections = props.connections; + } + + process(context: ProcessContext, llmOutput: string): State { + const sessionId = context.getDatastore().get('sessionId') as string; + logger.debug({ sessionId, llmOutputLength: llmOutput?.length }, `StateUpdateNode processing [length:${llmOutput?.length || 0}]`); + + // Get sessionId from dataStore (constant for graph execution) + + const connection = this.connections[sessionId]; + if (connection?.unloaded) { + throw Error(`Session unloaded for sessionId:${sessionId}`); + } + if (!connection) { + throw Error(`Failed to read connection for sessionId:${sessionId}`); + } + + // Only add assistant message if there's actual content + // When LLM returns tool calls only, llmOutput is empty string + if (llmOutput && llmOutput.trim().length > 0) { + logger.debug({ sessionId, content: llmOutput.substring(0, 100) }, `StateUpdateNode adding assistant message: "${llmOutput.substring(0, 50)}..."`); + connection.state.messages.push({ + role: 'assistant', + content: llmOutput, + id: connection.state.interactionId, + }); + } else { + logger.debug({ sessionId }, 'StateUpdateNode skipping empty message (likely tool call only)'); + } + + const dataStore = context.getDatastore(); + dataStore.add('c' + connection.state.interactionId, ''); + logger.info({ + sessionId, + interactionId: connection.state.interactionId, + }, `StateUpdateNode marking interaction completed ${formatContext(sessionId, undefined, connection.state.interactionId)}`); + + return connection.state; + } +} diff --git a/realtime-service/src/components/graphs/nodes/text_input_node.ts b/realtime-service/src/components/graphs/nodes/text_input_node.ts new file mode 100644 index 0000000..311a7d0 --- /dev/null +++ b/realtime-service/src/components/graphs/nodes/text_input_node.ts @@ -0,0 +1,75 @@ +import { CustomNode, ProcessContext } from '@inworld/runtime/graph'; +import logger from '../../../logger'; +import { formatSession } from '../../../log-helpers'; + +import { ConnectionsMap, State, TextInput } from '../../../types/index'; +import { TOOL_CALL_CONTINUATION_MARKER } from '../realtime_graph_executor'; + +/** + * TextInputNode updates the state with the user's input this turn. + * + * This node: + * - Receives user text input with interaction and session IDs + * - Updates the connection state with the user message + * - Returns the updated state for downstream processing + * + * Special handling for tool call continuation: + * - When text is TOOL_CALL_CONTINUATION_MARKER, this is a continuation after a tool call + * - In this case, we DON'T add a new user message (tool result is already in messages) + * - We just return the current state to trigger the LLM with existing conversation + */ +export class TextInputNode extends CustomNode { + private connections: ConnectionsMap; + + constructor(props: { + id: string; + connections: ConnectionsMap; + reportToClient?: boolean; + }) { + super({ + id: props.id, + reportToClient: props.reportToClient, + }); + this.connections = props.connections; + } + + process(context: ProcessContext, input: TextInput): State { + logger.debug({ sessionId: input.sessionId, text: input.text?.substring(0, 100) }, `TextInputNode processing: "${input.text?.substring(0, 50)}..."`); + + const { text, interactionId, sessionId } = input; + + const connection = this.connections[sessionId]; + if (connection?.unloaded) { + throw Error(`Session unloaded for sessionId:${sessionId}`); + } + if (!connection) { + throw Error(`Failed to read connection for sessionId:${sessionId}`); + } + const state = connection.state; + if (!state) { + throw Error( + `Failed to read state from connection for sessionId:${sessionId}`, + ); + } + + // Update interactionId + connection.state.interactionId = interactionId; + + // Check if this is a tool call continuation + // If so, the tool result is already in messages (added by createConversationItem) + // We skip adding a new user message and just return the state to continue the conversation + if (text === TOOL_CALL_CONTINUATION_MARKER) { + logger.info({ sessionId, interactionId }, 'TextInputNode: Tool call continuation - skipping user message, continuing with existing conversation'); + return connection.state; + } + + // Normal flow: add user message to conversation + connection.state.messages.push({ + role: 'user', + content: text, + id: interactionId, + }); + + return connection.state; + } +} diff --git a/realtime-service/src/components/graphs/nodes/text_output_stream_node.ts b/realtime-service/src/components/graphs/nodes/text_output_stream_node.ts new file mode 100644 index 0000000..ec6c6ac --- /dev/null +++ b/realtime-service/src/components/graphs/nodes/text_output_stream_node.ts @@ -0,0 +1,29 @@ +import { CustomNode, ProcessContext } from '@inworld/runtime/graph'; + +import { ConnectionsMap, State, TextInput } from '../../../types/index'; +import {TextStream} from "@inworld/runtime"; + +/** + * TextInputNode updates the state with the user's input this turn. + * + * This node: + * - Takes input from Text Chunking Node as a text stream + * - Outputs for graph handler to send to the client. Only used when modality is TextOnly + */ +export class TextOutputStreamNode extends CustomNode { + private connections: ConnectionsMap; + + constructor(props: { + id: string; + connections: ConnectionsMap; + }) { + super({ + id: props.id, + }); + this.connections = props.connections; + } + + process(context: ProcessContext, input: TextStream): TextStream { + return input; + } +} diff --git a/realtime-service/src/components/graphs/nodes/transcript_extractor_node.ts b/realtime-service/src/components/graphs/nodes/transcript_extractor_node.ts new file mode 100644 index 0000000..f3e8701 --- /dev/null +++ b/realtime-service/src/components/graphs/nodes/transcript_extractor_node.ts @@ -0,0 +1,63 @@ +import { DataStreamWithMetadata } from '@inworld/runtime'; +import { CustomNode, ProcessContext } from '@inworld/runtime/graph'; +import logger from '../../../logger'; +import { formatContext } from '../../../log-helpers'; + +import { InteractionInfo } from '../../../types/index'; + +/** + * TranscriptExtractorNode extracts transcript information from + * DataStreamWithMetadata (typically output from AssemblyAISTTNode) + * and converts it to InteractionInfo for downstream processing. + * + * This is a helper node to bridge Assembly.AI STT output with + * the rest of the graph that expects InteractionInfo. + */ +export class TranscriptExtractorNode extends CustomNode { + constructor(props?: { + id?: string; + reportToClient?: boolean; + }) { + super({ + id: props?.id || 'transcript-extractor-node', + reportToClient: props?.reportToClient, + }); + } + + /** + * Extract transcript from metadata and return as InteractionInfo + */ + process( + context: ProcessContext, + streamWithMetadata: DataStreamWithMetadata, + ): InteractionInfo { + const metadata = streamWithMetadata.getMetadata(); + const sessionId = context.getDatastore().get('sessionId') as string; + + // Extract transcript and related info from metadata + const transcript = (metadata.transcript as string) || ''; + const interactionComplete = + (metadata.interaction_complete as boolean) || false; + const iteration = (metadata.iteration as number) || 1; + const interactionId = String(metadata.interactionId || iteration); + + logger.info({ + sessionId, + interactionId, + iteration, + interactionComplete, + transcript, + }, `TranscriptExtractor processing [iteration:${iteration}]: "${transcript?.substring(0, 50)}..."`); + + // Return InteractionInfo + return { + sessionId, + interactionId: interactionId, + text: transcript, + }; + } + + async destroy(): Promise { + // No cleanup needed + } +} diff --git a/realtime-service/src/components/graphs/nodes/tts_request_builder_node.ts b/realtime-service/src/components/graphs/nodes/tts_request_builder_node.ts new file mode 100644 index 0000000..f1a0c3a --- /dev/null +++ b/realtime-service/src/components/graphs/nodes/tts_request_builder_node.ts @@ -0,0 +1,64 @@ +import { CustomNode, GraphTypes, ProcessContext } from '@inworld/runtime/graph'; +import logger from '../../../logger'; +import { formatSession } from '../../../log-helpers'; +import { ConnectionsMap } from '../../../types/index'; + +/** + * TTSRequestBuilderNode builds a TTSRequest with dynamic voiceId. + * For long-running graphs, it reads voiceId from connection state at processing time + * to ensure voice changes via session.update are reflected immediately. + */ +export class TTSRequestBuilderNode extends CustomNode { + private connections: ConnectionsMap; + private defaultVoiceId: string; + + constructor(props: { + id: string; + connections: ConnectionsMap; + defaultVoiceId: string; + reportToClient?: boolean; + }) { + super({ + id: props.id, + reportToClient: props.reportToClient, + }); + this.connections = props.connections; + this.defaultVoiceId = props.defaultVoiceId; + } + + /** + * Build a TTSRequest with the current voiceId from connection state + * Receives two inputs: + * 1. input - Graph input with sessionId (TextInput or State) + * 2. textStream - The text stream from TextChunkingNode + */ + process( + context: ProcessContext, + input: any, + textStream: GraphTypes.TextStream, + ): GraphTypes.TTSRequest { + const sessionId = context.getDatastore().get('sessionId') as string; + + // For long-running graphs, read voiceId from connection state at processing time + // This ensures voice changes via session.update are immediately reflected + const connection = this.connections[sessionId]; + const voiceId = connection?.state?.voiceId || input?.voiceId || this.defaultVoiceId; + + logger.debug({ + sessionId, + voiceId, + connectionVoiceId: connection?.state?.voiceId, + inputVoiceId: input?.voiceId, + defaultVoiceId: this.defaultVoiceId, + }, `TTSRequestBuilder building request [voice:${voiceId}]`); + + return GraphTypes.TTSRequest.withStream(textStream, { + id: voiceId + }); + } + + async destroy(): Promise { + // No cleanup needed + } +} + diff --git a/realtime-service/src/components/graphs/realtime_graph_executor.ts b/realtime-service/src/components/graphs/realtime_graph_executor.ts new file mode 100644 index 0000000..f067470 --- /dev/null +++ b/realtime-service/src/components/graphs/realtime_graph_executor.ts @@ -0,0 +1,1180 @@ +import { GraphOutputStream, GraphTypes } from '@inworld/runtime/graph'; +import { v4 as uuidv4 } from 'uuid'; +import logger from '../../logger'; +import * as RT from '../../types/realtime'; +import { InworldApp } from '../app'; +import { RealtimeEventFactory } from '../realtime/realtime_event_factory'; +import { InworldGraphWrapper } from '../graphs/graph'; +import { Connection } from '../../types/index'; +import { convertToPCM16Base64 } from '../audio/audio_utils'; +import { ToolCallInterface } from "@inworld/runtime"; +import { RealtimeSessionManager } from '../realtime/realtime_session_manager'; +import { MultimodalStreamManager } from '../audio/multimodal_stream_manager'; +import { abortStream } from '../../helpers'; + +// Marker used to signal tool call continuation - the graph nodes recognize this +// and skip adding a new user message, instead continuing with existing conversation state +export const TOOL_CALL_CONTINUATION_MARKER = '[TOOL_CALL_CONTINUATION]'; + +export class RealtimeGraphExecutor { + private isCancelled = false; + private currentTTSInteractionId: string | null = null; + private currentTranscriptionItemId: string | null = null; + private partialTranscripts: Map = new Map(); + + constructor( + private inworldApp: InworldApp, + private sessionKey: string, + private send: (data: RT.ServerEvent) => void, + private sessionManager: RealtimeSessionManager, + private sessionStartTime: number + ) { } + + cancelCurrentResponse(reason: 'turn_detected' | 'client_cancelled'): void { + const realtimeSession = this.sessionManager.getSession(); + if (!realtimeSession.currentResponse || this.isCancelled) { + return; // Nothing to cancel or already cancelled + } + logger.info({ + sessionId: this.sessionKey, + reason, + responseId: realtimeSession.currentResponse.id, + ttsInteractionId: this.currentTTSInteractionId || 'none', + }, 'Response Cancellation'); + + this.isCancelled = true; + const response = realtimeSession.currentResponse; + + // Abort active content and TTS streams + abortStream(realtimeSession.currentContentStream, 'content stream', this.sessionKey, 'due to response cancellation'); + realtimeSession.currentContentStream = null; + + abortStream(realtimeSession.currentTTSStream, 'TTS stream', this.sessionKey, 'due to response cancellation'); + realtimeSession.currentTTSStream = null; + + response.status = 'cancelled'; + response.status_details = { type: 'cancelled', reason }; + + this.send(RealtimeEventFactory.responseDone(response)); + realtimeSession.currentResponse = null; + } + + /** + * Execute audio graph with streaming multimodal input (audio and/or text) + */ + async executeAudioGraph({ + sessionId, + workspaceId, + apiKey, + input, + graphWrapper, + multimodalStreamManager, + }: { + sessionId: string; + workspaceId: string; + apiKey: string; + input: { sessionId: string; state: any }; + graphWrapper: InworldGraphWrapper; + multimodalStreamManager: MultimodalStreamManager; + }): Promise { + // Create a multimodal stream generator that yields MultimodalContent + async function* multimodalStreamGenerator() { + for await (const content of multimodalStreamManager.createStream()) { + yield content; + } + } + + // Create the tagged stream with metadata + const taggedStream = Object.assign(multimodalStreamGenerator(), { + type: 'MultimodalContent', + }); + + const { outputStream } = await graphWrapper.graph.start(taggedStream, { + executionId: input.sessionId, + dataStoreContent: { + sessionId: input.sessionId, + state: input.state, + }, + userContext: { + attributes: { + workspaceId: workspaceId, + }, + targetingKey: uuidv4(), + }, + userCredentials: { + inworldApiKey: apiKey, + }, + }); + + const connection = this.inworldApp.connections[sessionId]; + if (!connection) { + logger.debug({ sessionId }, 'Connection no longer exists, aborting audio graph execution'); + outputStream.abort(); + return; + } + + // Store the execution stream so it can be aborted if needed + connection.currentAudioExecutionStream = outputStream; + + // Handle multiple interactions from the stream + try { + let currentGraphInteractionId: string | undefined = undefined; + let resultCount = 0; + + for await (const result of outputStream) { + resultCount++; + logger.debug({ sessionId, resultCount }, `Processing audio interaction #${resultCount}`); + + // Check if result contains an error + if (result && result.isGraphError && result.isGraphError()) { + const errorData = result.data; + logger.error({ + sessionId, + err: errorData, + }, 'Graph error'); + if (!errorData.message.includes('recognition produced no text')) { + this.send( + RealtimeEventFactory.error({ + type: 'graph_error', + message: errorData.message, + }), + ); + if (errorData.code === 4) { + const connection = this.inworldApp.connections[sessionId]; + if (connection?.ws) { + // Close the websocket connection + // Using code 1011 (Internal Error) as it's a server-side error + connection.ws.close(1011, 'JS Call Timeout. We will end the call if the audio stream is not active in 60 seconds.'); + } + } + } + continue; + } + + // Process the result - this will handle transcription, LLM response, and TTS + await this.processAudioGraphOutput( + result, + connection, + sessionId, + currentGraphInteractionId, + ); + } + + logger.info({ sessionId, resultCount }, `Audio stream processing complete: ${resultCount} interactions`); + } catch (error) { + logger.error({ err: error, sessionId }, 'Error processing audio stream'); + throw error; + } finally { + // Clear the stream reference when done (if connection still exists) + const conn = this.inworldApp.connections[sessionId]; + if (conn && conn.currentAudioExecutionStream === outputStream) { + conn.currentAudioExecutionStream = undefined; + } + // Clean up stream manager + connection.multimodalStreamManager = undefined; + connection.currentAudioGraphExecution = undefined; + } + } + + /** + * Process a single result from the audio graph + */ + private async processAudioGraphOutput( + result: any, + connection: Connection, + sessionId: string, + currentGraphInteractionId: string | undefined, + ): Promise { + try { + logger.debug({ + sessionId, + handlers: Object.keys(result), + }, 'Audio Graph Result - Processing result'); + + const realtimeSession = this.sessionManager.getSession(); + + await result.processResponse({ + Content: async (content: GraphTypes.Content) => { + logger.debug({ + sessionId, + hasContent: !!content.content, + contentLength: content.content?.length, + hasToolCalls: !!content.toolCalls, + toolCallsCount: content.toolCalls?.length, + }, 'Audio Graph - Content received'); + + if (content.toolCalls && content.toolCalls.length > 0) { + logger.info({ sessionId, toolCalls: content.toolCalls }, `Audio Graph - ${content.toolCalls.length} tool calls received`); + } + }, + ContentStream: async (stream: GraphTypes.ContentStream) => { + // Ensure we have a response object + if (!realtimeSession.currentResponse) { + const responseId = uuidv4(); + const response: RT.Response = { + id: responseId, + object: 'realtime.response', + status: 'in_progress', + status_details: null, + output: [], + conversation_id: 'conv_' + realtimeSession.id, + output_modalities: realtimeSession.session.output_modalities, + max_output_tokens: realtimeSession.session.max_output_tokens, + audio: { + output: realtimeSession.session.audio.output, + }, + usage: null, + metadata: null, + }; + realtimeSession.currentResponse = response; + this.send(RealtimeEventFactory.responseCreated(response)); + } + + await this.handleContentStream(stream, realtimeSession.currentResponse!, connection); + }, + TTSOutputStream: async (ttsStream: GraphTypes.TTSOutputStream) => { + // Create a response if we don't have one + if (!realtimeSession.currentResponse) { + const responseId = uuidv4(); + const response: RT.Response = { + id: responseId, + object: 'realtime.response', + status: 'in_progress', + status_details: null, + output: [], + conversation_id: 'conv_' + realtimeSession.id, + output_modalities: realtimeSession.session.output_modalities, + max_output_tokens: realtimeSession.session.max_output_tokens, + audio: { + output: realtimeSession.session.audio.output, + }, + usage: null, + metadata: null, + }; + realtimeSession.currentResponse = response; + this.send(RealtimeEventFactory.responseCreated(response)); + } + + await this.handleTTSOutputStream(ttsStream, realtimeSession.currentResponse!, connection, 'Audio Input'); + }, + TextStream: async (stream: GraphTypes.TextStream) => { + // Create a response if we don't have one + if (!realtimeSession.currentResponse) { + const responseId = uuidv4(); + const response: RT.Response = { + id: responseId, + object: 'realtime.response', + status: 'in_progress', + status_details: null, + output: [], + conversation_id: 'conv_' + realtimeSession.id, + output_modalities: realtimeSession.session.output_modalities, + max_output_tokens: realtimeSession.session.max_output_tokens, + audio: { + output: realtimeSession.session.audio.output, + }, + usage: null, + metadata: null, + }; + realtimeSession.currentResponse = response; + this.send(RealtimeEventFactory.responseCreated(response)); + } + await this.handleTextOutputStream(stream, realtimeSession.currentResponse!, connection, 'Audio Input'); + }, + Custom: async (customData: GraphTypes.Custom) => { + // Handle custom events (transcription, etc) + + // Check if it's a transcription event + if (('text' in customData || customData.type === 'TRANSCRIPT') && !('messages' in customData)) { + const transcript = (customData as any).text || ''; + const interactionId = (customData as any).interactionId; + const isTextInput = (customData as any).is_text_input === true; + + // Skip tool continuation marker - this is an internal marker, not a real transcript + if (transcript === TOOL_CALL_CONTINUATION_MARKER) { + logger.info({ sessionId, interactionId }, + 'Skipping transcription event for tool call continuation marker'); + return; + } + + if (transcript && transcript.trim().length > 0) { + const itemId = interactionId || uuidv4(); + + // Check if this is a text input (simulated) - if so, skip transcription events + // Text inputs already have their conversation item created before being pushed to the graph + // We check: + // 1. If is_text_input flag is set in customData + // 2. If a conversation item with matching text already exists (text inputs create items first) + // 3. If the most recent user message matches this transcript (text inputs are processed immediately) + const existingItem = realtimeSession.conversationItems.find( + item => item.id === itemId || + (item.role === 'user' && + item.content?.[0]?.type === 'input_text' && + item.content[0].text === transcript) + ); + + // Also check the most recent user message (text inputs are typically the last item) + const lastUserItem = realtimeSession.conversationItems.length > 0 + ? realtimeSession.conversationItems[realtimeSession.conversationItems.length - 1] + : null; + const isRecentTextInput = lastUserItem?.role === 'user' && + lastUserItem?.content?.[0]?.type === 'input_text' && + lastUserItem.content[0].text === transcript; + + if (isTextInput || existingItem || isRecentTextInput) { + // This is a text input that was already processed + // Skip all transcription events since the conversation item already exists + logger.info({ sessionId, itemId, transcript: transcript.substring(0, 50) }, + `Skipping transcription events for text input (conversation item already exists)`); + return; + } + + const audioStartMs = Date.now() - this.sessionStartTime; + + // Send speech started event if we haven't already + if (!this.currentTranscriptionItemId) { + this.send( + RealtimeEventFactory.inputAudioBufferSpeechStarted(audioStartMs, itemId), + ); + } + + this.currentTranscriptionItemId = itemId; + + // Send speech stopped event + const audioEndMs = Date.now() - this.sessionStartTime; + this.send( + RealtimeEventFactory.inputAudioBufferSpeechStopped(audioEndMs, itemId), + ); + + const previousItemId = + realtimeSession.conversationItems.length > 0 + ? realtimeSession.conversationItems[ + realtimeSession.conversationItems.length - 1 + ].id + : null; + + // Send committed event + this.send( + RealtimeEventFactory.inputAudioBufferCommitted(itemId, previousItemId), + ); + + // Create conversation item for user transcription + const item: RT.ConversationItem = { + id: itemId, + type: 'message', + object: 'realtime.item', + status: 'completed', + role: 'user', + content: [ + { + type: 'input_text', + text: transcript, + }, + ], + }; + + realtimeSession.conversationItems.push(item); + this.send(RealtimeEventFactory.conversationItemAdded(item, previousItemId)); + + // Send transcription completed event + logger.info({ sessionId, transcript, itemId }, `Transcription completed: "${transcript.substring(0, 50)}..."`); + this.send( + RealtimeEventFactory.inputAudioTranscriptionCompleted( + itemId, + 0, + transcript, + ), + ); + + this.send(RealtimeEventFactory.conversationItemDone(item, previousItemId)); + + // Clear the transcription item ID + this.currentTranscriptionItemId = null; + this.partialTranscripts.delete(itemId); + } + } + }, + error: async (error: GraphTypes.GraphError) => { + logger.error({ sessionId, err: error }, 'Graph error'); + // Don't send errors for empty speech recognition + if (!error.message.includes('recognition produced no text')) { + this.send( + RealtimeEventFactory.error({ + type: 'server_error', + message: error.message, + }), + ); + } + }, + }); + } catch (error) { + logger.error({ sessionId, err: error }, 'Error processing audio graph result'); + this.send( + RealtimeEventFactory.error({ + type: 'server_error', + message: error instanceof Error ? error.message : 'Unknown error processing audio graph result', + }), + ); + } + } + + /** + * Process ContentStream for tool calls (shared between audio and text graphs) + */ + private async handleContentStream( + stream: GraphTypes.ContentStream, + response: RT.Response, + connection: Connection, + ): Promise { + const realtimeSession = this.sessionManager.getSession(); + const toolCallState = new Map(); + + // Store the stream object so it can be aborted + realtimeSession.currentContentStream = stream; + + try { + for await (const chunk of stream) { + if (chunk.toolCalls && chunk.toolCalls.length > 0) { + await this.handleToolCallChunk(chunk.toolCalls, response, toolCallState); + } + } + } finally { + // Clear the stream reference when done + realtimeSession.currentContentStream = null; + } + + // Send completion events for all tool calls + for (const [callId, state] of toolCallState.entries()) { + const outputIndex = response.output.indexOf(state.item); + + // Update the item with final arguments + state.item.arguments = state.args; + state.item.status = 'completed'; + + // Send done event + this.send( + RealtimeEventFactory.responseFunctionCallArgumentsDone( + response.id, + state.item.id!, + outputIndex, + callId, + state.args, + ), + ); + + this.send( + RealtimeEventFactory.conversationItemDone(state.item), + ); + + this.send( + RealtimeEventFactory.responseOutputItemDone( + response.id, + outputIndex, + state.item, + ), + ); + + // Add to conversation items + realtimeSession.conversationItems.push(state.item); + + // Add a minimal assistant message + connection.state.messages.push({ + role: 'assistant', + content: '[Function call executed]', + id: connection.state.interactionId, + }); + } + } + + /** + * Process TextOutputStream for text-only messages + */ + private async handleTextOutputStream( + stream: GraphTypes.TextStream, + response: RT.Response, + connection: Connection, + logPrefix: string, + ): Promise { + const realtimeSession = this.sessionManager.getSession(); + const isTextOnly = + response.output_modalities?.includes('text') && + !response.output_modalities?.includes('audio'); + + let item: RT.ConversationItem | undefined; + let itemId: string | undefined; + let outputIndex: number | undefined; + let contentPart: RT.ContentPart | undefined; + const contentIndex = 0; + + if (!isTextOnly) { + logger.info('[Text Output Stream] should not be called when modality is not text only!'); + return item; + } + + // Process text stream chunks + for await (const chunk of stream) { + // Extract text from chunk (handle both string and object with text property) + const text = typeof chunk === 'string' ? chunk : (chunk?.text || chunk?.toString() || ''); + + if (!text || text === '') { + continue; + } + + // Lazy create item on first chunk + if (!item) { + itemId = uuidv4(); + outputIndex = response.output.length; + + item = { + id: itemId, + type: 'message', + object: 'realtime.item', + status: 'in_progress', + role: 'assistant', + content: [], + }; + + const previousItemId = + realtimeSession.conversationItems.length > 0 + ? realtimeSession.conversationItems[ + realtimeSession.conversationItems.length - 1 + ].id + : null; + + response.output.push(item); + this.send( + RealtimeEventFactory.responseOutputItemAdded(response.id, outputIndex, item), + ); + this.send( + RealtimeEventFactory.conversationItemAdded(item, previousItemId), + ); + + contentPart = { type: 'text', text: '' }; + + item.content = [contentPart]; + this.send( + RealtimeEventFactory.responseContentPartAdded( + response.id, + itemId, + outputIndex, + contentIndex, + contentPart, + ), + ); + + // Track that we're streaming text for this interaction + const textInteractionId = connection.state.interactionId; + this.isCancelled = false; + + logger.info({ sessionId: this.sessionKey, textInteractionId, logPrefix }, `Text stream starting (${logPrefix})`); + } + + if (this.isCancelled) { + logger.info(`[Text Output] Stopping text stream - response was cancelled`); + break; + } + + // Send text delta event + this.send( + RealtimeEventFactory.responseTextDelta( + response.id, + itemId!, + outputIndex!, + contentIndex, + text, + ), + ); + // logger.info(`[TEXT DELTA] - ${text}`); + contentPart!.text = (contentPart!.text || '') + text; + } + + // Send completion events or mark as incomplete + if (item) { + if (this.isCancelled) { + item.status = 'incomplete'; + } else { + const previousItemId = + realtimeSession.conversationItems.length > 0 + ? realtimeSession.conversationItems[ + realtimeSession.conversationItems.length - 1 + ].id + : null; + + // Send text completion event + this.send(RealtimeEventFactory.responseTextDone( + response.id, itemId!, outputIndex!, contentIndex, contentPart!.text || '' + )); + + // Send common completion events + this.send(RealtimeEventFactory.responseContentPartDone( + response.id, itemId!, outputIndex!, contentIndex, contentPart! + )); + this.send(RealtimeEventFactory.conversationItemDone(item, previousItemId)); + + item.status = 'completed'; + this.send(RealtimeEventFactory.responseOutputItemDone(response.id, outputIndex!, item)); + } + + // Add to conversation items + realtimeSession.conversationItems.push(item); + } + + return item; + } + + /** + * Process TTSOutputStream for audio messages + * Note: Text-only modality is handled by handleTextOutputStream + */ + private async handleTTSOutputStream( + ttsStream: GraphTypes.TTSOutputStream, + response: RT.Response, + connection: Connection, + logPrefix: string, + ): Promise { + const realtimeSession = this.sessionManager.getSession(); + const isTextOnly = + response.output_modalities?.includes('text') && + !response.output_modalities?.includes('audio'); + + // Warn if called with text-only modality (should use handleTextOutputStream instead) + if (isTextOnly) { + logger.warn('[TTS Output Stream] Called with text-only modality - should use handleTextOutputStream instead'); + } + + let item: RT.ConversationItem | undefined; + let itemId: string | undefined; + let outputIndex: number | undefined; + let contentPart: RT.ContentPart | undefined; + const contentIndex = 0; + + // Store the TTSOutputStream object so it can be aborted using its abort() method + realtimeSession.currentTTSStream = ttsStream; + + try { + // Process TTS stream chunks + for await (const chunk of ttsStream) { + // Lazy create item on first chunk + if (!item) { + itemId = uuidv4(); + outputIndex = response.output.length; + + item = { + id: itemId, + type: 'message', + object: 'realtime.item', + status: 'in_progress', + role: 'assistant', + content: [], + }; + + const previousItemId = + realtimeSession.conversationItems.length > 0 + ? realtimeSession.conversationItems[ + realtimeSession.conversationItems.length - 1 + ].id + : null; + + response.output.push(item); + this.send( + RealtimeEventFactory.responseOutputItemAdded(response.id, outputIndex, item), + ); + this.send( + RealtimeEventFactory.conversationItemAdded(item, previousItemId), + ); + + // Always create audio content part (text-only is handled elsewhere) + contentPart = { type: 'audio', transcript: '' }; + + item.content = [contentPart]; + this.send( + RealtimeEventFactory.responseContentPartAdded( + response.id, + itemId, + outputIndex, + contentIndex, + contentPart, + ), + ); + + // Track that we're streaming TTS for this interaction + const ttsInteractionId = connection.state.interactionId; + this.currentTTSInteractionId = ttsInteractionId; + this.isCancelled = false; + + logger.info(`[TTS] Starting stream for ${ttsInteractionId} (${logPrefix})`); + } + + if (this.isCancelled) { + logger.info(`[TTS] Stopping TTS stream - response was cancelled`); + break; + } + + // Process audio chunk + if (chunk.text != null && chunk.text !== '') { + this.send( + RealtimeEventFactory.responseAudioTranscriptDelta( + response.id, + itemId!, + outputIndex!, + contentIndex, + chunk.text, + ), + ); + } + + // Convert audio data to PCM16 base64 + const audioBase64 = convertToPCM16Base64( + chunk.audio?.data, + chunk.audio?.sampleRate, + `TTS Audio (${logPrefix})` + ); + + if (!audioBase64) { + continue; + } + + // Send audio delta + this.send( + RealtimeEventFactory.responseAudioDelta( + response.id, + itemId!, + outputIndex!, + contentIndex, + audioBase64, + ), + ); + + // Update transcript + if (chunk.text) { + contentPart!.transcript = (contentPart!.transcript || '') + chunk.text; + } + } + + // Send completion events or mark as incomplete + if (item) { + if (this.isCancelled) { + item.status = 'incomplete'; + } else { + const previousItemId = + realtimeSession.conversationItems.length > 0 + ? realtimeSession.conversationItems[ + realtimeSession.conversationItems.length - 1 + ].id + : null; + + // Send audio completion events + this.send(RealtimeEventFactory.responseAudioTranscriptDone( + response.id, itemId!, outputIndex!, contentIndex, contentPart!.transcript || '' + )); + this.send(RealtimeEventFactory.responseAudioDone( + response.id, itemId!, outputIndex!, contentIndex + )); + + // Send common completion events + this.send(RealtimeEventFactory.responseContentPartDone( + response.id, itemId!, outputIndex!, contentIndex, contentPart! + )); + this.send(RealtimeEventFactory.conversationItemDone(item, previousItemId)); + + item.status = 'completed'; + this.send(RealtimeEventFactory.responseOutputItemDone(response.id, outputIndex!, item)); + + // Clear TTS tracking on successful completion + this.currentTTSInteractionId = null; + } + + // Add to conversation items + realtimeSession.conversationItems.push(item); + } + + return item; + } catch (error) { + // Check if this is a cancellation error (expected when response is cancelled) + const isCancellationError = + this.isCancelled && + error instanceof Error && + error.message.includes('Operation cancelled'); + + if (isCancellationError) { + logger.debug({ sessionId: this.sessionKey }, 'TTS stream cancelled (expected during response cancellation)'); + + // Mark item as incomplete if we had started creating one + if (item) { + item.status = 'incomplete'; + realtimeSession.conversationItems.push(item); + } + return item; + } + + // Re-throw unexpected errors + throw error; + } finally { + // Clear the stream reference when done + realtimeSession.currentTTSStream = null; + } + } + + /** + * Handle tool call chunks from LLM stream + */ + private async handleToolCallChunk( + toolCalls: ToolCallInterface[], + response: RT.Response, + toolCallState: Map, + ): Promise { + for (const toolCall of toolCalls) { + const callId = toolCall.id; + + if (!toolCallState.has(callId)) { + // New tool call + const itemId = uuidv4(); + const outputIndex = response.output.length; + const item: RT.ConversationItem = { + id: itemId, + type: 'function_call', + object: 'realtime.item', + status: 'in_progress', + call_id: callId, + name: toolCall.name, + arguments: '', + }; + + response.output.push(item); + this.send( + RealtimeEventFactory.responseOutputItemAdded( + response.id, + outputIndex, + item, + ), + ); + this.send(RealtimeEventFactory.conversationItemAdded(item)); + + toolCallState.set(callId, { item, args: toolCall.args || '' }); + + if (toolCall.args) { + this.send( + RealtimeEventFactory.responseFunctionCallArgumentsDelta( + response.id, + item.id!, + outputIndex, + callId, + toolCall.args, + ), + ); + } + } else { + // Existing tool call + const state = toolCallState.get(callId)!; + + if (toolCall.args) { + state.args += toolCall.args; + + const outputIndex = response.output.indexOf(state.item); + + this.send( + RealtimeEventFactory.responseFunctionCallArgumentsDelta( + response.id, + state.item.id!, + outputIndex, + callId, + toolCall.args, + ), + ); + } + } + } + } + + /** + * Handle partial transcription updates from AssemblyAI. + */ + handlePartialTranscriptDelta( + interactionId: string, + text: string, + ): void { + if (!interactionId || typeof text !== 'string') { + return; + } + + const previous = this.partialTranscripts.get(interactionId) ?? ''; + + if (text === previous) { + return; + } + + // Track that this interaction is the active transcription + this.currentTranscriptionItemId = interactionId; + + // Compute the longest common prefix to determine the delta + const commonPrefixLength = this.getCommonPrefixLength(previous, text); + const deletions = previous.length - commonPrefixLength; + let delta = ''; + + if (deletions > 0) { + delta += '\b'.repeat(deletions); + } + + delta += text.slice(commonPrefixLength); + + if (!delta) { + return; + } + + this.partialTranscripts.set(interactionId, text); + + this.send( + RealtimeEventFactory.inputAudioTranscriptionDelta( + interactionId, + 0, + delta, + ), + ); + } + + private getCommonPrefixLength(a: string, b: string): number { + const maxLen = Math.min(a.length, b.length); + let idx = 0; + while (idx < maxLen && a[idx] === b[idx]) { + idx++; + } + return idx; + } + + /** + * Create a response from the model + */ + async createResponse(inputItemId?: string): Promise { + // Reset cancellation flag for new response + this.isCancelled = false; + + const realtimeSession = this.sessionManager.getSession(); + const responseId = uuidv4(); + const response: RT.Response = { + id: responseId, + object: 'realtime.response', + status: 'in_progress', + status_details: null, + output: [], + conversation_id: 'conv_' + realtimeSession.id, + output_modalities: realtimeSession.session.output_modalities, + max_output_tokens: realtimeSession.session.max_output_tokens, + audio: { + output: realtimeSession.session.audio.output, + }, + usage: null, + metadata: null, + }; + + realtimeSession.currentResponse = response; + this.send(RealtimeEventFactory.responseCreated(response)); + + try { + const connection = this.inworldApp.connections[this.sessionKey]; + + // Check if this is a tool call continuation (response.create after function_call_output) + const lastItem = realtimeSession.conversationItems.length > 0 + ? realtimeSession.conversationItems[realtimeSession.conversationItems.length - 1] + : null; + + const isToolCallContinuation = lastItem?.type === 'function_call_output'; + + if (isToolCallContinuation) { + // This is a continuation after a tool call - the tool result is already in connection.state.messages + // We need to trigger the LLM with the current conversation state + logger.info({ sessionId: this.sessionKey, callId: lastItem.call_id }, 'Tool call continuation - triggering LLM with tool result'); + + if (!connection) { + throw new Error('No connection found for session'); + } + + // Trigger the LLM via the audio graph with a tool continuation marker + // This will be recognized by the graph and processed without adding a new user message + await this.handleToolCallContinuation(connection, response); + return; + } + + // Find the most recent user message to process + let userMessage: RT.ConversationItem | undefined; + + if (inputItemId) { + userMessage = realtimeSession.conversationItems.find( + (i) => i.id === inputItemId, + ); + } else { + // Find the last user message + for (let i = realtimeSession.conversationItems.length - 1; i >= 0; i--) { + const item = realtimeSession.conversationItems[i]; + if (item.role === 'user') { + userMessage = item; + break; + } + } + } + + if (!userMessage || !userMessage.content || userMessage.content.length === 0) { + // Send error event instead of throwing + response.status = 'failed'; + response.status_details = { + type: 'failed', + error: { + type: 'invalid_request_error', + code: 'no_user_message', + }, + }; + this.send(RealtimeEventFactory.responseDone(response)); + this.send( + RealtimeEventFactory.error({ + type: 'invalid_request_error', + code: 'no_user_message', + message: 'No user message found to generate response. Please create a conversation item before requesting a response.', + }), + ); + realtimeSession.currentResponse = null; + return; + } + + const content = userMessage.content[0]; + let input: any; + + if (content.type === 'input_text' && content.text) { + // Text inputs are processed through the continuous multimodal stream + // The graph runs continuously and automatically sends response events for each turn + // DO NOT wait for graph completion - it runs until the stream is ended + const hasActiveAudioGraph = connection?.currentAudioGraphExecution !== undefined; + + if (hasActiveAudioGraph) { + // Graph is running and processing text turns continuously + // Response events are sent automatically by processAudioGraphOutput + // Just acknowledge and return - don't block + logger.info({ sessionId: this.sessionKey, responseId }, + 'Text input being processed by continuous graph - response events sent automatically'); + } else { + // No active graph - text should have been pushed via conversation.item.create first + logger.warn({ sessionId: this.sessionKey }, + 'No active graph for text input - ensure conversation.item.create was called first'); + } + + // Cancel this response.create since the graph handles responses automatically + this.isCancelled = true; + realtimeSession.currentResponse = null; + return; + } else { + // For audio input, it should have been handled by the audio graph already? + // Or if we manually trigger response creation on an existing audio item? + // If content.type is input_audio, we probably can't easily re-run it through the text graph + // unless we have the transcript. + // But the logic in original file threw error. + throw new Error(`Unsupported content type: ${content.type}`); + } + + // Don't mark as completed if already cancelled + if (!this.isCancelled) { + // Mark response as completed + response.status = 'completed'; + response.status_details = { type: 'completed' }; + } + } catch (error) { + logger.error({ err: error, sessionId: this.sessionKey }, 'Error creating response'); + response.status = 'failed'; + response.status_details = { + type: 'failed', + error: { + type: 'server_error', + code: 'internal_error', + }, + }; + // Send error to websocket + this.send( + RealtimeEventFactory.error({ + type: 'server_error', + message: error instanceof Error ? error.message : 'Error creating response', + }), + ); + } + + // Only send response.done if not already cancelled + if (!this.isCancelled) { + this.send(RealtimeEventFactory.responseDone(response)); + realtimeSession.currentResponse = null; + } + } + + /** + * Handle tool call continuation by triggering the LLM with the current conversation state. + * The tool result is already in connection.state.messages (added by createConversationItem). + * This method triggers the graph without adding a new user message. + */ + private async handleToolCallContinuation( + connection: Connection, + response: RT.Response, + ): Promise { + const realtimeSession = this.sessionManager.getSession(); + + try { + // Create a multimodal stream manager for this continuation + const multimodalStreamManager = new MultimodalStreamManager(); + connection.multimodalStreamManager = multimodalStreamManager; + + const session = realtimeSession.session; + + // Start the audio graph execution with the stream + const audioStreamInput = { + sessionId: this.sessionKey, + state: connection.state, + voiceId: connection.state.voiceId || session.audio.output.voice, + }; + + // Use the Assembly.AI audio graph + const graphWrapper = this.inworldApp.graphWithAudioInput; + + // Push a special tool continuation marker that the graph nodes will recognize + // This signals that we're continuing after a tool call and shouldn't add a new user message + multimodalStreamManager.pushText(TOOL_CALL_CONTINUATION_MARKER); + + // Start graph execution and wait for completion + connection.currentAudioGraphExecution = this.executeAudioGraph({ + sessionId: this.sessionKey, + workspaceId: connection.workspaceId, + apiKey: connection.apiKey, + input: audioStreamInput, + graphWrapper, + multimodalStreamManager, + }); + + await connection.currentAudioGraphExecution; + + // Response should have been handled by the audio graph output processing + // Mark as completed if not already done + if (!this.isCancelled && realtimeSession.currentResponse?.id === response.id) { + response.status = 'completed'; + response.status_details = { type: 'completed' }; + this.send(RealtimeEventFactory.responseDone(response)); + realtimeSession.currentResponse = null; + } + } catch (error) { + logger.error({ err: error, sessionId: this.sessionKey }, 'Error in tool call continuation'); + response.status = 'failed'; + response.status_details = { + type: 'failed', + error: { + type: 'server_error', + code: 'tool_continuation_error', + }, + }; + this.send(RealtimeEventFactory.responseDone(response)); + this.send( + RealtimeEventFactory.error({ + type: 'server_error', + message: error instanceof Error ? error.message : 'Error in tool call continuation', + }), + ); + realtimeSession.currentResponse = null; + } finally { + // Clean up + connection.multimodalStreamManager = undefined; + connection.currentAudioGraphExecution = undefined; + } + } +} + diff --git a/realtime-service/src/components/realtime/realtime_event_factory.ts b/realtime-service/src/components/realtime/realtime_event_factory.ts new file mode 100644 index 0000000..1e94d4e --- /dev/null +++ b/realtime-service/src/components/realtime/realtime_event_factory.ts @@ -0,0 +1,499 @@ +import { v4 as uuidv4 } from 'uuid'; +import * as RT from '../../types/realtime'; + +/** + * Factory for creating OpenAI Realtime API server events + */ +export class RealtimeEventFactory { + /** + * Create a session.created event + */ + static sessionCreated(session: RT.Session): RT.SessionCreatedEvent { + return { + event_id: uuidv4(), + type: 'session.created', + session, + }; + } + + /** + * Create a session.updated event + */ + static sessionUpdated(session: RT.Session): RT.SessionUpdatedEvent { + return { + event_id: uuidv4(), + type: 'session.updated', + session, + }; + } + + /** + * Create a conversation.item.added event + */ + static conversationItemAdded( + item: RT.ConversationItem, + previousItemId: string | null = null, + ): RT.ConversationItemAddedEvent { + return { + event_id: uuidv4(), + type: 'conversation.item.added', + previous_item_id: previousItemId, + item, + }; + } + + /** + * Create a conversation.item.done event + */ + static conversationItemDone( + item: RT.ConversationItem, + previousItemId: string | null = null, + ): RT.ConversationItemDoneEvent { + return { + event_id: uuidv4(), + type: 'conversation.item.done', + previous_item_id: previousItemId, + item, + }; + } + + /** + * Create a conversation.item.retrieved event + */ + static conversationItemRetrieved( + item: RT.ConversationItem, + ): RT.ConversationItemRetrievedEvent { + return { + event_id: uuidv4(), + type: 'conversation.item.retrieved', + item, + }; + } + + /** + * Create a conversation.item.truncated event + */ + static conversationItemTruncated( + itemId: string, + contentIndex: number, + audioEndMs: number, + ): RT.ConversationItemTruncatedEvent { + return { + event_id: uuidv4(), + type: 'conversation.item.truncated', + item_id: itemId, + content_index: contentIndex, + audio_end_ms: audioEndMs, + }; + } + + /** + * Create a conversation.item.deleted event + */ + static conversationItemDeleted( + itemId: string, + ): RT.ConversationItemDeletedEvent { + return { + event_id: uuidv4(), + type: 'conversation.item.deleted', + item_id: itemId, + }; + } + + /** + * Create an input_audio_buffer.speech_started event + */ + static inputAudioBufferSpeechStarted( + audioStartMs: number, + itemId: string, + ): RT.InputAudioBufferSpeechStartedEvent { + return { + event_id: uuidv4(), + type: 'input_audio_buffer.speech_started', + audio_start_ms: audioStartMs, + item_id: itemId, + }; + } + + /** + * Create an input_audio_buffer.speech_stopped event + */ + static inputAudioBufferSpeechStopped( + audioEndMs: number, + itemId: string, + ): RT.InputAudioBufferSpeechStoppedEvent { + return { + event_id: uuidv4(), + type: 'input_audio_buffer.speech_stopped', + audio_end_ms: audioEndMs, + item_id: itemId, + }; + } + + /** + * Create an input_audio_buffer.committed event + */ + static inputAudioBufferCommitted( + itemId: string, + previousItemId: string | null = null, + ): RT.InputAudioBufferCommittedEvent { + return { + event_id: uuidv4(), + type: 'input_audio_buffer.committed', + previous_item_id: previousItemId, + item_id: itemId, + }; + } + + /** + * Create an input_audio_buffer.cleared event + */ + static inputAudioBufferCleared(): RT.InputAudioBufferClearedEvent { + return { + event_id: uuidv4(), + type: 'input_audio_buffer.cleared', + }; + } + + /** + * Create a conversation.item.input_audio_transcription.delta event + */ + static inputAudioTranscriptionDelta( + itemId: string, + contentIndex: number, + delta: string, + ): RT.ConversationItemInputAudioTranscriptionDeltaEvent { + return { + event_id: uuidv4(), + type: 'conversation.item.input_audio_transcription.delta', + item_id: itemId, + content_index: contentIndex, + delta, + }; + } + + /** + * Create a conversation.item.input_audio_transcription.completed event + */ + static inputAudioTranscriptionCompleted( + itemId: string, + contentIndex: number, + transcript: string, + ): RT.ConversationItemInputAudioTranscriptionCompletedEvent { + return { + event_id: uuidv4(), + type: 'conversation.item.input_audio_transcription.completed', + item_id: itemId, + content_index: contentIndex, + transcript, + }; + } + + /** + * Create a response.created event + */ + static responseCreated(response: RT.Response): RT.ResponseCreatedEvent { + return { + event_id: uuidv4(), + type: 'response.created', + response, + }; + } + + + /** + * Create a response.done event + */ + static responseDone(response: RT.Response): RT.ResponseDoneEvent { + return { + event_id: uuidv4(), + type: 'response.done', + response, + }; + } + + /** + * Create a response.output_item.added event + */ + static responseOutputItemAdded( + responseId: string, + outputIndex: number, + item: RT.ConversationItem, + ): RT.ResponseOutputItemAddedEvent { + return { + event_id: uuidv4(), + type: 'response.output_item.added', + response_id: responseId, + output_index: outputIndex, + item, + }; + } + + /** + * Create a response.output_item.done event + */ + static responseOutputItemDone( + responseId: string, + outputIndex: number, + item: RT.ConversationItem, + ): RT.ResponseOutputItemDoneEvent { + return { + event_id: uuidv4(), + type: 'response.output_item.done', + response_id: responseId, + output_index: outputIndex, + item, + }; + } + + /** + * Create a response.content_part.added event + */ + static responseContentPartAdded( + responseId: string, + itemId: string, + outputIndex: number, + contentIndex: number, + part: RT.ContentPart, + ): RT.ResponseContentPartAddedEvent { + return { + event_id: uuidv4(), + type: 'response.content_part.added', + response_id: responseId, + item_id: itemId, + output_index: outputIndex, + content_index: contentIndex, + part, + }; + } + + /** + * Create a response.content_part.done event + */ + static responseContentPartDone( + responseId: string, + itemId: string, + outputIndex: number, + contentIndex: number, + part: RT.ContentPart, + ): RT.ResponseContentPartDoneEvent { + return { + event_id: uuidv4(), + type: 'response.content_part.done', + response_id: responseId, + item_id: itemId, + output_index: outputIndex, + content_index: contentIndex, + part, + }; + } + + /** + * Create a response.output_audio.delta event + */ + static responseAudioDelta( + responseId: string, + itemId: string, + outputIndex: number, + contentIndex: number, + delta: string, + ): RT.ResponseAudioDeltaEvent { + return { + event_id: uuidv4(), + type: 'response.output_audio.delta', + response_id: responseId, + item_id: itemId, + output_index: outputIndex, + content_index: contentIndex, + delta, + }; + } + + /** + * Create a response.output_audio.done event + */ + static responseAudioDone( + responseId: string, + itemId: string, + outputIndex: number, + contentIndex: number, + ): RT.ResponseAudioDoneEvent { + return { + event_id: uuidv4(), + type: 'response.output_audio.done', + response_id: responseId, + item_id: itemId, + output_index: outputIndex, + content_index: contentIndex, + }; + } + + /** + * Create a response.output_audio_transcript.delta event + */ + static responseAudioTranscriptDelta( + responseId: string, + itemId: string, + outputIndex: number, + contentIndex: number, + delta: string, + ): RT.ResponseAudioTranscriptDeltaEvent { + return { + event_id: uuidv4(), + type: 'response.output_audio_transcript.delta', + response_id: responseId, + item_id: itemId, + output_index: outputIndex, + content_index: contentIndex, + delta, + }; + } + + /** + * Create a response.output_audio_transcript.done event + */ + static responseAudioTranscriptDone( + responseId: string, + itemId: string, + outputIndex: number, + contentIndex: number, + transcript: string, + ): RT.ResponseAudioTranscriptDoneEvent { + return { + event_id: uuidv4(), + type: 'response.output_audio_transcript.done', + response_id: responseId, + item_id: itemId, + output_index: outputIndex, + content_index: contentIndex, + transcript, + }; + } + + /** + * Create a response.function_call_arguments.delta event + */ + static responseFunctionCallArgumentsDelta( + responseId: string, + itemId: string, + outputIndex: number, + callId: string, + delta: string, + ): RT.ResponseFunctionCallArgumentsDeltaEvent { + return { + event_id: uuidv4(), + type: 'response.function_call_arguments.delta', + response_id: responseId, + item_id: itemId, + output_index: outputIndex, + call_id: callId, + delta, + }; + } + + /** + * Create a response.function_call_arguments.done event + */ + static responseFunctionCallArgumentsDone( + responseId: string, + itemId: string, + outputIndex: number, + callId: string, + args: string, + ): RT.ResponseFunctionCallArgumentsDoneEvent { + return { + event_id: uuidv4(), + type: 'response.function_call_arguments.done', + response_id: responseId, + item_id: itemId, + output_index: outputIndex, + call_id: callId, + arguments: args, + }; + } + + /** + * Create a response.output_text.delta event + */ + static responseTextDelta( + responseId: string, + itemId: string, + outputIndex: number, + contentIndex: number, + delta: string, + ): RT.ResponseTextDeltaEvent { + return { + event_id: uuidv4(), + type: 'response.output_text.delta', + response_id: responseId, + item_id: itemId, + output_index: outputIndex, + content_index: contentIndex, + delta, + }; + } + + /** + * Create a response.output_text.done event + */ + static responseTextDone( + responseId: string, + itemId: string, + outputIndex: number, + contentIndex: number, + text: string, + ): RT.ResponseTextDoneEvent { + return { + event_id: uuidv4(), + type: 'response.output_text.done', + response_id: responseId, + item_id: itemId, + output_index: outputIndex, + content_index: contentIndex, + text, + }; + } + + /** + * Create an error event + */ + static error( + error: { + type: string; + code?: string; + message: string; + param?: string; + event_id?: string; + }, + ): RT.ErrorEvent { + return { + event_id: uuidv4(), + type: 'error', + error: { + type: error.type, + code: error.code || null, + message: error.message, + param: error.param || null, + event_id: error.event_id || null, + }, + }; + } + + /** + * Create a rate_limits.updated event + */ + static rateLimitsUpdated( + rateLimits: Array<{ + name: 'requests' | 'tokens'; + limit: number; + remaining: number; + reset_seconds: number; + }>, + ): RT.RateLimitsUpdatedEvent { + return { + event_id: uuidv4(), + type: 'rate_limits.updated', + rate_limits: rateLimits, + }; + } +} diff --git a/realtime-service/src/components/realtime/realtime_message_handler.ts b/realtime-service/src/components/realtime/realtime_message_handler.ts new file mode 100644 index 0000000..7b35140 --- /dev/null +++ b/realtime-service/src/components/realtime/realtime_message_handler.ts @@ -0,0 +1,168 @@ +import { RawData } from 'ws'; +import logger from '../../logger'; +import { formatContext, formatSession, formatError } from '../../log-helpers'; +import * as RT from '../../types/realtime'; +import { InworldApp } from '../app'; +import { RealtimeEventFactory } from './realtime_event_factory'; +import { RealtimeSessionManager } from './realtime_session_manager'; +import { RealtimeAudioHandler } from '../audio/realtime_audio_handler'; +import { RealtimeGraphExecutor } from '../graphs/realtime_graph_executor'; + +export class RealtimeMessageHandler { + private sessionManager: RealtimeSessionManager; + private audioHandler: RealtimeAudioHandler; + private graphExecutor: RealtimeGraphExecutor; + private processingQueue: (() => Promise)[] = []; + private isProcessing = false; + private sessionStartTime: number = Date.now(); + + constructor( + private inworldApp: InworldApp, + private sessionKey: string, + private send: (data: RT.ServerEvent) => void, + ) { + this.sessionManager = new RealtimeSessionManager(inworldApp, sessionKey, send, this.sessionStartTime); + this.graphExecutor = new RealtimeGraphExecutor(inworldApp, sessionKey, send, this.sessionManager, this.sessionStartTime); + this.audioHandler = new RealtimeAudioHandler(inworldApp, sessionKey, send, this.graphExecutor, this.sessionManager); + } + + async initialize(): Promise { + const connection = this.inworldApp.connections[this.sessionKey]; + if (connection) { + connection.state.voiceId = this.sessionManager.getSession().session.audio.output.voice; + + connection.onSpeechDetected = (interactionId: string) => { + logger.info( + { interactionId, sessionId: this.sessionKey }, + `Speech detected ${formatContext(this.sessionKey, undefined, interactionId)}` + ); + + this.send(RealtimeEventFactory.inputAudioBufferSpeechStarted( + Date.now() - this.sessionStartTime, // audio_start_ms + interactionId // item_id + )); + }; + + // Partial transcript callback + connection.onPartialTranscript = (text: string, interactionId: string) => { + this.graphExecutor.handlePartialTranscriptDelta(interactionId, text); + }; + } + + this.send(RealtimeEventFactory.sessionCreated(this.sessionManager.getSession().session)); + } + + /** + * Handle incoming WebSocket messages + */ + async handleMessage(data: RawData): Promise { + try { + const event = JSON.parse(data.toString()) as RT.ClientEvent; + + // Handle these events immediately without queuing: + + // 1. Cancellation - needs to stop ongoing response generation immediately + if (event.type === 'response.cancel') { + logger.info({ + sessionId: this.sessionKey, + responseId: event.response_id, + }, `Cancelling response with id: ${event.response_id}`); + this.graphExecutor.cancelCurrentResponse('client_cancelled'); + return; + } + + // 2. Audio append - needs to flow continuously without blocking + if (event.type === 'input_audio_buffer.append') { + await this.audioHandler.handleInputAudioBufferAppend(event); + return; + } + + // Add all other events to the queue to ensure sequential processing + this.addToQueue(async () => { + try { + switch (event.type) { + case 'session.update': + await this.sessionManager.updateSession(event); + break; + + case 'input_audio_buffer.commit': + await this.audioHandler.handleInputAudioBufferCommit(event); + break; + + case 'input_audio_buffer.clear': + await this.audioHandler.handleInputAudioBufferClear(event); + break; + + case 'conversation.item.create': + await this.sessionManager.createConversationItem(event, this.audioHandler); + break; + + case 'conversation.item.truncate': + await this.sessionManager.truncateConversationItem(event); + break; + + case 'conversation.item.delete': + await this.sessionManager.deleteConversationItem(event); + break; + + case 'conversation.item.retrieve': + await this.sessionManager.retrieveConversationItem(event); + break; + + case 'response.create': + await this.graphExecutor.createResponse(); + break; + + default: + logger.warn({ eventType: (event as any).type, sessionId: this.sessionKey }, `Unknown event type: ${(event as any).type}`); + } + } catch (error) { + logger.error({ err: error, sessionId: this.sessionKey }, 'Error handling queued message'); + this.send( + RealtimeEventFactory.error({ + type: 'invalid_request_error', + message: error.message, + }), + ); + } + }); + } catch (error) { + logger.error({ err: error, sessionId: this.sessionKey }, 'Error parsing message'); + this.send( + RealtimeEventFactory.error({ + type: 'invalid_request_error', + message: 'Failed to parse message', + }), + ); + } + } + + /** + * Add task to processing queue + */ + private addToQueue(task: () => Promise): void { + this.processingQueue.push(task); + this.processQueue(); + } + + /** + * Process queued tasks + */ + private async processQueue(): Promise { + if (this.isProcessing) { + return; + } + this.isProcessing = true; + while (this.processingQueue.length > 0) { + const task = this.processingQueue.shift(); + if (task) { + try { + await task(); + } catch (error) { + logger.error({ err: error, sessionId: this.sessionKey }, 'Error processing task from queue'); + } + } + } + this.isProcessing = false; + } +} diff --git a/realtime-service/src/components/realtime/realtime_session_manager.ts b/realtime-service/src/components/realtime/realtime_session_manager.ts new file mode 100644 index 0000000..aa46204 --- /dev/null +++ b/realtime-service/src/components/realtime/realtime_session_manager.ts @@ -0,0 +1,508 @@ +import { v4 as uuidv4 } from 'uuid'; +import logger from '../../logger'; +import { formatSession, formatContext } from '../../log-helpers'; +import * as RT from '../../types/realtime'; +import { InworldApp } from '../app'; +import { RealtimeEventFactory } from './realtime_event_factory'; +import { Connection } from '../../types/index'; +import { getAssemblyAISettingsForEagerness } from '../../types/settings'; +import { RealtimeAudioHandler } from '../audio/realtime_audio_handler'; + +export class RealtimeSessionManager { + realtimeSession: RT.RealtimeSession; + private sessionStartTime: number; + + constructor( + private inworldApp: InworldApp, + private sessionKey: string, + private send: (data: RT.ServerEvent) => void, + sessionStartTime: number + ) { + this.sessionStartTime = sessionStartTime; + this.realtimeSession = this.createDefaultSession(); + } + + getSession(): RT.RealtimeSession { + return this.realtimeSession; + } + + + /** + * Create a default session configuration + */ + private createDefaultSession(): RT.RealtimeSession { + const sessionId = uuidv4(); + + const connection = this.inworldApp.connections[this.sessionKey]; + const instructions = connection?.state?.agent + ? `You are: "${connection.state.agent.name}". Your persona is: "${connection.state.agent.description}". Your motivation is: "${connection.state.agent.motivation}".` + : 'You are a helpful AI assistant.'; + + // Session expires in 15 minutes (900 seconds) + const expiresAt = Math.floor(Date.now() / 1000) + 900; + + const defaultOutputModalities: ('text' | 'audio')[] = ['audio', 'text']; + + // Sync default to connection state + if (connection) { + connection.state.output_modalities = defaultOutputModalities; + } + + return { + id: sessionId, + session: { + type: 'realtime', + id: sessionId, + object: 'realtime.session', + model: this.inworldApp.llmModelName, + output_modalities: defaultOutputModalities, + instructions, + audio: { + input: { + format: { + type: 'audio/pcm', + rate: 24000, + }, + transcription: null, + noise_reduction: null, + turn_detection: { + type: 'semantic_vad', + eagerness: 'medium', + create_response: true, + interrupt_response: true, + }, + }, + output: { + format: { + type: 'audio/pcm', + rate: 24000, + }, + voice: this.inworldApp.voiceId, + speed: 1, + }, + }, + tools: [], + tool_choice: 'auto', + temperature: 0.8, + max_output_tokens: 'inf', + truncation: 'auto', + prompt: null, + tracing: null, + expires_at: expiresAt, + include: null, + }, + conversationItems: [], + inputAudioBuffer: [], + currentResponse: null, + audioStartMs: 0, + currentContentStream: null, + currentTTSStream: null, + }; + } + + /** + * Handle session.update event + */ + async updateSession(event: RT.SessionUpdateEvent): Promise { + const sessionConfig = event.session; + + // Deep merge session configuration + if (sessionConfig.output_modalities !== undefined) { + this.realtimeSession.session.output_modalities = sessionConfig.output_modalities; + + // Sync to connection state for easy access by sessionId + const connection = this.inworldApp.connections[this.sessionKey]; + if (connection) { + connection.state.output_modalities = sessionConfig.output_modalities; + logger.info({ + sessionId: this.sessionKey, + output_modalities: sessionConfig.output_modalities, + state_modalities: connection.state.output_modalities + }, `Updated connection.state.output_modalities`); + } else { + logger.warn({ sessionId: this.sessionKey }, `No connection found when updating output_modalities`); + } + } + + if (sessionConfig.instructions !== undefined) { + this.realtimeSession.session.instructions = sessionConfig.instructions; + + // Inject system instructions as the first message in the conversation state + const connection = this.inworldApp.connections[this.sessionKey]; + if (connection && sessionConfig.instructions) { + // Check if there's already a system message at the start + const hasSystemMessage = + connection.state.messages.length > 0 && + connection.state.messages[0].role === 'system'; + + if (hasSystemMessage) { + // Update existing system message + connection.state.messages[0].content = sessionConfig.instructions; + logger.info({ sessionId: this.sessionKey }, 'Updated system instructions'); + } else { + // Insert new system message at the beginning + connection.state.messages.unshift({ + id: 'system_instructions', + role: 'system', + content: sessionConfig.instructions, + }); + logger.info({ sessionId: this.sessionKey }, 'Injected system instructions'); + } + } + } + + if (sessionConfig.audio) { + // Merge audio input configuration + if (sessionConfig.audio.input) { + if (sessionConfig.audio.input.format) { + this.realtimeSession.session.audio.input.format = { + ...this.realtimeSession.session.audio.input.format, + ...sessionConfig.audio.input.format, + }; + } + if (sessionConfig.audio.input.transcription !== undefined) { + this.realtimeSession.session.audio.input.transcription = sessionConfig.audio.input.transcription; + } + if (sessionConfig.audio.input.noise_reduction !== undefined) { + this.realtimeSession.session.audio.input.noise_reduction = sessionConfig.audio.input.noise_reduction; + } + if (sessionConfig.audio.input.turn_detection !== undefined) { + if (sessionConfig.audio.input.turn_detection === null) { + this.realtimeSession.session.audio.input.turn_detection = null; + } else { + this.realtimeSession.session.audio.input.turn_detection = { + ...this.realtimeSession.session.audio.input.turn_detection, + ...sessionConfig.audio.input.turn_detection, + }; + + // Handle semantic_vad eagerness settings + if (sessionConfig.audio.input.turn_detection.type === 'semantic_vad') { + const eagerness = sessionConfig.audio.input.turn_detection.eagerness; + if (eagerness && eagerness !== 'auto') { + const connection = this.inworldApp.connections[this.sessionKey]; + if (connection) { + const normalizedEagerness = eagerness as 'low' | 'medium' | 'high'; + connection.state.eagerness = normalizedEagerness; + logger.info({ sessionId: this.sessionKey, eagerness }, `Updated eagerness to ${eagerness}`); + + // Dynamically update AssemblyAI turn detection settings on the active graph + const assemblyAINode = this.inworldApp.graphWithAudioInput?.assemblyAINode; + if (assemblyAINode) { + const assemblySettings = getAssemblyAISettingsForEagerness(normalizedEagerness); + // Extract only the numeric settings for updateTurnDetectionSettings + const { endOfTurnConfidenceThreshold, minEndOfTurnSilenceWhenConfident, maxTurnSilence } = assemblySettings; + logger.info({ sessionId: this.sessionKey, endOfTurnConfidenceThreshold, minEndOfTurnSilenceWhenConfident, maxTurnSilence }, `Applying eagerness settings: threshold=${endOfTurnConfidenceThreshold}`); + assemblyAINode.updateTurnDetectionSettings( + this.sessionKey, + { endOfTurnConfidenceThreshold, minEndOfTurnSilenceWhenConfident, maxTurnSilence } + ) + } else { + logger.warn({ sessionId: this.sessionKey }, 'AssemblyAI node not found, settings will apply on next audio input'); + this.send( + RealtimeEventFactory.error({ + type: 'invalid_request_error', + code: 'no_STT_session', + message: `Server did not find active STT connection. Turning on Mic to start audio stream input session.`, + event_id: event.event_id, + }), + ); + } + } + } + } + } + } + } + + // Merge audio output configuration + if (sessionConfig.audio.output) { + if (sessionConfig.audio.output.format) { + this.realtimeSession.session.audio.output.format = { + ...this.realtimeSession.session.audio.output.format, + ...sessionConfig.audio.output.format, + }; + } + if (sessionConfig.audio.output.voice !== undefined) { + this.realtimeSession.session.audio.output.voice = sessionConfig.audio.output.voice; + + // Store voice in connection state for TTS node + const connection = this.inworldApp.connections[this.sessionKey]; + if (connection) { + connection.state.voiceId = sessionConfig.audio.output.voice; + logger.info({ sessionId: this.sessionKey, voice: sessionConfig.audio.output.voice }, `Updated TTS voice to ${sessionConfig.audio.output.voice}`); + } + } + if (sessionConfig.audio.output.speed !== undefined) { + this.realtimeSession.session.audio.output.speed = sessionConfig.audio.output.speed; + } + } + } + + if (sessionConfig.tools !== undefined) { + this.realtimeSession.session.tools = sessionConfig.tools; + + // Update connection state with tools + const connection = this.inworldApp.connections[this.sessionKey]; + if (connection) { + connection.state.tools = sessionConfig.tools; + } + } + + if (sessionConfig.tool_choice !== undefined) { + this.realtimeSession.session.tool_choice = sessionConfig.tool_choice; + + // Update connection state with toolChoice + const connection = this.inworldApp.connections[this.sessionKey]; + if (connection) { + connection.state.toolChoice = sessionConfig.tool_choice; + } + } + + if (sessionConfig.temperature !== undefined) { + this.realtimeSession.session.temperature = sessionConfig.temperature; + } + + if (sessionConfig.max_output_tokens !== undefined) { + this.realtimeSession.session.max_output_tokens = sessionConfig.max_output_tokens; + } + + if (sessionConfig.truncation !== undefined) { + this.realtimeSession.session.truncation = sessionConfig.truncation; + } + + if (sessionConfig.prompt !== undefined) { + this.realtimeSession.session.prompt = sessionConfig.prompt; + } + + if (sessionConfig.tracing !== undefined) { + this.realtimeSession.session.tracing = sessionConfig.tracing; + } + + if (sessionConfig.include !== undefined) { + this.realtimeSession.session.include = sessionConfig.include; + } + + // Send session.updated event + this.send(RealtimeEventFactory.sessionUpdated(this.realtimeSession.session)); + } + + /** + * Handle conversation.item.create event + */ + async createConversationItem( + event: RT.ConversationItemCreateEvent, + audioHandler?: RealtimeAudioHandler, + ): Promise { + const item = { + ...event.item, + id: event.item.id || uuidv4(), + object: 'realtime.item' as const, + status: event.item.status || ('completed' as const), + }; + + this.realtimeSession.conversationItems.push(item); + this.send( + RealtimeEventFactory.conversationItemAdded(item, event.previous_item_id), + ); + this.send( + RealtimeEventFactory.conversationItemDone(item, event.previous_item_id), + ); + + // Add to conversation state so LLM is aware of this item + const connection = this.inworldApp.connections[this.sessionKey]; + if (connection) { + // Handle different item types + if (item.type === 'function_call_output') { + // Function call output received + const functionOutputItem = item as RT.FunctionCallOutputItem; + logger.info({ + sessionId: this.sessionKey, + call_id: functionOutputItem.call_id, + output: functionOutputItem.output, + }, 'Function output received'); + + // Add a system message to inform the LLM that the function was executed + // The Inworld SDK/Groq combo doesn't properly support 'tool' role with tool_call_id + // So we use a system message to provide context about the function execution + connection.state.messages.push({ + role: 'system', + content: `[SYSTEM] Function executed. Result: ${functionOutputItem.output}`, + id: item.id, + }); + + logger.info({ sessionId: this.sessionKey }, 'Added function execution result to conversation'); + } else if (item.type === 'message') { + // Handle message items + const messageItem = item as RT.MessageItem; + if (messageItem.content && messageItem.content.length > 0) { + const content = messageItem.content[0]; + let textContent = ''; + + if (content.type === 'input_text') { + textContent = content.text || ''; + } else if (content.type === 'text') { + textContent = content.text || ''; + } else if (content.type === 'input_audio') { + // Audio inputs should already be handled by STT + return; + } + + if (textContent) { + logger.info({ + sessionId: this.sessionKey, + role: messageItem.role, + contentPreview: textContent.substring(0, 100), + }, `Adding ${messageItem.role} message: ${textContent.substring(0, 50)}...`); + // We don't need to call connection.state.messages.push here. That's handled in the graph's input node + } + + // If it's a text input, push it to the audio graph + if (content.type === 'input_text' && content.text && audioHandler) { + await audioHandler.handleTextInput(content.text); + } + } + } + } + } + + /** + * Handle conversation.item.truncate event + */ + async truncateConversationItem( + event: RT.ConversationItemTruncateEvent, + ): Promise { + // Find the item + const item = this.realtimeSession.conversationItems.find( + (i) => i.id === event.item_id, + ); + + if (!item) { + this.send( + RealtimeEventFactory.error({ + type: 'invalid_request_error', + code: 'item_not_found', + message: `Item with id ${event.item_id} not found`, + event_id: event.event_id, + }), + ); + return; + } + + // Only assistant message items can be truncated + if (item.role !== 'assistant') { + this.send( + RealtimeEventFactory.error({ + type: 'invalid_request_error', + code: 'invalid_item_type', + message: 'Only assistant message items can be truncated', + event_id: event.event_id, + }), + ); + return; + } + + // Truncate the content part at the specified index + if (item.content && item.content[event.content_index]) { + const contentPart = item.content[event.content_index]; + + // If it's an audio content part, truncate the audio and remove transcript + if (contentPart.type === 'audio' && contentPart.audio) { + // Note: In a real implementation, we would need to: + // 1. Decode the base64 audio + // 2. Calculate the number of samples to keep based on audio_end_ms + // 3. Truncate the audio data + // 4. Re-encode to base64 + // For now, we'll just remove the transcript to ensure no text is in context that hasn't been heard + delete contentPart.transcript; + + // In a full implementation, we would also truncate the audio data itself + logger.info({ + sessionId: this.sessionKey, + itemId: event.item_id, + audioEndMs: event.audio_end_ms, + contentIndex: event.content_index, + }, `Truncating item ${event.item_id} at ${event.audio_end_ms}ms`); + } + } + + // Send the truncated event + this.send( + RealtimeEventFactory.conversationItemTruncated( + event.item_id, + event.content_index, + event.audio_end_ms, + ), + ); + } + + /** + * Handle conversation.item.delete event + */ + async deleteConversationItem( + event: RT.ConversationItemDeleteEvent, + ): Promise { + const index = this.realtimeSession.conversationItems.findIndex( + (i) => i.id === event.item_id, + ); + + if (index === -1) { + this.send( + RealtimeEventFactory.error({ + type: 'invalid_request_error', + code: 'item_not_found', + message: `Item with id ${event.item_id} not found`, + event_id: event.event_id, + }), + ); + return; + } + + // Remove the item from conversation history + this.realtimeSession.conversationItems.splice(index, 1); + + // Also remove from connection.state.messages so the graph doesn't see it + const connection = this.inworldApp.connections[this.sessionKey]; + if (connection?.state?.messages) { + const messageIndex = connection.state.messages.findIndex( + (m) => m.id === event.item_id, + ); + if (messageIndex !== -1) { + connection.state.messages.splice(messageIndex, 1); + } + } + + // Send the deleted event + this.send( + RealtimeEventFactory.conversationItemDeleted(event.item_id), + ); + } + + /** + * Handle conversation.item.retrieve event + */ + async retrieveConversationItem( + event: RT.ConversationItemRetrieveEvent, + ): Promise { + const item = this.realtimeSession.conversationItems.find( + (i) => i.id === event.item_id, + ); + + if (!item) { + this.send( + RealtimeEventFactory.error({ + type: 'invalid_request_error', + code: 'item_not_found', + message: `Item with id ${event.item_id} not found`, + event_id: event.event_id, + }), + ); + return; + } + + // Send the retrieved item + this.send( + RealtimeEventFactory.conversationItemRetrieved(item), + ); + } +} + diff --git a/realtime-service/src/components/runtime_app_manager.ts b/realtime-service/src/components/runtime_app_manager.ts new file mode 100644 index 0000000..341c76e --- /dev/null +++ b/realtime-service/src/components/runtime_app_manager.ts @@ -0,0 +1,140 @@ +import { stopInworldRuntime } from '@inworld/runtime'; +import logger from '../logger'; +import { InworldApp } from './app'; +import { Connection } from '../types/index'; + +export interface AppConfig { + graphId?: string; + llmModelName?: string; + llmProvider?: string; + voiceId?: string; + ttsModelId?: string; + graphVisualizationEnabled?: boolean; + assemblyAIApiKey?: string; + useMocks?: boolean; +} + +export interface InworldAppConfig extends AppConfig { + sharedConnections?: { [sessionId: string]: Connection }; +} + +/** + * Helper to merge string config values with fallback chain + */ +function getStringConfig( + configValue: string | undefined, + defaultValue: string | undefined, + envVar?: string | undefined, + fallback: string = '', +): string { + return configValue || defaultValue || envVar || fallback; +} + +/** + * Helper to merge boolean config values with fallback chain + * Uses nullish coalescing to preserve false values + */ +function getBooleanConfig( + configValue: boolean | undefined, + defaultValue: boolean | undefined, + envVar: string | undefined, + fallback: boolean, +): boolean { + if (configValue !== undefined) return configValue; + if (defaultValue !== undefined) return defaultValue; + if (envVar !== undefined && envVar.trim() !== '') { + return envVar.toLowerCase().trim() === 'true'; + } + return fallback; +} + +/** + * InworldRuntimeAppManager manages a single InworldApp instance. + * The graph supports multitenancy natively, so we only need one instance. + */ +export class InworldRuntimeAppManager { + private app: InworldApp | null = null; + private defaultConfig: Partial; + private initPromise: Promise | null = null; + + // Shared connections map for all sessions + private sharedConnections: { [sessionId: string]: Connection } = {}; + + constructor(defaultConfig?: Partial) { + this.defaultConfig = defaultConfig || {}; + } + + /** + * Get the singleton InworldApp instance, creating it if needed. + * Uses lazy initialization with proper handling of concurrent calls. + */ + async getApp(configOverrides?: Partial): Promise { + // Return existing app if already initialized + if (this.app) { + return this.app; + } + + // If initialization is in progress, wait for it + if (this.initPromise) { + return this.initPromise; + } + + // Start initialization + this.initPromise = this.createApp(configOverrides); + + try { + this.app = await this.initPromise; + return this.app; + } finally { + this.initPromise = null; + } + } + + private async createApp(configOverrides?: Partial): Promise { + logger.info('Creating InworldApp instance'); + + const finalConfig: InworldAppConfig = { + graphId: getStringConfig(configOverrides?.graphId, this.defaultConfig.graphId, undefined), + llmModelName: getStringConfig(configOverrides?.llmModelName, this.defaultConfig.llmModelName), + llmProvider: getStringConfig(configOverrides?.llmProvider, this.defaultConfig.llmProvider), + voiceId: getStringConfig(configOverrides?.voiceId, this.defaultConfig.voiceId), + ttsModelId: getStringConfig(configOverrides?.ttsModelId, this.defaultConfig.ttsModelId), + graphVisualizationEnabled: getBooleanConfig(configOverrides?.graphVisualizationEnabled, this.defaultConfig.graphVisualizationEnabled, process.env.GRAPH_VISUALIZATION_ENABLED, false), + assemblyAIApiKey: getStringConfig(configOverrides?.assemblyAIApiKey, this.defaultConfig.assemblyAIApiKey, process.env.ASSEMBLYAI_API_KEY), + useMocks: getBooleanConfig(configOverrides?.useMocks, this.defaultConfig.useMocks, process.env.USE_MOCKS, false), + sharedConnections: this.sharedConnections, + }; + + // Validate required config + if (!finalConfig.assemblyAIApiKey) { + const error = new Error('Missing AssemblyAI API key'); + logger.error({ err: error }, 'Missing AssemblyAI API key'); + throw error; + } + + const app = await InworldApp.create(finalConfig); + logger.info('InworldApp initialized successfully'); + return app; + } + + /** + * Check if the app has been initialized + */ + isInitialized(): boolean { + return this.app !== null; + } + + /** + * Shutdown the app and clean up resources + */ + async shutdown(): Promise { + if (this.app) { + logger.info('Shutting down InworldApp'); + await this.app.shutdown(); + this.app = null; + } + + // Stop the global Inworld runtime + stopInworldRuntime(); + } +} diff --git a/realtime-service/src/config.ts b/realtime-service/src/config.ts new file mode 100644 index 0000000..7bbba6f --- /dev/null +++ b/realtime-service/src/config.ts @@ -0,0 +1,15 @@ +// Audio Configuration Constants +export const TTS_SAMPLE_RATE = 24000; // Sample rate for TTS output +export const INPUT_SAMPLE_RATE = 16000; // Sample rate for STT input (Assembly.AI) + +// Server Configuration +export const WS_APP_PORT = 4000; // WebSocket server port + +// LLM Model Configuration +export const DEFAULT_LLM_MODEL_NAME = process.env.LLM_MODEL_NAME || 'meta-llama/Llama-3.1-70b-Instruct'; +export const DEFAULT_LLM_PROVIDER = process.env.LLM_PROVIDER || 'inworld'; + +// Voice and TTS Configuration +export const DEFAULT_VOICE_ID = process.env.VOICE_ID || 'Dennis'; +export const DEFAULT_TTS_MODEL_ID = process.env.TTS_MODEL_ID || 'inworld-tts-1'; + diff --git a/realtime-service/src/helpers.ts b/realtime-service/src/helpers.ts new file mode 100644 index 0000000..855febb --- /dev/null +++ b/realtime-service/src/helpers.ts @@ -0,0 +1,65 @@ +import logger from './logger'; +import { DEFAULT_LLM_MODEL_NAME, DEFAULT_LLM_PROVIDER, DEFAULT_VOICE_ID, DEFAULT_TTS_MODEL_ID } from './config'; + +export const parseEnvironmentVariables = () => { + if (!process.env.INWORLD_API_KEY) { + throw new Error('INWORLD_API_KEY env variable is required'); + } + + // Assembly.AI is now the only STT provider for audio input + if (!process.env.ASSEMBLYAI_API_KEY) { + throw new Error('ASSEMBLYAI_API_KEY env variable is required'); + } + + logger.info('STT Service: Assembly.AI (only supported provider)'); + + return { + apiKey: process.env.INWORLD_API_KEY, + llmModelName: DEFAULT_LLM_MODEL_NAME, + llmProvider: DEFAULT_LLM_PROVIDER, + voiceId: DEFAULT_VOICE_ID, + ttsModelId: DEFAULT_TTS_MODEL_ID, + // Because the env variable is optional and it's a string, we need to convert it to a boolean safely + graphVisualizationEnabled: + (process.env.GRAPH_VISUALIZATION_ENABLED || '').toLowerCase().trim() === + 'true', + assemblyAIApiKey: process.env.ASSEMBLYAI_API_KEY, + appName: process.env.APP_NAME || 'realtime-service', + appVersion: process.env.APP_VERSION || '1.0.0', + }; +}; + +/** + * Safely aborts a stream with proper error handling and logging. + * Handles both regular streams with abort() method and ContentStreams with napiStream. + * + * @param stream The stream to abort (must have an abort method or napiStream with abort method) + * @param streamName Descriptive name for the stream (for logging) + * @param sessionId Session ID associated with the stream (for logging) + * @param context Optional context message for logging (e.g., 'on close', 'due to cancellation') + */ +export function abortStream( + stream: any | undefined, + streamName: string, + sessionId: string, + context: string = '' +): void { + if (!stream) { + return; + } + + const logContext = context ? ` ${context}` : ''; + logger.debug({ sessionId }, `Aborting ${streamName}${logContext}`); + + try { + if (typeof stream.abort === 'function') { + stream.abort(); + } else if (stream.napiStream && typeof stream.napiStream.abort === 'function') { + // ContentStream doesn't have an abort() method, but we can try to abort the underlying napiStream + stream.napiStream.abort(); + } + } catch (error) { + logger.error({ error, sessionId }, `Error aborting ${streamName}`); + } +} + diff --git a/realtime-service/src/index.ts b/realtime-service/src/index.ts new file mode 100644 index 0000000..55d24bc --- /dev/null +++ b/realtime-service/src/index.ts @@ -0,0 +1,319 @@ +import 'dotenv/config'; + +import {InworldError} from '@inworld/runtime/common'; +import {initTelemetry} from '@inworld/runtime/telemetry'; +import cors from 'cors'; +import express from 'express'; +import http from 'http'; +import {createServer} from 'http'; +import client from 'prom-client'; +import {parse} from 'url'; +import {RawData, WebSocketServer} from 'ws'; + +import {RealtimeMessageHandler} from './components/realtime/realtime_message_handler'; +import {InworldRuntimeAppManager} from './components/runtime_app_manager'; +import {WS_APP_PORT} from './config'; +import {abortStream, parseEnvironmentVariables} from './helpers'; +import {formatContext, formatError, formatSession, formatWorkspace} from './log-helpers'; +import logger from './logger'; + +const METRICS_PORT = 9000; +const register = new client.Registry(); +register.setDefaultLabels({app: 'realtime-service'}); + +// Enable collection of default metrics +client.collectDefaultMetrics({register}); + +const wsConnectionCounter = new client.Gauge({ + name: 'websocket_connections_total', + help: 'Total number of active WebSocket connections', + registers: [register], +}); + +const app = express(); +const server = createServer(app); +const webSocket = new WebSocketServer({noServer: true}); + +const metricsApp = express(); +const metricsServer = http.createServer(metricsApp); + +app.use(cors()); +app.use(express.json()); +app.use(express.static('frontend')); + +// InworldRuntimeAppManager manages a single graph instance +// The graph supports multitenancy natively via API key in execute method +const env = parseEnvironmentVariables(); +const inworldRuntimeAppManager = new InworldRuntimeAppManager({ + llmModelName: env.llmModelName, + llmProvider: env.llmProvider, + voiceId: env.voiceId, + ttsModelId: env.ttsModelId, + graphVisualizationEnabled: env.graphVisualizationEnabled, + assemblyAIApiKey: env.assemblyAIApiKey, +}); + +initTelemetry({ + apiKey: env.apiKey, + appName: env.appName, + appVersion: env.appVersion, +}); + +metricsApp.get('/metrics', async (req, res) => { + try { + res.set('Content-Type', register.contentType); + res.end(await register.metrics()); + } catch (ex) { + logger.error({error: ex}, `Error serving metrics${formatError(ex)}`); + res.status(500).end('Internal Server Error'); + } +}); + +app.get('/health', (req, res) => { + res.status(200).send('OK'); +}); + +/** + * Extracts the Inworld API key from the WebSocket protocol header. + * The key is embedded in the sec-websocket-protocol header with a 'basic_' + * prefix. This function removes the prefix and reconstructs any missing base64 + * padding. + */ +function extractInworldApiKey(headers: http.IncomingHttpHeaders): string| + undefined { + const wsProtocolHeader = headers['sec-websocket-protocol'] as string; + if (!wsProtocolHeader) { + return undefined; + } + + // The protocol header may contain comma-separated values, look for basic_ + // prefix + const protocols = wsProtocolHeader.split(',').map(p => p.trim()); + for (const protocol of protocols) { + if (protocol.startsWith('basic_')) { + let base64Key = protocol.substring( + 6); // Remove 'basic_' prefix to get base64 credentials + + // Reconstruct missing padding = symbols (sender cuts trailing = symbols) + // Base64 strings should be a multiple of 4 characters in length + const paddingNeeded = (4 - (base64Key.length % 4)) % 4; + if (paddingNeeded > 0) { + base64Key += '='.repeat(paddingNeeded); + } + return base64Key; + } + } + + return undefined; +} + +webSocket.on('connection', async (ws, request) => { + wsConnectionCounter.inc(1); + const workspaceId = (request.headers['workspace-id'] as string) || + process.env.WORKSPACE_ID || 'default'; + + try { + logger.info( + {workspaceId}, + `WebSocket connection received ${formatWorkspace(workspaceId)}`); + + // Extract the Inworld API key from sec-websocket-protocol header + const inworldApiKey = extractInworldApiKey(request.headers); + + const {query} = parse(request.url!, true); + const sessionId = query.key?.toString(); + + logger.info( + {sessionId, workspaceId}, + `WebSocket connection established ${ + formatContext(sessionId, workspaceId)}`); + + if (!sessionId) { + logger.error( + {workspaceId}, + `WebSocket connection rejected ${ + formatWorkspace(workspaceId)}: no session key provided`); + ws.close(1008, 'No session key provided'); + return; + } + + // Get the singleton InworldApp instance + const inworldApp = await inworldRuntimeAppManager.getApp(); + + // Create session on-the-fly if it doesn't exist + if (!inworldApp.connections?.[sessionId]) { + logger.info( + {sessionId, workspaceId}, + `Creating new session ${formatContext(sessionId, workspaceId)}`); + // Create a minimal connection for realtime protocol + // The session will be fully configured via session.update events + inworldApp.connections[sessionId] = { + workspaceId, + state: { + interactionId: '', // Will be set by graph nodes + messages: [], + agent: null, + userName: 'User', + }, + apiKey: inworldApiKey || '', + ws: null, + }; + } + + inworldApp.connections[sessionId].ws = + inworldApp.connections[sessionId].ws ?? ws; + + ws.on('error', (error) => { + logger.error( + {error, sessionId, workspaceId}, + `WebSocket error ${formatContext(sessionId, workspaceId)}${ + formatError(error)}`); + }); + + ws.on('close', async (code, reason) => { + logger.info( + {sessionId, workspaceId, code, reason: reason.toString()}, + `WebSocket closed ${formatContext(sessionId, workspaceId)} [code:${ + code}] [reason:${reason.toString()}]`); + wsConnectionCounter.dec(1); + + const connection = inworldApp.connections[sessionId]; + if (connection) { + // Step 1: Abort any active graph executions FIRST + abortStream( + connection.currentAudioExecutionStream, 'audio execution stream', + sessionId, 'on close'); + connection.currentAudioExecutionStream = undefined; + + // Step 2: Clean up multimodal stream if it exists + if (connection.multimodalStreamManager) { + logger.info( + {sessionId}, + `Ending multimodal stream ${ + formatSession(sessionId)} due to WebSocket close`); + connection.multimodalStreamManager.end(); + connection.multimodalStreamManager = undefined; + } + + // Step 3: Clean up audio graph execution reference + if (connection.currentAudioGraphExecution) { + connection.currentAudioGraphExecution = undefined; + } + } + + // Clean up AssemblyAI STT session if it exists + const assemblyAINode = inworldApp.graphWithAudioInput?.assemblyAINode; + if (assemblyAINode) { + try { + await assemblyAINode.closeSession(sessionId); + } catch (error) { + logger.error( + {error, sessionId, workspaceId}, + `Error during closing Assembly Session ${ + formatContext(sessionId, workspaceId)}${formatError(error)}`); + } + } + + // Clean up connection + inworldApp.removeSession(sessionId); + logger.info(`[Session ${sessionId}] Session removed and resources cleaned up`); + }); + + // Use OpenAI Realtime API protocol + const realtimeHandler = new RealtimeMessageHandler( + inworldApp, + sessionId, + (data: any) => ws.send(JSON.stringify(data)), + ); + + // Initialize session and send session.created event + logger.info( + {sessionId, workspaceId}, + `Initializing realtime session ${ + formatContext(sessionId, workspaceId)}`); + await realtimeHandler.initialize(); + logger.info( + {sessionId, workspaceId}, + `Realtime session initialized successfully ${ + formatContext(sessionId, workspaceId)}`); + + ws.on('message', (data: RawData) => realtimeHandler.handleMessage(data)); + } catch (error) { + logger.error( + {error, workspaceId}, + `WebSocket connection error ${formatWorkspace(workspaceId)}${ + formatError(error)}`); + ws.close(1011, 'Internal server error'); + } +}); + +server.on('upgrade', async (request, socket, head) => { + const {pathname, query} = parse(request.url!, true); + + if (pathname === '/session') { + const authToken = process.env.AUTH_TOKEN; + + // Validate token if configured + if (authToken) { + const providedToken = query?.token?.toString(); + + if (!providedToken || providedToken !== authToken) { + socket.write('HTTP/1.1 401 Unauthorized\r\n\r\n'); + socket.destroy(); + return; + } + } + + webSocket.handleUpgrade(request, socket, head, (ws) => { + webSocket.emit('connection', ws, request); + }); + } else { + socket.destroy(); + } +}); + +server.listen(WS_APP_PORT, () => { + logger.info( + {port: WS_APP_PORT}, + `Application Server listening on port ${WS_APP_PORT}`); + logger.info('InworldApp will be created on first connection'); +}); + +metricsServer.listen(METRICS_PORT, () => { + logger.info( + {port: METRICS_PORT}, `Metrics Server listening on port ${METRICS_PORT}`); +}); + +function done() { + logger.info('Server is closing'); + + // Handle the async shutdown properly + inworldRuntimeAppManager.shutdown() + .then(() => { + metricsServer.close(() => { + logger.info('Metrics server closed'); + process.exit(0); + }); + }) + .catch((err) => { + logger.error({error: err}, `Error during shutdown${formatError(err)}`); + process.exit(1); + }); +} + +process.on('SIGINT', done); +process.on('SIGTERM', done); +process.on('SIGUSR2', done); +process.on('unhandledRejection', (err: unknown) => { + if (err instanceof InworldError) { + logger.error( + { + message: err.message, + context: err.context, + }, + `Inworld Error - unhandled rejection${formatError(err)}`); + } else { + logger.error({error: err}, `Unhandled rejection${formatError(err)}`); + } + process.exit(1); +}); diff --git a/realtime-service/src/log-helpers.ts b/realtime-service/src/log-helpers.ts new file mode 100644 index 0000000..074f8d9 --- /dev/null +++ b/realtime-service/src/log-helpers.ts @@ -0,0 +1,88 @@ +/** + * Helper utilities for creating readable log messages with key context + * + * These helpers make it easy to include important information in both: + * 1. The message text (visible at a glance) + * 2. Structured fields (for querying/filtering) + */ + +/** + * Format a session context tag for log messages + * Usage: `Something happened ${formatSession(sessionId)}` + */ +export function formatSession(sessionId: string | undefined): string { + return sessionId ? `[session:${sessionId}]` : ''; +} + +/** + * Format a workspace context tag for log messages + * Usage: `Something happened ${formatWorkspace(workspaceId)}` + */ +export function formatWorkspace(workspaceId: string | undefined): string { + return workspaceId ? `[workspace:${workspaceId}]` : ''; +} + +/** + * Format an interaction context tag for log messages + * Usage: `Something happened ${formatInteraction(interactionId)}` + */ +export function formatInteraction(interactionId: string | undefined): string { + return interactionId ? `[interaction:${interactionId}]` : ''; +} + +/** + * Format a duration in milliseconds + * Usage: `Completed ${formatDuration(elapsed)}` + */ +export function formatDuration(ms: number | undefined): string { + return ms !== undefined ? `[duration:${ms}ms]` : ''; +} + +/** + * Format an error message + * Usage: `Failed ${formatError(error)}` + */ +export function formatError(error: unknown): string { + if (error instanceof Error) { + return `: ${error.message}`; + } + return error ? `: ${String(error)}` : ''; +} + +/** + * Combine session and workspace tags + * Usage: `Something happened ${formatContext(sessionId, workspaceId)}` + */ +export function formatContext( + sessionId?: string, + workspaceId?: string, + interactionId?: string +): string { + const parts = [ + formatSession(sessionId), + formatWorkspace(workspaceId), + formatInteraction(interactionId), + ].filter(Boolean); + + return parts.join(' '); +} + +/** + * Example usage: + * + * logger.info( + * { sessionId, workspaceId }, + * `WebSocket connected ${formatContext(sessionId, workspaceId)}` + * ); + * + * logger.error( + * { error, sessionId, operation }, + * `Operation failed ${formatSession(sessionId)}${formatError(error)}` + * ); + * + * logger.info( + * { sessionId, duration }, + * `Request completed ${formatSession(sessionId)} ${formatDuration(duration)}` + * ); + */ + diff --git a/realtime-service/src/logger.ts b/realtime-service/src/logger.ts new file mode 100644 index 0000000..7d20f50 --- /dev/null +++ b/realtime-service/src/logger.ts @@ -0,0 +1,66 @@ +import pino from 'pino'; + +/** + * Logger configuration for Google Cloud Logging compatibility + * + * DEFAULT: JSON logs formatted for Cloud Logging (production-ready) + * OPTIONAL: Pretty-print for local development + * + * Each log entry is a single JSON object, preventing multi-line splits in Cloud Logging + * + * Environment variables: + * - REALTIME_LOG_LEVEL: 'debug', 'info', 'warn', 'error' (default: 'info') + * - REALTIME_LOG_PRETTY: '1' to enable pretty logs locally (default: '0' = JSON) + * + * Examples: + * npm start # JSON logs (default, cloud-ready) + * REALTIME_LOG_PRETTY=1 npm start # Pretty logs (local dev) + * REALTIME_LOG_LEVEL=debug npm start # JSON with debug level + */ +const usePretty = process.env.REALTIME_LOG_PRETTY === '1'; + +const logger = pino({ + level: process.env.REALTIME_LOG_LEVEL || 'info', + + // Use pretty print locally (unless LOG_PRETTY=0), JSON in production + transport: usePretty ? { + target: 'pino-pretty', + options: { + colorize: true, + translateTime: 'HH:MM:ss.l', // Include milliseconds for debugging + ignore: 'pid,hostname,service,version', + // Show message and structured fields in a compact format + messageFormat: '{levelLabel} {msg}', + // Show structured fields in compact single-line format + singleLine: true, + // Limit object depth to avoid huge logs + depth: 3, + } + } : undefined, + + // Add common fields to all logs + base: { + service: 'realtime-service', + version: process.env.APP_VERSION || 'unknown', + }, + + // Format timestamps for Cloud Logging + timestamp: pino.stdTimeFunctions.isoTime, + + // Map Pino levels to Cloud Logging severity + formatters: { + level: (label: string) => { + return { severity: label.toUpperCase() }; + }, + }, + + // Serialize errors properly + serializers: { + error: pino.stdSerializers.err, + req: pino.stdSerializers.req, + res: pino.stdSerializers.res, + }, +}); + +export default logger; + diff --git a/realtime-service/src/package-lock.json b/realtime-service/src/package-lock.json new file mode 100644 index 0000000..3d3144e --- /dev/null +++ b/realtime-service/src/package-lock.json @@ -0,0 +1,3273 @@ +{ + "name": "server", + "lockfileVersion": 3, + "requires": true, + "packages": { + "": { + "name": "server", + "license": "SEE LICENSE IN LICENSE.md and LICENSE-CPP-BINARIES.md", + "dependencies": { + "@inworld/runtime": "https://storage.googleapis.com/assets-inworld-ai/node-packages/inworld-runtime-0.9.0-rc.13.tgz", + "cors": "^2.8.5", + "dotenv": "^16.4.7", + "express": "^5.1.0", + "express-validator": "^7.2.1", + "groq-sdk": "^0.7.0", + "pino": "^9.14.0", + "pino-pretty": "^13.1.2", + "prom-client": "^15.1.3", + "uuid": "^11.1.0", + "wav-encoder": "^1.3.0", + "ws": "^8.18.1", + "zod": "^4.1.13" + }, + "devDependencies": { + "@types/cors": "^2.8.17", + "@types/express": "^5.0.1", + "@types/express-validator": "^3.0.0", + "@types/node": "^22.14.0", + "@types/ws": "^8.18.1", + "nodemon": "^3.1.9", + "ts-node": "^10.9.2", + "tsconfig-paths": "^4.2.0", + "typescript": "^5.8.3" + } + }, + "node_modules/@cspotcode/source-map-support": { + "version": "0.8.1", + "resolved": "https://registry.npmjs.org/@cspotcode/source-map-support/-/source-map-support-0.8.1.tgz", + "integrity": "sha512-IchNf6dN4tHoMFIn/7OE8LWZ19Y6q/67Bmf6vnGREv8RSbBVb9LPJxEcnwrcwX6ixSvaiGoomAUvu4YSxXrVgw==", + "dev": true, + "license": "MIT", + "dependencies": { + "@jridgewell/trace-mapping": "0.3.9" + }, + "engines": { + "node": ">=12" + } + }, + "node_modules/@inworld/runtime": { + "version": "0.9.0-rc.13", + "resolved": "https://storage.googleapis.com/assets-inworld-ai/node-packages/inworld-runtime-0.9.0-rc.13.tgz", + "integrity": "sha512-6C/aND5PqOVzujwoS0rvUR963VxJ+MO0X+IZg9WpjrxBbuFag7fXay5yyamH0cKoTvmaTT2s8fca8pK0zQ0CYg==", + "hasInstallScript": true, + "license": "SEE LICENSE IN LICENSE.md and LICENSE-CPP-BINARIES.md", + "dependencies": { + "@types/lodash.snakecase": "^4.1.9", + "@types/protobufjs": "^6.0.0", + "@zod/core": "^0.11.6", + "ajv": "^8.17.1", + "ajv-formats": "^3.0.1", + "assemblyai": "^4.17.0", + "chalk": "^4.1.2", + "decompress": "^4.2.1", + "decompress-targz": "^4.1.1", + "decompress-unzip": "^4.0.1", + "groq-sdk": "^0.33.0", + "lodash.snakecase": "^4.1.1", + "node-record-lpcm16": "^1.0.1", + "protobufjs": "^7.5.4", + "uuid": "^11.1.0", + "zod-to-json-schema": "^3.25.0" + }, + "engines": { + "node": ">=18.0.0" + } + }, + "node_modules/@inworld/runtime/node_modules/@types/node": { + "version": "18.19.130", + "resolved": "https://registry.npmjs.org/@types/node/-/node-18.19.130.tgz", + "integrity": "sha512-GRaXQx6jGfL8sKfaIDD6OupbIHBr9jv7Jnaml9tB7l4v068PAOXqfcujMMo5PhbIs6ggR1XODELqahT2R8v0fg==", + "license": "MIT", + "dependencies": { + "undici-types": "~5.26.4" + } + }, + "node_modules/@inworld/runtime/node_modules/groq-sdk": { + "version": "0.33.0", + "resolved": "https://registry.npmjs.org/groq-sdk/-/groq-sdk-0.33.0.tgz", + "integrity": "sha512-wb7NrBq7LZDDhDPSpuAd9LpZ0MNjmWKGLfybYfjY3r63mSpfiP8+GQZQcSDJcX+jIMzSm+SwzxModDyVZ2T66Q==", + "license": "Apache-2.0", + "dependencies": { + "@types/node": "^18.11.18", + "@types/node-fetch": "^2.6.4", + "abort-controller": "^3.0.0", + "agentkeepalive": "^4.2.1", + "form-data-encoder": "1.7.2", + "formdata-node": "^4.3.2", + "node-fetch": "^2.6.7" + } + }, + "node_modules/@inworld/runtime/node_modules/undici-types": { + "version": "5.26.5", + "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-5.26.5.tgz", + "integrity": "sha512-JlCMO+ehdEIKqlFxk6IfVoAUVmgz7cU7zD/h9XZ0qzeosSHmUJVOzSQvvYSYWXkFXC+IfLKSIffhv0sVZup6pA==", + "license": "MIT" + }, + "node_modules/@jridgewell/resolve-uri": { + "version": "3.1.2", + "resolved": "https://registry.npmjs.org/@jridgewell/resolve-uri/-/resolve-uri-3.1.2.tgz", + "integrity": "sha512-bRISgCIjP20/tbWSPWMEi54QVPRZExkuD9lJL+UIxUKtwVJA8wW1Trb1jMs1RFXo1CBTNZ/5hpC9QvmKWdopKw==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=6.0.0" + } + }, + "node_modules/@jridgewell/sourcemap-codec": { + "version": "1.5.5", + "resolved": "https://registry.npmjs.org/@jridgewell/sourcemap-codec/-/sourcemap-codec-1.5.5.tgz", + "integrity": "sha512-cYQ9310grqxueWbl+WuIUIaiUaDcj7WOq5fVhEljNVgRfOUhY9fy2zTvfoqWsnebh8Sl70VScFbICvJnLKB0Og==", + "dev": true, + "license": "MIT" + }, + "node_modules/@jridgewell/trace-mapping": { + "version": "0.3.9", + "resolved": "https://registry.npmjs.org/@jridgewell/trace-mapping/-/trace-mapping-0.3.9.tgz", + "integrity": "sha512-3Belt6tdc8bPgAtbcmdtNJlirVoTmEb5e2gC94PnkwEW9jI6CAHUeoG85tjWP5WquqfavoMtMwiG4P926ZKKuQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "@jridgewell/resolve-uri": "^3.0.3", + "@jridgewell/sourcemap-codec": "^1.4.10" + } + }, + "node_modules/@opentelemetry/api": { + "version": "1.9.0", + "resolved": "https://registry.npmjs.org/@opentelemetry/api/-/api-1.9.0.tgz", + "integrity": "sha512-3giAOQvZiH5F9bMlMiv8+GSPMeqg0dbaeo58/0SlA9sxSqZhnUtxzX9/2FzyhS9sWQf5S0GJE0AKBrFqjpeYcg==", + "license": "Apache-2.0", + "engines": { + "node": ">=8.0.0" + } + }, + "node_modules/@pinojs/redact": { + "version": "0.4.0", + "resolved": "https://registry.npmjs.org/@pinojs/redact/-/redact-0.4.0.tgz", + "integrity": "sha512-k2ENnmBugE/rzQfEcdWHcCY+/FM3VLzH9cYEsbdsoqrvzAKRhUZeRNhAZvB8OitQJ1TBed3yqWtdjzS6wJKBwg==", + "license": "MIT" + }, + "node_modules/@protobufjs/aspromise": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/@protobufjs/aspromise/-/aspromise-1.1.2.tgz", + "integrity": "sha512-j+gKExEuLmKwvz3OgROXtrJ2UG2x8Ch2YZUxahh+s1F2HZ+wAceUNLkvy6zKCPVRkU++ZWQrdxsUeQXmcg4uoQ==", + "license": "BSD-3-Clause" + }, + "node_modules/@protobufjs/base64": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/@protobufjs/base64/-/base64-1.1.2.tgz", + "integrity": "sha512-AZkcAA5vnN/v4PDqKyMR5lx7hZttPDgClv83E//FMNhR2TMcLUhfRUBHCmSl0oi9zMgDDqRUJkSxO3wm85+XLg==", + "license": "BSD-3-Clause" + }, + "node_modules/@protobufjs/codegen": { + "version": "2.0.4", + "resolved": "https://registry.npmjs.org/@protobufjs/codegen/-/codegen-2.0.4.tgz", + "integrity": "sha512-YyFaikqM5sH0ziFZCN3xDC7zeGaB/d0IUb9CATugHWbd1FRFwWwt4ld4OYMPWu5a3Xe01mGAULCdqhMlPl29Jg==", + "license": "BSD-3-Clause" + }, + "node_modules/@protobufjs/eventemitter": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/@protobufjs/eventemitter/-/eventemitter-1.1.0.tgz", + "integrity": "sha512-j9ednRT81vYJ9OfVuXG6ERSTdEL1xVsNgqpkxMsbIabzSo3goCjDIveeGv5d03om39ML71RdmrGNjG5SReBP/Q==", + "license": "BSD-3-Clause" + }, + "node_modules/@protobufjs/fetch": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/@protobufjs/fetch/-/fetch-1.1.0.tgz", + "integrity": "sha512-lljVXpqXebpsijW71PZaCYeIcE5on1w5DlQy5WH6GLbFryLUrBD4932W/E2BSpfRJWseIL4v/KPgBFxDOIdKpQ==", + "license": "BSD-3-Clause", + "dependencies": { + "@protobufjs/aspromise": "^1.1.1", + "@protobufjs/inquire": "^1.1.0" + } + }, + "node_modules/@protobufjs/float": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/@protobufjs/float/-/float-1.0.2.tgz", + "integrity": "sha512-Ddb+kVXlXst9d+R9PfTIxh1EdNkgoRe5tOX6t01f1lYWOvJnSPDBlG241QLzcyPdoNTsblLUdujGSE4RzrTZGQ==", + "license": "BSD-3-Clause" + }, + "node_modules/@protobufjs/inquire": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/@protobufjs/inquire/-/inquire-1.1.0.tgz", + "integrity": "sha512-kdSefcPdruJiFMVSbn801t4vFK7KB/5gd2fYvrxhuJYg8ILrmn9SKSX2tZdV6V+ksulWqS7aXjBcRXl3wHoD9Q==", + "license": "BSD-3-Clause" + }, + "node_modules/@protobufjs/path": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/@protobufjs/path/-/path-1.1.2.tgz", + "integrity": "sha512-6JOcJ5Tm08dOHAbdR3GrvP+yUUfkjG5ePsHYczMFLq3ZmMkAD98cDgcT2iA1lJ9NVwFd4tH/iSSoe44YWkltEA==", + "license": "BSD-3-Clause" + }, + "node_modules/@protobufjs/pool": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/@protobufjs/pool/-/pool-1.1.0.tgz", + "integrity": "sha512-0kELaGSIDBKvcgS4zkjz1PeddatrjYcmMWOlAuAPwAeccUrPHdUqo/J6LiymHHEiJT5NrF1UVwxY14f+fy4WQw==", + "license": "BSD-3-Clause" + }, + "node_modules/@protobufjs/utf8": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/@protobufjs/utf8/-/utf8-1.1.0.tgz", + "integrity": "sha512-Vvn3zZrhQZkkBE8LSuW3em98c0FwgO4nxzv6OdSxPKJIEKY2bGbHn+mhGIPerzI4twdxaP8/0+06HBpwf345Lw==", + "license": "BSD-3-Clause" + }, + "node_modules/@tsconfig/node10": { + "version": "1.0.12", + "resolved": "https://registry.npmjs.org/@tsconfig/node10/-/node10-1.0.12.tgz", + "integrity": "sha512-UCYBaeFvM11aU2y3YPZ//O5Rhj+xKyzy7mvcIoAjASbigy8mHMryP5cK7dgjlz2hWxh1g5pLw084E0a/wlUSFQ==", + "dev": true, + "license": "MIT" + }, + "node_modules/@tsconfig/node12": { + "version": "1.0.11", + "resolved": "https://registry.npmjs.org/@tsconfig/node12/-/node12-1.0.11.tgz", + "integrity": "sha512-cqefuRsh12pWyGsIoBKJA9luFu3mRxCA+ORZvA4ktLSzIuCUtWVxGIuXigEwO5/ywWFMZ2QEGKWvkZG1zDMTag==", + "dev": true, + "license": "MIT" + }, + "node_modules/@tsconfig/node14": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/@tsconfig/node14/-/node14-1.0.3.tgz", + "integrity": "sha512-ysT8mhdixWK6Hw3i1V2AeRqZ5WfXg1G43mqoYlM2nc6388Fq5jcXyr5mRsqViLx/GJYdoL0bfXD8nmF+Zn/Iow==", + "dev": true, + "license": "MIT" + }, + "node_modules/@tsconfig/node16": { + "version": "1.0.4", + "resolved": "https://registry.npmjs.org/@tsconfig/node16/-/node16-1.0.4.tgz", + "integrity": "sha512-vxhUy4J8lyeyinH7Azl1pdd43GJhZH/tP2weN8TntQblOY+A0XbT8DJk1/oCPuOOyg/Ja757rG0CgHcWC8OfMA==", + "dev": true, + "license": "MIT" + }, + "node_modules/@types/body-parser": { + "version": "1.19.6", + "resolved": "https://registry.npmjs.org/@types/body-parser/-/body-parser-1.19.6.tgz", + "integrity": "sha512-HLFeCYgz89uk22N5Qg3dvGvsv46B8GLvKKo1zKG4NybA8U2DiEO3w9lqGg29t/tfLRJpJ6iQxnVw4OnB7MoM9g==", + "dev": true, + "license": "MIT", + "dependencies": { + "@types/connect": "*", + "@types/node": "*" + } + }, + "node_modules/@types/connect": { + "version": "3.4.38", + "resolved": "https://registry.npmjs.org/@types/connect/-/connect-3.4.38.tgz", + "integrity": "sha512-K6uROf1LD88uDQqJCktA4yzL1YYAK6NgfsI0v/mTgyPKWsX1CnJ0XPSDhViejru1GcRkLWb8RlzFYJRqGUbaug==", + "dev": true, + "license": "MIT", + "dependencies": { + "@types/node": "*" + } + }, + "node_modules/@types/cors": { + "version": "2.8.19", + "resolved": "https://registry.npmjs.org/@types/cors/-/cors-2.8.19.tgz", + "integrity": "sha512-mFNylyeyqN93lfe/9CSxOGREz8cpzAhH+E93xJ4xWQf62V8sQ/24reV2nyzUWM6H6Xji+GGHpkbLe7pVoUEskg==", + "dev": true, + "license": "MIT", + "dependencies": { + "@types/node": "*" + } + }, + "node_modules/@types/express": { + "version": "5.0.6", + "resolved": "https://registry.npmjs.org/@types/express/-/express-5.0.6.tgz", + "integrity": "sha512-sKYVuV7Sv9fbPIt/442koC7+IIwK5olP1KWeD88e/idgoJqDm3JV/YUiPwkoKK92ylff2MGxSz1CSjsXelx0YA==", + "dev": true, + "license": "MIT", + "dependencies": { + "@types/body-parser": "*", + "@types/express-serve-static-core": "^5.0.0", + "@types/serve-static": "^2" + } + }, + "node_modules/@types/express-serve-static-core": { + "version": "5.1.0", + "resolved": "https://registry.npmjs.org/@types/express-serve-static-core/-/express-serve-static-core-5.1.0.tgz", + "integrity": "sha512-jnHMsrd0Mwa9Cf4IdOzbz543y4XJepXrbia2T4b6+spXC2We3t1y6K44D3mR8XMFSXMCf3/l7rCgddfx7UNVBA==", + "dev": true, + "license": "MIT", + "dependencies": { + "@types/node": "*", + "@types/qs": "*", + "@types/range-parser": "*", + "@types/send": "*" + } + }, + "node_modules/@types/express-validator": { + "version": "3.0.2", + "resolved": "https://registry.npmjs.org/@types/express-validator/-/express-validator-3.0.2.tgz", + "integrity": "sha512-dV+1u6absDDEIVe5jd5ID1XYvPPvOSvggMWh09VYM3TeBjAjBDtwJWyJg/5PnK7s8FRgbOtHgD6oX3N2MKWwDw==", + "deprecated": "This is a stub types definition. express-validator provides its own type definitions, so you do not need this installed.", + "dev": true, + "license": "MIT", + "dependencies": { + "express-validator": "*" + } + }, + "node_modules/@types/http-errors": { + "version": "2.0.5", + "resolved": "https://registry.npmjs.org/@types/http-errors/-/http-errors-2.0.5.tgz", + "integrity": "sha512-r8Tayk8HJnX0FztbZN7oVqGccWgw98T/0neJphO91KkmOzug1KkofZURD4UaD5uH8AqcFLfdPErnBod0u71/qg==", + "dev": true, + "license": "MIT" + }, + "node_modules/@types/lodash": { + "version": "4.17.21", + "resolved": "https://registry.npmjs.org/@types/lodash/-/lodash-4.17.21.tgz", + "integrity": "sha512-FOvQ0YPD5NOfPgMzJihoT+Za5pdkDJWcbpuj1DjaKZIr/gxodQjY/uWEFlTNqW2ugXHUiL8lRQgw63dzKHZdeQ==", + "license": "MIT" + }, + "node_modules/@types/lodash.snakecase": { + "version": "4.1.9", + "resolved": "https://registry.npmjs.org/@types/lodash.snakecase/-/lodash.snakecase-4.1.9.tgz", + "integrity": "sha512-emBZJUiNlo+QPXr1junMKXwzHJK9zbFvTVdyAoorFcm1YRsbzkZCYPTVMM9AW+dlnA6utG7vpfvOs8alxv/TMw==", + "license": "MIT", + "dependencies": { + "@types/lodash": "*" + } + }, + "node_modules/@types/node": { + "version": "22.19.2", + "resolved": "https://registry.npmjs.org/@types/node/-/node-22.19.2.tgz", + "integrity": "sha512-LPM2G3Syo1GLzXLGJAKdqoU35XvrWzGJ21/7sgZTUpbkBaOasTj8tjwn6w+hCkqaa1TfJ/w67rJSwYItlJ2mYw==", + "license": "MIT", + "dependencies": { + "undici-types": "~6.21.0" + } + }, + "node_modules/@types/node-fetch": { + "version": "2.6.13", + "resolved": "https://registry.npmjs.org/@types/node-fetch/-/node-fetch-2.6.13.tgz", + "integrity": "sha512-QGpRVpzSaUs30JBSGPjOg4Uveu384erbHBoT1zeONvyCfwQxIkUshLAOqN/k9EjGviPRmWTTe6aH2qySWKTVSw==", + "license": "MIT", + "dependencies": { + "@types/node": "*", + "form-data": "^4.0.4" + } + }, + "node_modules/@types/protobufjs": { + "version": "6.0.0", + "resolved": "https://registry.npmjs.org/@types/protobufjs/-/protobufjs-6.0.0.tgz", + "integrity": "sha512-A27RDExpAf3rdDjIrHKiJK6x8kqqJ4CmoChwtipfhVAn1p7+wviQFFP7dppn8FslSbHtQeVPvi8wNKkDjSYjHw==", + "deprecated": "This is a stub types definition for protobufjs (https://github.com/dcodeIO/ProtoBuf.js). protobufjs provides its own type definitions, so you don't need @types/protobufjs installed!", + "license": "MIT", + "dependencies": { + "protobufjs": "*" + } + }, + "node_modules/@types/qs": { + "version": "6.14.0", + "resolved": "https://registry.npmjs.org/@types/qs/-/qs-6.14.0.tgz", + "integrity": "sha512-eOunJqu0K1923aExK6y8p6fsihYEn/BYuQ4g0CxAAgFc4b/ZLN4CrsRZ55srTdqoiLzU2B2evC+apEIxprEzkQ==", + "dev": true, + "license": "MIT" + }, + "node_modules/@types/range-parser": { + "version": "1.2.7", + "resolved": "https://registry.npmjs.org/@types/range-parser/-/range-parser-1.2.7.tgz", + "integrity": "sha512-hKormJbkJqzQGhziax5PItDUTMAM9uE2XXQmM37dyd4hVM+5aVl7oVxMVUiVQn2oCQFN/LKCZdvSM0pFRqbSmQ==", + "dev": true, + "license": "MIT" + }, + "node_modules/@types/send": { + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/@types/send/-/send-1.2.1.tgz", + "integrity": "sha512-arsCikDvlU99zl1g69TcAB3mzZPpxgw0UQnaHeC1Nwb015xp8bknZv5rIfri9xTOcMuaVgvabfIRA7PSZVuZIQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "@types/node": "*" + } + }, + "node_modules/@types/serve-static": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/@types/serve-static/-/serve-static-2.2.0.tgz", + "integrity": "sha512-8mam4H1NHLtu7nmtalF7eyBH14QyOASmcxHhSfEoRyr0nP/YdoesEtU+uSRvMe96TW/HPTtkoKqQLl53N7UXMQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "@types/http-errors": "*", + "@types/node": "*" + } + }, + "node_modules/@types/ws": { + "version": "8.18.1", + "resolved": "https://registry.npmjs.org/@types/ws/-/ws-8.18.1.tgz", + "integrity": "sha512-ThVF6DCVhA8kUGy+aazFQ4kXQ7E1Ty7A3ypFOe0IcJV8O/M511G99AW24irKrW56Wt44yG9+ij8FaqoBGkuBXg==", + "dev": true, + "license": "MIT", + "dependencies": { + "@types/node": "*" + } + }, + "node_modules/@zod/core": { + "version": "0.11.6", + "resolved": "https://registry.npmjs.org/@zod/core/-/core-0.11.6.tgz", + "integrity": "sha512-03Bv82fFSfjDAvMfdHHdGSS6SOJs0iCcJlWJv1kJHRtoTT02hZpyip/2Lk6oo4l4FtjuwTrsEQTwg/LD8I7dJA==", + "license": "MIT", + "funding": { + "url": "https://github.com/sponsors/colinhacks" + } + }, + "node_modules/abort-controller": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/abort-controller/-/abort-controller-3.0.0.tgz", + "integrity": "sha512-h8lQ8tacZYnR3vNQTgibj+tODHI5/+l06Au2Pcriv/Gmet0eaj4TwWH41sO9wnHDiQsEj19q0drzdWdeAHtweg==", + "license": "MIT", + "dependencies": { + "event-target-shim": "^5.0.0" + }, + "engines": { + "node": ">=6.5" + } + }, + "node_modules/accepts": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/accepts/-/accepts-2.0.0.tgz", + "integrity": "sha512-5cvg6CtKwfgdmVqY1WIiXKc3Q1bkRqGLi+2W/6ao+6Y7gu/RCwRuAhGEzh5B4KlszSuTLgZYuqFqo5bImjNKng==", + "license": "MIT", + "dependencies": { + "mime-types": "^3.0.0", + "negotiator": "^1.0.0" + }, + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/acorn": { + "version": "8.15.0", + "resolved": "https://registry.npmjs.org/acorn/-/acorn-8.15.0.tgz", + "integrity": "sha512-NZyJarBfL7nWwIq+FDL6Zp/yHEhePMNnnJ0y3qfieCrmNvYct8uvtiV41UvlSe6apAfk0fY1FbWx+NwfmpvtTg==", + "dev": true, + "license": "MIT", + "bin": { + "acorn": "bin/acorn" + }, + "engines": { + "node": ">=0.4.0" + } + }, + "node_modules/acorn-walk": { + "version": "8.3.4", + "resolved": "https://registry.npmjs.org/acorn-walk/-/acorn-walk-8.3.4.tgz", + "integrity": "sha512-ueEepnujpqee2o5aIYnvHU6C0A42MNdsIDeqy5BydrkuC5R1ZuUFnm27EeFJGoEHJQgn3uleRvmTXaJgfXbt4g==", + "dev": true, + "license": "MIT", + "dependencies": { + "acorn": "^8.11.0" + }, + "engines": { + "node": ">=0.4.0" + } + }, + "node_modules/agentkeepalive": { + "version": "4.6.0", + "resolved": "https://registry.npmjs.org/agentkeepalive/-/agentkeepalive-4.6.0.tgz", + "integrity": "sha512-kja8j7PjmncONqaTsB8fQ+wE2mSU2DJ9D4XKoJ5PFWIdRMa6SLSN1ff4mOr4jCbfRSsxR4keIiySJU0N9T5hIQ==", + "license": "MIT", + "dependencies": { + "humanize-ms": "^1.2.1" + }, + "engines": { + "node": ">= 8.0.0" + } + }, + "node_modules/ajv": { + "version": "8.17.1", + "resolved": "https://registry.npmjs.org/ajv/-/ajv-8.17.1.tgz", + "integrity": "sha512-B/gBuNg5SiMTrPkC+A2+cW0RszwxYmn6VYxB/inlBStS5nx6xHIt/ehKRhIMhqusl7a8LjQoZnjCs5vhwxOQ1g==", + "license": "MIT", + "dependencies": { + "fast-deep-equal": "^3.1.3", + "fast-uri": "^3.0.1", + "json-schema-traverse": "^1.0.0", + "require-from-string": "^2.0.2" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/epoberezkin" + } + }, + "node_modules/ajv-formats": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/ajv-formats/-/ajv-formats-3.0.1.tgz", + "integrity": "sha512-8iUql50EUR+uUcdRQ3HDqa6EVyo3docL8g5WJ3FNcWmu62IbkGUue/pEyLBW8VGKKucTPgqeks4fIU1DA4yowQ==", + "license": "MIT", + "dependencies": { + "ajv": "^8.0.0" + }, + "peerDependencies": { + "ajv": "^8.0.0" + }, + "peerDependenciesMeta": { + "ajv": { + "optional": true + } + } + }, + "node_modules/ansi-styles": { + "version": "4.3.0", + "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-4.3.0.tgz", + "integrity": "sha512-zbB9rCJAT1rbjiVDb2hqKFHNYLxgtk8NURxZ3IZwD3F6NtxbXZQCnnSi1Lkx+IDohdPlFp222wVALIheZJQSEg==", + "license": "MIT", + "dependencies": { + "color-convert": "^2.0.1" + }, + "engines": { + "node": ">=8" + }, + "funding": { + "url": "https://github.com/chalk/ansi-styles?sponsor=1" + } + }, + "node_modules/anymatch": { + "version": "3.1.3", + "resolved": "https://registry.npmjs.org/anymatch/-/anymatch-3.1.3.tgz", + "integrity": "sha512-KMReFUr0B4t+D+OBkjR3KYqvocp2XaSzO55UcB6mgQMd3KbcE+mWTyvVV7D/zsdEbNnV6acZUutkiHQXvTr1Rw==", + "dev": true, + "license": "ISC", + "dependencies": { + "normalize-path": "^3.0.0", + "picomatch": "^2.0.4" + }, + "engines": { + "node": ">= 8" + } + }, + "node_modules/arg": { + "version": "4.1.3", + "resolved": "https://registry.npmjs.org/arg/-/arg-4.1.3.tgz", + "integrity": "sha512-58S9QDqG0Xx27YwPSt9fJxivjYl432YCwfDMfZ+71RAqUrZef7LrKQZ3LHLOwCS4FLNBplP533Zx895SeOCHvA==", + "dev": true, + "license": "MIT" + }, + "node_modules/assemblyai": { + "version": "4.21.0", + "resolved": "https://registry.npmjs.org/assemblyai/-/assemblyai-4.21.0.tgz", + "integrity": "sha512-Nbbtk3wwHB1vhi7+98kYi/e9N9EFY0wAv5OuKXNmGw9eHkHG2r3ytgYOigcXyzC8cCvQdhA8Y0JCQm2BFDOOlQ==", + "license": "MIT", + "dependencies": { + "ws": "^8.18.0" + }, + "engines": { + "node": ">=18" + } + }, + "node_modules/asynckit": { + "version": "0.4.0", + "resolved": "https://registry.npmjs.org/asynckit/-/asynckit-0.4.0.tgz", + "integrity": "sha512-Oei9OH4tRh0YqU3GxhX79dM/mwVgvbZJaSNaRk+bshkj0S5cfHcgYakreBjrHwatXKbz+IoIdYLxrKim2MjW0Q==", + "license": "MIT" + }, + "node_modules/atomic-sleep": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/atomic-sleep/-/atomic-sleep-1.0.0.tgz", + "integrity": "sha512-kNOjDqAh7px0XWNI+4QbzoiR/nTkHAWNud2uvnJquD1/x5a7EQZMJT0AczqK0Qn67oY/TTQ1LbUKajZpp3I9tQ==", + "license": "MIT", + "engines": { + "node": ">=8.0.0" + } + }, + "node_modules/available-typed-arrays": { + "version": "1.0.7", + "resolved": "https://registry.npmjs.org/available-typed-arrays/-/available-typed-arrays-1.0.7.tgz", + "integrity": "sha512-wvUjBtSGN7+7SjNpq/9M2Tg350UZD3q62IFZLbRAR1bSMlCo1ZaeW+BJ+D090e4hIIZLBcTDWe4Mh4jvUDajzQ==", + "license": "MIT", + "dependencies": { + "possible-typed-array-names": "^1.0.0" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/balanced-match": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/balanced-match/-/balanced-match-1.0.2.tgz", + "integrity": "sha512-3oSeUO0TMV67hN1AmbXsK4yaqU7tjiHlbxRDZOpH0KW9+CeX4bRAaX0Anxt0tx2MrpRpWwQaPwIlISEJhYU5Pw==", + "dev": true, + "license": "MIT" + }, + "node_modules/base64-js": { + "version": "1.5.1", + "resolved": "https://registry.npmjs.org/base64-js/-/base64-js-1.5.1.tgz", + "integrity": "sha512-AKpaYlHn8t4SVbOHCy+b5+KKgvR4vrsD8vbvrbiQJps7fKDTkjkDry6ji0rUJjC0kzbNePLwzxq8iypo41qeWA==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/feross" + }, + { + "type": "patreon", + "url": "https://www.patreon.com/feross" + }, + { + "type": "consulting", + "url": "https://feross.org/support" + } + ], + "license": "MIT" + }, + "node_modules/binary-extensions": { + "version": "2.3.0", + "resolved": "https://registry.npmjs.org/binary-extensions/-/binary-extensions-2.3.0.tgz", + "integrity": "sha512-Ceh+7ox5qe7LJuLHoY0feh3pHuUDHAcRUeyL2VYghZwfpkNIy/+8Ocg0a3UuSoYzavmylwuLWQOf3hl0jjMMIw==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=8" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/bintrees": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/bintrees/-/bintrees-1.0.2.tgz", + "integrity": "sha512-VOMgTMwjAaUG580SXn3LacVgjurrbMme7ZZNYGSSV7mmtY6QQRh0Eg3pwIcntQ77DErK1L0NxkbetjcoXzVwKw==", + "license": "MIT" + }, + "node_modules/bl": { + "version": "1.2.3", + "resolved": "https://registry.npmjs.org/bl/-/bl-1.2.3.tgz", + "integrity": "sha512-pvcNpa0UU69UT341rO6AYy4FVAIkUHuZXRIWbq+zHnsVcRzDDjIAhGuuYoi0d//cwIwtt4pkpKycWEfjdV+vww==", + "license": "MIT", + "dependencies": { + "readable-stream": "^2.3.5", + "safe-buffer": "^5.1.1" + } + }, + "node_modules/body-parser": { + "version": "2.2.1", + "resolved": "https://registry.npmjs.org/body-parser/-/body-parser-2.2.1.tgz", + "integrity": "sha512-nfDwkulwiZYQIGwxdy0RUmowMhKcFVcYXUU7m4QlKYim1rUtg83xm2yjZ40QjDuc291AJjjeSc9b++AWHSgSHw==", + "license": "MIT", + "dependencies": { + "bytes": "^3.1.2", + "content-type": "^1.0.5", + "debug": "^4.4.3", + "http-errors": "^2.0.0", + "iconv-lite": "^0.7.0", + "on-finished": "^2.4.1", + "qs": "^6.14.0", + "raw-body": "^3.0.1", + "type-is": "^2.0.1" + }, + "engines": { + "node": ">=18" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" + } + }, + "node_modules/brace-expansion": { + "version": "1.1.12", + "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.12.tgz", + "integrity": "sha512-9T9UjW3r0UW5c1Q7GTwllptXwhvYmEzFhzMfZ9H7FQWt+uZePjZPjBP/W1ZEyZ1twGWom5/56TF4lPcqjnDHcg==", + "dev": true, + "license": "MIT", + "dependencies": { + "balanced-match": "^1.0.0", + "concat-map": "0.0.1" + } + }, + "node_modules/braces": { + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/braces/-/braces-3.0.3.tgz", + "integrity": "sha512-yQbXgO/OSZVD2IsiLlro+7Hf6Q18EJrKSEsdoMzKePKXct3gvD8oLcOQdIzGupr5Fj+EDe8gO/lxc1BzfMpxvA==", + "dev": true, + "license": "MIT", + "dependencies": { + "fill-range": "^7.1.1" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/buffer": { + "version": "5.7.1", + "resolved": "https://registry.npmjs.org/buffer/-/buffer-5.7.1.tgz", + "integrity": "sha512-EHcyIPBQ4BSGlvjB16k5KgAJ27CIsHY/2JBmCRReo48y9rQ3MaUzWX3KVlBa4U7MyX02HdVj0K7C3WaB3ju7FQ==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/feross" + }, + { + "type": "patreon", + "url": "https://www.patreon.com/feross" + }, + { + "type": "consulting", + "url": "https://feross.org/support" + } + ], + "license": "MIT", + "dependencies": { + "base64-js": "^1.3.1", + "ieee754": "^1.1.13" + } + }, + "node_modules/buffer-alloc": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/buffer-alloc/-/buffer-alloc-1.2.0.tgz", + "integrity": "sha512-CFsHQgjtW1UChdXgbyJGtnm+O/uLQeZdtbDo8mfUgYXCHSM1wgrVxXm6bSyrUuErEb+4sYVGCzASBRot7zyrow==", + "license": "MIT", + "dependencies": { + "buffer-alloc-unsafe": "^1.1.0", + "buffer-fill": "^1.0.0" + } + }, + "node_modules/buffer-alloc-unsafe": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/buffer-alloc-unsafe/-/buffer-alloc-unsafe-1.1.0.tgz", + "integrity": "sha512-TEM2iMIEQdJ2yjPJoSIsldnleVaAk1oW3DBVUykyOLsEsFmEc9kn+SFFPz+gl54KQNxlDnAwCXosOS9Okx2xAg==", + "license": "MIT" + }, + "node_modules/buffer-crc32": { + "version": "0.2.13", + "resolved": "https://registry.npmjs.org/buffer-crc32/-/buffer-crc32-0.2.13.tgz", + "integrity": "sha512-VO9Ht/+p3SN7SKWqcrgEzjGbRSJYTx+Q1pTQC0wrWqHx0vpJraQ6GtHx8tvcg1rlK1byhU5gccxgOgj7B0TDkQ==", + "license": "MIT", + "engines": { + "node": "*" + } + }, + "node_modules/buffer-fill": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/buffer-fill/-/buffer-fill-1.0.0.tgz", + "integrity": "sha512-T7zexNBwiiaCOGDg9xNX9PBmjrubblRkENuptryuI64URkXDFum9il/JGL8Lm8wYfAXpredVXXZz7eMHilimiQ==", + "license": "MIT" + }, + "node_modules/bytes": { + "version": "3.1.2", + "resolved": "https://registry.npmjs.org/bytes/-/bytes-3.1.2.tgz", + "integrity": "sha512-/Nf7TyzTx6S3yRJObOAV7956r8cr2+Oj8AC5dt8wSP3BQAoeX58NoHyCU8P8zGkNXStjTSi6fzO6F0pBdcYbEg==", + "license": "MIT", + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/call-bind": { + "version": "1.0.8", + "resolved": "https://registry.npmjs.org/call-bind/-/call-bind-1.0.8.tgz", + "integrity": "sha512-oKlSFMcMwpUg2ednkhQ454wfWiU/ul3CkJe/PEHcTKuiX6RpbehUiFMXu13HalGZxfUwCQzZG747YXBn1im9ww==", + "license": "MIT", + "dependencies": { + "call-bind-apply-helpers": "^1.0.0", + "es-define-property": "^1.0.0", + "get-intrinsic": "^1.2.4", + "set-function-length": "^1.2.2" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/call-bind-apply-helpers": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/call-bind-apply-helpers/-/call-bind-apply-helpers-1.0.2.tgz", + "integrity": "sha512-Sp1ablJ0ivDkSzjcaJdxEunN5/XvksFJ2sMBFfq6x0ryhQV/2b/KwFe21cMpmHtPOSij8K99/wSfoEuTObmuMQ==", + "license": "MIT", + "dependencies": { + "es-errors": "^1.3.0", + "function-bind": "^1.1.2" + }, + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/call-bound": { + "version": "1.0.4", + "resolved": "https://registry.npmjs.org/call-bound/-/call-bound-1.0.4.tgz", + "integrity": "sha512-+ys997U96po4Kx/ABpBCqhA9EuxJaQWDQg7295H4hBphv3IZg0boBKuwYpt4YXp6MZ5AmZQnU/tyMTlRpaSejg==", + "license": "MIT", + "dependencies": { + "call-bind-apply-helpers": "^1.0.2", + "get-intrinsic": "^1.3.0" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/chalk": { + "version": "4.1.2", + "resolved": "https://registry.npmjs.org/chalk/-/chalk-4.1.2.tgz", + "integrity": "sha512-oKnbhFyRIXpUuez8iBMmyEa4nbj4IOQyuhc/wy9kY7/WVPcwIO9VA668Pu8RkO7+0G76SLROeyw9CpQ061i4mA==", + "license": "MIT", + "dependencies": { + "ansi-styles": "^4.1.0", + "supports-color": "^7.1.0" + }, + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/chalk/chalk?sponsor=1" + } + }, + "node_modules/chokidar": { + "version": "3.6.0", + "resolved": "https://registry.npmjs.org/chokidar/-/chokidar-3.6.0.tgz", + "integrity": "sha512-7VT13fmjotKpGipCW9JEQAusEPE+Ei8nl6/g4FBAmIm0GOOLMua9NDDo/DWp0ZAxCr3cPq5ZpBqmPAQgDda2Pw==", + "dev": true, + "license": "MIT", + "dependencies": { + "anymatch": "~3.1.2", + "braces": "~3.0.2", + "glob-parent": "~5.1.2", + "is-binary-path": "~2.1.0", + "is-glob": "~4.0.1", + "normalize-path": "~3.0.0", + "readdirp": "~3.6.0" + }, + "engines": { + "node": ">= 8.10.0" + }, + "funding": { + "url": "https://paulmillr.com/funding/" + }, + "optionalDependencies": { + "fsevents": "~2.3.2" + } + }, + "node_modules/color-convert": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-2.0.1.tgz", + "integrity": "sha512-RRECPsj7iu/xb5oKYcsFHSppFNnsj/52OVTRKb4zP5onXwVF3zVmmToNcOfGC+CRDpfK/U584fMg38ZHCaElKQ==", + "license": "MIT", + "dependencies": { + "color-name": "~1.1.4" + }, + "engines": { + "node": ">=7.0.0" + } + }, + "node_modules/color-name": { + "version": "1.1.4", + "resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.4.tgz", + "integrity": "sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA==", + "license": "MIT" + }, + "node_modules/colorette": { + "version": "2.0.20", + "resolved": "https://registry.npmjs.org/colorette/-/colorette-2.0.20.tgz", + "integrity": "sha512-IfEDxwoWIjkeXL1eXcDiow4UbKjhLdq6/EuSVR9GMN7KVH3r9gQ83e73hsz1Nd1T3ijd5xv1wcWRYO+D6kCI2w==", + "license": "MIT" + }, + "node_modules/combined-stream": { + "version": "1.0.8", + "resolved": "https://registry.npmjs.org/combined-stream/-/combined-stream-1.0.8.tgz", + "integrity": "sha512-FQN4MRfuJeHf7cBbBMJFXhKSDq+2kAArBlmRBvcvFE5BB1HZKXtSFASDhdlz9zOYwxh8lDdnvmMOe/+5cdoEdg==", + "license": "MIT", + "dependencies": { + "delayed-stream": "~1.0.0" + }, + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/commander": { + "version": "2.20.3", + "resolved": "https://registry.npmjs.org/commander/-/commander-2.20.3.tgz", + "integrity": "sha512-GpVkmM8vF2vQUkj2LvZmD35JxeJOLCwJ9cUkugyk2nuhbv3+mJvpLYYt+0+USMxE+oj+ey/lJEnhZw75x/OMcQ==", + "license": "MIT" + }, + "node_modules/concat-map": { + "version": "0.0.1", + "resolved": "https://registry.npmjs.org/concat-map/-/concat-map-0.0.1.tgz", + "integrity": "sha512-/Srv4dswyQNBfohGpz9o6Yb3Gz3SrUDqBH5rTuhGR7ahtlbYKnVxw2bCFMRljaA7EXHaXZ8wsHdodFvbkhKmqg==", + "dev": true, + "license": "MIT" + }, + "node_modules/content-disposition": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/content-disposition/-/content-disposition-1.0.1.tgz", + "integrity": "sha512-oIXISMynqSqm241k6kcQ5UwttDILMK4BiurCfGEREw6+X9jkkpEe5T9FZaApyLGGOnFuyMWZpdolTXMtvEJ08Q==", + "license": "MIT", + "engines": { + "node": ">=18" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" + } + }, + "node_modules/content-type": { + "version": "1.0.5", + "resolved": "https://registry.npmjs.org/content-type/-/content-type-1.0.5.tgz", + "integrity": "sha512-nTjqfcBFEipKdXCv4YDQWCfmcLZKm81ldF0pAopTvyrFGVbcR6P/VAAd5G7N+0tTr8QqiU0tFadD6FK4NtJwOA==", + "license": "MIT", + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/cookie": { + "version": "0.7.2", + "resolved": "https://registry.npmjs.org/cookie/-/cookie-0.7.2.tgz", + "integrity": "sha512-yki5XnKuf750l50uGTllt6kKILY4nQ1eNIQatoXEByZ5dWgnKqbnqmTrBE5B4N7lrMJKQ2ytWMiTO2o0v6Ew/w==", + "license": "MIT", + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/cookie-signature": { + "version": "1.2.2", + "resolved": "https://registry.npmjs.org/cookie-signature/-/cookie-signature-1.2.2.tgz", + "integrity": "sha512-D76uU73ulSXrD1UXF4KE2TMxVVwhsnCgfAyTg9k8P6KGZjlXKrOLe4dJQKI3Bxi5wjesZoFXJWElNWBjPZMbhg==", + "license": "MIT", + "engines": { + "node": ">=6.6.0" + } + }, + "node_modules/core-util-is": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/core-util-is/-/core-util-is-1.0.3.tgz", + "integrity": "sha512-ZQBvi1DcpJ4GDqanjucZ2Hj3wEO5pZDS89BWbkcrvdxksJorwUDDZamX9ldFkp9aw2lmBDLgkObEA4DWNJ9FYQ==", + "license": "MIT" + }, + "node_modules/cors": { + "version": "2.8.5", + "resolved": "https://registry.npmjs.org/cors/-/cors-2.8.5.tgz", + "integrity": "sha512-KIHbLJqu73RGr/hnbrO9uBeixNGuvSQjul/jdFvS/KFSIH1hWVd1ng7zOHx+YrEfInLG7q4n6GHQ9cDtxv/P6g==", + "license": "MIT", + "dependencies": { + "object-assign": "^4", + "vary": "^1" + }, + "engines": { + "node": ">= 0.10" + } + }, + "node_modules/create-require": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/create-require/-/create-require-1.1.1.tgz", + "integrity": "sha512-dcKFX3jn0MpIaXjisoRvexIJVEKzaq7z2rZKxf+MSr9TkdmHmsU4m2lcLojrj/FHl8mk5VxMmYA+ftRkP/3oKQ==", + "dev": true, + "license": "MIT" + }, + "node_modules/dateformat": { + "version": "4.6.3", + "resolved": "https://registry.npmjs.org/dateformat/-/dateformat-4.6.3.tgz", + "integrity": "sha512-2P0p0pFGzHS5EMnhdxQi7aJN+iMheud0UhG4dlE1DLAlvL8JHjJJTX/CSm4JXwV0Ka5nGk3zC5mcb5bUQUxxMA==", + "license": "MIT", + "engines": { + "node": "*" + } + }, + "node_modules/debug": { + "version": "4.4.3", + "resolved": "https://registry.npmjs.org/debug/-/debug-4.4.3.tgz", + "integrity": "sha512-RGwwWnwQvkVfavKVt22FGLw+xYSdzARwm0ru6DhTVA3umU5hZc28V3kO4stgYryrTlLpuvgI9GiijltAjNbcqA==", + "license": "MIT", + "dependencies": { + "ms": "^2.1.3" + }, + "engines": { + "node": ">=6.0" + }, + "peerDependenciesMeta": { + "supports-color": { + "optional": true + } + } + }, + "node_modules/decompress": { + "version": "4.2.1", + "resolved": "https://registry.npmjs.org/decompress/-/decompress-4.2.1.tgz", + "integrity": "sha512-e48kc2IjU+2Zw8cTb6VZcJQ3lgVbS4uuB1TfCHbiZIP/haNXm+SVyhu+87jts5/3ROpd82GSVCoNs/z8l4ZOaQ==", + "license": "MIT", + "dependencies": { + "decompress-tar": "^4.0.0", + "decompress-tarbz2": "^4.0.0", + "decompress-targz": "^4.0.0", + "decompress-unzip": "^4.0.1", + "graceful-fs": "^4.1.10", + "make-dir": "^1.0.0", + "pify": "^2.3.0", + "strip-dirs": "^2.0.0" + }, + "engines": { + "node": ">=4" + } + }, + "node_modules/decompress-tar": { + "version": "4.1.1", + "resolved": "https://registry.npmjs.org/decompress-tar/-/decompress-tar-4.1.1.tgz", + "integrity": "sha512-JdJMaCrGpB5fESVyxwpCx4Jdj2AagLmv3y58Qy4GE6HMVjWz1FeVQk1Ct4Kye7PftcdOo/7U7UKzYBJgqnGeUQ==", + "license": "MIT", + "dependencies": { + "file-type": "^5.2.0", + "is-stream": "^1.1.0", + "tar-stream": "^1.5.2" + }, + "engines": { + "node": ">=4" + } + }, + "node_modules/decompress-tarbz2": { + "version": "4.1.1", + "resolved": "https://registry.npmjs.org/decompress-tarbz2/-/decompress-tarbz2-4.1.1.tgz", + "integrity": "sha512-s88xLzf1r81ICXLAVQVzaN6ZmX4A6U4z2nMbOwobxkLoIIfjVMBg7TeguTUXkKeXni795B6y5rnvDw7rxhAq9A==", + "license": "MIT", + "dependencies": { + "decompress-tar": "^4.1.0", + "file-type": "^6.1.0", + "is-stream": "^1.1.0", + "seek-bzip": "^1.0.5", + "unbzip2-stream": "^1.0.9" + }, + "engines": { + "node": ">=4" + } + }, + "node_modules/decompress-tarbz2/node_modules/file-type": { + "version": "6.2.0", + "resolved": "https://registry.npmjs.org/file-type/-/file-type-6.2.0.tgz", + "integrity": "sha512-YPcTBDV+2Tm0VqjybVd32MHdlEGAtuxS3VAYsumFokDSMG+ROT5wawGlnHDoz7bfMcMDt9hxuXvXwoKUx2fkOg==", + "license": "MIT", + "engines": { + "node": ">=4" + } + }, + "node_modules/decompress-targz": { + "version": "4.1.1", + "resolved": "https://registry.npmjs.org/decompress-targz/-/decompress-targz-4.1.1.tgz", + "integrity": "sha512-4z81Znfr6chWnRDNfFNqLwPvm4db3WuZkqV+UgXQzSngG3CEKdBkw5jrv3axjjL96glyiiKjsxJG3X6WBZwX3w==", + "license": "MIT", + "dependencies": { + "decompress-tar": "^4.1.1", + "file-type": "^5.2.0", + "is-stream": "^1.1.0" + }, + "engines": { + "node": ">=4" + } + }, + "node_modules/decompress-unzip": { + "version": "4.0.1", + "resolved": "https://registry.npmjs.org/decompress-unzip/-/decompress-unzip-4.0.1.tgz", + "integrity": "sha512-1fqeluvxgnn86MOh66u8FjbtJpAFv5wgCT9Iw8rcBqQcCo5tO8eiJw7NNTrvt9n4CRBVq7CstiS922oPgyGLrw==", + "license": "MIT", + "dependencies": { + "file-type": "^3.8.0", + "get-stream": "^2.2.0", + "pify": "^2.3.0", + "yauzl": "^2.4.2" + }, + "engines": { + "node": ">=4" + } + }, + "node_modules/decompress-unzip/node_modules/file-type": { + "version": "3.9.0", + "resolved": "https://registry.npmjs.org/file-type/-/file-type-3.9.0.tgz", + "integrity": "sha512-RLoqTXE8/vPmMuTI88DAzhMYC99I8BWv7zYP4A1puo5HIjEJ5EX48ighy4ZyKMG9EDXxBgW6e++cn7d1xuFghA==", + "license": "MIT", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/define-data-property": { + "version": "1.1.4", + "resolved": "https://registry.npmjs.org/define-data-property/-/define-data-property-1.1.4.tgz", + "integrity": "sha512-rBMvIzlpA8v6E+SJZoo++HAYqsLrkg7MSfIinMPFhmkorw7X+dOXVJQs+QT69zGkzMyfDnIMN2Wid1+NbL3T+A==", + "license": "MIT", + "dependencies": { + "es-define-property": "^1.0.0", + "es-errors": "^1.3.0", + "gopd": "^1.0.1" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/delayed-stream": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/delayed-stream/-/delayed-stream-1.0.0.tgz", + "integrity": "sha512-ZySD7Nf91aLB0RxL4KGrKHBXl7Eds1DAmEdcoVawXnLD7SDhpNgtuII2aAkg7a7QS41jxPSZ17p4VdGnMHk3MQ==", + "license": "MIT", + "engines": { + "node": ">=0.4.0" + } + }, + "node_modules/depd": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/depd/-/depd-2.0.0.tgz", + "integrity": "sha512-g7nH6P6dyDioJogAAGprGpCtVImJhpPk/roCzdb3fIh61/s/nPsfR6onyMwkCAR/OlC3yBC0lESvUoQEAssIrw==", + "license": "MIT", + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/diff": { + "version": "4.0.2", + "resolved": "https://registry.npmjs.org/diff/-/diff-4.0.2.tgz", + "integrity": "sha512-58lmxKSA4BNyLz+HHMUzlOEpg09FV+ev6ZMe3vJihgdxzgcwZ8VoEEPmALCZG9LmqfVoNMMKpttIYTVG6uDY7A==", + "dev": true, + "license": "BSD-3-Clause", + "engines": { + "node": ">=0.3.1" + } + }, + "node_modules/dotenv": { + "version": "16.6.1", + "resolved": "https://registry.npmjs.org/dotenv/-/dotenv-16.6.1.tgz", + "integrity": "sha512-uBq4egWHTcTt33a72vpSG0z3HnPuIl6NqYcTrKEg2azoEyl2hpW0zqlxysq2pK9HlDIHyHyakeYaYnSAwd8bow==", + "license": "BSD-2-Clause", + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://dotenvx.com" + } + }, + "node_modules/dunder-proto": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/dunder-proto/-/dunder-proto-1.0.1.tgz", + "integrity": "sha512-KIN/nDJBQRcXw0MLVhZE9iQHmG68qAVIBg9CqmUYjmQIhgij9U5MFvrqkUL5FbtyyzZuOeOt0zdeRe4UY7ct+A==", + "license": "MIT", + "dependencies": { + "call-bind-apply-helpers": "^1.0.1", + "es-errors": "^1.3.0", + "gopd": "^1.2.0" + }, + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/ee-first": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/ee-first/-/ee-first-1.1.1.tgz", + "integrity": "sha512-WMwm9LhRUo+WUaRN+vRuETqG89IgZphVSNkdFgeb6sS/E4OrDIN7t48CAewSHXc6C8lefD8KKfr5vY61brQlow==", + "license": "MIT" + }, + "node_modules/encodeurl": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/encodeurl/-/encodeurl-2.0.0.tgz", + "integrity": "sha512-Q0n9HRi4m6JuGIV1eFlmvJB7ZEVxu93IrMyiMsGC0lrMJMWzRgx6WGquyfQgZVb31vhGgXnfmPNNXmxnOkRBrg==", + "license": "MIT", + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/end-of-stream": { + "version": "1.4.5", + "resolved": "https://registry.npmjs.org/end-of-stream/-/end-of-stream-1.4.5.tgz", + "integrity": "sha512-ooEGc6HP26xXq/N+GCGOT0JKCLDGrq2bQUZrQ7gyrJiZANJ/8YDTxTpQBXGMn+WbIQXNVpyWymm7KYVICQnyOg==", + "license": "MIT", + "dependencies": { + "once": "^1.4.0" + } + }, + "node_modules/es-define-property": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/es-define-property/-/es-define-property-1.0.1.tgz", + "integrity": "sha512-e3nRfgfUZ4rNGL232gUgX06QNyyez04KdjFrF+LTRoOXmrOgFKDg4BCdsjW8EnT69eqdYGmRpJwiPVYNrCaW3g==", + "license": "MIT", + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/es-errors": { + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/es-errors/-/es-errors-1.3.0.tgz", + "integrity": "sha512-Zf5H2Kxt2xjTvbJvP2ZWLEICxA6j+hAmMzIlypy4xcBg1vKVnx89Wy0GbS+kf5cwCVFFzdCFh2XSCFNULS6csw==", + "license": "MIT", + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/es-object-atoms": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/es-object-atoms/-/es-object-atoms-1.1.1.tgz", + "integrity": "sha512-FGgH2h8zKNim9ljj7dankFPcICIK9Cp5bm+c2gQSYePhpaG5+esrLODihIorn+Pe6FGJzWhXQotPv73jTaldXA==", + "license": "MIT", + "dependencies": { + "es-errors": "^1.3.0" + }, + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/es-set-tostringtag": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/es-set-tostringtag/-/es-set-tostringtag-2.1.0.tgz", + "integrity": "sha512-j6vWzfrGVfyXxge+O0x5sh6cvxAog0a/4Rdd2K36zCMV5eJ+/+tOAngRO8cODMNWbVRdVlmGZQL2YS3yR8bIUA==", + "license": "MIT", + "dependencies": { + "es-errors": "^1.3.0", + "get-intrinsic": "^1.2.6", + "has-tostringtag": "^1.0.2", + "hasown": "^2.0.2" + }, + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/escape-html": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/escape-html/-/escape-html-1.0.3.tgz", + "integrity": "sha512-NiSupZ4OeuGwr68lGIeym/ksIZMJodUGOSCZ/FSnTxcrekbvqrgdUxlJOMpijaKZVjAJrWrGs/6Jy8OMuyj9ow==", + "license": "MIT" + }, + "node_modules/etag": { + "version": "1.8.1", + "resolved": "https://registry.npmjs.org/etag/-/etag-1.8.1.tgz", + "integrity": "sha512-aIL5Fx7mawVa300al2BnEE4iNvo1qETxLrPI/o05L7z6go7fCw1J6EQmbK4FmJ2AS7kgVF/KEZWufBfdClMcPg==", + "license": "MIT", + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/event-target-shim": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/event-target-shim/-/event-target-shim-5.0.1.tgz", + "integrity": "sha512-i/2XbnSz/uxRCU6+NdVJgKWDTM427+MqYbkQzD321DuCQJUqOuJKIA0IM2+W2xtYHdKOmZ4dR6fExsd4SXL+WQ==", + "license": "MIT", + "engines": { + "node": ">=6" + } + }, + "node_modules/express": { + "version": "5.2.1", + "resolved": "https://registry.npmjs.org/express/-/express-5.2.1.tgz", + "integrity": "sha512-hIS4idWWai69NezIdRt2xFVofaF4j+6INOpJlVOLDO8zXGpUVEVzIYk12UUi2JzjEzWL3IOAxcTubgz9Po0yXw==", + "license": "MIT", + "dependencies": { + "accepts": "^2.0.0", + "body-parser": "^2.2.1", + "content-disposition": "^1.0.0", + "content-type": "^1.0.5", + "cookie": "^0.7.1", + "cookie-signature": "^1.2.1", + "debug": "^4.4.0", + "depd": "^2.0.0", + "encodeurl": "^2.0.0", + "escape-html": "^1.0.3", + "etag": "^1.8.1", + "finalhandler": "^2.1.0", + "fresh": "^2.0.0", + "http-errors": "^2.0.0", + "merge-descriptors": "^2.0.0", + "mime-types": "^3.0.0", + "on-finished": "^2.4.1", + "once": "^1.4.0", + "parseurl": "^1.3.3", + "proxy-addr": "^2.0.7", + "qs": "^6.14.0", + "range-parser": "^1.2.1", + "router": "^2.2.0", + "send": "^1.1.0", + "serve-static": "^2.2.0", + "statuses": "^2.0.1", + "type-is": "^2.0.1", + "vary": "^1.1.2" + }, + "engines": { + "node": ">= 18" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" + } + }, + "node_modules/express-validator": { + "version": "7.3.1", + "resolved": "https://registry.npmjs.org/express-validator/-/express-validator-7.3.1.tgz", + "integrity": "sha512-IGenaSf+DnWc69lKuqlRE9/i/2t5/16VpH5bXoqdxWz1aCpRvEdrBuu1y95i/iL5QP8ZYVATiwLFhwk3EDl5vg==", + "license": "MIT", + "dependencies": { + "lodash": "^4.17.21", + "validator": "~13.15.23" + }, + "engines": { + "node": ">= 8.0.0" + } + }, + "node_modules/fast-copy": { + "version": "4.0.1", + "resolved": "https://registry.npmjs.org/fast-copy/-/fast-copy-4.0.1.tgz", + "integrity": "sha512-+uUOQlhsaswsizHFmEFAQhB3lSiQ+lisxl50N6ZP0wywlZeWsIESxSi9ftPEps8UGfiBzyYP7x27zA674WUvXw==", + "license": "MIT" + }, + "node_modules/fast-deep-equal": { + "version": "3.1.3", + "resolved": "https://registry.npmjs.org/fast-deep-equal/-/fast-deep-equal-3.1.3.tgz", + "integrity": "sha512-f3qQ9oQy9j2AhBe/H9VC91wLmKBCCU/gDOnKNAYG5hswO7BLKj09Hc5HYNz9cGI++xlpDCIgDaitVs03ATR84Q==", + "license": "MIT" + }, + "node_modules/fast-safe-stringify": { + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/fast-safe-stringify/-/fast-safe-stringify-2.1.1.tgz", + "integrity": "sha512-W+KJc2dmILlPplD/H4K9l9LcAHAfPtP6BY84uVLXQ6Evcz9Lcg33Y2z1IVblT6xdY54PXYVHEv+0Wpq8Io6zkA==", + "license": "MIT" + }, + "node_modules/fast-uri": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/fast-uri/-/fast-uri-3.1.0.tgz", + "integrity": "sha512-iPeeDKJSWf4IEOasVVrknXpaBV0IApz/gp7S2bb7Z4Lljbl2MGJRqInZiUrQwV16cpzw/D3S5j5Julj/gT52AA==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/fastify" + }, + { + "type": "opencollective", + "url": "https://opencollective.com/fastify" + } + ], + "license": "BSD-3-Clause" + }, + "node_modules/fd-slicer": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/fd-slicer/-/fd-slicer-1.1.0.tgz", + "integrity": "sha512-cE1qsB/VwyQozZ+q1dGxR8LBYNZeofhEdUNGSMbQD3Gw2lAzX9Zb3uIU6Ebc/Fmyjo9AWWfnn0AUCHqtevs/8g==", + "license": "MIT", + "dependencies": { + "pend": "~1.2.0" + } + }, + "node_modules/file-type": { + "version": "5.2.0", + "resolved": "https://registry.npmjs.org/file-type/-/file-type-5.2.0.tgz", + "integrity": "sha512-Iq1nJ6D2+yIO4c8HHg4fyVb8mAJieo1Oloy1mLLaB2PvezNedhBVm+QU7g0qM42aiMbRXTxKKwGD17rjKNJYVQ==", + "license": "MIT", + "engines": { + "node": ">=4" + } + }, + "node_modules/fill-range": { + "version": "7.1.1", + "resolved": "https://registry.npmjs.org/fill-range/-/fill-range-7.1.1.tgz", + "integrity": "sha512-YsGpe3WHLK8ZYi4tWDg2Jy3ebRz2rXowDxnld4bkQB00cc/1Zw9AWnC0i9ztDJitivtQvaI9KaLyKrc+hBW0yg==", + "dev": true, + "license": "MIT", + "dependencies": { + "to-regex-range": "^5.0.1" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/finalhandler": { + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/finalhandler/-/finalhandler-2.1.1.tgz", + "integrity": "sha512-S8KoZgRZN+a5rNwqTxlZZePjT/4cnm0ROV70LedRHZ0p8u9fRID0hJUZQpkKLzro8LfmC8sx23bY6tVNxv8pQA==", + "license": "MIT", + "dependencies": { + "debug": "^4.4.0", + "encodeurl": "^2.0.0", + "escape-html": "^1.0.3", + "on-finished": "^2.4.1", + "parseurl": "^1.3.3", + "statuses": "^2.0.1" + }, + "engines": { + "node": ">= 18.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" + } + }, + "node_modules/for-each": { + "version": "0.3.5", + "resolved": "https://registry.npmjs.org/for-each/-/for-each-0.3.5.tgz", + "integrity": "sha512-dKx12eRCVIzqCxFGplyFKJMPvLEWgmNtUrpTiJIR5u97zEhRG8ySrtboPHZXx7daLxQVrl643cTzbab2tkQjxg==", + "license": "MIT", + "dependencies": { + "is-callable": "^1.2.7" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/form-data": { + "version": "4.0.5", + "resolved": "https://registry.npmjs.org/form-data/-/form-data-4.0.5.tgz", + "integrity": "sha512-8RipRLol37bNs2bhoV67fiTEvdTrbMUYcFTiy3+wuuOnUog2QBHCZWXDRijWQfAkhBj2Uf5UnVaiWwA5vdd82w==", + "license": "MIT", + "dependencies": { + "asynckit": "^0.4.0", + "combined-stream": "^1.0.8", + "es-set-tostringtag": "^2.1.0", + "hasown": "^2.0.2", + "mime-types": "^2.1.12" + }, + "engines": { + "node": ">= 6" + } + }, + "node_modules/form-data-encoder": { + "version": "1.7.2", + "resolved": "https://registry.npmjs.org/form-data-encoder/-/form-data-encoder-1.7.2.tgz", + "integrity": "sha512-qfqtYan3rxrnCk1VYaA4H+Ms9xdpPqvLZa6xmMgFvhO32x7/3J/ExcTd6qpxM0vH2GdMI+poehyBZvqfMTto8A==", + "license": "MIT" + }, + "node_modules/form-data/node_modules/mime-db": { + "version": "1.52.0", + "resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.52.0.tgz", + "integrity": "sha512-sPU4uV7dYlvtWJxwwxHD0PuihVNiE7TyAbQ5SWxDCB9mUYvOgroQOwYQQOKPJ8CIbE+1ETVlOoK1UC2nU3gYvg==", + "license": "MIT", + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/form-data/node_modules/mime-types": { + "version": "2.1.35", + "resolved": "https://registry.npmjs.org/mime-types/-/mime-types-2.1.35.tgz", + "integrity": "sha512-ZDY+bPm5zTTF+YpCrAU9nK0UgICYPT0QtT1NZWFv4s++TNkcgVaT0g6+4R2uI4MjQjzysHB1zxuWL50hzaeXiw==", + "license": "MIT", + "dependencies": { + "mime-db": "1.52.0" + }, + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/formdata-node": { + "version": "4.4.1", + "resolved": "https://registry.npmjs.org/formdata-node/-/formdata-node-4.4.1.tgz", + "integrity": "sha512-0iirZp3uVDjVGt9p49aTaqjk84TrglENEDuqfdlZQ1roC9CWlPk6Avf8EEnZNcAqPonwkG35x4n3ww/1THYAeQ==", + "license": "MIT", + "dependencies": { + "node-domexception": "1.0.0", + "web-streams-polyfill": "4.0.0-beta.3" + }, + "engines": { + "node": ">= 12.20" + } + }, + "node_modules/forwarded": { + "version": "0.2.0", + "resolved": "https://registry.npmjs.org/forwarded/-/forwarded-0.2.0.tgz", + "integrity": "sha512-buRG0fpBtRHSTCOASe6hD258tEubFoRLb4ZNA6NxMVHNw2gOcwHo9wyablzMzOA5z9xA9L1KNjk/Nt6MT9aYow==", + "license": "MIT", + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/fresh": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/fresh/-/fresh-2.0.0.tgz", + "integrity": "sha512-Rx/WycZ60HOaqLKAi6cHRKKI7zxWbJ31MhntmtwMoaTeF7XFH9hhBp8vITaMidfljRQ6eYWCKkaTK+ykVJHP2A==", + "license": "MIT", + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/fs-constants": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/fs-constants/-/fs-constants-1.0.0.tgz", + "integrity": "sha512-y6OAwoSIf7FyjMIv94u+b5rdheZEjzR63GTyZJm5qh4Bi+2YgwLCcI/fPFZkL5PSixOt6ZNKm+w+Hfp/Bciwow==", + "license": "MIT" + }, + "node_modules/fsevents": { + "version": "2.3.3", + "resolved": "https://registry.npmjs.org/fsevents/-/fsevents-2.3.3.tgz", + "integrity": "sha512-5xoDfX+fL7faATnagmWPpbFtwh/R77WmMMqqHGS65C3vvB0YHrgF+B1YmZ3441tMj5n63k0212XNoJwzlhffQw==", + "dev": true, + "hasInstallScript": true, + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": "^8.16.0 || ^10.6.0 || >=11.0.0" + } + }, + "node_modules/function-bind": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/function-bind/-/function-bind-1.1.2.tgz", + "integrity": "sha512-7XHNxH7qX9xG5mIwxkhumTox/MIRNcOgDrxWsMt2pAr23WHp6MrRlN7FBSFpCpr+oVO0F744iUgR82nJMfG2SA==", + "license": "MIT", + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/get-intrinsic": { + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/get-intrinsic/-/get-intrinsic-1.3.0.tgz", + "integrity": "sha512-9fSjSaos/fRIVIp+xSJlE6lfwhES7LNtKaCBIamHsjr2na1BiABJPo0mOjjz8GJDURarmCPGqaiVg5mfjb98CQ==", + "license": "MIT", + "dependencies": { + "call-bind-apply-helpers": "^1.0.2", + "es-define-property": "^1.0.1", + "es-errors": "^1.3.0", + "es-object-atoms": "^1.1.1", + "function-bind": "^1.1.2", + "get-proto": "^1.0.1", + "gopd": "^1.2.0", + "has-symbols": "^1.1.0", + "hasown": "^2.0.2", + "math-intrinsics": "^1.1.0" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/get-proto": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/get-proto/-/get-proto-1.0.1.tgz", + "integrity": "sha512-sTSfBjoXBp89JvIKIefqw7U2CCebsc74kiY6awiGogKtoSGbgjYE/G/+l9sF3MWFPNc9IcoOC4ODfKHfxFmp0g==", + "license": "MIT", + "dependencies": { + "dunder-proto": "^1.0.1", + "es-object-atoms": "^1.0.0" + }, + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/get-stream": { + "version": "2.3.1", + "resolved": "https://registry.npmjs.org/get-stream/-/get-stream-2.3.1.tgz", + "integrity": "sha512-AUGhbbemXxrZJRD5cDvKtQxLuYaIbNtDTK8YqupCI393Q2KSTreEsLUN3ZxAWFGiKTzL6nKuzfcIvieflUX9qA==", + "license": "MIT", + "dependencies": { + "object-assign": "^4.0.1", + "pinkie-promise": "^2.0.0" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/glob-parent": { + "version": "5.1.2", + "resolved": "https://registry.npmjs.org/glob-parent/-/glob-parent-5.1.2.tgz", + "integrity": "sha512-AOIgSQCepiJYwP3ARnGx+5VnTu2HBYdzbGP45eLw1vr3zB3vZLeyed1sC9hnbcOc9/SrMyM5RPQrkGz4aS9Zow==", + "dev": true, + "license": "ISC", + "dependencies": { + "is-glob": "^4.0.1" + }, + "engines": { + "node": ">= 6" + } + }, + "node_modules/gopd": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/gopd/-/gopd-1.2.0.tgz", + "integrity": "sha512-ZUKRh6/kUFoAiTAtTYPZJ3hw9wNxx+BIBOijnlG9PnrJsCcSjs1wyyD6vJpaYtgnzDrKYRSqf3OO6Rfa93xsRg==", + "license": "MIT", + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/graceful-fs": { + "version": "4.2.11", + "resolved": "https://registry.npmjs.org/graceful-fs/-/graceful-fs-4.2.11.tgz", + "integrity": "sha512-RbJ5/jmFcNNCcDV5o9eTnBLJ/HszWV0P73bc+Ff4nS/rJj+YaS6IGyiOL0VoBYX+l1Wrl3k63h/KrH+nhJ0XvQ==", + "license": "ISC" + }, + "node_modules/groq-sdk": { + "version": "0.7.0", + "resolved": "https://registry.npmjs.org/groq-sdk/-/groq-sdk-0.7.0.tgz", + "integrity": "sha512-OgPqrRtti5MjEVclR8sgBHrhSkTLdFCmi47yrEF29uJZaiCkX3s7bXpnMhq8Lwoe1f4AwgC0qGOeHXpeSgu5lg==", + "license": "Apache-2.0", + "dependencies": { + "@types/node": "^18.11.18", + "@types/node-fetch": "^2.6.4", + "abort-controller": "^3.0.0", + "agentkeepalive": "^4.2.1", + "form-data-encoder": "1.7.2", + "formdata-node": "^4.3.2", + "node-fetch": "^2.6.7" + } + }, + "node_modules/groq-sdk/node_modules/@types/node": { + "version": "18.19.130", + "resolved": "https://registry.npmjs.org/@types/node/-/node-18.19.130.tgz", + "integrity": "sha512-GRaXQx6jGfL8sKfaIDD6OupbIHBr9jv7Jnaml9tB7l4v068PAOXqfcujMMo5PhbIs6ggR1XODELqahT2R8v0fg==", + "license": "MIT", + "dependencies": { + "undici-types": "~5.26.4" + } + }, + "node_modules/groq-sdk/node_modules/undici-types": { + "version": "5.26.5", + "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-5.26.5.tgz", + "integrity": "sha512-JlCMO+ehdEIKqlFxk6IfVoAUVmgz7cU7zD/h9XZ0qzeosSHmUJVOzSQvvYSYWXkFXC+IfLKSIffhv0sVZup6pA==", + "license": "MIT" + }, + "node_modules/has-flag": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/has-flag/-/has-flag-4.0.0.tgz", + "integrity": "sha512-EykJT/Q1KjTWctppgIAgfSO0tKVuZUjhgMr17kqTumMl6Afv3EISleU7qZUzoXDFTAHTDC4NOoG/ZxU3EvlMPQ==", + "license": "MIT", + "engines": { + "node": ">=8" + } + }, + "node_modules/has-property-descriptors": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/has-property-descriptors/-/has-property-descriptors-1.0.2.tgz", + "integrity": "sha512-55JNKuIW+vq4Ke1BjOTjM2YctQIvCT7GFzHwmfZPGo5wnrgkid0YQtnAleFSqumZm4az3n2BS+erby5ipJdgrg==", + "license": "MIT", + "dependencies": { + "es-define-property": "^1.0.0" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/has-symbols": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/has-symbols/-/has-symbols-1.1.0.tgz", + "integrity": "sha512-1cDNdwJ2Jaohmb3sg4OmKaMBwuC48sYni5HUw2DvsC8LjGTLK9h+eb1X6RyuOHe4hT0ULCW68iomhjUoKUqlPQ==", + "license": "MIT", + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/has-tostringtag": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/has-tostringtag/-/has-tostringtag-1.0.2.tgz", + "integrity": "sha512-NqADB8VjPFLM2V0VvHUewwwsw0ZWBaIdgo+ieHtK3hasLz4qeCRjYcqfB6AQrBggRKppKF8L52/VqdVsO47Dlw==", + "license": "MIT", + "dependencies": { + "has-symbols": "^1.0.3" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/hasown": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/hasown/-/hasown-2.0.2.tgz", + "integrity": "sha512-0hJU9SCPvmMzIBdZFqNPXWa6dqh7WdH0cII9y+CyS8rG3nL48Bclra9HmKhVVUHyPWNH5Y7xDwAB7bfgSjkUMQ==", + "license": "MIT", + "dependencies": { + "function-bind": "^1.1.2" + }, + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/help-me": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/help-me/-/help-me-5.0.0.tgz", + "integrity": "sha512-7xgomUX6ADmcYzFik0HzAxh/73YlKR9bmFzf51CZwR+b6YtzU2m0u49hQCqV6SvlqIqsaxovfwdvbnsw3b/zpg==", + "license": "MIT" + }, + "node_modules/http-errors": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/http-errors/-/http-errors-2.0.1.tgz", + "integrity": "sha512-4FbRdAX+bSdmo4AUFuS0WNiPz8NgFt+r8ThgNWmlrjQjt1Q7ZR9+zTlce2859x4KSXrwIsaeTqDoKQmtP8pLmQ==", + "license": "MIT", + "dependencies": { + "depd": "~2.0.0", + "inherits": "~2.0.4", + "setprototypeof": "~1.2.0", + "statuses": "~2.0.2", + "toidentifier": "~1.0.1" + }, + "engines": { + "node": ">= 0.8" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" + } + }, + "node_modules/humanize-ms": { + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/humanize-ms/-/humanize-ms-1.2.1.tgz", + "integrity": "sha512-Fl70vYtsAFb/C06PTS9dZBo7ihau+Tu/DNCk/OyHhea07S+aeMWpFFkUaXRa8fI+ScZbEI8dfSxwY7gxZ9SAVQ==", + "license": "MIT", + "dependencies": { + "ms": "^2.0.0" + } + }, + "node_modules/iconv-lite": { + "version": "0.7.1", + "resolved": "https://registry.npmjs.org/iconv-lite/-/iconv-lite-0.7.1.tgz", + "integrity": "sha512-2Tth85cXwGFHfvRgZWszZSvdo+0Xsqmw8k8ZwxScfcBneNUraK+dxRxRm24nszx80Y0TVio8kKLt5sLE7ZCLlw==", + "license": "MIT", + "dependencies": { + "safer-buffer": ">= 2.1.2 < 3.0.0" + }, + "engines": { + "node": ">=0.10.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" + } + }, + "node_modules/ieee754": { + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/ieee754/-/ieee754-1.2.1.tgz", + "integrity": "sha512-dcyqhDvX1C46lXZcVqCpK+FtMRQVdIMN6/Df5js2zouUsqG7I6sFxitIC+7KYK29KdXOLHdu9zL4sFnoVQnqaA==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/feross" + }, + { + "type": "patreon", + "url": "https://www.patreon.com/feross" + }, + { + "type": "consulting", + "url": "https://feross.org/support" + } + ], + "license": "BSD-3-Clause" + }, + "node_modules/ignore-by-default": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/ignore-by-default/-/ignore-by-default-1.0.1.tgz", + "integrity": "sha512-Ius2VYcGNk7T90CppJqcIkS5ooHUZyIQK+ClZfMfMNFEF9VSE73Fq+906u/CWu92x4gzZMWOwfFYckPObzdEbA==", + "dev": true, + "license": "ISC" + }, + "node_modules/inherits": { + "version": "2.0.4", + "resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.4.tgz", + "integrity": "sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ==", + "license": "ISC" + }, + "node_modules/ipaddr.js": { + "version": "1.9.1", + "resolved": "https://registry.npmjs.org/ipaddr.js/-/ipaddr.js-1.9.1.tgz", + "integrity": "sha512-0KI/607xoxSToH7GjN1FfSbLoU0+btTicjsQSWQlh/hZykN8KpmMf7uYwPW3R+akZ6R/w18ZlXSHBYXiYUPO3g==", + "license": "MIT", + "engines": { + "node": ">= 0.10" + } + }, + "node_modules/is-binary-path": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/is-binary-path/-/is-binary-path-2.1.0.tgz", + "integrity": "sha512-ZMERYes6pDydyuGidse7OsHxtbI7WVeUEozgR/g7rd0xUimYNlvZRE/K2MgZTjWy725IfelLeVcEM97mmtRGXw==", + "dev": true, + "license": "MIT", + "dependencies": { + "binary-extensions": "^2.0.0" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/is-callable": { + "version": "1.2.7", + "resolved": "https://registry.npmjs.org/is-callable/-/is-callable-1.2.7.tgz", + "integrity": "sha512-1BC0BVFhS/p0qtw6enp8e+8OD0UrK0oFLztSjNzhcKA3WDuJxxAPXzPuPtKkjEY9UUoEWlX/8fgKeu2S8i9JTA==", + "license": "MIT", + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/is-extglob": { + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/is-extglob/-/is-extglob-2.1.1.tgz", + "integrity": "sha512-SbKbANkN603Vi4jEZv49LeVJMn4yGwsbzZworEoyEiutsN3nJYdbO36zfhGJ6QEDpOZIFkDtnq5JRxmvl3jsoQ==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/is-glob": { + "version": "4.0.3", + "resolved": "https://registry.npmjs.org/is-glob/-/is-glob-4.0.3.tgz", + "integrity": "sha512-xelSayHH36ZgE7ZWhli7pW34hNbNl8Ojv5KVmkJD4hBdD3th8Tfk9vYasLM+mXWOZhFkgZfxhLSnrwRr4elSSg==", + "dev": true, + "license": "MIT", + "dependencies": { + "is-extglob": "^2.1.1" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/is-natural-number": { + "version": "4.0.1", + "resolved": "https://registry.npmjs.org/is-natural-number/-/is-natural-number-4.0.1.tgz", + "integrity": "sha512-Y4LTamMe0DDQIIAlaer9eKebAlDSV6huy+TWhJVPlzZh2o4tRP5SQWFlLn5N0To4mDD22/qdOq+veo1cSISLgQ==", + "license": "MIT" + }, + "node_modules/is-number": { + "version": "7.0.0", + "resolved": "https://registry.npmjs.org/is-number/-/is-number-7.0.0.tgz", + "integrity": "sha512-41Cifkg6e8TylSpdtTpeLVMqvSBEVzTttHvERD741+pnZ8ANv0004MRL43QKPDlK9cGvNp6NZWZUBlbGXYxxng==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=0.12.0" + } + }, + "node_modules/is-promise": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/is-promise/-/is-promise-4.0.0.tgz", + "integrity": "sha512-hvpoI6korhJMnej285dSg6nu1+e6uxs7zG3BYAm5byqDsgJNWwxzM6z6iZiAgQR4TJ30JmBTOwqZUw3WlyH3AQ==", + "license": "MIT" + }, + "node_modules/is-stream": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/is-stream/-/is-stream-1.1.0.tgz", + "integrity": "sha512-uQPm8kcs47jx38atAcWTVxyltQYoPT68y9aWYdV6yWXSyW8mzSat0TL6CiWdZeCdF3KrAvpVtnHbTv4RN+rqdQ==", + "license": "MIT", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/is-typed-array": { + "version": "1.1.15", + "resolved": "https://registry.npmjs.org/is-typed-array/-/is-typed-array-1.1.15.tgz", + "integrity": "sha512-p3EcsicXjit7SaskXHs1hA91QxgTw46Fv6EFKKGS5DRFLD8yKnohjF3hxoju94b/OcMZoQukzpPpBE9uLVKzgQ==", + "license": "MIT", + "dependencies": { + "which-typed-array": "^1.1.16" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/isarray": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/isarray/-/isarray-1.0.0.tgz", + "integrity": "sha512-VLghIWNM6ELQzo7zwmcg0NmTVyWKYjvIeM83yjp0wRDTmUnrM678fQbcKBo6n2CJEF0szoG//ytg+TKla89ALQ==", + "license": "MIT" + }, + "node_modules/joycon": { + "version": "3.1.1", + "resolved": "https://registry.npmjs.org/joycon/-/joycon-3.1.1.tgz", + "integrity": "sha512-34wB/Y7MW7bzjKRjUKTa46I2Z7eV62Rkhva+KkopW7Qvv/OSWBqvkSY7vusOPrNuZcUG3tApvdVgNB8POj3SPw==", + "license": "MIT", + "engines": { + "node": ">=10" + } + }, + "node_modules/json-schema-traverse": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/json-schema-traverse/-/json-schema-traverse-1.0.0.tgz", + "integrity": "sha512-NM8/P9n3XjXhIZn1lLhkFaACTOURQXjWhV4BA/RnOv8xvgqtqpAX9IO4mRQxSx1Rlo4tqzeqb0sOlruaOy3dug==", + "license": "MIT" + }, + "node_modules/json5": { + "version": "2.2.3", + "resolved": "https://registry.npmjs.org/json5/-/json5-2.2.3.tgz", + "integrity": "sha512-XmOWe7eyHYH14cLdVPoyg+GOH3rYX++KpzrylJwSW98t3Nk+U8XOl8FWKOgwtzdb8lXGf6zYwDUzeHMWfxasyg==", + "dev": true, + "license": "MIT", + "bin": { + "json5": "lib/cli.js" + }, + "engines": { + "node": ">=6" + } + }, + "node_modules/lodash": { + "version": "4.17.21", + "resolved": "https://registry.npmjs.org/lodash/-/lodash-4.17.21.tgz", + "integrity": "sha512-v2kDEe57lecTulaDIuNTPy3Ry4gLGJ6Z1O3vE1krgXZNrsQ+LFTGHVxVjcXPs17LhbZVGedAJv8XZ1tvj5FvSg==", + "license": "MIT" + }, + "node_modules/lodash.snakecase": { + "version": "4.1.1", + "resolved": "https://registry.npmjs.org/lodash.snakecase/-/lodash.snakecase-4.1.1.tgz", + "integrity": "sha512-QZ1d4xoBHYUeuouhEq3lk3Uq7ldgyFXGBhg04+oRLnIz8o9T65Eh+8YdroUwn846zchkA9yDsDl5CVVaV2nqYw==", + "license": "MIT" + }, + "node_modules/long": { + "version": "5.3.2", + "resolved": "https://registry.npmjs.org/long/-/long-5.3.2.tgz", + "integrity": "sha512-mNAgZ1GmyNhD7AuqnTG3/VQ26o760+ZYBPKjPvugO8+nLbYfX6TVpJPseBvopbdY+qpZ/lKUnmEc1LeZYS3QAA==", + "license": "Apache-2.0" + }, + "node_modules/make-dir": { + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/make-dir/-/make-dir-1.3.0.tgz", + "integrity": "sha512-2w31R7SJtieJJnQtGc7RVL2StM2vGYVfqUOvUDxH6bC6aJTxPxTF0GnIgCyu7tjockiUWAYQRbxa7vKn34s5sQ==", + "license": "MIT", + "dependencies": { + "pify": "^3.0.0" + }, + "engines": { + "node": ">=4" + } + }, + "node_modules/make-dir/node_modules/pify": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/pify/-/pify-3.0.0.tgz", + "integrity": "sha512-C3FsVNH1udSEX48gGX1xfvwTWfsYWj5U+8/uK15BGzIGrKoUpghX8hWZwa/OFnakBiiVNmBvemTJR5mcy7iPcg==", + "license": "MIT", + "engines": { + "node": ">=4" + } + }, + "node_modules/make-error": { + "version": "1.3.6", + "resolved": "https://registry.npmjs.org/make-error/-/make-error-1.3.6.tgz", + "integrity": "sha512-s8UhlNe7vPKomQhC1qFelMokr/Sc3AgNbso3n74mVPA5LTZwkB9NlXf4XPamLxJE8h0gh73rM94xvwRT2CVInw==", + "dev": true, + "license": "ISC" + }, + "node_modules/math-intrinsics": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/math-intrinsics/-/math-intrinsics-1.1.0.tgz", + "integrity": "sha512-/IXtbwEk5HTPyEwyKX6hGkYXxM9nbj64B+ilVJnC/R6B0pH5G4V3b0pVbL7DBj4tkhBAppbQUlf6F6Xl9LHu1g==", + "license": "MIT", + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/media-typer": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/media-typer/-/media-typer-1.1.0.tgz", + "integrity": "sha512-aisnrDP4GNe06UcKFnV5bfMNPBUw4jsLGaWwWfnH3v02GnBuXX2MCVn5RbrWo0j3pczUilYblq7fQ7Nw2t5XKw==", + "license": "MIT", + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/merge-descriptors": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/merge-descriptors/-/merge-descriptors-2.0.0.tgz", + "integrity": "sha512-Snk314V5ayFLhp3fkUREub6WtjBfPdCPY1Ln8/8munuLuiYhsABgBVWsozAG+MWMbVEvcdcpbi9R7ww22l9Q3g==", + "license": "MIT", + "engines": { + "node": ">=18" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/mime-db": { + "version": "1.54.0", + "resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.54.0.tgz", + "integrity": "sha512-aU5EJuIN2WDemCcAp2vFBfp/m4EAhWJnUNSSw0ixs7/kXbd6Pg64EmwJkNdFhB8aWt1sH2CTXrLxo/iAGV3oPQ==", + "license": "MIT", + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/mime-types": { + "version": "3.0.2", + "resolved": "https://registry.npmjs.org/mime-types/-/mime-types-3.0.2.tgz", + "integrity": "sha512-Lbgzdk0h4juoQ9fCKXW4by0UJqj+nOOrI9MJ1sSj4nI8aI2eo1qmvQEie4VD1glsS250n15LsWsYtCugiStS5A==", + "license": "MIT", + "dependencies": { + "mime-db": "^1.54.0" + }, + "engines": { + "node": ">=18" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" + } + }, + "node_modules/minimatch": { + "version": "3.1.2", + "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.1.2.tgz", + "integrity": "sha512-J7p63hRiAjw1NDEww1W7i37+ByIrOWO5XQQAzZ3VOcL0PNybwpfmV/N05zFAzwQ9USyEcX6t3UO+K5aqBQOIHw==", + "dev": true, + "license": "ISC", + "dependencies": { + "brace-expansion": "^1.1.7" + }, + "engines": { + "node": "*" + } + }, + "node_modules/minimist": { + "version": "1.2.8", + "resolved": "https://registry.npmjs.org/minimist/-/minimist-1.2.8.tgz", + "integrity": "sha512-2yyAR8qBkN3YuheJanUpWC5U3bb5osDywNB8RzDVlDwDHbocAJveqqj1u8+SVD7jkWT4yvsHCpWqqWqAxb0zCA==", + "license": "MIT", + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/ms": { + "version": "2.1.3", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz", + "integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==", + "license": "MIT" + }, + "node_modules/negotiator": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/negotiator/-/negotiator-1.0.0.tgz", + "integrity": "sha512-8Ofs/AUQh8MaEcrlq5xOX0CQ9ypTF5dl78mjlMNfOK08fzpgTHQRQPBxcPlEtIw0yRpws+Zo/3r+5WRby7u3Gg==", + "license": "MIT", + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/node-domexception": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/node-domexception/-/node-domexception-1.0.0.tgz", + "integrity": "sha512-/jKZoMpw0F8GRwl4/eLROPA3cfcXtLApP0QzLmUT/HuPCZWyB7IY9ZrMeKw2O/nFIqPQB3PVM9aYm0F312AXDQ==", + "deprecated": "Use your platform's native DOMException instead", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/jimmywarting" + }, + { + "type": "github", + "url": "https://paypal.me/jimmywarting" + } + ], + "license": "MIT", + "engines": { + "node": ">=10.5.0" + } + }, + "node_modules/node-fetch": { + "version": "2.7.0", + "resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-2.7.0.tgz", + "integrity": "sha512-c4FRfUm/dbcWZ7U+1Wq0AwCyFL+3nt2bEw05wfxSz+DWpWsitgmSgYmy2dQdWyKC1694ELPqMs/YzUSNozLt8A==", + "license": "MIT", + "dependencies": { + "whatwg-url": "^5.0.0" + }, + "engines": { + "node": "4.x || >=6.0.0" + }, + "peerDependencies": { + "encoding": "^0.1.0" + }, + "peerDependenciesMeta": { + "encoding": { + "optional": true + } + } + }, + "node_modules/node-record-lpcm16": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/node-record-lpcm16/-/node-record-lpcm16-1.0.1.tgz", + "integrity": "sha512-H75GMOP8ErnF67m21+qSgj4USnzv5RLfm7OkEItdIi+soNKoJZpMQPX6umM8Cn9nVPSgd/dBUtc1msst5MmABA==", + "license": "ISC", + "dependencies": { + "debug": "^2.6.8" + } + }, + "node_modules/node-record-lpcm16/node_modules/debug": { + "version": "2.6.9", + "resolved": "https://registry.npmjs.org/debug/-/debug-2.6.9.tgz", + "integrity": "sha512-bC7ElrdJaJnPbAP+1EotYvqZsb3ecl5wi6Bfi6BJTUcNowp6cvspg0jXznRTKDjm/E7AdgFBVeAPVMNcKGsHMA==", + "license": "MIT", + "dependencies": { + "ms": "2.0.0" + } + }, + "node_modules/node-record-lpcm16/node_modules/ms": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.0.0.tgz", + "integrity": "sha512-Tpp60P6IUJDTuOq/5Z8cdskzJujfwqfOTkrwIwj7IRISpnkJnT6SyJ4PCPnGMoFjC9ddhal5KVIYtAt97ix05A==", + "license": "MIT" + }, + "node_modules/nodemon": { + "version": "3.1.11", + "resolved": "https://registry.npmjs.org/nodemon/-/nodemon-3.1.11.tgz", + "integrity": "sha512-is96t8F/1//UHAjNPHpbsNY46ELPpftGUoSVNXwUfMk/qdjSylYrWSu1XavVTBOn526kFiOR733ATgNBCQyH0g==", + "dev": true, + "license": "MIT", + "dependencies": { + "chokidar": "^3.5.2", + "debug": "^4", + "ignore-by-default": "^1.0.1", + "minimatch": "^3.1.2", + "pstree.remy": "^1.1.8", + "semver": "^7.5.3", + "simple-update-notifier": "^2.0.0", + "supports-color": "^5.5.0", + "touch": "^3.1.0", + "undefsafe": "^2.0.5" + }, + "bin": { + "nodemon": "bin/nodemon.js" + }, + "engines": { + "node": ">=10" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/nodemon" + } + }, + "node_modules/nodemon/node_modules/has-flag": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/has-flag/-/has-flag-3.0.0.tgz", + "integrity": "sha512-sKJf1+ceQBr4SMkvQnBDNDtf4TXpVhVGateu0t918bl30FnbE2m4vNLX+VWe/dpjlb+HugGYzW7uQXH98HPEYw==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=4" + } + }, + "node_modules/nodemon/node_modules/supports-color": { + "version": "5.5.0", + "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-5.5.0.tgz", + "integrity": "sha512-QjVjwdXIt408MIiAqCX4oUKsgU2EqAGzs2Ppkm4aQYbjm+ZEWEcW4SfFNTr4uMNZma0ey4f5lgLrkB0aX0QMow==", + "dev": true, + "license": "MIT", + "dependencies": { + "has-flag": "^3.0.0" + }, + "engines": { + "node": ">=4" + } + }, + "node_modules/normalize-path": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/normalize-path/-/normalize-path-3.0.0.tgz", + "integrity": "sha512-6eZs5Ls3WtCisHWp9S2GUy8dqkpGi4BVSz3GaqiE6ezub0512ESztXUwUB6C6IKbQkY2Pnb/mD4WYojCRwcwLA==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/object-assign": { + "version": "4.1.1", + "resolved": "https://registry.npmjs.org/object-assign/-/object-assign-4.1.1.tgz", + "integrity": "sha512-rJgTQnkUnH1sFw8yT6VSU3zD3sWmu6sZhIseY8VX+GRu3P6F7Fu+JNDoXfklElbLJSnc3FUQHVe4cU5hj+BcUg==", + "license": "MIT", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/object-inspect": { + "version": "1.13.4", + "resolved": "https://registry.npmjs.org/object-inspect/-/object-inspect-1.13.4.tgz", + "integrity": "sha512-W67iLl4J2EXEGTbfeHCffrjDfitvLANg0UlX3wFUUSTx92KXRFegMHUVgSqE+wvhAbi4WqjGg9czysTV2Epbew==", + "license": "MIT", + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/on-exit-leak-free": { + "version": "2.1.2", + "resolved": "https://registry.npmjs.org/on-exit-leak-free/-/on-exit-leak-free-2.1.2.tgz", + "integrity": "sha512-0eJJY6hXLGf1udHwfNftBqH+g73EU4B504nZeKpz1sYRKafAghwxEJunB2O7rDZkL4PGfsMVnTXZ2EjibbqcsA==", + "license": "MIT", + "engines": { + "node": ">=14.0.0" + } + }, + "node_modules/on-finished": { + "version": "2.4.1", + "resolved": "https://registry.npmjs.org/on-finished/-/on-finished-2.4.1.tgz", + "integrity": "sha512-oVlzkg3ENAhCk2zdv7IJwd/QUD4z2RxRwpkcGY8psCVcCYZNq4wYnVWALHM+brtuJjePWiYF/ClmuDr8Ch5+kg==", + "license": "MIT", + "dependencies": { + "ee-first": "1.1.1" + }, + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/once": { + "version": "1.4.0", + "resolved": "https://registry.npmjs.org/once/-/once-1.4.0.tgz", + "integrity": "sha512-lNaJgI+2Q5URQBkccEKHTQOPaXdUxnZZElQTZY0MFUAuaEqe1E+Nyvgdz/aIyNi6Z9MzO5dv1H8n58/GELp3+w==", + "license": "ISC", + "dependencies": { + "wrappy": "1" + } + }, + "node_modules/parseurl": { + "version": "1.3.3", + "resolved": "https://registry.npmjs.org/parseurl/-/parseurl-1.3.3.tgz", + "integrity": "sha512-CiyeOxFT/JZyN5m0z9PfXw4SCBJ6Sygz1Dpl0wqjlhDEGGBP1GnsUVEL0p63hoG1fcj3fHynXi9NYO4nWOL+qQ==", + "license": "MIT", + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/path-to-regexp": { + "version": "8.3.0", + "resolved": "https://registry.npmjs.org/path-to-regexp/-/path-to-regexp-8.3.0.tgz", + "integrity": "sha512-7jdwVIRtsP8MYpdXSwOS0YdD0Du+qOoF/AEPIt88PcCFrZCzx41oxku1jD88hZBwbNUIEfpqvuhjFaMAqMTWnA==", + "license": "MIT", + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" + } + }, + "node_modules/pend": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/pend/-/pend-1.2.0.tgz", + "integrity": "sha512-F3asv42UuXchdzt+xXqfW1OGlVBe+mxa2mqI0pg5yAHZPvFmY3Y6drSf/GQ1A86WgWEN9Kzh/WrgKa6iGcHXLg==", + "license": "MIT" + }, + "node_modules/picomatch": { + "version": "2.3.1", + "resolved": "https://registry.npmjs.org/picomatch/-/picomatch-2.3.1.tgz", + "integrity": "sha512-JU3teHTNjmE2VCGFzuY8EXzCDVwEqB2a8fsIvwaStHhAWJEeVd1o1QD80CU6+ZdEXXSLbSsuLwJjkCBWqRQUVA==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=8.6" + }, + "funding": { + "url": "https://github.com/sponsors/jonschlinkert" + } + }, + "node_modules/pify": { + "version": "2.3.0", + "resolved": "https://registry.npmjs.org/pify/-/pify-2.3.0.tgz", + "integrity": "sha512-udgsAY+fTnvv7kI7aaxbqwWNb0AHiB0qBO89PZKPkoTmGOgdbrHDKD+0B2X4uTfJ/FT1R09r9gTsjUjNJotuog==", + "license": "MIT", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/pinkie": { + "version": "2.0.4", + "resolved": "https://registry.npmjs.org/pinkie/-/pinkie-2.0.4.tgz", + "integrity": "sha512-MnUuEycAemtSaeFSjXKW/aroV7akBbY+Sv+RkyqFjgAe73F+MR0TBWKBRDkmfWq/HiFmdavfZ1G7h4SPZXaCSg==", + "license": "MIT", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/pinkie-promise": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/pinkie-promise/-/pinkie-promise-2.0.1.tgz", + "integrity": "sha512-0Gni6D4UcLTbv9c57DfxDGdr41XfgUjqWZu492f0cIGr16zDU06BWP/RAEvOuo7CQ0CNjHaLlM59YJJFm3NWlw==", + "license": "MIT", + "dependencies": { + "pinkie": "^2.0.0" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/pino": { + "version": "9.14.0", + "resolved": "https://registry.npmjs.org/pino/-/pino-9.14.0.tgz", + "integrity": "sha512-8OEwKp5juEvb/MjpIc4hjqfgCNysrS94RIOMXYvpYCdm/jglrKEiAYmiumbmGhCvs+IcInsphYDFwqrjr7398w==", + "license": "MIT", + "dependencies": { + "@pinojs/redact": "^0.4.0", + "atomic-sleep": "^1.0.0", + "on-exit-leak-free": "^2.1.0", + "pino-abstract-transport": "^2.0.0", + "pino-std-serializers": "^7.0.0", + "process-warning": "^5.0.0", + "quick-format-unescaped": "^4.0.3", + "real-require": "^0.2.0", + "safe-stable-stringify": "^2.3.1", + "sonic-boom": "^4.0.1", + "thread-stream": "^3.0.0" + }, + "bin": { + "pino": "bin.js" + } + }, + "node_modules/pino-abstract-transport": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/pino-abstract-transport/-/pino-abstract-transport-2.0.0.tgz", + "integrity": "sha512-F63x5tizV6WCh4R6RHyi2Ml+M70DNRXt/+HANowMflpgGFMAym/VKm6G7ZOQRjqN7XbGxK1Lg9t6ZrtzOaivMw==", + "license": "MIT", + "dependencies": { + "split2": "^4.0.0" + } + }, + "node_modules/pino-pretty": { + "version": "13.1.3", + "resolved": "https://registry.npmjs.org/pino-pretty/-/pino-pretty-13.1.3.tgz", + "integrity": "sha512-ttXRkkOz6WWC95KeY9+xxWL6AtImwbyMHrL1mSwqwW9u+vLp/WIElvHvCSDg0xO/Dzrggz1zv3rN5ovTRVowKg==", + "license": "MIT", + "dependencies": { + "colorette": "^2.0.7", + "dateformat": "^4.6.3", + "fast-copy": "^4.0.0", + "fast-safe-stringify": "^2.1.1", + "help-me": "^5.0.0", + "joycon": "^3.1.1", + "minimist": "^1.2.6", + "on-exit-leak-free": "^2.1.0", + "pino-abstract-transport": "^3.0.0", + "pump": "^3.0.0", + "secure-json-parse": "^4.0.0", + "sonic-boom": "^4.0.1", + "strip-json-comments": "^5.0.2" + }, + "bin": { + "pino-pretty": "bin.js" + } + }, + "node_modules/pino-pretty/node_modules/pino-abstract-transport": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/pino-abstract-transport/-/pino-abstract-transport-3.0.0.tgz", + "integrity": "sha512-wlfUczU+n7Hy/Ha5j9a/gZNy7We5+cXp8YL+X+PG8S0KXxw7n/JXA3c46Y0zQznIJ83URJiwy7Lh56WLokNuxg==", + "license": "MIT", + "dependencies": { + "split2": "^4.0.0" + } + }, + "node_modules/pino-std-serializers": { + "version": "7.0.0", + "resolved": "https://registry.npmjs.org/pino-std-serializers/-/pino-std-serializers-7.0.0.tgz", + "integrity": "sha512-e906FRY0+tV27iq4juKzSYPbUj2do2X2JX4EzSca1631EB2QJQUqGbDuERal7LCtOpxl6x3+nvo9NPZcmjkiFA==", + "license": "MIT" + }, + "node_modules/possible-typed-array-names": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/possible-typed-array-names/-/possible-typed-array-names-1.1.0.tgz", + "integrity": "sha512-/+5VFTchJDoVj3bhoqi6UeymcD00DAwb1nJwamzPvHEszJ4FpF6SNNbUbOS8yI56qHzdV8eK0qEfOSiodkTdxg==", + "license": "MIT", + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/process-nextick-args": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/process-nextick-args/-/process-nextick-args-2.0.1.tgz", + "integrity": "sha512-3ouUOpQhtgrbOa17J7+uxOTpITYWaGP7/AhoR3+A+/1e9skrzelGi/dXzEYyvbxubEF6Wn2ypscTKiKJFFn1ag==", + "license": "MIT" + }, + "node_modules/process-warning": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/process-warning/-/process-warning-5.0.0.tgz", + "integrity": "sha512-a39t9ApHNx2L4+HBnQKqxxHNs1r7KF+Intd8Q/g1bUh6q0WIp9voPXJ/x0j+ZL45KF1pJd9+q2jLIRMfvEshkA==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/fastify" + }, + { + "type": "opencollective", + "url": "https://opencollective.com/fastify" + } + ], + "license": "MIT" + }, + "node_modules/prom-client": { + "version": "15.1.3", + "resolved": "https://registry.npmjs.org/prom-client/-/prom-client-15.1.3.tgz", + "integrity": "sha512-6ZiOBfCywsD4k1BN9IX0uZhF+tJkV8q8llP64G5Hajs4JOeVLPCwpPVcpXy3BwYiUGgyJzsJJQeOIv7+hDSq8g==", + "license": "Apache-2.0", + "dependencies": { + "@opentelemetry/api": "^1.4.0", + "tdigest": "^0.1.1" + }, + "engines": { + "node": "^16 || ^18 || >=20" + } + }, + "node_modules/protobufjs": { + "version": "7.5.4", + "resolved": "https://registry.npmjs.org/protobufjs/-/protobufjs-7.5.4.tgz", + "integrity": "sha512-CvexbZtbov6jW2eXAvLukXjXUW1TzFaivC46BpWc/3BpcCysb5Vffu+B3XHMm8lVEuy2Mm4XGex8hBSg1yapPg==", + "hasInstallScript": true, + "license": "BSD-3-Clause", + "dependencies": { + "@protobufjs/aspromise": "^1.1.2", + "@protobufjs/base64": "^1.1.2", + "@protobufjs/codegen": "^2.0.4", + "@protobufjs/eventemitter": "^1.1.0", + "@protobufjs/fetch": "^1.1.0", + "@protobufjs/float": "^1.0.2", + "@protobufjs/inquire": "^1.1.0", + "@protobufjs/path": "^1.1.2", + "@protobufjs/pool": "^1.1.0", + "@protobufjs/utf8": "^1.1.0", + "@types/node": ">=13.7.0", + "long": "^5.0.0" + }, + "engines": { + "node": ">=12.0.0" + } + }, + "node_modules/proxy-addr": { + "version": "2.0.7", + "resolved": "https://registry.npmjs.org/proxy-addr/-/proxy-addr-2.0.7.tgz", + "integrity": "sha512-llQsMLSUDUPT44jdrU/O37qlnifitDP+ZwrmmZcoSKyLKvtZxpyV0n2/bD/N4tBAAZ/gJEdZU7KMraoK1+XYAg==", + "license": "MIT", + "dependencies": { + "forwarded": "0.2.0", + "ipaddr.js": "1.9.1" + }, + "engines": { + "node": ">= 0.10" + } + }, + "node_modules/pstree.remy": { + "version": "1.1.8", + "resolved": "https://registry.npmjs.org/pstree.remy/-/pstree.remy-1.1.8.tgz", + "integrity": "sha512-77DZwxQmxKnu3aR542U+X8FypNzbfJ+C5XQDk3uWjWxn6151aIMGthWYRXTqT1E5oJvg+ljaa2OJi+VfvCOQ8w==", + "dev": true, + "license": "MIT" + }, + "node_modules/pump": { + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/pump/-/pump-3.0.3.tgz", + "integrity": "sha512-todwxLMY7/heScKmntwQG8CXVkWUOdYxIvY2s0VWAAMh/nd8SoYiRaKjlr7+iCs984f2P8zvrfWcDDYVb73NfA==", + "license": "MIT", + "dependencies": { + "end-of-stream": "^1.1.0", + "once": "^1.3.1" + } + }, + "node_modules/qs": { + "version": "6.14.0", + "resolved": "https://registry.npmjs.org/qs/-/qs-6.14.0.tgz", + "integrity": "sha512-YWWTjgABSKcvs/nWBi9PycY/JiPJqOD4JA6o9Sej2AtvSGarXxKC3OQSk4pAarbdQlKAh5D4FCQkJNkW+GAn3w==", + "license": "BSD-3-Clause", + "dependencies": { + "side-channel": "^1.1.0" + }, + "engines": { + "node": ">=0.6" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/quick-format-unescaped": { + "version": "4.0.4", + "resolved": "https://registry.npmjs.org/quick-format-unescaped/-/quick-format-unescaped-4.0.4.tgz", + "integrity": "sha512-tYC1Q1hgyRuHgloV/YXs2w15unPVh8qfu/qCTfhTYamaw7fyhumKa2yGpdSo87vY32rIclj+4fWYQXUMs9EHvg==", + "license": "MIT" + }, + "node_modules/range-parser": { + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/range-parser/-/range-parser-1.2.1.tgz", + "integrity": "sha512-Hrgsx+orqoygnmhFbKaHE6c296J+HTAQXoxEF6gNupROmmGJRoyzfG3ccAveqCBrwr/2yxQ5BVd/GTl5agOwSg==", + "license": "MIT", + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/raw-body": { + "version": "3.0.2", + "resolved": "https://registry.npmjs.org/raw-body/-/raw-body-3.0.2.tgz", + "integrity": "sha512-K5zQjDllxWkf7Z5xJdV0/B0WTNqx6vxG70zJE4N0kBs4LovmEYWJzQGxC9bS9RAKu3bgM40lrd5zoLJ12MQ5BA==", + "license": "MIT", + "dependencies": { + "bytes": "~3.1.2", + "http-errors": "~2.0.1", + "iconv-lite": "~0.7.0", + "unpipe": "~1.0.0" + }, + "engines": { + "node": ">= 0.10" + } + }, + "node_modules/readable-stream": { + "version": "2.3.8", + "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-2.3.8.tgz", + "integrity": "sha512-8p0AUk4XODgIewSi0l8Epjs+EVnWiK7NoDIEGU0HhE7+ZyY8D1IMY7odu5lRrFXGg71L15KG8QrPmum45RTtdA==", + "license": "MIT", + "dependencies": { + "core-util-is": "~1.0.0", + "inherits": "~2.0.3", + "isarray": "~1.0.0", + "process-nextick-args": "~2.0.0", + "safe-buffer": "~5.1.1", + "string_decoder": "~1.1.1", + "util-deprecate": "~1.0.1" + } + }, + "node_modules/readable-stream/node_modules/safe-buffer": { + "version": "5.1.2", + "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.1.2.tgz", + "integrity": "sha512-Gd2UZBJDkXlY7GbJxfsE8/nvKkUEU1G38c1siN6QP6a9PT9MmHB8GnpscSmMJSoF8LOIrt8ud/wPtojys4G6+g==", + "license": "MIT" + }, + "node_modules/readdirp": { + "version": "3.6.0", + "resolved": "https://registry.npmjs.org/readdirp/-/readdirp-3.6.0.tgz", + "integrity": "sha512-hOS089on8RduqdbhvQ5Z37A0ESjsqz6qnRcffsMU3495FuTdqSm+7bhJ29JvIOsBDEEnan5DPu9t3To9VRlMzA==", + "dev": true, + "license": "MIT", + "dependencies": { + "picomatch": "^2.2.1" + }, + "engines": { + "node": ">=8.10.0" + } + }, + "node_modules/real-require": { + "version": "0.2.0", + "resolved": "https://registry.npmjs.org/real-require/-/real-require-0.2.0.tgz", + "integrity": "sha512-57frrGM/OCTLqLOAh0mhVA9VBMHd+9U7Zb2THMGdBUoZVOtGbJzjxsYGDJ3A9AYYCP4hn6y1TVbaOfzWtm5GFg==", + "license": "MIT", + "engines": { + "node": ">= 12.13.0" + } + }, + "node_modules/require-from-string": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/require-from-string/-/require-from-string-2.0.2.tgz", + "integrity": "sha512-Xf0nWe6RseziFMu+Ap9biiUbmplq6S9/p+7w7YXP/JBHhrUDDUhwa+vANyubuqfZWTveU//DYVGsDG7RKL/vEw==", + "license": "MIT", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/router": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/router/-/router-2.2.0.tgz", + "integrity": "sha512-nLTrUKm2UyiL7rlhapu/Zl45FwNgkZGaCpZbIHajDYgwlJCOzLSk+cIPAnsEqV955GjILJnKbdQC1nVPz+gAYQ==", + "license": "MIT", + "dependencies": { + "debug": "^4.4.0", + "depd": "^2.0.0", + "is-promise": "^4.0.0", + "parseurl": "^1.3.3", + "path-to-regexp": "^8.0.0" + }, + "engines": { + "node": ">= 18" + } + }, + "node_modules/safe-buffer": { + "version": "5.2.1", + "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.2.1.tgz", + "integrity": "sha512-rp3So07KcdmmKbGvgaNxQSJr7bGVSVk5S9Eq1F+ppbRo70+YeaDxkw5Dd8NPN+GD6bjnYm2VuPuCXmpuYvmCXQ==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/feross" + }, + { + "type": "patreon", + "url": "https://www.patreon.com/feross" + }, + { + "type": "consulting", + "url": "https://feross.org/support" + } + ], + "license": "MIT" + }, + "node_modules/safe-stable-stringify": { + "version": "2.5.0", + "resolved": "https://registry.npmjs.org/safe-stable-stringify/-/safe-stable-stringify-2.5.0.tgz", + "integrity": "sha512-b3rppTKm9T+PsVCBEOUR46GWI7fdOs00VKZ1+9c1EWDaDMvjQc6tUwuFyIprgGgTcWoVHSKrU8H31ZHA2e0RHA==", + "license": "MIT", + "engines": { + "node": ">=10" + } + }, + "node_modules/safer-buffer": { + "version": "2.1.2", + "resolved": "https://registry.npmjs.org/safer-buffer/-/safer-buffer-2.1.2.tgz", + "integrity": "sha512-YZo3K82SD7Riyi0E1EQPojLz7kpepnSQI9IyPbHHg1XXXevb5dJI7tpyN2ADxGcQbHG7vcyRHk0cbwqcQriUtg==", + "license": "MIT" + }, + "node_modules/secure-json-parse": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/secure-json-parse/-/secure-json-parse-4.1.0.tgz", + "integrity": "sha512-l4KnYfEyqYJxDwlNVyRfO2E4NTHfMKAWdUuA8J0yve2Dz/E/PdBepY03RvyJpssIpRFwJoCD55wA+mEDs6ByWA==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/fastify" + }, + { + "type": "opencollective", + "url": "https://opencollective.com/fastify" + } + ], + "license": "BSD-3-Clause" + }, + "node_modules/seek-bzip": { + "version": "1.0.6", + "resolved": "https://registry.npmjs.org/seek-bzip/-/seek-bzip-1.0.6.tgz", + "integrity": "sha512-e1QtP3YL5tWww8uKaOCQ18UxIT2laNBXHjV/S2WYCiK4udiv8lkG89KRIoCjUagnAmCBurjF4zEVX2ByBbnCjQ==", + "license": "MIT", + "dependencies": { + "commander": "^2.8.1" + }, + "bin": { + "seek-bunzip": "bin/seek-bunzip", + "seek-table": "bin/seek-bzip-table" + } + }, + "node_modules/semver": { + "version": "7.7.3", + "resolved": "https://registry.npmjs.org/semver/-/semver-7.7.3.tgz", + "integrity": "sha512-SdsKMrI9TdgjdweUSR9MweHA4EJ8YxHn8DFaDisvhVlUOe4BF1tLD7GAj0lIqWVl+dPb/rExr0Btby5loQm20Q==", + "dev": true, + "license": "ISC", + "bin": { + "semver": "bin/semver.js" + }, + "engines": { + "node": ">=10" + } + }, + "node_modules/send": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/send/-/send-1.2.0.tgz", + "integrity": "sha512-uaW0WwXKpL9blXE2o0bRhoL2EGXIrZxQ2ZQ4mgcfoBxdFmQold+qWsD2jLrfZ0trjKL6vOw0j//eAwcALFjKSw==", + "license": "MIT", + "dependencies": { + "debug": "^4.3.5", + "encodeurl": "^2.0.0", + "escape-html": "^1.0.3", + "etag": "^1.8.1", + "fresh": "^2.0.0", + "http-errors": "^2.0.0", + "mime-types": "^3.0.1", + "ms": "^2.1.3", + "on-finished": "^2.4.1", + "range-parser": "^1.2.1", + "statuses": "^2.0.1" + }, + "engines": { + "node": ">= 18" + } + }, + "node_modules/serve-static": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/serve-static/-/serve-static-2.2.0.tgz", + "integrity": "sha512-61g9pCh0Vnh7IutZjtLGGpTA355+OPn2TyDv/6ivP2h/AdAVX9azsoxmg2/M6nZeQZNYBEwIcsne1mJd9oQItQ==", + "license": "MIT", + "dependencies": { + "encodeurl": "^2.0.0", + "escape-html": "^1.0.3", + "parseurl": "^1.3.3", + "send": "^1.2.0" + }, + "engines": { + "node": ">= 18" + } + }, + "node_modules/set-function-length": { + "version": "1.2.2", + "resolved": "https://registry.npmjs.org/set-function-length/-/set-function-length-1.2.2.tgz", + "integrity": "sha512-pgRc4hJ4/sNjWCSS9AmnS40x3bNMDTknHgL5UaMBTMyJnU90EgWh1Rz+MC9eFu4BuN/UwZjKQuY/1v3rM7HMfg==", + "license": "MIT", + "dependencies": { + "define-data-property": "^1.1.4", + "es-errors": "^1.3.0", + "function-bind": "^1.1.2", + "get-intrinsic": "^1.2.4", + "gopd": "^1.0.1", + "has-property-descriptors": "^1.0.2" + }, + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/setprototypeof": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/setprototypeof/-/setprototypeof-1.2.0.tgz", + "integrity": "sha512-E5LDX7Wrp85Kil5bhZv46j8jOeboKq5JMmYM3gVGdGH8xFpPWXUMsNrlODCrkoxMEeNi/XZIwuRvY4XNwYMJpw==", + "license": "ISC" + }, + "node_modules/side-channel": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/side-channel/-/side-channel-1.1.0.tgz", + "integrity": "sha512-ZX99e6tRweoUXqR+VBrslhda51Nh5MTQwou5tnUDgbtyM0dBgmhEDtWGP/xbKn6hqfPRHujUNwz5fy/wbbhnpw==", + "license": "MIT", + "dependencies": { + "es-errors": "^1.3.0", + "object-inspect": "^1.13.3", + "side-channel-list": "^1.0.0", + "side-channel-map": "^1.0.1", + "side-channel-weakmap": "^1.0.2" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/side-channel-list": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/side-channel-list/-/side-channel-list-1.0.0.tgz", + "integrity": "sha512-FCLHtRD/gnpCiCHEiJLOwdmFP+wzCmDEkc9y7NsYxeF4u7Btsn1ZuwgwJGxImImHicJArLP4R0yX4c2KCrMrTA==", + "license": "MIT", + "dependencies": { + "es-errors": "^1.3.0", + "object-inspect": "^1.13.3" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/side-channel-map": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/side-channel-map/-/side-channel-map-1.0.1.tgz", + "integrity": "sha512-VCjCNfgMsby3tTdo02nbjtM/ewra6jPHmpThenkTYh8pG9ucZ/1P8So4u4FGBek/BjpOVsDCMoLA/iuBKIFXRA==", + "license": "MIT", + "dependencies": { + "call-bound": "^1.0.2", + "es-errors": "^1.3.0", + "get-intrinsic": "^1.2.5", + "object-inspect": "^1.13.3" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/side-channel-weakmap": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/side-channel-weakmap/-/side-channel-weakmap-1.0.2.tgz", + "integrity": "sha512-WPS/HvHQTYnHisLo9McqBHOJk2FkHO/tlpvldyrnem4aeQp4hai3gythswg6p01oSoTl58rcpiFAjF2br2Ak2A==", + "license": "MIT", + "dependencies": { + "call-bound": "^1.0.2", + "es-errors": "^1.3.0", + "get-intrinsic": "^1.2.5", + "object-inspect": "^1.13.3", + "side-channel-map": "^1.0.1" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/simple-update-notifier": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/simple-update-notifier/-/simple-update-notifier-2.0.0.tgz", + "integrity": "sha512-a2B9Y0KlNXl9u/vsW6sTIu9vGEpfKu2wRV6l1H3XEas/0gUIzGzBoP/IouTcUQbm9JWZLH3COxyn03TYlFax6w==", + "dev": true, + "license": "MIT", + "dependencies": { + "semver": "^7.5.3" + }, + "engines": { + "node": ">=10" + } + }, + "node_modules/sonic-boom": { + "version": "4.2.0", + "resolved": "https://registry.npmjs.org/sonic-boom/-/sonic-boom-4.2.0.tgz", + "integrity": "sha512-INb7TM37/mAcsGmc9hyyI6+QR3rR1zVRu36B0NeGXKnOOLiZOfER5SA+N7X7k3yUYRzLWafduTDvJAfDswwEww==", + "license": "MIT", + "dependencies": { + "atomic-sleep": "^1.0.0" + } + }, + "node_modules/split2": { + "version": "4.2.0", + "resolved": "https://registry.npmjs.org/split2/-/split2-4.2.0.tgz", + "integrity": "sha512-UcjcJOWknrNkF6PLX83qcHM6KHgVKNkV62Y8a5uYDVv9ydGQVwAHMKqHdJje1VTWpljG0WYpCDhrCdAOYH4TWg==", + "license": "ISC", + "engines": { + "node": ">= 10.x" + } + }, + "node_modules/statuses": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/statuses/-/statuses-2.0.2.tgz", + "integrity": "sha512-DvEy55V3DB7uknRo+4iOGT5fP1slR8wQohVdknigZPMpMstaKJQWhwiYBACJE3Ul2pTnATihhBYnRhZQHGBiRw==", + "license": "MIT", + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/string_decoder": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.1.1.tgz", + "integrity": "sha512-n/ShnvDi6FHbbVfviro+WojiFzv+s8MPMHBczVePfUpDJLwoLT0ht1l4YwBCbi8pJAveEEdnkHyPyTP/mzRfwg==", + "license": "MIT", + "dependencies": { + "safe-buffer": "~5.1.0" + } + }, + "node_modules/string_decoder/node_modules/safe-buffer": { + "version": "5.1.2", + "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.1.2.tgz", + "integrity": "sha512-Gd2UZBJDkXlY7GbJxfsE8/nvKkUEU1G38c1siN6QP6a9PT9MmHB8GnpscSmMJSoF8LOIrt8ud/wPtojys4G6+g==", + "license": "MIT" + }, + "node_modules/strip-bom": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/strip-bom/-/strip-bom-3.0.0.tgz", + "integrity": "sha512-vavAMRXOgBVNF6nyEEmL3DBK19iRpDcoIwW+swQ+CbGiu7lju6t+JklA1MHweoWtadgt4ISVUsXLyDq34ddcwA==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=4" + } + }, + "node_modules/strip-dirs": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/strip-dirs/-/strip-dirs-2.1.0.tgz", + "integrity": "sha512-JOCxOeKLm2CAS73y/U4ZeZPTkE+gNVCzKt7Eox84Iej1LT/2pTWYpZKJuxwQpvX1LiZb1xokNR7RLfuBAa7T3g==", + "license": "MIT", + "dependencies": { + "is-natural-number": "^4.0.1" + } + }, + "node_modules/strip-json-comments": { + "version": "5.0.3", + "resolved": "https://registry.npmjs.org/strip-json-comments/-/strip-json-comments-5.0.3.tgz", + "integrity": "sha512-1tB5mhVo7U+ETBKNf92xT4hrQa3pm0MZ0PQvuDnWgAAGHDsfp4lPSpiS6psrSiet87wyGPh9ft6wmhOMQ0hDiw==", + "license": "MIT", + "engines": { + "node": ">=14.16" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/supports-color": { + "version": "7.2.0", + "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-7.2.0.tgz", + "integrity": "sha512-qpCAvRl9stuOHveKsn7HncJRvv501qIacKzQlO/+Lwxc9+0q2wLyv4Dfvt80/DPn2pqOBsJdDiogXGR9+OvwRw==", + "license": "MIT", + "dependencies": { + "has-flag": "^4.0.0" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/tar-stream": { + "version": "1.6.2", + "resolved": "https://registry.npmjs.org/tar-stream/-/tar-stream-1.6.2.tgz", + "integrity": "sha512-rzS0heiNf8Xn7/mpdSVVSMAWAoy9bfb1WOTYC78Z0UQKeKa/CWS8FOq0lKGNa8DWKAn9gxjCvMLYc5PGXYlK2A==", + "license": "MIT", + "dependencies": { + "bl": "^1.0.0", + "buffer-alloc": "^1.2.0", + "end-of-stream": "^1.0.0", + "fs-constants": "^1.0.0", + "readable-stream": "^2.3.0", + "to-buffer": "^1.1.1", + "xtend": "^4.0.0" + }, + "engines": { + "node": ">= 0.8.0" + } + }, + "node_modules/tdigest": { + "version": "0.1.2", + "resolved": "https://registry.npmjs.org/tdigest/-/tdigest-0.1.2.tgz", + "integrity": "sha512-+G0LLgjjo9BZX2MfdvPfH+MKLCrxlXSYec5DaPYP1fe6Iyhf0/fSmJ0bFiZ1F8BT6cGXl2LpltQptzjXKWEkKA==", + "license": "MIT", + "dependencies": { + "bintrees": "1.0.2" + } + }, + "node_modules/thread-stream": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/thread-stream/-/thread-stream-3.1.0.tgz", + "integrity": "sha512-OqyPZ9u96VohAyMfJykzmivOrY2wfMSf3C5TtFJVgN+Hm6aj+voFhlK+kZEIv2FBh1X6Xp3DlnCOfEQ3B2J86A==", + "license": "MIT", + "dependencies": { + "real-require": "^0.2.0" + } + }, + "node_modules/through": { + "version": "2.3.8", + "resolved": "https://registry.npmjs.org/through/-/through-2.3.8.tgz", + "integrity": "sha512-w89qg7PI8wAdvX60bMDP+bFoD5Dvhm9oLheFp5O4a2QF0cSBGsBX4qZmadPMvVqlLJBBci+WqGGOAPvcDeNSVg==", + "license": "MIT" + }, + "node_modules/to-buffer": { + "version": "1.2.2", + "resolved": "https://registry.npmjs.org/to-buffer/-/to-buffer-1.2.2.tgz", + "integrity": "sha512-db0E3UJjcFhpDhAF4tLo03oli3pwl3dbnzXOUIlRKrp+ldk/VUxzpWYZENsw2SZiuBjHAk7DfB0VU7NKdpb6sw==", + "license": "MIT", + "dependencies": { + "isarray": "^2.0.5", + "safe-buffer": "^5.2.1", + "typed-array-buffer": "^1.0.3" + }, + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/to-buffer/node_modules/isarray": { + "version": "2.0.5", + "resolved": "https://registry.npmjs.org/isarray/-/isarray-2.0.5.tgz", + "integrity": "sha512-xHjhDr3cNBK0BzdUJSPXZntQUx/mwMS5Rw4A7lPJ90XGAO6ISP/ePDNuo0vhqOZU+UD5JoodwCAAoZQd3FeAKw==", + "license": "MIT" + }, + "node_modules/to-regex-range": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/to-regex-range/-/to-regex-range-5.0.1.tgz", + "integrity": "sha512-65P7iz6X5yEr1cwcgvQxbbIw7Uk3gOy5dIdtZ4rDveLqhrdJP+Li/Hx6tyK0NEb+2GCyneCMJiGqrADCSNk8sQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "is-number": "^7.0.0" + }, + "engines": { + "node": ">=8.0" + } + }, + "node_modules/toidentifier": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/toidentifier/-/toidentifier-1.0.1.tgz", + "integrity": "sha512-o5sSPKEkg/DIQNmH43V0/uerLrpzVedkUh8tGNvaeXpfpuwjKenlSox/2O/BTlZUtEe+JG7s5YhEz608PlAHRA==", + "license": "MIT", + "engines": { + "node": ">=0.6" + } + }, + "node_modules/touch": { + "version": "3.1.1", + "resolved": "https://registry.npmjs.org/touch/-/touch-3.1.1.tgz", + "integrity": "sha512-r0eojU4bI8MnHr8c5bNo7lJDdI2qXlWWJk6a9EAFG7vbhTjElYhBVS3/miuE0uOuoLdb8Mc/rVfsmm6eo5o9GA==", + "dev": true, + "license": "ISC", + "bin": { + "nodetouch": "bin/nodetouch.js" + } + }, + "node_modules/tr46": { + "version": "0.0.3", + "resolved": "https://registry.npmjs.org/tr46/-/tr46-0.0.3.tgz", + "integrity": "sha512-N3WMsuqV66lT30CrXNbEjx4GEwlow3v6rr4mCcv6prnfwhS01rkgyFdjPNBYd9br7LpXV1+Emh01fHnq2Gdgrw==", + "license": "MIT" + }, + "node_modules/ts-node": { + "version": "10.9.2", + "resolved": "https://registry.npmjs.org/ts-node/-/ts-node-10.9.2.tgz", + "integrity": "sha512-f0FFpIdcHgn8zcPSbf1dRevwt047YMnaiJM3u2w2RewrB+fob/zePZcrOyQoLMMO7aBIddLcQIEK5dYjkLnGrQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "@cspotcode/source-map-support": "^0.8.0", + "@tsconfig/node10": "^1.0.7", + "@tsconfig/node12": "^1.0.7", + "@tsconfig/node14": "^1.0.0", + "@tsconfig/node16": "^1.0.2", + "acorn": "^8.4.1", + "acorn-walk": "^8.1.1", + "arg": "^4.1.0", + "create-require": "^1.1.0", + "diff": "^4.0.1", + "make-error": "^1.1.1", + "v8-compile-cache-lib": "^3.0.1", + "yn": "3.1.1" + }, + "bin": { + "ts-node": "dist/bin.js", + "ts-node-cwd": "dist/bin-cwd.js", + "ts-node-esm": "dist/bin-esm.js", + "ts-node-script": "dist/bin-script.js", + "ts-node-transpile-only": "dist/bin-transpile.js", + "ts-script": "dist/bin-script-deprecated.js" + }, + "peerDependencies": { + "@swc/core": ">=1.2.50", + "@swc/wasm": ">=1.2.50", + "@types/node": "*", + "typescript": ">=2.7" + }, + "peerDependenciesMeta": { + "@swc/core": { + "optional": true + }, + "@swc/wasm": { + "optional": true + } + } + }, + "node_modules/tsconfig-paths": { + "version": "4.2.0", + "resolved": "https://registry.npmjs.org/tsconfig-paths/-/tsconfig-paths-4.2.0.tgz", + "integrity": "sha512-NoZ4roiN7LnbKn9QqE1amc9DJfzvZXxF4xDavcOWt1BPkdx+m+0gJuPM+S0vCe7zTJMYUP0R8pO2XMr+Y8oLIg==", + "dev": true, + "license": "MIT", + "dependencies": { + "json5": "^2.2.2", + "minimist": "^1.2.6", + "strip-bom": "^3.0.0" + }, + "engines": { + "node": ">=6" + } + }, + "node_modules/type-is": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/type-is/-/type-is-2.0.1.tgz", + "integrity": "sha512-OZs6gsjF4vMp32qrCbiVSkrFmXtG/AZhY3t0iAMrMBiAZyV9oALtXO8hsrHbMXF9x6L3grlFuwW2oAz7cav+Gw==", + "license": "MIT", + "dependencies": { + "content-type": "^1.0.5", + "media-typer": "^1.1.0", + "mime-types": "^3.0.0" + }, + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/typed-array-buffer": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/typed-array-buffer/-/typed-array-buffer-1.0.3.tgz", + "integrity": "sha512-nAYYwfY3qnzX30IkA6AQZjVbtK6duGontcQm1WSG1MD94YLqK0515GNApXkoxKOWMusVssAHWLh9SeaoefYFGw==", + "license": "MIT", + "dependencies": { + "call-bound": "^1.0.3", + "es-errors": "^1.3.0", + "is-typed-array": "^1.1.14" + }, + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/typescript": { + "version": "5.9.3", + "resolved": "https://registry.npmjs.org/typescript/-/typescript-5.9.3.tgz", + "integrity": "sha512-jl1vZzPDinLr9eUt3J/t7V6FgNEw9QjvBPdysz9KfQDD41fQrC2Y4vKQdiaUpFT4bXlb1RHhLpp8wtm6M5TgSw==", + "dev": true, + "license": "Apache-2.0", + "bin": { + "tsc": "bin/tsc", + "tsserver": "bin/tsserver" + }, + "engines": { + "node": ">=14.17" + } + }, + "node_modules/unbzip2-stream": { + "version": "1.4.3", + "resolved": "https://registry.npmjs.org/unbzip2-stream/-/unbzip2-stream-1.4.3.tgz", + "integrity": "sha512-mlExGW4w71ebDJviH16lQLtZS32VKqsSfk80GCfUlwT/4/hNRFsoscrF/c++9xinkMzECL1uL9DDwXqFWkruPg==", + "license": "MIT", + "dependencies": { + "buffer": "^5.2.1", + "through": "^2.3.8" + } + }, + "node_modules/undefsafe": { + "version": "2.0.5", + "resolved": "https://registry.npmjs.org/undefsafe/-/undefsafe-2.0.5.tgz", + "integrity": "sha512-WxONCrssBM8TSPRqN5EmsjVrsv4A8X12J4ArBiiayv3DyyG3ZlIg6yysuuSYdZsVz3TKcTg2fd//Ujd4CHV1iA==", + "dev": true, + "license": "MIT" + }, + "node_modules/undici-types": { + "version": "6.21.0", + "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-6.21.0.tgz", + "integrity": "sha512-iwDZqg0QAGrg9Rav5H4n0M64c3mkR59cJ6wQp+7C4nI0gsmExaedaYLNO44eT4AtBBwjbTiGPMlt2Md0T9H9JQ==", + "license": "MIT" + }, + "node_modules/unpipe": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/unpipe/-/unpipe-1.0.0.tgz", + "integrity": "sha512-pjy2bYhSsufwWlKwPc+l3cN7+wuJlK6uz0YdJEOlQDbl6jo/YlPi4mb8agUkVC8BF7V8NuzeyPNqRksA3hztKQ==", + "license": "MIT", + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/util-deprecate": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/util-deprecate/-/util-deprecate-1.0.2.tgz", + "integrity": "sha512-EPD5q1uXyFxJpCrLnCc1nHnq3gOa6DZBocAIiI2TaSCA7VCJ1UJDMagCzIkXNsUYfD1daK//LTEQ8xiIbrHtcw==", + "license": "MIT" + }, + "node_modules/uuid": { + "version": "11.1.0", + "resolved": "https://registry.npmjs.org/uuid/-/uuid-11.1.0.tgz", + "integrity": "sha512-0/A9rDy9P7cJ+8w1c9WD9V//9Wj15Ce2MPz8Ri6032usz+NfePxx5AcN3bN+r6ZL6jEo066/yNYB3tn4pQEx+A==", + "funding": [ + "https://github.com/sponsors/broofa", + "https://github.com/sponsors/ctavan" + ], + "license": "MIT", + "bin": { + "uuid": "dist/esm/bin/uuid" + } + }, + "node_modules/v8-compile-cache-lib": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/v8-compile-cache-lib/-/v8-compile-cache-lib-3.0.1.tgz", + "integrity": "sha512-wa7YjyUGfNZngI/vtK0UHAN+lgDCxBPCylVXGp0zu59Fz5aiGtNXaq3DhIov063MorB+VfufLh3JlF2KdTK3xg==", + "dev": true, + "license": "MIT" + }, + "node_modules/validator": { + "version": "13.15.23", + "resolved": "https://registry.npmjs.org/validator/-/validator-13.15.23.tgz", + "integrity": "sha512-4yoz1kEWqUjzi5zsPbAS/903QXSYp0UOtHsPpp7p9rHAw/W+dkInskAE386Fat3oKRROwO98d9ZB0G4cObgUyw==", + "license": "MIT", + "engines": { + "node": ">= 0.10" + } + }, + "node_modules/vary": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/vary/-/vary-1.1.2.tgz", + "integrity": "sha512-BNGbWLfd0eUPabhkXUVm0j8uuvREyTh5ovRa/dyow/BqAbZJyC+5fU+IzQOzmAKzYqYRAISoRhdQr3eIZ/PXqg==", + "license": "MIT", + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/wav-encoder": { + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/wav-encoder/-/wav-encoder-1.3.0.tgz", + "integrity": "sha512-FXJdEu2qDOI+wbVYZpu21CS1vPEg5NaxNskBr4SaULpOJMrLE6xkH8dECa7PiS+ZoeyvP7GllWUAxPN3AvFSEw==", + "license": "MIT" + }, + "node_modules/web-streams-polyfill": { + "version": "4.0.0-beta.3", + "resolved": "https://registry.npmjs.org/web-streams-polyfill/-/web-streams-polyfill-4.0.0-beta.3.tgz", + "integrity": "sha512-QW95TCTaHmsYfHDybGMwO5IJIM93I/6vTRk+daHTWFPhwh+C8Cg7j7XyKrwrj8Ib6vYXe0ocYNrmzY4xAAN6ug==", + "license": "MIT", + "engines": { + "node": ">= 14" + } + }, + "node_modules/webidl-conversions": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/webidl-conversions/-/webidl-conversions-3.0.1.tgz", + "integrity": "sha512-2JAn3z8AR6rjK8Sm8orRC0h/bcl/DqL7tRPdGZ4I1CjdF+EaMLmYxBHyXuKL849eucPFhvBoxMsflfOb8kxaeQ==", + "license": "BSD-2-Clause" + }, + "node_modules/whatwg-url": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/whatwg-url/-/whatwg-url-5.0.0.tgz", + "integrity": "sha512-saE57nupxk6v3HY35+jzBwYa0rKSy0XR8JSxZPwgLr7ys0IBzhGviA1/TUGJLmSVqs8pb9AnvICXEuOHLprYTw==", + "license": "MIT", + "dependencies": { + "tr46": "~0.0.3", + "webidl-conversions": "^3.0.0" + } + }, + "node_modules/which-typed-array": { + "version": "1.1.19", + "resolved": "https://registry.npmjs.org/which-typed-array/-/which-typed-array-1.1.19.tgz", + "integrity": "sha512-rEvr90Bck4WZt9HHFC4DJMsjvu7x+r6bImz0/BrbWb7A2djJ8hnZMrWnHo9F8ssv0OMErasDhftrfROTyqSDrw==", + "license": "MIT", + "dependencies": { + "available-typed-arrays": "^1.0.7", + "call-bind": "^1.0.8", + "call-bound": "^1.0.4", + "for-each": "^0.3.5", + "get-proto": "^1.0.1", + "gopd": "^1.2.0", + "has-tostringtag": "^1.0.2" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/wrappy": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/wrappy/-/wrappy-1.0.2.tgz", + "integrity": "sha512-l4Sp/DRseor9wL6EvV2+TuQn63dMkPjZ/sp9XkghTEbV9KlPS1xUsZ3u7/IQO4wxtcFB4bgpQPRcR3QCvezPcQ==", + "license": "ISC" + }, + "node_modules/ws": { + "version": "8.18.3", + "resolved": "https://registry.npmjs.org/ws/-/ws-8.18.3.tgz", + "integrity": "sha512-PEIGCY5tSlUt50cqyMXfCzX+oOPqN0vuGqWzbcJ2xvnkzkq46oOpz7dQaTDBdfICb4N14+GARUDw2XV2N4tvzg==", + "license": "MIT", + "engines": { + "node": ">=10.0.0" + }, + "peerDependencies": { + "bufferutil": "^4.0.1", + "utf-8-validate": ">=5.0.2" + }, + "peerDependenciesMeta": { + "bufferutil": { + "optional": true + }, + "utf-8-validate": { + "optional": true + } + } + }, + "node_modules/xtend": { + "version": "4.0.2", + "resolved": "https://registry.npmjs.org/xtend/-/xtend-4.0.2.tgz", + "integrity": "sha512-LKYU1iAXJXUgAXn9URjiu+MWhyUXHsvfp7mcuYm9dSUKK0/CjtrUwFAxD82/mCWbtLsGjFIad0wIsod4zrTAEQ==", + "license": "MIT", + "engines": { + "node": ">=0.4" + } + }, + "node_modules/yauzl": { + "version": "2.10.0", + "resolved": "https://registry.npmjs.org/yauzl/-/yauzl-2.10.0.tgz", + "integrity": "sha512-p4a9I6X6nu6IhoGmBqAcbJy1mlC4j27vEPZX9F4L4/vZT3Lyq1VkFHw/V/PUcB9Buo+DG3iHkT0x3Qya58zc3g==", + "license": "MIT", + "dependencies": { + "buffer-crc32": "~0.2.3", + "fd-slicer": "~1.1.0" + } + }, + "node_modules/yn": { + "version": "3.1.1", + "resolved": "https://registry.npmjs.org/yn/-/yn-3.1.1.tgz", + "integrity": "sha512-Ux4ygGWsu2c7isFWe8Yu1YluJmqVhxqK2cLXNQA5AcC3QfbGNpM7fu0Y8b/z16pXLnFxZYvWhd3fhBY9DLmC6Q==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=6" + } + }, + "node_modules/zod": { + "version": "4.1.13", + "resolved": "https://registry.npmjs.org/zod/-/zod-4.1.13.tgz", + "integrity": "sha512-AvvthqfqrAhNH9dnfmrfKzX5upOdjUVJYFqNSlkmGf64gRaTzlPwz99IHYnVs28qYAybvAlBV+H7pn0saFY4Ig==", + "license": "MIT", + "funding": { + "url": "https://github.com/sponsors/colinhacks" + } + }, + "node_modules/zod-to-json-schema": { + "version": "3.25.0", + "resolved": "https://registry.npmjs.org/zod-to-json-schema/-/zod-to-json-schema-3.25.0.tgz", + "integrity": "sha512-HvWtU2UG41LALjajJrML6uQejQhNJx+JBO9IflpSja4R03iNWfKXrj6W2h7ljuLyc1nKS+9yDyL/9tD1U/yBnQ==", + "license": "ISC", + "peerDependencies": { + "zod": "^3.25 || ^4" + } + } + } +} diff --git a/realtime-service/src/package.json b/realtime-service/src/package.json new file mode 100644 index 0000000..d12a8f1 --- /dev/null +++ b/realtime-service/src/package.json @@ -0,0 +1,36 @@ +{ + "name": "server", + "main": "index.js", + "license": "SEE LICENSE IN LICENSE.md and LICENSE-CPP-BINARIES.md", + "scripts": { + "start": "nodemon -r tsconfig-paths/register index.ts", + "start:debug": "node-ts index.ts", + "build": "tsc" + }, + "devDependencies": { + "@types/cors": "^2.8.17", + "@types/express": "^5.0.1", + "@types/express-validator": "^3.0.0", + "@types/node": "^22.14.0", + "@types/ws": "^8.18.1", + "nodemon": "^3.1.9", + "ts-node": "^10.9.2", + "tsconfig-paths": "^4.2.0", + "typescript": "^5.8.3" + }, + "dependencies": { + "@inworld/runtime": "https://storage.googleapis.com/assets-inworld-ai/node-packages/inworld-runtime-0.9.0-rc.13.tgz", + "cors": "^2.8.5", + "dotenv": "^16.4.7", + "express": "^5.1.0", + "express-validator": "^7.2.1", + "groq-sdk": "^0.7.0", + "pino": "^9.14.0", + "pino-pretty": "^13.1.2", + "prom-client": "^15.1.3", + "uuid": "^11.1.0", + "wav-encoder": "^1.3.0", + "ws": "^8.18.1", + "zod": "^4.1.13" + } +} diff --git a/realtime-service/src/tsconfig.json b/realtime-service/src/tsconfig.json new file mode 100644 index 0000000..fc82f5d --- /dev/null +++ b/realtime-service/src/tsconfig.json @@ -0,0 +1,23 @@ +{ + "compilerOptions": { + "declaration": true, + "module": "Node16", + "moduleResolution": "node16", + "noImplicitAny": true, + "resolveJsonModule": true, + "target": "es6", + "types": ["node"], + "outDir": "./dist", + "rootDir": "./", + "baseUrl": ".", + "paths": { + "@inworld/runtime/common": ["./node_modules/@inworld/runtime/build/common/export"], + "@inworld/runtime/graph": ["./node_modules/@inworld/runtime/build/graph/export"], + "@inworld/runtime/graph/nodes": ["./node_modules/@inworld/runtime/build/graph/dsl/nodes/builtin/export"], + "@inworld/runtime/graph/components": ["./node_modules/@inworld/runtime/build/graph/dsl/components/export"], + "@inworld/runtime/core": ["./node_modules/@inworld/runtime/build/core/export"], + "@inworld/runtime/primitives/*": ["./node_modules/@inworld/runtime/build/primitives/*/index"], + "@inworld/runtime/telemetry": ["./node_modules/@inworld/runtime/build/telemetry/export"] + } + } +} diff --git a/realtime-service/src/types/index.ts b/realtime-service/src/types/index.ts new file mode 100644 index 0000000..2a80e1f --- /dev/null +++ b/realtime-service/src/types/index.ts @@ -0,0 +1,108 @@ +import { AudioChunkInterface } from '@inworld/runtime/common'; +import { GraphOutputStream } from '@inworld/runtime/graph'; + +import { MultimodalStreamManager } from '../components/audio/multimodal_stream_manager'; + +export enum EVENT_TYPE { + TEXT = 'TEXT', + AUDIO = 'AUDIO', + AUDIO_SESSION_END = 'audioSessionEnd', + NEW_INTERACTION = 'newInteraction', + CANCEL_RESPONSE = 'CANCEL_RESPONSE', +} + +export enum AUDIO_SESSION_STATE { + PROCESSING = 'PROCESSING', + ACTIVE = 'ACTIVE', + INACTIVE = 'INACTIVE', +} + +export interface ChatMessage { + id: string; + role: string; + content: string; + tool_call_id?: string; // For tool/function call outputs +} + +export interface Agent { + id: string; + name: string; + description: string; + motivation: string; + knowledge?: string[]; +} + +export interface TextInput { + sessionId: string; + text: string; + interactionId: string; + voiceId?: string; +} + +export interface AudioInput { + sessionId: string; + audio: AudioChunkInterface; + state: State; + interactionId: string; +} + +export interface AudioStreamInput { + sessionId: string; + state: State; + voiceId?: string; +} + +export interface State { + interactionId: string; + agent: Agent; + userName: string; + messages: ChatMessage[]; + voiceId?: string; + tools?: any[]; + toolChoice?: string | object; + eagerness?: 'low' | 'medium' | 'high'; + output_modalities?: ('text' | 'audio')[]; +} + +export interface Connection { + apiKey: string; + workspaceId: string; + state: State; + ws: any; + unloaded?: true; + multimodalStreamManager?: MultimodalStreamManager; + currentAudioGraphExecution?: Promise; + // Track execution streams so they can be aborted + currentAudioExecutionStream?: GraphOutputStream; + onSpeechDetected?: (interactionId: string) => void; // Callback when speech is detected (triggers input_audio_buffer.speech_started event) + onPartialTranscript?: (text: string, interactionId: string) => void; // Callback for partial transcripts +} + +export type ConnectionsMap = { + [sessionId: string]: Connection; +}; + +export interface PromptInput { + agent: Agent; + messages: ChatMessage[]; + userName: string; + userQuery: string; +} + +export interface CreateGraphPropsInterface { + llmModelName: string; + llmProvider: string; + voiceId: string; + graphVisualizationEnabled: boolean; + connections: ConnectionsMap; + ttsModelId: string; + useAssemblyAI?: boolean; // Use Assembly.AI streaming STT (should always be true for audio input) + assemblyAIApiKey?: string; // Assembly.AI API key (required for audio input) + useMocks?: boolean; // Use mock components (FakeTTSComponent, FakeRemoteLLMComponent) instead of real ones +} + +export interface InteractionInfo { + sessionId: string; + interactionId: string; + text: string; +} diff --git a/realtime-service/src/types/realtime.ts b/realtime-service/src/types/realtime.ts new file mode 100644 index 0000000..a71c088 --- /dev/null +++ b/realtime-service/src/types/realtime.ts @@ -0,0 +1,526 @@ +/** + * OpenAI Realtime API Type Definitions + * Based on the OpenAI Realtime API specification + */ + +import { GraphTypes } from '@inworld/runtime/graph'; + +// ============================================================================ +// Client Events (sent from client to server) +// ============================================================================ + +export interface ClientEventBase { + event_id?: string; +} + +export interface SessionUpdateEvent extends ClientEventBase { + type: 'session.update'; + session: Partial; +} + +export interface InputAudioBufferAppendEvent extends ClientEventBase { + type: 'input_audio_buffer.append'; + audio: string; // base64-encoded audio +} + +export interface InputAudioBufferCommitEvent extends ClientEventBase { + type: 'input_audio_buffer.commit'; +} + +export interface InputAudioBufferClearEvent extends ClientEventBase { + type: 'input_audio_buffer.clear'; +} + +export interface ConversationItemCreateEvent extends ClientEventBase { + type: 'conversation.item.create'; + previous_item_id?: string; + item: ConversationItem; +} + +export interface ConversationItemTruncateEvent extends ClientEventBase { + type: 'conversation.item.truncate'; + item_id: string; + content_index: number; + audio_end_ms: number; +} + +export interface ConversationItemDeleteEvent extends ClientEventBase { + type: 'conversation.item.delete'; + item_id: string; +} + +export interface ConversationItemRetrieveEvent extends ClientEventBase { + type: 'conversation.item.retrieve'; + item_id: string; +} + +export interface ResponseCreateEvent extends ClientEventBase { + type: 'response.create'; + response?: ResponseConfig; +} + +export interface ResponseCancelEvent extends ClientEventBase { + type: 'response.cancel'; + response_id?: string; +} + +export type ClientEvent = + | SessionUpdateEvent + | InputAudioBufferAppendEvent + | InputAudioBufferCommitEvent + | InputAudioBufferClearEvent + | ConversationItemCreateEvent + | ConversationItemTruncateEvent + | ConversationItemDeleteEvent + | ConversationItemRetrieveEvent + | ResponseCreateEvent + | ResponseCancelEvent; + +// ============================================================================ +// Server Events (sent from server to client) +// ============================================================================ + +export interface ServerEventBase { + event_id: string; +} + +export interface SessionCreatedEvent extends ServerEventBase { + type: 'session.created'; + session: Session; +} + +export interface SessionUpdatedEvent extends ServerEventBase { + type: 'session.updated'; + session: Session; +} + +export interface ConversationItemAddedEvent extends ServerEventBase { + type: 'conversation.item.added'; + previous_item_id: string | null; + item: ConversationItem; +} + +export interface ConversationItemDoneEvent extends ServerEventBase { + type: 'conversation.item.done'; + previous_item_id: string | null; + item: ConversationItem; +} + +export interface ConversationItemRetrievedEvent extends ServerEventBase { + type: 'conversation.item.retrieved'; + item: ConversationItem; +} + +export interface ConversationItemTruncatedEvent extends ServerEventBase { + type: 'conversation.item.truncated'; + item_id: string; + content_index: number; + audio_end_ms: number; +} + +export interface ConversationItemDeletedEvent extends ServerEventBase { + type: 'conversation.item.deleted'; + item_id: string; +} + +export interface InputAudioBufferCommittedEvent extends ServerEventBase { + type: 'input_audio_buffer.committed'; + previous_item_id: string | null; + item_id: string; +} + +export interface InputAudioBufferClearedEvent extends ServerEventBase { + type: 'input_audio_buffer.cleared'; +} + +export interface InputAudioBufferSpeechStartedEvent extends ServerEventBase { + type: 'input_audio_buffer.speech_started'; + audio_start_ms: number; + item_id: string; +} + +export interface InputAudioBufferSpeechStoppedEvent extends ServerEventBase { + type: 'input_audio_buffer.speech_stopped'; + audio_end_ms: number; + item_id: string; +} + +export interface ConversationItemInputAudioTranscriptionDeltaEvent + extends ServerEventBase { + type: 'conversation.item.input_audio_transcription.delta'; + item_id: string; + content_index: number; + delta: string; +} + +export interface ConversationItemInputAudioTranscriptionCompletedEvent + extends ServerEventBase { + type: 'conversation.item.input_audio_transcription.completed'; + item_id: string; + content_index: number; + transcript: string; +} + +export interface ResponseCreatedEvent extends ServerEventBase { + type: 'response.created'; + response: Response; +} + + +export interface ResponseDoneEvent extends ServerEventBase { + type: 'response.done'; + response: Response; +} + +export interface ResponseOutputItemAddedEvent extends ServerEventBase { + type: 'response.output_item.added'; + response_id: string; + output_index: number; + item: ConversationItem; +} + +export interface ResponseOutputItemDoneEvent extends ServerEventBase { + type: 'response.output_item.done'; + response_id: string; + output_index: number; + item: ConversationItem; +} + +export interface ResponseContentPartAddedEvent extends ServerEventBase { + type: 'response.content_part.added'; + response_id: string; + item_id: string; + output_index: number; + content_index: number; + part: ContentPart; +} + +export interface ResponseContentPartDoneEvent extends ServerEventBase { + type: 'response.content_part.done'; + response_id: string; + item_id: string; + output_index: number; + content_index: number; + part: ContentPart; +} + +export interface ResponseAudioDeltaEvent extends ServerEventBase { + type: 'response.output_audio.delta'; + response_id: string; + item_id: string; + output_index: number; + content_index: number; + delta: string; // base64-encoded audio +} + +export interface ResponseAudioDoneEvent extends ServerEventBase { + type: 'response.output_audio.done'; + response_id: string; + item_id: string; + output_index: number; + content_index: number; +} + +export interface ResponseAudioTranscriptDeltaEvent extends ServerEventBase { + type: 'response.output_audio_transcript.delta'; + response_id: string; + item_id: string; + output_index: number; + content_index: number; + delta: string; +} + +export interface ResponseAudioTranscriptDoneEvent extends ServerEventBase { + type: 'response.output_audio_transcript.done'; + response_id: string; + item_id: string; + output_index: number; + content_index: number; + transcript: string; +} + +export interface ResponseFunctionCallArgumentsDeltaEvent extends ServerEventBase { + type: 'response.function_call_arguments.delta'; + response_id: string; + item_id: string; + output_index: number; + call_id: string; + delta: string; +} + +export interface ResponseFunctionCallArgumentsDoneEvent extends ServerEventBase { + type: 'response.function_call_arguments.done'; + response_id: string; + item_id: string; + output_index: number; + call_id: string; + arguments: string; +} + +export interface ResponseTextDeltaEvent extends ServerEventBase { + type: 'response.output_text.delta'; + response_id: string; + item_id: string; + output_index: number; + content_index: number; + delta: string; +} + +export interface ResponseTextDoneEvent extends ServerEventBase { + type: 'response.output_text.done'; + response_id: string; + item_id: string; + output_index: number; + content_index: number; + text: string; +} + +export interface ErrorEvent extends ServerEventBase { + type: 'error'; + error: { + type: string; + code: string | null; + message: string; + param: string | null; + event_id: string | null; + }; +} + +export interface RateLimitsUpdatedEvent extends ServerEventBase { + type: 'rate_limits.updated'; + rate_limits: Array<{ + name: 'requests' | 'tokens'; + limit: number; + remaining: number; + reset_seconds: number; + }>; +} + +export type ServerEvent = + | SessionCreatedEvent + | SessionUpdatedEvent + | ConversationItemAddedEvent + | ConversationItemDoneEvent + | ConversationItemRetrievedEvent + | ConversationItemTruncatedEvent + | ConversationItemDeletedEvent + | InputAudioBufferCommittedEvent + | InputAudioBufferClearedEvent + | InputAudioBufferSpeechStartedEvent + | InputAudioBufferSpeechStoppedEvent + | ConversationItemInputAudioTranscriptionDeltaEvent + | ConversationItemInputAudioTranscriptionCompletedEvent + | ResponseCreatedEvent + | ResponseDoneEvent + | ResponseOutputItemAddedEvent + | ResponseOutputItemDoneEvent + | ResponseContentPartAddedEvent + | ResponseContentPartDoneEvent + | ResponseAudioDeltaEvent + | ResponseAudioDoneEvent + | ResponseAudioTranscriptDeltaEvent + | ResponseAudioTranscriptDoneEvent + | ResponseFunctionCallArgumentsDeltaEvent + | ResponseFunctionCallArgumentsDoneEvent + | ResponseTextDeltaEvent + | ResponseTextDoneEvent + | ErrorEvent + | RateLimitsUpdatedEvent; + +// ============================================================================ +// Data Types +// ============================================================================ + +export interface AudioInputConfig { + format: { + type: 'audio/pcm'; + rate: 24000; + }; + transcription?: { + model: string; + language?: string; + } | null; + noise_reduction?: { + type: 'near_field' | 'far_field'; + } | null; + turn_detection?: TurnDetection | null; +} + +export interface AudioOutputConfig { + format: { + type: 'audio/pcm'; + rate: 24000; + }; + voice: string; + speed?: number; +} + +export interface SessionConfig { + output_modalities?: ('text' | 'audio')[]; + instructions?: string; + audio?: { + input?: Partial; + output?: Partial; + }; + tools?: Tool[]; + tool_choice?: 'auto' | 'none' | 'required'; + temperature?: number; + max_output_tokens?: number | 'inf'; + truncation?: 'auto' | 'disabled'; + prompt?: string | null; + tracing?: string | null; + include?: string[] | null; +} + +export interface Session { + type: 'realtime'; + id: string; + object: 'realtime.session'; + model: string; + output_modalities: ('text' | 'audio')[]; + instructions: string; + audio: { + input: AudioInputConfig; + output: AudioOutputConfig; + }; + tools: Tool[]; + tool_choice: 'auto' | 'none' | 'required'; + temperature: number; + max_output_tokens: number | 'inf'; + truncation: 'auto' | 'disabled'; + prompt: string | null; + tracing: string | null; + expires_at: number; + include: string[] | null; +} + +// Turn detection can be either server_vad (simple) or semantic_vad (advanced with eagerness) +export interface ServerVADTurnDetection { + type: 'server_vad'; + threshold?: number; + prefix_padding_ms?: number; + silence_duration_ms?: number; + idle_timeout_ms?: number | null; + create_response?: boolean; + interrupt_response?: boolean; +} + +export interface SemanticVADTurnDetection { + type: 'semantic_vad'; + eagerness?: 'low' | 'medium' | 'high' | 'auto'; + create_response?: boolean; + interrupt_response?: boolean; +} + +export type TurnDetection = ServerVADTurnDetection | SemanticVADTurnDetection; + +export interface Tool { + type: 'function'; + name: string; + description?: string; + parameters?: Record; +} + +export interface ConversationItem { + id?: string; + type: 'message' | 'function_call' | 'function_call_output'; + object?: 'realtime.item'; + status?: 'completed' | 'in_progress' | 'incomplete'; + role?: 'user' | 'assistant' | 'system'; + content?: ContentPart[]; + call_id?: string; + name?: string; + arguments?: string; + output?: string; +} + +// Specific types for different conversation items +export interface MessageItem extends ConversationItem { + type: 'message'; + role: 'user' | 'assistant' | 'system'; + content: ContentPart[]; +} + +export interface FunctionCallItem extends ConversationItem { + type: 'function_call'; + call_id: string; + name: string; + arguments: string; +} + +export interface FunctionCallOutputItem extends ConversationItem { + type: 'function_call_output'; + call_id: string; + output: string; +} + +export interface ContentPart { + type: 'text' | 'audio' | 'input_text' | 'input_audio'; + text?: string; + audio?: string; // base64-encoded + transcript?: string; +} + +export interface ResponseConfig { + modalities?: ('text' | 'audio')[]; + instructions?: string; + voice?: string; + output_audio_format?: 'pcm16' | 'g711_ulaw' | 'g711_alaw'; + tools?: Tool[]; + tool_choice?: 'auto' | 'none' | 'required'; + temperature?: number; + max_output_tokens?: number | 'inf'; +} + +export interface Response { + id: string; + object: 'realtime.response'; + status: 'in_progress' | 'completed' | 'cancelled' | 'failed' | 'incomplete'; + status_details?: { + type: 'completed' | 'cancelled' | 'incomplete' | 'failed'; + reason?: string; + error?: { + type: string; + code?: string; + }; + } | null; + output: ConversationItem[]; + conversation_id?: string | null; + output_modalities?: ('text' | 'audio')[]; + max_output_tokens?: number | 'inf'; + audio?: { + output: AudioOutputConfig; + }; + usage?: { + total_tokens: number; + input_tokens: number; + output_tokens: number; + input_token_details?: { + cached_tokens?: number; + text_tokens?: number; + image_tokens?: number; + audio_tokens?: number; + }; + output_token_details?: { + text_tokens?: number; + audio_tokens?: number; + }; + } | null; + metadata?: any | null; +} + +// ============================================================================ +// Session State +// ============================================================================ + +export interface RealtimeSession { + id: string; + session: Session; + conversationItems: ConversationItem[]; + inputAudioBuffer: number[]; + currentResponse: Response | null; + audioStartMs: number; + // Store active streams so they can be aborted on cancel + currentContentStream?: GraphTypes.ContentStream | null; + currentTTSStream?: GraphTypes.TTSOutputStream | null; +} diff --git a/realtime-service/src/types/settings.ts b/realtime-service/src/types/settings.ts new file mode 100644 index 0000000..4fd332e --- /dev/null +++ b/realtime-service/src/types/settings.ts @@ -0,0 +1,45 @@ +/** + * Maps OpenAI eagerness levels to AssemblyAI turn detection settings + * Based on AssemblyAI's recommended configurations for different use cases + */ + +export interface AssemblyAITurnDetectionSettings { + endOfTurnConfidenceThreshold: number; + minEndOfTurnSilenceWhenConfident: number; + maxTurnSilence: number; + description: string; +} + +/** + * Get AssemblyAI turn detection settings for a given eagerness level + * @param eagerness - The eagerness level ('low' | 'medium' | 'high') + * @returns AssemblyAI turn detection settings including threshold values and description + */ +export function getAssemblyAISettingsForEagerness( + eagerness: 'low' | 'medium' | 'high' = 'medium' +): AssemblyAITurnDetectionSettings { + switch (eagerness) { + case 'high': // Aggressive - VERY responsive + return { + endOfTurnConfidenceThreshold: 0.4, + minEndOfTurnSilenceWhenConfident: 160, + maxTurnSilence: 320, + description: 'Aggressive - VERY quick responses, ideal for rapid Q&A (Agent Assist, IVR)', + }; + case 'medium': // Balanced (default) + return { + endOfTurnConfidenceThreshold: 0.4, + minEndOfTurnSilenceWhenConfident: 400, + maxTurnSilence: 1280, + description: 'Balanced - Natural conversation flow (Customer Support, Tech Support)', + }; + case 'low': // Conservative - VERY patient + return { + endOfTurnConfidenceThreshold: 0.7, + minEndOfTurnSilenceWhenConfident: 800, + maxTurnSilence: 3000, + description: 'Conservative - VERY patient, allows long thinking pauses (Complex inquiries)', + }; + } +} + diff --git a/realtime-service/tsconfig.test.json b/realtime-service/tsconfig.test.json new file mode 100644 index 0000000..96174c8 --- /dev/null +++ b/realtime-service/tsconfig.test.json @@ -0,0 +1,21 @@ +{ + "extends": "./src/tsconfig.json", + "compilerOptions": { + "module": "commonjs", + "moduleResolution": "node", + "types": ["node", "jest"], + "esModuleInterop": true, + "allowSyntheticDefaultImports": true + }, + "include": [ + "src/**/*.ts", + "src/**/*.test.ts", + "src/**/*.spec.ts", + "__tests__/**/*.ts" + ], + "exclude": [ + "node_modules", + "dist" + ] +} + diff --git a/realtime-service/yarn.lock b/realtime-service/yarn.lock new file mode 100644 index 0000000..42d12b0 --- /dev/null +++ b/realtime-service/yarn.lock @@ -0,0 +1,8 @@ +# THIS IS AN AUTOGENERATED FILE. DO NOT EDIT THIS FILE DIRECTLY. +# yarn lockfile v1 + + +ws@^8.18.3: + version "8.18.3" + resolved "https://registry.npmjs.org/ws/-/ws-8.18.3.tgz" + integrity sha512-PEIGCY5tSlUt50cqyMXfCzX+oOPqN0vuGqWzbcJ2xvnkzkq46oOpz7dQaTDBdfICb4N14+GARUDw2XV2N4tvzg== From 5a0a505587fd3a74992cc28d4262ffa861d72d3b Mon Sep 17 00:00:00 2001 From: Cale Shapera <25466659+cshape@users.noreply.github.com> Date: Tue, 30 Dec 2025 12:01:33 -0800 Subject: [PATCH 03/49] 0.9 working ok end to end in Spanish --- backend/components/audio/audio_utils.ts | 71 - .../graphs/conversation-graph.ts | 6 +- .../graphs/nodes/assembly_ai_stt_ws_node.ts | 166 +- .../nodes/dialog_prompt_builder_node.ts | 6 +- .../graphs/nodes/interaction_queue_node.ts | 2 +- .../graphs/nodes/state_update_node.ts | 2 +- .../graphs/nodes/text_input_node.ts | 2 +- .../graphs/nodes/transcript_extractor_node.ts | 2 +- .../graphs/nodes/tts_request_builder_node.ts | 4 +- backend/helpers/audio_utils.ts | 199 + backend/helpers/connection-manager.ts | 13 +- .../multimodal_stream_manager.ts | 2 + backend/server.ts | 2 +- backend/types/index.ts | 2 +- frontend/.gitignore | 24 + frontend/README.md | 73 + frontend/eslint.config.js | 23 + frontend/index.html | 108 +- frontend/js/audio-processor.js | 59 - frontend/js/chat-ui.js | 417 --- frontend/js/flashcard-ui.js | 179 - frontend/js/ios-audio-workarounds.js | 578 --- frontend/js/main.js | 923 ----- frontend/package-lock.json | 3212 ++++++++++++++++ frontend/package.json | 30 + frontend/public/audio-processor.js | 72 + frontend/public/vite.svg | 1 + frontend/src/App.tsx | 33 + frontend/src/components/ChatSection.tsx | 166 + frontend/src/components/Flashcard.tsx | 61 + frontend/src/components/FlashcardsSection.tsx | 125 + frontend/src/components/Header.tsx | 50 + frontend/src/components/Message.tsx | 86 + frontend/src/components/StreamingMessage.tsx | 94 + .../src/components/TranslationTooltip.tsx | 70 + frontend/src/context/AppContext.tsx | 619 ++++ frontend/src/hooks/useTranslator.ts | 26 + frontend/src/hooks/useTypewriter.ts | 63 + frontend/src/main.tsx | 9 + .../services/AudioHandler.ts} | 106 +- .../services/AudioPlayer.ts} | 204 +- .../storage.js => src/services/Storage.ts} | 99 +- .../services/Translator.ts} | 61 +- .../services/WebSocketClient.ts} | 145 +- frontend/{ => src}/styles/main.css | 0 frontend/src/types/global.d.ts | 11 + frontend/src/types/index.ts | 139 + frontend/src/vite-env.d.ts | 1 + frontend/tsconfig.app.json | 28 + frontend/tsconfig.json | 7 + frontend/tsconfig.node.json | 26 + frontend/vite.config.ts | 15 + realtime-service/.env-sample | 39 - realtime-service/Makefile | 53 - realtime-service/README.md | 91 - realtime-service/__tests__/README.md | 469 --- realtime-service/__tests__/api/README.md | 312 -- .../__tests__/api/realtime-api.spec.ts | 885 ----- .../__tests__/api/websocket-server-helper.ts | 133 - .../__tests__/api/websocket-test-helper.ts | 311 -- realtime-service/__tests__/config.ts | 8 - realtime-service/__tests__/setup.ts | 9 - realtime-service/__tests__/test-setup.ts | 32 - .../__tests__/utils/graph-test-helpers.ts | 71 - .../__tests__/utils/mock-helpers.ts | 93 - realtime-service/jest.config.js | 34 - realtime-service/load-tests/README.md | 61 - realtime-service/load-tests/load_test.js | 425 --- realtime-service/load-tests/package.json | 30 - realtime-service/mock/echo_ws_server.go | 50 - realtime-service/package-lock.json | 33 - realtime-service/package.json | 26 - realtime-service/src/.gitignore | 32 - realtime-service/src/REALTIME_API.md | 313 -- realtime-service/src/components/app.ts | 78 - .../src/components/audio/audio_utils.ts | 91 - .../audio/multimodal_stream_manager.ts | 125 - .../audio/realtime_audio_handler.ts | 213 -- .../src/components/graphs/graph.ts | 308 -- .../graphs/nodes/assembly_ai_stt_ws_node.ts | 855 ----- .../nodes/dialog_prompt_builder_node.ts | 87 - .../graphs/nodes/interaction_queue_node.ts | 178 - .../graphs/nodes/state_update_node.ts | 67 - .../graphs/nodes/text_input_node.ts | 75 - .../graphs/nodes/text_output_stream_node.ts | 29 - .../graphs/nodes/transcript_extractor_node.ts | 63 - .../graphs/nodes/tts_request_builder_node.ts | 64 - .../graphs/realtime_graph_executor.ts | 1180 ------ .../realtime/realtime_event_factory.ts | 499 --- .../realtime/realtime_message_handler.ts | 168 - .../realtime/realtime_session_manager.ts | 508 --- .../src/components/runtime_app_manager.ts | 140 - realtime-service/src/config.ts | 15 - realtime-service/src/helpers.ts | 65 - realtime-service/src/index.ts | 319 -- realtime-service/src/log-helpers.ts | 88 - realtime-service/src/logger.ts | 66 - realtime-service/src/package-lock.json | 3273 ----------------- realtime-service/src/package.json | 36 - realtime-service/src/tsconfig.json | 23 - realtime-service/src/types/index.ts | 108 - realtime-service/src/types/realtime.ts | 526 --- realtime-service/src/types/settings.ts | 45 - realtime-service/tsconfig.test.json | 21 - realtime-service/yarn.lock | 8 - 105 files changed, 5764 insertions(+), 15489 deletions(-) delete mode 100644 backend/components/audio/audio_utils.ts rename backend/{components => }/graphs/conversation-graph.ts (98%) rename backend/{components => }/graphs/nodes/assembly_ai_stt_ws_node.ts (80%) rename backend/{components => }/graphs/nodes/dialog_prompt_builder_node.ts (93%) rename backend/{components => }/graphs/nodes/interaction_queue_node.ts (99%) rename backend/{components => }/graphs/nodes/state_update_node.ts (96%) rename backend/{components => }/graphs/nodes/text_input_node.ts (95%) rename backend/{components => }/graphs/nodes/transcript_extractor_node.ts (96%) rename backend/{components => }/graphs/nodes/tts_request_builder_node.ts (93%) create mode 100644 backend/helpers/audio_utils.ts rename backend/{components/audio => helpers}/multimodal_stream_manager.ts (95%) create mode 100644 frontend/.gitignore create mode 100644 frontend/README.md create mode 100644 frontend/eslint.config.js delete mode 100644 frontend/js/audio-processor.js delete mode 100644 frontend/js/chat-ui.js delete mode 100644 frontend/js/flashcard-ui.js delete mode 100644 frontend/js/ios-audio-workarounds.js delete mode 100644 frontend/js/main.js create mode 100644 frontend/package-lock.json create mode 100644 frontend/package.json create mode 100644 frontend/public/audio-processor.js create mode 100644 frontend/public/vite.svg create mode 100644 frontend/src/App.tsx create mode 100644 frontend/src/components/ChatSection.tsx create mode 100644 frontend/src/components/Flashcard.tsx create mode 100644 frontend/src/components/FlashcardsSection.tsx create mode 100644 frontend/src/components/Header.tsx create mode 100644 frontend/src/components/Message.tsx create mode 100644 frontend/src/components/StreamingMessage.tsx create mode 100644 frontend/src/components/TranslationTooltip.tsx create mode 100644 frontend/src/context/AppContext.tsx create mode 100644 frontend/src/hooks/useTranslator.ts create mode 100644 frontend/src/hooks/useTypewriter.ts create mode 100644 frontend/src/main.tsx rename frontend/{js/audio-handler.js => src/services/AudioHandler.ts} (74%) rename frontend/{js/audio-player.js => src/services/AudioPlayer.ts} (54%) rename frontend/{js/storage.js => src/services/Storage.ts} (70%) rename frontend/{js/translator.js => src/services/Translator.ts} (53%) rename frontend/{js/websocket-client.js => src/services/WebSocketClient.ts} (56%) rename frontend/{ => src}/styles/main.css (100%) create mode 100644 frontend/src/types/global.d.ts create mode 100644 frontend/src/types/index.ts create mode 100644 frontend/src/vite-env.d.ts create mode 100644 frontend/tsconfig.app.json create mode 100644 frontend/tsconfig.json create mode 100644 frontend/tsconfig.node.json create mode 100644 frontend/vite.config.ts delete mode 100644 realtime-service/.env-sample delete mode 100644 realtime-service/Makefile delete mode 100644 realtime-service/README.md delete mode 100644 realtime-service/__tests__/README.md delete mode 100644 realtime-service/__tests__/api/README.md delete mode 100644 realtime-service/__tests__/api/realtime-api.spec.ts delete mode 100644 realtime-service/__tests__/api/websocket-server-helper.ts delete mode 100644 realtime-service/__tests__/api/websocket-test-helper.ts delete mode 100644 realtime-service/__tests__/config.ts delete mode 100644 realtime-service/__tests__/setup.ts delete mode 100644 realtime-service/__tests__/test-setup.ts delete mode 100644 realtime-service/__tests__/utils/graph-test-helpers.ts delete mode 100644 realtime-service/__tests__/utils/mock-helpers.ts delete mode 100644 realtime-service/jest.config.js delete mode 100644 realtime-service/load-tests/README.md delete mode 100644 realtime-service/load-tests/load_test.js delete mode 100644 realtime-service/load-tests/package.json delete mode 100644 realtime-service/mock/echo_ws_server.go delete mode 100644 realtime-service/package-lock.json delete mode 100644 realtime-service/package.json delete mode 100644 realtime-service/src/.gitignore delete mode 100644 realtime-service/src/REALTIME_API.md delete mode 100644 realtime-service/src/components/app.ts delete mode 100644 realtime-service/src/components/audio/audio_utils.ts delete mode 100644 realtime-service/src/components/audio/multimodal_stream_manager.ts delete mode 100644 realtime-service/src/components/audio/realtime_audio_handler.ts delete mode 100644 realtime-service/src/components/graphs/graph.ts delete mode 100644 realtime-service/src/components/graphs/nodes/assembly_ai_stt_ws_node.ts delete mode 100644 realtime-service/src/components/graphs/nodes/dialog_prompt_builder_node.ts delete mode 100644 realtime-service/src/components/graphs/nodes/interaction_queue_node.ts delete mode 100644 realtime-service/src/components/graphs/nodes/state_update_node.ts delete mode 100644 realtime-service/src/components/graphs/nodes/text_input_node.ts delete mode 100644 realtime-service/src/components/graphs/nodes/text_output_stream_node.ts delete mode 100644 realtime-service/src/components/graphs/nodes/transcript_extractor_node.ts delete mode 100644 realtime-service/src/components/graphs/nodes/tts_request_builder_node.ts delete mode 100644 realtime-service/src/components/graphs/realtime_graph_executor.ts delete mode 100644 realtime-service/src/components/realtime/realtime_event_factory.ts delete mode 100644 realtime-service/src/components/realtime/realtime_message_handler.ts delete mode 100644 realtime-service/src/components/realtime/realtime_session_manager.ts delete mode 100644 realtime-service/src/components/runtime_app_manager.ts delete mode 100644 realtime-service/src/config.ts delete mode 100644 realtime-service/src/helpers.ts delete mode 100644 realtime-service/src/index.ts delete mode 100644 realtime-service/src/log-helpers.ts delete mode 100644 realtime-service/src/logger.ts delete mode 100644 realtime-service/src/package-lock.json delete mode 100644 realtime-service/src/package.json delete mode 100644 realtime-service/src/tsconfig.json delete mode 100644 realtime-service/src/types/index.ts delete mode 100644 realtime-service/src/types/realtime.ts delete mode 100644 realtime-service/src/types/settings.ts delete mode 100644 realtime-service/tsconfig.test.json delete mode 100644 realtime-service/yarn.lock diff --git a/backend/components/audio/audio_utils.ts b/backend/components/audio/audio_utils.ts deleted file mode 100644 index f404a38..0000000 --- a/backend/components/audio/audio_utils.ts +++ /dev/null @@ -1,71 +0,0 @@ -/** - * Audio utility functions for format conversion - */ - -/** - * Convert Float32Array audio data to Int16Array (PCM16) - */ -export function float32ToPCM16(float32Data: Float32Array): Int16Array { - const pcm16 = new Int16Array(float32Data.length); - for (let i = 0; i < float32Data.length; i++) { - // Clamp to [-1, 1] range and convert to Int16 range [-32768, 32767] - const s = Math.max(-1, Math.min(1, float32Data[i])); - pcm16[i] = s < 0 ? s * 0x8000 : s * 0x7fff; - } - return pcm16; -} - -/** - * Convert Int16Array (PCM16) to Float32Array - */ -export function pcm16ToFloat32(pcm16Data: Int16Array): Float32Array { - const float32 = new Float32Array(pcm16Data.length); - for (let i = 0; i < pcm16Data.length; i++) { - float32[i] = pcm16Data[i] / 32768.0; - } - return float32; -} - -/** - * Convert audio data to PCM16 base64 string for WebSocket transmission - */ -export function convertToPCM16Base64( - audioData: number[] | Float32Array | string | undefined, - _sampleRate: number | undefined, - _logPrefix: string = 'Audio' -): string | null { - if (!audioData) { - return null; - } - - let base64Data: string; - - if (typeof audioData === 'string') { - // Already base64 encoded - base64Data = audioData; - } else { - // Convert Float32 array to PCM16 base64 - const float32Data = Array.isArray(audioData) - ? new Float32Array(audioData) - : audioData; - const pcm16Data = float32ToPCM16(float32Data); - base64Data = Buffer.from(pcm16Data.buffer).toString('base64'); - } - - return base64Data; -} - -/** - * Decode base64 audio to Float32Array - * Note: Node.js Buffer objects share ArrayBuffers with offsets, so we need to copy - */ -export function decodeBase64ToFloat32(base64Audio: string): Float32Array { - const buffer = Buffer.from(base64Audio, 'base64'); - // Create a clean copy to avoid Node.js Buffer's internal ArrayBuffer sharing - const cleanArray = new Uint8Array(buffer.length); - for (let i = 0; i < buffer.length; i++) { - cleanArray[i] = buffer[i]; - } - const int16Array = new Int16Array(cleanArray.buffer); - return pcm16ToFloat32(int16Array); -} diff --git a/backend/components/graphs/conversation-graph.ts b/backend/graphs/conversation-graph.ts similarity index 98% rename from backend/components/graphs/conversation-graph.ts rename to backend/graphs/conversation-graph.ts index b873b7d..5f64f43 100644 --- a/backend/components/graphs/conversation-graph.ts +++ b/backend/graphs/conversation-graph.ts @@ -35,12 +35,12 @@ import { TextInput, INPUT_SAMPLE_RATE, TTS_SAMPLE_RATE, -} from '../../types/index.js'; +} from '../types/index.js'; import { getLanguageConfig, DEFAULT_LANGUAGE_CODE, -} from '../../config/languages.js'; -import { getAssemblyAISettingsForEagerness } from '../../types/settings.js'; +} from '../config/languages.js'; +import { getAssemblyAISettingsForEagerness } from '../types/settings.js'; export interface ConversationGraphConfig { assemblyAIApiKey: string; diff --git a/backend/components/graphs/nodes/assembly_ai_stt_ws_node.ts b/backend/graphs/nodes/assembly_ai_stt_ws_node.ts similarity index 80% rename from backend/components/graphs/nodes/assembly_ai_stt_ws_node.ts rename to backend/graphs/nodes/assembly_ai_stt_ws_node.ts index ec4c7c5..7dd0bec 100644 --- a/backend/components/graphs/nodes/assembly_ai_stt_ws_node.ts +++ b/backend/graphs/nodes/assembly_ai_stt_ws_node.ts @@ -16,10 +16,10 @@ import { CustomNode, GraphTypes, ProcessContext } from '@inworld/runtime/graph'; import WebSocket from 'ws'; import { v4 as uuidv4 } from 'uuid'; -import { Connection } from '../../../types/index.js'; +import { Connection } from '../../types/index.js'; // Settings imported but used via constructor config -// import { getAssemblyAISettingsForEagerness } from '../../../types/settings.js'; -import { float32ToPCM16 } from '../../audio/audio_utils.js'; +// import { getAssemblyAISettingsForEagerness } from '../../types/settings.js'; +import { float32ToPCM16 } from '../../helpers/audio_utils.js'; /** * Configuration interface for AssemblyAISTTWebSocketNode @@ -59,10 +59,11 @@ class AssemblyAISession { private lastActivityTime: number = Date.now(); private readonly INACTIVITY_TIMEOUT_MS = 60000; // 60 seconds - // Audio buffering for AssemblyAI's chunk size requirements (50-1000ms) - private audioBuffer: Int16Array = new Int16Array(0); - // Buffer to ~100ms worth of audio before sending (1600 samples at 16kHz) - private readonly MIN_SAMPLES_TO_SEND = 1600; + // Track completed turns to prevent duplicate processing + private lastCompletedTranscript: string = ''; + private newSpeechReceived: boolean = false; + private lastTurnTimestamp: number = 0; + private readonly TURN_DEBOUNCE_MS = 500; // Minimum time between accepting turns constructor( public readonly sessionId: string, @@ -158,40 +159,13 @@ class AssemblyAISession { } } - public sendAudio(pcm16Data: Int16Array): void { - if (!this.ws || this.ws.readyState !== WebSocket.OPEN) { - return; - } - - // Append new audio to buffer - const newBuffer = new Int16Array(this.audioBuffer.length + pcm16Data.length); - newBuffer.set(this.audioBuffer, 0); - newBuffer.set(pcm16Data, this.audioBuffer.length); - this.audioBuffer = newBuffer; - - // Send if we have enough samples (100ms worth at 16kHz = 1600 samples) - if (this.audioBuffer.length >= this.MIN_SAMPLES_TO_SEND) { - this.ws.send(Buffer.from(this.audioBuffer.buffer)); - this.audioBuffer = new Int16Array(0); - this.resetInactivityTimer(); - } - } - /** - * Flush any remaining buffered audio (call before closing) + * Send audio data directly to AssemblyAI (no buffering) */ - public flushAudio(): void { - if (this.ws && this.ws.readyState === WebSocket.OPEN && this.audioBuffer.length > 0) { - // Pad to minimum size if needed - if (this.audioBuffer.length < 800) { - // Pad to at least 50ms (800 samples) - const paddedBuffer = new Int16Array(800); - paddedBuffer.set(this.audioBuffer, 0); - this.ws.send(Buffer.from(paddedBuffer.buffer)); - } else { - this.ws.send(Buffer.from(this.audioBuffer.buffer)); - } - this.audioBuffer = new Int16Array(0); + public sendAudio(pcm16Data: Int16Array): void { + if (this.ws && this.ws.readyState === WebSocket.OPEN) { + this.ws.send(Buffer.from(pcm16Data.buffer)); + this.resetInactivityTimer(); } } @@ -246,6 +220,68 @@ class AssemblyAISession { this.closeWebSocket(); } + + /** + * Mark that new speech was received (partial transcript) + */ + public markNewSpeechReceived(): void { + this.newSpeechReceived = true; + } + + /** + * Check if a turn should be accepted or rejected as duplicate + * Returns true if the turn is valid and should be processed + */ + public shouldAcceptTurn(transcript: string): boolean { + const now = Date.now(); + + // Check debounce - if a turn was just completed, reject + if (this.lastTurnTimestamp > 0 && now - this.lastTurnTimestamp < this.TURN_DEBOUNCE_MS) { + console.log(`[AssemblyAI] Rejecting turn - within debounce period (${now - this.lastTurnTimestamp}ms)`); + return false; + } + + // Normalize transcripts for comparison (lowercase, remove punctuation, trim) + const normalizeText = (text: string) => + text.toLowerCase().replace(/[.,!?;:'"]/g, '').replace(/\s+/g, ' ').trim(); + + const normalizedNew = normalizeText(transcript); + const normalizedLast = normalizeText(this.lastCompletedTranscript); + + // If no new speech was received since last turn, check for duplicate content + if (!this.newSpeechReceived && this.lastCompletedTranscript) { + // Check if transcripts are similar (allowing for minor formatting differences) + const isSimilar = normalizedNew === normalizedLast || + normalizedNew.includes(normalizedLast) || + normalizedLast.includes(normalizedNew); + + if (isSimilar) { + console.log(`[AssemblyAI] Rejecting turn - duplicate content without new speech`); + console.log(` Last: "${this.lastCompletedTranscript.substring(0, 50)}..."`); + console.log(` New: "${transcript.substring(0, 50)}..."`); + return false; + } + } + + return true; + } + + /** + * Mark that a turn was completed - store for deduplication + */ + public markTurnCompleted(transcript: string): void { + this.lastTurnTimestamp = Date.now(); + this.lastCompletedTranscript = transcript; + this.newSpeechReceived = false; // Reset for next turn + console.log(`[AssemblyAI] Turn completed, transcript stored for deduplication`); + } + + /** + * Reset turn tracking state (call when starting fresh interaction) + */ + public resetTurnTracking(): void { + this.newSpeechReceived = false; + } } /** @@ -300,8 +336,11 @@ export class AssemblyAISTTWebSocketNode extends CustomNode { this.maxTurnSilence = config.maxTurnSilence || 1280; this.defaultLanguage = config.language || 'es'; + const defaultModel = this.defaultLanguage === 'en' + ? 'universal-streaming-english' + : 'universal-streaming-multilingual'; console.log( - `[AssemblyAI] Configured with [threshold:${this.endOfTurnConfidenceThreshold}] [silence:${this.minEndOfTurnSilenceWhenConfident}ms] [lang:${this.defaultLanguage}]` + `[AssemblyAI] Configured with [model:${defaultModel}] [threshold:${this.endOfTurnConfidenceThreshold}] [silence:${this.minEndOfTurnSilenceWhenConfident}ms] [lang:${this.defaultLanguage}]` ); } @@ -324,18 +363,31 @@ export class AssemblyAISTTWebSocketNode extends CustomNode { } } + // Determine speech model based on language + // AssemblyAI's streaming v3 API: + // - 'universal-streaming-english' for English only (default) + // - 'universal-streaming-multilingual' for other languages (Spanish, Japanese, etc.) + const isEnglish = language === 'en'; + const speechModel = isEnglish + ? 'universal-streaming-english' + : 'universal-streaming-multilingual'; + + const languageCode = language === 'en' ? 'en' : 'multi'; + const params = new URLSearchParams({ sample_rate: this.sampleRate.toString(), + encoding: 'pcm_s16le', // Signed 16-bit little-endian PCM format_turns: this.formatTurns.toString(), end_of_turn_confidence_threshold: endOfTurnThreshold.toString(), min_end_of_turn_silence_when_confident: minSilenceWhenConfident.toString(), max_turn_silence: maxSilence.toString(), - language: language, + speech_model: speechModel, + language: languageCode }); const url = `${this.wsEndpointBaseUrl}?${params.toString()}`; console.log( - `[AssemblyAI] Connecting with [lang:${language}] [threshold:${endOfTurnThreshold}]` + `[AssemblyAI] Connecting with [model:${speechModel}] [lang:${language}] [threshold:${endOfTurnThreshold}]` ); return url; @@ -443,6 +495,7 @@ export class AssemblyAISTTWebSocketNode extends CustomNode { if (msgType === 'Turn') { if (session?.shouldStopProcessing) { + console.log(`[AssemblyAI] Ignoring Turn - processing stopped [iteration:${iteration}]`); return; } @@ -453,9 +506,14 @@ export class AssemblyAISTTWebSocketNode extends CustomNode { if (!transcript) return; if (!isFinal) { - // Partial transcript + // Partial transcript - mark that we're receiving new speech const textToSend = utterance || transcript; if (textToSend) { + console.log(`[AssemblyAI] Partial [iteration:${iteration}]: "${textToSend.substring(0, 30)}..."`); + + // Mark new speech received BEFORE sending partial + session?.markNewSpeechReceived(); + this.sendPartialTranscript(sessionId, nextInteractionId, textToSend); if (connection?.onSpeechDetected && !speechDetected) { @@ -467,14 +525,28 @@ export class AssemblyAISTTWebSocketNode extends CustomNode { return; } - // Final transcript + // Final transcript - check for duplicates console.log( - `[AssemblyAI] Turn detected [iteration:${iteration}]: "${transcript.substring(0, 50)}..."` + `[AssemblyAI] Turn candidate [iteration:${iteration}]: "${transcript.substring(0, 50)}..."` + ); + + // Check if we should accept this turn (not a duplicate) + if (session && !session.shouldAcceptTurn(transcript)) { + console.log(`[AssemblyAI] Turn rejected as duplicate [iteration:${iteration}]`); + return; + } + + // Accept the turn + console.log( + `[AssemblyAI] Turn ACCEPTED [iteration:${iteration}]: "${transcript.substring(0, 50)}..."` ); transcriptText = transcript; turnDetected = true; - if (session) session.shouldStopProcessing = true; + if (session) { + session.shouldStopProcessing = true; + session.markTurnCompleted(transcript); + } turnResolve(transcript); } else if (msgType === 'Termination') { console.log(`[AssemblyAI] Session terminated [iteration:${iteration}]`); @@ -562,8 +634,6 @@ export class AssemblyAISTTWebSocketNode extends CustomNode { if (maxDurationTimeout) { clearTimeout(maxDurationTimeout); } - // Flush any remaining buffered audio - session?.flushAudio(); } })(); diff --git a/backend/components/graphs/nodes/dialog_prompt_builder_node.ts b/backend/graphs/nodes/dialog_prompt_builder_node.ts similarity index 93% rename from backend/components/graphs/nodes/dialog_prompt_builder_node.ts rename to backend/graphs/nodes/dialog_prompt_builder_node.ts index 73d05f7..036f079 100644 --- a/backend/components/graphs/nodes/dialog_prompt_builder_node.ts +++ b/backend/graphs/nodes/dialog_prompt_builder_node.ts @@ -10,9 +10,9 @@ import { CustomNode, GraphTypes, ProcessContext } from '@inworld/runtime/graph'; import { PromptBuilder } from '@inworld/runtime/primitives/llm'; -import { ConnectionsMap, State } from '../../../types/index.js'; -import { getLanguageConfig } from '../../../config/languages.js'; -import { conversationTemplate } from '../../../helpers/prompt-templates.js'; +import { ConnectionsMap, State } from '../../types/index.js'; +import { getLanguageConfig } from '../../config/languages.js'; +import { conversationTemplate } from '../../helpers/prompt-templates.js'; export class DialogPromptBuilderNode extends CustomNode { constructor(props: { diff --git a/backend/components/graphs/nodes/interaction_queue_node.ts b/backend/graphs/nodes/interaction_queue_node.ts similarity index 99% rename from backend/components/graphs/nodes/interaction_queue_node.ts rename to backend/graphs/nodes/interaction_queue_node.ts index 5c129ae..ae345d2 100644 --- a/backend/components/graphs/nodes/interaction_queue_node.ts +++ b/backend/graphs/nodes/interaction_queue_node.ts @@ -9,7 +9,7 @@ */ import { CustomNode, ProcessContext } from '@inworld/runtime/graph'; -import { ConnectionsMap, InteractionInfo, State, TextInput } from '../../../types/index.js'; +import { ConnectionsMap, InteractionInfo, State, TextInput } from '../../types/index.js'; export class InteractionQueueNode extends CustomNode { private connections: ConnectionsMap; diff --git a/backend/components/graphs/nodes/state_update_node.ts b/backend/graphs/nodes/state_update_node.ts similarity index 96% rename from backend/components/graphs/nodes/state_update_node.ts rename to backend/graphs/nodes/state_update_node.ts index fb013c7..23f7fed 100644 --- a/backend/components/graphs/nodes/state_update_node.ts +++ b/backend/graphs/nodes/state_update_node.ts @@ -10,7 +10,7 @@ import { CustomNode, ProcessContext } from '@inworld/runtime/graph'; import { v4 as uuidv4 } from 'uuid'; -import { ConnectionsMap, State } from '../../../types/index.js'; +import { ConnectionsMap, State } from '../../types/index.js'; export class StateUpdateNode extends CustomNode { private connections: ConnectionsMap; diff --git a/backend/components/graphs/nodes/text_input_node.ts b/backend/graphs/nodes/text_input_node.ts similarity index 95% rename from backend/components/graphs/nodes/text_input_node.ts rename to backend/graphs/nodes/text_input_node.ts index ba3111d..fd91f6a 100644 --- a/backend/components/graphs/nodes/text_input_node.ts +++ b/backend/graphs/nodes/text_input_node.ts @@ -9,7 +9,7 @@ import { CustomNode, ProcessContext } from '@inworld/runtime/graph'; import { v4 as uuidv4 } from 'uuid'; -import { ConnectionsMap, State, TextInput } from '../../../types/index.js'; +import { ConnectionsMap, State, TextInput } from '../../types/index.js'; export class TextInputNode extends CustomNode { private connections: ConnectionsMap; diff --git a/backend/components/graphs/nodes/transcript_extractor_node.ts b/backend/graphs/nodes/transcript_extractor_node.ts similarity index 96% rename from backend/components/graphs/nodes/transcript_extractor_node.ts rename to backend/graphs/nodes/transcript_extractor_node.ts index c4b33d4..1b47e1c 100644 --- a/backend/components/graphs/nodes/transcript_extractor_node.ts +++ b/backend/graphs/nodes/transcript_extractor_node.ts @@ -6,7 +6,7 @@ import { DataStreamWithMetadata } from '@inworld/runtime'; import { CustomNode, ProcessContext } from '@inworld/runtime/graph'; -import { InteractionInfo } from '../../../types/index.js'; +import { InteractionInfo } from '../../types/index.js'; export class TranscriptExtractorNode extends CustomNode { constructor(props?: { id?: string; reportToClient?: boolean }) { diff --git a/backend/components/graphs/nodes/tts_request_builder_node.ts b/backend/graphs/nodes/tts_request_builder_node.ts similarity index 93% rename from backend/components/graphs/nodes/tts_request_builder_node.ts rename to backend/graphs/nodes/tts_request_builder_node.ts index 1af2ca8..df1caa4 100644 --- a/backend/components/graphs/nodes/tts_request_builder_node.ts +++ b/backend/graphs/nodes/tts_request_builder_node.ts @@ -6,8 +6,8 @@ */ import { CustomNode, GraphTypes, ProcessContext } from '@inworld/runtime/graph'; -import { ConnectionsMap } from '../../../types/index.js'; -import { getLanguageConfig } from '../../../config/languages.js'; +import { ConnectionsMap } from '../../types/index.js'; +import { getLanguageConfig } from '../../config/languages.js'; export class TTSRequestBuilderNode extends CustomNode { private connections: ConnectionsMap; diff --git a/backend/helpers/audio_utils.ts b/backend/helpers/audio_utils.ts new file mode 100644 index 0000000..6084f08 --- /dev/null +++ b/backend/helpers/audio_utils.ts @@ -0,0 +1,199 @@ +/** + * Audio utility functions for format conversion + */ + +import * as fs from 'fs'; +import * as path from 'path'; + +// Audio debug logging +const DEBUG_AUDIO = process.env.DEBUG_AUDIO === 'true'; +const AUDIO_DEBUG_DIR = path.join(process.cwd(), 'audio-debug'); + +// Ensure debug directory exists +if (DEBUG_AUDIO && !fs.existsSync(AUDIO_DEBUG_DIR)) { + fs.mkdirSync(AUDIO_DEBUG_DIR, { recursive: true }); + console.log(`[AudioDebug] Created debug directory: ${AUDIO_DEBUG_DIR}`); +} + +// Audio buffer for accumulating chunks per session +const audioBuffers: Map = new Map(); + +/** + * Add audio chunk to debug buffer + */ +export function debugAddAudioChunk(sessionId: string, float32Data: Float32Array): void { + if (!DEBUG_AUDIO) return; + + if (!audioBuffers.has(sessionId)) { + audioBuffers.set(sessionId, []); + console.log(`[AudioDebug] Started collecting audio for session ${sessionId}`); + } + + audioBuffers.get(sessionId)!.push(new Float32Array(float32Data)); +} + +/** + * Save accumulated audio to WAV file + */ +export function debugSaveAudio(sessionId: string, sampleRate: number = 16000): string | null { + if (!DEBUG_AUDIO) return null; + + const chunks = audioBuffers.get(sessionId); + if (!chunks || chunks.length === 0) { + console.log(`[AudioDebug] No audio to save for session ${sessionId}`); + return null; + } + + // Combine all chunks + const totalLength = chunks.reduce((sum, chunk) => sum + chunk.length, 0); + const combined = new Float32Array(totalLength); + let offset = 0; + for (const chunk of chunks) { + combined.set(chunk, offset); + offset += chunk.length; + } + + // Convert to WAV + const wavBuffer = float32ToWav(combined, sampleRate); + + // Save to file + const filename = `audio_${sessionId}_${Date.now()}.wav`; + const filepath = path.join(AUDIO_DEBUG_DIR, filename); + fs.writeFileSync(filepath, wavBuffer); + + console.log(`[AudioDebug] Saved ${combined.length} samples (${(combined.length / sampleRate).toFixed(2)}s) to ${filepath}`); + + // Clear buffer + audioBuffers.delete(sessionId); + + return filepath; +} + +/** + * Log audio stats for debugging + */ +export function debugLogAudioStats(sessionId: string, float32Data: Float32Array): void { + if (!DEBUG_AUDIO) return; + + const min = Math.min(...float32Data); + const max = Math.max(...float32Data); + const avg = float32Data.reduce((a, b) => a + b, 0) / float32Data.length; + const nonZero = float32Data.filter(v => Math.abs(v) > 0.001).length; + + console.log(`[AudioDebug] Session ${sessionId.slice(-8)}: samples=${float32Data.length}, min=${min.toFixed(4)}, max=${max.toFixed(4)}, avg=${avg.toFixed(4)}, nonZero=${nonZero}/${float32Data.length}`); +} + +/** + * Convert Float32 audio to WAV buffer + */ +function float32ToWav(samples: Float32Array, sampleRate: number): Buffer { + const numChannels = 1; + const bitsPerSample = 16; + const bytesPerSample = bitsPerSample / 8; + const blockAlign = numChannels * bytesPerSample; + const byteRate = sampleRate * blockAlign; + const dataSize = samples.length * bytesPerSample; + const headerSize = 44; + const totalSize = headerSize + dataSize; + + const buffer = Buffer.alloc(totalSize); + let offset = 0; + + // RIFF header + buffer.write('RIFF', offset); offset += 4; + buffer.writeUInt32LE(totalSize - 8, offset); offset += 4; + buffer.write('WAVE', offset); offset += 4; + + // fmt chunk + buffer.write('fmt ', offset); offset += 4; + buffer.writeUInt32LE(16, offset); offset += 4; // chunk size + buffer.writeUInt16LE(1, offset); offset += 2; // PCM format + buffer.writeUInt16LE(numChannels, offset); offset += 2; + buffer.writeUInt32LE(sampleRate, offset); offset += 4; + buffer.writeUInt32LE(byteRate, offset); offset += 4; + buffer.writeUInt16LE(blockAlign, offset); offset += 2; + buffer.writeUInt16LE(bitsPerSample, offset); offset += 2; + + // data chunk + buffer.write('data', offset); offset += 4; + buffer.writeUInt32LE(dataSize, offset); offset += 4; + + // Write samples as 16-bit PCM + for (let i = 0; i < samples.length; i++) { + const s = Math.max(-1, Math.min(1, samples[i])); + const val = s < 0 ? s * 0x8000 : s * 0x7FFF; + buffer.writeInt16LE(Math.round(val), offset); + offset += 2; + } + + return buffer; +} + +/** + * Convert Float32Array audio data to Int16Array (PCM16) + */ +export function float32ToPCM16(float32Data: Float32Array): Int16Array { + const pcm16 = new Int16Array(float32Data.length); + for (let i = 0; i < float32Data.length; i++) { + // Clamp to [-1, 1] range and convert to Int16 range [-32768, 32767] + const s = Math.max(-1, Math.min(1, float32Data[i])); + pcm16[i] = s < 0 ? s * 0x8000 : s * 0x7fff; + } + return pcm16; +} + +/** + * Convert Int16Array (PCM16) to Float32Array + */ +export function pcm16ToFloat32(pcm16Data: Int16Array): Float32Array { + const float32 = new Float32Array(pcm16Data.length); + for (let i = 0; i < pcm16Data.length; i++) { + float32[i] = pcm16Data[i] / 32768.0; + } + return float32; +} + +/** + * Convert audio data to PCM16 base64 string for WebSocket transmission + */ +export function convertToPCM16Base64( + audioData: number[] | Float32Array | string | undefined, + _sampleRate: number | undefined, + _logPrefix: string = 'Audio' +): string | null { + if (!audioData) { + return null; + } + + let base64Data: string; + + if (typeof audioData === 'string') { + // Already base64 encoded + base64Data = audioData; + } else { + // Convert Float32 array to PCM16 base64 + const float32Data = Array.isArray(audioData) + ? new Float32Array(audioData) + : audioData; + const pcm16Data = float32ToPCM16(float32Data); + base64Data = Buffer.from(pcm16Data.buffer).toString('base64'); + } + + return base64Data; +} + +/** + * Decode base64 audio to Float32Array + * Frontend sends Float32 audio data directly (4 bytes per sample) + * Note: Node.js Buffer objects share ArrayBuffers with offsets, so we need to copy + */ +export function decodeBase64ToFloat32(base64Audio: string): Float32Array { + const buffer = Buffer.from(base64Audio, 'base64'); + // Create a clean copy to avoid Node.js Buffer's internal ArrayBuffer sharing + const cleanArray = new Uint8Array(buffer.length); + for (let i = 0; i < buffer.length; i++) { + cleanArray[i] = buffer[i]; + } + // Interpret bytes directly as Float32 (4 bytes per sample) + return new Float32Array(cleanArray.buffer); +} diff --git a/backend/helpers/connection-manager.ts b/backend/helpers/connection-manager.ts index f395862..219cd17 100644 --- a/backend/helpers/connection-manager.ts +++ b/backend/helpers/connection-manager.ts @@ -11,9 +11,9 @@ import { WebSocket } from 'ws'; import { GraphTypes } from '@inworld/runtime/graph'; -import { ConversationGraphWrapper } from '../components/graphs/conversation-graph.js'; -import { MultimodalStreamManager } from '../components/audio/multimodal_stream_manager.js'; -import { decodeBase64ToFloat32 } from '../components/audio/audio_utils.js'; +import { ConversationGraphWrapper } from '../graphs/conversation-graph.js'; +import { MultimodalStreamManager } from './multimodal_stream_manager.js'; +import { decodeBase64ToFloat32, debugAddAudioChunk, debugLogAudioStats, debugSaveAudio } from './audio_utils.js'; import { ConnectionsMap, INPUT_SAMPLE_RATE, TTS_SAMPLE_RATE } from '../types/index.js'; import { getLanguageConfig, @@ -327,6 +327,10 @@ export class ConnectionManager { // Decode base64 to Float32Array const float32Data = decodeBase64ToFloat32(base64Audio); + // Debug: log audio stats and collect for WAV export + debugLogAudioStats(this.sessionId, float32Data); + debugAddAudioChunk(this.sessionId, float32Data); + // Push to multimodal stream this.multimodalStreamManager.pushAudio({ data: Array.from(float32Data), @@ -566,6 +570,9 @@ export class ConnectionManager { console.log(`[ConnectionManager] Destroying session ${this.sessionId}`); this.isDestroyed = true; + // Save debug audio if enabled (DEBUG_AUDIO=true) + debugSaveAudio(this.sessionId, INPUT_SAMPLE_RATE); + // End the multimodal stream this.multimodalStreamManager.end(); diff --git a/backend/components/audio/multimodal_stream_manager.ts b/backend/helpers/multimodal_stream_manager.ts similarity index 95% rename from backend/components/audio/multimodal_stream_manager.ts rename to backend/helpers/multimodal_stream_manager.ts index 212fe55..175bec8 100644 --- a/backend/components/audio/multimodal_stream_manager.ts +++ b/backend/helpers/multimodal_stream_manager.ts @@ -29,6 +29,8 @@ export class MultimodalStreamManager { } // Create GraphTypes.Audio object and wrap in MultimodalContent + // Note: GraphTypes.Audio requires number[] - conversion from Float32Array is unavoidable + // due to SDK constraint. If chunk.data is already number[], we pass it directly. const audioData = new GraphTypes.Audio({ data: Array.isArray(chunk.data) ? chunk.data : Array.from(chunk.data), sampleRate: chunk.sampleRate, diff --git a/backend/server.ts b/backend/server.ts index 110f7e2..a07f2ce 100644 --- a/backend/server.ts +++ b/backend/server.ts @@ -27,7 +27,7 @@ import { telemetry, stopInworldRuntime } from '@inworld/runtime'; import { MetricType } from '@inworld/runtime/telemetry'; // Import new 0.9 components -import { ConversationGraphWrapper } from './components/graphs/conversation-graph.js'; +import { ConversationGraphWrapper } from './graphs/conversation-graph.js'; import { ConnectionManager } from './helpers/connection-manager.js'; import { ConnectionsMap } from './types/index.js'; diff --git a/backend/types/index.ts b/backend/types/index.ts index 90c7965..230273d 100644 --- a/backend/types/index.ts +++ b/backend/types/index.ts @@ -3,7 +3,7 @@ */ import { WebSocket } from 'ws'; -import type { MultimodalStreamManager } from '../components/audio/multimodal_stream_manager.js'; +import type { MultimodalStreamManager } from '../helpers/multimodal_stream_manager.js'; import type { GraphOutputStream } from '@inworld/runtime/graph'; import type { IntroductionState } from '../helpers/introduction-state-processor.js'; diff --git a/frontend/.gitignore b/frontend/.gitignore new file mode 100644 index 0000000..a547bf3 --- /dev/null +++ b/frontend/.gitignore @@ -0,0 +1,24 @@ +# Logs +logs +*.log +npm-debug.log* +yarn-debug.log* +yarn-error.log* +pnpm-debug.log* +lerna-debug.log* + +node_modules +dist +dist-ssr +*.local + +# Editor directories and files +.vscode/* +!.vscode/extensions.json +.idea +.DS_Store +*.suo +*.ntvs* +*.njsproj +*.sln +*.sw? diff --git a/frontend/README.md b/frontend/README.md new file mode 100644 index 0000000..d2e7761 --- /dev/null +++ b/frontend/README.md @@ -0,0 +1,73 @@ +# React + TypeScript + Vite + +This template provides a minimal setup to get React working in Vite with HMR and some ESLint rules. + +Currently, two official plugins are available: + +- [@vitejs/plugin-react](https://github.com/vitejs/vite-plugin-react/blob/main/packages/plugin-react) uses [Babel](https://babeljs.io/) (or [oxc](https://oxc.rs) when used in [rolldown-vite](https://vite.dev/guide/rolldown)) for Fast Refresh +- [@vitejs/plugin-react-swc](https://github.com/vitejs/vite-plugin-react/blob/main/packages/plugin-react-swc) uses [SWC](https://swc.rs/) for Fast Refresh + +## React Compiler + +The React Compiler is not enabled on this template because of its impact on dev & build performances. To add it, see [this documentation](https://react.dev/learn/react-compiler/installation). + +## Expanding the ESLint configuration + +If you are developing a production application, we recommend updating the configuration to enable type-aware lint rules: + +```js +export default defineConfig([ + globalIgnores(['dist']), + { + files: ['**/*.{ts,tsx}'], + extends: [ + // Other configs... + + // Remove tseslint.configs.recommended and replace with this + tseslint.configs.recommendedTypeChecked, + // Alternatively, use this for stricter rules + tseslint.configs.strictTypeChecked, + // Optionally, add this for stylistic rules + tseslint.configs.stylisticTypeChecked, + + // Other configs... + ], + languageOptions: { + parserOptions: { + project: ['./tsconfig.node.json', './tsconfig.app.json'], + tsconfigRootDir: import.meta.dirname, + }, + // other options... + }, + }, +]) +``` + +You can also install [eslint-plugin-react-x](https://github.com/Rel1cx/eslint-react/tree/main/packages/plugins/eslint-plugin-react-x) and [eslint-plugin-react-dom](https://github.com/Rel1cx/eslint-react/tree/main/packages/plugins/eslint-plugin-react-dom) for React-specific lint rules: + +```js +// eslint.config.js +import reactX from 'eslint-plugin-react-x' +import reactDom from 'eslint-plugin-react-dom' + +export default defineConfig([ + globalIgnores(['dist']), + { + files: ['**/*.{ts,tsx}'], + extends: [ + // Other configs... + // Enable lint rules for React + reactX.configs['recommended-typescript'], + // Enable lint rules for React DOM + reactDom.configs.recommended, + ], + languageOptions: { + parserOptions: { + project: ['./tsconfig.node.json', './tsconfig.app.json'], + tsconfigRootDir: import.meta.dirname, + }, + // other options... + }, + }, +]) +``` diff --git a/frontend/eslint.config.js b/frontend/eslint.config.js new file mode 100644 index 0000000..5e6b472 --- /dev/null +++ b/frontend/eslint.config.js @@ -0,0 +1,23 @@ +import js from '@eslint/js' +import globals from 'globals' +import reactHooks from 'eslint-plugin-react-hooks' +import reactRefresh from 'eslint-plugin-react-refresh' +import tseslint from 'typescript-eslint' +import { defineConfig, globalIgnores } from 'eslint/config' + +export default defineConfig([ + globalIgnores(['dist']), + { + files: ['**/*.{ts,tsx}'], + extends: [ + js.configs.recommended, + tseslint.configs.recommended, + reactHooks.configs.flat.recommended, + reactRefresh.configs.vite, + ], + languageOptions: { + ecmaVersion: 2020, + globals: globals.browser, + }, + }, +]) diff --git a/frontend/index.html b/frontend/index.html index 7eb9aab..edf10b7 100644 --- a/frontend/index.html +++ b/frontend/index.html @@ -1,100 +1,16 @@ - + - - - - Aprendemo - + + + + - - - -
-
-

Aprendemo

-
-
- -
-
- - Connecting... -
-
-
-
- -
-
-
- -
-
-

Conversation

-
- - -
-
- -
-
-
-
-
- - -
-
-

Flashcards

- 0 cards -
- -
-
-
-

-
-
-
-
-
-
-
- - - - - - - - - + + Aprendemo + + +
+ + diff --git a/frontend/js/audio-processor.js b/frontend/js/audio-processor.js deleted file mode 100644 index d4517f6..0000000 --- a/frontend/js/audio-processor.js +++ /dev/null @@ -1,59 +0,0 @@ -class AudioProcessor extends AudioWorkletProcessor { - constructor(options) { - super(); - this.sourceSampleRate = options.processorOptions.sourceSampleRate; - this.targetSampleRate = 16000; - this.resampleRatio = this.sourceSampleRate / this.targetSampleRate; - this.buffer = null; - } - - process(inputs) { - const inputChannel = inputs[0][0]; - if (!inputChannel) return true; - - const currentLength = this.buffer ? this.buffer.length : 0; - const newBuffer = new Float32Array(currentLength + inputChannel.length); - if (this.buffer) { - newBuffer.set(this.buffer, 0); - } - newBuffer.set(inputChannel, currentLength); - this.buffer = newBuffer; - - // Put stuff back into 16k - const numOutputSamples = Math.floor( - this.buffer.length / this.resampleRatio - ); - if (numOutputSamples === 0) return true; - - const resampledData = new Float32Array(numOutputSamples); - for (let i = 0; i < numOutputSamples; i++) { - const correspondingInputIndex = i * this.resampleRatio; - const lowerIndex = Math.floor(correspondingInputIndex); - const upperIndex = Math.ceil(correspondingInputIndex); - const interpolationFactor = correspondingInputIndex - lowerIndex; - - const lowerValue = this.buffer[lowerIndex] || 0; - const upperValue = this.buffer[upperIndex] || 0; - - resampledData[i] = - lowerValue + (upperValue - lowerValue) * interpolationFactor; - } - - const consumedInputSamples = numOutputSamples * this.resampleRatio; - this.buffer = this.buffer.slice(Math.round(consumedInputSamples)); - - // Convert Float32Array to Int16Array - const int16Array = new Int16Array(resampledData.length); - for (let i = 0; i < resampledData.length; i++) { - int16Array[i] = Math.max( - -32768, - Math.min(32767, resampledData[i] * 32768) - ); - } - - this.port.postMessage(int16Array.buffer, [int16Array.buffer]); - return true; - } -} - -registerProcessor('audio-processor', AudioProcessor); diff --git a/frontend/js/chat-ui.js b/frontend/js/chat-ui.js deleted file mode 100644 index 3ebfd61..0000000 --- a/frontend/js/chat-ui.js +++ /dev/null @@ -1,417 +0,0 @@ -import { translator } from './translator.js'; - -export class ChatUI { - constructor() { - this.messagesContainer = document.getElementById('messages'); - this.transcriptContainer = document.getElementById('currentTranscript'); - this.typewriterTimers = new Map(); // Track active typewriter effects - this.typewriterSpeed = 25; // milliseconds per character - this.llmTypewriterCallback = null; // Callback for when LLM typewriter completes - - // Translation tooltip - this.translationTooltip = this._createTranslationTooltip(); - this.activeHoverElement = null; - this.hoverTimeout = null; - this.hideTimeout = null; - } - - _createTranslationTooltip() { - const tooltip = document.createElement('div'); - tooltip.className = 'translation-tooltip'; - tooltip.innerHTML = ` -
- -
-
- -
- `; - document.body.appendChild(tooltip); - - // Keep tooltip visible when hovering over it - tooltip.addEventListener('mouseenter', () => { - if (this.hideTimeout) { - clearTimeout(this.hideTimeout); - this.hideTimeout = null; - } - }); - - tooltip.addEventListener('mouseleave', () => { - this._hideTooltip(); - }); - - return tooltip; - } - - _showTooltip(element, text) { - const rect = element.getBoundingClientRect(); - const tooltip = this.translationTooltip; - - // Position tooltip above the message - tooltip.style.left = `${rect.left + window.scrollX}px`; - tooltip.style.top = `${rect.top + window.scrollY - 8}px`; - tooltip.style.maxWidth = `${Math.min(rect.width + 40, 400)}px`; - - // Show loading state - tooltip.classList.add('visible', 'loading'); - tooltip.querySelector('.translation-text').textContent = ''; - - // Fetch translation - translator.translate(text, 'en', 'auto') - .then(translation => { - if (this.activeHoverElement === element) { - tooltip.querySelector('.translation-text').textContent = translation; - tooltip.classList.remove('loading'); - - // Reposition after content loads (in case size changed) - requestAnimationFrame(() => { - const tooltipRect = tooltip.getBoundingClientRect(); - tooltip.style.top = `${rect.top + window.scrollY - tooltipRect.height - 8}px`; - }); - } - }) - .catch(error => { - console.error('[ChatUI] Translation failed:', error); - if (this.activeHoverElement === element) { - tooltip.querySelector('.translation-text').textContent = 'Translation unavailable'; - tooltip.classList.remove('loading'); - } - }); - } - - _hideTooltip() { - this.translationTooltip.classList.remove('visible', 'loading'); - this.activeHoverElement = null; - } - - _setupTranslationHover(element) { - element.addEventListener('mouseenter', () => { - if (this.hideTimeout) { - clearTimeout(this.hideTimeout); - this.hideTimeout = null; - } - - // Small delay before showing tooltip to avoid flickering - this.hoverTimeout = setTimeout(() => { - this.activeHoverElement = element; - const text = element.textContent.replace('▊', '').trim(); // Remove cursor - if (text) { - this._showTooltip(element, text); - } - }, 300); - }); - - element.addEventListener('mouseleave', () => { - if (this.hoverTimeout) { - clearTimeout(this.hoverTimeout); - this.hoverTimeout = null; - } - - // Delay hiding to allow moving to tooltip - this.hideTimeout = setTimeout(() => { - this._hideTooltip(); - }, 150); - }); - } - - render( - chatHistory, - currentTranscript, - currentLLMResponse, - pendingTranscription, - streamingLLMResponse, - isRecording, - speechDetected - ) { - this.renderMessages( - chatHistory, - pendingTranscription, - streamingLLMResponse, - currentTranscript, - isRecording, - speechDetected - ); - this.renderCurrentTranscript(currentTranscript); - } - - renderMessages( - messages, - pendingTranscription, - streamingLLMResponse, - currentTranscript, - isRecording, - speechDetected - ) { - // Only clear and rebuild if the conversation history changed - const currentHistoryLength = this.messagesContainer.querySelectorAll( - '.message:not(.streaming)' - ).length; - if (currentHistoryLength !== messages.length) { - this.messagesContainer.innerHTML = ''; - - // Render existing conversation history - messages.forEach((message) => { - const messageElement = this.createMessageElement(message); - this.messagesContainer.appendChild(messageElement); - }); - } - - // Handle real-time transcript updates (while recording and speech detected) - // Show immediately when VAD activates, even before transcript text arrives - const existingRealtimeTranscript = document.getElementById( - 'realtime-transcript' - ); - if (speechDetected && isRecording && !pendingTranscription) { - if (!existingRealtimeTranscript) { - const userMessage = this.createRealtimeTranscriptElement(); - userMessage.id = 'realtime-transcript'; - this.messagesContainer.appendChild(userMessage); - } - // Update the transcript in real-time (no typewriter effect) - // Show 3-dot loading animation if no text yet, otherwise show the actual transcript - const transcriptElement = document.getElementById('realtime-transcript'); - if (transcriptElement) { - const textNode = transcriptElement.querySelector('.transcript-text'); - const loadingDots = transcriptElement.querySelector('.loading-dots'); - if (currentTranscript) { - if (textNode) textNode.textContent = currentTranscript; - if (loadingDots) loadingDots.style.display = 'none'; - } else { - if (textNode) textNode.textContent = ''; - if (loadingDots) loadingDots.style.display = 'flex'; - } - this.scrollToBottom(); - } - } else if (existingRealtimeTranscript) { - existingRealtimeTranscript.remove(); - } - - // Handle pending user transcription (final transcription) - const existingUserStreaming = document.getElementById( - 'pending-transcription' - ); - if (pendingTranscription) { - // Remove real-time transcript if it exists - if (existingRealtimeTranscript) { - existingRealtimeTranscript.remove(); - } - if (!existingUserStreaming) { - const userMessage = this.createMessageElement({ - role: 'learner', - content: '', - }); - userMessage.classList.add('streaming'); - userMessage.id = 'pending-transcription'; - this.messagesContainer.appendChild(userMessage); - // Only start typewriter for new transcriptions - this.startTypewriter( - 'pending-transcription', - pendingTranscription, - this.typewriterSpeed * 0.8 - ); - } - // Don't restart typewriter if element already exists - } else if (existingUserStreaming) { - existingUserStreaming.remove(); - this.clearTypewriter('pending-transcription'); - } - - // Handle streaming LLM response - const existingLLMStreaming = document.getElementById( - 'streaming-llm-response' - ); - if (streamingLLMResponse) { - console.log( - '[ChatUI] Handling streaming LLM response:', - streamingLLMResponse - ); - if (!existingLLMStreaming) { - console.log( - '[ChatUI] Creating new streaming LLM element and starting typewriter' - ); - const assistantMessage = this.createStreamingMessageElement(''); - assistantMessage.id = 'streaming-llm-response'; - this.messagesContainer.appendChild(assistantMessage); - // Only start typewriter for new LLM responses - this.startTypewriter( - 'streaming-llm-response', - streamingLLMResponse, - this.typewriterSpeed, - this.llmTypewriterCallback - ); - } - // Don't restart typewriter if element already exists - } else if (existingLLMStreaming) { - console.log('[ChatUI] Removing existing LLM streaming element'); - // Clear typewriter first (this will call the completion callback) - this.clearTypewriter('streaming-llm-response'); - // Then remove the element - existingLLMStreaming.remove(); - } - - this.scrollToBottom(); - } - - createMessageElement(message) { - const div = document.createElement('div'); - div.className = `message ${message.role}`; - div.textContent = message.content; - - // Add translation hover for teacher (LLM) messages - if (message.role === 'teacher') { - this._setupTranslationHover(div); - } - - return div; - } - - createStreamingMessageElement(content) { - const div = document.createElement('div'); - div.className = 'message teacher streaming'; - - // Create text node for content - const textNode = document.createTextNode(content); - div.appendChild(textNode); - - // Add typing indicator - const cursor = document.createElement('span'); - cursor.className = 'streaming-cursor'; - cursor.textContent = '▊'; - div.appendChild(cursor); - - // Add translation hover for streaming teacher messages too - this._setupTranslationHover(div); - - return div; - } - - createRealtimeTranscriptElement() { - const div = document.createElement('div'); - div.className = 'message learner streaming realtime'; - - // Create container for text - const textNode = document.createElement('span'); - textNode.className = 'transcript-text'; - div.appendChild(textNode); - - // Add 3-dot loading animation - const loadingDots = document.createElement('span'); - loadingDots.className = 'loading-dots'; - loadingDots.innerHTML = ''; - div.appendChild(loadingDots); - - return div; - } - - renderCurrentTranscript(transcript) { - if (transcript) { - this.transcriptContainer.textContent = transcript; - } else { - this.transcriptContainer.textContent = ''; - } - } - - startTypewriter(elementId, fullText, speed, onComplete) { - const element = document.getElementById(elementId); - if (!element) { - // Element doesn't exist, call completion callback immediately if provided - if (onComplete) { - console.log( - `[Typewriter] Element ${elementId} not found, calling completion callback immediately` - ); - onComplete(); - } - return; - } - - // Clear any existing timer for this element - if (this.typewriterTimers.has(elementId)) { - const existingTimer = this.typewriterTimers.get(elementId); - clearInterval(existingTimer.timer || existingTimer); - this.typewriterTimers.delete(elementId); - - // If there was a previous timer with a callback, call it now since we're replacing it - if (existingTimer.onComplete) { - existingTimer.onComplete(); - } - } - - // Get the text content node (accounting for streaming cursor) - const textContent = element.querySelector('.streaming-cursor') - ? element.childNodes[0] - : element; - - // Start fresh - clear the content and start typing from beginning - textContent.textContent = ''; - let currentIndex = 0; - - console.log(`[Typewriter] Starting ${elementId}, target: "${fullText}"`); - - const timer = setInterval(() => { - // Check if element still exists - const currentElement = document.getElementById(elementId); - if (!currentElement) { - console.log( - `[Typewriter] Element ${elementId} removed, completing immediately` - ); - clearInterval(timer); - this.typewriterTimers.delete(elementId); - if (onComplete) { - onComplete(); - } - return; - } - - if (currentIndex < fullText.length) { - const newText = fullText.substring(0, currentIndex + 1); - textContent.textContent = newText; - currentIndex++; - this.scrollToBottom(); - } else { - console.log(`[Typewriter] Complete: ${elementId}`); - clearInterval(timer); - this.typewriterTimers.delete(elementId); - - // Call completion callback if provided - if (onComplete) { - onComplete(); - } - } - }, speed); - - this.typewriterTimers.set(elementId, { timer, fullText, onComplete }); - } - - clearTypewriter(elementId) { - if (this.typewriterTimers.has(elementId)) { - const timerData = this.typewriterTimers.get(elementId); - clearInterval(timerData.timer || timerData); - - // If there's a completion callback, call it before clearing - // This ensures the text gets finalized even if typewriter is interrupted - if (timerData.onComplete) { - console.log( - `[Typewriter] Clearing ${elementId}, calling completion callback` - ); - timerData.onComplete(); - } - - this.typewriterTimers.delete(elementId); - } - } - - clearAllTypewriters() { - this.typewriterTimers.forEach((timerData, elementId) => { - clearInterval(timerData.timer || timerData); - }); - this.typewriterTimers.clear(); - } - - setLLMTypewriterCallback(callback) { - this.llmTypewriterCallback = callback; - } - - scrollToBottom() { - this.messagesContainer.scrollTop = this.messagesContainer.scrollHeight; - } -} diff --git a/frontend/js/flashcard-ui.js b/frontend/js/flashcard-ui.js deleted file mode 100644 index 1d887da..0000000 --- a/frontend/js/flashcard-ui.js +++ /dev/null @@ -1,179 +0,0 @@ -export class FlashcardUI { - constructor() { - this.flashcardsGrid = document.getElementById('flashcardsGrid'); - this.cardCount = document.getElementById('cardCount'); - this.flashcards = []; - this.currentLanguage = 'es'; - } - - render(flashcards, languageCode = 'es') { - this.flashcards = flashcards; - this.currentLanguage = languageCode; - this.updateCardCount(flashcards.length); - this.renderFlashcards(flashcards); - } - - addFlashcards(newFlashcards) { - // Add new flashcards to the existing collection - this.flashcards = [...this.flashcards, ...newFlashcards]; - this.render(this.flashcards, this.currentLanguage); - } - - updateCardCount(count) { - if (count >= 1) { - this.cardCount.textContent = `Export ${count} card${count !== 1 ? 's' : ''} to Anki`; - this.cardCount.style.cursor = 'pointer'; - this.cardCount.style.color = '#666'; - this.cardCount.style.textDecoration = 'underline'; - this.cardCount.onclick = () => this.exportToAnki(); - } else { - this.cardCount.textContent = `${count} card${count !== 1 ? 's' : ''}`; - this.cardCount.style.cursor = 'default'; - this.cardCount.style.color = 'inherit'; - this.cardCount.style.textDecoration = 'none'; - this.cardCount.onclick = null; - } - } - - renderFlashcards(flashcards) { - this.flashcardsGrid.innerHTML = ''; - - if (flashcards.length === 0) { - this.renderEmptyState(); - return; - } - - // Show only the latest flashcards, scroll to see older ones - const sortedFlashcards = [...flashcards].sort((a, b) => { - const timeA = new Date(a.timestamp || 0).getTime(); - const timeB = new Date(b.timestamp || 0).getTime(); - return timeB - timeA; // Most recent first - }); - - sortedFlashcards.forEach((flashcard) => { - const cardElement = this.createFlashcardElement(flashcard); - this.flashcardsGrid.appendChild(cardElement); - }); - } - - renderEmptyState() { - const emptyState = document.createElement('div'); - emptyState.className = 'empty-state'; - emptyState.innerHTML = ''; - this.flashcardsGrid.appendChild(emptyState); - } - - createFlashcardElement(flashcard) { - const card = document.createElement('div'); - card.className = 'flashcard'; - - // Support both new 'targetWord' and legacy 'spanish' field - const targetWord = flashcard.targetWord || flashcard.spanish || flashcard.word || ''; - - card.innerHTML = ` -
-
-
${this.escapeHtml(targetWord)}
-
-
-
${this.escapeHtml(flashcard.english || flashcard.translation || '')}
-
${this.escapeHtml(flashcard.example || flashcard.example_sentence || '')}
-
- Remember: - ${this.escapeHtml(flashcard.mnemonic || '')} -
-
-
- `; - - card.addEventListener('click', () => { - this.flipCard(card); - if (typeof this.onCardClick === 'function') { - this.onCardClick(flashcard); - } - }); - - return card; - } - - flipCard(cardElement) { - cardElement.classList.toggle('flipped'); - } - - escapeHtml(text) { - const div = document.createElement('div'); - div.textContent = text; - return div.innerHTML; - } - - async exportToAnki() { - try { - // Filter out invalid flashcards - const validFlashcards = this.flashcards.filter((flashcard) => { - const targetWord = flashcard.targetWord || flashcard.spanish; - return ( - targetWord && - flashcard.english && - targetWord.trim() !== '' && - flashcard.english.trim() !== '' - ); - }); - - if (validFlashcards.length === 0) { - alert('No valid flashcards to export'); - return; - } - - // Show loading state - const originalText = this.cardCount.textContent; - this.cardCount.textContent = 'Exporting...'; - this.cardCount.style.cursor = 'wait'; - - // Get language name for deck naming - const languageNames = { - es: 'Spanish', - ja: 'Japanese', - fr: 'French', - }; - const languageName = languageNames[this.currentLanguage] || 'Language'; - - const response = await fetch('/api/export-anki', { - method: 'POST', - headers: { - 'Content-Type': 'application/json', - }, - body: JSON.stringify({ - flashcards: validFlashcards, - deckName: `Aprendemo ${languageName} Cards`, - languageCode: this.currentLanguage, - }), - }); - - if (!response.ok) { - throw new Error('Export failed'); - } - - // Create download link - const blob = await response.blob(); - const url = window.URL.createObjectURL(blob); - const a = document.createElement('a'); - a.href = url; - a.download = `aprendemo_${this.currentLanguage}_cards.apkg`; - document.body.appendChild(a); - a.click(); - document.body.removeChild(a); - window.URL.revokeObjectURL(url); - - console.log('ANKI deck exported successfully!'); - - // Restore original state - this.updateCardCount(this.flashcards.length); - } catch (error) { - console.error('Error exporting to ANKI:', error); - alert('Failed to export flashcards to ANKI'); - - // Restore original state - this.updateCardCount(this.flashcards.length); - } - } -} diff --git a/frontend/js/ios-audio-workarounds.js b/frontend/js/ios-audio-workarounds.js deleted file mode 100644 index e195b9c..0000000 --- a/frontend/js/ios-audio-workarounds.js +++ /dev/null @@ -1,578 +0,0 @@ -/** - * iOS Audio Workarounds for WebSocket Audio Streaming - * Handles iOS-specific audio limitations and provides fallback solutions - */ - -class IOSAudioHandler { - constructor() { - this.isIOS = this.detectIOS(); - this.audioContext = null; - this.audioUnlocked = false; - this.microphoneStream = null; - this.audioProcessor = null; - this.audioChunks = []; - this.currentAudioUrl = null; - this.audioQueue = []; - this.isPlaying = false; - - // iOS-specific audio constraints - this.audioConstraints = { - audio: { - echoCancellation: true, - noiseSuppression: true, - autoGainControl: true, - sampleRate: 16000, - channelCount: 1, - // iOS-specific constraints - latency: 0.02, - sampleSize: 16, - volume: 1.0, - }, - }; - - if (this.isIOS) { - console.log('[iOS Audio] iOS device detected, initializing workarounds'); - this.initializeIOSWorkarounds(); - } - } - - detectIOS() { - const userAgent = navigator.userAgent || navigator.vendor || window.opera; - - // Check for iOS devices - if (/iPad|iPhone|iPod/.test(userAgent) && !window.MSStream) { - return true; - } - - // Check for iPad on iOS 13+ (reports as Mac) - if (navigator.platform === 'MacIntel' && navigator.maxTouchPoints > 1) { - return true; - } - - // Additional check for iOS Safari - const isIOSSafari = - !!navigator.userAgent.match(/Version\/[\d\.]+.*Safari/) && - /iPad|iPhone|iPod/.test(navigator.userAgent); - - return isIOSSafari; - } - - initializeIOSWorkarounds() { - // Add touch event listeners for audio unlock - this.setupTouchHandlers(); - - // Setup visibility change handler to resume audio - this.setupVisibilityHandler(); - - // Add iOS-specific meta tags if not present - this.addIOSMetaTags(); - - // Prevent iOS zoom on double-tap - this.preventDoubleTapZoom(); - } - - setupTouchHandlers() { - // Unlock audio on first user interaction - const unlockAudio = async () => { - if (!this.audioUnlocked) { - await this.unlockAudioContext(); - this.audioUnlocked = true; - console.log('[iOS Audio] Audio unlocked via user interaction'); - } - }; - - // Add multiple event types to catch user interaction - ['touchstart', 'touchend', 'click'].forEach((eventType) => { - document.addEventListener(eventType, unlockAudio, { - once: true, - passive: true, - }); - }); - } - - setupVisibilityHandler() { - // Resume audio context when app becomes visible - document.addEventListener('visibilitychange', async () => { - if (!document.hidden && this.audioContext) { - if (this.audioContext.state === 'suspended') { - await this.audioContext.resume(); - console.log( - '[iOS Audio] Resumed audio context after visibility change' - ); - } - } - }); - } - - addIOSMetaTags() { - // Ensure proper viewport settings for iOS - let viewport = document.querySelector('meta[name="viewport"]'); - if (!viewport) { - viewport = document.createElement('meta'); - viewport.name = 'viewport'; - document.head.appendChild(viewport); - } - viewport.content = - 'width=device-width, initial-scale=1.0, maximum-scale=1.0, user-scalable=no'; - - // Add iOS web app capable meta tag - if (!document.querySelector('meta[name="apple-mobile-web-app-capable"]')) { - const webAppMeta = document.createElement('meta'); - webAppMeta.name = 'apple-mobile-web-app-capable'; - webAppMeta.content = 'yes'; - document.head.appendChild(webAppMeta); - } - } - - preventDoubleTapZoom() { - let lastTouchEnd = 0; - document.addEventListener( - 'touchend', - (event) => { - const now = Date.now(); - if (now - lastTouchEnd <= 300) { - event.preventDefault(); - } - lastTouchEnd = now; - }, - false - ); - } - - async unlockAudioContext() { - try { - // Create or resume audio context - if (!this.audioContext) { - this.audioContext = new (window.AudioContext || - window.webkitAudioContext)({ - sampleRate: 16000, - latencyHint: 'interactive', - }); - } - - if (this.audioContext.state === 'suspended') { - await this.audioContext.resume(); - } - - // Play a silent buffer to unlock audio - const buffer = this.audioContext.createBuffer(1, 1, 22050); - const source = this.audioContext.createBufferSource(); - source.buffer = buffer; - source.connect(this.audioContext.destination); - source.start(0); - - console.log( - '[iOS Audio] AudioContext unlocked, state:', - this.audioContext.state - ); - - // Dispatch custom event to notify app - window.dispatchEvent( - new CustomEvent('ios-audio-unlocked', { - detail: { audioContext: this.audioContext }, - }) - ); - - return this.audioContext; - } catch (error) { - console.error('[iOS Audio] Failed to unlock audio context:', error); - throw error; - } - } - - async startMicrophone(onAudioData) { - try { - console.log('[iOS Audio] Starting microphone...'); - - // Ensure audio context is ready - if (!this.audioContext || this.audioContext.state === 'suspended') { - await this.unlockAudioContext(); - } - - // Request microphone permission with iOS-optimized constraints - this.microphoneStream = await navigator.mediaDevices.getUserMedia( - this.audioConstraints - ); - console.log('[iOS Audio] Microphone access granted'); - - // Create audio processing pipeline - const source = this.audioContext.createMediaStreamSource( - this.microphoneStream - ); - - // Use ScriptProcessorNode for iOS compatibility (AudioWorklet may not work) - const bufferSize = 4096; - this.audioProcessor = this.audioContext.createScriptProcessor( - bufferSize, - 1, - 1 - ); - - this.audioProcessor.onaudioprocess = (event) => { - const inputData = event.inputBuffer.getChannelData(0); - - // Convert Float32Array to Int16Array - const int16Array = new Int16Array(inputData.length); - for (let i = 0; i < inputData.length; i++) { - const s = Math.max(-1, Math.min(1, inputData[i])); - int16Array[i] = s < 0 ? s * 0x8000 : s * 0x7fff; - } - - // Convert to base64 - const base64Audio = this.arrayBufferToBase64(int16Array.buffer); - - // Call the callback with audio data - if (onAudioData) { - onAudioData(base64Audio); - } - }; - - // Connect the pipeline - source.connect(this.audioProcessor); - this.audioProcessor.connect(this.audioContext.destination); - - console.log('[iOS Audio] Microphone pipeline connected'); - return true; - } catch (error) { - console.error('[iOS Audio] Failed to start microphone:', error); - - // Provide user-friendly error message - if (error.name === 'NotAllowedError') { - alert( - 'Microphone access denied. Please enable microphone permissions in Settings > Safari > Microphone.' - ); - } else if (error.name === 'NotFoundError') { - alert( - 'No microphone found. Please ensure your device has a working microphone.' - ); - } else { - alert(`Microphone error: ${error.message}`); - } - - throw error; - } - } - - stopMicrophone() { - console.log('[iOS Audio] Stopping microphone...'); - - if (this.audioProcessor) { - this.audioProcessor.disconnect(); - this.audioProcessor = null; - } - - if (this.microphoneStream) { - this.microphoneStream.getTracks().forEach((track) => track.stop()); - this.microphoneStream = null; - } - - console.log('[iOS Audio] Microphone stopped'); - } - - // PCM to WAV conversion for iOS - createWAVFromPCM(pcmData, sampleRate = 16000) { - // PCM data is Int16Array - const numChannels = 1; - const bitsPerSample = 16; - const dataLength = pcmData.byteLength; - - // Create WAV header - const headerLength = 44; - const fileLength = headerLength + dataLength; - const arrayBuffer = new ArrayBuffer(fileLength); - const view = new DataView(arrayBuffer); - - // RIFF header - const writeString = (offset, string) => { - for (let i = 0; i < string.length; i++) { - view.setUint8(offset + i, string.charCodeAt(i)); - } - }; - - writeString(0, 'RIFF'); - view.setUint32(4, fileLength - 8, true); // File size - 8 - writeString(8, 'WAVE'); - - // fmt chunk - writeString(12, 'fmt '); - view.setUint32(16, 16, true); // fmt chunk size - view.setUint16(20, 1, true); // Audio format (1 = PCM) - view.setUint16(22, numChannels, true); - view.setUint32(24, sampleRate, true); - view.setUint32(28, (sampleRate * numChannels * bitsPerSample) / 8, true); // Byte rate - view.setUint16(32, (numChannels * bitsPerSample) / 8, true); // Block align - view.setUint16(34, bitsPerSample, true); - - // data chunk - writeString(36, 'data'); - view.setUint32(40, dataLength, true); - - // Copy PCM data - const dataOffset = 44; - const dataView = new Uint8Array(arrayBuffer, dataOffset); - dataView.set(new Uint8Array(pcmData.buffer || pcmData)); - - return arrayBuffer; - } - - // Audio playback methods for iOS - async playAudioChunk(base64Audio, isLastChunk = false) { - try { - // Add chunk to queue - this.audioChunks.push(base64Audio); - - // If this is the last chunk or we have enough chunks, create and play audio - if (isLastChunk || this.audioChunks.length > 10) { - await this.playAccumulatedAudio(); - } - } catch (error) { - console.error('[iOS Audio] Failed to play audio chunk:', error); - } - } - - async playAccumulatedAudio() { - if (this.audioChunks.length === 0) return; - - try { - console.log( - '[iOS Audio] Processing accumulated audio chunks:', - this.audioChunks.length - ); - - // Decode and combine all base64 chunks into one Int16Array - let totalLength = 0; - const decodedChunks = this.audioChunks.map((base64) => { - const binaryString = atob(base64); - const bytes = new Uint8Array(binaryString.length); - for (let i = 0; i < binaryString.length; i++) { - bytes[i] = binaryString.charCodeAt(i); - } - totalLength += bytes.length; - return bytes; - }); - - // Combine all chunks - const combinedData = new Uint8Array(totalLength); - let offset = 0; - for (const chunk of decodedChunks) { - combinedData.set(chunk, offset); - offset += chunk.length; - } - - // Clear chunks - this.audioChunks = []; - - // Convert PCM to WAV - const wavData = this.createWAVFromPCM(combinedData, 16000); - - // Create blob with WAV data - const audioBlob = new Blob([wavData], { type: 'audio/wav' }); - const audioUrl = URL.createObjectURL(audioBlob); - - // Clean up previous audio URL - if (this.currentAudioUrl) { - URL.revokeObjectURL(this.currentAudioUrl); - } - this.currentAudioUrl = audioUrl; - - // Get audio element - use the one we added to HTML - const audioElement = - document.getElementById('iosAudioElement') || - document.querySelector('audio'); - - if (!audioElement) { - console.error('[iOS Audio] No audio element found!'); - return; - } - - // iOS-specific audio element setup - audioElement.setAttribute('playsinline', ''); - audioElement.setAttribute('webkit-playsinline', ''); - audioElement.preload = 'auto'; - - // Add to queue for sequential playback - this.audioQueue.push(audioUrl); - - // Start playback if not already playing - if (!this.isPlaying) { - await this.playNextInQueue(audioElement); - } - } catch (error) { - console.error('[iOS Audio] Failed to play accumulated audio:', error); - } - } - - async playNextInQueue(audioElement) { - if (this.audioQueue.length === 0) { - this.isPlaying = false; - window.dispatchEvent(new CustomEvent('ios-audio-ended')); - return; - } - - this.isPlaying = true; - const audioUrl = this.audioQueue.shift(); - - return new Promise((resolve) => { - audioElement.src = audioUrl; - - audioElement.onended = async () => { - URL.revokeObjectURL(audioUrl); - await this.playNextInQueue(audioElement); - resolve(); - }; - - audioElement.onerror = (error) => { - console.error('[iOS Audio] Playback error:', error); - URL.revokeObjectURL(audioUrl); - this.playNextInQueue(audioElement); - resolve(); - }; - - // Attempt to play with user gesture handling - const playPromise = audioElement.play(); - - if (playPromise !== undefined) { - playPromise - .then(() => { - console.log('[iOS Audio] Playback started successfully'); - }) - .catch(async (error) => { - console.error('[iOS Audio] Playback failed:', error); - - // If playback fails due to user interaction requirement - if (error.name === 'NotAllowedError') { - // Wait for user interaction - console.log( - '[iOS Audio] Waiting for user interaction to play audio...' - ); - - const playOnInteraction = async () => { - try { - await audioElement.play(); - console.log( - '[iOS Audio] Playback started after user interaction' - ); - document.removeEventListener('touchstart', playOnInteraction); - document.removeEventListener('click', playOnInteraction); - } catch (e) { - console.error('[iOS Audio] Still cannot play:', e); - } - }; - - document.addEventListener('touchstart', playOnInteraction, { - once: true, - }); - document.addEventListener('click', playOnInteraction, { - once: true, - }); - } - }); - } - }); - } - - stopAudioPlayback() { - console.log('[iOS Audio] Stopping audio playback...'); - - // Clear audio queue - this.audioQueue = []; - this.audioChunks = []; - this.isPlaying = false; - - // Stop current audio - const audioElement = document.querySelector('audio'); - if (audioElement) { - audioElement.pause(); - audioElement.src = ''; - } - - // Clean up audio URLs - if (this.currentAudioUrl) { - URL.revokeObjectURL(this.currentAudioUrl); - this.currentAudioUrl = null; - } - } - - // Utility methods - arrayBufferToBase64(buffer) { - const bytes = new Uint8Array(buffer); - let binary = ''; - for (let i = 0; i < bytes.byteLength; i++) { - binary += String.fromCharCode(bytes[i]); - } - return btoa(binary); - } - - base64ToArrayBuffer(base64) { - const binaryString = atob(base64); - const len = binaryString.length; - const bytes = new Uint8Array(len); - for (let i = 0; i < len; i++) { - bytes[i] = binaryString.charCodeAt(i); - } - return bytes.buffer; - } - - // WebSocket connection helper for iOS - getOptimizedWebSocketURL() { - const protocol = location.protocol === 'https:' ? 'wss:' : 'ws:'; - const hostname = location.hostname; - const port = location.port || (protocol === 'wss:' ? '443' : '80'); - - // For production - if (hostname !== 'localhost' && hostname !== '127.0.0.1') { - return `${protocol}//${hostname}:${port}`; - } - - // For local development with iOS device - // Try to use the computer's local IP address - if (this.isIOS && hostname === 'localhost') { - // You'll need to replace this with your computer's local IP - // Or implement auto-discovery - console.warn( - '[iOS Audio] Using localhost, consider using computer IP address for iOS testing' - ); - } - - return `${protocol}//${hostname}:8765`; - } - - // Enhanced error handling for iOS - handleIOSError(error, context) { - console.error(`[iOS Audio] Error in ${context}:`, error); - - const errorMessages = { - NotAllowedError: - 'Permission denied. Please allow microphone/audio access.', - NotFoundError: 'Required audio hardware not found.', - NotReadableError: 'Audio hardware is in use by another application.', - OverconstrainedError: 'Audio constraints cannot be satisfied.', - SecurityError: 'Audio access blocked due to security settings.', - TypeError: 'Invalid audio configuration.', - }; - - const message = - errorMessages[error.name] || `Audio error: ${error.message}`; - - // Dispatch error event for app to handle - window.dispatchEvent( - new CustomEvent('ios-audio-error', { - detail: { error, context, message }, - }) - ); - - return message; - } -} - -// Export for use in main application -window.IOSAudioHandler = IOSAudioHandler; - -// Auto-initialize if on iOS -if (document.readyState === 'loading') { - document.addEventListener('DOMContentLoaded', () => { - window.iosAudioHandler = new IOSAudioHandler(); - }); -} else { - window.iosAudioHandler = new IOSAudioHandler(); -} diff --git a/frontend/js/main.js b/frontend/js/main.js deleted file mode 100644 index 4cfcc8b..0000000 --- a/frontend/js/main.js +++ /dev/null @@ -1,923 +0,0 @@ -import { WebSocketClient } from './websocket-client.js'; -import { AudioHandler } from './audio-handler.js'; -import { AudioPlayer } from './audio-player.js'; -import { ChatUI } from './chat-ui.js'; -import { FlashcardUI } from './flashcard-ui.js'; -import { Storage } from './storage.js'; - -class App { - constructor() { - this.storage = new Storage(); - this.wsClient = new WebSocketClient('ws://localhost:3000'); - this.audioHandler = new AudioHandler(); - this.audioPlayer = new AudioPlayer(); - this.chatUI = new ChatUI(); - this.flashcardUI = new FlashcardUI(); - this.userId = this.getOrCreateUserId(); - this.currentAudioElement = null; // Track current Audio element to prevent simultaneous playback - this.flashcardUI.onCardClick = (card) => { - this.wsClient.send({ type: 'flashcard_clicked', card }); - }; - - // Language state - this.currentLanguage = this.storage.getLanguage() || 'es'; - this.availableLanguages = []; - - this.state = { - chatHistory: [], - flashcards: [], - isRecording: false, - connectionStatus: 'connecting', - currentTranscript: '', - currentLLMResponse: '', - pendingTranscription: null, - pendingLLMResponse: null, - streamingLLMResponse: '', - lastPendingTranscription: null, - speechDetected: false, - llmResponseComplete: false, // Track if LLM response is complete to prevent chunk accumulation - currentResponseId: null, // Track current response to match audio with text - }; - - this.init(); - } - - async init() { - this.loadState(); - this.setupEventListeners(); - await this.fetchLanguages(); - await this.connectWebSocket(); - await this.initializeAudioPlayer(); - this.render(); - } - - async fetchLanguages() { - try { - const response = await fetch('/api/languages'); - if (response.ok) { - const data = await response.json(); - this.availableLanguages = data.languages; - - // If current language is not in available languages, use default - const isValidLanguage = this.availableLanguages.some( - (lang) => lang.code === this.currentLanguage - ); - if (!isValidLanguage) { - this.currentLanguage = data.defaultLanguage || 'es'; - } - - this.populateLanguageDropdown(); - } - } catch (error) { - console.error('Failed to fetch languages:', error); - // Fallback to default language options - this.availableLanguages = [ - { code: 'es', name: 'Spanish', nativeName: 'Español', flag: '🇲🇽' }, - { code: 'ja', name: 'Japanese', nativeName: '日本語', flag: '🇯🇵' }, - { code: 'fr', name: 'French', nativeName: 'Français', flag: '🇫🇷' }, - ]; - this.populateLanguageDropdown(); - } - } - - populateLanguageDropdown() { - const languageSelect = document.getElementById('languageSelect'); - if (!languageSelect) return; - - languageSelect.innerHTML = ''; - this.availableLanguages.forEach((lang) => { - const option = document.createElement('option'); - option.value = lang.code; - option.textContent = `${lang.flag} ${lang.name}`; - if (lang.code === this.currentLanguage) { - option.selected = true; - } - languageSelect.appendChild(option); - }); - - languageSelect.disabled = false; - } - - async initializeAudioPlayer() { - try { - await this.audioPlayer.initialize(); - console.log('Audio player initialized'); - } catch (error) { - console.error('Failed to initialize audio player:', error); - } - } - - loadState() { - const savedState = this.storage.getState(); - if (savedState) { - this.state.chatHistory = savedState.chatHistory || []; - } - - // Load flashcards from storage (for current language) - this.state.flashcards = this.storage.getFlashcards(this.currentLanguage); - - // Load existing conversation history - const existingConversation = this.storage.getConversationHistory(); - console.log( - 'Loading existing conversation history:', - existingConversation.messages.length, - 'messages' - ); - console.log( - 'Loading existing flashcards:', - this.state.flashcards.length, - 'flashcards' - ); - } - - saveState() { - this.storage.saveState({ - chatHistory: this.state.chatHistory, - }); - // Flashcards are saved separately through storage.addFlashcards() - } - - setupEventListeners() { - const micButton = document.getElementById('micButton'); - const restartButton = document.getElementById('restartButton'); - const languageSelect = document.getElementById('languageSelect'); - - // Check for iOS - const isIOS = - /iPad|iPhone|iPod/.test(navigator.userAgent) || - (navigator.platform === 'MacIntel' && navigator.maxTouchPoints > 1); - - // Add both click and touch events for better iOS support - micButton.addEventListener('click', (e) => { - e.preventDefault(); - this.toggleStreaming(); - }); - - if (isIOS) { - // Add touch event for iOS - micButton.addEventListener( - 'touchend', - (e) => { - e.preventDefault(); - this.toggleStreaming(); - }, - { passive: false } - ); - - // Prevent double-tap zoom on mic button - let lastTouchEnd = 0; - micButton.addEventListener( - 'touchend', - (e) => { - const now = Date.now(); - if (now - lastTouchEnd <= 300) { - e.preventDefault(); - } - lastTouchEnd = now; - }, - false - ); - } - - // Restart button event listener - restartButton.addEventListener('click', (e) => { - e.preventDefault(); - this.restartConversation(); - }); - - if (isIOS) { - restartButton.addEventListener( - 'touchend', - (e) => { - e.preventDefault(); - this.restartConversation(); - }, - { passive: false } - ); - } - - // Language selector event listener - languageSelect.addEventListener('change', (e) => { - const newLanguage = e.target.value; - if (newLanguage !== this.currentLanguage) { - this.changeLanguage(newLanguage); - } - }); - - this.wsClient.on('connection', (status) => { - this.state.connectionStatus = status; - - // Send existing conversation history to backend when connected - if (status === 'connected') { - // Send language preference first - this.wsClient.send({ - type: 'set_language', - languageCode: this.currentLanguage, - }); - - const existingConversation = this.storage.getConversationHistory(); - if (existingConversation.messages.length > 0) { - console.log( - 'Sending existing conversation history to backend:', - existingConversation.messages.length, - 'messages' - ); - this.wsClient.send({ - type: 'conversation_update', - data: existingConversation, - }); - } - } - - this.render(); - }); - - this.wsClient.on('transcript_update', (text) => { - this.state.currentTranscript = text; - this.state.speechDetected = true; // Ensure speech detected is true when we get transcript - this.render(); - }); - - this.wsClient.on('ai_response', (response) => { - this.addMessage('teacher', response.text); - this.state.currentTranscript = ''; - this.state.speechDetected = false; - // Note: ai_response is legacy - audio should come via audio_stream now - // But if it does come, stop any current playback first - if (response.audio) { - this.audioPlayer.stop(); - // Convert WAV base64 to PCM and play through AudioPlayer - // For now, use the legacy playAudio but stop current playback first - this.playAudio(response.audio); - } - }); - - this.wsClient.on('flashcard_generated', (flashcard) => { - this.addFlashcard(flashcard); - }); - - this.wsClient.on('flashcards_generated', (flashcards) => { - console.log('Received new flashcards:', flashcards); - this.addMultipleFlashcards(flashcards); - }); - - this.wsClient.on('speech_detected', (data) => { - // Show listening indicator immediately when VAD detects speech - // Use empty string or placeholder - transcript will update as it arrives - this.state.currentTranscript = data.text || ''; - this.state.speechDetected = true; - - // Trigger interrupt behavior immediately on VAD detection - // This freezes the current response if one is streaming - this.handleInterrupt(); - - this.render(); - }); - - this.wsClient.on('partial_transcript', (data) => { - // Update the transcript in real-time as AssemblyAI processes speech - if (data.text) { - this.state.currentTranscript = data.text; - this.state.speechDetected = true; - this.render(); - } - }); - - this.wsClient.on('speech_ended', (data) => { - // VAD stopped detecting speech - clear the real-time transcript bubble - console.log( - '[Main] VAD stopped detecting speech, clearing real-time transcript' - ); - - // Clear speech detected state to hide the real-time transcript bubble - // Only clear if we don't have a pending transcription (which means speech was processed) - // If we have a pending transcription, it will be handled by the transcription handler - if (!this.state.pendingTranscription) { - this.state.currentTranscript = ''; - this.state.speechDetected = false; - this.render(); - } - }); - - this.wsClient.on('transcription', (data) => { - // Stop any ongoing audio from previous response to prevent audio/text mismatch - // A new transcription means a new conversation turn is starting - this.audioPlayer.stop(); - if (this.currentAudioElement) { - this.currentAudioElement.pause(); - this.currentAudioElement.src = ''; - this.currentAudioElement = null; - } - - this.state.pendingTranscription = data.text; - this.state.currentTranscript = ''; - this.state.speechDetected = false; // Clear speech detected when transcription is complete - - // If we have a frozen LLM response from an interrupt, finalize it now - if (this.state.pendingLLMResponse && !this.state.streamingLLMResponse) { - // We have a frozen response from an interrupt, finalize it with this transcription - console.log( - '[Main] Finalizing frozen LLM response with new transcription' - ); - this.checkAndUpdateConversation(); - } - - // Before resetting LLM streaming, finalize any pending LLM response - // This ensures the text is added to history even if typewriter was interrupted - if ( - this.state.streamingLLMResponse && - this.state.streamingLLMResponse.trim() && - this.state.llmResponseComplete && - !this.state.pendingLLMResponse - ) { - console.log( - '[Main] Finalizing LLM response before clearing streaming state' - ); - this.state.pendingLLMResponse = this.state.streamingLLMResponse; - this.checkAndUpdateConversation(); - } - - // Reset LLM streaming for new conversation turn - this.state.streamingLLMResponse = ''; // Reset LLM streaming for new conversation - this.state.llmResponseComplete = false; // Reset completion flag for new response - this.state.currentResponseId = null; // Reset response ID - - // Only render if the transcription changed to avoid restarting typewriter - if (this.state.lastPendingTranscription !== data.text) { - this.state.lastPendingTranscription = data.text; - this.render(); - } - - // Check if we can update conversation (will happen if we just finalized a frozen response) - this.checkAndUpdateConversation(); - }); - - this.wsClient.on('llm_response_chunk', (data) => { - // Only accumulate chunks if response is not yet complete - // This prevents chunks from being added after llm_response_complete - if (!this.state.llmResponseComplete) { - this.state.streamingLLMResponse += data.text; - } else { - console.warn( - '[Main] Ignoring llm_response_chunk after response complete' - ); - } - }); - - this.wsClient.on('llm_response_complete', (data) => { - console.log( - '[Main] LLM response complete, starting typewriter with:', - data.text - ); - - // Generate a unique ID for this response to match audio with text - const responseId = `response_${Date.now()}_${Math.random().toString(36).slice(2, 9)}`; - this.state.currentResponseId = responseId; - - // Mark response as complete to prevent further chunk accumulation - this.state.llmResponseComplete = true; - - // Use the provided text (which should be the complete response) - // This ensures we use the backend's final text, not accumulated chunks - const finalText = data.text || this.state.streamingLLMResponse; - this.state.streamingLLMResponse = finalText; - - console.log('[Main] About to render for typewriter effect'); - - // Set up callback for when typewriter finishes - this.chatUI.setLLMTypewriterCallback(() => { - console.log('[Main] LLM typewriter finished, updating conversation'); - // Only update if this is still the current response (not interrupted) - if (this.state.currentResponseId === responseId) { - this.state.pendingLLMResponse = finalText; - this.checkAndUpdateConversation(); - } else { - console.log( - '[Main] Response was interrupted, skipping conversation update' - ); - } - }); - - this.render(); // Start typewriter effect - }); - - this.wsClient.on('audio_stream', (data) => { - this.handleAudioStream(data); - }); - - this.wsClient.on('audio_stream_complete', (data) => { - console.log('Audio stream complete signal received'); - this.audioPlayer.markStreamComplete(); - }); - - // Interrupt handling: stop audio playback and freeze current response - // This is triggered both by 'interrupt' message and 'speech_detected' (VAD) - this.wsClient.on('interrupt', (_data) => { - console.log('[Main] Interrupt message received'); - this.handleInterrupt(); - }); - - // Handle language change confirmation from backend - this.wsClient.on('language_changed', (data) => { - console.log(`[Main] Language changed to ${data.languageName}`); - // Optionally show a notification or update UI - }); - - this.audioHandler.on('audioChunk', (audioData) => { - this.wsClient.sendAudioChunk(audioData); - }); - - this.audioPlayer.on('playback_started', () => { - console.log('Audio playback started'); - }); - - this.audioPlayer.on('playback_finished', () => { - console.log('Audio playback finished'); - }); - } - - async changeLanguage(newLanguage) { - console.log(`[Main] Changing language from ${this.currentLanguage} to ${newLanguage}`); - - // Stop any ongoing recording - if (this.state.isRecording) { - this.audioHandler.stopStreaming(); - this.state.isRecording = false; - } - - // Stop audio playback - try { - this.audioPlayer.stop(); - } catch (error) { - console.error('Error stopping audio:', error); - } - - // Update language - this.currentLanguage = newLanguage; - this.storage.saveLanguage(newLanguage); - - // Clear conversation for new language - this.storage.clearConversation(); - this.state.chatHistory = []; - this.state.currentTranscript = ''; - this.state.currentLLMResponse = ''; - this.state.pendingTranscription = null; - this.state.pendingLLMResponse = null; - this.state.streamingLLMResponse = ''; - this.state.lastPendingTranscription = null; - this.state.speechDetected = false; - this.state.llmResponseComplete = false; - this.state.currentResponseId = null; - - // Clear typewriters - this.chatUI.clearAllTypewriters(); - - // Load flashcards for new language - this.state.flashcards = this.storage.getFlashcards(newLanguage); - - // Send language change to backend - if (this.wsClient && this.state.connectionStatus === 'connected') { - this.wsClient.send({ - type: 'set_language', - languageCode: newLanguage, - }); - } - - // Re-render UI - this.render(); - } - - async connectWebSocket() { - try { - await this.wsClient.connect(); - // After connection, send lightweight user context (timezone) - try { - const tz = Intl.DateTimeFormat().resolvedOptions().timeZone || ''; - this.wsClient.send({ - type: 'user_context', - timezone: tz, - userId: this.userId, - languageCode: this.currentLanguage, - }); - } catch (e) { - // ignore - } - } catch (error) { - console.error('WebSocket connection failed:', error); - this.state.connectionStatus = 'disconnected'; - this.render(); - } - } - - getOrCreateUserId() { - try { - const key = 'aprende-user-id'; - let id = localStorage.getItem(key); - if (!id) { - id = - typeof crypto !== 'undefined' && crypto.randomUUID - ? crypto.randomUUID() - : `u_${Date.now()}_${Math.random().toString(36).slice(2, 10)}`; - localStorage.setItem(key, id); - } - return id; - } catch (_) { - return `u_${Date.now()}_${Math.random().toString(36).slice(2, 10)}`; - } - } - - async toggleStreaming() { - if (!this.state.isRecording) { - try { - await this.audioHandler.startStreaming(); - this.state.isRecording = true; - this.state.currentTranscript = ''; - this.state.speechDetected = false; - } catch (error) { - console.error('Failed to start streaming:', error); - alert( - 'Microphone access denied. Please enable microphone permissions.' - ); - return; - } - } else { - this.audioHandler.stopStreaming(); - this.state.isRecording = false; - this.state.currentTranscript = ''; - this.state.speechDetected = false; - } - this.render(); - } - - restartConversation() { - // Stop any ongoing recording - if (this.state.isRecording) { - this.audioHandler.stopStreaming(); - this.state.isRecording = false; - } - - // Stop audio playback - try { - this.audioPlayer.stop(); - } catch (error) { - console.error('Error stopping audio:', error); - } - - // Clear conversation history from storage - this.storage.clearConversation(); - - // Clear chat history from state - this.state.chatHistory = []; - this.state.currentTranscript = ''; - this.state.currentLLMResponse = ''; - this.state.pendingTranscription = null; - this.state.pendingLLMResponse = null; - this.state.streamingLLMResponse = ''; - this.state.lastPendingTranscription = null; - this.state.speechDetected = false; - this.state.llmResponseComplete = false; - this.state.currentResponseId = null; - - // Clear typewriters - this.chatUI.clearAllTypewriters(); - - // Save cleared state - this.saveState(); - - // Send restart message to backend - if (this.wsClient && this.state.connectionStatus === 'connected') { - this.wsClient.send({ - type: 'restart_conversation', - }); - } - - // Re-render UI - this.render(); - } - - checkAndUpdateConversation() { - // Only proceed when we have both transcription and LLM response - console.log( - 'checkAndUpdateConversation called - pending transcription:', - this.state.pendingTranscription, - 'pending LLM:', - this.state.pendingLLMResponse - ); - - if (this.state.pendingTranscription && this.state.pendingLLMResponse) { - // Check if we've already added both messages together to prevent duplicates - const lastUserMessage = this.state.chatHistory - .filter((m) => m.role === 'learner') - .pop(); - const lastTeacherMessage = this.state.chatHistory - .filter((m) => m.role === 'teacher') - .pop(); - - // Check if both messages together are duplicates - const isDuplicate = - lastUserMessage?.content === this.state.pendingTranscription && - lastTeacherMessage?.content === this.state.pendingLLMResponse; - - if (isDuplicate) { - console.log( - '[Main] Duplicate conversation turn detected, skipping update' - ); - // Still clear the pending state - this.state.pendingTranscription = null; - this.state.pendingLLMResponse = null; - this.state.streamingLLMResponse = ''; - this.state.lastPendingTranscription = null; - this.state.llmResponseComplete = false; - return; - } - - // Check if teacher message was already added (from interrupt) but user message wasn't - // In this case, we just need to add the user message - const teacherAlreadyAdded = - lastTeacherMessage?.content === this.state.pendingLLMResponse; - const userAlreadyAdded = - lastUserMessage?.content === this.state.pendingTranscription; - - if (teacherAlreadyAdded && !userAlreadyAdded) { - console.log( - '[Main] Teacher message already added from interrupt, adding user message' - ); - // Just add the user message - this.storage.addMessage('user', this.state.pendingTranscription); - this.addMessageToHistory('learner', this.state.pendingTranscription); - - // Send conversation update - const conversationHistory = this.storage.getConversationHistory(); - this.wsClient.send({ - type: 'conversation_update', - data: conversationHistory, - }); - - // Clear pending state - this.state.pendingTranscription = null; - this.state.pendingLLMResponse = null; - this.state.streamingLLMResponse = ''; - this.state.lastPendingTranscription = null; - this.state.llmResponseComplete = false; - this.state.currentResponseId = null; - - this.render(); - return; - } - - console.log('Adding messages to conversation history...'); - - // Add both messages to conversation history (with automatic truncation) - const userHistory = this.storage.addMessage( - 'user', - this.state.pendingTranscription - ); - const assistantHistory = this.storage.addMessage( - 'assistant', - this.state.pendingLLMResponse - ); - - console.log( - 'User message added, total messages:', - userHistory.messages.length - ); - console.log( - 'Assistant message added, total messages:', - assistantHistory.messages.length - ); - - // Add to chat history for display - this.addMessageToHistory('learner', this.state.pendingTranscription); - this.addMessageToHistory('teacher', this.state.pendingLLMResponse); - - // Get updated conversation history and send to backend - const conversationHistory = this.storage.getConversationHistory(); - console.log( - 'Sending conversation update to backend:', - conversationHistory.messages.length, - 'messages' - ); - - this.wsClient.send({ - type: 'conversation_update', - data: conversationHistory, - }); - - // Clear pending messages and streaming state - this.state.pendingTranscription = null; - this.state.pendingLLMResponse = null; - this.state.streamingLLMResponse = ''; - this.state.lastPendingTranscription = null; - this.state.llmResponseComplete = false; - this.state.currentResponseId = null; - - // Clear any active typewriters before rendering final state - this.chatUI.clearAllTypewriters(); - this.render(); - } - } - - addMessage(role, content) { - // This method is kept for backward compatibility with existing addMessage calls - this.addMessageToHistory(role, content); - } - - addMessageToHistory(role, content) { - const message = { role, content }; - this.state.chatHistory.push(message); - this.saveState(); - this.render(); - } - - addFlashcard(flashcard) { - // Use targetWord for deduplication (backwards compatible with spanish) - const targetWord = flashcard.targetWord || flashcard.spanish || flashcard.word; - const exists = this.state.flashcards.some((card) => { - const cardWord = card.targetWord || card.spanish || card.word; - return cardWord === targetWord; - }); - - if (!exists) { - this.state.flashcards.push(flashcard); - this.storage.addFlashcards([flashcard], this.currentLanguage); - this.saveState(); - this.render(); - } - } - - addMultipleFlashcards(flashcards) { - // Use storage method which handles deduplication and persistence - const updatedFlashcards = this.storage.addFlashcards(flashcards, this.currentLanguage); - this.state.flashcards = updatedFlashcards; - this.saveState(); - this.render(); - } - - getContextForBackend() { - return { - chatHistory: this.state.chatHistory.slice(-10), - flashcards: this.state.flashcards, - }; - } - - handleInterrupt() { - console.log( - '[Main] Handling interrupt - stopping audio and freezing current response' - ); - try { - // Stop audio playback - this.audioPlayer.stop(); - // Also stop any Audio element playback - if (this.currentAudioElement) { - this.currentAudioElement.pause(); - this.currentAudioElement.src = ''; - this.currentAudioElement = null; - } - - // Freeze the current LLM response if there is one - // This means finalizing it and adding it to chat history, but keeping it visible - if ( - this.state.streamingLLMResponse && - this.state.streamingLLMResponse.trim() - ) { - console.log( - '[Main] Freezing current LLM response:', - this.state.streamingLLMResponse - ); - - // Stop typewriter effect immediately - this.chatUI.clearAllTypewriters(); - - // Get the frozen text - const frozenText = this.state.streamingLLMResponse; - - // Save the frozen response - this.state.pendingLLMResponse = frozenText; - - // If we have a pending transcription, finalize the conversation turn now - // Otherwise, we'll finalize it when the transcription arrives - if (this.state.pendingTranscription) { - // We have both, so we can finalize this conversation turn immediately - // This adds both messages to chat history - this.checkAndUpdateConversation(); - } else { - // No transcription yet - add just the LLM response to chat history now - // so it stays visible. We'll add the user message when transcription arrives. - // Check if this message is already in history to avoid duplicates - const lastTeacherMessage = this.state.chatHistory - .filter((m) => m.role === 'teacher') - .pop(); - - if (lastTeacherMessage?.content !== frozenText) { - console.log('[Main] Adding frozen LLM response to chat history'); - this.addMessageToHistory('teacher', frozenText); - } - } - - // Clear streaming state - this will remove the streaming element - // But the message is now in chat history, so it will stay visible - this.state.streamingLLMResponse = ''; - this.state.llmResponseComplete = false; - this.state.currentResponseId = null; - } else { - // No streaming response, just clear state - this.state.streamingLLMResponse = ''; - this.state.llmResponseComplete = false; - this.state.currentResponseId = null; - } - - // Render to update UI - this.render(); - } catch (error) { - console.warn('Error handling interrupt:', error); - } - } - - async handleAudioStream(data) { - try { - if (data.audio && data.audio.length > 0) { - console.log( - `Received audio stream: ${data.audio.length} bytes at ${data.sampleRate}Hz [format:${data.audioFormat}]${data.text ? ` with text: "${data.text}"` : ''}` - ); - await this.audioPlayer.addAudioStream(data.audio, data.sampleRate, false, data.audioFormat); - } - } catch (error) { - console.error('Error handling audio stream:', error); - } - } - - playAudio(audioData) { - // Stop any current AudioPlayer playback to prevent simultaneous audio - this.audioPlayer.stop(); - - // Stop any existing Audio element playback - if (this.currentAudioElement) { - try { - this.currentAudioElement.pause(); - this.currentAudioElement.src = ''; - this.currentAudioElement = null; - } catch (error) { - console.warn('Error stopping previous audio element:', error); - } - } - - const audio = new Audio(); - this.currentAudioElement = audio; - audio.src = `data:audio/wav;base64,${audioData}`; - audio.play().catch(console.error); - - // Clean up when audio finishes - audio.onended = () => { - if (this.currentAudioElement === audio) { - this.currentAudioElement = null; - } - }; - - audio.onerror = () => { - if (this.currentAudioElement === audio) { - this.currentAudioElement = null; - } - }; - } - - render() { - this.updateConnectionStatus(); - this.chatUI.render( - this.state.chatHistory, - this.state.currentTranscript, - this.state.currentLLMResponse, - this.state.pendingTranscription, - this.state.streamingLLMResponse, - this.state.isRecording, - this.state.speechDetected - ); - this.flashcardUI.render(this.state.flashcards, this.currentLanguage); - - const micButton = document.getElementById('micButton'); - const restartButton = document.getElementById('restartButton'); - const languageSelect = document.getElementById('languageSelect'); - - micButton.disabled = this.state.connectionStatus !== 'connected'; - micButton.classList.toggle('recording', this.state.isRecording); - restartButton.disabled = this.state.connectionStatus !== 'connected'; - languageSelect.disabled = this.state.connectionStatus !== 'connected'; - } - - updateConnectionStatus() { - const statusDot = document.getElementById('statusDot'); - const statusText = document.getElementById('statusText'); - - statusDot.className = `status-dot ${this.state.connectionStatus}`; - - const statusMessages = { - connecting: 'Connecting...', - connected: 'Connected', - disconnected: 'Disconnected', - }; - - statusText.textContent = - statusMessages[this.state.connectionStatus] || 'Unknown'; - } -} - -new App(); diff --git a/frontend/package-lock.json b/frontend/package-lock.json new file mode 100644 index 0000000..805265b --- /dev/null +++ b/frontend/package-lock.json @@ -0,0 +1,3212 @@ +{ + "name": "frontend", + "version": "0.0.0", + "lockfileVersion": 3, + "requires": true, + "packages": { + "": { + "name": "frontend", + "version": "0.0.0", + "dependencies": { + "react": "^19.2.0", + "react-dom": "^19.2.0" + }, + "devDependencies": { + "@eslint/js": "^9.39.1", + "@types/node": "^24.10.1", + "@types/react": "^19.2.5", + "@types/react-dom": "^19.2.3", + "@vitejs/plugin-react": "^5.1.1", + "eslint": "^9.39.1", + "eslint-plugin-react-hooks": "^7.0.1", + "eslint-plugin-react-refresh": "^0.4.24", + "globals": "^16.5.0", + "typescript": "~5.9.3", + "typescript-eslint": "^8.46.4", + "vite": "^7.2.4" + } + }, + "node_modules/@babel/code-frame": { + "version": "7.27.1", + "resolved": "https://registry.npmjs.org/@babel/code-frame/-/code-frame-7.27.1.tgz", + "integrity": "sha512-cjQ7ZlQ0Mv3b47hABuTevyTuYN4i+loJKGeV9flcCgIK37cCXRh+L1bd3iBHlynerhQ7BhCkn2BPbQUL+rGqFg==", + "dev": true, + "license": "MIT", + "dependencies": { + "@babel/helper-validator-identifier": "^7.27.1", + "js-tokens": "^4.0.0", + "picocolors": "^1.1.1" + }, + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@babel/compat-data": { + "version": "7.28.5", + "resolved": "https://registry.npmjs.org/@babel/compat-data/-/compat-data-7.28.5.tgz", + "integrity": "sha512-6uFXyCayocRbqhZOB+6XcuZbkMNimwfVGFji8CTZnCzOHVGvDqzvitu1re2AU5LROliz7eQPhB8CpAMvnx9EjA==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@babel/core": { + "version": "7.28.5", + "resolved": "https://registry.npmjs.org/@babel/core/-/core-7.28.5.tgz", + "integrity": "sha512-e7jT4DxYvIDLk1ZHmU/m/mB19rex9sv0c2ftBtjSBv+kVM/902eh0fINUzD7UwLLNR+jU585GxUJ8/EBfAM5fw==", + "dev": true, + "license": "MIT", + "dependencies": { + "@babel/code-frame": "^7.27.1", + "@babel/generator": "^7.28.5", + "@babel/helper-compilation-targets": "^7.27.2", + "@babel/helper-module-transforms": "^7.28.3", + "@babel/helpers": "^7.28.4", + "@babel/parser": "^7.28.5", + "@babel/template": "^7.27.2", + "@babel/traverse": "^7.28.5", + "@babel/types": "^7.28.5", + "@jridgewell/remapping": "^2.3.5", + "convert-source-map": "^2.0.0", + "debug": "^4.1.0", + "gensync": "^1.0.0-beta.2", + "json5": "^2.2.3", + "semver": "^6.3.1" + }, + "engines": { + "node": ">=6.9.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/babel" + } + }, + "node_modules/@babel/generator": { + "version": "7.28.5", + "resolved": "https://registry.npmjs.org/@babel/generator/-/generator-7.28.5.tgz", + "integrity": "sha512-3EwLFhZ38J4VyIP6WNtt2kUdW9dokXA9Cr4IVIFHuCpZ3H8/YFOl5JjZHisrn1fATPBmKKqXzDFvh9fUwHz6CQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "@babel/parser": "^7.28.5", + "@babel/types": "^7.28.5", + "@jridgewell/gen-mapping": "^0.3.12", + "@jridgewell/trace-mapping": "^0.3.28", + "jsesc": "^3.0.2" + }, + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@babel/helper-compilation-targets": { + "version": "7.27.2", + "resolved": "https://registry.npmjs.org/@babel/helper-compilation-targets/-/helper-compilation-targets-7.27.2.tgz", + "integrity": "sha512-2+1thGUUWWjLTYTHZWK1n8Yga0ijBz1XAhUXcKy81rd5g6yh7hGqMp45v7cadSbEHc9G3OTv45SyneRN3ps4DQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "@babel/compat-data": "^7.27.2", + "@babel/helper-validator-option": "^7.27.1", + "browserslist": "^4.24.0", + "lru-cache": "^5.1.1", + "semver": "^6.3.1" + }, + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@babel/helper-globals": { + "version": "7.28.0", + "resolved": "https://registry.npmjs.org/@babel/helper-globals/-/helper-globals-7.28.0.tgz", + "integrity": "sha512-+W6cISkXFa1jXsDEdYA8HeevQT/FULhxzR99pxphltZcVaugps53THCeiWA8SguxxpSp3gKPiuYfSWopkLQ4hw==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@babel/helper-module-imports": { + "version": "7.27.1", + "resolved": "https://registry.npmjs.org/@babel/helper-module-imports/-/helper-module-imports-7.27.1.tgz", + "integrity": "sha512-0gSFWUPNXNopqtIPQvlD5WgXYI5GY2kP2cCvoT8kczjbfcfuIljTbcWrulD1CIPIX2gt1wghbDy08yE1p+/r3w==", + "dev": true, + "license": "MIT", + "dependencies": { + "@babel/traverse": "^7.27.1", + "@babel/types": "^7.27.1" + }, + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@babel/helper-module-transforms": { + "version": "7.28.3", + "resolved": "https://registry.npmjs.org/@babel/helper-module-transforms/-/helper-module-transforms-7.28.3.tgz", + "integrity": "sha512-gytXUbs8k2sXS9PnQptz5o0QnpLL51SwASIORY6XaBKF88nsOT0Zw9szLqlSGQDP/4TljBAD5y98p2U1fqkdsw==", + "dev": true, + "license": "MIT", + "dependencies": { + "@babel/helper-module-imports": "^7.27.1", + "@babel/helper-validator-identifier": "^7.27.1", + "@babel/traverse": "^7.28.3" + }, + "engines": { + "node": ">=6.9.0" + }, + "peerDependencies": { + "@babel/core": "^7.0.0" + } + }, + "node_modules/@babel/helper-plugin-utils": { + "version": "7.27.1", + "resolved": "https://registry.npmjs.org/@babel/helper-plugin-utils/-/helper-plugin-utils-7.27.1.tgz", + "integrity": "sha512-1gn1Up5YXka3YYAHGKpbideQ5Yjf1tDa9qYcgysz+cNCXukyLl6DjPXhD3VRwSb8c0J9tA4b2+rHEZtc6R0tlw==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@babel/helper-string-parser": { + "version": "7.27.1", + "resolved": "https://registry.npmjs.org/@babel/helper-string-parser/-/helper-string-parser-7.27.1.tgz", + "integrity": "sha512-qMlSxKbpRlAridDExk92nSobyDdpPijUq2DW6oDnUqd0iOGxmQjyqhMIihI9+zv4LPyZdRje2cavWPbCbWm3eA==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@babel/helper-validator-identifier": { + "version": "7.28.5", + "resolved": "https://registry.npmjs.org/@babel/helper-validator-identifier/-/helper-validator-identifier-7.28.5.tgz", + "integrity": "sha512-qSs4ifwzKJSV39ucNjsvc6WVHs6b7S03sOh2OcHF9UHfVPqWWALUsNUVzhSBiItjRZoLHx7nIarVjqKVusUZ1Q==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@babel/helper-validator-option": { + "version": "7.27.1", + "resolved": "https://registry.npmjs.org/@babel/helper-validator-option/-/helper-validator-option-7.27.1.tgz", + "integrity": "sha512-YvjJow9FxbhFFKDSuFnVCe2WxXk1zWc22fFePVNEaWJEu8IrZVlda6N0uHwzZrUM1il7NC9Mlp4MaJYbYd9JSg==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@babel/helpers": { + "version": "7.28.4", + "resolved": "https://registry.npmjs.org/@babel/helpers/-/helpers-7.28.4.tgz", + "integrity": "sha512-HFN59MmQXGHVyYadKLVumYsA9dBFun/ldYxipEjzA4196jpLZd8UjEEBLkbEkvfYreDqJhZxYAWFPtrfhNpj4w==", + "dev": true, + "license": "MIT", + "dependencies": { + "@babel/template": "^7.27.2", + "@babel/types": "^7.28.4" + }, + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@babel/parser": { + "version": "7.28.5", + "resolved": "https://registry.npmjs.org/@babel/parser/-/parser-7.28.5.tgz", + "integrity": "sha512-KKBU1VGYR7ORr3At5HAtUQ+TV3SzRCXmA/8OdDZiLDBIZxVyzXuztPjfLd3BV1PRAQGCMWWSHYhL0F8d5uHBDQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "@babel/types": "^7.28.5" + }, + "bin": { + "parser": "bin/babel-parser.js" + }, + "engines": { + "node": ">=6.0.0" + } + }, + "node_modules/@babel/plugin-transform-react-jsx-self": { + "version": "7.27.1", + "resolved": "https://registry.npmjs.org/@babel/plugin-transform-react-jsx-self/-/plugin-transform-react-jsx-self-7.27.1.tgz", + "integrity": "sha512-6UzkCs+ejGdZ5mFFC/OCUrv028ab2fp1znZmCZjAOBKiBK2jXD1O+BPSfX8X2qjJ75fZBMSnQn3Rq2mrBJK2mw==", + "dev": true, + "license": "MIT", + "dependencies": { + "@babel/helper-plugin-utils": "^7.27.1" + }, + "engines": { + "node": ">=6.9.0" + }, + "peerDependencies": { + "@babel/core": "^7.0.0-0" + } + }, + "node_modules/@babel/plugin-transform-react-jsx-source": { + "version": "7.27.1", + "resolved": "https://registry.npmjs.org/@babel/plugin-transform-react-jsx-source/-/plugin-transform-react-jsx-source-7.27.1.tgz", + "integrity": "sha512-zbwoTsBruTeKB9hSq73ha66iFeJHuaFkUbwvqElnygoNbj/jHRsSeokowZFN3CZ64IvEqcmmkVe89OPXc7ldAw==", + "dev": true, + "license": "MIT", + "dependencies": { + "@babel/helper-plugin-utils": "^7.27.1" + }, + "engines": { + "node": ">=6.9.0" + }, + "peerDependencies": { + "@babel/core": "^7.0.0-0" + } + }, + "node_modules/@babel/template": { + "version": "7.27.2", + "resolved": "https://registry.npmjs.org/@babel/template/-/template-7.27.2.tgz", + "integrity": "sha512-LPDZ85aEJyYSd18/DkjNh4/y1ntkE5KwUHWTiqgRxruuZL2F1yuHligVHLvcHY2vMHXttKFpJn6LwfI7cw7ODw==", + "dev": true, + "license": "MIT", + "dependencies": { + "@babel/code-frame": "^7.27.1", + "@babel/parser": "^7.27.2", + "@babel/types": "^7.27.1" + }, + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@babel/traverse": { + "version": "7.28.5", + "resolved": "https://registry.npmjs.org/@babel/traverse/-/traverse-7.28.5.tgz", + "integrity": "sha512-TCCj4t55U90khlYkVV/0TfkJkAkUg3jZFA3Neb7unZT8CPok7iiRfaX0F+WnqWqt7OxhOn0uBKXCw4lbL8W0aQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "@babel/code-frame": "^7.27.1", + "@babel/generator": "^7.28.5", + "@babel/helper-globals": "^7.28.0", + "@babel/parser": "^7.28.5", + "@babel/template": "^7.27.2", + "@babel/types": "^7.28.5", + "debug": "^4.3.1" + }, + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@babel/types": { + "version": "7.28.5", + "resolved": "https://registry.npmjs.org/@babel/types/-/types-7.28.5.tgz", + "integrity": "sha512-qQ5m48eI/MFLQ5PxQj4PFaprjyCTLI37ElWMmNs0K8Lk3dVeOdNpB3ks8jc7yM5CDmVC73eMVk/trk3fgmrUpA==", + "dev": true, + "license": "MIT", + "dependencies": { + "@babel/helper-string-parser": "^7.27.1", + "@babel/helper-validator-identifier": "^7.28.5" + }, + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@esbuild/aix-ppc64": { + "version": "0.27.2", + "resolved": "https://registry.npmjs.org/@esbuild/aix-ppc64/-/aix-ppc64-0.27.2.tgz", + "integrity": "sha512-GZMB+a0mOMZs4MpDbj8RJp4cw+w1WV5NYD6xzgvzUJ5Ek2jerwfO2eADyI6ExDSUED+1X8aMbegahsJi+8mgpw==", + "cpu": [ + "ppc64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "aix" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/android-arm": { + "version": "0.27.2", + "resolved": "https://registry.npmjs.org/@esbuild/android-arm/-/android-arm-0.27.2.tgz", + "integrity": "sha512-DVNI8jlPa7Ujbr1yjU2PfUSRtAUZPG9I1RwW4F4xFB1Imiu2on0ADiI/c3td+KmDtVKNbi+nffGDQMfcIMkwIA==", + "cpu": [ + "arm" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "android" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/android-arm64": { + "version": "0.27.2", + "resolved": "https://registry.npmjs.org/@esbuild/android-arm64/-/android-arm64-0.27.2.tgz", + "integrity": "sha512-pvz8ZZ7ot/RBphf8fv60ljmaoydPU12VuXHImtAs0XhLLw+EXBi2BLe3OYSBslR4rryHvweW5gmkKFwTiFy6KA==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "android" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/android-x64": { + "version": "0.27.2", + "resolved": "https://registry.npmjs.org/@esbuild/android-x64/-/android-x64-0.27.2.tgz", + "integrity": "sha512-z8Ank4Byh4TJJOh4wpz8g2vDy75zFL0TlZlkUkEwYXuPSgX8yzep596n6mT7905kA9uHZsf/o2OJZubl2l3M7A==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "android" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/darwin-arm64": { + "version": "0.27.2", + "resolved": "https://registry.npmjs.org/@esbuild/darwin-arm64/-/darwin-arm64-0.27.2.tgz", + "integrity": "sha512-davCD2Zc80nzDVRwXTcQP/28fiJbcOwvdolL0sOiOsbwBa72kegmVU0Wrh1MYrbuCL98Omp5dVhQFWRKR2ZAlg==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/darwin-x64": { + "version": "0.27.2", + "resolved": "https://registry.npmjs.org/@esbuild/darwin-x64/-/darwin-x64-0.27.2.tgz", + "integrity": "sha512-ZxtijOmlQCBWGwbVmwOF/UCzuGIbUkqB1faQRf5akQmxRJ1ujusWsb3CVfk/9iZKr2L5SMU5wPBi1UWbvL+VQA==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/freebsd-arm64": { + "version": "0.27.2", + "resolved": "https://registry.npmjs.org/@esbuild/freebsd-arm64/-/freebsd-arm64-0.27.2.tgz", + "integrity": "sha512-lS/9CN+rgqQ9czogxlMcBMGd+l8Q3Nj1MFQwBZJyoEKI50XGxwuzznYdwcav6lpOGv5BqaZXqvBSiB/kJ5op+g==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "freebsd" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/freebsd-x64": { + "version": "0.27.2", + "resolved": "https://registry.npmjs.org/@esbuild/freebsd-x64/-/freebsd-x64-0.27.2.tgz", + "integrity": "sha512-tAfqtNYb4YgPnJlEFu4c212HYjQWSO/w/h/lQaBK7RbwGIkBOuNKQI9tqWzx7Wtp7bTPaGC6MJvWI608P3wXYA==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "freebsd" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-arm": { + "version": "0.27.2", + "resolved": "https://registry.npmjs.org/@esbuild/linux-arm/-/linux-arm-0.27.2.tgz", + "integrity": "sha512-vWfq4GaIMP9AIe4yj1ZUW18RDhx6EPQKjwe7n8BbIecFtCQG4CfHGaHuh7fdfq+y3LIA2vGS/o9ZBGVxIDi9hw==", + "cpu": [ + "arm" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-arm64": { + "version": "0.27.2", + "resolved": "https://registry.npmjs.org/@esbuild/linux-arm64/-/linux-arm64-0.27.2.tgz", + "integrity": "sha512-hYxN8pr66NsCCiRFkHUAsxylNOcAQaxSSkHMMjcpx0si13t1LHFphxJZUiGwojB1a/Hd5OiPIqDdXONia6bhTw==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-ia32": { + "version": "0.27.2", + "resolved": "https://registry.npmjs.org/@esbuild/linux-ia32/-/linux-ia32-0.27.2.tgz", + "integrity": "sha512-MJt5BRRSScPDwG2hLelYhAAKh9imjHK5+NE/tvnRLbIqUWa+0E9N4WNMjmp/kXXPHZGqPLxggwVhz7QP8CTR8w==", + "cpu": [ + "ia32" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-loong64": { + "version": "0.27.2", + "resolved": "https://registry.npmjs.org/@esbuild/linux-loong64/-/linux-loong64-0.27.2.tgz", + "integrity": "sha512-lugyF1atnAT463aO6KPshVCJK5NgRnU4yb3FUumyVz+cGvZbontBgzeGFO1nF+dPueHD367a2ZXe1NtUkAjOtg==", + "cpu": [ + "loong64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-mips64el": { + "version": "0.27.2", + "resolved": "https://registry.npmjs.org/@esbuild/linux-mips64el/-/linux-mips64el-0.27.2.tgz", + "integrity": "sha512-nlP2I6ArEBewvJ2gjrrkESEZkB5mIoaTswuqNFRv/WYd+ATtUpe9Y09RnJvgvdag7he0OWgEZWhviS1OTOKixw==", + "cpu": [ + "mips64el" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-ppc64": { + "version": "0.27.2", + "resolved": "https://registry.npmjs.org/@esbuild/linux-ppc64/-/linux-ppc64-0.27.2.tgz", + "integrity": "sha512-C92gnpey7tUQONqg1n6dKVbx3vphKtTHJaNG2Ok9lGwbZil6DrfyecMsp9CrmXGQJmZ7iiVXvvZH6Ml5hL6XdQ==", + "cpu": [ + "ppc64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-riscv64": { + "version": "0.27.2", + "resolved": "https://registry.npmjs.org/@esbuild/linux-riscv64/-/linux-riscv64-0.27.2.tgz", + "integrity": "sha512-B5BOmojNtUyN8AXlK0QJyvjEZkWwy/FKvakkTDCziX95AowLZKR6aCDhG7LeF7uMCXEJqwa8Bejz5LTPYm8AvA==", + "cpu": [ + "riscv64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-s390x": { + "version": "0.27.2", + "resolved": "https://registry.npmjs.org/@esbuild/linux-s390x/-/linux-s390x-0.27.2.tgz", + "integrity": "sha512-p4bm9+wsPwup5Z8f4EpfN63qNagQ47Ua2znaqGH6bqLlmJ4bx97Y9JdqxgGZ6Y8xVTixUnEkoKSHcpRlDnNr5w==", + "cpu": [ + "s390x" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-x64": { + "version": "0.27.2", + "resolved": "https://registry.npmjs.org/@esbuild/linux-x64/-/linux-x64-0.27.2.tgz", + "integrity": "sha512-uwp2Tip5aPmH+NRUwTcfLb+W32WXjpFejTIOWZFw/v7/KnpCDKG66u4DLcurQpiYTiYwQ9B7KOeMJvLCu/OvbA==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/netbsd-arm64": { + "version": "0.27.2", + "resolved": "https://registry.npmjs.org/@esbuild/netbsd-arm64/-/netbsd-arm64-0.27.2.tgz", + "integrity": "sha512-Kj6DiBlwXrPsCRDeRvGAUb/LNrBASrfqAIok+xB0LxK8CHqxZ037viF13ugfsIpePH93mX7xfJp97cyDuTZ3cw==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "netbsd" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/netbsd-x64": { + "version": "0.27.2", + "resolved": "https://registry.npmjs.org/@esbuild/netbsd-x64/-/netbsd-x64-0.27.2.tgz", + "integrity": "sha512-HwGDZ0VLVBY3Y+Nw0JexZy9o/nUAWq9MlV7cahpaXKW6TOzfVno3y3/M8Ga8u8Yr7GldLOov27xiCnqRZf0tCA==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "netbsd" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/openbsd-arm64": { + "version": "0.27.2", + "resolved": "https://registry.npmjs.org/@esbuild/openbsd-arm64/-/openbsd-arm64-0.27.2.tgz", + "integrity": "sha512-DNIHH2BPQ5551A7oSHD0CKbwIA/Ox7+78/AWkbS5QoRzaqlev2uFayfSxq68EkonB+IKjiuxBFoV8ESJy8bOHA==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "openbsd" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/openbsd-x64": { + "version": "0.27.2", + "resolved": "https://registry.npmjs.org/@esbuild/openbsd-x64/-/openbsd-x64-0.27.2.tgz", + "integrity": "sha512-/it7w9Nb7+0KFIzjalNJVR5bOzA9Vay+yIPLVHfIQYG/j+j9VTH84aNB8ExGKPU4AzfaEvN9/V4HV+F+vo8OEg==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "openbsd" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/openharmony-arm64": { + "version": "0.27.2", + "resolved": "https://registry.npmjs.org/@esbuild/openharmony-arm64/-/openharmony-arm64-0.27.2.tgz", + "integrity": "sha512-LRBbCmiU51IXfeXk59csuX/aSaToeG7w48nMwA6049Y4J4+VbWALAuXcs+qcD04rHDuSCSRKdmY63sruDS5qag==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "openharmony" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/sunos-x64": { + "version": "0.27.2", + "resolved": "https://registry.npmjs.org/@esbuild/sunos-x64/-/sunos-x64-0.27.2.tgz", + "integrity": "sha512-kMtx1yqJHTmqaqHPAzKCAkDaKsffmXkPHThSfRwZGyuqyIeBvf08KSsYXl+abf5HDAPMJIPnbBfXvP2ZC2TfHg==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "sunos" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/win32-arm64": { + "version": "0.27.2", + "resolved": "https://registry.npmjs.org/@esbuild/win32-arm64/-/win32-arm64-0.27.2.tgz", + "integrity": "sha512-Yaf78O/B3Kkh+nKABUF++bvJv5Ijoy9AN1ww904rOXZFLWVc5OLOfL56W+C8F9xn5JQZa3UX6m+IktJnIb1Jjg==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/win32-ia32": { + "version": "0.27.2", + "resolved": "https://registry.npmjs.org/@esbuild/win32-ia32/-/win32-ia32-0.27.2.tgz", + "integrity": "sha512-Iuws0kxo4yusk7sw70Xa2E2imZU5HoixzxfGCdxwBdhiDgt9vX9VUCBhqcwY7/uh//78A1hMkkROMJq9l27oLQ==", + "cpu": [ + "ia32" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/win32-x64": { + "version": "0.27.2", + "resolved": "https://registry.npmjs.org/@esbuild/win32-x64/-/win32-x64-0.27.2.tgz", + "integrity": "sha512-sRdU18mcKf7F+YgheI/zGf5alZatMUTKj/jNS6l744f9u3WFu4v7twcUI9vu4mknF4Y9aDlblIie0IM+5xxaqQ==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@eslint-community/eslint-utils": { + "version": "4.9.0", + "resolved": "https://registry.npmjs.org/@eslint-community/eslint-utils/-/eslint-utils-4.9.0.tgz", + "integrity": "sha512-ayVFHdtZ+hsq1t2Dy24wCmGXGe4q9Gu3smhLYALJrr473ZH27MsnSL+LKUlimp4BWJqMDMLmPpx/Q9R3OAlL4g==", + "dev": true, + "license": "MIT", + "dependencies": { + "eslint-visitor-keys": "^3.4.3" + }, + "engines": { + "node": "^12.22.0 || ^14.17.0 || >=16.0.0" + }, + "funding": { + "url": "https://opencollective.com/eslint" + }, + "peerDependencies": { + "eslint": "^6.0.0 || ^7.0.0 || >=8.0.0" + } + }, + "node_modules/@eslint-community/eslint-utils/node_modules/eslint-visitor-keys": { + "version": "3.4.3", + "resolved": "https://registry.npmjs.org/eslint-visitor-keys/-/eslint-visitor-keys-3.4.3.tgz", + "integrity": "sha512-wpc+LXeiyiisxPlEkUzU6svyS1frIO3Mgxj1fdy7Pm8Ygzguax2N3Fa/D/ag1WqbOprdI+uY6wMUl8/a2G+iag==", + "dev": true, + "license": "Apache-2.0", + "engines": { + "node": "^12.22.0 || ^14.17.0 || >=16.0.0" + }, + "funding": { + "url": "https://opencollective.com/eslint" + } + }, + "node_modules/@eslint-community/regexpp": { + "version": "4.12.2", + "resolved": "https://registry.npmjs.org/@eslint-community/regexpp/-/regexpp-4.12.2.tgz", + "integrity": "sha512-EriSTlt5OC9/7SXkRSCAhfSxxoSUgBm33OH+IkwbdpgoqsSsUg7y3uh+IICI/Qg4BBWr3U2i39RpmycbxMq4ew==", + "dev": true, + "license": "MIT", + "engines": { + "node": "^12.0.0 || ^14.0.0 || >=16.0.0" + } + }, + "node_modules/@eslint/config-array": { + "version": "0.21.1", + "resolved": "https://registry.npmjs.org/@eslint/config-array/-/config-array-0.21.1.tgz", + "integrity": "sha512-aw1gNayWpdI/jSYVgzN5pL0cfzU02GT3NBpeT/DXbx1/1x7ZKxFPd9bwrzygx/qiwIQiJ1sw/zD8qY/kRvlGHA==", + "dev": true, + "license": "Apache-2.0", + "dependencies": { + "@eslint/object-schema": "^2.1.7", + "debug": "^4.3.1", + "minimatch": "^3.1.2" + }, + "engines": { + "node": "^18.18.0 || ^20.9.0 || >=21.1.0" + } + }, + "node_modules/@eslint/config-helpers": { + "version": "0.4.2", + "resolved": "https://registry.npmjs.org/@eslint/config-helpers/-/config-helpers-0.4.2.tgz", + "integrity": "sha512-gBrxN88gOIf3R7ja5K9slwNayVcZgK6SOUORm2uBzTeIEfeVaIhOpCtTox3P6R7o2jLFwLFTLnC7kU/RGcYEgw==", + "dev": true, + "license": "Apache-2.0", + "dependencies": { + "@eslint/core": "^0.17.0" + }, + "engines": { + "node": "^18.18.0 || ^20.9.0 || >=21.1.0" + } + }, + "node_modules/@eslint/core": { + "version": "0.17.0", + "resolved": "https://registry.npmjs.org/@eslint/core/-/core-0.17.0.tgz", + "integrity": "sha512-yL/sLrpmtDaFEiUj1osRP4TI2MDz1AddJL+jZ7KSqvBuliN4xqYY54IfdN8qD8Toa6g1iloph1fxQNkjOxrrpQ==", + "dev": true, + "license": "Apache-2.0", + "dependencies": { + "@types/json-schema": "^7.0.15" + }, + "engines": { + "node": "^18.18.0 || ^20.9.0 || >=21.1.0" + } + }, + "node_modules/@eslint/eslintrc": { + "version": "3.3.3", + "resolved": "https://registry.npmjs.org/@eslint/eslintrc/-/eslintrc-3.3.3.tgz", + "integrity": "sha512-Kr+LPIUVKz2qkx1HAMH8q1q6azbqBAsXJUxBl/ODDuVPX45Z9DfwB8tPjTi6nNZ8BuM3nbJxC5zCAg5elnBUTQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "ajv": "^6.12.4", + "debug": "^4.3.2", + "espree": "^10.0.1", + "globals": "^14.0.0", + "ignore": "^5.2.0", + "import-fresh": "^3.2.1", + "js-yaml": "^4.1.1", + "minimatch": "^3.1.2", + "strip-json-comments": "^3.1.1" + }, + "engines": { + "node": "^18.18.0 || ^20.9.0 || >=21.1.0" + }, + "funding": { + "url": "https://opencollective.com/eslint" + } + }, + "node_modules/@eslint/eslintrc/node_modules/globals": { + "version": "14.0.0", + "resolved": "https://registry.npmjs.org/globals/-/globals-14.0.0.tgz", + "integrity": "sha512-oahGvuMGQlPw/ivIYBjVSrWAfWLBeku5tpPE2fOPLi+WHffIWbuh2tCjhyQhTBPMf5E9jDEH4FOmTYgYwbKwtQ==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=18" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/@eslint/js": { + "version": "9.39.2", + "resolved": "https://registry.npmjs.org/@eslint/js/-/js-9.39.2.tgz", + "integrity": "sha512-q1mjIoW1VX4IvSocvM/vbTiveKC4k9eLrajNEuSsmjymSDEbpGddtpfOoN7YGAqBK3NG+uqo8ia4PDTt8buCYA==", + "dev": true, + "license": "MIT", + "engines": { + "node": "^18.18.0 || ^20.9.0 || >=21.1.0" + }, + "funding": { + "url": "https://eslint.org/donate" + } + }, + "node_modules/@eslint/object-schema": { + "version": "2.1.7", + "resolved": "https://registry.npmjs.org/@eslint/object-schema/-/object-schema-2.1.7.tgz", + "integrity": "sha512-VtAOaymWVfZcmZbp6E2mympDIHvyjXs/12LqWYjVw6qjrfF+VK+fyG33kChz3nnK+SU5/NeHOqrTEHS8sXO3OA==", + "dev": true, + "license": "Apache-2.0", + "engines": { + "node": "^18.18.0 || ^20.9.0 || >=21.1.0" + } + }, + "node_modules/@eslint/plugin-kit": { + "version": "0.4.1", + "resolved": "https://registry.npmjs.org/@eslint/plugin-kit/-/plugin-kit-0.4.1.tgz", + "integrity": "sha512-43/qtrDUokr7LJqoF2c3+RInu/t4zfrpYdoSDfYyhg52rwLV6TnOvdG4fXm7IkSB3wErkcmJS9iEhjVtOSEjjA==", + "dev": true, + "license": "Apache-2.0", + "dependencies": { + "@eslint/core": "^0.17.0", + "levn": "^0.4.1" + }, + "engines": { + "node": "^18.18.0 || ^20.9.0 || >=21.1.0" + } + }, + "node_modules/@humanfs/core": { + "version": "0.19.1", + "resolved": "https://registry.npmjs.org/@humanfs/core/-/core-0.19.1.tgz", + "integrity": "sha512-5DyQ4+1JEUzejeK1JGICcideyfUbGixgS9jNgex5nqkW+cY7WZhxBigmieN5Qnw9ZosSNVC9KQKyb+GUaGyKUA==", + "dev": true, + "license": "Apache-2.0", + "engines": { + "node": ">=18.18.0" + } + }, + "node_modules/@humanfs/node": { + "version": "0.16.7", + "resolved": "https://registry.npmjs.org/@humanfs/node/-/node-0.16.7.tgz", + "integrity": "sha512-/zUx+yOsIrG4Y43Eh2peDeKCxlRt/gET6aHfaKpuq267qXdYDFViVHfMaLyygZOnl0kGWxFIgsBy8QFuTLUXEQ==", + "dev": true, + "license": "Apache-2.0", + "dependencies": { + "@humanfs/core": "^0.19.1", + "@humanwhocodes/retry": "^0.4.0" + }, + "engines": { + "node": ">=18.18.0" + } + }, + "node_modules/@humanwhocodes/module-importer": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/@humanwhocodes/module-importer/-/module-importer-1.0.1.tgz", + "integrity": "sha512-bxveV4V8v5Yb4ncFTT3rPSgZBOpCkjfK0y4oVVVJwIuDVBRMDXrPyXRL988i5ap9m9bnyEEjWfm5WkBmtffLfA==", + "dev": true, + "license": "Apache-2.0", + "engines": { + "node": ">=12.22" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/nzakas" + } + }, + "node_modules/@humanwhocodes/retry": { + "version": "0.4.3", + "resolved": "https://registry.npmjs.org/@humanwhocodes/retry/-/retry-0.4.3.tgz", + "integrity": "sha512-bV0Tgo9K4hfPCek+aMAn81RppFKv2ySDQeMoSZuvTASywNTnVJCArCZE2FWqpvIatKu7VMRLWlR1EazvVhDyhQ==", + "dev": true, + "license": "Apache-2.0", + "engines": { + "node": ">=18.18" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/nzakas" + } + }, + "node_modules/@jridgewell/gen-mapping": { + "version": "0.3.13", + "resolved": "https://registry.npmjs.org/@jridgewell/gen-mapping/-/gen-mapping-0.3.13.tgz", + "integrity": "sha512-2kkt/7niJ6MgEPxF0bYdQ6etZaA+fQvDcLKckhy1yIQOzaoKjBBjSj63/aLVjYE3qhRt5dvM+uUyfCg6UKCBbA==", + "dev": true, + "license": "MIT", + "dependencies": { + "@jridgewell/sourcemap-codec": "^1.5.0", + "@jridgewell/trace-mapping": "^0.3.24" + } + }, + "node_modules/@jridgewell/remapping": { + "version": "2.3.5", + "resolved": "https://registry.npmjs.org/@jridgewell/remapping/-/remapping-2.3.5.tgz", + "integrity": "sha512-LI9u/+laYG4Ds1TDKSJW2YPrIlcVYOwi2fUC6xB43lueCjgxV4lffOCZCtYFiH6TNOX+tQKXx97T4IKHbhyHEQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "@jridgewell/gen-mapping": "^0.3.5", + "@jridgewell/trace-mapping": "^0.3.24" + } + }, + "node_modules/@jridgewell/resolve-uri": { + "version": "3.1.2", + "resolved": "https://registry.npmjs.org/@jridgewell/resolve-uri/-/resolve-uri-3.1.2.tgz", + "integrity": "sha512-bRISgCIjP20/tbWSPWMEi54QVPRZExkuD9lJL+UIxUKtwVJA8wW1Trb1jMs1RFXo1CBTNZ/5hpC9QvmKWdopKw==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=6.0.0" + } + }, + "node_modules/@jridgewell/sourcemap-codec": { + "version": "1.5.5", + "resolved": "https://registry.npmjs.org/@jridgewell/sourcemap-codec/-/sourcemap-codec-1.5.5.tgz", + "integrity": "sha512-cYQ9310grqxueWbl+WuIUIaiUaDcj7WOq5fVhEljNVgRfOUhY9fy2zTvfoqWsnebh8Sl70VScFbICvJnLKB0Og==", + "dev": true, + "license": "MIT" + }, + "node_modules/@jridgewell/trace-mapping": { + "version": "0.3.31", + "resolved": "https://registry.npmjs.org/@jridgewell/trace-mapping/-/trace-mapping-0.3.31.tgz", + "integrity": "sha512-zzNR+SdQSDJzc8joaeP8QQoCQr8NuYx2dIIytl1QeBEZHJ9uW6hebsrYgbz8hJwUQao3TWCMtmfV8Nu1twOLAw==", + "dev": true, + "license": "MIT", + "dependencies": { + "@jridgewell/resolve-uri": "^3.1.0", + "@jridgewell/sourcemap-codec": "^1.4.14" + } + }, + "node_modules/@rolldown/pluginutils": { + "version": "1.0.0-beta.53", + "resolved": "https://registry.npmjs.org/@rolldown/pluginutils/-/pluginutils-1.0.0-beta.53.tgz", + "integrity": "sha512-vENRlFU4YbrwVqNDZ7fLvy+JR1CRkyr01jhSiDpE1u6py3OMzQfztQU2jxykW3ALNxO4kSlqIDeYyD0Y9RcQeQ==", + "dev": true, + "license": "MIT" + }, + "node_modules/@rollup/rollup-android-arm-eabi": { + "version": "4.54.0", + "resolved": "https://registry.npmjs.org/@rollup/rollup-android-arm-eabi/-/rollup-android-arm-eabi-4.54.0.tgz", + "integrity": "sha512-OywsdRHrFvCdvsewAInDKCNyR3laPA2mc9bRYJ6LBp5IyvF3fvXbbNR0bSzHlZVFtn6E0xw2oZlyjg4rKCVcng==", + "cpu": [ + "arm" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "android" + ] + }, + "node_modules/@rollup/rollup-android-arm64": { + "version": "4.54.0", + "resolved": "https://registry.npmjs.org/@rollup/rollup-android-arm64/-/rollup-android-arm64-4.54.0.tgz", + "integrity": "sha512-Skx39Uv+u7H224Af+bDgNinitlmHyQX1K/atIA32JP3JQw6hVODX5tkbi2zof/E69M1qH2UoN3Xdxgs90mmNYw==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "android" + ] + }, + "node_modules/@rollup/rollup-darwin-arm64": { + "version": "4.54.0", + "resolved": "https://registry.npmjs.org/@rollup/rollup-darwin-arm64/-/rollup-darwin-arm64-4.54.0.tgz", + "integrity": "sha512-k43D4qta/+6Fq+nCDhhv9yP2HdeKeP56QrUUTW7E6PhZP1US6NDqpJj4MY0jBHlJivVJD5P8NxrjuobZBJTCRw==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ] + }, + "node_modules/@rollup/rollup-darwin-x64": { + "version": "4.54.0", + "resolved": "https://registry.npmjs.org/@rollup/rollup-darwin-x64/-/rollup-darwin-x64-4.54.0.tgz", + "integrity": "sha512-cOo7biqwkpawslEfox5Vs8/qj83M/aZCSSNIWpVzfU2CYHa2G3P1UN5WF01RdTHSgCkri7XOlTdtk17BezlV3A==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ] + }, + "node_modules/@rollup/rollup-freebsd-arm64": { + "version": "4.54.0", + "resolved": "https://registry.npmjs.org/@rollup/rollup-freebsd-arm64/-/rollup-freebsd-arm64-4.54.0.tgz", + "integrity": "sha512-miSvuFkmvFbgJ1BevMa4CPCFt5MPGw094knM64W9I0giUIMMmRYcGW/JWZDriaw/k1kOBtsWh1z6nIFV1vPNtA==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "freebsd" + ] + }, + "node_modules/@rollup/rollup-freebsd-x64": { + "version": "4.54.0", + "resolved": "https://registry.npmjs.org/@rollup/rollup-freebsd-x64/-/rollup-freebsd-x64-4.54.0.tgz", + "integrity": "sha512-KGXIs55+b/ZfZsq9aR026tmr/+7tq6VG6MsnrvF4H8VhwflTIuYh+LFUlIsRdQSgrgmtM3fVATzEAj4hBQlaqQ==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "freebsd" + ] + }, + "node_modules/@rollup/rollup-linux-arm-gnueabihf": { + "version": "4.54.0", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-arm-gnueabihf/-/rollup-linux-arm-gnueabihf-4.54.0.tgz", + "integrity": "sha512-EHMUcDwhtdRGlXZsGSIuXSYwD5kOT9NVnx9sqzYiwAc91wfYOE1g1djOEDseZJKKqtHAHGwnGPQu3kytmfaXLQ==", + "cpu": [ + "arm" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@rollup/rollup-linux-arm-musleabihf": { + "version": "4.54.0", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-arm-musleabihf/-/rollup-linux-arm-musleabihf-4.54.0.tgz", + "integrity": "sha512-+pBrqEjaakN2ySv5RVrj/qLytYhPKEUwk+e3SFU5jTLHIcAtqh2rLrd/OkbNuHJpsBgxsD8ccJt5ga/SeG0JmA==", + "cpu": [ + "arm" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@rollup/rollup-linux-arm64-gnu": { + "version": "4.54.0", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-arm64-gnu/-/rollup-linux-arm64-gnu-4.54.0.tgz", + "integrity": "sha512-NSqc7rE9wuUaRBsBp5ckQ5CVz5aIRKCwsoa6WMF7G01sX3/qHUw/z4pv+D+ahL1EIKy6Enpcnz1RY8pf7bjwng==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@rollup/rollup-linux-arm64-musl": { + "version": "4.54.0", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-arm64-musl/-/rollup-linux-arm64-musl-4.54.0.tgz", + "integrity": "sha512-gr5vDbg3Bakga5kbdpqx81m2n9IX8M6gIMlQQIXiLTNeQW6CucvuInJ91EuCJ/JYvc+rcLLsDFcfAD1K7fMofg==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@rollup/rollup-linux-loong64-gnu": { + "version": "4.54.0", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-loong64-gnu/-/rollup-linux-loong64-gnu-4.54.0.tgz", + "integrity": "sha512-gsrtB1NA3ZYj2vq0Rzkylo9ylCtW/PhpLEivlgWe0bpgtX5+9j9EZa0wtZiCjgu6zmSeZWyI/e2YRX1URozpIw==", + "cpu": [ + "loong64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@rollup/rollup-linux-ppc64-gnu": { + "version": "4.54.0", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-ppc64-gnu/-/rollup-linux-ppc64-gnu-4.54.0.tgz", + "integrity": "sha512-y3qNOfTBStmFNq+t4s7Tmc9hW2ENtPg8FeUD/VShI7rKxNW7O4fFeaYbMsd3tpFlIg1Q8IapFgy7Q9i2BqeBvA==", + "cpu": [ + "ppc64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@rollup/rollup-linux-riscv64-gnu": { + "version": "4.54.0", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-riscv64-gnu/-/rollup-linux-riscv64-gnu-4.54.0.tgz", + "integrity": "sha512-89sepv7h2lIVPsFma8iwmccN7Yjjtgz0Rj/Ou6fEqg3HDhpCa+Et+YSufy27i6b0Wav69Qv4WBNl3Rs6pwhebQ==", + "cpu": [ + "riscv64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@rollup/rollup-linux-riscv64-musl": { + "version": "4.54.0", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-riscv64-musl/-/rollup-linux-riscv64-musl-4.54.0.tgz", + "integrity": "sha512-ZcU77ieh0M2Q8Ur7D5X7KvK+UxbXeDHwiOt/CPSBTI1fBmeDMivW0dPkdqkT4rOgDjrDDBUed9x4EgraIKoR2A==", + "cpu": [ + "riscv64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@rollup/rollup-linux-s390x-gnu": { + "version": "4.54.0", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-s390x-gnu/-/rollup-linux-s390x-gnu-4.54.0.tgz", + "integrity": "sha512-2AdWy5RdDF5+4YfG/YesGDDtbyJlC9LHmL6rZw6FurBJ5n4vFGupsOBGfwMRjBYH7qRQowT8D/U4LoSvVwOhSQ==", + "cpu": [ + "s390x" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@rollup/rollup-linux-x64-gnu": { + "version": "4.54.0", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-x64-gnu/-/rollup-linux-x64-gnu-4.54.0.tgz", + "integrity": "sha512-WGt5J8Ij/rvyqpFexxk3ffKqqbLf9AqrTBbWDk7ApGUzaIs6V+s2s84kAxklFwmMF/vBNGrVdYgbblCOFFezMQ==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@rollup/rollup-linux-x64-musl": { + "version": "4.54.0", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-x64-musl/-/rollup-linux-x64-musl-4.54.0.tgz", + "integrity": "sha512-JzQmb38ATzHjxlPHuTH6tE7ojnMKM2kYNzt44LO/jJi8BpceEC8QuXYA908n8r3CNuG/B3BV8VR3Hi1rYtmPiw==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@rollup/rollup-openharmony-arm64": { + "version": "4.54.0", + "resolved": "https://registry.npmjs.org/@rollup/rollup-openharmony-arm64/-/rollup-openharmony-arm64-4.54.0.tgz", + "integrity": "sha512-huT3fd0iC7jigGh7n3q/+lfPcXxBi+om/Rs3yiFxjvSxbSB6aohDFXbWvlspaqjeOh+hx7DDHS+5Es5qRkWkZg==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "openharmony" + ] + }, + "node_modules/@rollup/rollup-win32-arm64-msvc": { + "version": "4.54.0", + "resolved": "https://registry.npmjs.org/@rollup/rollup-win32-arm64-msvc/-/rollup-win32-arm64-msvc-4.54.0.tgz", + "integrity": "sha512-c2V0W1bsKIKfbLMBu/WGBz6Yci8nJ/ZJdheE0EwB73N3MvHYKiKGs3mVilX4Gs70eGeDaMqEob25Tw2Gb9Nqyw==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "win32" + ] + }, + "node_modules/@rollup/rollup-win32-ia32-msvc": { + "version": "4.54.0", + "resolved": "https://registry.npmjs.org/@rollup/rollup-win32-ia32-msvc/-/rollup-win32-ia32-msvc-4.54.0.tgz", + "integrity": "sha512-woEHgqQqDCkAzrDhvDipnSirm5vxUXtSKDYTVpZG3nUdW/VVB5VdCYA2iReSj/u3yCZzXID4kuKG7OynPnB3WQ==", + "cpu": [ + "ia32" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "win32" + ] + }, + "node_modules/@rollup/rollup-win32-x64-gnu": { + "version": "4.54.0", + "resolved": "https://registry.npmjs.org/@rollup/rollup-win32-x64-gnu/-/rollup-win32-x64-gnu-4.54.0.tgz", + "integrity": "sha512-dzAc53LOuFvHwbCEOS0rPbXp6SIhAf2txMP5p6mGyOXXw5mWY8NGGbPMPrs4P1WItkfApDathBj/NzMLUZ9rtQ==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "win32" + ] + }, + "node_modules/@rollup/rollup-win32-x64-msvc": { + "version": "4.54.0", + "resolved": "https://registry.npmjs.org/@rollup/rollup-win32-x64-msvc/-/rollup-win32-x64-msvc-4.54.0.tgz", + "integrity": "sha512-hYT5d3YNdSh3mbCU1gwQyPgQd3T2ne0A3KG8KSBdav5TiBg6eInVmV+TeR5uHufiIgSFg0XsOWGW5/RhNcSvPg==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "win32" + ] + }, + "node_modules/@types/babel__core": { + "version": "7.20.5", + "resolved": "https://registry.npmjs.org/@types/babel__core/-/babel__core-7.20.5.tgz", + "integrity": "sha512-qoQprZvz5wQFJwMDqeseRXWv3rqMvhgpbXFfVyWhbx9X47POIA6i/+dXefEmZKoAgOaTdaIgNSMqMIU61yRyzA==", + "dev": true, + "license": "MIT", + "dependencies": { + "@babel/parser": "^7.20.7", + "@babel/types": "^7.20.7", + "@types/babel__generator": "*", + "@types/babel__template": "*", + "@types/babel__traverse": "*" + } + }, + "node_modules/@types/babel__generator": { + "version": "7.27.0", + "resolved": "https://registry.npmjs.org/@types/babel__generator/-/babel__generator-7.27.0.tgz", + "integrity": "sha512-ufFd2Xi92OAVPYsy+P4n7/U7e68fex0+Ee8gSG9KX7eo084CWiQ4sdxktvdl0bOPupXtVJPY19zk6EwWqUQ8lg==", + "dev": true, + "license": "MIT", + "dependencies": { + "@babel/types": "^7.0.0" + } + }, + "node_modules/@types/babel__template": { + "version": "7.4.4", + "resolved": "https://registry.npmjs.org/@types/babel__template/-/babel__template-7.4.4.tgz", + "integrity": "sha512-h/NUaSyG5EyxBIp8YRxo4RMe2/qQgvyowRwVMzhYhBCONbW8PUsg4lkFMrhgZhUe5z3L3MiLDuvyJ/CaPa2A8A==", + "dev": true, + "license": "MIT", + "dependencies": { + "@babel/parser": "^7.1.0", + "@babel/types": "^7.0.0" + } + }, + "node_modules/@types/babel__traverse": { + "version": "7.28.0", + "resolved": "https://registry.npmjs.org/@types/babel__traverse/-/babel__traverse-7.28.0.tgz", + "integrity": "sha512-8PvcXf70gTDZBgt9ptxJ8elBeBjcLOAcOtoO/mPJjtji1+CdGbHgm77om1GrsPxsiE+uXIpNSK64UYaIwQXd4Q==", + "dev": true, + "license": "MIT", + "dependencies": { + "@babel/types": "^7.28.2" + } + }, + "node_modules/@types/estree": { + "version": "1.0.8", + "resolved": "https://registry.npmjs.org/@types/estree/-/estree-1.0.8.tgz", + "integrity": "sha512-dWHzHa2WqEXI/O1E9OjrocMTKJl2mSrEolh1Iomrv6U+JuNwaHXsXx9bLu5gG7BUWFIN0skIQJQ/L1rIex4X6w==", + "dev": true, + "license": "MIT" + }, + "node_modules/@types/json-schema": { + "version": "7.0.15", + "resolved": "https://registry.npmjs.org/@types/json-schema/-/json-schema-7.0.15.tgz", + "integrity": "sha512-5+fP8P8MFNC+AyZCDxrB2pkZFPGzqQWUzpSeuuVLvm8VMcorNYavBqoFcxK8bQz4Qsbn4oUEEem4wDLfcysGHA==", + "dev": true, + "license": "MIT" + }, + "node_modules/@types/node": { + "version": "24.10.4", + "resolved": "https://registry.npmjs.org/@types/node/-/node-24.10.4.tgz", + "integrity": "sha512-vnDVpYPMzs4wunl27jHrfmwojOGKya0xyM3sH+UE5iv5uPS6vX7UIoh6m+vQc5LGBq52HBKPIn/zcSZVzeDEZg==", + "dev": true, + "license": "MIT", + "dependencies": { + "undici-types": "~7.16.0" + } + }, + "node_modules/@types/react": { + "version": "19.2.7", + "resolved": "https://registry.npmjs.org/@types/react/-/react-19.2.7.tgz", + "integrity": "sha512-MWtvHrGZLFttgeEj28VXHxpmwYbor/ATPYbBfSFZEIRK0ecCFLl2Qo55z52Hss+UV9CRN7trSeq1zbgx7YDWWg==", + "dev": true, + "license": "MIT", + "dependencies": { + "csstype": "^3.2.2" + } + }, + "node_modules/@types/react-dom": { + "version": "19.2.3", + "resolved": "https://registry.npmjs.org/@types/react-dom/-/react-dom-19.2.3.tgz", + "integrity": "sha512-jp2L/eY6fn+KgVVQAOqYItbF0VY/YApe5Mz2F0aykSO8gx31bYCZyvSeYxCHKvzHG5eZjc+zyaS5BrBWya2+kQ==", + "dev": true, + "license": "MIT", + "peerDependencies": { + "@types/react": "^19.2.0" + } + }, + "node_modules/@typescript-eslint/eslint-plugin": { + "version": "8.51.0", + "resolved": "https://registry.npmjs.org/@typescript-eslint/eslint-plugin/-/eslint-plugin-8.51.0.tgz", + "integrity": "sha512-XtssGWJvypyM2ytBnSnKtHYOGT+4ZwTnBVl36TA4nRO2f4PRNGz5/1OszHzcZCvcBMh+qb7I06uoCmLTRdR9og==", + "dev": true, + "license": "MIT", + "dependencies": { + "@eslint-community/regexpp": "^4.10.0", + "@typescript-eslint/scope-manager": "8.51.0", + "@typescript-eslint/type-utils": "8.51.0", + "@typescript-eslint/utils": "8.51.0", + "@typescript-eslint/visitor-keys": "8.51.0", + "ignore": "^7.0.0", + "natural-compare": "^1.4.0", + "ts-api-utils": "^2.2.0" + }, + "engines": { + "node": "^18.18.0 || ^20.9.0 || >=21.1.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/typescript-eslint" + }, + "peerDependencies": { + "@typescript-eslint/parser": "^8.51.0", + "eslint": "^8.57.0 || ^9.0.0", + "typescript": ">=4.8.4 <6.0.0" + } + }, + "node_modules/@typescript-eslint/eslint-plugin/node_modules/ignore": { + "version": "7.0.5", + "resolved": "https://registry.npmjs.org/ignore/-/ignore-7.0.5.tgz", + "integrity": "sha512-Hs59xBNfUIunMFgWAbGX5cq6893IbWg4KnrjbYwX3tx0ztorVgTDA6B2sxf8ejHJ4wz8BqGUMYlnzNBer5NvGg==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">= 4" + } + }, + "node_modules/@typescript-eslint/parser": { + "version": "8.51.0", + "resolved": "https://registry.npmjs.org/@typescript-eslint/parser/-/parser-8.51.0.tgz", + "integrity": "sha512-3xP4XzzDNQOIqBMWogftkwxhg5oMKApqY0BAflmLZiFYHqyhSOxv/cd/zPQLTcCXr4AkaKb25joocY0BD1WC6A==", + "dev": true, + "license": "MIT", + "dependencies": { + "@typescript-eslint/scope-manager": "8.51.0", + "@typescript-eslint/types": "8.51.0", + "@typescript-eslint/typescript-estree": "8.51.0", + "@typescript-eslint/visitor-keys": "8.51.0", + "debug": "^4.3.4" + }, + "engines": { + "node": "^18.18.0 || ^20.9.0 || >=21.1.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/typescript-eslint" + }, + "peerDependencies": { + "eslint": "^8.57.0 || ^9.0.0", + "typescript": ">=4.8.4 <6.0.0" + } + }, + "node_modules/@typescript-eslint/project-service": { + "version": "8.51.0", + "resolved": "https://registry.npmjs.org/@typescript-eslint/project-service/-/project-service-8.51.0.tgz", + "integrity": "sha512-Luv/GafO07Z7HpiI7qeEW5NW8HUtZI/fo/kE0YbtQEFpJRUuR0ajcWfCE5bnMvL7QQFrmT/odMe8QZww8X2nfQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "@typescript-eslint/tsconfig-utils": "^8.51.0", + "@typescript-eslint/types": "^8.51.0", + "debug": "^4.3.4" + }, + "engines": { + "node": "^18.18.0 || ^20.9.0 || >=21.1.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/typescript-eslint" + }, + "peerDependencies": { + "typescript": ">=4.8.4 <6.0.0" + } + }, + "node_modules/@typescript-eslint/scope-manager": { + "version": "8.51.0", + "resolved": "https://registry.npmjs.org/@typescript-eslint/scope-manager/-/scope-manager-8.51.0.tgz", + "integrity": "sha512-JhhJDVwsSx4hiOEQPeajGhCWgBMBwVkxC/Pet53EpBVs7zHHtayKefw1jtPaNRXpI9RA2uocdmpdfE7T+NrizA==", + "dev": true, + "license": "MIT", + "dependencies": { + "@typescript-eslint/types": "8.51.0", + "@typescript-eslint/visitor-keys": "8.51.0" + }, + "engines": { + "node": "^18.18.0 || ^20.9.0 || >=21.1.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/typescript-eslint" + } + }, + "node_modules/@typescript-eslint/tsconfig-utils": { + "version": "8.51.0", + "resolved": "https://registry.npmjs.org/@typescript-eslint/tsconfig-utils/-/tsconfig-utils-8.51.0.tgz", + "integrity": "sha512-Qi5bSy/vuHeWyir2C8u/uqGMIlIDu8fuiYWv48ZGlZ/k+PRPHtaAu7erpc7p5bzw2WNNSniuxoMSO4Ar6V9OXw==", + "dev": true, + "license": "MIT", + "engines": { + "node": "^18.18.0 || ^20.9.0 || >=21.1.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/typescript-eslint" + }, + "peerDependencies": { + "typescript": ">=4.8.4 <6.0.0" + } + }, + "node_modules/@typescript-eslint/type-utils": { + "version": "8.51.0", + "resolved": "https://registry.npmjs.org/@typescript-eslint/type-utils/-/type-utils-8.51.0.tgz", + "integrity": "sha512-0XVtYzxnobc9K0VU7wRWg1yiUrw4oQzexCG2V2IDxxCxhqBMSMbjB+6o91A+Uc0GWtgjCa3Y8bi7hwI0Tu4n5Q==", + "dev": true, + "license": "MIT", + "dependencies": { + "@typescript-eslint/types": "8.51.0", + "@typescript-eslint/typescript-estree": "8.51.0", + "@typescript-eslint/utils": "8.51.0", + "debug": "^4.3.4", + "ts-api-utils": "^2.2.0" + }, + "engines": { + "node": "^18.18.0 || ^20.9.0 || >=21.1.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/typescript-eslint" + }, + "peerDependencies": { + "eslint": "^8.57.0 || ^9.0.0", + "typescript": ">=4.8.4 <6.0.0" + } + }, + "node_modules/@typescript-eslint/types": { + "version": "8.51.0", + "resolved": "https://registry.npmjs.org/@typescript-eslint/types/-/types-8.51.0.tgz", + "integrity": "sha512-TizAvWYFM6sSscmEakjY3sPqGwxZRSywSsPEiuZF6d5GmGD9Gvlsv0f6N8FvAAA0CD06l3rIcWNbsN1e5F/9Ag==", + "dev": true, + "license": "MIT", + "engines": { + "node": "^18.18.0 || ^20.9.0 || >=21.1.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/typescript-eslint" + } + }, + "node_modules/@typescript-eslint/typescript-estree": { + "version": "8.51.0", + "resolved": "https://registry.npmjs.org/@typescript-eslint/typescript-estree/-/typescript-estree-8.51.0.tgz", + "integrity": "sha512-1qNjGqFRmlq0VW5iVlcyHBbCjPB7y6SxpBkrbhNWMy/65ZoncXCEPJxkRZL8McrseNH6lFhaxCIaX+vBuFnRng==", + "dev": true, + "license": "MIT", + "dependencies": { + "@typescript-eslint/project-service": "8.51.0", + "@typescript-eslint/tsconfig-utils": "8.51.0", + "@typescript-eslint/types": "8.51.0", + "@typescript-eslint/visitor-keys": "8.51.0", + "debug": "^4.3.4", + "minimatch": "^9.0.4", + "semver": "^7.6.0", + "tinyglobby": "^0.2.15", + "ts-api-utils": "^2.2.0" + }, + "engines": { + "node": "^18.18.0 || ^20.9.0 || >=21.1.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/typescript-eslint" + }, + "peerDependencies": { + "typescript": ">=4.8.4 <6.0.0" + } + }, + "node_modules/@typescript-eslint/typescript-estree/node_modules/brace-expansion": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-2.0.2.tgz", + "integrity": "sha512-Jt0vHyM+jmUBqojB7E1NIYadt0vI0Qxjxd2TErW94wDz+E2LAm5vKMXXwg6ZZBTHPuUlDgQHKXvjGBdfcF1ZDQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "balanced-match": "^1.0.0" + } + }, + "node_modules/@typescript-eslint/typescript-estree/node_modules/minimatch": { + "version": "9.0.5", + "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-9.0.5.tgz", + "integrity": "sha512-G6T0ZX48xgozx7587koeX9Ys2NYy6Gmv//P89sEte9V9whIapMNF4idKxnW2QtCcLiTWlb/wfCabAtAFWhhBow==", + "dev": true, + "license": "ISC", + "dependencies": { + "brace-expansion": "^2.0.1" + }, + "engines": { + "node": ">=16 || 14 >=14.17" + }, + "funding": { + "url": "https://github.com/sponsors/isaacs" + } + }, + "node_modules/@typescript-eslint/typescript-estree/node_modules/semver": { + "version": "7.7.3", + "resolved": "https://registry.npmjs.org/semver/-/semver-7.7.3.tgz", + "integrity": "sha512-SdsKMrI9TdgjdweUSR9MweHA4EJ8YxHn8DFaDisvhVlUOe4BF1tLD7GAj0lIqWVl+dPb/rExr0Btby5loQm20Q==", + "dev": true, + "license": "ISC", + "bin": { + "semver": "bin/semver.js" + }, + "engines": { + "node": ">=10" + } + }, + "node_modules/@typescript-eslint/utils": { + "version": "8.51.0", + "resolved": "https://registry.npmjs.org/@typescript-eslint/utils/-/utils-8.51.0.tgz", + "integrity": "sha512-11rZYxSe0zabiKaCP2QAwRf/dnmgFgvTmeDTtZvUvXG3UuAdg/GU02NExmmIXzz3vLGgMdtrIosI84jITQOxUA==", + "dev": true, + "license": "MIT", + "dependencies": { + "@eslint-community/eslint-utils": "^4.7.0", + "@typescript-eslint/scope-manager": "8.51.0", + "@typescript-eslint/types": "8.51.0", + "@typescript-eslint/typescript-estree": "8.51.0" + }, + "engines": { + "node": "^18.18.0 || ^20.9.0 || >=21.1.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/typescript-eslint" + }, + "peerDependencies": { + "eslint": "^8.57.0 || ^9.0.0", + "typescript": ">=4.8.4 <6.0.0" + } + }, + "node_modules/@typescript-eslint/visitor-keys": { + "version": "8.51.0", + "resolved": "https://registry.npmjs.org/@typescript-eslint/visitor-keys/-/visitor-keys-8.51.0.tgz", + "integrity": "sha512-mM/JRQOzhVN1ykejrvwnBRV3+7yTKK8tVANVN3o1O0t0v7o+jqdVu9crPy5Y9dov15TJk/FTIgoUGHrTOVL3Zg==", + "dev": true, + "license": "MIT", + "dependencies": { + "@typescript-eslint/types": "8.51.0", + "eslint-visitor-keys": "^4.2.1" + }, + "engines": { + "node": "^18.18.0 || ^20.9.0 || >=21.1.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/typescript-eslint" + } + }, + "node_modules/@vitejs/plugin-react": { + "version": "5.1.2", + "resolved": "https://registry.npmjs.org/@vitejs/plugin-react/-/plugin-react-5.1.2.tgz", + "integrity": "sha512-EcA07pHJouywpzsoTUqNh5NwGayl2PPVEJKUSinGGSxFGYn+shYbqMGBg6FXDqgXum9Ou/ecb+411ssw8HImJQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "@babel/core": "^7.28.5", + "@babel/plugin-transform-react-jsx-self": "^7.27.1", + "@babel/plugin-transform-react-jsx-source": "^7.27.1", + "@rolldown/pluginutils": "1.0.0-beta.53", + "@types/babel__core": "^7.20.5", + "react-refresh": "^0.18.0" + }, + "engines": { + "node": "^20.19.0 || >=22.12.0" + }, + "peerDependencies": { + "vite": "^4.2.0 || ^5.0.0 || ^6.0.0 || ^7.0.0" + } + }, + "node_modules/acorn": { + "version": "8.15.0", + "resolved": "https://registry.npmjs.org/acorn/-/acorn-8.15.0.tgz", + "integrity": "sha512-NZyJarBfL7nWwIq+FDL6Zp/yHEhePMNnnJ0y3qfieCrmNvYct8uvtiV41UvlSe6apAfk0fY1FbWx+NwfmpvtTg==", + "dev": true, + "license": "MIT", + "bin": { + "acorn": "bin/acorn" + }, + "engines": { + "node": ">=0.4.0" + } + }, + "node_modules/acorn-jsx": { + "version": "5.3.2", + "resolved": "https://registry.npmjs.org/acorn-jsx/-/acorn-jsx-5.3.2.tgz", + "integrity": "sha512-rq9s+JNhf0IChjtDXxllJ7g41oZk5SlXtp0LHwyA5cejwn7vKmKp4pPri6YEePv2PU65sAsegbXtIinmDFDXgQ==", + "dev": true, + "license": "MIT", + "peerDependencies": { + "acorn": "^6.0.0 || ^7.0.0 || ^8.0.0" + } + }, + "node_modules/ajv": { + "version": "6.12.6", + "resolved": "https://registry.npmjs.org/ajv/-/ajv-6.12.6.tgz", + "integrity": "sha512-j3fVLgvTo527anyYyJOGTYJbG+vnnQYvE0m5mmkc1TK+nxAppkCLMIL0aZ4dblVCNoGShhm+kzE4ZUykBoMg4g==", + "dev": true, + "license": "MIT", + "dependencies": { + "fast-deep-equal": "^3.1.1", + "fast-json-stable-stringify": "^2.0.0", + "json-schema-traverse": "^0.4.1", + "uri-js": "^4.2.2" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/epoberezkin" + } + }, + "node_modules/ansi-styles": { + "version": "4.3.0", + "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-4.3.0.tgz", + "integrity": "sha512-zbB9rCJAT1rbjiVDb2hqKFHNYLxgtk8NURxZ3IZwD3F6NtxbXZQCnnSi1Lkx+IDohdPlFp222wVALIheZJQSEg==", + "dev": true, + "license": "MIT", + "dependencies": { + "color-convert": "^2.0.1" + }, + "engines": { + "node": ">=8" + }, + "funding": { + "url": "https://github.com/chalk/ansi-styles?sponsor=1" + } + }, + "node_modules/argparse": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/argparse/-/argparse-2.0.1.tgz", + "integrity": "sha512-8+9WqebbFzpX9OR+Wa6O29asIogeRMzcGtAINdpMHHyAg10f05aSFVBbcEqGf/PXw1EjAZ+q2/bEBg3DvurK3Q==", + "dev": true, + "license": "Python-2.0" + }, + "node_modules/balanced-match": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/balanced-match/-/balanced-match-1.0.2.tgz", + "integrity": "sha512-3oSeUO0TMV67hN1AmbXsK4yaqU7tjiHlbxRDZOpH0KW9+CeX4bRAaX0Anxt0tx2MrpRpWwQaPwIlISEJhYU5Pw==", + "dev": true, + "license": "MIT" + }, + "node_modules/baseline-browser-mapping": { + "version": "2.9.11", + "resolved": "https://registry.npmjs.org/baseline-browser-mapping/-/baseline-browser-mapping-2.9.11.tgz", + "integrity": "sha512-Sg0xJUNDU1sJNGdfGWhVHX0kkZ+HWcvmVymJbj6NSgZZmW/8S9Y2HQ5euytnIgakgxN6papOAWiwDo1ctFDcoQ==", + "dev": true, + "license": "Apache-2.0", + "bin": { + "baseline-browser-mapping": "dist/cli.js" + } + }, + "node_modules/brace-expansion": { + "version": "1.1.12", + "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.12.tgz", + "integrity": "sha512-9T9UjW3r0UW5c1Q7GTwllptXwhvYmEzFhzMfZ9H7FQWt+uZePjZPjBP/W1ZEyZ1twGWom5/56TF4lPcqjnDHcg==", + "dev": true, + "license": "MIT", + "dependencies": { + "balanced-match": "^1.0.0", + "concat-map": "0.0.1" + } + }, + "node_modules/browserslist": { + "version": "4.28.1", + "resolved": "https://registry.npmjs.org/browserslist/-/browserslist-4.28.1.tgz", + "integrity": "sha512-ZC5Bd0LgJXgwGqUknZY/vkUQ04r8NXnJZ3yYi4vDmSiZmC/pdSN0NbNRPxZpbtO4uAfDUAFffO8IZoM3Gj8IkA==", + "dev": true, + "funding": [ + { + "type": "opencollective", + "url": "https://opencollective.com/browserslist" + }, + { + "type": "tidelift", + "url": "https://tidelift.com/funding/github/npm/browserslist" + }, + { + "type": "github", + "url": "https://github.com/sponsors/ai" + } + ], + "license": "MIT", + "dependencies": { + "baseline-browser-mapping": "^2.9.0", + "caniuse-lite": "^1.0.30001759", + "electron-to-chromium": "^1.5.263", + "node-releases": "^2.0.27", + "update-browserslist-db": "^1.2.0" + }, + "bin": { + "browserslist": "cli.js" + }, + "engines": { + "node": "^6 || ^7 || ^8 || ^9 || ^10 || ^11 || ^12 || >=13.7" + } + }, + "node_modules/callsites": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/callsites/-/callsites-3.1.0.tgz", + "integrity": "sha512-P8BjAsXvZS+VIDUI11hHCQEv74YT67YUi5JJFNWIqL235sBmjX4+qx9Muvls5ivyNENctx46xQLQ3aTuE7ssaQ==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=6" + } + }, + "node_modules/caniuse-lite": { + "version": "1.0.30001762", + "resolved": "https://registry.npmjs.org/caniuse-lite/-/caniuse-lite-1.0.30001762.tgz", + "integrity": "sha512-PxZwGNvH7Ak8WX5iXzoK1KPZttBXNPuaOvI2ZYU7NrlM+d9Ov+TUvlLOBNGzVXAntMSMMlJPd+jY6ovrVjSmUw==", + "dev": true, + "funding": [ + { + "type": "opencollective", + "url": "https://opencollective.com/browserslist" + }, + { + "type": "tidelift", + "url": "https://tidelift.com/funding/github/npm/caniuse-lite" + }, + { + "type": "github", + "url": "https://github.com/sponsors/ai" + } + ], + "license": "CC-BY-4.0" + }, + "node_modules/chalk": { + "version": "4.1.2", + "resolved": "https://registry.npmjs.org/chalk/-/chalk-4.1.2.tgz", + "integrity": "sha512-oKnbhFyRIXpUuez8iBMmyEa4nbj4IOQyuhc/wy9kY7/WVPcwIO9VA668Pu8RkO7+0G76SLROeyw9CpQ061i4mA==", + "dev": true, + "license": "MIT", + "dependencies": { + "ansi-styles": "^4.1.0", + "supports-color": "^7.1.0" + }, + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/chalk/chalk?sponsor=1" + } + }, + "node_modules/color-convert": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-2.0.1.tgz", + "integrity": "sha512-RRECPsj7iu/xb5oKYcsFHSppFNnsj/52OVTRKb4zP5onXwVF3zVmmToNcOfGC+CRDpfK/U584fMg38ZHCaElKQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "color-name": "~1.1.4" + }, + "engines": { + "node": ">=7.0.0" + } + }, + "node_modules/color-name": { + "version": "1.1.4", + "resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.4.tgz", + "integrity": "sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA==", + "dev": true, + "license": "MIT" + }, + "node_modules/concat-map": { + "version": "0.0.1", + "resolved": "https://registry.npmjs.org/concat-map/-/concat-map-0.0.1.tgz", + "integrity": "sha512-/Srv4dswyQNBfohGpz9o6Yb3Gz3SrUDqBH5rTuhGR7ahtlbYKnVxw2bCFMRljaA7EXHaXZ8wsHdodFvbkhKmqg==", + "dev": true, + "license": "MIT" + }, + "node_modules/convert-source-map": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/convert-source-map/-/convert-source-map-2.0.0.tgz", + "integrity": "sha512-Kvp459HrV2FEJ1CAsi1Ku+MY3kasH19TFykTz2xWmMeq6bk2NU3XXvfJ+Q61m0xktWwt+1HSYf3JZsTms3aRJg==", + "dev": true, + "license": "MIT" + }, + "node_modules/cross-spawn": { + "version": "7.0.6", + "resolved": "https://registry.npmjs.org/cross-spawn/-/cross-spawn-7.0.6.tgz", + "integrity": "sha512-uV2QOWP2nWzsy2aMp8aRibhi9dlzF5Hgh5SHaB9OiTGEyDTiJJyx0uy51QXdyWbtAHNua4XJzUKca3OzKUd3vA==", + "dev": true, + "license": "MIT", + "dependencies": { + "path-key": "^3.1.0", + "shebang-command": "^2.0.0", + "which": "^2.0.1" + }, + "engines": { + "node": ">= 8" + } + }, + "node_modules/csstype": { + "version": "3.2.3", + "resolved": "https://registry.npmjs.org/csstype/-/csstype-3.2.3.tgz", + "integrity": "sha512-z1HGKcYy2xA8AGQfwrn0PAy+PB7X/GSj3UVJW9qKyn43xWa+gl5nXmU4qqLMRzWVLFC8KusUX8T/0kCiOYpAIQ==", + "dev": true, + "license": "MIT" + }, + "node_modules/debug": { + "version": "4.4.3", + "resolved": "https://registry.npmjs.org/debug/-/debug-4.4.3.tgz", + "integrity": "sha512-RGwwWnwQvkVfavKVt22FGLw+xYSdzARwm0ru6DhTVA3umU5hZc28V3kO4stgYryrTlLpuvgI9GiijltAjNbcqA==", + "dev": true, + "license": "MIT", + "dependencies": { + "ms": "^2.1.3" + }, + "engines": { + "node": ">=6.0" + }, + "peerDependenciesMeta": { + "supports-color": { + "optional": true + } + } + }, + "node_modules/deep-is": { + "version": "0.1.4", + "resolved": "https://registry.npmjs.org/deep-is/-/deep-is-0.1.4.tgz", + "integrity": "sha512-oIPzksmTg4/MriiaYGO+okXDT7ztn/w3Eptv/+gSIdMdKsJo0u4CfYNFJPy+4SKMuCqGw2wxnA+URMg3t8a/bQ==", + "dev": true, + "license": "MIT" + }, + "node_modules/electron-to-chromium": { + "version": "1.5.267", + "resolved": "https://registry.npmjs.org/electron-to-chromium/-/electron-to-chromium-1.5.267.tgz", + "integrity": "sha512-0Drusm6MVRXSOJpGbaSVgcQsuB4hEkMpHXaVstcPmhu5LIedxs1xNK/nIxmQIU/RPC0+1/o0AVZfBTkTNJOdUw==", + "dev": true, + "license": "ISC" + }, + "node_modules/esbuild": { + "version": "0.27.2", + "resolved": "https://registry.npmjs.org/esbuild/-/esbuild-0.27.2.tgz", + "integrity": "sha512-HyNQImnsOC7X9PMNaCIeAm4ISCQXs5a5YasTXVliKv4uuBo1dKrG0A+uQS8M5eXjVMnLg3WgXaKvprHlFJQffw==", + "dev": true, + "hasInstallScript": true, + "license": "MIT", + "bin": { + "esbuild": "bin/esbuild" + }, + "engines": { + "node": ">=18" + }, + "optionalDependencies": { + "@esbuild/aix-ppc64": "0.27.2", + "@esbuild/android-arm": "0.27.2", + "@esbuild/android-arm64": "0.27.2", + "@esbuild/android-x64": "0.27.2", + "@esbuild/darwin-arm64": "0.27.2", + "@esbuild/darwin-x64": "0.27.2", + "@esbuild/freebsd-arm64": "0.27.2", + "@esbuild/freebsd-x64": "0.27.2", + "@esbuild/linux-arm": "0.27.2", + "@esbuild/linux-arm64": "0.27.2", + "@esbuild/linux-ia32": "0.27.2", + "@esbuild/linux-loong64": "0.27.2", + "@esbuild/linux-mips64el": "0.27.2", + "@esbuild/linux-ppc64": "0.27.2", + "@esbuild/linux-riscv64": "0.27.2", + "@esbuild/linux-s390x": "0.27.2", + "@esbuild/linux-x64": "0.27.2", + "@esbuild/netbsd-arm64": "0.27.2", + "@esbuild/netbsd-x64": "0.27.2", + "@esbuild/openbsd-arm64": "0.27.2", + "@esbuild/openbsd-x64": "0.27.2", + "@esbuild/openharmony-arm64": "0.27.2", + "@esbuild/sunos-x64": "0.27.2", + "@esbuild/win32-arm64": "0.27.2", + "@esbuild/win32-ia32": "0.27.2", + "@esbuild/win32-x64": "0.27.2" + } + }, + "node_modules/escalade": { + "version": "3.2.0", + "resolved": "https://registry.npmjs.org/escalade/-/escalade-3.2.0.tgz", + "integrity": "sha512-WUj2qlxaQtO4g6Pq5c29GTcWGDyd8itL8zTlipgECz3JesAiiOKotd8JU6otB3PACgG6xkJUyVhboMS+bje/jA==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=6" + } + }, + "node_modules/escape-string-regexp": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/escape-string-regexp/-/escape-string-regexp-4.0.0.tgz", + "integrity": "sha512-TtpcNJ3XAzx3Gq8sWRzJaVajRs0uVxA2YAkdb1jm2YkPz4G6egUFAyA3n5vtEIZefPk5Wa4UXbKuS5fKkJWdgA==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/eslint": { + "version": "9.39.2", + "resolved": "https://registry.npmjs.org/eslint/-/eslint-9.39.2.tgz", + "integrity": "sha512-LEyamqS7W5HB3ujJyvi0HQK/dtVINZvd5mAAp9eT5S/ujByGjiZLCzPcHVzuXbpJDJF/cxwHlfceVUDZ2lnSTw==", + "dev": true, + "license": "MIT", + "dependencies": { + "@eslint-community/eslint-utils": "^4.8.0", + "@eslint-community/regexpp": "^4.12.1", + "@eslint/config-array": "^0.21.1", + "@eslint/config-helpers": "^0.4.2", + "@eslint/core": "^0.17.0", + "@eslint/eslintrc": "^3.3.1", + "@eslint/js": "9.39.2", + "@eslint/plugin-kit": "^0.4.1", + "@humanfs/node": "^0.16.6", + "@humanwhocodes/module-importer": "^1.0.1", + "@humanwhocodes/retry": "^0.4.2", + "@types/estree": "^1.0.6", + "ajv": "^6.12.4", + "chalk": "^4.0.0", + "cross-spawn": "^7.0.6", + "debug": "^4.3.2", + "escape-string-regexp": "^4.0.0", + "eslint-scope": "^8.4.0", + "eslint-visitor-keys": "^4.2.1", + "espree": "^10.4.0", + "esquery": "^1.5.0", + "esutils": "^2.0.2", + "fast-deep-equal": "^3.1.3", + "file-entry-cache": "^8.0.0", + "find-up": "^5.0.0", + "glob-parent": "^6.0.2", + "ignore": "^5.2.0", + "imurmurhash": "^0.1.4", + "is-glob": "^4.0.0", + "json-stable-stringify-without-jsonify": "^1.0.1", + "lodash.merge": "^4.6.2", + "minimatch": "^3.1.2", + "natural-compare": "^1.4.0", + "optionator": "^0.9.3" + }, + "bin": { + "eslint": "bin/eslint.js" + }, + "engines": { + "node": "^18.18.0 || ^20.9.0 || >=21.1.0" + }, + "funding": { + "url": "https://eslint.org/donate" + }, + "peerDependencies": { + "jiti": "*" + }, + "peerDependenciesMeta": { + "jiti": { + "optional": true + } + } + }, + "node_modules/eslint-plugin-react-hooks": { + "version": "7.0.1", + "resolved": "https://registry.npmjs.org/eslint-plugin-react-hooks/-/eslint-plugin-react-hooks-7.0.1.tgz", + "integrity": "sha512-O0d0m04evaNzEPoSW+59Mezf8Qt0InfgGIBJnpC0h3NH/WjUAR7BIKUfysC6todmtiZ/A0oUVS8Gce0WhBrHsA==", + "dev": true, + "license": "MIT", + "dependencies": { + "@babel/core": "^7.24.4", + "@babel/parser": "^7.24.4", + "hermes-parser": "^0.25.1", + "zod": "^3.25.0 || ^4.0.0", + "zod-validation-error": "^3.5.0 || ^4.0.0" + }, + "engines": { + "node": ">=18" + }, + "peerDependencies": { + "eslint": "^3.0.0 || ^4.0.0 || ^5.0.0 || ^6.0.0 || ^7.0.0 || ^8.0.0-0 || ^9.0.0" + } + }, + "node_modules/eslint-plugin-react-refresh": { + "version": "0.4.26", + "resolved": "https://registry.npmjs.org/eslint-plugin-react-refresh/-/eslint-plugin-react-refresh-0.4.26.tgz", + "integrity": "sha512-1RETEylht2O6FM/MvgnyvT+8K21wLqDNg4qD51Zj3guhjt433XbnnkVttHMyaVyAFD03QSV4LPS5iE3VQmO7XQ==", + "dev": true, + "license": "MIT", + "peerDependencies": { + "eslint": ">=8.40" + } + }, + "node_modules/eslint-scope": { + "version": "8.4.0", + "resolved": "https://registry.npmjs.org/eslint-scope/-/eslint-scope-8.4.0.tgz", + "integrity": "sha512-sNXOfKCn74rt8RICKMvJS7XKV/Xk9kA7DyJr8mJik3S7Cwgy3qlkkmyS2uQB3jiJg6VNdZd/pDBJu0nvG2NlTg==", + "dev": true, + "license": "BSD-2-Clause", + "dependencies": { + "esrecurse": "^4.3.0", + "estraverse": "^5.2.0" + }, + "engines": { + "node": "^18.18.0 || ^20.9.0 || >=21.1.0" + }, + "funding": { + "url": "https://opencollective.com/eslint" + } + }, + "node_modules/eslint-visitor-keys": { + "version": "4.2.1", + "resolved": "https://registry.npmjs.org/eslint-visitor-keys/-/eslint-visitor-keys-4.2.1.tgz", + "integrity": "sha512-Uhdk5sfqcee/9H/rCOJikYz67o0a2Tw2hGRPOG2Y1R2dg7brRe1uG0yaNQDHu+TO/uQPF/5eCapvYSmHUjt7JQ==", + "dev": true, + "license": "Apache-2.0", + "engines": { + "node": "^18.18.0 || ^20.9.0 || >=21.1.0" + }, + "funding": { + "url": "https://opencollective.com/eslint" + } + }, + "node_modules/espree": { + "version": "10.4.0", + "resolved": "https://registry.npmjs.org/espree/-/espree-10.4.0.tgz", + "integrity": "sha512-j6PAQ2uUr79PZhBjP5C5fhl8e39FmRnOjsD5lGnWrFU8i2G776tBK7+nP8KuQUTTyAZUwfQqXAgrVH5MbH9CYQ==", + "dev": true, + "license": "BSD-2-Clause", + "dependencies": { + "acorn": "^8.15.0", + "acorn-jsx": "^5.3.2", + "eslint-visitor-keys": "^4.2.1" + }, + "engines": { + "node": "^18.18.0 || ^20.9.0 || >=21.1.0" + }, + "funding": { + "url": "https://opencollective.com/eslint" + } + }, + "node_modules/esquery": { + "version": "1.6.0", + "resolved": "https://registry.npmjs.org/esquery/-/esquery-1.6.0.tgz", + "integrity": "sha512-ca9pw9fomFcKPvFLXhBKUK90ZvGibiGOvRJNbjljY7s7uq/5YO4BOzcYtJqExdx99rF6aAcnRxHmcUHcz6sQsg==", + "dev": true, + "license": "BSD-3-Clause", + "dependencies": { + "estraverse": "^5.1.0" + }, + "engines": { + "node": ">=0.10" + } + }, + "node_modules/esrecurse": { + "version": "4.3.0", + "resolved": "https://registry.npmjs.org/esrecurse/-/esrecurse-4.3.0.tgz", + "integrity": "sha512-KmfKL3b6G+RXvP8N1vr3Tq1kL/oCFgn2NYXEtqP8/L3pKapUA4G8cFVaoF3SU323CD4XypR/ffioHmkti6/Tag==", + "dev": true, + "license": "BSD-2-Clause", + "dependencies": { + "estraverse": "^5.2.0" + }, + "engines": { + "node": ">=4.0" + } + }, + "node_modules/estraverse": { + "version": "5.3.0", + "resolved": "https://registry.npmjs.org/estraverse/-/estraverse-5.3.0.tgz", + "integrity": "sha512-MMdARuVEQziNTeJD8DgMqmhwR11BRQ/cBP+pLtYdSTnf3MIO8fFeiINEbX36ZdNlfU/7A9f3gUw49B3oQsvwBA==", + "dev": true, + "license": "BSD-2-Clause", + "engines": { + "node": ">=4.0" + } + }, + "node_modules/esutils": { + "version": "2.0.3", + "resolved": "https://registry.npmjs.org/esutils/-/esutils-2.0.3.tgz", + "integrity": "sha512-kVscqXk4OCp68SZ0dkgEKVi6/8ij300KBWTJq32P/dYeWTSwK41WyTxalN1eRmA5Z9UU/LX9D7FWSmV9SAYx6g==", + "dev": true, + "license": "BSD-2-Clause", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/fast-deep-equal": { + "version": "3.1.3", + "resolved": "https://registry.npmjs.org/fast-deep-equal/-/fast-deep-equal-3.1.3.tgz", + "integrity": "sha512-f3qQ9oQy9j2AhBe/H9VC91wLmKBCCU/gDOnKNAYG5hswO7BLKj09Hc5HYNz9cGI++xlpDCIgDaitVs03ATR84Q==", + "dev": true, + "license": "MIT" + }, + "node_modules/fast-json-stable-stringify": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/fast-json-stable-stringify/-/fast-json-stable-stringify-2.1.0.tgz", + "integrity": "sha512-lhd/wF+Lk98HZoTCtlVraHtfh5XYijIjalXck7saUtuanSDyLMxnHhSXEDJqHxD7msR8D0uCmqlkwjCV8xvwHw==", + "dev": true, + "license": "MIT" + }, + "node_modules/fast-levenshtein": { + "version": "2.0.6", + "resolved": "https://registry.npmjs.org/fast-levenshtein/-/fast-levenshtein-2.0.6.tgz", + "integrity": "sha512-DCXu6Ifhqcks7TZKY3Hxp3y6qphY5SJZmrWMDrKcERSOXWQdMhU9Ig/PYrzyw/ul9jOIyh0N4M0tbC5hodg8dw==", + "dev": true, + "license": "MIT" + }, + "node_modules/fdir": { + "version": "6.5.0", + "resolved": "https://registry.npmjs.org/fdir/-/fdir-6.5.0.tgz", + "integrity": "sha512-tIbYtZbucOs0BRGqPJkshJUYdL+SDH7dVM8gjy+ERp3WAUjLEFJE+02kanyHtwjWOnwrKYBiwAmM0p4kLJAnXg==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=12.0.0" + }, + "peerDependencies": { + "picomatch": "^3 || ^4" + }, + "peerDependenciesMeta": { + "picomatch": { + "optional": true + } + } + }, + "node_modules/file-entry-cache": { + "version": "8.0.0", + "resolved": "https://registry.npmjs.org/file-entry-cache/-/file-entry-cache-8.0.0.tgz", + "integrity": "sha512-XXTUwCvisa5oacNGRP9SfNtYBNAMi+RPwBFmblZEF7N7swHYQS6/Zfk7SRwx4D5j3CH211YNRco1DEMNVfZCnQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "flat-cache": "^4.0.0" + }, + "engines": { + "node": ">=16.0.0" + } + }, + "node_modules/find-up": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/find-up/-/find-up-5.0.0.tgz", + "integrity": "sha512-78/PXT1wlLLDgTzDs7sjq9hzz0vXD+zn+7wypEe4fXQxCmdmqfGsEPQxmiCSQI3ajFV91bVSsvNtrJRiW6nGng==", + "dev": true, + "license": "MIT", + "dependencies": { + "locate-path": "^6.0.0", + "path-exists": "^4.0.0" + }, + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/flat-cache": { + "version": "4.0.1", + "resolved": "https://registry.npmjs.org/flat-cache/-/flat-cache-4.0.1.tgz", + "integrity": "sha512-f7ccFPK3SXFHpx15UIGyRJ/FJQctuKZ0zVuN3frBo4HnK3cay9VEW0R6yPYFHC0AgqhukPzKjq22t5DmAyqGyw==", + "dev": true, + "license": "MIT", + "dependencies": { + "flatted": "^3.2.9", + "keyv": "^4.5.4" + }, + "engines": { + "node": ">=16" + } + }, + "node_modules/flatted": { + "version": "3.3.3", + "resolved": "https://registry.npmjs.org/flatted/-/flatted-3.3.3.tgz", + "integrity": "sha512-GX+ysw4PBCz0PzosHDepZGANEuFCMLrnRTiEy9McGjmkCQYwRq4A/X786G/fjM/+OjsWSU1ZrY5qyARZmO/uwg==", + "dev": true, + "license": "ISC" + }, + "node_modules/fsevents": { + "version": "2.3.3", + "resolved": "https://registry.npmjs.org/fsevents/-/fsevents-2.3.3.tgz", + "integrity": "sha512-5xoDfX+fL7faATnagmWPpbFtwh/R77WmMMqqHGS65C3vvB0YHrgF+B1YmZ3441tMj5n63k0212XNoJwzlhffQw==", + "dev": true, + "hasInstallScript": true, + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": "^8.16.0 || ^10.6.0 || >=11.0.0" + } + }, + "node_modules/gensync": { + "version": "1.0.0-beta.2", + "resolved": "https://registry.npmjs.org/gensync/-/gensync-1.0.0-beta.2.tgz", + "integrity": "sha512-3hN7NaskYvMDLQY55gnW3NQ+mesEAepTqlg+VEbj7zzqEMBVNhzcGYYeqFo/TlYz6eQiFcp1HcsCZO+nGgS8zg==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/glob-parent": { + "version": "6.0.2", + "resolved": "https://registry.npmjs.org/glob-parent/-/glob-parent-6.0.2.tgz", + "integrity": "sha512-XxwI8EOhVQgWp6iDL+3b0r86f4d6AX6zSU55HfB4ydCEuXLXc5FcYeOu+nnGftS4TEju/11rt4KJPTMgbfmv4A==", + "dev": true, + "license": "ISC", + "dependencies": { + "is-glob": "^4.0.3" + }, + "engines": { + "node": ">=10.13.0" + } + }, + "node_modules/globals": { + "version": "16.5.0", + "resolved": "https://registry.npmjs.org/globals/-/globals-16.5.0.tgz", + "integrity": "sha512-c/c15i26VrJ4IRt5Z89DnIzCGDn9EcebibhAOjw5ibqEHsE1wLUgkPn9RDmNcUKyU87GeaL633nyJ+pplFR2ZQ==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=18" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/has-flag": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/has-flag/-/has-flag-4.0.0.tgz", + "integrity": "sha512-EykJT/Q1KjTWctppgIAgfSO0tKVuZUjhgMr17kqTumMl6Afv3EISleU7qZUzoXDFTAHTDC4NOoG/ZxU3EvlMPQ==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=8" + } + }, + "node_modules/hermes-estree": { + "version": "0.25.1", + "resolved": "https://registry.npmjs.org/hermes-estree/-/hermes-estree-0.25.1.tgz", + "integrity": "sha512-0wUoCcLp+5Ev5pDW2OriHC2MJCbwLwuRx+gAqMTOkGKJJiBCLjtrvy4PWUGn6MIVefecRpzoOZ/UV6iGdOr+Cw==", + "dev": true, + "license": "MIT" + }, + "node_modules/hermes-parser": { + "version": "0.25.1", + "resolved": "https://registry.npmjs.org/hermes-parser/-/hermes-parser-0.25.1.tgz", + "integrity": "sha512-6pEjquH3rqaI6cYAXYPcz9MS4rY6R4ngRgrgfDshRptUZIc3lw0MCIJIGDj9++mfySOuPTHB4nrSW99BCvOPIA==", + "dev": true, + "license": "MIT", + "dependencies": { + "hermes-estree": "0.25.1" + } + }, + "node_modules/ignore": { + "version": "5.3.2", + "resolved": "https://registry.npmjs.org/ignore/-/ignore-5.3.2.tgz", + "integrity": "sha512-hsBTNUqQTDwkWtcdYI2i06Y/nUBEsNEDJKjWdigLvegy8kDuJAS8uRlpkkcQpyEXL0Z/pjDy5HBmMjRCJ2gq+g==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">= 4" + } + }, + "node_modules/import-fresh": { + "version": "3.3.1", + "resolved": "https://registry.npmjs.org/import-fresh/-/import-fresh-3.3.1.tgz", + "integrity": "sha512-TR3KfrTZTYLPB6jUjfx6MF9WcWrHL9su5TObK4ZkYgBdWKPOFoSoQIdEuTuR82pmtxH2spWG9h6etwfr1pLBqQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "parent-module": "^1.0.0", + "resolve-from": "^4.0.0" + }, + "engines": { + "node": ">=6" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/imurmurhash": { + "version": "0.1.4", + "resolved": "https://registry.npmjs.org/imurmurhash/-/imurmurhash-0.1.4.tgz", + "integrity": "sha512-JmXMZ6wuvDmLiHEml9ykzqO6lwFbof0GG4IkcGaENdCRDDmMVnny7s5HsIgHCbaq0w2MyPhDqkhTUgS2LU2PHA==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=0.8.19" + } + }, + "node_modules/is-extglob": { + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/is-extglob/-/is-extglob-2.1.1.tgz", + "integrity": "sha512-SbKbANkN603Vi4jEZv49LeVJMn4yGwsbzZworEoyEiutsN3nJYdbO36zfhGJ6QEDpOZIFkDtnq5JRxmvl3jsoQ==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/is-glob": { + "version": "4.0.3", + "resolved": "https://registry.npmjs.org/is-glob/-/is-glob-4.0.3.tgz", + "integrity": "sha512-xelSayHH36ZgE7ZWhli7pW34hNbNl8Ojv5KVmkJD4hBdD3th8Tfk9vYasLM+mXWOZhFkgZfxhLSnrwRr4elSSg==", + "dev": true, + "license": "MIT", + "dependencies": { + "is-extglob": "^2.1.1" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/isexe": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/isexe/-/isexe-2.0.0.tgz", + "integrity": "sha512-RHxMLp9lnKHGHRng9QFhRCMbYAcVpn69smSGcq3f36xjgVVWThj4qqLbTLlq7Ssj8B+fIQ1EuCEGI2lKsyQeIw==", + "dev": true, + "license": "ISC" + }, + "node_modules/js-tokens": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/js-tokens/-/js-tokens-4.0.0.tgz", + "integrity": "sha512-RdJUflcE3cUzKiMqQgsCu06FPu9UdIJO0beYbPhHN4k6apgJtifcoCtT9bcxOpYBtpD2kCM6Sbzg4CausW/PKQ==", + "dev": true, + "license": "MIT" + }, + "node_modules/js-yaml": { + "version": "4.1.1", + "resolved": "https://registry.npmjs.org/js-yaml/-/js-yaml-4.1.1.tgz", + "integrity": "sha512-qQKT4zQxXl8lLwBtHMWwaTcGfFOZviOJet3Oy/xmGk2gZH677CJM9EvtfdSkgWcATZhj/55JZ0rmy3myCT5lsA==", + "dev": true, + "license": "MIT", + "dependencies": { + "argparse": "^2.0.1" + }, + "bin": { + "js-yaml": "bin/js-yaml.js" + } + }, + "node_modules/jsesc": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/jsesc/-/jsesc-3.1.0.tgz", + "integrity": "sha512-/sM3dO2FOzXjKQhJuo0Q173wf2KOo8t4I8vHy6lF9poUp7bKT0/NHE8fPX23PwfhnykfqnC2xRxOnVw5XuGIaA==", + "dev": true, + "license": "MIT", + "bin": { + "jsesc": "bin/jsesc" + }, + "engines": { + "node": ">=6" + } + }, + "node_modules/json-buffer": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/json-buffer/-/json-buffer-3.0.1.tgz", + "integrity": "sha512-4bV5BfR2mqfQTJm+V5tPPdf+ZpuhiIvTuAB5g8kcrXOZpTT/QwwVRWBywX1ozr6lEuPdbHxwaJlm9G6mI2sfSQ==", + "dev": true, + "license": "MIT" + }, + "node_modules/json-schema-traverse": { + "version": "0.4.1", + "resolved": "https://registry.npmjs.org/json-schema-traverse/-/json-schema-traverse-0.4.1.tgz", + "integrity": "sha512-xbbCH5dCYU5T8LcEhhuh7HJ88HXuW3qsI3Y0zOZFKfZEHcpWiHU/Jxzk629Brsab/mMiHQti9wMP+845RPe3Vg==", + "dev": true, + "license": "MIT" + }, + "node_modules/json-stable-stringify-without-jsonify": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/json-stable-stringify-without-jsonify/-/json-stable-stringify-without-jsonify-1.0.1.tgz", + "integrity": "sha512-Bdboy+l7tA3OGW6FjyFHWkP5LuByj1Tk33Ljyq0axyzdk9//JSi2u3fP1QSmd1KNwq6VOKYGlAu87CisVir6Pw==", + "dev": true, + "license": "MIT" + }, + "node_modules/json5": { + "version": "2.2.3", + "resolved": "https://registry.npmjs.org/json5/-/json5-2.2.3.tgz", + "integrity": "sha512-XmOWe7eyHYH14cLdVPoyg+GOH3rYX++KpzrylJwSW98t3Nk+U8XOl8FWKOgwtzdb8lXGf6zYwDUzeHMWfxasyg==", + "dev": true, + "license": "MIT", + "bin": { + "json5": "lib/cli.js" + }, + "engines": { + "node": ">=6" + } + }, + "node_modules/keyv": { + "version": "4.5.4", + "resolved": "https://registry.npmjs.org/keyv/-/keyv-4.5.4.tgz", + "integrity": "sha512-oxVHkHR/EJf2CNXnWxRLW6mg7JyCCUcG0DtEGmL2ctUo1PNTin1PUil+r/+4r5MpVgC/fn1kjsx7mjSujKqIpw==", + "dev": true, + "license": "MIT", + "dependencies": { + "json-buffer": "3.0.1" + } + }, + "node_modules/levn": { + "version": "0.4.1", + "resolved": "https://registry.npmjs.org/levn/-/levn-0.4.1.tgz", + "integrity": "sha512-+bT2uH4E5LGE7h/n3evcS/sQlJXCpIp6ym8OWJ5eV6+67Dsql/LaaT7qJBAt2rzfoa/5QBGBhxDix1dMt2kQKQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "prelude-ls": "^1.2.1", + "type-check": "~0.4.0" + }, + "engines": { + "node": ">= 0.8.0" + } + }, + "node_modules/locate-path": { + "version": "6.0.0", + "resolved": "https://registry.npmjs.org/locate-path/-/locate-path-6.0.0.tgz", + "integrity": "sha512-iPZK6eYjbxRu3uB4/WZ3EsEIMJFMqAoopl3R+zuq0UjcAm/MO6KCweDgPfP3elTztoKP3KtnVHxTn2NHBSDVUw==", + "dev": true, + "license": "MIT", + "dependencies": { + "p-locate": "^5.0.0" + }, + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/lodash.merge": { + "version": "4.6.2", + "resolved": "https://registry.npmjs.org/lodash.merge/-/lodash.merge-4.6.2.tgz", + "integrity": "sha512-0KpjqXRVvrYyCsX1swR/XTK0va6VQkQM6MNo7PqW77ByjAhoARA8EfrP1N4+KlKj8YS0ZUCtRT/YUuhyYDujIQ==", + "dev": true, + "license": "MIT" + }, + "node_modules/lru-cache": { + "version": "5.1.1", + "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-5.1.1.tgz", + "integrity": "sha512-KpNARQA3Iwv+jTA0utUVVbrh+Jlrr1Fv0e56GGzAFOXN7dk/FviaDW8LHmK52DlcH4WP2n6gI8vN1aesBFgo9w==", + "dev": true, + "license": "ISC", + "dependencies": { + "yallist": "^3.0.2" + } + }, + "node_modules/minimatch": { + "version": "3.1.2", + "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.1.2.tgz", + "integrity": "sha512-J7p63hRiAjw1NDEww1W7i37+ByIrOWO5XQQAzZ3VOcL0PNybwpfmV/N05zFAzwQ9USyEcX6t3UO+K5aqBQOIHw==", + "dev": true, + "license": "ISC", + "dependencies": { + "brace-expansion": "^1.1.7" + }, + "engines": { + "node": "*" + } + }, + "node_modules/ms": { + "version": "2.1.3", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz", + "integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==", + "dev": true, + "license": "MIT" + }, + "node_modules/nanoid": { + "version": "3.3.11", + "resolved": "https://registry.npmjs.org/nanoid/-/nanoid-3.3.11.tgz", + "integrity": "sha512-N8SpfPUnUp1bK+PMYW8qSWdl9U+wwNWI4QKxOYDy9JAro3WMX7p2OeVRF9v+347pnakNevPmiHhNmZ2HbFA76w==", + "dev": true, + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/ai" + } + ], + "license": "MIT", + "bin": { + "nanoid": "bin/nanoid.cjs" + }, + "engines": { + "node": "^10 || ^12 || ^13.7 || ^14 || >=15.0.1" + } + }, + "node_modules/natural-compare": { + "version": "1.4.0", + "resolved": "https://registry.npmjs.org/natural-compare/-/natural-compare-1.4.0.tgz", + "integrity": "sha512-OWND8ei3VtNC9h7V60qff3SVobHr996CTwgxubgyQYEpg290h9J0buyECNNJexkFm5sOajh5G116RYA1c8ZMSw==", + "dev": true, + "license": "MIT" + }, + "node_modules/node-releases": { + "version": "2.0.27", + "resolved": "https://registry.npmjs.org/node-releases/-/node-releases-2.0.27.tgz", + "integrity": "sha512-nmh3lCkYZ3grZvqcCH+fjmQ7X+H0OeZgP40OierEaAptX4XofMh5kwNbWh7lBduUzCcV/8kZ+NDLCwm2iorIlA==", + "dev": true, + "license": "MIT" + }, + "node_modules/optionator": { + "version": "0.9.4", + "resolved": "https://registry.npmjs.org/optionator/-/optionator-0.9.4.tgz", + "integrity": "sha512-6IpQ7mKUxRcZNLIObR0hz7lxsapSSIYNZJwXPGeF0mTVqGKFIXj1DQcMoT22S3ROcLyY/rz0PWaWZ9ayWmad9g==", + "dev": true, + "license": "MIT", + "dependencies": { + "deep-is": "^0.1.3", + "fast-levenshtein": "^2.0.6", + "levn": "^0.4.1", + "prelude-ls": "^1.2.1", + "type-check": "^0.4.0", + "word-wrap": "^1.2.5" + }, + "engines": { + "node": ">= 0.8.0" + } + }, + "node_modules/p-limit": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/p-limit/-/p-limit-3.1.0.tgz", + "integrity": "sha512-TYOanM3wGwNGsZN2cVTYPArw454xnXj5qmWF1bEoAc4+cU/ol7GVh7odevjp1FNHduHc3KZMcFduxU5Xc6uJRQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "yocto-queue": "^0.1.0" + }, + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/p-locate": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/p-locate/-/p-locate-5.0.0.tgz", + "integrity": "sha512-LaNjtRWUBY++zB5nE/NwcaoMylSPk+S+ZHNB1TzdbMJMny6dynpAGt7X/tl/QYq3TIeE6nxHppbo2LGymrG5Pw==", + "dev": true, + "license": "MIT", + "dependencies": { + "p-limit": "^3.0.2" + }, + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/parent-module": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/parent-module/-/parent-module-1.0.1.tgz", + "integrity": "sha512-GQ2EWRpQV8/o+Aw8YqtfZZPfNRWZYkbidE9k5rpl/hC3vtHHBfGm2Ifi6qWV+coDGkrUKZAxE3Lot5kcsRlh+g==", + "dev": true, + "license": "MIT", + "dependencies": { + "callsites": "^3.0.0" + }, + "engines": { + "node": ">=6" + } + }, + "node_modules/path-exists": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/path-exists/-/path-exists-4.0.0.tgz", + "integrity": "sha512-ak9Qy5Q7jYb2Wwcey5Fpvg2KoAc/ZIhLSLOSBmRmygPsGwkVVt0fZa0qrtMz+m6tJTAHfZQ8FnmB4MG4LWy7/w==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=8" + } + }, + "node_modules/path-key": { + "version": "3.1.1", + "resolved": "https://registry.npmjs.org/path-key/-/path-key-3.1.1.tgz", + "integrity": "sha512-ojmeN0qd+y0jszEtoY48r0Peq5dwMEkIlCOu6Q5f41lfkswXuKtYrhgoTpLnyIcHm24Uhqx+5Tqm2InSwLhE6Q==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=8" + } + }, + "node_modules/picocolors": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/picocolors/-/picocolors-1.1.1.tgz", + "integrity": "sha512-xceH2snhtb5M9liqDsmEw56le376mTZkEX/jEb/RxNFyegNul7eNslCXP9FDj/Lcu0X8KEyMceP2ntpaHrDEVA==", + "dev": true, + "license": "ISC" + }, + "node_modules/picomatch": { + "version": "4.0.3", + "resolved": "https://registry.npmjs.org/picomatch/-/picomatch-4.0.3.tgz", + "integrity": "sha512-5gTmgEY/sqK6gFXLIsQNH19lWb4ebPDLA4SdLP7dsWkIXHWlG66oPuVvXSGFPppYZz8ZDZq0dYYrbHfBCVUb1Q==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://github.com/sponsors/jonschlinkert" + } + }, + "node_modules/postcss": { + "version": "8.5.6", + "resolved": "https://registry.npmjs.org/postcss/-/postcss-8.5.6.tgz", + "integrity": "sha512-3Ybi1tAuwAP9s0r1UQ2J4n5Y0G05bJkpUIO0/bI9MhwmD70S5aTWbXGBwxHrelT+XM1k6dM0pk+SwNkpTRN7Pg==", + "dev": true, + "funding": [ + { + "type": "opencollective", + "url": "https://opencollective.com/postcss/" + }, + { + "type": "tidelift", + "url": "https://tidelift.com/funding/github/npm/postcss" + }, + { + "type": "github", + "url": "https://github.com/sponsors/ai" + } + ], + "license": "MIT", + "dependencies": { + "nanoid": "^3.3.11", + "picocolors": "^1.1.1", + "source-map-js": "^1.2.1" + }, + "engines": { + "node": "^10 || ^12 || >=14" + } + }, + "node_modules/prelude-ls": { + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/prelude-ls/-/prelude-ls-1.2.1.tgz", + "integrity": "sha512-vkcDPrRZo1QZLbn5RLGPpg/WmIQ65qoWWhcGKf/b5eplkkarX0m9z8ppCat4mlOqUsWpyNuYgO3VRyrYHSzX5g==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">= 0.8.0" + } + }, + "node_modules/punycode": { + "version": "2.3.1", + "resolved": "https://registry.npmjs.org/punycode/-/punycode-2.3.1.tgz", + "integrity": "sha512-vYt7UD1U9Wg6138shLtLOvdAu+8DsC/ilFtEVHcH+wydcSpNE20AfSOduf6MkRFahL5FY7X1oU7nKVZFtfq8Fg==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=6" + } + }, + "node_modules/react": { + "version": "19.2.3", + "resolved": "https://registry.npmjs.org/react/-/react-19.2.3.tgz", + "integrity": "sha512-Ku/hhYbVjOQnXDZFv2+RibmLFGwFdeeKHFcOTlrt7xplBnya5OGn/hIRDsqDiSUcfORsDC7MPxwork8jBwsIWA==", + "license": "MIT", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/react-dom": { + "version": "19.2.3", + "resolved": "https://registry.npmjs.org/react-dom/-/react-dom-19.2.3.tgz", + "integrity": "sha512-yELu4WmLPw5Mr/lmeEpox5rw3RETacE++JgHqQzd2dg+YbJuat3jH4ingc+WPZhxaoFzdv9y33G+F7Nl5O0GBg==", + "license": "MIT", + "dependencies": { + "scheduler": "^0.27.0" + }, + "peerDependencies": { + "react": "^19.2.3" + } + }, + "node_modules/react-refresh": { + "version": "0.18.0", + "resolved": "https://registry.npmjs.org/react-refresh/-/react-refresh-0.18.0.tgz", + "integrity": "sha512-QgT5//D3jfjJb6Gsjxv0Slpj23ip+HtOpnNgnb2S5zU3CB26G/IDPGoy4RJB42wzFE46DRsstbW6tKHoKbhAxw==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/resolve-from": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/resolve-from/-/resolve-from-4.0.0.tgz", + "integrity": "sha512-pb/MYmXstAkysRFx8piNI1tGFNQIFA3vkE3Gq4EuA1dF6gHp/+vgZqsCGJapvy8N3Q+4o7FwvquPJcnZ7RYy4g==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=4" + } + }, + "node_modules/rollup": { + "version": "4.54.0", + "resolved": "https://registry.npmjs.org/rollup/-/rollup-4.54.0.tgz", + "integrity": "sha512-3nk8Y3a9Ea8szgKhinMlGMhGMw89mqule3KWczxhIzqudyHdCIOHw8WJlj/r329fACjKLEh13ZSk7oE22kyeIw==", + "dev": true, + "license": "MIT", + "dependencies": { + "@types/estree": "1.0.8" + }, + "bin": { + "rollup": "dist/bin/rollup" + }, + "engines": { + "node": ">=18.0.0", + "npm": ">=8.0.0" + }, + "optionalDependencies": { + "@rollup/rollup-android-arm-eabi": "4.54.0", + "@rollup/rollup-android-arm64": "4.54.0", + "@rollup/rollup-darwin-arm64": "4.54.0", + "@rollup/rollup-darwin-x64": "4.54.0", + "@rollup/rollup-freebsd-arm64": "4.54.0", + "@rollup/rollup-freebsd-x64": "4.54.0", + "@rollup/rollup-linux-arm-gnueabihf": "4.54.0", + "@rollup/rollup-linux-arm-musleabihf": "4.54.0", + "@rollup/rollup-linux-arm64-gnu": "4.54.0", + "@rollup/rollup-linux-arm64-musl": "4.54.0", + "@rollup/rollup-linux-loong64-gnu": "4.54.0", + "@rollup/rollup-linux-ppc64-gnu": "4.54.0", + "@rollup/rollup-linux-riscv64-gnu": "4.54.0", + "@rollup/rollup-linux-riscv64-musl": "4.54.0", + "@rollup/rollup-linux-s390x-gnu": "4.54.0", + "@rollup/rollup-linux-x64-gnu": "4.54.0", + "@rollup/rollup-linux-x64-musl": "4.54.0", + "@rollup/rollup-openharmony-arm64": "4.54.0", + "@rollup/rollup-win32-arm64-msvc": "4.54.0", + "@rollup/rollup-win32-ia32-msvc": "4.54.0", + "@rollup/rollup-win32-x64-gnu": "4.54.0", + "@rollup/rollup-win32-x64-msvc": "4.54.0", + "fsevents": "~2.3.2" + } + }, + "node_modules/scheduler": { + "version": "0.27.0", + "resolved": "https://registry.npmjs.org/scheduler/-/scheduler-0.27.0.tgz", + "integrity": "sha512-eNv+WrVbKu1f3vbYJT/xtiF5syA5HPIMtf9IgY/nKg0sWqzAUEvqY/xm7OcZc/qafLx/iO9FgOmeSAp4v5ti/Q==", + "license": "MIT" + }, + "node_modules/semver": { + "version": "6.3.1", + "resolved": "https://registry.npmjs.org/semver/-/semver-6.3.1.tgz", + "integrity": "sha512-BR7VvDCVHO+q2xBEWskxS6DJE1qRnb7DxzUrogb71CWoSficBxYsiAGd+Kl0mmq/MprG9yArRkyrQxTO6XjMzA==", + "dev": true, + "license": "ISC", + "bin": { + "semver": "bin/semver.js" + } + }, + "node_modules/shebang-command": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/shebang-command/-/shebang-command-2.0.0.tgz", + "integrity": "sha512-kHxr2zZpYtdmrN1qDjrrX/Z1rR1kG8Dx+gkpK1G4eXmvXswmcE1hTWBWYUzlraYw1/yZp6YuDY77YtvbN0dmDA==", + "dev": true, + "license": "MIT", + "dependencies": { + "shebang-regex": "^3.0.0" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/shebang-regex": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/shebang-regex/-/shebang-regex-3.0.0.tgz", + "integrity": "sha512-7++dFhtcx3353uBaq8DDR4NuxBetBzC7ZQOhmTQInHEd6bSrXdiEyzCvG07Z44UYdLShWUyXt5M/yhz8ekcb1A==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=8" + } + }, + "node_modules/source-map-js": { + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/source-map-js/-/source-map-js-1.2.1.tgz", + "integrity": "sha512-UXWMKhLOwVKb728IUtQPXxfYU+usdybtUrK/8uGE8CQMvrhOpwvzDBwj0QhSL7MQc7vIsISBG8VQ8+IDQxpfQA==", + "dev": true, + "license": "BSD-3-Clause", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/strip-json-comments": { + "version": "3.1.1", + "resolved": "https://registry.npmjs.org/strip-json-comments/-/strip-json-comments-3.1.1.tgz", + "integrity": "sha512-6fPc+R4ihwqP6N/aIv2f1gMH8lOVtWQHoqC4yK6oSDVVocumAsfCqjkXnqiYMhmMwS/mEHLp7Vehlt3ql6lEig==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=8" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/supports-color": { + "version": "7.2.0", + "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-7.2.0.tgz", + "integrity": "sha512-qpCAvRl9stuOHveKsn7HncJRvv501qIacKzQlO/+Lwxc9+0q2wLyv4Dfvt80/DPn2pqOBsJdDiogXGR9+OvwRw==", + "dev": true, + "license": "MIT", + "dependencies": { + "has-flag": "^4.0.0" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/tinyglobby": { + "version": "0.2.15", + "resolved": "https://registry.npmjs.org/tinyglobby/-/tinyglobby-0.2.15.tgz", + "integrity": "sha512-j2Zq4NyQYG5XMST4cbs02Ak8iJUdxRM0XI5QyxXuZOzKOINmWurp3smXu3y5wDcJrptwpSjgXHzIQxR0omXljQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "fdir": "^6.5.0", + "picomatch": "^4.0.3" + }, + "engines": { + "node": ">=12.0.0" + }, + "funding": { + "url": "https://github.com/sponsors/SuperchupuDev" + } + }, + "node_modules/ts-api-utils": { + "version": "2.3.0", + "resolved": "https://registry.npmjs.org/ts-api-utils/-/ts-api-utils-2.3.0.tgz", + "integrity": "sha512-6eg3Y9SF7SsAvGzRHQvvc1skDAhwI4YQ32ui1scxD1Ccr0G5qIIbUBT3pFTKX8kmWIQClHobtUdNuaBgwdfdWg==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=18.12" + }, + "peerDependencies": { + "typescript": ">=4.8.4" + } + }, + "node_modules/type-check": { + "version": "0.4.0", + "resolved": "https://registry.npmjs.org/type-check/-/type-check-0.4.0.tgz", + "integrity": "sha512-XleUoc9uwGXqjWwXaUTZAmzMcFZ5858QA2vvx1Ur5xIcixXIP+8LnFDgRplU30us6teqdlskFfu+ae4K79Ooew==", + "dev": true, + "license": "MIT", + "dependencies": { + "prelude-ls": "^1.2.1" + }, + "engines": { + "node": ">= 0.8.0" + } + }, + "node_modules/typescript": { + "version": "5.9.3", + "resolved": "https://registry.npmjs.org/typescript/-/typescript-5.9.3.tgz", + "integrity": "sha512-jl1vZzPDinLr9eUt3J/t7V6FgNEw9QjvBPdysz9KfQDD41fQrC2Y4vKQdiaUpFT4bXlb1RHhLpp8wtm6M5TgSw==", + "dev": true, + "license": "Apache-2.0", + "bin": { + "tsc": "bin/tsc", + "tsserver": "bin/tsserver" + }, + "engines": { + "node": ">=14.17" + } + }, + "node_modules/typescript-eslint": { + "version": "8.51.0", + "resolved": "https://registry.npmjs.org/typescript-eslint/-/typescript-eslint-8.51.0.tgz", + "integrity": "sha512-jh8ZuM5oEh2PSdyQG9YAEM1TCGuWenLSuSUhf/irbVUNW9O5FhbFVONviN2TgMTBnUmyHv7E56rYnfLZK6TkiA==", + "dev": true, + "license": "MIT", + "dependencies": { + "@typescript-eslint/eslint-plugin": "8.51.0", + "@typescript-eslint/parser": "8.51.0", + "@typescript-eslint/typescript-estree": "8.51.0", + "@typescript-eslint/utils": "8.51.0" + }, + "engines": { + "node": "^18.18.0 || ^20.9.0 || >=21.1.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/typescript-eslint" + }, + "peerDependencies": { + "eslint": "^8.57.0 || ^9.0.0", + "typescript": ">=4.8.4 <6.0.0" + } + }, + "node_modules/undici-types": { + "version": "7.16.0", + "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-7.16.0.tgz", + "integrity": "sha512-Zz+aZWSj8LE6zoxD+xrjh4VfkIG8Ya6LvYkZqtUQGJPZjYl53ypCaUwWqo7eI0x66KBGeRo+mlBEkMSeSZ38Nw==", + "dev": true, + "license": "MIT" + }, + "node_modules/update-browserslist-db": { + "version": "1.2.3", + "resolved": "https://registry.npmjs.org/update-browserslist-db/-/update-browserslist-db-1.2.3.tgz", + "integrity": "sha512-Js0m9cx+qOgDxo0eMiFGEueWztz+d4+M3rGlmKPT+T4IS/jP4ylw3Nwpu6cpTTP8R1MAC1kF4VbdLt3ARf209w==", + "dev": true, + "funding": [ + { + "type": "opencollective", + "url": "https://opencollective.com/browserslist" + }, + { + "type": "tidelift", + "url": "https://tidelift.com/funding/github/npm/browserslist" + }, + { + "type": "github", + "url": "https://github.com/sponsors/ai" + } + ], + "license": "MIT", + "dependencies": { + "escalade": "^3.2.0", + "picocolors": "^1.1.1" + }, + "bin": { + "update-browserslist-db": "cli.js" + }, + "peerDependencies": { + "browserslist": ">= 4.21.0" + } + }, + "node_modules/uri-js": { + "version": "4.4.1", + "resolved": "https://registry.npmjs.org/uri-js/-/uri-js-4.4.1.tgz", + "integrity": "sha512-7rKUyy33Q1yc98pQ1DAmLtwX109F7TIfWlW1Ydo8Wl1ii1SeHieeh0HHfPeL2fMXK6z0s8ecKs9frCuLJvndBg==", + "dev": true, + "license": "BSD-2-Clause", + "dependencies": { + "punycode": "^2.1.0" + } + }, + "node_modules/vite": { + "version": "7.3.0", + "resolved": "https://registry.npmjs.org/vite/-/vite-7.3.0.tgz", + "integrity": "sha512-dZwN5L1VlUBewiP6H9s2+B3e3Jg96D0vzN+Ry73sOefebhYr9f94wwkMNN/9ouoU8pV1BqA1d1zGk8928cx0rg==", + "dev": true, + "license": "MIT", + "dependencies": { + "esbuild": "^0.27.0", + "fdir": "^6.5.0", + "picomatch": "^4.0.3", + "postcss": "^8.5.6", + "rollup": "^4.43.0", + "tinyglobby": "^0.2.15" + }, + "bin": { + "vite": "bin/vite.js" + }, + "engines": { + "node": "^20.19.0 || >=22.12.0" + }, + "funding": { + "url": "https://github.com/vitejs/vite?sponsor=1" + }, + "optionalDependencies": { + "fsevents": "~2.3.3" + }, + "peerDependencies": { + "@types/node": "^20.19.0 || >=22.12.0", + "jiti": ">=1.21.0", + "less": "^4.0.0", + "lightningcss": "^1.21.0", + "sass": "^1.70.0", + "sass-embedded": "^1.70.0", + "stylus": ">=0.54.8", + "sugarss": "^5.0.0", + "terser": "^5.16.0", + "tsx": "^4.8.1", + "yaml": "^2.4.2" + }, + "peerDependenciesMeta": { + "@types/node": { + "optional": true + }, + "jiti": { + "optional": true + }, + "less": { + "optional": true + }, + "lightningcss": { + "optional": true + }, + "sass": { + "optional": true + }, + "sass-embedded": { + "optional": true + }, + "stylus": { + "optional": true + }, + "sugarss": { + "optional": true + }, + "terser": { + "optional": true + }, + "tsx": { + "optional": true + }, + "yaml": { + "optional": true + } + } + }, + "node_modules/which": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/which/-/which-2.0.2.tgz", + "integrity": "sha512-BLI3Tl1TW3Pvl70l3yq3Y64i+awpwXqsGBYWkkqMtnbXgrMD+yj7rhW0kuEDxzJaYXGjEW5ogapKNMEKNMjibA==", + "dev": true, + "license": "ISC", + "dependencies": { + "isexe": "^2.0.0" + }, + "bin": { + "node-which": "bin/node-which" + }, + "engines": { + "node": ">= 8" + } + }, + "node_modules/word-wrap": { + "version": "1.2.5", + "resolved": "https://registry.npmjs.org/word-wrap/-/word-wrap-1.2.5.tgz", + "integrity": "sha512-BN22B5eaMMI9UMtjrGd5g5eCYPpCPDUy0FJXbYsaT5zYxjFOckS53SQDE3pWkVoWpHXVb3BrYcEN4Twa55B5cA==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/yallist": { + "version": "3.1.1", + "resolved": "https://registry.npmjs.org/yallist/-/yallist-3.1.1.tgz", + "integrity": "sha512-a4UGQaWPH59mOXUYnAG2ewncQS4i4F43Tv3JoAM+s2VDAmS9NsK8GpDMLrCHPksFT7h3K6TOoUNn2pb7RoXx4g==", + "dev": true, + "license": "ISC" + }, + "node_modules/yocto-queue": { + "version": "0.1.0", + "resolved": "https://registry.npmjs.org/yocto-queue/-/yocto-queue-0.1.0.tgz", + "integrity": "sha512-rVksvsnNCdJ/ohGc6xgPwyN8eheCxsiLM8mxuE/t/mOVqJewPuO1miLpTHQiRgTKCLexL4MeAFVagts7HmNZ2Q==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/zod": { + "version": "4.2.1", + "resolved": "https://registry.npmjs.org/zod/-/zod-4.2.1.tgz", + "integrity": "sha512-0wZ1IRqGGhMP76gLqz8EyfBXKk0J2qo2+H3fi4mcUP/KtTocoX08nmIAHl1Z2kJIZbZee8KOpBCSNPRgauucjw==", + "dev": true, + "license": "MIT", + "funding": { + "url": "https://github.com/sponsors/colinhacks" + } + }, + "node_modules/zod-validation-error": { + "version": "4.0.2", + "resolved": "https://registry.npmjs.org/zod-validation-error/-/zod-validation-error-4.0.2.tgz", + "integrity": "sha512-Q6/nZLe6jxuU80qb/4uJ4t5v2VEZ44lzQjPDhYJNztRQ4wyWc6VF3D3Kb/fAuPetZQnhS3hnajCf9CsWesghLQ==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=18.0.0" + }, + "peerDependencies": { + "zod": "^3.25.0 || ^4.0.0" + } + } + } +} diff --git a/frontend/package.json b/frontend/package.json new file mode 100644 index 0000000..eded7cd --- /dev/null +++ b/frontend/package.json @@ -0,0 +1,30 @@ +{ + "name": "frontend", + "private": true, + "version": "0.0.0", + "type": "module", + "scripts": { + "dev": "vite", + "build": "tsc -b && vite build", + "lint": "eslint .", + "preview": "vite preview" + }, + "dependencies": { + "react": "^19.2.0", + "react-dom": "^19.2.0" + }, + "devDependencies": { + "@eslint/js": "^9.39.1", + "@types/node": "^24.10.1", + "@types/react": "^19.2.5", + "@types/react-dom": "^19.2.3", + "@vitejs/plugin-react": "^5.1.1", + "eslint": "^9.39.1", + "eslint-plugin-react-hooks": "^7.0.1", + "eslint-plugin-react-refresh": "^0.4.24", + "globals": "^16.5.0", + "typescript": "~5.9.3", + "typescript-eslint": "^8.46.4", + "vite": "^7.2.4" + } +} diff --git a/frontend/public/audio-processor.js b/frontend/public/audio-processor.js new file mode 100644 index 0000000..d6fc838 --- /dev/null +++ b/frontend/public/audio-processor.js @@ -0,0 +1,72 @@ +/** + * AudioWorklet processor for capturing and resampling microphone audio + * Buffers to 100ms chunks (1600 samples at 16kHz) to meet AssemblyAI requirements + * Outputs Float32 audio (backend handles conversion to PCM16) + */ +class AudioProcessor extends AudioWorkletProcessor { + constructor(options) { + super(); + this.sourceSampleRate = options.processorOptions.sourceSampleRate; + this.targetSampleRate = 16000; + this.resampleRatio = this.sourceSampleRate / this.targetSampleRate; + + // Input buffer for resampling + this.inputBuffer = null; + + // Output buffer to collect 100ms of resampled audio (1600 samples at 16kHz) + // AssemblyAI requires chunks between 50-1000ms + this.outputBuffer = []; + this.outputBufferSize = 1600; // 100ms at 16kHz + } + + process(inputs) { + const inputChannel = inputs[0][0]; + if (!inputChannel) return true; + + // Accumulate input samples + const currentLength = this.inputBuffer ? this.inputBuffer.length : 0; + const newBuffer = new Float32Array(currentLength + inputChannel.length); + if (this.inputBuffer) { + newBuffer.set(this.inputBuffer, 0); + } + newBuffer.set(inputChannel, currentLength); + this.inputBuffer = newBuffer; + + // Resample to 16kHz + const numOutputSamples = Math.floor(this.inputBuffer.length / this.resampleRatio); + if (numOutputSamples === 0) return true; + + const resampledData = new Float32Array(numOutputSamples); + for (let i = 0; i < numOutputSamples; i++) { + const correspondingInputIndex = i * this.resampleRatio; + const lowerIndex = Math.floor(correspondingInputIndex); + const upperIndex = Math.ceil(correspondingInputIndex); + const interpolationFactor = correspondingInputIndex - lowerIndex; + + const lowerValue = this.inputBuffer[lowerIndex] || 0; + const upperValue = this.inputBuffer[upperIndex] || 0; + + resampledData[i] = lowerValue + (upperValue - lowerValue) * interpolationFactor; + } + + // Keep unconsumed input samples + const consumedInputSamples = numOutputSamples * this.resampleRatio; + this.inputBuffer = this.inputBuffer.slice(Math.round(consumedInputSamples)); + + // Add Float32 samples to output buffer + for (let i = 0; i < resampledData.length; i++) { + this.outputBuffer.push(resampledData[i]); + + // When we have 100ms of audio (1600 samples), send it as Float32 + if (this.outputBuffer.length >= this.outputBufferSize) { + const float32Array = new Float32Array(this.outputBuffer); + this.port.postMessage(float32Array.buffer, [float32Array.buffer]); + this.outputBuffer = []; + } + } + + return true; + } +} + +registerProcessor('audio-processor', AudioProcessor); diff --git a/frontend/public/vite.svg b/frontend/public/vite.svg new file mode 100644 index 0000000..e7b8dfb --- /dev/null +++ b/frontend/public/vite.svg @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/frontend/src/App.tsx b/frontend/src/App.tsx new file mode 100644 index 0000000..e6f1697 --- /dev/null +++ b/frontend/src/App.tsx @@ -0,0 +1,33 @@ +import { AppProvider } from './context/AppContext'; +import { Header } from './components/Header'; +import { ChatSection } from './components/ChatSection'; +import { FlashcardsSection } from './components/FlashcardsSection'; +import './styles/main.css'; + +function AppContent() { + return ( + <> +
+
+
+
+ + +
+
+
+ {/* Hidden audio element for iOS compatibility */} +