Example: Half-cascade with Cartesia

1
/**
2
 * Voximplant + OpenAI Realtime API + Cartesia TTS demo
3
 * Scenario: OpenAI handles STT/LLM, Cartesia handles TTS (half-cascade).
4
 */
5
6
require(Modules.OpenAI);
7
require(Modules.Cartesia);
8
9
const OPENAI_MODEL = "gpt-realtime-2.1";
10
11
const SYSTEM_PROMPT = `
12
You are Voxi, a helpful phone assistant.
13
Keep responses short and telephony-friendly.
14
Reply in English.
15
`;
16
17
const CARTESIA_MODEL_ID = "sonic-2";
18
const CARTESIA_VOICE_ID = "a0e99841-438c-4a64-b679-ae501e7d6091";
19
20
const SESSION_CONFIG = {
21
    session: {
22
        type: "realtime",
23
        model: OPENAI_MODEL,
24
        instructions: SYSTEM_PROMPT,
25
        output_modalities: ["text"],
26
        audio: {
27
            input: {
28
                turn_detection: {type: "server_vad", interrupt_response: true},
29
            },
30
        },
31
    },
32
};
33
34
VoxEngine.addEventListener(AppEvents.CallAlerting, async ({call}) => {
35
    let voiceAIClient;
36
    let ttsPlayer;
37
38
    call.addEventListener(CallEvents.Disconnected, () => VoxEngine.terminate());
39
    call.addEventListener(CallEvents.Failed, () => VoxEngine.terminate());
40
41
    try {
42
        call.answer();
43
        // call.record({hd_audio: true, stereo: true}); // Optional: record the call
44
45
        const openAiKey = VoxEngine.getSecretValue("OPENAI_API_KEY");
46
47
        voiceAIClient = await OpenAI.createRealtimeAPIClient({
48
            apiKey: openAiKey,
49
            model: OPENAI_MODEL,
50
            onWebSocketClose: (event) => {
51
                Logger.write("===OpenAI.WebSocket.Close===");
52
                if (event) Logger.write(JSON.stringify(event));
53
                VoxEngine.terminate();
54
            },
55
        });
56
57
        voiceAIClient.addEventListener(OpenAI.RealtimeAPIEvents.SessionCreated, () => {
58
            voiceAIClient.sessionUpdate(SESSION_CONFIG);
59
        });
60
61
        voiceAIClient.addEventListener(OpenAI.RealtimeAPIEvents.SessionUpdated, () => {
62
            call.sendMediaTo(voiceAIClient);
63
            voiceAIClient.responseCreate({instructions: "Hello! How can I help today?"});
64
        });
65
66
        voiceAIClient.addEventListener(OpenAI.RealtimeAPIEvents.ResponseOutputTextDone, (event) => {
67
            const payload = event?.data?.payload || event?.data || {};
68
            const text = payload.text || payload.delta;
69
            if (!text) return;
70
            Logger.write(`===AGENT_TEXT=== ${text}`);
71
72
            // Cartesia TTS requires input on initialization, so we lazily create the player here as needed
73
            if (!ttsPlayer) {
74
                const contextId = `openai-cartesia-${Date.now()}`;
75
                const cartesiaOptions = {
76
                    // apikey: VoxEngine.getSecretValue('CARTESIA_API_KEY'), // optional
77
                    generationRequestParameters: {
78
                        model_id: CARTESIA_MODEL_ID,
79
                        transcript: text,
80
                        language: "en",
81
                        voice: {mode: "id", id: CARTESIA_VOICE_ID},
82
                        context_id: contextId,
83
                        continue: false,
84
                    },
85
                };
86
87
                ttsPlayer = Cartesia.createRealtimeTTSPlayer(text, cartesiaOptions);
88
                ttsPlayer.sendMediaTo(call);
89
                return;
90
            }
91
92
            const contextId = `openai-cartesia-${Date.now()}`;
93
            ttsPlayer.generationRequest({
94
                model_id: CARTESIA_MODEL_ID,
95
                transcript: text,
96
                language: "en",
97
                voice: {mode: "id", id: CARTESIA_VOICE_ID},
98
                context_id: contextId,
99
                continue: false,
100
            });
101
        });
102
103
        // Barge-in: clear both OpenAI and Cartesia buffers
104
        voiceAIClient.addEventListener(OpenAI.RealtimeAPIEvents.InputAudioBufferSpeechStarted, () => {
105
            Logger.write("===BARGE-IN: OpenAI.InputAudioBufferSpeechStarted===");
106
            voiceAIClient.clearMediaBuffer();
107
            ttsPlayer?.clearBuffer();
108
        });
109
110
        // ---------------------- Log all other events for debugging -----------------------
111
        [
112
            OpenAI.RealtimeAPIEvents.ResponseCreated,
113
            OpenAI.RealtimeAPIEvents.ResponseDone,
114
            OpenAI.RealtimeAPIEvents.ResponseOutputTextDelta,
115
            OpenAI.RealtimeAPIEvents.ConnectorInformation,
116
            OpenAI.RealtimeAPIEvents.HTTPResponse,
117
            OpenAI.RealtimeAPIEvents.WebSocketError,
118
            OpenAI.RealtimeAPIEvents.Unknown,
119
            OpenAI.Events.WebSocketMediaStarted,
120
            OpenAI.Events.WebSocketMediaEnded,
121
        ].forEach((eventName) => {
122
            voiceAIClient.addEventListener(eventName, (event) => {
123
                Logger.write(`===${event.name}===`);
124
                if (event?.data) Logger.write(JSON.stringify(event.data));
125
            });
126
        });
127
    } catch (error) {
128
        Logger.write("===UNHANDLED_ERROR===");
129
        Logger.write(error);
130
        voiceAIClient?.close();
131
        VoxEngine.terminate();
132
    }
133
});