Voice AI ClientsOpenAI

Example: Half-cascade with ElevenLabs

View as Markdown

For the complete documentation index, see llms.txt.

Overview

This half-cascade example uses OpenAI Realtime for speech‑to‑text and reasoning, then sends OpenAI text responses to ElevenLabs Realtime TTS.

⬇️ Jump to the Full VoxEngine scenario.

Prerequisites

  • Store your OpenAI API key in Voximplant Secrets under OPENAI_API_KEY.
  • (Optional) Update the ELEVENLABS_VOICE_ID constant in the example to your preferred voice.
  • (Optional) Store your ElevenLabs API key in Voximplant Secrets under ELEVENLABS_API_KEY if you want to use your own ElevenLabs account.

How it works

  • OpenAI runs in text mode (output_modalities: ["text"]).
  • Caller audio is sent to OpenAI: call.sendMediaTo(voiceAIClient).
  • ElevenLabs generates speech from OpenAI text and streams it to the call.

Notes

  • The example uses eleven_turbo_v2_5.
  • The example instructs the agent to reply in English.
  • Do not set audio format parameters (for example ulaw_8000) in half-cascade connector requests. VoxEngine’s WebSocket gateway handles media format negotiation automatically.
  • If no ElevenLabs API key is provided, Voximplant’s default account and billing are used.
  • Custom / cloned voices are only available when using your own API key.
  • Use append(text, true) for each response chunk so playback stays responsive.
  • No audio format/codec params should be passed for this half-cascade flow.

More info

Full VoxEngine scenario

voxeengine-openai-half-cascade-elevenlabs.js
1/**
2 * Voximplant + OpenAI Realtime API + ElevenLabs TTS demo
3 * Scenario: OpenAI handles STT/LLM, ElevenLabs handles TTS (half-cascade).
4 */
5
6require(Modules.OpenAI);
7require(Modules.ElevenLabs);
8const SYSTEM_PROMPT = `
9You are Voxi, a helpful phone assistant.
10Keep responses short and telephony-friendly.
11Always reply in English.
12`;
13
14const ELEVENLABS_VOICE_ID = "21m00Tcm4TlvDq8ikWAM";
15const ELEVENLABS_MODEL_ID = "eleven_turbo_v2_5";
16
17const SESSION_CONFIG = {
18    session: {
19        type: "realtime",
20        instructions: SYSTEM_PROMPT,
21        output_modalities: ["text"],
22        turn_detection: {type: "server_vad", interrupt_response: true},
23    },
24};
25
26VoxEngine.addEventListener(AppEvents.CallAlerting, async ({call}) => {
27    let voiceAIClient;
28    let ttsPlayer;
29
30    call.addEventListener(CallEvents.Disconnected, () => VoxEngine.terminate());
31    call.addEventListener(CallEvents.Failed, () => VoxEngine.terminate());
32
33    try {
34        call.answer();
35        // call.record({hd_audio: true, stereo: true}); // Optional: record the call
36
37        const openAiKey = VoxEngine.getSecretValue('OPENAI_API_KEY');
38
39        voiceAIClient = await OpenAI.createRealtimeAPIClient({
40            apiKey: openAiKey,
41            model: "gpt-realtime-1.5",
42            onWebSocketClose: (event) => {
43                Logger.write("===OpenAI.WebSocket.Close===");
44                if (event) Logger.write(JSON.stringify(event));
45                VoxEngine.terminate();
46            },
47        });
48
49        voiceAIClient.addEventListener(OpenAI.RealtimeAPIEvents.SessionCreated, () => {
50            voiceAIClient.sessionUpdate(SESSION_CONFIG);
51        });
52
53        voiceAIClient.addEventListener(OpenAI.RealtimeAPIEvents.SessionUpdated, () => {
54            call.sendMediaTo(voiceAIClient);  // bridge media between the call and OpenAI
55
56            // create the TTS player and pass the config parameters
57            ttsPlayer = ElevenLabs.createRealtimeTTSPlayer(" ", {
58                // headers: [{name: "xi-api-key", value: VoxEngine.getSecretValue('ELEVENLABS_API_KEY')}], // optional
59                pathParameters: {voice_id: ELEVENLABS_VOICE_ID},
60                queryParameters: {
61                    model_id: ELEVENLABS_MODEL_ID,
62                },
63            });
64            ttsPlayer.sendMediaTo(call); // bridge media between the TTS player and the call
65
66            voiceAIClient.responseCreate({instructions: "Hello! How can I help today?"});
67        });
68
69        voiceAIClient.addEventListener(OpenAI.RealtimeAPIEvents.ResponseOutputTextDone, (event) => {
70            const payload = event?.data?.payload || event?.data || {};
71            const text = payload.text || payload.delta;
72            if (!text || !ttsPlayer) return;
73            Logger.write(`===AGENT_TEXT=== ${text}`);
74            ttsPlayer.append(text, true);
75        });
76
77        // Barge-in: clear both OpenAI and ElevenLabs buffers
78        voiceAIClient.addEventListener(OpenAI.RealtimeAPIEvents.InputAudioBufferSpeechStarted, () => {
79            Logger.write("===BARGE-IN: OpenAI.InputAudioBufferSpeechStarted===");
80            voiceAIClient.clearMediaBuffer();
81            ttsPlayer?.clearBuffer();
82        });
83
84        // ---------------------- Log all other events for debugging -----------------------
85        [
86            OpenAI.RealtimeAPIEvents.ResponseCreated,
87            OpenAI.RealtimeAPIEvents.ResponseDone,
88            OpenAI.RealtimeAPIEvents.ResponseOutputTextDelta,
89            OpenAI.RealtimeAPIEvents.ConnectorInformation,
90            OpenAI.RealtimeAPIEvents.HTTPResponse,
91            OpenAI.RealtimeAPIEvents.WebSocketError,
92            OpenAI.RealtimeAPIEvents.Unknown,
93            OpenAI.Events.WebSocketMediaStarted,
94            OpenAI.Events.WebSocketMediaEnded,
95        ].forEach((eventName) => {
96            voiceAIClient.addEventListener(eventName, (event) => {
97                Logger.write(`===${event.name}===`);
98                if (event?.data) Logger.write(JSON.stringify(event.data));
99            });
100        });
101    } catch (error) {
102        Logger.write("===UNHANDLED_ERROR===");
103        Logger.write(error);
104        voiceAIClient?.close();
105        VoxEngine.terminate();
106    }
107});