OpenAI Realtime

Example: Function calling

View as Markdown

This example answers an inbound Voximplant call, connects it to OpenAI Realtime, and handles function calls in VoxEngine.

It includes three tools:

  • get_weather
  • hang_up
  • warm_transfer

⬇️ Jump to the Full VoxEngine scenario.

Prerequisites

Session setup

The scenario uses sessionUpdate to define:

  • Realtime model instructions
  • server_vad turn detection
  • tool schemas (tools)
  • tool_choice: "auto"

Tool definitions are declared in SESSION_CONFIG.session.tools and sent right after SessionCreated.

Connect call audio

After SessionUpdated, the example bridges call audio to OpenAI:

Connect call audio
1VoxEngine.sendMediaBetween(call, voiceAIClient);
2voiceAIClient.responseCreate({ instructions: "Hello! How can I help today?" });

Function calling flow

The example listens for OpenAI.RealtimeAPIEvents.ResponseFunctionCallArgumentsDone, parses arguments, and returns tool output with conversationItemCreate (type: "function_call_output"), then calls responseCreate so the assistant continues.

get_weather

Returns a stub weather payload for the requested location.

hang_up

Sets pendingHangup = true, returns hangup_scheduled, and hangs up after assistant audio completes (ResponseOutputAudioDone or WebSocketMediaEnded).

warm_transfer

Places a PSTN leg, plays a brief intro to the callee, then bridges the original caller to the transfer leg after a delay.

Barge-in

The scenario clears buffered model audio when caller speech starts:

Barge-in
1voiceAIClient.addEventListener(
2  OpenAI.RealtimeAPIEvents.InputAudioBufferSpeechStarted,
3  () => {
4    voiceAIClient.clearMediaBuffer();
5  }
6);

Notes

  • Tool implementations are demo stubs. Replace with real APIs and transfer logic for production.
  • The warm transfer demo uses a default destination if the model does not provide one.

See the VoxEngine API Reference for more details.

Full VoxEngine scenario

voxeengine-openai-function-calling.js
1/**
2 * Voximplant + OpenAI Realtime API connector demo
3 * Scenario: answer an incoming call and handle OpenAI function calls.
4 */
5
6require(Modules.OpenAI);
7require(Modules.ApplicationStorage);
8
9const SYSTEM_PROMPT = `
10You are Voxi, a helpful phone assistant.
11Keep responses short and telephony-friendly.
12If the caller asks for weather, call get_weather.
13If the caller wants to end the call, say a brief goodbye and call hang_up.
14If the caller asks for a warm transfer, call warm_transfer with destination_number.
15`;
16
17const WEATHER_TOOL = "get_weather";
18const HANGUP_TOOL = "hang_up";
19const WARM_TRANSFER_TOOL = "warm_transfer";
20
21const DEFAULT_TRANSFER_NUMBER = "+18889277255";
22const DEFAULT_TRANSFER_DELAY_MS = 3000;
23const DEFAULT_TRANSFER_GREETING =
24    "Hi, this is Voxi. I'm warm transferring a caller. Please hold for a brief note, then I'll connect you.";
25
26const SESSION_CONFIG = {
27    session: {
28        type: "realtime",
29        instructions: SYSTEM_PROMPT,
30        voice: "alloy",
31        output_modalities: ["audio"],
32        turn_detection: {type: "server_vad", interrupt_response: true},
33        tools: [
34            {
35                type: "function",
36                name: WEATHER_TOOL,
37                description: "Get the weather for a given location",
38                parameters: {
39                    type: "object",
40                    properties: {
41                        location: {type: "string"},
42                    },
43                    required: ["location"],
44                },
45            },
46            {
47                type: "function",
48                name: HANGUP_TOOL,
49                description: "Hang up the call",
50                parameters: {
51                    type: "object",
52                    properties: {},
53                    required: [],
54                },
55            },
56            {
57                type: "function",
58                name: WARM_TRANSFER_TOOL,
59                description: "Warm transfer the caller to a phone number",
60                parameters: {
61                    type: "object",
62                    properties: {
63                        destination_number: {type: "string"},
64                        message: {type: "string"},
65                        delay_ms: {type: "integer"},
66                    },
67                    required: [],
68                },
69            },
70        ],
71        tool_choice: "auto",
72    },
73};
74
75VoxEngine.addEventListener(AppEvents.CallAlerting, async ({call}) => {
76    let voiceAIClient;
77    let transferCall;
78    let transferInProgress = false;
79    let pendingHangup = false;
80
81    call.addEventListener(CallEvents.Disconnected, () => VoxEngine.terminate());
82    call.addEventListener(CallEvents.Failed, () => VoxEngine.terminate());
83
84    try {
85        call.answer();
86        // call.record({hd_audio: true, stereo: true}); // Optional: record the call
87
88        voiceAIClient = await OpenAI.createRealtimeAPIClient({
89            apiKey: (await ApplicationStorage.get("OPENAI_API_KEY")).value,
90            model: "gpt-realtime",
91            onWebSocketClose: (event) => {
92                Logger.write("===OpenAI.WebSocket.Close===");
93                if (event) Logger.write(JSON.stringify(event));
94                VoxEngine.terminate();
95            },
96        });
97
98        voiceAIClient.addEventListener(OpenAI.RealtimeAPIEvents.SessionCreated, () => {
99            voiceAIClient.sessionUpdate(SESSION_CONFIG);
100        });
101
102        voiceAIClient.addEventListener(OpenAI.RealtimeAPIEvents.SessionUpdated, () => {
103            VoxEngine.sendMediaBetween(call, voiceAIClient);
104            voiceAIClient.responseCreate({instructions: "Hello! How can I help today?"});
105        });
106
107        voiceAIClient.addEventListener(
108            OpenAI.RealtimeAPIEvents.InputAudioBufferSpeechStarted,
109            () => {
110                Logger.write("===BARGE-IN: OpenAI.InputAudioBufferSpeechStarted===");
111                voiceAIClient.clearMediaBuffer();
112            }
113        );
114
115        // Handle function calls
116        voiceAIClient.addEventListener(
117            OpenAI.RealtimeAPIEvents.ResponseFunctionCallArgumentsDone,
118            async (event) => {
119                const payload = event?.data?.payload || event?.data || {};
120                const toolName = payload.name || payload.tool_name;
121                const toolCallId = payload.call_id || payload.callId;
122                const rawArgs = payload.arguments;
123
124                if (!toolName || !toolCallId) {
125                    Logger.write("===TOOL_CALL_MISSING_FIELDS===");
126                    Logger.write(JSON.stringify(payload));
127                    return;
128                }
129
130                let args = {};
131                if (typeof rawArgs === "string") {
132                    try {
133                        args = JSON.parse(rawArgs);
134                    } catch (error) {
135                        Logger.write("===TOOL_ARGS_PARSE_ERROR===");
136                        Logger.write(rawArgs);
137                        Logger.write(error);
138                    }
139                } else if (rawArgs && typeof rawArgs === "object") {
140                    args = rawArgs;
141                }
142
143                Logger.write("===TOOL_CALL_RECEIVED===");
144                Logger.write(JSON.stringify({toolName, args}));
145
146                if (toolName === WEATHER_TOOL) {
147                    const location = args.location || "Unknown";
148                    const result = {
149                        location,
150                        temperature_f: 72,
151                        condition: "sunny",
152                    };
153                    voiceAIClient.conversationItemCreate({
154                        item: {
155                            type: "function_call_output",
156                            call_id: toolCallId,
157                            output: JSON.stringify(result),
158                        },
159                    });
160                    voiceAIClient.responseCreate({});
161                    return;
162                }
163
164                if (toolName === HANGUP_TOOL) {
165                    pendingHangup = true;
166                    voiceAIClient.conversationItemCreate({
167                        item: {
168                            type: "function_call_output",
169                            call_id: toolCallId,
170                            output: JSON.stringify({status: "hangup_scheduled"}),
171                        },
172                    });
173                    voiceAIClient.responseCreate({});
174                    return;
175                }
176
177                if (toolName === WARM_TRANSFER_TOOL) {
178                    if (transferInProgress) {
179                        voiceAIClient.conversationItemCreate({
180                            item: {
181                                type: "function_call_output",
182                                call_id: toolCallId,
183                                output: JSON.stringify({status: "transfer_already_in_progress"}),
184                            },
185                        });
186                        voiceAIClient.responseCreate({});
187                        return;
188                    }
189
190                    transferInProgress = true;
191
192                    const destination =
193                        args.destination_number ||
194                        args.destination ||
195                        args.phone_number ||
196                        args.number ||
197                        DEFAULT_TRANSFER_NUMBER;
198                    const delayMs = Number.isFinite(args.delay_ms)
199                        ? args.delay_ms
200                        : DEFAULT_TRANSFER_DELAY_MS;
201                    const message = args.message || DEFAULT_TRANSFER_GREETING;
202
203                    try {
204                        transferCall = VoxEngine.callPSTN(destination, call.callerid());
205                        // transferCall = VoxEngine.callUser({username: destination, callerid: call.callerid()});
206                        // transferCall = VoxEngine.callSIP(`sip:${destination}@your-sip-domain`, call.callerid());
207                        // transferCall = VoxEngine.callWhatsappUser({number: destination, callerid: call.callerid()});
208
209                        transferCall.addEventListener(CallEvents.Connected, () => {
210                            Logger.write(`===WARM_TRANSFER_CONNECTED=== ${destination}`);
211                            transferCall.say(message);
212
213                            setTimeout(() => {
214                                try {
215                                    voiceAIClient.clearMediaBuffer();
216                                    call.stopMediaTo(voiceAIClient);
217                                    voiceAIClient.stopMediaTo(call);
218                                    VoxEngine.sendMediaBetween(call, transferCall);
219                                    voiceAIClient.close();
220                                    Logger.write("===WARM_TRANSFER_BRIDGED===");
221                                } catch (bridgeError) {
222                                    Logger.write("===WARM_TRANSFER_BRIDGE_ERROR===");
223                                    Logger.write(bridgeError);
224                                }
225                            }, delayMs);
226                        });
227
228                        transferCall.addEventListener(CallEvents.Failed, (event) => {
229                            Logger.write("===WARM_TRANSFER_FAILED===");
230                            Logger.write(JSON.stringify(event));
231                            transferInProgress = false;
232                        });
233
234                        voiceAIClient.conversationItemCreate({
235                            item: {
236                                type: "function_call_output",
237                                call_id: toolCallId,
238                                output: JSON.stringify({
239                                    status: "transfer_started",
240                                    destination,
241                                    delay_ms: delayMs,
242                                }),
243                            },
244                        });
245                        voiceAIClient.responseCreate({});
246                    } catch (transferError) {
247                        Logger.write("===WARM_TRANSFER_ERROR===");
248                        Logger.write(transferError);
249                        transferInProgress = false;
250                        voiceAIClient.conversationItemCreate({
251                            item: {
252                                type: "function_call_output",
253                                call_id: toolCallId,
254                                output: JSON.stringify({error: "warm_transfer_failed"}),
255                            },
256                        });
257                        voiceAIClient.responseCreate({});
258                    }
259                    return;
260                }
261
262                voiceAIClient.conversationItemCreate({
263                    item: {
264                        type: "function_call_output",
265                        call_id: toolCallId,
266                        output: JSON.stringify({error: `Unhandled tool: ${toolName}`}),
267                    },
268                });
269                voiceAIClient.responseCreate({});
270            }
271        );
272
273        voiceAIClient.addEventListener(OpenAI.RealtimeAPIEvents.ResponseOutputAudioDone, () => {
274            if (!pendingHangup) return;
275            Logger.write("===HANGUP_AFTER_AGENT_AUDIO===");
276            pendingHangup = false;
277            call.hangup();
278        });
279
280        voiceAIClient.addEventListener(OpenAI.Events.WebSocketMediaEnded, () => {
281            if (!pendingHangup) return;
282            Logger.write("===HANGUP_AFTER_MEDIA_ENDED===");
283            pendingHangup = false;
284            call.hangup();
285        });
286
287        // Consolidated "log-only" handlers
288        [
289            OpenAI.RealtimeAPIEvents.ResponseCreated,
290            OpenAI.RealtimeAPIEvents.ResponseDone,
291            OpenAI.RealtimeAPIEvents.ResponseOutputAudioTranscriptDone,
292            OpenAI.RealtimeAPIEvents.ResponseFunctionCallArgumentsDelta,
293            OpenAI.RealtimeAPIEvents.ConnectorInformation,
294            OpenAI.RealtimeAPIEvents.HTTPResponse,
295            OpenAI.RealtimeAPIEvents.WebSocketError,
296            OpenAI.RealtimeAPIEvents.Unknown,
297            OpenAI.Events.WebSocketMediaStarted,
298            OpenAI.Events.WebSocketMediaEnded,
299        ].forEach((eventName) => {
300            voiceAIClient.addEventListener(eventName, (event) => {
301                Logger.write(`===${event.name}===`);
302                if (event?.data) Logger.write(JSON.stringify(event.data));
303            });
304        });
305    } catch (error) {
306        Logger.write("===UNHANDLED_ERROR===");
307        Logger.write(error);
308        voiceAIClient?.close();
309        VoxEngine.terminate();
310    }
311});