Voice AI ClientsOpenAI

Example: Function calling

View as Markdown

For the complete documentation index, see llms.txt.

This example answers an inbound Voximplant call, connects it to OpenAI Realtime, and handles function calls in VoxEngine.

It includes three tools:

  • get_weather
  • hang_up
  • warm_transfer

⬇️ Jump to the Full VoxEngine scenario.

Prerequisites

Session setup

The scenario uses sessionUpdate to define:

  • Realtime model instructions
  • server_vad turn detection
  • tool schemas (tools)
  • tool_choice: "auto"

Tool definitions are declared in SESSION_CONFIG.session.tools and sent right after SessionCreated.

Connect call audio

After SessionUpdated, the example bridges call audio to OpenAI:

Connect call audio
1VoxEngine.sendMediaBetween(call, voiceAIClient);
2voiceAIClient.responseCreate({ instructions: "Hello! How can I help today?" });

Function calling flow

The example listens for OpenAI.RealtimeAPIEvents.ResponseFunctionCallArgumentsDone, parses arguments, and returns tool output with conversationItemCreate (type: "function_call_output"), then calls responseCreate so the assistant continues.

get_weather

Returns a stub weather payload for the requested location.

hang_up

Sets pendingHangup = true, returns hangup_scheduled, and hangs up after assistant audio completes (ResponseOutputAudioDone or WebSocketMediaEnded).

warm_transfer

Places a PSTN leg, plays a brief intro to the callee, then bridges the original caller to the transfer leg after a delay.

Barge-in

The scenario clears buffered model audio when caller speech starts:

Barge-in
1voiceAIClient.addEventListener(
2  OpenAI.RealtimeAPIEvents.InputAudioBufferSpeechStarted,
3  () => {
4    voiceAIClient.clearMediaBuffer();
5  }
6);

Notes

  • Tool implementations are demo stubs. Replace with real APIs and transfer logic for production.
  • The warm transfer demo uses a default destination if the model does not provide one.

See the VoxEngine API Reference for more details.

Full VoxEngine scenario

voxeengine-openai-function-calling.js
1/**
2 * Voximplant + OpenAI Realtime API connector demo
3 * Scenario: answer an incoming call and handle OpenAI function calls.
4 */
5
6require(Modules.OpenAI);
7const SYSTEM_PROMPT = `
8You are Voxi, a helpful phone assistant.
9Keep responses short and telephony-friendly.
10If the caller asks for weather, call get_weather.
11If the caller wants to end the call, say a brief goodbye and call hang_up.
12If the caller asks for a warm transfer, call warm_transfer with destination_number.
13`;
14
15const WEATHER_TOOL = "get_weather";
16const HANGUP_TOOL = "hang_up";
17const WARM_TRANSFER_TOOL = "warm_transfer";
18
19const DEFAULT_TRANSFER_NUMBER = "+18889277255";
20const DEFAULT_TRANSFER_DELAY_MS = 3000;
21const DEFAULT_TRANSFER_GREETING =
22    "Hi, this is Voxi. I'm warm transferring a caller. Please hold for a brief note, then I'll connect you.";
23
24const SESSION_CONFIG = {
25    session: {
26        type: "realtime",
27        instructions: SYSTEM_PROMPT,
28        voice: "alloy",
29        output_modalities: ["audio"],
30        turn_detection: {type: "server_vad", interrupt_response: true},
31        tools: [
32            {
33                type: "function",
34                name: WEATHER_TOOL,
35                description: "Get the weather for a given location",
36                parameters: {
37                    type: "object",
38                    properties: {
39                        location: {type: "string"},
40                    },
41                    required: ["location"],
42                },
43            },
44            {
45                type: "function",
46                name: HANGUP_TOOL,
47                description: "Hang up the call",
48                parameters: {
49                    type: "object",
50                    properties: {},
51                    required: [],
52                },
53            },
54            {
55                type: "function",
56                name: WARM_TRANSFER_TOOL,
57                description: "Warm transfer the caller to a phone number",
58                parameters: {
59                    type: "object",
60                    properties: {
61                        destination_number: {type: "string"},
62                        message: {type: "string"},
63                        delay_ms: {type: "integer"},
64                    },
65                    required: [],
66                },
67            },
68        ],
69        tool_choice: "auto",
70    },
71};
72
73VoxEngine.addEventListener(AppEvents.CallAlerting, async ({call}) => {
74    let voiceAIClient;
75    let transferCall;
76    let transferInProgress = false;
77    let pendingHangup = false;
78
79    call.addEventListener(CallEvents.Disconnected, () => VoxEngine.terminate());
80    call.addEventListener(CallEvents.Failed, () => VoxEngine.terminate());
81
82    try {
83        call.answer();
84        // call.record({hd_audio: true, stereo: true}); // Optional: record the call
85
86        voiceAIClient = await OpenAI.createRealtimeAPIClient({
87            apiKey: VoxEngine.getSecretValue('OPENAI_API_KEY'),
88            model: "gpt-realtime-1.5",
89            onWebSocketClose: (event) => {
90                Logger.write("===OpenAI.WebSocket.Close===");
91                if (event) Logger.write(JSON.stringify(event));
92                VoxEngine.terminate();
93            },
94        });
95
96        voiceAIClient.addEventListener(OpenAI.RealtimeAPIEvents.SessionCreated, () => {
97            voiceAIClient.sessionUpdate(SESSION_CONFIG);
98        });
99
100        voiceAIClient.addEventListener(OpenAI.RealtimeAPIEvents.SessionUpdated, () => {
101            VoxEngine.sendMediaBetween(call, voiceAIClient);
102            voiceAIClient.responseCreate({instructions: "Hello! How can I help today?"});
103        });
104
105        voiceAIClient.addEventListener(
106            OpenAI.RealtimeAPIEvents.InputAudioBufferSpeechStarted,
107            () => {
108                Logger.write("===BARGE-IN: OpenAI.InputAudioBufferSpeechStarted===");
109                voiceAIClient.clearMediaBuffer();
110            }
111        );
112
113        // Handle function calls
114        voiceAIClient.addEventListener(
115            OpenAI.RealtimeAPIEvents.ResponseFunctionCallArgumentsDone,
116            async (event) => {
117                const payload = event?.data?.payload || event?.data || {};
118                const toolName = payload.name || payload.tool_name;
119                const toolCallId = payload.call_id || payload.callId;
120                const rawArgs = payload.arguments;
121
122                if (!toolName || !toolCallId) {
123                    Logger.write("===TOOL_CALL_MISSING_FIELDS===");
124                    Logger.write(JSON.stringify(payload));
125                    return;
126                }
127
128                let args = {};
129                if (typeof rawArgs === "string") {
130                    try {
131                        args = JSON.parse(rawArgs);
132                    } catch (error) {
133                        Logger.write("===TOOL_ARGS_PARSE_ERROR===");
134                        Logger.write(rawArgs);
135                        Logger.write(error);
136                    }
137                } else if (rawArgs && typeof rawArgs === "object") {
138                    args = rawArgs;
139                }
140
141                Logger.write("===TOOL_CALL_RECEIVED===");
142                Logger.write(JSON.stringify({toolName, args}));
143
144                if (toolName === WEATHER_TOOL) {
145                    const location = args.location || "Unknown";
146                    const result = {
147                        location,
148                        temperature_f: 72,
149                        condition: "sunny",
150                    };
151                    voiceAIClient.conversationItemCreate({
152                        item: {
153                            type: "function_call_output",
154                            call_id: toolCallId,
155                            output: JSON.stringify(result),
156                        },
157                    });
158                    voiceAIClient.responseCreate({});
159                    return;
160                }
161
162                if (toolName === HANGUP_TOOL) {
163                    pendingHangup = true;
164                    voiceAIClient.conversationItemCreate({
165                        item: {
166                            type: "function_call_output",
167                            call_id: toolCallId,
168                            output: JSON.stringify({status: "hangup_scheduled"}),
169                        },
170                    });
171                    voiceAIClient.responseCreate({});
172                    return;
173                }
174
175                if (toolName === WARM_TRANSFER_TOOL) {
176                    if (transferInProgress) {
177                        voiceAIClient.conversationItemCreate({
178                            item: {
179                                type: "function_call_output",
180                                call_id: toolCallId,
181                                output: JSON.stringify({status: "transfer_already_in_progress"}),
182                            },
183                        });
184                        voiceAIClient.responseCreate({});
185                        return;
186                    }
187
188                    transferInProgress = true;
189
190                    const destination =
191                        args.destination_number ||
192                        args.destination ||
193                        args.phone_number ||
194                        args.number ||
195                        DEFAULT_TRANSFER_NUMBER;
196                    const delayMs = Number.isFinite(args.delay_ms)
197                        ? args.delay_ms
198                        : DEFAULT_TRANSFER_DELAY_MS;
199                    const message = args.message || DEFAULT_TRANSFER_GREETING;
200
201                    try {
202                        transferCall = VoxEngine.callPSTN(destination, call.callerid());
203                        // transferCall = VoxEngine.callUser({username: destination, callerid: call.callerid()});
204                        // transferCall = VoxEngine.callSIP(`sip:${destination}@your-sip-domain`, call.callerid());
205                        // transferCall = VoxEngine.callWhatsappUser({number: destination, callerid: call.callerid()});
206
207                        transferCall.addEventListener(CallEvents.Connected, () => {
208                            Logger.write(`===WARM_TRANSFER_CONNECTED=== ${destination}`);
209                            transferCall.say(message);
210
211                            setTimeout(() => {
212                                try {
213                                    voiceAIClient.clearMediaBuffer();
214                                    call.stopMediaTo(voiceAIClient);
215                                    voiceAIClient.stopMediaTo(call);
216                                    VoxEngine.sendMediaBetween(call, transferCall);
217                                    voiceAIClient.close();
218                                    Logger.write("===WARM_TRANSFER_BRIDGED===");
219                                } catch (bridgeError) {
220                                    Logger.write("===WARM_TRANSFER_BRIDGE_ERROR===");
221                                    Logger.write(bridgeError);
222                                }
223                            }, delayMs);
224                        });
225
226                        transferCall.addEventListener(CallEvents.Failed, (event) => {
227                            Logger.write("===WARM_TRANSFER_FAILED===");
228                            Logger.write(JSON.stringify(event));
229                            transferInProgress = false;
230                        });
231
232                        voiceAIClient.conversationItemCreate({
233                            item: {
234                                type: "function_call_output",
235                                call_id: toolCallId,
236                                output: JSON.stringify({
237                                    status: "transfer_started",
238                                    destination,
239                                    delay_ms: delayMs,
240                                }),
241                            },
242                        });
243                        voiceAIClient.responseCreate({});
244                    } catch (transferError) {
245                        Logger.write("===WARM_TRANSFER_ERROR===");
246                        Logger.write(transferError);
247                        transferInProgress = false;
248                        voiceAIClient.conversationItemCreate({
249                            item: {
250                                type: "function_call_output",
251                                call_id: toolCallId,
252                                output: JSON.stringify({error: "warm_transfer_failed"}),
253                            },
254                        });
255                        voiceAIClient.responseCreate({});
256                    }
257                    return;
258                }
259
260                voiceAIClient.conversationItemCreate({
261                    item: {
262                        type: "function_call_output",
263                        call_id: toolCallId,
264                        output: JSON.stringify({error: `Unhandled tool: ${toolName}`}),
265                    },
266                });
267                voiceAIClient.responseCreate({});
268            }
269        );
270
271        voiceAIClient.addEventListener(OpenAI.RealtimeAPIEvents.ResponseOutputAudioDone, () => {
272            if (!pendingHangup) return;
273            Logger.write("===HANGUP_AFTER_AGENT_AUDIO===");
274            pendingHangup = false;
275            call.hangup();
276        });
277
278        voiceAIClient.addEventListener(OpenAI.Events.WebSocketMediaEnded, () => {
279            if (!pendingHangup) return;
280            Logger.write("===HANGUP_AFTER_MEDIA_ENDED===");
281            pendingHangup = false;
282            call.hangup();
283        });
284
285        // Consolidated "log-only" handlers
286        [
287            OpenAI.RealtimeAPIEvents.ResponseCreated,
288            OpenAI.RealtimeAPIEvents.ResponseDone,
289            OpenAI.RealtimeAPIEvents.ResponseOutputAudioTranscriptDone,
290            OpenAI.RealtimeAPIEvents.ResponseFunctionCallArgumentsDelta,
291            OpenAI.RealtimeAPIEvents.ConnectorInformation,
292            OpenAI.RealtimeAPIEvents.HTTPResponse,
293            OpenAI.RealtimeAPIEvents.WebSocketError,
294            OpenAI.RealtimeAPIEvents.Unknown,
295            OpenAI.Events.WebSocketMediaStarted,
296            OpenAI.Events.WebSocketMediaEnded,
297        ].forEach((eventName) => {
298            voiceAIClient.addEventListener(eventName, (event) => {
299                Logger.write(`===${event.name}===`);
300                if (event?.data) Logger.write(JSON.stringify(event.data));
301            });
302        });
303    } catch (error) {
304        Logger.write("===UNHANDLED_ERROR===");
305        Logger.write(error);
306        voiceAIClient?.close();
307        VoxEngine.terminate();
308    }
309});