OpenAI

Example: Function calling

View as Markdown

For the complete documentation index, see llms.txt.

This example answers an inbound Voximplant call, connects it to OpenAI Realtime, and handles function calls in VoxEngine.

It includes three tools:

  • get_weather
  • hang_up
  • warm_transfer

⬇️ Jump to the Full VoxEngine scenario.

Prerequisites

Session setup

The scenario uses sessionUpdate to define:

  • Realtime model instructions
  • server_vad turn detection
  • tool schemas (tools)
  • tool_choice: "auto"

Tool definitions are declared in SESSION_CONFIG.session.tools and sent right after SessionCreated.

Connect call audio

After SessionUpdated, the example bridges call audio to OpenAI:

Connect call audio
1VoxEngine.sendMediaBetween(call, voiceAIClient);
2voiceAIClient.responseCreate({ instructions: "Hello! How can I help today?" });

Function calling flow

The example listens for OpenAI.RealtimeAPIEvents.ResponseFunctionCallArgumentsDone, parses arguments, and returns tool output with conversationItemCreate (type: "function_call_output"), then calls responseCreate so the assistant continues.

get_weather

Returns a stub weather payload for the requested location.

hang_up

Sets pendingHangup = true, returns hangup_scheduled, and hangs up after assistant audio completes (ResponseOutputAudioDone or WebSocketMediaEnded).

warm_transfer

Places a PSTN leg, plays a brief intro to the callee, then bridges the original caller to the transfer leg after a delay.

Barge-in

The scenario clears buffered model audio when caller speech starts:

Barge-in
1voiceAIClient.addEventListener(
2  OpenAI.RealtimeAPIEvents.InputAudioBufferSpeechStarted,
3  () => {
4    voiceAIClient.clearMediaBuffer();
5  }
6);

Notes

  • Tool implementations are demo stubs. Replace with real APIs and transfer logic for production.
  • The warm transfer demo uses a default destination if the model does not provide one.

See the VoxEngine API Reference for more details.

Full VoxEngine scenario

voxeengine-openai-function-calling.js
1/**
2 * Voximplant + OpenAI Realtime API connector demo
3 * Scenario: answer an incoming call and handle OpenAI function calls.
4 */
5
6require(Modules.OpenAI);
7require(Modules.ApplicationStorage);
8
9const SYSTEM_PROMPT = `
10You are Voxi, a helpful phone assistant.
11Keep responses short and telephony-friendly.
12If the caller asks for weather, call get_weather.
13If the caller wants to end the call, say a brief goodbye and call hang_up.
14If the caller asks for a warm transfer, call warm_transfer with destination_number.
15`;
16
17const WEATHER_TOOL = "get_weather";
18const HANGUP_TOOL = "hang_up";
19const WARM_TRANSFER_TOOL = "warm_transfer";
20
21const DEFAULT_TRANSFER_NUMBER = "+18889277255";
22const DEFAULT_TRANSFER_DELAY_MS = 3000;
23const DEFAULT_TRANSFER_GREETING =
24    "Hi, this is Voxi. I'm warm transferring a caller. Please hold for a brief note, then I'll connect you.";
25
26const SESSION_CONFIG = {
27    session: {
28        type: "realtime",
29        instructions: SYSTEM_PROMPT,
30        voice: "alloy",
31        output_modalities: ["audio"],
32        turn_detection: {type: "server_vad", interrupt_response: true},
33        tools: [
34            {
35                type: "function",
36                name: WEATHER_TOOL,
37                description: "Get the weather for a given location",
38                parameters: {
39                    type: "object",
40                    properties: {
41                        location: {type: "string"},
42                    },
43                    required: ["location"],
44                },
45            },
46            {
47                type: "function",
48                name: HANGUP_TOOL,
49                description: "Hang up the call",
50                parameters: {
51                    type: "object",
52                    properties: {},
53                    required: [],
54                },
55            },
56            {
57                type: "function",
58                name: WARM_TRANSFER_TOOL,
59                description: "Warm transfer the caller to a phone number",
60                parameters: {
61                    type: "object",
62                    properties: {
63                        destination_number: {type: "string"},
64                        message: {type: "string"},
65                        delay_ms: {type: "integer"},
66                    },
67                    required: [],
68                },
69            },
70        ],
71        tool_choice: "auto",
72    },
73};
74
75VoxEngine.addEventListener(AppEvents.CallAlerting, async ({call}) => {
76    let voiceAIClient;
77    let transferCall;
78    let transferInProgress = false;
79    let pendingHangup = false;
80
81    call.addEventListener(CallEvents.Disconnected, () => VoxEngine.terminate());
82    call.addEventListener(CallEvents.Failed, () => VoxEngine.terminate());
83
84    try {
85        call.answer();
86        // call.record({hd_audio: true, stereo: true}); // Optional: record the call
87
88        voiceAIClient = await OpenAI.createRealtimeAPIClient({
89            apiKey: (await ApplicationStorage.get("OPENAI_API_KEY")).value,
90            model: "gpt-realtime-1.5",
91            onWebSocketClose: (event) => {
92                Logger.write("===OpenAI.WebSocket.Close===");
93                if (event) Logger.write(JSON.stringify(event));
94                VoxEngine.terminate();
95            },
96        });
97
98        voiceAIClient.addEventListener(OpenAI.RealtimeAPIEvents.SessionCreated, () => {
99            voiceAIClient.sessionUpdate(SESSION_CONFIG);
100        });
101
102        voiceAIClient.addEventListener(OpenAI.RealtimeAPIEvents.SessionUpdated, () => {
103            VoxEngine.sendMediaBetween(call, voiceAIClient);
104            voiceAIClient.responseCreate({instructions: "Hello! How can I help today?"});
105        });
106
107        voiceAIClient.addEventListener(
108            OpenAI.RealtimeAPIEvents.InputAudioBufferSpeechStarted,
109            () => {
110                Logger.write("===BARGE-IN: OpenAI.InputAudioBufferSpeechStarted===");
111                voiceAIClient.clearMediaBuffer();
112            }
113        );
114
115        // Handle function calls
116        voiceAIClient.addEventListener(
117            OpenAI.RealtimeAPIEvents.ResponseFunctionCallArgumentsDone,
118            async (event) => {
119                const payload = event?.data?.payload || event?.data || {};
120                const toolName = payload.name || payload.tool_name;
121                const toolCallId = payload.call_id || payload.callId;
122                const rawArgs = payload.arguments;
123
124                if (!toolName || !toolCallId) {
125                    Logger.write("===TOOL_CALL_MISSING_FIELDS===");
126                    Logger.write(JSON.stringify(payload));
127                    return;
128                }
129
130                let args = {};
131                if (typeof rawArgs === "string") {
132                    try {
133                        args = JSON.parse(rawArgs);
134                    } catch (error) {
135                        Logger.write("===TOOL_ARGS_PARSE_ERROR===");
136                        Logger.write(rawArgs);
137                        Logger.write(error);
138                    }
139                } else if (rawArgs && typeof rawArgs === "object") {
140                    args = rawArgs;
141                }
142
143                Logger.write("===TOOL_CALL_RECEIVED===");
144                Logger.write(JSON.stringify({toolName, args}));
145
146                if (toolName === WEATHER_TOOL) {
147                    const location = args.location || "Unknown";
148                    const result = {
149                        location,
150                        temperature_f: 72,
151                        condition: "sunny",
152                    };
153                    voiceAIClient.conversationItemCreate({
154                        item: {
155                            type: "function_call_output",
156                            call_id: toolCallId,
157                            output: JSON.stringify(result),
158                        },
159                    });
160                    voiceAIClient.responseCreate({});
161                    return;
162                }
163
164                if (toolName === HANGUP_TOOL) {
165                    pendingHangup = true;
166                    voiceAIClient.conversationItemCreate({
167                        item: {
168                            type: "function_call_output",
169                            call_id: toolCallId,
170                            output: JSON.stringify({status: "hangup_scheduled"}),
171                        },
172                    });
173                    voiceAIClient.responseCreate({});
174                    return;
175                }
176
177                if (toolName === WARM_TRANSFER_TOOL) {
178                    if (transferInProgress) {
179                        voiceAIClient.conversationItemCreate({
180                            item: {
181                                type: "function_call_output",
182                                call_id: toolCallId,
183                                output: JSON.stringify({status: "transfer_already_in_progress"}),
184                            },
185                        });
186                        voiceAIClient.responseCreate({});
187                        return;
188                    }
189
190                    transferInProgress = true;
191
192                    const destination =
193                        args.destination_number ||
194                        args.destination ||
195                        args.phone_number ||
196                        args.number ||
197                        DEFAULT_TRANSFER_NUMBER;
198                    const delayMs = Number.isFinite(args.delay_ms)
199                        ? args.delay_ms
200                        : DEFAULT_TRANSFER_DELAY_MS;
201                    const message = args.message || DEFAULT_TRANSFER_GREETING;
202
203                    try {
204                        transferCall = VoxEngine.callPSTN(destination, call.callerid());
205                        // transferCall = VoxEngine.callUser({username: destination, callerid: call.callerid()});
206                        // transferCall = VoxEngine.callSIP(`sip:${destination}@your-sip-domain`, call.callerid());
207                        // transferCall = VoxEngine.callWhatsappUser({number: destination, callerid: call.callerid()});
208
209                        transferCall.addEventListener(CallEvents.Connected, () => {
210                            Logger.write(`===WARM_TRANSFER_CONNECTED=== ${destination}`);
211                            transferCall.say(message);
212
213                            setTimeout(() => {
214                                try {
215                                    voiceAIClient.clearMediaBuffer();
216                                    call.stopMediaTo(voiceAIClient);
217                                    voiceAIClient.stopMediaTo(call);
218                                    VoxEngine.sendMediaBetween(call, transferCall);
219                                    voiceAIClient.close();
220                                    Logger.write("===WARM_TRANSFER_BRIDGED===");
221                                } catch (bridgeError) {
222                                    Logger.write("===WARM_TRANSFER_BRIDGE_ERROR===");
223                                    Logger.write(bridgeError);
224                                }
225                            }, delayMs);
226                        });
227
228                        transferCall.addEventListener(CallEvents.Failed, (event) => {
229                            Logger.write("===WARM_TRANSFER_FAILED===");
230                            Logger.write(JSON.stringify(event));
231                            transferInProgress = false;
232                        });
233
234                        voiceAIClient.conversationItemCreate({
235                            item: {
236                                type: "function_call_output",
237                                call_id: toolCallId,
238                                output: JSON.stringify({
239                                    status: "transfer_started",
240                                    destination,
241                                    delay_ms: delayMs,
242                                }),
243                            },
244                        });
245                        voiceAIClient.responseCreate({});
246                    } catch (transferError) {
247                        Logger.write("===WARM_TRANSFER_ERROR===");
248                        Logger.write(transferError);
249                        transferInProgress = false;
250                        voiceAIClient.conversationItemCreate({
251                            item: {
252                                type: "function_call_output",
253                                call_id: toolCallId,
254                                output: JSON.stringify({error: "warm_transfer_failed"}),
255                            },
256                        });
257                        voiceAIClient.responseCreate({});
258                    }
259                    return;
260                }
261
262                voiceAIClient.conversationItemCreate({
263                    item: {
264                        type: "function_call_output",
265                        call_id: toolCallId,
266                        output: JSON.stringify({error: `Unhandled tool: ${toolName}`}),
267                    },
268                });
269                voiceAIClient.responseCreate({});
270            }
271        );
272
273        voiceAIClient.addEventListener(OpenAI.RealtimeAPIEvents.ResponseOutputAudioDone, () => {
274            if (!pendingHangup) return;
275            Logger.write("===HANGUP_AFTER_AGENT_AUDIO===");
276            pendingHangup = false;
277            call.hangup();
278        });
279
280        voiceAIClient.addEventListener(OpenAI.Events.WebSocketMediaEnded, () => {
281            if (!pendingHangup) return;
282            Logger.write("===HANGUP_AFTER_MEDIA_ENDED===");
283            pendingHangup = false;
284            call.hangup();
285        });
286
287        // Consolidated "log-only" handlers
288        [
289            OpenAI.RealtimeAPIEvents.ResponseCreated,
290            OpenAI.RealtimeAPIEvents.ResponseDone,
291            OpenAI.RealtimeAPIEvents.ResponseOutputAudioTranscriptDone,
292            OpenAI.RealtimeAPIEvents.ResponseFunctionCallArgumentsDelta,
293            OpenAI.RealtimeAPIEvents.ConnectorInformation,
294            OpenAI.RealtimeAPIEvents.HTTPResponse,
295            OpenAI.RealtimeAPIEvents.WebSocketError,
296            OpenAI.RealtimeAPIEvents.Unknown,
297            OpenAI.Events.WebSocketMediaStarted,
298            OpenAI.Events.WebSocketMediaEnded,
299        ].forEach((eventName) => {
300            voiceAIClient.addEventListener(eventName, (event) => {
301                Logger.write(`===${event.name}===`);
302                if (event?.data) Logger.write(JSON.stringify(event.data));
303            });
304        });
305    } catch (error) {
306        Logger.write("===UNHANDLED_ERROR===");
307        Logger.write(error);
308        voiceAIClient?.close();
309        VoxEngine.terminate();
310    }
311});