Example: OpenAI Realtime on Azure

1
/**
2
 * Voximplant + Azure OpenAI Realtime API connector demo
3
 * Scenario: answer an incoming call and bridge it to Azure OpenAI Realtime.
4
 */
5
6
require(Modules.OpenAI);
7
8
const SYSTEM_PROMPT = `
9
You are Voxi, a helpful voice assistant for Voximplant phone callers.
10
Keep responses short and telephony-friendly (usually 1-2 sentences).
11
`;
12
13
const SESSION_CONFIG = {
14
    session: {
15
        type: "realtime",
16
        instructions: SYSTEM_PROMPT,
17
        voice: "alloy",
18
        output_modalities: ["audio"],
19
        turn_detection: {type: "server_vad", interrupt_response: true},
20
    },
21
};
22
23
VoxEngine.addEventListener(AppEvents.CallAlerting, async ({call}) => {
24
    let voiceAIClient;
25
26
    function terminate(){
27
        voiceAIClient?.close();
28
        VoxEngine.terminate();
29
    }
30
31
    call.addEventListener(CallEvents.Disconnected, terminate);
32
    call.addEventListener(CallEvents.Failed, terminate);
33
34
    try {
35
        call.answer();
36
        // call.record({hd_audio: true, stereo: true}); // Optional: record the call
37
38
        voiceAIClient = await OpenAI.createRealtimeAPIClient({
39
            apiKey: VoxEngine.getSecretValue("AZURE_API_KEY"),          // API key from Azure
40
            baseUrl: VoxEngine.getSecretValue("AZURE_OPENAI_BASE_URL"), // https://{{resource}}.openai.azure.com
41
            model: "gpt-realtime-1.5",                                        // align with deployed model
42
            onWebSocketClose: (event) => {
43
                Logger.write("===AzureOpenAI.WebSocket.Close===");
44
                if (event) Logger.write(JSON.stringify(event));
45
                VoxEngine.terminate();
46
            },
47
        });
48
49
        //---------------------- Event handlers -----------------------
50
        voiceAIClient.addEventListener(OpenAI.RealtimeAPIEvents.SessionCreated, () => {
51
            voiceAIClient.sessionUpdate(SESSION_CONFIG);
52
        });
53
54
        voiceAIClient.addEventListener(OpenAI.RealtimeAPIEvents.SessionUpdated, () => {
55
            VoxEngine.sendMediaBetween(call, voiceAIClient);
56
            voiceAIClient.responseCreate({instructions: "Greet the caller and ask how you can help."});
57
        });
58
59
        // Barge-in: clear buffered audio when the caller starts speaking
60
        voiceAIClient.addEventListener(OpenAI.RealtimeAPIEvents.InputAudioBufferSpeechStarted, () => {
61
            Logger.write("===BARGE-IN: AzureOpenAI.InputAudioBufferSpeechStarted===");
62
            voiceAIClient.clearMediaBuffer();
63
        });
64
65
66
        voiceAIClient.addEventListener(
67
            OpenAI.RealtimeAPIEvents.ConversationItemInputAudioTranscriptionCompleted,
68
            (event) => {
69
                const {transcript} = event.data.payload;
70
                if (transcript) Logger.write(`===USER=== ${transcript}`);
71
            },
72
        );
73
74
        voiceAIClient.addEventListener(
75
            OpenAI.RealtimeAPIEvents.ResponseOutputAudioTranscriptDelta,
76
            (event) => {
77
                const {delta} = event.data.payload;
78
                if (delta) Logger.write(`===AGENT_DELTA=== ${delta}`);
79
            },
80
        );
81
82
        voiceAIClient.addEventListener(
83
            OpenAI.RealtimeAPIEvents.ResponseOutputAudioTranscriptDone,
84
            (event) => {
85
                const {transcript} = event.data.payload;
86
                if (transcript) Logger.write(`===AGENT=== ${transcript}`);
87
            },
88
        );
89
90
        // Consolidated "log-only" handlers - key OpenAI/VoxEngine debugging events
91
        [
92
            OpenAI.RealtimeAPIEvents.ResponseCreated,
93
            OpenAI.RealtimeAPIEvents.ResponseDone,
94
            OpenAI.RealtimeAPIEvents.ResponseOutputAudioDone,
95
            OpenAI.RealtimeAPIEvents.ConnectorInformation,
96
            OpenAI.RealtimeAPIEvents.HTTPResponse,
97
            OpenAI.RealtimeAPIEvents.WebSocketError,
98
            OpenAI.RealtimeAPIEvents.Unknown,
99
            OpenAI.Events.WebSocketMediaStarted,
100
            OpenAI.Events.WebSocketMediaEnded,
101
        ].forEach((eventName) => {
102
            voiceAIClient.addEventListener(eventName, (event) => {
103
                Logger.write(`===${event.name}===`);
104
                if (event?.data) Logger.write(JSON.stringify(event.data));
105
            });
106
        });
107
    } catch (error) {
108
        Logger.write("===UNHANDLED_ERROR===");
109
        Logger.write(error);
110
        terminate();
111
    }
112
});