Example: Using Vertex AI

1
/**
2
 * Voximplant + Gemini Live API connector demo
3
 * Scenario: load Vertex AI credentials for Gemini Live API (example only).
4
 *
5
 * Include this scenario BEFORE the main Vertex AI scenario in your routing rule.
6
 */
7
8
// eslint-disable-next-line no-unused-vars
9
var GEMINI_VERTEX_CREDENTIALS = {
10
    type: "service_account",
11
    project_id: "your-gcp-project-id",
12
    private_key_id: "example-private-key-id",
13
    private_key: "-----BEGIN PRIVATE KEY-----\nREDACTED\n-----END PRIVATE KEY-----\n",
14
    client_email: "vertex-express@your-gcp-project-id.iam.gserviceaccount.com",
15
    client_id: "123456789012345678901",
16
    auth_uri: "https://accounts.google.com/o/oauth2/auth",
17
    token_uri: "https://oauth2.googleapis.com/token",
18
    auth_provider_x509_cert_url: "https://www.googleapis.com/oauth2/v1/certs",
19
    client_x509_cert_url: "https://www.googleapis.com/robot/v1/metadata/x509/vertex-express%40your-gcp-project-id.iam.gserviceaccount.com",
20
    universe_domain: "googleapis.com",
21
};
22
23
// This will be global to the next scenarios in the routing rule sequence.

1
VoxEngine.sendMediaBetween(call, geminiLiveAPIClient);

1
geminiLiveAPIClient.sendClientContent({
2
  turns: [{ role: "user", parts: [{ text: "Say hello and ask how you can help." }] }],
3
  turnComplete: true,
4
});

1
/**
2
 * Voximplant + Gemini Live API connector demo
3
 * Scenario: answer an incoming call using Gemini Live API via Vertex AI.
4
 */
5
6
require(Modules.Gemini);
7
require(Modules.ApplicationStorage);
8
9
const SYSTEM_PROMPT = `You are Voxi, a helpful voice assistant for phone callers. Keep responses short and telephony-friendly (usually 1-2 sentences).`;
10
11
// -------------------- Gemini Live API settings --------------------
12
const CONNECT_CONFIG = {
13
    responseModalities: ["AUDIO"],
14
    speechConfig: {
15
        voiceConfig: {
16
            prebuiltVoiceConfig: {voiceName: "Aoede"},
17
        },
18
    },
19
    systemInstruction: {
20
        parts: [{text: SYSTEM_PROMPT}],
21
    },
22
    inputAudioTranscription: {},
23
    outputAudioTranscription: {},
24
};
25
26
VoxEngine.addEventListener(AppEvents.CallAlerting, async ({call}) => {
27
    let voiceAIClient;
28
29
    // Termination functions - add cleanup and logging as needed
30
    call.addEventListener(CallEvents.Disconnected, ()=>VoxEngine.terminate());
31
    call.addEventListener(CallEvents.Failed, ()=>VoxEngine.terminate());
32
33
    try {
34
        call.answer();
35
        // call.record({hd_audio: true, stereo: true}); // optional: call recording
36
37
        voiceAIClient = await Gemini.createLiveAPIClient({
38
            credentials: GEMINI_VERTEX_CREDENTIALS,                           // stored in a different scenario
39
            model: "gemini-live-2.5-flash-preview-native-audio-09-17",
40
            backend: Gemini.Backend.VERTEX_AI,                                // Set the Vertex back-end
41
            project: (await ApplicationStorage.get("GCP_PROJECT_ID")).value,  // required for Vertex
42
            location: (await ApplicationStorage.get("GCP_REGION")).value,     // required for Vertex
43
            connectConfig: CONNECT_CONFIG,
44
            onWebSocketClose: (event) => {
45
                Logger.write("===Gemini.WebSocket.Close===");
46
                if (event) Logger.write(JSON.stringify(event));
47
                VoxEngine.terminate();
48
            },
49
        });
50
51
        // ---------------------- Event handlers -----------------------
52
        // Wait for Gemini setup, then bridge audio and trigger the greeting
53
        voiceAIClient.addEventListener(Gemini.LiveAPIEvents.SetupComplete, () => {
54
            VoxEngine.sendMediaBetween(call, voiceAIClient);
55
            voiceAIClient.sendClientContent({
56
                turns: [{role: "user", parts: [{text: "Say hello and ask how you can help."}]}],
57
                turnComplete: true,
58
            });
59
        });
60
61
        // Capture transcripts + handle barge-in
62
        voiceAIClient.addEventListener(Gemini.LiveAPIEvents.ServerContent, (event) => {
63
            const payload = event?.data?.payload || {};
64
            if (payload.inputTranscription?.text) {
65
                Logger.write(`===USER=== ${payload.inputTranscription.text}`);
66
            }
67
            if (payload.outputTranscription?.text) {
68
                Logger.write(`===AGENT=== ${payload.outputTranscription.text}`);
69
            }
70
            if (payload.interrupted) {
71
                Logger.write("===BARGE-IN=== Gemini.LiveAPIEvents.ServerContent");
72
                voiceAIClient.clearMediaBuffer();
73
            }
74
        });
75
76
        // Log all Gemini events for illustration/debugging
77
        [
78
            Gemini.LiveAPIEvents.SetupComplete,
79
            Gemini.LiveAPIEvents.ServerContent,
80
            Gemini.LiveAPIEvents.ToolCall,
81
            Gemini.LiveAPIEvents.ToolCallCancellation,
82
            Gemini.LiveAPIEvents.ConnectorInformation,
83
            Gemini.LiveAPIEvents.Unknown,
84
            Gemini.Events.WebSocketMediaStarted,
85
            Gemini.Events.WebSocketMediaEnded,
86
        ].forEach((eventName) => {
87
            voiceAIClient.addEventListener(eventName, (event) => {
88
                Logger.write(`===${event.name}===`);
89
                if (event?.data) Logger.write(JSON.stringify(event.data));
90
            });
91
        });
92
    } catch (error) {
93
        Logger.write("===SOMETHING_WENT_WRONG===");
94
        Logger.write(error);
95
        VoxEngine.terminate();
96
    }
97
});