Example: Chat Completions with Together AI

1
/**
2
 * Bring your own LLM demo: Chat Completions through an OpenAI-compatible API.
3
 * Scenario: answer an incoming call, transcribe caller speech, and respond with an external LLM.
4
 */
5
6
require(Modules.OpenAI);
7
require(Modules.ASR);
8
9
// -------------------- OpenAI-compatible LLM settings --------------------
10
const LLM_BASE_URL = "https://api.together.ai/v1";
11
const LLM_MODEL = "meta-llama/Llama-3.3-70B-Instruct-Turbo";
12
const SYSTEM_PROMPT = "You are a helpful phone assistant. Keep replies short and natural.";
13
14
VoxEngine.addEventListener(AppEvents.CallAlerting, async ({call}) => {
15
    let asr;
16
    let chatClient;
17
    let responseText = "";
18
    let responseCompleted = false;
19
    let isResponding = false;
20
    const messages = [{role: "system", content: SYSTEM_PROMPT}];
21
22
    // Termination function - clean up ASR and the LLM client
23
    const terminate = () => {
24
        asr?.stop();
25
        chatClient?.close();
26
        VoxEngine.terminate();
27
    };
28
29
    // Speak the completed assistant response back to the caller
30
    const playFinalResponse = (text) => {
31
        if (responseCompleted) return;
32
        responseCompleted = true;
33
        isResponding = false;
34
        const finalText = text || responseText || "The LLM responded successfully.";
35
        Logger.write(`===LLM_DONE=== ${finalText}`);
36
        messages.push({role: "assistant", content: finalText});
37
        call.say(finalText);
38
    };
39
40
    // Send the caller's final transcript to the LLM
41
    const askLLM = (input) => {
42
        if (isResponding) {
43
            Logger.write("===SKIP_INPUT_WHILE_RESPONDING===");
44
            Logger.write(input);
45
            return;
46
        }
47
        isResponding = true;
48
        responseCompleted = false;
49
        responseText = "";
50
        messages.push({role: "user", content: input});
51
52
        chatClient.createChatCompletions({
53
            model: LLM_MODEL,
54
            messages,
55
            stream: true,
56
            max_tokens: 80,
57
        });
58
    };
59
60
    call.addEventListener(CallEvents.Disconnected, terminate);
61
    call.addEventListener(CallEvents.Failed, terminate);
62
63
    try {
64
        call.answer();
65
        // call.record({hd_audio: true, stereo: true}); // Optional: record the call
66
67
        // Create an OpenAI-compatible Chat Completions client
68
        chatClient = await OpenAI.createChatCompletionsAPIClient({
69
            apiKey: VoxEngine.getSecretValue("TOGETHER_API_KEY"),
70
            baseUrl: LLM_BASE_URL,
71
            storeContext: false,
72
            onWebSocketClose: (event) => {
73
                Logger.write("===LLM.WebSocket.Close===");
74
                if (event) Logger.write(JSON.stringify(event));
75
                terminate();
76
            },
77
        });
78
79
        // Stream text deltas as they arrive
80
        chatClient.addEventListener(OpenAI.ChatCompletionsAPIEvents.ContentDelta, (event) => {
81
            const delta = event.data.payload.delta;
82
            if (!delta) return;
83
            responseText += delta;
84
            Logger.write(`===LLM_DELTA=== ${delta}`);
85
        });
86
87
        // Final content event from the Chat Completions client
88
        chatClient.addEventListener(OpenAI.ChatCompletionsAPIEvents.ContentDone, (event) => {
89
            playFinalResponse(event.data.payload.content);
90
        });
91
92
        // Raw chunk event used as a fallback completion signal
93
        chatClient.addEventListener(OpenAI.ChatCompletionsAPIEvents.Chunk, (event) => {
94
            const payload = event.data.payload;
95
            const finishReason = payload.choices?.[0]?.finish_reason;
96
            if (finishReason === "stop") playFinalResponse();
97
        });
98
99
        // ---------------------- Error and diagnostic events -----------------------
100
        chatClient.addEventListener(OpenAI.ChatCompletionsAPIEvents.ChatCompletionsAPIError, (event) => {
101
            Logger.write("===ChatCompletions.Error===");
102
            Logger.write(JSON.stringify(event));
103
        });
104
105
        chatClient.addEventListener(OpenAI.ChatCompletionsAPIEvents.ConnectorInformation, (event) => {
106
            Logger.write("===ChatCompletions.ConnectorInformation===");
107
            if (event?.data) Logger.write(JSON.stringify(event.data));
108
        });
109
110
        // Create ASR and send caller audio to it
111
        asr = VoxEngine.createASR({
112
            profile: ASRProfileList.Google.en_US,
113
        });
114
115
        // Send final ASR transcripts to the LLM
116
        asr.addEventListener(ASREvents.Result, (event) => {
117
            if (event.isFinal === false) return;
118
            const text = event.text?.trim();
119
            if (!text) return;
120
            Logger.write(`===USER=== ${text}`);
121
            askLLM(text);
122
        });
123
124
        asr.addEventListener(ASREvents.ASRError, (event) => {
125
            Logger.write("===ASR_ERROR===");
126
            Logger.write(JSON.stringify(event));
127
        });
128
129
        // Bridge caller audio to ASR and greet the caller
130
        call.sendMediaTo(asr);
131
        call.say("Connected to the LLM. Ask me something.");
132
    } catch (error) {
133
        Logger.write("===UNHANDLED_ERROR===");
134
        Logger.write(error);
135
        terminate();
136
    }
137
});