OpenAI Realtime 低延迟,多模态,实时语音交互

https://github.com/openai/openai-realtime-console.git

# 拉取仓库代码git clone https://github.com/openai/openai-realtime-console.git

cd openai-realtime-console# 安装依赖包npm i

# 启动服务npm start

import WebSocket from "ws";
const url = "wss://api.openai.com/v1/realtime?model=gpt-4o-realtime-preview-2024-10-01";const ws = new WebSocket(url, {headers: {"Authorization": "Bearer " + process.env.OPENAI_API_KEY,"OpenAI-Beta": "realtime=v1",},});
ws.on("open", function open() {console.log("Connected to server.");ws.send(JSON.stringify({type: "response.create",response: {modalities: ["text"],instructions: "Please assist the user.",}}));});
ws.on("message", function incoming(message) {console.log(JSON.parse(message.toString()));});

const event = {type: 'conversation.item.create',item: {type: 'message',role: 'user',content: [{type: 'input_text',text: 'Hello!'}]}};ws.send(JSON.stringify(event));ws.send(JSON.stringify({type: 'response.create'}));

import fs from 'fs';import decodeAudio from 'audio-decode';
// Converts Float32Array of audio data to PCM16 ArrayBufferfunction floatTo16BitPCM(float32Array) {const buffer = new ArrayBuffer(float32Array.length * 2);const view = new DataView(buffer);let offset = 0;for (let i = 0; i < float32Array.length; i++, offset += 2) {let s = Math.max(-1, Math.min(1, float32Array[i]));view.setInt16(offset, s < 0 ? s * 0x8000 : s * 0x7fff, true);}return buffer;}
// Converts a Float32Array to base64-encoded PCM16 database64EncodeAudio(float32Array) {const arrayBuffer = floatTo16BitPCM(float32Array);let binary = '';let bytes = new Uint8Array(arrayBuffer);const chunkSize = 0x8000; // 32KB chunk sizefor (let i = 0; i < bytes.length; i += chunkSize) {let chunk = bytes.subarray(i, i + chunkSize);binary += String.fromCharCode.apply(null, chunk);}return btoa(binary);}
// Using the "audio-decode" library to get raw audio bytesconst myAudio = fs.readFileSync('./path/to/audio.wav');const audioBuffer = await decodeAudio(myAudio);const channelData = audioBuffer.getChannelData(0); // only accepts monoconst base64AudioData = base64EncodeAudio(channelData);
const event = {type: 'conversation.item.create',item: {type: 'message',role: 'user',content: [{type: 'input_audio',audio: base64AudioData}]}};ws.send(JSON.stringify(event));ws.send(JSON.stringify({type: 'response.create'}));

https://platform.openai.com/docs/guides/realtime