仿生人AI服务端测试网页

2025-11-05 18:05:09 +08:00
parent 179604931d
commit f50521e548
14 changed files with 6521 additions and 0 deletions
@@ -0,0 +1,149 @@
+import BlockingQueue from './utils/BlockingQueue.js';
+import { log } from './utils/logger.js';
+
+// 音频流播放上下文类
+export class StreamingContext {
+    constructor(opusDecoder, audioContext, sampleRate, channels, minAudioDuration) {
+        this.opusDecoder = opusDecoder;
+        this.audioContext = audioContext;
+
+        // 音频参数
+        this.sampleRate = sampleRate;
+        this.channels = channels;
+        this.minAudioDuration = minAudioDuration;
+
+        // 初始化队列和状态
+        this.queue = [];          // 已解码的PCM队列。正在播放
+        this.activeQueue = new BlockingQueue(); // 已解码的PCM队列。准备播放
+        this.pendingAudioBufferQueue = [];  // 待处理的缓存队列
+        this.audioBufferQueue = new BlockingQueue();  // 缓存队列
+        this.playing = false;     // 是否正在播放
+        this.endOfStream = false; // 是否收到结束信号
+        this.source = null;       // 当前音频源
+        this.totalSamples = 0;    // 累积的总样本数
+        this.lastPlayTime = 0;    // 上次播放的时间戳
+    }
+
+    // 缓存音频数组
+    pushAudioBuffer(item) {
+        this.audioBufferQueue.enqueue(...item);
+    }
+
+    // 获取需要处理缓存队列，单线程：在audioBufferQueue一直更新的状态下不会出现安全问题
+    async getPendingAudioBufferQueue() {
+        // 原子交换 + 清空
+        [this.pendingAudioBufferQueue, this.audioBufferQueue] = [await this.audioBufferQueue.dequeue(), new BlockingQueue()];
+    }
+
+    // 获取正在播放已解码的PCM队列，单线程：在activeQueue一直更新的状态下不会出现安全问题
+    async getQueue(minSamples) {
+        let TepArray = [];
+        const num = minSamples - this.queue.length > 0 ? minSamples - this.queue.length : 1;
+        // 原子交换 + 清空
+        [TepArray, this.activeQueue] = [await this.activeQueue.dequeue(num), new BlockingQueue()];
+        this.queue.push(...TepArray);
+    }
+
+    // 将Int16音频数据转换为Float32音频数据
+    convertInt16ToFloat32(int16Data) {
+        const float32Data = new Float32Array(int16Data.length);
+        for (let i = 0; i < int16Data.length; i++) {
+            // 将[-32768,32767]范围转换为[-1,1]
+            float32Data[i] = int16Data[i] / (int16Data[i] < 0 ? 0x8000 : 0x7FFF);
+        }
+        return float32Data;
+    }
+
+    // 将Opus数据解码为PCM
+    async decodeOpusFrames() {
+        if (!this.opusDecoder) {
+            log('Opus解码器未初始化，无法解码', 'error');
+            return;
+        } else {
+            log('Opus解码器启动', 'info');
+        }
+
+        while (true) {
+            let decodedSamples = [];
+            for (const frame of this.pendingAudioBufferQueue) {
+                try {
+                    // 使用Opus解码器解码
+                    const frameData = this.opusDecoder.decode(frame);
+                    if (frameData && frameData.length > 0) {
+                        // 转换为Float32
+                        const floatData = this.convertInt16ToFloat32(frameData);
+                        // 使用循环替代展开运算符
+                        for (let i = 0; i < floatData.length; i++) {
+                            decodedSamples.push(floatData[i]);
+                        }
+                    }
+                } catch (error) {
+                    log("Opus解码失败: " + error.message, 'error');
+                }
+            }
+
+            if (decodedSamples.length > 0) {
+                // 使用循环替代展开运算符
+                for (let i = 0; i < decodedSamples.length; i++) {
+                    this.activeQueue.enqueue(decodedSamples[i]);
+                }
+                this.totalSamples += decodedSamples.length;
+            } else {
+                log('没有成功解码的样本', 'warning');
+            }
+            await this.getPendingAudioBufferQueue();
+        }
+    }
+
+    // 开始播放音频
+    async startPlaying() {
+        while (true) {
+            // 如果累积了至少0.3秒的音频，开始播放
+            const minSamples = this.sampleRate * this.minAudioDuration * 3;
+            if (!this.playing && this.queue.length < minSamples) {
+                await this.getQueue(minSamples);
+            }
+            this.playing = true;
+            while (this.playing && this.queue.length) {
+                // 创建新的音频缓冲区
+                const minPlaySamples = Math.min(this.queue.length, this.sampleRate);
+                const currentSamples = this.queue.splice(0, minPlaySamples);
+
+                const audioBuffer = this.audioContext.createBuffer(this.channels, currentSamples.length, this.sampleRate);
+                audioBuffer.copyToChannel(new Float32Array(currentSamples), 0);
+
+                // 创建音频源
+                this.source = this.audioContext.createBufferSource();
+                this.source.buffer = audioBuffer;
+
+                // 创建增益节点用于平滑过渡
+                const gainNode = this.audioContext.createGain();
+
+                // 应用淡入淡出效果避免爆音
+                const fadeDuration = 0.02; // 20毫秒
+                gainNode.gain.setValueAtTime(0, this.audioContext.currentTime);
+                gainNode.gain.linearRampToValueAtTime(1, this.audioContext.currentTime + fadeDuration);
+
+                const duration = audioBuffer.duration;
+                if (duration > fadeDuration * 2) {
+                    gainNode.gain.setValueAtTime(1, this.audioContext.currentTime + duration - fadeDuration);
+                    gainNode.gain.linearRampToValueAtTime(0, this.audioContext.currentTime + duration);
+                }
+
+                // 连接节点并开始播放
+                this.source.connect(gainNode);
+                gainNode.connect(this.audioContext.destination);
+
+                this.lastPlayTime = this.audioContext.currentTime;
+                log(`开始播放 ${currentSamples.length} 个样本，约 ${(currentSamples.length / this.sampleRate).toFixed(2)} 秒`, 'info');
+                this.source.start();
+            }
+            await this.getQueue(minSamples);
+        }
+    }
+}
+
+// 创建streamingContext实例的工厂函数
+export function createStreamingContext(opusDecoder, audioContext, sampleRate, channels, minAudioDuration) {
+    return new StreamingContext(opusDecoder, audioContext, sampleRate, channels, minAudioDuration);
+}
@@ -0,0 +1,49 @@
+// DOM元素
+const connectButton = document.getElementById('connectButton');
+const serverUrlInput = document.getElementById('serverUrl');
+const connectionStatus = document.getElementById('connectionStatus');
+const messageInput = document.getElementById('messageInput');
+const sendTextButton = document.getElementById('sendTextButton');
+const recordButton = document.getElementById('recordButton');
+const stopButton = document.getElementById('stopButton');
+// 会话记录
+const conversationDiv = document.getElementById('conversation');
+const logContainer = document.getElementById('logContainer');
+let visualizerCanvas = document.getElementById('audioVisualizer');
+
+// ota 是否连接成功，修改成对应的样式
+export function otaStatusStyle (flan) {
+    if(flan){
+        document.getElementById('otaStatus').textContent = 'ota已连接';
+        document.getElementById('otaStatus').style.color = 'green';
+    }else{
+        document.getElementById('otaStatus').textContent = 'ota未连接';
+        document.getElementById('otaStatus').style.color = 'red';
+    }
+}
+
+// ota 是否连接成功，修改成对应的样式
+export function getLogContainer (flan) {
+    return  logContainer;
+}
+
+// 更新Opus库状态显示
+export function updateScriptStatus(message, type) {
+    const statusElement = document.getElementById('scriptStatus');
+    if (statusElement) {
+        statusElement.textContent = message;
+        statusElement.className = `script-status ${type}`;
+        statusElement.style.display = 'block';
+        statusElement.style.width = 'auto';
+    }
+}
+
+// 添加消息到会话记录
+export function addMessage(text, isUser = false) {
+    const messageDiv = document.createElement('div');
+    messageDiv.className = `message ${isUser ? 'user' : 'server'}`;
+    messageDiv.textContent = text;
+    conversationDiv.appendChild(messageDiv);
+    conversationDiv.scrollTop = conversationDiv.scrollHeight;
+}
+
@@ -0,0 +1,186 @@
+import { log } from './utils/logger.js';
+import { updateScriptStatus } from './document.js'
+
+
+// 检查Opus库是否已加载
+export function checkOpusLoaded() {
+    try {
+        // 检查Module是否存在（本地库导出的全局变量）
+        if (typeof Module === 'undefined') {
+            throw new Error('Opus库未加载，Module对象不存在');
+        }
+
+        // 尝试先使用Module.instance（libopus.js最后一行导出方式）
+        if (typeof Module.instance !== 'undefined' && typeof Module.instance._opus_decoder_get_size === 'function') {
+            // 使用Module.instance对象替换全局Module对象
+            window.ModuleInstance = Module.instance;
+            log('Opus库加载成功（使用Module.instance）', 'success');
+            updateScriptStatus('Opus库加载成功', 'success');
+
+            // 3秒后隐藏状态
+            const statusElement = document.getElementById('scriptStatus');
+            if (statusElement) statusElement.style.display = 'none';
+            return;
+        }
+
+        // 如果没有Module.instance，检查全局Module函数
+        if (typeof Module._opus_decoder_get_size === 'function') {
+            window.ModuleInstance = Module;
+            log('Opus库加载成功（使用全局Module）', 'success');
+            updateScriptStatus('Opus库加载成功', 'success');
+
+            // 3秒后隐藏状态
+            const statusElement = document.getElementById('scriptStatus');
+            if (statusElement) statusElement.style.display = 'none';
+            return;
+        }
+
+        throw new Error('Opus解码函数未找到，可能Module结构不正确');
+    } catch (err) {
+        log(`Opus库加载失败，请检查libopus.js文件是否存在且正确: ${err.message}`, 'error');
+        updateScriptStatus('Opus库加载失败，请检查libopus.js文件是否存在且正确', 'error');
+    }
+}
+
+
+// 创建一个Opus编码器
+let opusEncoder = null;
+export function initOpusEncoder() {
+    try {
+        if (opusEncoder) {
+            return opusEncoder; // 已经初始化过
+        }
+
+        if (!window.ModuleInstance) {
+            log('无法创建Opus编码器：ModuleInstance不可用', 'error');
+            return;
+        }
+
+        // 初始化一个Opus编码器
+        const mod = window.ModuleInstance;
+        const sampleRate = 16000; // 16kHz采样率
+        const channels = 1;       // 单声道
+        const application = 2048; // OPUS_APPLICATION_VOIP = 2048
+
+        // 创建编码器
+        opusEncoder = {
+            channels: channels,
+            sampleRate: sampleRate,
+            frameSize: 960, // 60ms @ 16kHz = 60 * 16 = 960 samples
+            maxPacketSize: 4000, // 最大包大小
+            module: mod,
+
+            // 初始化编码器
+            init: function () {
+                try {
+                    // 获取编码器大小
+                    const encoderSize = mod._opus_encoder_get_size(this.channels);
+                    log(`Opus编码器大小: ${encoderSize}字节`, 'info');
+
+                    // 分配内存
+                    this.encoderPtr = mod._malloc(encoderSize);
+                    if (!this.encoderPtr) {
+                        throw new Error("无法分配编码器内存");
+                    }
+
+                    // 初始化编码器
+                    const err = mod._opus_encoder_init(
+                        this.encoderPtr,
+                        this.sampleRate,
+                        this.channels,
+                        application
+                    );
+
+                    if (err < 0) {
+                        throw new Error(`Opus编码器初始化失败: ${err}`);
+                    }
+
+                    // 设置位率 (16kbps)
+                    mod._opus_encoder_ctl(this.encoderPtr, 4002, 16000); // OPUS_SET_BITRATE
+
+                    // 设置复杂度 (0-10, 越高质量越好但CPU使用越多)
+                    mod._opus_encoder_ctl(this.encoderPtr, 4010, 5);     // OPUS_SET_COMPLEXITY
+
+                    // 设置使用DTX (不传输静音帧)
+                    mod._opus_encoder_ctl(this.encoderPtr, 4016, 1);     // OPUS_SET_DTX
+
+                    log("Opus编码器初始化成功", 'success');
+                    return true;
+                } catch (error) {
+                    if (this.encoderPtr) {
+                        mod._free(this.encoderPtr);
+                        this.encoderPtr = null;
+                    }
+                    log(`Opus编码器初始化失败: ${error.message}`, 'error');
+                    return false;
+                }
+            },
+
+            // 编码PCM数据为Opus
+            encode: function (pcmData) {
+                if (!this.encoderPtr) {
+                    if (!this.init()) {
+                        return null;
+                    }
+                }
+
+                try {
+                    const mod = this.module;
+
+                    // 为PCM数据分配内存
+                    const pcmPtr = mod._malloc(pcmData.length * 2); // 2字节/int16
+
+                    // 将PCM数据复制到HEAP
+                    for (let i = 0; i < pcmData.length; i++) {
+                        mod.HEAP16[(pcmPtr >> 1) + i] = pcmData[i];
+                    }
+
+                    // 为输出分配内存
+                    const outPtr = mod._malloc(this.maxPacketSize);
+
+                    // 进行编码
+                    const encodedLen = mod._opus_encode(
+                        this.encoderPtr,
+                        pcmPtr,
+                        this.frameSize,
+                        outPtr,
+                        this.maxPacketSize
+                    );
+
+                    if (encodedLen < 0) {
+                        throw new Error(`Opus编码失败: ${encodedLen}`);
+                    }
+
+                    // 复制编码后的数据
+                    const opusData = new Uint8Array(encodedLen);
+                    for (let i = 0; i < encodedLen; i++) {
+                        opusData[i] = mod.HEAPU8[outPtr + i];
+                    }
+
+                    // 释放内存
+                    mod._free(pcmPtr);
+                    mod._free(outPtr);
+
+                    return opusData;
+                } catch (error) {
+                    log(`Opus编码出错: ${error.message}`, 'error');
+                    return null;
+                }
+            },
+
+            // 销毁编码器
+            destroy: function () {
+                if (this.encoderPtr) {
+                    this.module._free(this.encoderPtr);
+                    this.encoderPtr = null;
+                }
+            }
+        };
+
+        opusEncoder.init();
+        return opusEncoder;
+    } catch (error) {
+        log(`创建Opus编码器失败: ${error.message}`, 'error');
+        return false;
+    }
+}
@@ -0,0 +1,98 @@
+export default class BlockingQueue {
+    #items   = [];
+    #waiters = [];          // {resolve, reject, min, timer, onTimeout}
+
+    /* 空队列一次性闸门 */
+    #emptyPromise = null;
+    #emptyResolve = null;
+
+    /* 生产者：把数据塞进去 */
+    enqueue(item, ...restItems) {
+        if (restItems.length === 0) {
+            this.#items.push(item);
+        }
+        // 如果有额外参数，批量处理所有项
+        else {
+            const items = [item, ...restItems].filter(i => i);
+            if (items.length === 0) return;
+            this.#items.push(...items);
+        }
+        // 若有空队列闸门，一次性放行所有等待者
+        if (this.#emptyResolve) {
+            this.#emptyResolve();
+            this.#emptyResolve = null;
+            this.#emptyPromise = null;
+        }
+
+        // 唤醒所有正在等的 waiter
+        this.#wakeWaiters();
+    }
+
+    /* 消费者：min 条或 timeout ms 先到谁 */
+    async dequeue(min = 1, timeout = Infinity, onTimeout = null) {
+        // 1. 若空，等第一次数据到达（所有调用共享同一个 promise）
+        if (this.#items.length === 0) {
+            await this.#waitForFirstItem();
+        }
+
+        // 立即满足
+        if (this.#items.length >= min) {
+            return this.#flush();
+        }
+
+        // 需要等待
+        return new Promise((resolve, reject) => {
+            let timer = null;
+            const waiter = { resolve, reject, min, onTimeout, timer };
+
+            // 超时逻辑
+            if (Number.isFinite(timeout)) {
+                waiter.timer = setTimeout(() => {
+                    this.#removeWaiter(waiter);
+                    if (onTimeout) onTimeout(this.#items.length);
+                    resolve(this.#flush());
+                }, timeout);
+            }
+
+            this.#waiters.push(waiter);
+        });
+    }
+
+    /* 空队列闸门生成器 */
+    #waitForFirstItem() {
+        if (!this.#emptyPromise) {
+            this.#emptyPromise = new Promise(r => (this.#emptyResolve = r));
+        }
+        return this.#emptyPromise;
+    }
+
+    /* 内部：每次数据变动后，检查哪些 waiter 已满足 */
+    #wakeWaiters() {
+        for (let i = this.#waiters.length - 1; i >= 0; i--) {
+            const w = this.#waiters[i];
+            if (this.#items.length >= w.min) {
+                this.#removeWaiter(w);
+                w.resolve(this.#flush());
+            }
+        }
+    }
+
+    #removeWaiter(waiter) {
+        const idx = this.#waiters.indexOf(waiter);
+        if (idx !== -1) {
+            this.#waiters.splice(idx, 1);
+            if (waiter.timer) clearTimeout(waiter.timer);
+        }
+    }
+
+    #flush() {
+        const snapshot = [...this.#items];
+        this.#items.length = 0;
+        return snapshot;
+    }
+
+    /* 当前缓存长度（不含等待者） */
+    get length() {
+        return this.#items.length;
+    }
+}
@@ -0,0 +1,37 @@
+import { getLogContainer } from '../document.js'
+
+const logContainer = getLogContainer();
+// 日志记录函数
+export function log(message, type = 'info') {
+    // 将消息按换行符分割成多行
+    const lines = message.split('\n');
+    const now = new Date();
+    // const timestamp = `[${now.toLocaleTimeString()}] `;
+    const timestamp = `[${now.toLocaleTimeString()}.${now.getMilliseconds().toString().padStart(3, '0')}] `;
+    // 为每一行创建日志条目
+    lines.forEach((line, index) => {
+        const logEntry = document.createElement('div');
+        logEntry.className = `log-entry log-${type}`;
+        // 如果是第一条日志，显示时间戳
+        const prefix = index === 0 ? timestamp : ' '.repeat(timestamp.length);
+        logEntry.textContent = `${prefix}${line}`;
+        // logEntry.textContent = `[${new Date().toLocaleTimeString()}] ${message}`;
+        // logEntry.style 保留起始的空格
+        logEntry.style.whiteSpace = 'pre';
+        if (type === 'error') {
+            logEntry.style.color = 'red';
+        } else if (type === 'debug') {
+            logEntry.style.color = 'gray';
+            return;
+        } else if (type === 'warning') {
+            logEntry.style.color = 'orange';
+        } else if (type === 'success') {
+            logEntry.style.color = 'green';
+        } else {
+            logEntry.style.color = 'black';
+        }
+        logContainer.appendChild(logEntry);
+    });
+
+    logContainer.scrollTop = logContainer.scrollHeight;
+}
@@ -0,0 +1,124 @@
+import { otaStatusStyle } from './document.js';
+import { log } from './utils/logger.js';
+
+// WebSocket 连接
+export async function webSocketConnect(otaUrl, config) {
+
+    if (!validateConfig(config)) {
+        return;
+    }
+
+    // 发送OTA请求并获取返回的websocket信息
+    const otaResult = await sendOTA(otaUrl, config);
+    if (!otaResult) {
+        log('无法从OTA服务器获取信息', 'error');
+        return;
+    }
+
+    // 从OTA响应中提取websocket信息
+    const { websocket } = otaResult;
+    if (!websocket || !websocket.url) {
+        log('OTA响应中缺少websocket信息', 'error');
+        return;
+    }
+
+    // 使用OTA返回的websocket URL
+    let connUrl = new URL(websocket.url);
+
+    // 添加token参数（从OTA响应中获取）
+    if (websocket.token) {
+        if (websocket.token.startsWith("Bearer ")) {
+            connUrl.searchParams.append('authorization', websocket.token);
+        } else {
+            connUrl.searchParams.append('authorization', 'Bearer ' + websocket.token);
+        }
+    }
+
+    // 添加认证参数（保持原有逻辑）
+    connUrl.searchParams.append('device-id', config.deviceId);
+    connUrl.searchParams.append('client-id', config.clientId);
+
+    const wsurl = connUrl.toString()
+
+    log(`正在连接: ${wsurl}`, 'info');
+
+    if (wsurl) {
+        document.getElementById('serverUrl').value = wsurl;
+    }
+
+    return new WebSocket(connUrl.toString());
+}
+
+// 验证配置
+function validateConfig(config) {
+    if (!config.deviceMac) {
+        log('设备MAC地址不能为空', 'error');
+        return false;
+    }
+    if (!config.clientId) {
+        log('客户端ID不能为空', 'error');
+        return false;
+    }
+    return true;
+}
+
+// 判断wsUrl路径是否存在错误
+function validateWsUrl(wsUrl) {
+    if (wsUrl === '') return false;
+    // 检查URL格式
+    if (!wsUrl.startsWith('ws://') && !wsUrl.startsWith('wss://')) {
+        log('URL格式错误，必须以ws://或wss://开头', 'error');
+        return false;
+    }
+    return true
+}
+
+
+// OTA发送请求，验证状态，并返回响应数据
+async function sendOTA(otaUrl, config) {
+    try {
+        const res = await fetch(otaUrl, {
+            method: 'POST',
+            headers: {
+                'Content-Type': 'application/json',
+                'Device-Id': config.deviceId,
+                'Client-Id': config.clientId
+            },
+            body: JSON.stringify({
+                version: 0,
+                uuid: '',
+                application: {
+                    name: 'xiaozhi-web-test',
+                    version: '1.0.0',
+                    compile_time: '2025-04-16 10:00:00',
+                    idf_version: '4.4.3',
+                    elf_sha256: '1234567890abcdef1234567890abcdef1234567890abcdef'
+                },
+                ota: { label: 'xiaozhi-web-test' },
+                board: {
+                    type: 'xiaozhi-web-test',
+                    ssid: 'xiaozhi-web-test',
+                    rssi: 0,
+                    channel: 0,
+                    ip: '192.168.1.1',
+                    mac: config.deviceMac
+                },
+                flash_size: 0,
+                minimum_free_heap_size: 0,
+                mac_address: config.deviceMac,
+                chip_model_name: '',
+                chip_info: { model: 0, cores: 0, revision: 0, features: 0 },
+                partition_table: [{ label: '', type: 0, subtype: 0, address: 0, size: 0 }]
+            })
+        });
+
+        if (!res.ok) throw new Error(`${res.status} ${res.statusText}`);
+
+        const result = await res.json();
+        otaStatusStyle(true)
+        return result; // 返回完整的响应数据
+    } catch (err) {
+        otaStatusStyle(false)
+        return null; // 失败返回null
+    }
+}