@re-ai/inner-tool-sdk - npm Package Compare versions

Comparing version 0.2.14 to 0.2.15

dist/entrys/index.d.ts

@@ -13,2 +13,3 @@ import WebSocket from "ws";
 import { NetBridgeService } from "../services/NetBridgeService";
+import { Seamless } from "../services/SeamlessService";
 export declare class ReAITool {
@@ -26,2 +27,3 @@ static sms(options?: RPCClientOptions): SMSService;
     static bridgeAgentSet(id: string, ws: WebSocket): void;
+    static seamless(host: string, roomId?: string): Seamless;
 }

dist/entrys/index.js

@@ -12,2 +12,3 @@ "use strict";
 const NetBridgeService_1 = require("../services/NetBridgeService");
+const SeamlessService_1 = require("../services/SeamlessService");
 class ReAITool {
@@ -65,3 +66,9 @@ static sms(options) {
     }
+    static seamless(host, roomId) {
+        return new SeamlessService_1.Seamless({
+            host: host,
+            roomId: roomId
+        });
+    }
 }
 exports.ReAITool = ReAITool;

dist/services/SeamlessService.d.ts

 /// <reference types="node" />
-import WebSocket, { Data } from "ws";
-import { ISeamless } from "../interfaces/seamless";
-import { SeamlessHooks, SeamlessLanguege, SeamlessModelType, SeamlessOptions } from "../types/seamless";
-export declare class Seamless implements ISeamless {
+import WebSocket from "ws";
+import { SeamlessCallbackParams, SeamlessLangueges, SeamlessModelType, SeamlessOptions } from "../types/seamless";
+import { ITranslate } from "../interfaces/translate";
+export declare class Seamless implements ITranslate {
     host: string;
     ws?: WebSocket;
-    sid?: string;
+    serverId?: string;
     roomId?: string | null;
     msgId?: string;
-    callback?: (hook: SeamlessHooks, id?: string, data?: any) => void;
-    isSpeech: boolean;
+    actionIndex: number;
+    isReady: boolean;
+    callback?: (data: SeamlessCallbackParams) => void;
     constructor(options?: SeamlessOptions);
-    connect(clientId: string, callback: (hook: string) => void): Promise<void>;
-    send(data: any): void;
-    speech(data: Buffer): void;
-    onMessage(callback: (data: any) => void): void;
-    _onMessage(data: Data): void;
-}
-export declare class SeamlessRoom extends Seamless {
-    constructor(options?: SeamlessOptions);
+    ready(clientId: string, callback?: (data: SeamlessCallbackParams) => void): Promise<this>;
+    private send;
+    private _onMessage;
+    private _onMessageInfo;
+    private _onMessageAction;
+    private getJson;
     join(memberId: string): void;
-    targetLanguageSet(lan: SeamlessLanguege, expressive?: boolean | null): void;
+    langSet(lan: SeamlessLangueges, expressive?: boolean | null): void;
     configSet(data: {
         modelType?: SeamlessModelType;
         rate?: number;
+        debug?: boolean;
     }): void;
-    speechStart(lan: SeamlessLanguege, config: {
+    start(lan: SeamlessLangueges, config: {
         modelType?: SeamlessModelType;
         rate?: number;
+        debug?: boolean;
     }): void;
-    speechStop(): void;
+    stop(): void;
+    speech(data: Buffer): void;
+    private floatArrayTo16BitPcmBuffer;
 }

dist/services/SeamlessService.js

@@ -6,3 +6,3 @@ "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.SeamlessRoom = exports.Seamless = void 0;
+exports.Seamless = void 0;
 const ws_1 = __importDefault(require("ws"));
@@ -15,17 +15,25 @@ const Logger_1 = require("../utils/Logger");
     ws;
-    sid;
+    serverId;
     roomId;
     msgId;
+    actionIndex = 0;
+    isReady = false;
     callback;
-    isSpeech = false;
     constructor(options) {
         // address = "/socket.io/?clientID=1c2b94f6-7501-44eb-a4a5-46540072dcfd&EIO=4&transport=websocket"
         this.host = options?.host || process.env.REAI_SEAMLESS_HOST || "wss://seamless.cn.reai.com/seamless/ws";
+        this.roomId = options?.roomId || null;
+        this.callback = options?.onReceive || undefined;
+        this.isReady = false;
     }
-    async connect(clientId, callback) {
+    async ready(clientId, callback) {
         const address = `${this.host}/ws/socket.io/?clientID=${clientId}&EIO=4&transport=websocket`;
+        Logger_1.Logger.info(`[Seamless] connect ${address}`);
         this.ws = new ws_1.default(address);
         this.ws.onopen = (e) => {
             Logger_1.Logger.warn(`[Seamless] open .... ${e.type}`);
-            this.callback = callback;
+            // this.callback = callback
+            if (callback) {
+                this.callback = callback;
+            }
         };
@@ -44,74 +52,141 @@ this.ws.onclose = (e) => {
         };
+        return this;
     }
     send(data) {
-        this.ws?.send(JSON.stringify(data));
+        Logger_1.Logger.debug(`[Seamless] send ${JSON.stringify(data)}`);
+        this.ws?.send(seamless_1.SeamlessMsgSendNo.ACTION.toString() + this.actionIndex.toString() + JSON.stringify(data));
+        this.actionIndex++;
         return;
     }
-    speech(data) {
-        if (this.isSpeech == false) {
+    _onMessage(data) {
+        if (typeof data === "string") {
+            Logger_1.Logger.debug(`[Seamless] receive ${data}`);
+            if (data == seamless_1.SeamlessMsgReceiveNo.PING.toString()) {
+                this.ws?.send(seamless_1.SeamlessMsgSendNo.PONG.toString());
+                return;
+            }
+            else if (data[0] === seamless_1.SeamlessMsgReceiveNo.CONNECT.toString()) {
+                // ws连接
+                this.ws?.send(seamless_1.SeamlessMsgSendNo.CONNECT.toString());
+                return;
+            }
+            else if (data.startsWith(seamless_1.SeamlessMsgReceiveNo.READY.toString())) {
+                this.callback && this.callback({
+                    hook: seamless_1.SeamlessHooks.READY
+                });
+            }
+            else if (data.startsWith(seamless_1.SeamlessMsgReceiveNo.INFO.toString())) {
+                // 消息
+                this._onMessageInfo(data);
+                return;
+            }
+            else if (data.startsWith(seamless_1.SeamlessMsgReceiveNo.ACTION.toString())) {
+                // 
+                this._onMessageAction(data);
+                return;
+            }
+        }
+        else {
+            Logger_1.Logger.debug(`[Seamless] receive data buffer`);
+        }
+    }
+    _onMessageInfo(message) {
+        const json = this.getJson(message);
+        if (!json) {
             return;
         }
-        this.ws?.send(["incoming_audio", { _placeholder: true, num: 0 }]);
-        this.ws?.send(data);
-    }
-    onMessage(callback) {
-        this.callback = callback;
-    }
-    _onMessage(data) {
-        if (typeof data === "string") {
-            if (data.startsWith("{") && data.endsWith("}")) {
-                try {
-                    const json = JSON.parse(data);
-                    // 判断是否数组
-                    if (Array.isArray(json)) {
-                        if (json[0] === "room_state_update") {
-                            this.roomId = json[1].room_id;
+        if (Array.isArray(json)) {
+            const infoKey = json[0];
+            if (infoKey === seamless_1.SeamlessReceiveInfoTypes.server_id) {
+                this.serverId = json[1];
+            }
+            else if (infoKey === seamless_1.SeamlessReceiveInfoTypes.room_state_update) {
+                this.roomId = json[1].room_id;
+            }
+            else if (infoKey === seamless_1.SeamlessReceiveInfoTypes.server_state_update) {
+                // 
+            }
+            else if (infoKey === seamless_1.SeamlessReceiveInfoTypes.translation_speech || infoKey === seamless_1.SeamlessReceiveInfoTypes.translation_text) {
+                const data = json[1];
+                if (data.event === "translation_speech") {
+                    this.callback && this.callback({
+                        hook: seamless_1.SeamlessHooks.TRANSLATION_SPEECH,
+                        msgId: this.msgId,
+                        data: {
+                            type: "speech",
+                            payload: this.floatArrayTo16BitPcmBuffer(data.payload),
+                            eos: data.eos
                         }
-                        else if (json[0] === "translation_text") {
-                            const payload = json[1].payload;
-                            this.callback && this.callback(seamless_1.SeamlessHooks.TRANSLATION_TEXT, this.msgId, payload);
+                    });
+                }
+                else if (data.event === "translation_text") {
+                    this.callback && this.callback({
+                        hook: seamless_1.SeamlessHooks.TRANSLATION_TEXT,
+                        msgId: this.msgId,
+                        data: {
+                            type: "text",
+                            payload: data.payload,
+                            eos: data.eos
                         }
-                        else if (json[0] === "translation_speech") {
-                            const payload = json[1].payload;
-                            this.callback && this.callback(seamless_1.SeamlessHooks.TRANSLATION_SPEECH, this.msgId, payload);
-                        }
-                        else if (typeof json[0] === "object") {
-                            if (json[0].status) {
-                                const status = json[0].status;
-                                const message = json[0].message;
-                                if (status === "ok" && message === "server_ready") {
-                                    if (this.isSpeech) {
-                                        this.callback && this.callback(seamless_1.SeamlessHooks.SPEECH_START, this.msgId);
-                                    }
-                                }
-                                else if (status === "ok" && message === "Stream stopped") {
-                                    this.callback && this.callback(seamless_1.SeamlessHooks.SPEECH_STOP, this.msgId);
-                                    this.msgId = undefined;
-                                }
-                            }
-                        }
-                    }
-                    else {
-                        if (json.sid) {
-                            this.sid = json.sid;
-                        }
-                    }
+                    });
                 }
-                catch (err) {
+            }
+        }
+    }
+    _onMessageAction(message) {
+        const json = this.getJson(message);
+        if (!json) {
+            return;
+        }
+        if (Array.isArray(json)) {
+            const data = json[0];
+            if (data.status && data.status === "ok") {
+                const actionMessage = data.message || "";
+                if (actionMessage === "server_ready") {
+                    this.callback && this.callback({
+                        hook: seamless_1.SeamlessHooks.SPEECH_START
+                    });
                 }
+                else if (actionMessage === "Stream stopped") {
+                    this.callback && this.callback({
+                        hook: seamless_1.SeamlessHooks.SPEECH_STOP
+                    });
+                }
             }
+            else if (data.roomsJoined) {
+                this.callback && this.callback({
+                    hook: seamless_1.SeamlessHooks.SPEECH_READY
+                });
+            }
         }
     }
-}
-exports.Seamless = Seamless;
-class SeamlessRoom extends Seamless {
-    constructor(options) {
-        super();
-        this.roomId = options?.roomId || null;
+    getJson(inputString) {
+        const jsonStrArr = [];
+        let isAdd = false;
+        for (let index = 0; index < inputString.length; index++) {
+            const char = inputString[index];
+            if (char === "{" || char === "[") {
+                isAdd = true;
+            }
+            if (isAdd) {
+                jsonStrArr.push(char);
+            }
+        }
+        const jsonStr = jsonStrArr.join("");
+        if (!jsonStr) {
+            return undefined;
+        }
+        try {
+            const json = JSON.parse(jsonStr);
+            return json;
+        }
+        catch (err) {
+            return null;
+        }
     }
     join(memberId) {
         this.send([
-            "join_room",
+            seamless_1.SeamlessSendMsgActionTypes.join_room,
             memberId,
-            this.roomId,
+            this.roomId || null,
             {
@@ -124,5 +199,5 @@ roles: ["speaker", "listener"],
     // 设置语言
-    targetLanguageSet(lan, expressive = null) {
+    langSet(lan, expressive = null) {
         this.send([
-            "set_dynamic_config",
+            seamless_1.SeamlessSendMsgActionTypes.set_dynamic_config,
             {
@@ -136,27 +211,50 @@ targetLanguage: lan,
         this.send([
-            "configure_stream",
+            seamless_1.SeamlessSendMsgActionTypes.configure_stream,
             {
                 async_processing: true,
                 buffer_limit: 1,
-                debug: false,
+                debug: data.debug || false,
                 event: "config",
                 model_name: "SeamlessStreaming",
                 model_type: data.modelType || "s2s&t",
-                rate: data.rate || 48000,
+                rate: data.rate || 16000,
             }
         ]);
     }
-    speechStart(lan, config) {
-        this.isSpeech = true;
+    start(lan, config) {
+        this.isReady = true;
         this.msgId = (0, uuid_1.v4)();
-        this.targetLanguageSet(lan);
-        this.configSet(config);
+        this.langSet(lan);
+        setTimeout(() => {
+            this.configSet(config);
+        }, 500);
     }
-    speechStop() {
+    stop() {
         this.send([
-            "stop_stream"
+            seamless_1.SeamlessSendMsgActionTypes.stop_stream,
         ]);
-        this.isSpeech = false;
+        this.isReady = false;
     }
+    speech(data) {
+        if (this.isReady == false) {
+            return;
+        }
+        const msg = "451-" + JSON.stringify(["incoming_audio", { _placeholder: true, num: 0 }]);
+        Logger_1.Logger.debug("send msg: " + msg);
+        this.ws?.send(msg);
+        Logger_1.Logger.debug("send data: ", data);
+        this.ws?.send(data, { binary: true });
+    }
+    floatArrayTo16BitPcmBuffer(floatArray) {
+        // 乘以32768以映射到16位整数范围(-32768 to 32767)，并确保值在范围内
+        const intArray = floatArray.map(sample => Math.max(-32768, Math.min(32767, sample * 32768)));
+        // 创建一个Uint16Array视图，用于存放量化后的整数数据
+        const uint16Array = new Uint16Array(intArray.length);
+        for (let i = 0; i < intArray.length; i++) {
+            uint16Array[i] = intArray[i];
+        }
+        // 从Uint16Array创建Buffer
+        return Buffer.from(uint16Array.buffer);
+    }
 }
-exports.SeamlessRoom = SeamlessRoom;
+exports.Seamless = Seamless;

dist/types/seamless.d.ts

@@ -0,6 +1,8 @@
+/// <reference types="node" />
 export type SeamlessOptions = {
     host?: string;
     roomId?: string;
+    onReceive?: (data: SeamlessCallbackParams) => void;
 };
-export type SeamlessMsgReceive = SeamlessMsgReceiveRoom | ("room_state_update" | SeamlessMsgReceiveRoomStateUpdate)[] | SeamlessMsgReceiveStatus[];
+export type SeamlessMsgReceive = SeamlessMsgReceiveRoom | (string | SeamlessMsgReceiveRoomStateUpdate)[] | SeamlessMsgReceiveStatus[];
 export type SeamlessMsgReceiveRoom = {
@@ -26,3 +28,3 @@ sid?: string;
 export type SeamlessTranscoderDynamicConfig = {
-    targetLanguage: "cmn";
+    targetLanguage: SeamlessLangueges;
     expressive: null;
@@ -35,7 +37,6 @@ };
 export type SeamlessModelType = "s2s&t" | "s2t" | "s2s";
-export declare enum SeamlessLanguege {
-    ENG = "eng",
-    CMN = "cmn"
-}
+export type SeamlessLangueges = "eng" | "arb" | "ben" | "cat" | "ces" | "cmn" | "cym" | "dan" | "deu" | "est" | "fin" | "fra" | "hin" | "ind" | "ita" | "jpn" | "kor" | "mlt" | "nld" | "pes" | "pol" | "por" | "ron" | "rus" | "slk" | "spa" | "swe" | "swh" | "tel" | "tgl" | "tha" | "tur" | "ukr" | "urd" | "uzn" | "vie";
 export declare enum SeamlessHooks {
+    READY = "ready",
+    SPEECH_READY = "speech_ready",
     SPEECH_START = "speech_start",
@@ -46,1 +47,43 @@ SPEECH_STOP = "speech_stop",
 }
+export declare enum SeamlessMsgSendNo {
+    PONG = 3,
+    CONNECT = 40,
+    ACTION = 42
+}
+export declare enum SeamlessMsgReceiveNo {
+    CONNECT = 0,
+    PING = 2,
+    READY = 40,
+    INFO = 42,
+    ACTION = 43
+}
+export type SeamlessCallbackParams = {
+    hook: SeamlessHooks;
+    msgId?: string;
+    data?: SeamlessTranslationResult;
+};
+export declare enum SeamlessReceiveInfoTypes {
+    server_id = "server_id",
+    room_state_update = "room_state_update",
+    server_state_update = "server_state_update",
+    clear_transcript = "clear_transcript",
+    translation_speech = "translation_speech",
+    translation_text = "translation_text"
+}
+export declare enum SeamlessSendMsgActionTypes {
+    join_room = "join_room",
+    set_dynamic_config = "set_dynamic_config",
+    configure_stream = "configure_stream",
+    stop_stream = "stop_stream"
+}
+export type SeamlessTranslationData = {
+    event: "translation_speech" | "translation_text";
+    payload: string | number[];
+    eos: boolean;
+    sample_rate?: number;
+};
+export type SeamlessTranslationResult = {
+    type: "speech" | "text";
+    payload: string | Buffer;
+    eos: boolean;
+};

dist/types/seamless.js

 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.SeamlessHooks = exports.SeamlessLanguege = void 0;
-var SeamlessLanguege;
-(function (SeamlessLanguege) {
-    SeamlessLanguege["ENG"] = "eng";
-    SeamlessLanguege["CMN"] = "cmn";
-})(SeamlessLanguege || (exports.SeamlessLanguege = SeamlessLanguege = {}));
+exports.SeamlessSendMsgActionTypes = exports.SeamlessReceiveInfoTypes = exports.SeamlessMsgReceiveNo = exports.SeamlessMsgSendNo = exports.SeamlessHooks = void 0;
 var SeamlessHooks;
 (function (SeamlessHooks) {
+    SeamlessHooks["READY"] = "ready";
+    SeamlessHooks["SPEECH_READY"] = "speech_ready";
     SeamlessHooks["SPEECH_START"] = "speech_start";
@@ -16,1 +13,31 @@ SeamlessHooks["SPEECH_STOP"] = "speech_stop";
 })(SeamlessHooks || (exports.SeamlessHooks = SeamlessHooks = {}));
+var SeamlessMsgSendNo;
+(function (SeamlessMsgSendNo) {
+    SeamlessMsgSendNo[SeamlessMsgSendNo["PONG"] = 3] = "PONG";
+    SeamlessMsgSendNo[SeamlessMsgSendNo["CONNECT"] = 40] = "CONNECT";
+    SeamlessMsgSendNo[SeamlessMsgSendNo["ACTION"] = 42] = "ACTION";
+})(SeamlessMsgSendNo || (exports.SeamlessMsgSendNo = SeamlessMsgSendNo = {}));
+var SeamlessMsgReceiveNo;
+(function (SeamlessMsgReceiveNo) {
+    SeamlessMsgReceiveNo[SeamlessMsgReceiveNo["CONNECT"] = 0] = "CONNECT";
+    SeamlessMsgReceiveNo[SeamlessMsgReceiveNo["PING"] = 2] = "PING";
+    SeamlessMsgReceiveNo[SeamlessMsgReceiveNo["READY"] = 40] = "READY";
+    SeamlessMsgReceiveNo[SeamlessMsgReceiveNo["INFO"] = 42] = "INFO";
+    SeamlessMsgReceiveNo[SeamlessMsgReceiveNo["ACTION"] = 43] = "ACTION";
+})(SeamlessMsgReceiveNo || (exports.SeamlessMsgReceiveNo = SeamlessMsgReceiveNo = {}));
+var SeamlessReceiveInfoTypes;
+(function (SeamlessReceiveInfoTypes) {
+    SeamlessReceiveInfoTypes["server_id"] = "server_id";
+    SeamlessReceiveInfoTypes["room_state_update"] = "room_state_update";
+    SeamlessReceiveInfoTypes["server_state_update"] = "server_state_update";
+    SeamlessReceiveInfoTypes["clear_transcript"] = "clear_transcript";
+    SeamlessReceiveInfoTypes["translation_speech"] = "translation_speech";
+    SeamlessReceiveInfoTypes["translation_text"] = "translation_text";
+})(SeamlessReceiveInfoTypes || (exports.SeamlessReceiveInfoTypes = SeamlessReceiveInfoTypes = {}));
+var SeamlessSendMsgActionTypes;
+(function (SeamlessSendMsgActionTypes) {
+    SeamlessSendMsgActionTypes["join_room"] = "join_room";
+    SeamlessSendMsgActionTypes["set_dynamic_config"] = "set_dynamic_config";
+    SeamlessSendMsgActionTypes["configure_stream"] = "configure_stream";
+    SeamlessSendMsgActionTypes["stop_stream"] = "stop_stream";
+})(SeamlessSendMsgActionTypes || (exports.SeamlessSendMsgActionTypes = SeamlessSendMsgActionTypes = {}));

package.json

 {
   "name": "@re-ai/inner-tool-sdk",
-  "version": "0.2.14",
+  "version": "0.2.15",
   "description": "ReAI内部接口sdk",
@@ -5,0 +5,0 @@ "main": "dist/index.js",

src/entrys/index.ts

@@ -14,2 +14,3 @@
 import { NetBridgeService } from "../services/NetBridgeService";
+import { Seamless } from "../services/SeamlessService";
 
@@ -77,2 +78,9 @@ export class ReAITool {
 
+    static seamless(host: string, roomId?: string) {
+        return new Seamless({
+            host: host,
+            roomId: roomId
+        })
+    }
+
 }

src/services/SeamlessService.ts

 import WebSocket, { CloseEvent, Data, ErrorEvent, MessageEvent, OpenEvent } from "ws";
-import { ISeamless } from "../interfaces/seamless";
 import { Logger } from "../utils/Logger";
-import { SeamlessHooks, SeamlessLanguege, SeamlessModelType, SeamlessMsgReceive, SeamlessMsgReceiveStatus, SeamlessOptions } from "../types/seamless";
+import { SeamlessCallbackParams, SeamlessHooks, SeamlessLangueges, SeamlessModelType, SeamlessMsgReceive, SeamlessMsgReceiveNo, SeamlessMsgReceiveStatus, SeamlessMsgSendNo, SeamlessOptions, SeamlessReceiveInfoTypes, SeamlessSendMsgActionTypes, SeamlessTranscoderDynamicConfig, SeamlessTranslationData, SeamlessTranslationResult } from "../types/seamless";
 import { v4 } from "uuid";
+import { ITranslate } from "../interfaces/translate";
 
-export class Seamless implements ISeamless {
+export class Seamless implements ITranslate {
 
@@ -12,17 +12,22 @@ host: string;
     ws?: WebSocket;
-    sid?: string;
+    serverId?: string;
     roomId?: string | null;
     msgId?: string;
-    
+    actionIndex: number = 0;
+    isReady: boolean = false;
 
-    callback?: (hook: SeamlessHooks, id?: string, data?: any) => void
+    callback?: (data: SeamlessCallbackParams) => void
 
-    isSpeech: boolean = false
     constructor(options?: SeamlessOptions) {
         // address = "/socket.io/?clientID=1c2b94f6-7501-44eb-a4a5-46540072dcfd&EIO=4&transport=websocket"
         this.host = options?.host || process.env.REAI_SEAMLESS_HOST || "wss://seamless.cn.reai.com/seamless/ws"
+        this.roomId = options?.roomId || null
+        this.callback = options?.onReceive || undefined
+
+        this.isReady = false
     }
 
-    async connect(clientId: string, callback: (hook: string) => void) {
+    async ready(clientId: string, callback?: (data: SeamlessCallbackParams) => void) {
         const address = `${this.host}/ws/socket.io/?clientID=${clientId}&EIO=4&transport=websocket`
+        Logger.info(`[Seamless] connect ${address}`)
         this.ws = new WebSocket(address)
@@ -32,3 +37,7 @@
             Logger.warn(`[Seamless] open .... ${e.type}`)
-            this.callback = callback
+            // this.callback = callback
+            if (callback) {
+                this.callback = callback
+            }
+            
         }
@@ -50,72 +59,139 @@
         }
+
+        return this
     }
 
-    send(data: any) {
-        this.ws?.send(JSON.stringify(data))
+    private send(data: any) {
+        Logger.debug(`[Seamless] send ${JSON.stringify(data)}`)
+        this.ws?.send(SeamlessMsgSendNo.ACTION.toString() + this.actionIndex.toString() + JSON.stringify(data))
+        this.actionIndex ++
         return
     }
 
-    speech(data: Buffer) {
-        if (this.isSpeech == false) {
-            return
+    private _onMessage(data: Data) {
+        
+        if (typeof data === "string") {
+            Logger.debug(`[Seamless] receive ${data}`)
+            if (data == SeamlessMsgReceiveNo.PING.toString()) {
+                this.ws?.send(SeamlessMsgSendNo.PONG.toString())
+                return
+            } else if (data[0] === SeamlessMsgReceiveNo.CONNECT.toString()) {
+                // ws连接
+                this.ws?.send(SeamlessMsgSendNo.CONNECT.toString())
+                return
+            } else if (data.startsWith(SeamlessMsgReceiveNo.READY.toString())) {
+                this.callback && this.callback({
+                    hook: SeamlessHooks.READY
+                })
+            } else if (data.startsWith(SeamlessMsgReceiveNo.INFO.toString())) {
+                // 消息
+                this._onMessageInfo(data)
+                return
+            } else if (data.startsWith(SeamlessMsgReceiveNo.ACTION.toString())) {
+                // 
+                this._onMessageAction(data)
+                return
+            }
+  
+        } else {
+            Logger.debug(`[Seamless] receive data buffer`)
         }
-        this.ws?.send(["incoming_audio", { _placeholder: true, num: 0 }])
-        this.ws?.send(data)
     }
 
-    onMessage(callback: (data: any) => void): void {
-        this.callback = callback
+    private _onMessageInfo(message: string) {
+        const json = this.getJson(message)
+        if (!json) {
+            return
+        }
+
+        if (Array.isArray(json)) {
+            const infoKey = json[0]
+            if (infoKey === SeamlessReceiveInfoTypes.server_id) {
+                this.serverId = json[1]
+            } else if (infoKey === SeamlessReceiveInfoTypes.room_state_update) {
+                this.roomId = json[1].room_id
+            } else if (infoKey === SeamlessReceiveInfoTypes.server_state_update) {
+                // 
+            } else if (infoKey === SeamlessReceiveInfoTypes.translation_speech || infoKey === SeamlessReceiveInfoTypes.translation_text) {
+                const data = json[1] as SeamlessTranslationData
+                if (data.event === "translation_speech") {
+                   this.callback && this.callback({
+                        hook: SeamlessHooks.TRANSLATION_SPEECH,
+                        msgId: this.msgId,
+                        data: {
+                            type: "speech",
+                            payload: this.floatArrayTo16BitPcmBuffer(data.payload as number[]),
+                            eos: data.eos
+                        } as SeamlessTranslationResult
+                    })
+                } else if (data.event === "translation_text") {
+                    this.callback && this.callback({
+                        hook: SeamlessHooks.TRANSLATION_TEXT,
+                        msgId: this.msgId,
+                        data: {
+                            type: "text",
+                            payload: data.payload as string,
+                            eos: data.eos
+                        } as SeamlessTranslationResult
+                    })
+                }
+            }
+        }
     }
 
-    _onMessage(data: Data) {
-        if (typeof data === "string") {
-            if (data.startsWith("{") && data.endsWith("}")) {
-                try {
-                    const json = JSON.parse(data)
-                    // 判断是否数组
-                    if (Array.isArray(json)) {
+    private _onMessageAction(message: string) {
+        const json = this.getJson(message)
+        if (!json) {
+            return
+        }
 
-                        if (json[0] === "room_state_update") {
-                            this.roomId = json[1].room_id
-                        } else if (json[0] === "translation_text") {
-                            const payload = json[1].payload
-                            this.callback && this.callback(SeamlessHooks.TRANSLATION_TEXT, this.msgId, payload)
-                        } else if (json[0] === "translation_speech") {
-                            const payload = json[1].payload
-                            this.callback && this.callback(SeamlessHooks.TRANSLATION_SPEECH, this.msgId, payload)
-                        } else if (typeof json[0] === "object") {
-                            if (json[0].status) {
-                                const status = json[0].status
-                                const message = json[0].message
-                                if (status === "ok" && message === "server_ready") {
-                                    if (this.isSpeech) {
-                                        this.callback && this.callback(SeamlessHooks.SPEECH_START, this.msgId)
-                                    }
-                                } else if (status === "ok" && message === "Stream stopped") {
-                                    this.callback && this.callback(SeamlessHooks.SPEECH_STOP, this.msgId)
-                                    this.msgId = undefined
-                                }
-                            }
-                        }
-                    } else {
-                        if (json.sid) {
-                            this.sid = json.sid
-                        }
-                    }
+        if (Array.isArray(json)) {
+            const data = json[0]
+            if (data.status && data.status === "ok") {
+                const actionMessage = data.message || ""
+                if (actionMessage === "server_ready") {
+                    this.callback && this.callback({
+                        hook: SeamlessHooks.SPEECH_START
+                    })
+                } else if (actionMessage === "Stream stopped") {
+                    this.callback && this.callback({
+                        hook: SeamlessHooks.SPEECH_STOP
+                    })
+                }
+            } else if (data.roomsJoined) {
+                this.callback && this.callback({
+                    hook: SeamlessHooks.SPEECH_READY
+                })
+            }
+        }
+    }
 
-                } catch (err) {
+    
 
-                }
+    private getJson(inputString: string) {
+        const jsonStrArr: string[] = []
+        let isAdd = false
+        for (let index = 0; index < inputString.length; index++) {
+            const char = inputString[index]
+            if (char === "{" || char === "[") {
+                isAdd = true
+            }
 
+            if (isAdd) {
+                jsonStrArr.push(char)
             }
+            
         }
-    }
-}
 
-export class SeamlessRoom extends Seamless {
+        const jsonStr = jsonStrArr.join("")
+        if (!jsonStr) {
+            return undefined
+        }
 
-    constructor(options?: SeamlessOptions) {
-   
-        super()
-        this.roomId = options?.roomId || null
+        try {
+            const json = JSON.parse(jsonStr)
+            return json
+        } catch (err) {
+            return null
+        }
     }
@@ -125,5 +201,5 @@
         this.send([
-            "join_room",
+            SeamlessSendMsgActionTypes.join_room,
             memberId,
-            this.roomId,
+            this.roomId || null,
             {
@@ -137,5 +213,5 @@ roles: ["speaker", "listener"],
     // 设置语言
-    targetLanguageSet(lan: SeamlessLanguege, expressive: boolean | null = null) {
+    langSet(lan: SeamlessLangueges, expressive: boolean | null = null) {
         this.send([
-            "set_dynamic_config",
+            SeamlessSendMsgActionTypes.set_dynamic_config,
             {
@@ -150,14 +226,15 @@ targetLanguage: lan,
         modelType?: SeamlessModelType
-        rate?: number
+        rate?: number,
+        debug?: boolean
     }) {
         this.send([
-            "configure_stream",
+            SeamlessSendMsgActionTypes.configure_stream,
             {
                 async_processing: true,
                 buffer_limit: 1,
-                debug: false,
+                debug: data.debug || false,
                 event: "config",
                 model_name: "SeamlessStreaming",
                 model_type: data.modelType || "s2s&t",
-                rate: data.rate || 48000,
+                rate: data.rate || 16000,
             }
@@ -167,23 +244,50 @@ ])
 
-    speechStart(lan: SeamlessLanguege, config: {
+    start(lan: SeamlessLangueges, config: {
         modelType?: SeamlessModelType
-        rate?: number
+        rate?: number,
+        debug?: boolean
     }) {
-        this.isSpeech = true
+        this.isReady = true
         this.msgId = v4()
 
-        this.targetLanguageSet(lan)
-        this.configSet(config)
+        this.langSet(lan)
+        setTimeout(() => {
+            this.configSet(config)
+        }, 500)
 
     }
 
-    speechStop() {
+    stop() {
         this.send([
-            "stop_stream"
+            SeamlessSendMsgActionTypes.stop_stream,
         ])
-        this.isSpeech = false
+        this.isReady = false
     }
 
+    
+    speech(data: Buffer) {
+        if (this.isReady == false) {
+            return
+        }
+        const msg = "451-" + JSON.stringify(["incoming_audio", { _placeholder: true, num: 0 }])
+        Logger.debug("send msg: " + msg)
+        this.ws?.send(msg)
+        Logger.debug("send data: " , data)
+        this.ws?.send(data, { binary: true })
+    }
 
+    private floatArrayTo16BitPcmBuffer(floatArray: number[]): Buffer {
+        // 乘以32768以映射到16位整数范围(-32768 to 32767)，并确保值在范围内
+        const intArray = floatArray.map(sample => Math.max(-32768, Math.min(32767, sample * 32768)));
+    
+        // 创建一个Uint16Array视图，用于存放量化后的整数数据
+        const uint16Array = new Uint16Array(intArray.length);
+        for (let i = 0; i < intArray.length; i++) {
+            uint16Array[i] = intArray[i];
+        }
+    
+        // 从Uint16Array创建Buffer
+        return Buffer.from(uint16Array.buffer);
+    }
+
 }
-

src/types/seamless.ts

 export type SeamlessOptions = {
     host?: string,
-    roomId?: string
+    roomId?: string,
+    onReceive?: (data: SeamlessCallbackParams) => void
 }
 
-export type SeamlessMsgReceive = SeamlessMsgReceiveRoom | ("room_state_update" | SeamlessMsgReceiveRoomStateUpdate)[] | SeamlessMsgReceiveStatus[]
+export type SeamlessMsgReceive = SeamlessMsgReceiveRoom | (string | SeamlessMsgReceiveRoomStateUpdate)[] | SeamlessMsgReceiveStatus[]
 
@@ -31,3 +32,3 @@ export type SeamlessMsgReceiveRoom = {
 export type SeamlessTranscoderDynamicConfig = {
-    targetLanguage: "cmn",
+    targetLanguage: SeamlessLangueges,
     expressive: null
@@ -42,8 +43,8 @@ }
 export type SeamlessModelType = "s2s&t" | "s2t" | "s2s"
-export enum SeamlessLanguege {
-    ENG = "eng",
-    CMN = "cmn"
-}
 
+export type SeamlessLangueges = "eng" |"arb" | "ben" | "cat" | "ces" | "cmn" | "cym" | "dan" | "deu" | "est" | "fin" | "fra" | "hin" | "ind" | "ita" | "jpn" | "kor" | "mlt" | "nld" | "pes" | "pol" | "por" | "ron" | "rus" | "slk" | "spa" | "swe" | "swh" | "tel" | "tgl" | "tha" | "tur" | "ukr" | "urd" | "uzn" | "vie"
+
 export enum SeamlessHooks {
+    READY = "ready",
+    SPEECH_READY = "speech_ready",
     SPEECH_START = "speech_start",
@@ -53,2 +54,52 @@ SPEECH_STOP = "speech_stop",
     TRANSLATION_SPEECH = "translation_speech"
+}
+
+export enum SeamlessMsgSendNo {
+    PONG = 3,
+    CONNECT = 40,
+    ACTION = 42
+}
+
+export enum SeamlessMsgReceiveNo {
+    CONNECT = 0,
+    PING = 2,
+    READY = 40,
+    INFO = 42,
+    ACTION = 43,
+    
+}
+
+export type SeamlessCallbackParams = {
+    hook: SeamlessHooks,
+    msgId?: string,
+    data?: SeamlessTranslationResult
+}
+
+export enum SeamlessReceiveInfoTypes {
+    server_id = "server_id",
+    room_state_update = "room_state_update",
+    server_state_update = "server_state_update",
+    clear_transcript = "clear_transcript",
+    translation_speech = "translation_speech",
+    translation_text = "translation_text"
+}
+
+export enum SeamlessSendMsgActionTypes {
+    join_room = "join_room",
+    set_dynamic_config = "set_dynamic_config",
+    configure_stream = "configure_stream",
+    stop_stream = "stop_stream"
+}
+
+export type SeamlessTranslationData = {
+    event: "translation_speech" | "translation_text",
+    payload: string | number[],
+    eos: boolean,
+    sample_rate?: number,
+}
+
+export type SeamlessTranslationResult = {
+    type: "speech" | "text",
+    payload: string | Buffer,
+    eos: boolean
 }

New alerts

License Policy Violation

License

This package is not allowed per your license policy. Review the package's license to ensure compliance.

Found 1 instance in 1 package

Fixed alerts

License Policy Violation

License

This package is not allowed per your license policy. Review the package's license to ensure compliance.

Found 1 instance in 1 package

Improved metrics

Total package byte prevSize

189667

increased by7.56%

Number of package files

119

increased by1.71%

Lines of code

4958

increased by7.18%

No dependency changes