simli-client - npm Package Compare versions

Comparing version 3.0.1 to 3.0.2

dist/client.d.ts

+import { SimliClientEvents } from './Events';
+import { Logger, LogLevel } from './Logger';
+interface SimliSessionRequest {
+    faceId: string;
+    handleSilence: boolean;
+    maxSessionLength: number;
+    maxIdleTime: number;
+    model?: "fasttalk" | "artalk";
+}
+interface TokenRequestData {
+    config: SimliSessionRequest;
+    apiKey: string;
+}
+interface SimliSessionToken {
+    session_token: string;
+}
+type TransportMode = "livekit" | "p2p";
+type SignalingMode = "websockets";
+type session_token = string;
+declare function generateSimliSessionToken(request: TokenRequestData, SimliURL?: string): Promise<SimliSessionToken>;
+declare function generateIceServers(apiKey: string, SimliURL?: string): Promise<RTCIceServer[]>;
+declare class SimliClient {
+    session_token: string;
+    transport: TransportMode;
+    signaling: SignalingMode;
+    videoElement: HTMLVideoElement;
+    audioElement: HTMLAudioElement;
+    audioBufferSize: number;
+    private connection;
+    private connectionTimeout;
+    private connectionResolve;
+    private connectionReject;
+    private connectionPromise;
+    private sourceNode;
+    private audioWorklet;
+    private readonly MAX_RETRY_ATTEMPTS;
+    private RETRY_DELAY;
+    private readonly CONNECTION_TIMEOUT_MS;
+    private retryAttempt;
+    private SimliWSURL;
+    private audioContext;
+    private logger;
+    private iceServers;
+    private persistent_events;
+    private failReason;
+    private shouldStop;
+    on<K extends keyof SimliClientEvents>(event: K, callback: SimliClientEvents[K]): void;
+    off<K extends keyof SimliClientEvents>(event: K, callback: SimliClientEvents[K]): void;
+    constructor(session_token: session_token, videoElement: HTMLVideoElement, audioElement: HTMLAudioElement, iceServers: RTCIceServer[] | null, logLevel?: LogLevel, transport_mode?: TransportMode, signaling?: SignalingMode, SimliWSURL?: string, audioBufferSize?: number);
+    private resetConnections;
+    start(): Promise<void>;
+    stop(): Promise<void>;
+    listenToMediastreamTrack(stream: MediaStreamTrack): void;
+    listenToAudioElement(audioEl: HTMLAudioElement): void;
+    private attachSourceToWorklet;
+    ClearBuffer: () => void;
+    sendAudioData(audioData: Uint8Array): void;
+    sendAudioDataImmediate(audioData: Uint8Array): void;
+}
+export { SimliClient, generateSimliSessionToken, generateIceServers, Logger, LogLevel };
+export type { SimliSessionRequest };

dist/client.js

+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.LogLevel = exports.Logger = exports.SimliClient = void 0;
+exports.generateSimliSessionToken = generateSimliSessionToken;
+exports.generateIceServers = generateIceServers;
+const LivekitTransport_1 = require("./Transports/LivekitTransport");
+const P2PTransport_1 = require("./Transports/P2PTransport");
+const Logger_1 = require("./Logger");
+Object.defineProperty(exports, "Logger", { enumerable: true, get: function () { return Logger_1.Logger; } });
+Object.defineProperty(exports, "LogLevel", { enumerable: true, get: function () { return Logger_1.LogLevel; } });
+const AudioProcessor = (buffer) => {
+    if (buffer <= 0) {
+        throw "Invalid Buffer Size, Can't be negative";
+    }
+    if (Math.floor(buffer) - buffer != 0) {
+        throw "Invalid Buffer Size, Can't be a float";
+    }
+    return `
+        class AudioProcessor extends AudioWorkletProcessor {
+          constructor() {
+            super();
+            this.buffer = new Int16Array(${buffer});
+            this.bufferIndex = 0;
+        }
+
+          process(inputs, outputs, parameters) {
+            const input = inputs[0];
+            const inputChannel = input[0];
+            if (inputChannel) {
+              for (let i = 0; i < inputChannel.length; i++) {
+                this.buffer[this.bufferIndex] = Math.max(-32768, Math.min(32767, Math.round(inputChannel[i] * 32767)));
+                this.bufferIndex++;
+
+                if (this.bufferIndex === this.buffer.length){
+                  this.port.postMessage({type: 'audioData', data: this.buffer.slice(0, this.bufferIndex)});
+                  this.bufferIndex = 0;
+                }
+              }
+            }
+            return true;
+          }
+        }
+
+        registerProcessor('audio-processor', AudioProcessor);
+      `;
+};
+async function generateSimliSessionToken(request, SimliURL = "https://api.simli.ai") {
+    const url = `${SimliURL}/compose/token`;
+    const response = await fetch(url, {
+        method: "POST",
+        body: JSON.stringify(request.config),
+        headers: {
+            "Content-Type": "application/json",
+            "x-simli-api-key": request.apiKey
+        },
+    });
+    if (!response.ok) {
+        const errorText = await response.text();
+        throw errorText;
+    }
+    const resJSON = await response.json();
+    return resJSON;
+}
+async function generateIceServers(apiKey, SimliURL = "https://api.simli.ai") {
+    try {
+        const url = `${SimliURL}/compose/ice`;
+        const response = await fetch(url, {
+            headers: {
+                "Content-Type": "application/json",
+                "x-simli-api-key": apiKey,
+            },
+            method: "GET",
+        });
+        if (!response.ok) {
+            throw new Error(`SIMLI: HTTP error! status: ${response.status}`);
+        }
+        const iceServers = await response.json();
+        if (!iceServers || iceServers.length === 0) {
+            throw new Error("SIMLI: No ICE servers returned");
+        }
+        return iceServers;
+    }
+    catch (error) {
+        return [{ urls: ["stun:stun.l.google.com:19302"] }];
+    }
+}
+class SimliClient {
+    session_token;
+    transport = "livekit";
+    signaling = "websockets";
+    videoElement;
+    audioElement;
+    audioBufferSize = 3000;
+    connection;
+    connectionTimeout;
+    connectionResolve;
+    connectionReject;
+    connectionPromise;
+    sourceNode = null;
+    audioWorklet = null;
+    MAX_RETRY_ATTEMPTS = 10;
+    RETRY_DELAY = 2000;
+    CONNECTION_TIMEOUT_MS = 15000;
+    retryAttempt = 0;
+    SimliWSURL = "wss://api.simli.ai";
+    audioContext = new (window.AudioContext ||
+        window.webkitAudioContext)({
+        sampleRate: 16000,
+    });
+    logger;
+    iceServers;
+    persistent_events;
+    failReason = null;
+    shouldStop = false;
+    // Type-safe event methods
+    on(event, callback) {
+        if (!this.persistent_events.has(event)) {
+            this.persistent_events.set(event, new Set());
+        }
+        this.persistent_events.get(event)?.add(callback);
+        this.logger.debug("Registered Callback for Event: " + event);
+        this.connection.on(event, callback);
+    }
+    off(event, callback) {
+        if (!this.persistent_events.has(event)) {
+            throw "Event Not Regsitered";
+        }
+        this.persistent_events.get(event)?.delete(callback);
+        this.connection.off(event, callback);
+    }
+    constructor(session_token, videoElement, audioElement, iceServers, logLevel = Logger_1.LogLevel.DEBUG, transport_mode = "p2p", signaling = "websockets", SimliWSURL = "wss://api.simli.ai", audioBufferSize = 3000) {
+        if (audioBufferSize <= 0) {
+            throw "Invalid Buffer Size, Can't be negative";
+        }
+        if (Math.floor(audioBufferSize) - audioBufferSize != 0) {
+            throw "Invalid Buffer Size, Can't be a float";
+        }
+        if (!(SimliWSURL.startsWith("ws://") || SimliWSURL.startsWith("wss://")) || SimliWSURL.endsWith("/")) {
+            throw "Invalid Simli WS URL";
+        }
+        this.audioBufferSize = audioBufferSize;
+        this.session_token = session_token;
+        this.transport = transport_mode;
+        this.signaling = signaling;
+        this.SimliWSURL = SimliWSURL;
+        this.videoElement = videoElement;
+        this.audioElement = audioElement;
+        this.iceServers = iceServers;
+        this.logger = new Logger_1.Logger(logLevel);
+        let resolveFn;
+        let rejectFn;
+        this.connectionPromise = new Promise((resolve, reject) => {
+            resolveFn = resolve;
+            rejectFn = reject;
+        });
+        this.connectionResolve = resolveFn;
+        this.connectionReject = rejectFn;
+        this.persistent_events = new Map();
+        this.connectionTimeout = setTimeout(() => this.connectionReject("CONNECTION TIMED OUT"), this.CONNECTION_TIMEOUT_MS);
+        switch (this.transport) {
+            case "livekit":
+                this.connection = new LivekitTransport_1.LivekitTransport(this.SimliWSURL, this.session_token, videoElement, audioElement, this.logger, this.connectionReject);
+                break;
+            case "p2p":
+                if (!iceServers || iceServers.length == 0) {
+                    throw "Ice Servers Required for P2P Mode";
+                }
+                this.connection = new P2PTransport_1.P2PTransport(this.SimliWSURL, this.session_token, true, iceServers, videoElement, audioElement, this.logger, this.connectionReject);
+                break;
+            default:
+                throw new Error("Not Implemented Yet");
+        }
+        this.connection.on("start", () => {
+            this.connectionResolve();
+            clearTimeout(this.connectionTimeout);
+        });
+        this.connection.on("unknown", (message) => this.logger.debug("UNKOWN MESSAGE FROM SERVER: " + message));
+        this.connection.on("error", (message) => { this.failReason = message; this.connectionReject(message); });
+    }
+    resetConnections(videoElement, audioElement, iceServers) {
+        this.failReason = null;
+        let resolveFn;
+        let rejectFn;
+        this.connectionPromise = new Promise((resolve, reject) => {
+            resolveFn = resolve;
+            rejectFn = reject;
+        });
+        this.connectionResolve = resolveFn;
+        this.connectionReject = rejectFn;
+        this.connectionTimeout = setTimeout(() => this.connectionReject("Connection Timed Out"), this.CONNECTION_TIMEOUT_MS);
+        switch (this.transport) {
+            case "livekit":
+                this.connection = new LivekitTransport_1.LivekitTransport(this.SimliWSURL, this.session_token, videoElement, audioElement, this.logger, this.connectionReject);
+                break;
+            case "p2p":
+                if (!iceServers || iceServers.length == 0) {
+                    throw "Ice Servers Required for P2P Mode";
+                }
+                this.connection = new P2PTransport_1.P2PTransport(this.SimliWSURL, this.session_token, true, iceServers, videoElement, audioElement, this.logger, this.connectionReject);
+                break;
+            default:
+                throw new Error("Not Implemented Yet");
+        }
+        this.connection.on("start", () => {
+            this.connectionResolve();
+            clearTimeout(this.connectionTimeout);
+        });
+        this.connection.on("error", (message) => { this.retryAttempt = this.MAX_RETRY_ATTEMPTS; this.connectionReject(message); });
+        this.connection.on("unknown", (message) => this.logger.debug("UNKOWN MESSAGE FROM SERVER: " + message));
+        // Re-register all user event handlers on the new connection
+        this.persistent_events.forEach((callbacks, event) => {
+            callbacks.forEach((callback) => {
+                this.connection.on(event, callback);
+            });
+        });
+    }
+    async start() {
+        if (this.shouldStop) {
+            throw new Error("Disconnect Already Called, Can't reuse same SimliClient multiple times create a new SimliClient Object");
+        }
+        try {
+            await this.connection.connect();
+            await this.connectionPromise;
+            this.retryAttempt = 0;
+        }
+        catch (error) {
+            if (this.failReason) {
+                throw error;
+            }
+            if (this.retryAttempt >= this.MAX_RETRY_ATTEMPTS)
+                throw new Error("Too Many Retry Attempts Failed to connect");
+            if (this.shouldStop) {
+                this.shouldStop = false;
+                throw new Error("Called Disconnect Before A Connecction succeeded");
+            }
+            this.logger.error("FAILED: " + error);
+            await this.connection.disconnect();
+            await new Promise(resolve => setTimeout(resolve, this.RETRY_DELAY));
+            this.retryAttempt += 1;
+            if (this.retryAttempt > 2)
+                this.transport = "livekit";
+            this.resetConnections(this.videoElement, this.audioElement, this.iceServers);
+            await this.start();
+        }
+    }
+    async stop() {
+        this.shouldStop = true;
+        await this.connection.disconnect();
+    }
+    listenToMediastreamTrack(stream) {
+        const source = this.audioContext.createMediaStreamSource(new MediaStream([stream]));
+        this.sourceNode = source;
+        this.attachSourceToWorklet(this.audioContext, source);
+    }
+    listenToAudioElement(audioEl) {
+        const source = this.audioContext.createMediaElementSource(audioEl);
+        // No connection to audioContext.destination on purpose: createMediaElementSource
+        // reroutes the element's audio into the graph, so leaving the destination
+        // unconnected silences the <audio> while samples still flow to Simli.
+        this.attachSourceToWorklet(this.audioContext, source);
+    }
+    attachSourceToWorklet(audioContext, source) {
+        audioContext.audioWorklet
+            .addModule(URL.createObjectURL(new Blob([AudioProcessor(this.audioBufferSize)], {
+            type: "application/javascript",
+        })))
+            .then(() => {
+            this.audioWorklet = new AudioWorkletNode(audioContext, "audio-processor");
+            if (this.audioWorklet === null) {
+                throw new Error("SIMLI: AudioWorklet not initialized");
+            }
+            source.connect(this.audioWorklet);
+            this.audioWorklet.port.onmessage = (event) => {
+                if (event.data.type === "audioData") {
+                    this.connection.signalingConnection.sendAudioData(new Uint8Array(event.data.data.buffer));
+                }
+            };
+        });
+    }
+    ClearBuffer = () => {
+        this.connection.signalingConnection.sendSignal("SKIP");
+    };
+    sendAudioData(audioData) {
+        this.connection.signalingConnection.sendAudioData(audioData);
+    }
+    sendAudioDataImmediate(audioData) {
+        this.connection.signalingConnection.sendAudioDataImmediate(audioData);
+    }
+}
+exports.SimliClient = SimliClient;

dist/config.d.ts

+interface SimliClientConfig {
+    faceID: string;
+    handleSilence: boolean;
+    maxSessionLength: number;
+    maxIdleTime: number;
+    enableSFU: boolean;
+    model: "fasttalk" | "artalk";
+}
+export type { SimliClientConfig };

dist/config.js

+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });

lib/client.ts

+// src/index.ts
+import { SimliClientEvents } from './Events';
+import { BaseTransport, EventCallback, EventMap } from './Transports/BaseTransport';
+import { LivekitTransport } from './Transports/LivekitTransport';
+import { P2PTransport } from './Transports/P2PTransport';
+import { Logger, LogLevel } from './Logger';
+
+const AudioProcessor = (buffer: number) => {
+    if (buffer <= 0) {
+        throw "Invalid Buffer Size, Can't be negative"
+    }
+    if (Math.floor(buffer) - buffer != 0) {
+        throw "Invalid Buffer Size, Can't be a float"
+    }
+    return `
+        class AudioProcessor extends AudioWorkletProcessor {
+          constructor() {
+            super();
+            this.buffer = new Int16Array(${buffer});
+            this.bufferIndex = 0;
+        }
+
+          process(inputs, outputs, parameters) {
+            const input = inputs[0];
+            const inputChannel = input[0];
+            if (inputChannel) {
+              for (let i = 0; i < inputChannel.length; i++) {
+                this.buffer[this.bufferIndex] = Math.max(-32768, Math.min(32767, Math.round(inputChannel[i] * 32767)));
+                this.bufferIndex++;
+
+                if (this.bufferIndex === this.buffer.length){
+                  this.port.postMessage({type: 'audioData', data: this.buffer.slice(0, this.bufferIndex)});
+                  this.bufferIndex = 0;
+                }
+              }
+            }
+            return true;
+          }
+        }
+
+        registerProcessor('audio-processor', AudioProcessor);
+      `};
+
+// Custom event handler types
+interface SimliSessionRequest {
+    faceId: string;
+    handleSilence: boolean;
+    maxSessionLength: number;
+    maxIdleTime: number;
+    model?: "fasttalk" | "artalk";
+}
+interface TokenRequestData {
+    config: SimliSessionRequest
+    apiKey: string
+}
+
+interface SimliSessionToken {
+    session_token: string;
+}
+
+type TransportMode = "livekit" | "p2p"
+type SignalingMode = "websockets"
+type session_token = string
+
+async function generateSimliSessionToken(
+    request: TokenRequestData,
+    SimliURL: string = "https://api.simli.ai",
+): Promise<SimliSessionToken> {
+    const url = `${SimliURL}/compose/token`;
+    const response = await fetch(url, {
+        method: "POST",
+        body: JSON.stringify(request.config),
+        headers: {
+            "Content-Type": "application/json",
+            "x-simli-api-key": request.apiKey
+        },
+    });
+
+    if (!response.ok) {
+        const errorText = await response.text();
+        throw errorText;
+    }
+
+    const resJSON = await response.json();
+    return resJSON;
+}
+
+async function generateIceServers(
+    apiKey: string,
+    SimliURL: string = "https://api.simli.ai",
+): Promise<RTCIceServer[]> {
+    try {
+        const url = `${SimliURL}/compose/ice`;
+        const response: any = await fetch(url, {
+            headers: {
+                "Content-Type": "application/json",
+                "x-simli-api-key": apiKey,
+            },
+            method: "GET",
+        })
+
+        if (!response.ok) {
+            throw new Error(`SIMLI: HTTP error! status: ${response.status}`);
+        }
+
+        const iceServers = await response.json();
+        if (!iceServers || iceServers.length === 0) {
+            throw new Error("SIMLI: No ICE servers returned");
+        }
+        return iceServers;
+    } catch (error) {
+        return [{ urls: ["stun:stun.l.google.com:19302"] }];
+    }
+}
+
+class SimliClient {
+
+    session_token: string;
+    transport: TransportMode = "livekit";
+    signaling: SignalingMode = "websockets"
+    videoElement: HTMLVideoElement
+    audioElement: HTMLAudioElement
+    audioBufferSize: number = 3000
+    private connection: BaseTransport
+    private connectionTimeout: NodeJS.Timeout
+    private connectionResolve: () => void
+    private connectionReject: (message: string) => void
+    private connectionPromise: Promise<void>
+    private sourceNode: MediaStreamAudioSourceNode | null = null;
+    private audioWorklet: AudioWorkletNode | null = null;
+    private readonly MAX_RETRY_ATTEMPTS = 10;
+    private RETRY_DELAY = 2000;
+    private readonly CONNECTION_TIMEOUT_MS = 15000;
+    private retryAttempt: number = 0;
+    private SimliWSURL: string = "wss://api.simli.ai";
+    private audioContext: AudioContext = new (window.AudioContext ||
+        (window as any).webkitAudioContext)({
+            sampleRate: 16000,
+        });
+    private logger: Logger;
+    private iceServers: RTCIceServer[] | null;
+    private persistent_events: EventMap;
+    private failReason: string | null = null;
+    private shouldStop: boolean = false;
+
+    // Type-safe event methods
+    public on<K extends keyof SimliClientEvents>(
+        event: K,
+        callback: SimliClientEvents[K]
+    ): void {
+        if (!this.persistent_events.has(event)) {
+            this.persistent_events.set(event, new Set());
+        }
+        this.persistent_events.get(event)?.add(callback as EventCallback);
+        this.logger.debug("Registered Callback for Event: " + event)
+        this.connection.on(event, callback)
+    }
+
+    public off<K extends keyof SimliClientEvents>(
+        event: K,
+        callback: SimliClientEvents[K],
+    ): void {
+        if (!this.persistent_events.has(event)) {
+            throw "Event Not Regsitered"
+        }
+        this.persistent_events.get(event)?.delete(callback as EventCallback);
+        this.connection.off(event, callback)
+    }
+
+    constructor(
+        session_token: session_token,
+        videoElement: HTMLVideoElement,
+        audioElement: HTMLAudioElement,
+        iceServers: RTCIceServer[] | null,
+        logLevel: LogLevel = LogLevel.DEBUG,
+        transport_mode: TransportMode = "p2p",
+        signaling: SignalingMode = "websockets",
+        SimliWSURL: string = "wss://api.simli.ai",
+        audioBufferSize: number = 3000,
+    ) {
+        if (audioBufferSize <= 0) {
+            throw "Invalid Buffer Size, Can't be negative"
+        }
+        if (Math.floor(audioBufferSize) - audioBufferSize != 0) {
+            throw "Invalid Buffer Size, Can't be a float"
+        }
+        if (!(SimliWSURL.startsWith("ws://") || SimliWSURL.startsWith("wss://")) || SimliWSURL.endsWith("/")) {
+            throw "Invalid Simli WS URL"
+        }
+        this.audioBufferSize = audioBufferSize
+        this.session_token = session_token
+        this.transport = transport_mode
+        this.signaling = signaling
+        this.SimliWSURL = SimliWSURL
+        this.videoElement = videoElement
+        this.audioElement = audioElement
+        this.iceServers = iceServers
+        this.logger = new Logger(logLevel);
+
+        let resolveFn: () => void;
+        let rejectFn: () => void;
+
+        this.connectionPromise = new Promise<void>((resolve, reject) => {
+            resolveFn = resolve;
+            rejectFn = reject;
+        });
+
+        this.connectionResolve = resolveFn!;
+        this.connectionReject = rejectFn!;
+        this.persistent_events = new Map()
+        this.connectionTimeout = setTimeout(() => this.connectionReject("CONNECTION TIMED OUT"), this.CONNECTION_TIMEOUT_MS)
+
+        switch (this.transport) {
+            case "livekit":
+                this.connection = new LivekitTransport(this.SimliWSURL, this.session_token, videoElement, audioElement, this.logger, this.connectionReject)
+                break;
+            case "p2p":
+                if (!iceServers || iceServers.length == 0) {
+                    throw "Ice Servers Required for P2P Mode"
+                }
+                this.connection = new P2PTransport(this.SimliWSURL, this.session_token, true, iceServers, videoElement, audioElement, this.logger, this.connectionReject)
+                break
+            default:
+                throw new Error("Not Implemented Yet")
+
+        }
+        this.connection.on("start", () => {
+            this.connectionResolve()
+            clearTimeout(this.connectionTimeout)
+        })
+        this.connection.on("unknown", (message) => this.logger.debug("UNKOWN MESSAGE FROM SERVER: " + message))
+        this.connection.on("error", (message) => { this.failReason = message; this.connectionReject(message) })
+
+    }
+
+    private resetConnections(videoElement: HTMLVideoElement, audioElement: HTMLAudioElement, iceServers: RTCIceServer[] | null) {
+        this.failReason = null
+        let resolveFn: () => void;
+        let rejectFn: () => void;
+
+        this.connectionPromise = new Promise<void>((resolve, reject) => {
+            resolveFn = resolve;
+            rejectFn = reject;
+        });
+
+        this.connectionResolve = resolveFn!;
+        this.connectionReject = rejectFn!;
+        this.connectionTimeout = setTimeout(() => this.connectionReject("Connection Timed Out"), this.CONNECTION_TIMEOUT_MS)
+
+        switch (this.transport) {
+            case "livekit":
+                this.connection = new LivekitTransport(this.SimliWSURL, this.session_token, videoElement, audioElement, this.logger, this.connectionReject)
+                break;
+            case "p2p":
+                if (!iceServers || iceServers.length == 0) {
+                    throw "Ice Servers Required for P2P Mode"
+                }
+                this.connection = new P2PTransport(this.SimliWSURL, this.session_token, true, iceServers, videoElement, audioElement, this.logger, this.connectionReject)
+                break
+            default:
+                throw new Error("Not Implemented Yet")
+
+        }
+        this.connection.on("start", () => {
+            this.connectionResolve()
+            clearTimeout(this.connectionTimeout)
+        })
+        this.connection.on("error", (message) => { this.retryAttempt = this.MAX_RETRY_ATTEMPTS; this.connectionReject(message) })
+        this.connection.on("unknown", (message) => this.logger.debug("UNKOWN MESSAGE FROM SERVER: " + message))
+        // Re-register all user event handlers on the new connection
+        this.persistent_events.forEach((callbacks, event) => {
+            callbacks.forEach((callback) => {
+                this.connection.on(event as keyof SimliClientEvents, callback);
+            });
+        });
+    }
+    async start(): Promise<void> {
+        if (this.shouldStop) {
+            throw new Error("Disconnect Already Called, Can't reuse same SimliClient multiple times create a new SimliClient Object")
+        }
+        try {
+            await this.connection.connect()
+            await this.connectionPromise
+            this.retryAttempt = 0
+        } catch (error) {
+            if (this.failReason) {
+                throw error
+            }
+            if (this.retryAttempt >= this.MAX_RETRY_ATTEMPTS)
+                throw new Error("Too Many Retry Attempts Failed to connect")
+            if (this.shouldStop) {
+                this.shouldStop = false
+                throw new Error("Called Disconnect Before A Connecction succeeded")
+            }
+            this.logger.error("FAILED: " + error)
+            await this.connection.disconnect()
+            await new Promise(resolve => setTimeout(resolve, this.RETRY_DELAY));
+            this.retryAttempt += 1
+            if (this.retryAttempt > 2)
+                this.transport = "livekit"
+            this.resetConnections(this.videoElement, this.audioElement, this.iceServers)
+            await this.start()
+        }
+    }
+
+    async stop() {
+        this.shouldStop = true
+        await this.connection.disconnect()
+    }
+
+    listenToMediastreamTrack(stream: MediaStreamTrack) {
+        const source = this.audioContext.createMediaStreamSource(
+            new MediaStream([stream]),
+        );
+        this.sourceNode = source;
+        this.attachSourceToWorklet(this.audioContext, source);
+    }
+
+    listenToAudioElement(audioEl: HTMLAudioElement) {
+        const source = this.audioContext.createMediaElementSource(audioEl);
+        // No connection to audioContext.destination on purpose: createMediaElementSource
+        // reroutes the element's audio into the graph, so leaving the destination
+        // unconnected silences the <audio> while samples still flow to Simli.
+        this.attachSourceToWorklet(this.audioContext, source);
+    }
+
+    private attachSourceToWorklet(
+        audioContext: AudioContext,
+        source: AudioNode,
+    ) {
+        audioContext.audioWorklet
+            .addModule(
+                URL.createObjectURL(
+                    new Blob([AudioProcessor(this.audioBufferSize)], {
+                        type: "application/javascript",
+                    }),
+                ),
+            )
+            .then(() => {
+                this.audioWorklet = new AudioWorkletNode(
+                    audioContext,
+                    "audio-processor",
+                );
+                if (this.audioWorklet === null) {
+                    throw new Error("SIMLI: AudioWorklet not initialized");
+                }
+                source.connect(this.audioWorklet);
+                this.audioWorklet.port.onmessage = (event) => {
+                    if (event.data.type === "audioData") {
+                        this.connection.signalingConnection.sendAudioData(new Uint8Array(event.data.data.buffer));
+                    }
+                };
+            })
+    }
+
+
+    public ClearBuffer = () => {
+        this.connection.signalingConnection.sendSignal("SKIP");
+    };
+    sendAudioData(audioData: Uint8Array) {
+        this.connection.signalingConnection.sendAudioData(audioData);
+    }
+
+    sendAudioDataImmediate(audioData: Uint8Array) {
+        this.connection.signalingConnection.sendAudioDataImmediate(audioData);
+    }
+
+}
+
+export { SimliClient, generateSimliSessionToken, generateIceServers, Logger, LogLevel }
+export type { SimliSessionRequest };

lib/config.ts

+
+interface SimliClientConfig {
+    faceID: string;
+    handleSilence: boolean;
+    maxSessionLength: number;
+    maxIdleTime: number;
+    enableSFU: boolean;
+    model: "fasttalk" | "artalk";
+}
+export type { SimliClientConfig }

lib/events.ts

+interface SimliClientEvents {
+    start: () => void;
+    stop: () => void;
+    error: (detail: string) => void;
+    ack: () => void;
+    connection_info: (serialized_info: string) => void;
+    video_info: (serialized_info: string) => void;
+    destination: (serialized_info: string) => void;
+    speaking: () => void;
+    silent: () => void;
+    unknown: (message: string) => void;
+    startup_error: (message: string) => void
+}
+export type { SimliClientEvents }

lib/signaling/BaseSignaling.ts

+import { Logger } from "../Logger";
+
+type ClientSignals = "DONE" | "SKIP"
+interface BaseSignaling {
+    logger: Logger
+    connect(connected: () => void): Promise<void>;
+    disconnect(): void;
+    sendSignal(data: ClientSignals): void;
+    sendAudioData(audioData: Uint8Array): void;
+    sendAudioDataImmediate(audioData: Uint8Array): void;
+}
+export type { BaseSignaling, ClientSignals }

lib/signaling/WebSocketSignaling.ts

+import { Logger, LogLevel } from "../Logger"
+import { BaseSignaling, ClientSignals } from "./BaseSignaling"
+
+class WebSocketSignaling implements BaseSignaling {
+    wsURL: string | URL
+    wsConnection: WebSocket
+    logger: Logger
+    constructor(wsURL: string | URL, logger: Logger) {
+        this.wsURL = wsURL
+        this.wsConnection = new WebSocket(this.wsURL)
+        this.wsConnection.addEventListener("message", (message) => (this.logger.debug(message.data)))
+        this.logger = logger
+    }
+    async connect(connected: () => void): Promise<void> {
+        this.wsConnection.onopen = connected
+    }
+
+    disconnect(): void {
+        this.wsConnection.close()
+    }
+    private send(data: Uint8Array | string) {
+        if (this.wsConnection.readyState != WebSocket.OPEN) {
+            throw `Invalid State, WS Connection ${this.wsConnection.readyState.toString()}`
+        }
+        this.wsConnection.send(data)
+    }
+
+    sendOffer(offer: RTCSessionDescription): void {
+        this.send(JSON.stringify(offer))
+    }
+    sendSignal(data: ClientSignals): void {
+        this.send(data)
+    }
+
+    sendAudioData(audioData: Uint8Array) {
+        if (this.logger.getLevel() === LogLevel.DEBUG)
+            this.logger.debug("Sent Audio of length: " + (audioData.length / 32000).toString())
+        this.send(audioData);
+    }
+
+    sendAudioDataImmediate(audioData: Uint8Array) {
+        if (this.logger.getLevel() === LogLevel.DEBUG)
+            this.logger.debug("Sent Audio of length for immediate playback: " + (audioData.length / 32000).toString())
+        const asciiStr = "PLAY_IMMEDIATE";
+        const encoder = new TextEncoder(); // Default is utf-8
+        const strBytes = encoder.encode(asciiStr); // Uint8Array of " World!"
+        const buffer = new Uint8Array(strBytes.length + audioData.length);
+        buffer.set(strBytes, 0);
+        buffer.set(audioData, strBytes.length);
+        this.send(buffer);
+    }
+}
+export { WebSocketSignaling }

lib/transports/BaseTransport.ts

+import { SimliClientEvents } from "../Events";
+import { Logger } from "../Logger";
+import { BaseSignaling } from "../Signaling/BaseSignaling";
+
+type EventCallback = (...args: any[]) => void;
+type EventMap = Map<string, Set<EventCallback>>;
+
+interface BaseTransport {
+    videoElementAnchor: HTMLVideoElement
+    audioElementAnchor: HTMLAudioElement
+    signalingConnection: BaseSignaling
+    session_token: string
+    events: EventMap;
+    logger: Logger;
+    connect(): Promise<void>;
+    disconnect(): void;
+    on<K extends keyof SimliClientEvents>(
+        event: K,
+        callback: SimliClientEvents[K]
+    ): void;
+
+    off<K extends keyof SimliClientEvents>(
+        event: K,
+        callback: SimliClientEvents[K],
+    ): void;
+
+    emit<K extends keyof SimliClientEvents>(
+        event: K,
+        ...args: Parameters<SimliClientEvents[K]>
+    ): void;
+
+}
+
+function register_destination(logger: Logger, serialized_info: string) {
+    const parsed = JSON.parse(serialized_info)
+    logger.destination = parsed.destination
+    logger.session_id = parsed.session_id
+}
+async function handleMessage(transport: BaseTransport, message: MessageEvent): Promise<void> {
+    const firstToken = (message.data as string).toUpperCase().split(" ")[0]
+    switch (firstToken) {
+        case "START": {
+            // SOFT IGNORE
+            break
+        }
+        case "ACK": {
+            transport.emit("ack")
+            break;
+        }
+        case "STOP": {
+            transport.disconnect();
+            transport.emit("stop")
+            break;
+        }
+        case "CLOSING":
+        case "RATE":
+        case "ERROR":
+        case "ERROR:": {
+            transport.disconnect()
+            transport.emit("error", message.data as string)
+        }
+        case "SPEAK": {
+            transport.emit("speaking");
+            break
+        }
+        case "SILENT": {
+            transport.emit("silent");
+            break
+        }
+        default: {
+            if (firstToken.includes("SDP") || firstToken.includes("LIVEKIT")) {
+                transport.emit("connection_info", message.data)
+            } else if (firstToken.includes("VIDEO_METADATA")) {
+                transport.emit("video_info", message.data)
+            } else if (firstToken.includes("ENDFRAME")) {
+                transport.emit("stop")
+                transport.disconnect()
+            } else if (firstToken.includes("DESTINATION")) {
+                transport.emit("destination", message.data)
+            } else {
+                transport.emit("unknown", message.data)
+            }
+        }
+
+    }
+}
+
+export { handleMessage, register_destination };
+export type {
+    BaseTransport, EventCallback, EventMap
+};

lib/transports/LivekitTransport.ts

+import { RemoteParticipant, RemoteTrack, RemoteTrackPublication, Room, RoomEvent, RoomOptions, Track } from "livekit-client";
+import { WebSocketSignaling } from "../Signaling/WebSocketSignaling";
+import { SimliClientEvents } from "../Events";
+import { BaseTransport, EventCallback, EventMap, handleMessage, register_destination } from "./BaseTransport";
+import { Logger } from "../Logger";
+
+class LivekitTransport implements BaseTransport {
+    videoElementAnchor: HTMLVideoElement
+    audioElementAnchor: HTMLAudioElement
+    signalingConnection: WebSocketSignaling;
+    session_token: string;
+    pc: Room
+    logger: Logger
+    events: EventMap = new Map()
+    private websocketPromise: Promise<unknown>;
+    private websocketReject: ((reason: string) => void) | null = null;
+    constructor(
+        simliBaseWSURL: string,
+        session_token: string,
+        videoElementAnchor: HTMLVideoElement,
+        audioElementAnchor: HTMLAudioElement,
+        logger: Logger,
+        failSignal: (message: string) => void
+    ) {
+        this.logger = logger
+        this.on("startup_error", failSignal)
+
+        this.session_token = session_token
+        const wsURL = new URL(simliBaseWSURL + "/compose/webrtc/livekit")
+        wsURL.searchParams.set("session_token", session_token)
+        this.signalingConnection = new WebSocketSignaling(wsURL, this.logger)
+        this.on("destination", (serilized_info) => register_destination(this.logger, serilized_info))
+        this.websocketPromise = new Promise(
+            (resolve, reject) => {
+                this.websocketReject = reject;
+                this.signalingConnection.connect(() => {
+                    resolve("success")
+                    this.logger.debug("LK WebSocket Connected")
+                })
+            }
+        )
+        this.signalingConnection.wsConnection.onmessage = (message) => { handleMessage(this, message) }
+        this.signalingConnection.wsConnection.onerror = (evt) => {
+            this.emit("startup_error", "Websocket Failed");
+            if (this.websocketReject) {
+                this.websocketReject("Websocket Failed");
+                this.websocketReject = null; // Prevent multiple rejections
+            }
+        }
+        const options: RoomOptions = { adaptiveStream: false, dynacast: true }
+        this.pc = new Room(options);
+        this.on("connection_info", (serialized_info) => this.join_lk_room(serialized_info))
+        this.videoElementAnchor = videoElementAnchor
+        this.audioElementAnchor = audioElementAnchor
+
+    }
+    public on<K extends keyof SimliClientEvents>(
+        event: K,
+        callback: SimliClientEvents[K]
+    ): void {
+        if (!this.events.has(event)) {
+            this.events.set(event, new Set());
+        }
+        this.events.get(event)?.add(callback as EventCallback);
+        this.logger.debug("Registered Callback for Event: " + event)
+    }
+
+    public off<K extends keyof SimliClientEvents>(
+        event: K,
+        callback: SimliClientEvents[K],
+    ): void {
+        if (!this.events.has(event)) {
+            throw "Event Not Regsitered"
+        }
+        this.events.get(event)?.delete(callback as EventCallback);
+    }
+
+    emit<K extends keyof SimliClientEvents>(
+        event: K,
+        ...args: Parameters<SimliClientEvents[K]>
+    ): void {
+        this.logger.debug("Event: " + event)
+        this.events.get(event)?.forEach((callback) => {
+            callback(...args);
+        });
+    }
+
+    async connect() {
+        this.logger.info("Connecting")
+        this.setupConnectionStateHandler()
+        await this.websocketPromise
+    }
+
+    async disconnect() {
+        this.logger.info("Disconnecting")
+        try {
+            this.signalingConnection.sendSignal("DONE")
+        }
+        catch {
+            this.logger.error("FAILED TO SEND FINAL MESSAGE")
+        }
+        try {
+            this.signalingConnection.disconnect()
+        } catch {
+            this.logger.error("SIGNALING ALREADY DISCONNECTED")
+        }
+        try {
+            await this.pc.disconnect()
+        } catch {
+            this.logger.error("LOCAL PEER ALREADY CLOSED")
+        }
+
+    }
+
+    private async join_lk_room(serialized_info: string) {
+        const info = JSON.parse(serialized_info)
+        this.logger.debug(info)
+        if (info.livekit_url && info.livekit_token) {
+            await this.pc.connect(info.livekit_url, info.livekit_token);
+        } else {
+            this.disconnect()
+            this.emit("error", "Invalid Join Info, Contact Simli For Support")
+        }
+    }
+    private setupConnectionStateHandler() {
+        this.pc.on(RoomEvent.Disconnected, () => {
+            this.disconnect();
+        })
+
+        this.pc.on(RoomEvent.Connected, () => {
+        })
+        this.pc.on(RoomEvent.TrackSubscribed, (track: RemoteTrack,
+            publication: RemoteTrackPublication,
+            participant: RemoteParticipant,
+        ) => {
+            this.logger.debug("Track Received: " + track.kind)
+            if (track.kind === Track.Kind.Video) {
+                track.attach(this.videoElementAnchor)
+                this.videoElementAnchor.requestVideoFrameCallback(() => {
+                    this.emit("start");
+                });
+            } else if (track.kind === Track.Kind.Audio) {
+                track.attach(this.audioElementAnchor)
+            }
+        })
+    };
+
+
+}
+
+export { LivekitTransport }

lib/transports/P2PTransport.ts

+import { WebSocketSignaling } from "../Signaling/WebSocketSignaling";
+import { SimliClientEvents } from "../Events";
+import { BaseTransport, EventCallback, EventMap, handleMessage, register_destination } from "./BaseTransport";
+import { Logger } from "../Logger";
+
+class P2PTransport implements BaseTransport {
+    videoElementAnchor: HTMLVideoElement
+    audioElementAnchor: HTMLAudioElement
+    signalingConnection: WebSocketSignaling;
+    session_token: string;
+    pc: RTCPeerConnection;
+    events: EventMap = new Map()
+    logger: Logger;
+    private iceCandidateCount: number;
+    private previousIceCandidateCount: number;
+    private iceTimeout: NodeJS.Timeout | null = null;
+    private websocketPromise: Promise<unknown>;
+    private websocketReject: ((reason: string) => void) | null = null;
+    constructor(
+        simliBaseWSURL: string,
+        session_token: string,
+        enableSFU: boolean,
+        iceServers: RTCIceServer[],
+        videoElementAnchor: HTMLVideoElement,
+        audioElementAnchor: HTMLAudioElement,
+        logger: Logger,
+        failSignal: (message: string) => void,
+    ) {
+        this.logger = logger
+        this.on("startup_error", failSignal)
+
+        this.session_token = session_token
+        const wsURL = new URL(simliBaseWSURL + "/compose/webrtc/p2p")
+        wsURL.searchParams.set("session_token", session_token)
+        wsURL.searchParams.set("enableSFU", String(enableSFU))
+        this.on("destination", (serilized_info) => register_destination(this.logger, serilized_info))
+        this.signalingConnection = new WebSocketSignaling(wsURL, this.logger)
+        this.websocketPromise = new Promise(
+            (resolve, reject) => {
+                this.websocketReject = reject;
+                this.signalingConnection.connect(() => {
+                    resolve("success")
+                    this.logger.debug("P2P WebSocket Connected")
+                })
+            }
+        )
+        this.signalingConnection.wsConnection.onmessage = (message) => { handleMessage(this, message) }
+        this.signalingConnection.wsConnection.onerror = (evt) => {
+            this.emit("startup_error", "Websocket Failed");
+            if (this.websocketReject) {
+                this.websocketReject("Websocket Failed");
+                this.websocketReject = null; // Prevent multiple rejections
+            }
+        }
+        this.on("connection_info", (serialized_info) => this.registerPeerInfo(serialized_info))
+        this.videoElementAnchor = videoElementAnchor
+        this.audioElementAnchor = audioElementAnchor
+        this.iceCandidateCount = 0
+        this.previousIceCandidateCount = 0
+        const config = {
+            sdpSemantics: "unified-plan",
+            iceServers: iceServers,
+        };
+        this.pc = new window.RTCPeerConnection(config);
+        this.pc.addTransceiver("audio", {
+            direction: "recvonly",
+        });
+        this.pc.addTransceiver("video", {
+            direction: "recvonly",
+        })
+    }
+    public on<K extends keyof SimliClientEvents>(
+        event: K,
+        callback: SimliClientEvents[K]
+    ): void {
+        if (!this.events.has(event)) {
+            this.events.set(event, new Set());
+        }
+        this.events.get(event)?.add(callback as EventCallback);
+    }
+
+    public off<K extends keyof SimliClientEvents>(
+        event: K,
+        callback: SimliClientEvents[K],
+    ): void {
+        this.events.get(event)?.delete(callback as EventCallback);
+    }
+
+    emit<K extends keyof SimliClientEvents>(
+        event: K,
+        ...args: Parameters<SimliClientEvents[K]>
+    ): void {
+        this.events.get(event)?.forEach((callback) => {
+            try {
+                callback(...args);
+            }
+            catch {
+                this.logger.error("CALLBACK FAILED: " + callback.name)
+            }
+        });
+
+    }
+    async connect() {
+
+        const offer = await this.pc.createOffer();
+        await this.pc.setLocalDescription(offer);
+        await this.waitForIceGathering();
+        this.setupPeerConnectionListeners()
+        await this.websocketPromise
+        if (this.pc.localDescription) {
+            this.signalingConnection.sendOffer(this.pc.localDescription)
+        }
+
+    }
+
+    async disconnect() {
+        this.logger.info("Disconnecting")
+        try {
+            this.signalingConnection.sendSignal("DONE")
+        }
+        catch {
+            this.logger.error("FAILED TO SEND FINAL MESSAGE")
+        }
+        try {
+            this.signalingConnection.disconnect()
+        } catch {
+            this.logger.error("SIGNALING ALREADY DISCONNECTED")
+        }
+        try {
+            this.pc.close()
+        } catch {
+            this.logger.error("LOCAL PEER ALREADY CLOSED")
+        }
+
+    }
+    private async registerPeerInfo(serialized_info: string) {
+        const info = JSON.parse(serialized_info)
+        if (info.sdp && info.type == "answer") {
+            await this.pc.setRemoteDescription(new RTCSessionDescription(info));
+        } else {
+            this.disconnect()
+            this.emit("error", "Invalid Join Info, Contact Simli For Support")
+        }
+    }
+
+    private async waitForIceGathering(): Promise<void> {
+        this.iceCandidateCount = 0;
+        this.previousIceCandidateCount = 0;
+        if (this.pc.iceGatheringState === "complete") {
+            return;
+        }
+
+        return new Promise<void>((resolve, reject) => {
+            if (!this.iceTimeout) {
+                this.iceTimeout = setTimeout(() => {
+                    reject(new Error("ICE gathering timeout"));
+                }, 10000);
+            }
+
+            const checkIceCandidates = () => {
+                if (
+                    this.pc.iceGatheringState === "complete" ||
+                    this.iceCandidateCount === this.previousIceCandidateCount
+                ) {
+                    if (this.iceTimeout) {
+                        clearTimeout(this.iceTimeout);
+                    }
+                    resolve();
+                } else {
+                    this.previousIceCandidateCount = this.iceCandidateCount;
+                    setTimeout(checkIceCandidates, 150);
+                }
+            };
+
+            checkIceCandidates();
+        });
+    }
+
+    private setupPeerConnectionListeners() {
+        this.pc.addEventListener("track", (evt) => {
+            if (evt.track.kind === "video") {
+                this.videoElementAnchor.srcObject = evt.streams[0];
+                this.videoElementAnchor.requestVideoFrameCallback(() => {
+                    this.emit("start")
+                });
+            } else if (evt.track.kind === "audio" && this.audioElementAnchor) {
+                this.audioElementAnchor.srcObject = evt.streams[0];
+            }
+        });
+
+        this.pc.onicecandidate = (event) => {
+            if (event.candidate !== null) {
+                this.iceCandidateCount += 1;
+            }
+        };
+    }
+}
+
+export { P2PTransport }

dist/Transports/LivekitTransport.js

@@ -40,3 +40,3 @@ "use strict";
         };
-        const options = { adaptiveStream: true, dynacast: true };
+        const options = { adaptiveStream: false, dynacast: true };
         this.pc = new livekit_client_1.Room(options);
@@ -43,0 +43,0 @@ this.on("connection_info", (serialized_info) => this.join_lk_room(serialized_info));

package.json

 {
     "name": "simli-client",
-    "version": "3.0.1",
+    "version": "3.0.2",
     "description": "Simli WebRTC Client",
@@ -5,0 +5,0 @@ "main": "dist/index.js",

dist/Client.d.ts

-import { SimliClientEvents } from './Events';
-import { Logger, LogLevel } from './Logger';
-interface SimliSessionRequest {
-    faceId: string;
-    handleSilence: boolean;
-    maxSessionLength: number;
-    maxIdleTime: number;
-    model?: "fasttalk" | "artalk";
-}
-interface TokenRequestData {
-    config: SimliSessionRequest;
-    apiKey: string;
-}
-interface SimliSessionToken {
-    session_token: string;
-}
-type TransportMode = "livekit" | "p2p";
-type SignalingMode = "websockets";
-type session_token = string;
-declare function generateSimliSessionToken(request: TokenRequestData, SimliURL?: string): Promise<SimliSessionToken>;
-declare function generateIceServers(apiKey: string, SimliURL?: string): Promise<RTCIceServer[]>;
-declare class SimliClient {
-    session_token: string;
-    transport: TransportMode;
-    signaling: SignalingMode;
-    videoElement: HTMLVideoElement;
-    audioElement: HTMLAudioElement;
-    audioBufferSize: number;
-    private connection;
-    private connectionTimeout;
-    private connectionResolve;
-    private connectionReject;
-    private connectionPromise;
-    private sourceNode;
-    private audioWorklet;
-    private readonly MAX_RETRY_ATTEMPTS;
-    private RETRY_DELAY;
-    private readonly CONNECTION_TIMEOUT_MS;
-    private retryAttempt;
-    private SimliWSURL;
-    private audioContext;
-    private logger;
-    private iceServers;
-    private persistent_events;
-    private failReason;
-    private shouldStop;
-    on<K extends keyof SimliClientEvents>(event: K, callback: SimliClientEvents[K]): void;
-    off<K extends keyof SimliClientEvents>(event: K, callback: SimliClientEvents[K]): void;
-    constructor(session_token: session_token, videoElement: HTMLVideoElement, audioElement: HTMLAudioElement, iceServers: RTCIceServer[] | null, logLevel?: LogLevel, transport_mode?: TransportMode, signaling?: SignalingMode, SimliWSURL?: string, audioBufferSize?: number);
-    private resetConnections;
-    start(): Promise<void>;
-    stop(): Promise<void>;
-    listenToMediastreamTrack(stream: MediaStreamTrack): void;
-    private initializeAudioWorklet;
-    ClearBuffer: () => void;
-    sendAudioData(audioData: Uint8Array): void;
-    sendAudioDataImmediate(audioData: Uint8Array): void;
-}
-export { SimliClient, generateSimliSessionToken, generateIceServers, Logger, LogLevel };
-export type { SimliSessionRequest };

dist/Client.js

-"use strict";
-Object.defineProperty(exports, "__esModule", { value: true });
-exports.LogLevel = exports.Logger = exports.SimliClient = void 0;
-exports.generateSimliSessionToken = generateSimliSessionToken;
-exports.generateIceServers = generateIceServers;
-const LivekitTransport_1 = require("./Transports/LivekitTransport");
-const P2PTransport_1 = require("./Transports/P2PTransport");
-const Logger_1 = require("./Logger");
-Object.defineProperty(exports, "Logger", { enumerable: true, get: function () { return Logger_1.Logger; } });
-Object.defineProperty(exports, "LogLevel", { enumerable: true, get: function () { return Logger_1.LogLevel; } });
-const AudioProcessor = (buffer) => {
-    if (buffer <= 0) {
-        throw "Invalid Buffer Size, Can't be negative";
-    }
-    if (Math.floor(buffer) - buffer != 0) {
-        throw "Invalid Buffer Size, Can't be a float";
-    }
-    return `
-        class AudioProcessor extends AudioWorkletProcessor {
-          constructor() {
-            super();
-            this.buffer = new Int16Array(${buffer});
-            this.bufferIndex = 0;
-        }
-
-          process(inputs, outputs, parameters) {
-            const input = inputs[0];
-            const inputChannel = input[0];
-            if (inputChannel) {
-              for (let i = 0; i < inputChannel.length; i++) {
-                this.buffer[this.bufferIndex] = Math.max(-32768, Math.min(32767, Math.round(inputChannel[i] * 32767)));
-                this.bufferIndex++;
-
-                if (this.bufferIndex === this.buffer.length){
-                  this.port.postMessage({type: 'audioData', data: this.buffer.slice(0, this.bufferIndex)});
-                  this.bufferIndex = 0;
-                }
-              }
-            }
-            return true;
-          }
-        }
-
-        registerProcessor('audio-processor', AudioProcessor);
-      `;
-};
-async function generateSimliSessionToken(request, SimliURL = "https://api.simli.ai") {
-    const url = `${SimliURL}/compose/token`;
-    const response = await fetch(url, {
-        method: "POST",
-        body: JSON.stringify(request.config),
-        headers: {
-            "Content-Type": "application/json",
-            "x-simli-api-key": request.apiKey
-        },
-    });
-    if (!response.ok) {
-        const errorText = await response.text();
-        throw errorText;
-    }
-    const resJSON = await response.json();
-    return resJSON;
-}
-async function generateIceServers(apiKey, SimliURL = "https://api.simli.ai") {
-    try {
-        const url = `${SimliURL}/compose/ice`;
-        const response = await fetch(url, {
-            headers: {
-                "Content-Type": "application/json",
-                "x-simli-api-key": apiKey,
-            },
-            method: "GET",
-        });
-        if (!response.ok) {
-            throw new Error(`SIMLI: HTTP error! status: ${response.status}`);
-        }
-        const iceServers = await response.json();
-        if (!iceServers || iceServers.length === 0) {
-            throw new Error("SIMLI: No ICE servers returned");
-        }
-        return iceServers;
-    }
-    catch (error) {
-        return [{ urls: ["stun:stun.l.google.com:19302"] }];
-    }
-}
-class SimliClient {
-    session_token;
-    transport = "livekit";
-    signaling = "websockets";
-    videoElement;
-    audioElement;
-    audioBufferSize = 3000;
-    connection;
-    connectionTimeout;
-    connectionResolve;
-    connectionReject;
-    connectionPromise;
-    sourceNode = null;
-    audioWorklet = null;
-    MAX_RETRY_ATTEMPTS = 10;
-    RETRY_DELAY = 2000;
-    CONNECTION_TIMEOUT_MS = 15000;
-    retryAttempt = 0;
-    SimliWSURL = "wss://api.simli.ai";
-    audioContext = new (window.AudioContext ||
-        window.webkitAudioContext)({
-        sampleRate: 16000,
-    });
-    logger;
-    iceServers;
-    persistent_events;
-    failReason = null;
-    shouldStop = false;
-    // Type-safe event methods
-    on(event, callback) {
-        if (!this.persistent_events.has(event)) {
-            this.persistent_events.set(event, new Set());
-        }
-        this.persistent_events.get(event)?.add(callback);
-        this.logger.debug("Registered Callback for Event: " + event);
-        this.connection.on(event, callback);
-    }
-    off(event, callback) {
-        if (!this.persistent_events.has(event)) {
-            throw "Event Not Regsitered";
-        }
-        this.persistent_events.get(event)?.delete(callback);
-        this.connection.off(event, callback);
-    }
-    constructor(session_token, videoElement, audioElement, iceServers, logLevel = Logger_1.LogLevel.DEBUG, transport_mode = "p2p", signaling = "websockets", SimliWSURL = "wss://api.simli.ai", audioBufferSize = 3000) {
-        if (audioBufferSize <= 0) {
-            throw "Invalid Buffer Size, Can't be negative";
-        }
-        if (Math.floor(audioBufferSize) - audioBufferSize != 0) {
-            throw "Invalid Buffer Size, Can't be a float";
-        }
-        if (!(SimliWSURL.startsWith("ws://") || SimliWSURL.startsWith("wss://")) || SimliWSURL.endsWith("/")) {
-            throw "Invalid Simli WS URL";
-        }
-        this.session_token = session_token;
-        this.transport = transport_mode;
-        this.signaling = signaling;
-        this.SimliWSURL = SimliWSURL;
-        this.videoElement = videoElement;
-        this.audioElement = audioElement;
-        this.iceServers = iceServers;
-        this.logger = new Logger_1.Logger(logLevel);
-        let resolveFn;
-        let rejectFn;
-        this.connectionPromise = new Promise((resolve, reject) => {
-            resolveFn = resolve;
-            rejectFn = reject;
-        });
-        this.connectionResolve = resolveFn;
-        this.connectionReject = rejectFn;
-        this.persistent_events = new Map();
-        this.connectionTimeout = setTimeout(() => this.connectionReject("CONNECTION TIMED OUT"), this.CONNECTION_TIMEOUT_MS);
-        switch (this.transport) {
-            case "livekit":
-                this.connection = new LivekitTransport_1.LivekitTransport(this.SimliWSURL, this.session_token, videoElement, audioElement, this.logger, this.connectionReject);
-                break;
-            case "p2p":
-                if (!iceServers || iceServers.length == 0) {
-                    throw "Ice Servers Required for P2P Mode";
-                }
-                this.connection = new P2PTransport_1.P2PTransport(this.SimliWSURL, this.session_token, true, iceServers, videoElement, audioElement, this.logger, this.connectionReject);
-                break;
-            default:
-                throw new Error("Not Implemented Yet");
-        }
-        this.connection.on("start", () => {
-            this.connectionResolve();
-            clearTimeout(this.connectionTimeout);
-        });
-        this.connection.on("unknown", (message) => this.logger.debug("UNKOWN MESSAGE FROM SERVER: " + message));
-        this.connection.on("error", (message) => { this.failReason = message; this.connectionReject(message); });
-    }
-    resetConnections(videoElement, audioElement, iceServers) {
-        this.failReason = null;
-        let resolveFn;
-        let rejectFn;
-        this.connectionPromise = new Promise((resolve, reject) => {
-            resolveFn = resolve;
-            rejectFn = reject;
-        });
-        this.connectionResolve = resolveFn;
-        this.connectionReject = rejectFn;
-        this.connectionTimeout = setTimeout(() => this.connectionReject("Connection Timed Out"), this.CONNECTION_TIMEOUT_MS);
-        switch (this.transport) {
-            case "livekit":
-                this.connection = new LivekitTransport_1.LivekitTransport(this.SimliWSURL, this.session_token, videoElement, audioElement, this.logger, this.connectionReject);
-                break;
-            case "p2p":
-                if (!iceServers || iceServers.length == 0) {
-                    throw "Ice Servers Required for P2P Mode";
-                }
-                this.connection = new P2PTransport_1.P2PTransport(this.SimliWSURL, this.session_token, true, iceServers, videoElement, audioElement, this.logger, this.connectionReject);
-                break;
-            default:
-                throw new Error("Not Implemented Yet");
-        }
-        this.connection.on("start", () => {
-            this.connectionResolve();
-            clearTimeout(this.connectionTimeout);
-        });
-        this.connection.on("error", (message) => { this.retryAttempt = this.MAX_RETRY_ATTEMPTS; this.connectionReject(message); });
-        this.connection.on("unknown", (message) => this.logger.debug("UNKOWN MESSAGE FROM SERVER: " + message));
-        // Re-register all user event handlers on the new connection
-        this.persistent_events.forEach((callbacks, event) => {
-            callbacks.forEach((callback) => {
-                this.connection.on(event, callback);
-            });
-        });
-    }
-    async start() {
-        if (this.shouldStop) {
-            throw new Error("Disconnect Already Called, Can't reuse same SimliClient multiple times create a new SimliClient Object");
-        }
-        try {
-            await this.connection.connect();
-            await this.connectionPromise;
-            this.retryAttempt = 0;
-        }
-        catch (error) {
-            if (this.failReason) {
-                throw error;
-            }
-            if (this.retryAttempt >= this.MAX_RETRY_ATTEMPTS)
-                throw new Error("Too Many Retry Attempts Failed to connect");
-            if (this.shouldStop) {
-                this.shouldStop = false;
-                throw new Error("Called Disconnect Before A Connecction succeeded");
-            }
-            this.logger.error("FAILED: " + error);
-            await this.connection.disconnect();
-            await new Promise(resolve => setTimeout(resolve, this.RETRY_DELAY));
-            this.retryAttempt += 1;
-            if (this.retryAttempt > 2)
-                this.transport = "livekit";
-            this.resetConnections(this.videoElement, this.audioElement, this.iceServers);
-            await this.start();
-        }
-    }
-    async stop() {
-        this.shouldStop = true;
-        await this.connection.disconnect();
-    }
-    listenToMediastreamTrack(stream) {
-        this.initializeAudioWorklet(this.audioContext, stream);
-    }
-    initializeAudioWorklet(audioContext, stream) {
-        audioContext.audioWorklet
-            .addModule(URL.createObjectURL(new Blob([AudioProcessor(this.audioBufferSize)], {
-            type: "application/javascript",
-        })))
-            .then(() => {
-            this.audioWorklet = new AudioWorkletNode(audioContext, "audio-processor");
-            this.sourceNode = audioContext.createMediaStreamSource(new MediaStream([stream]));
-            if (this.audioWorklet === null) {
-                throw new Error("SIMLI: AudioWorklet not initialized");
-            }
-            this.sourceNode.connect(this.audioWorklet);
-            this.audioWorklet.port.onmessage = (event) => {
-                if (event.data.type === "audioData") {
-                    this.connection.signalingConnection.sendAudioData(new Uint8Array(event.data.data.buffer));
-                }
-            };
-        });
-    }
-    ClearBuffer = () => {
-        this.connection.signalingConnection.sendSignal("SKIP");
-    };
-    sendAudioData(audioData) {
-        this.connection.signalingConnection.sendAudioData(audioData);
-    }
-    sendAudioDataImmediate(audioData) {
-        this.connection.signalingConnection.sendAudioDataImmediate(audioData);
-    }
-}
-exports.SimliClient = SimliClient;

dist/Config.d.ts

-interface SimliClientConfig {
-    faceID: string;
-    handleSilence: boolean;
-    maxSessionLength: number;
-    maxIdleTime: number;
-    enableSFU: boolean;
-    model: "fasttalk" | "artalk";
-}
-export type { SimliClientConfig };

dist/Config.js

-"use strict";
-Object.defineProperty(exports, "__esModule", { value: true });

lib/Client.ts

-// src/index.ts
-import { SimliClientEvents } from './Events';
-import { BaseTransport, EventCallback, EventMap } from './Transports/BaseTransport';
-import { LivekitTransport } from './Transports/LivekitTransport';
-import { P2PTransport } from './Transports/P2PTransport';
-import { Logger, LogLevel } from './Logger';
-
-const AudioProcessor = (buffer: number) => {
-    if (buffer <= 0) {
-        throw "Invalid Buffer Size, Can't be negative"
-    }
-    if (Math.floor(buffer) - buffer != 0) {
-        throw "Invalid Buffer Size, Can't be a float"
-    }
-    return `
-        class AudioProcessor extends AudioWorkletProcessor {
-          constructor() {
-            super();
-            this.buffer = new Int16Array(${buffer});
-            this.bufferIndex = 0;
-        }
-
-          process(inputs, outputs, parameters) {
-            const input = inputs[0];
-            const inputChannel = input[0];
-            if (inputChannel) {
-              for (let i = 0; i < inputChannel.length; i++) {
-                this.buffer[this.bufferIndex] = Math.max(-32768, Math.min(32767, Math.round(inputChannel[i] * 32767)));
-                this.bufferIndex++;
-
-                if (this.bufferIndex === this.buffer.length){
-                  this.port.postMessage({type: 'audioData', data: this.buffer.slice(0, this.bufferIndex)});
-                  this.bufferIndex = 0;
-                }
-              }
-            }
-            return true;
-          }
-        }
-
-        registerProcessor('audio-processor', AudioProcessor);
-      `};
-
-// Custom event handler types
-interface SimliSessionRequest {
-    faceId: string;
-    handleSilence: boolean;
-    maxSessionLength: number;
-    maxIdleTime: number;
-    model?: "fasttalk" | "artalk";
-}
-interface TokenRequestData {
-    config: SimliSessionRequest
-    apiKey: string
-}
-
-interface SimliSessionToken {
-    session_token: string;
-}
-
-type TransportMode = "livekit" | "p2p"
-type SignalingMode = "websockets"
-type session_token = string
-
-async function generateSimliSessionToken(
-    request: TokenRequestData,
-    SimliURL: string = "https://api.simli.ai",
-): Promise<SimliSessionToken> {
-    const url = `${SimliURL}/compose/token`;
-    const response = await fetch(url, {
-        method: "POST",
-        body: JSON.stringify(request.config),
-        headers: {
-            "Content-Type": "application/json",
-            "x-simli-api-key": request.apiKey
-        },
-    });
-
-    if (!response.ok) {
-        const errorText = await response.text();
-        throw errorText;
-    }
-
-    const resJSON = await response.json();
-    return resJSON;
-}
-
-async function generateIceServers(
-    apiKey: string,
-    SimliURL: string = "https://api.simli.ai",
-): Promise<RTCIceServer[]> {
-    try {
-        const url = `${SimliURL}/compose/ice`;
-        const response: any = await fetch(url, {
-            headers: {
-                "Content-Type": "application/json",
-                "x-simli-api-key": apiKey,
-            },
-            method: "GET",
-        })
-
-        if (!response.ok) {
-            throw new Error(`SIMLI: HTTP error! status: ${response.status}`);
-        }
-
-        const iceServers = await response.json();
-        if (!iceServers || iceServers.length === 0) {
-            throw new Error("SIMLI: No ICE servers returned");
-        }
-        return iceServers;
-    } catch (error) {
-        return [{ urls: ["stun:stun.l.google.com:19302"] }];
-    }
-}
-
-class SimliClient {
-
-    session_token: string;
-    transport: TransportMode = "livekit";
-    signaling: SignalingMode = "websockets"
-    videoElement: HTMLVideoElement
-    audioElement: HTMLAudioElement
-    audioBufferSize: number = 3000
-    private connection: BaseTransport
-    private connectionTimeout: NodeJS.Timeout
-    private connectionResolve: () => void
-    private connectionReject: (message: string) => void
-    private connectionPromise: Promise<void>
-    private sourceNode: MediaStreamAudioSourceNode | null = null;
-    private audioWorklet: AudioWorkletNode | null = null;
-    private readonly MAX_RETRY_ATTEMPTS = 10;
-    private RETRY_DELAY = 2000;
-    private readonly CONNECTION_TIMEOUT_MS = 15000;
-    private retryAttempt: number = 0;
-    private SimliWSURL: string = "wss://api.simli.ai";
-    private audioContext: AudioContext = new (window.AudioContext ||
-        (window as any).webkitAudioContext)({
-            sampleRate: 16000,
-        });
-    private logger: Logger;
-    private iceServers: RTCIceServer[] | null;
-    private persistent_events: EventMap;
-    private failReason: string | null = null;
-    private shouldStop: boolean = false;
-
-    // Type-safe event methods
-    public on<K extends keyof SimliClientEvents>(
-        event: K,
-        callback: SimliClientEvents[K]
-    ): void {
-        if (!this.persistent_events.has(event)) {
-            this.persistent_events.set(event, new Set());
-        }
-        this.persistent_events.get(event)?.add(callback as EventCallback);
-        this.logger.debug("Registered Callback for Event: " + event)
-        this.connection.on(event, callback)
-    }
-
-    public off<K extends keyof SimliClientEvents>(
-        event: K,
-        callback: SimliClientEvents[K],
-    ): void {
-        if (!this.persistent_events.has(event)) {
-            throw "Event Not Regsitered"
-        }
-        this.persistent_events.get(event)?.delete(callback as EventCallback);
-        this.connection.off(event, callback)
-    }
-
-    constructor(
-        session_token: session_token,
-        videoElement: HTMLVideoElement,
-        audioElement: HTMLAudioElement,
-        iceServers: RTCIceServer[] | null,
-        logLevel: LogLevel = LogLevel.DEBUG,
-        transport_mode: TransportMode = "p2p",
-        signaling: SignalingMode = "websockets",
-        SimliWSURL: string = "wss://api.simli.ai",
-        audioBufferSize: number = 3000,
-    ) {
-        if (audioBufferSize <= 0) {
-            throw "Invalid Buffer Size, Can't be negative"
-        }
-        if (Math.floor(audioBufferSize) - audioBufferSize != 0) {
-            throw "Invalid Buffer Size, Can't be a float"
-        }
-        if (!(SimliWSURL.startsWith("ws://") || SimliWSURL.startsWith("wss://")) || SimliWSURL.endsWith("/")) {
-            throw "Invalid Simli WS URL"
-        }
-        this.session_token = session_token
-        this.transport = transport_mode
-        this.signaling = signaling
-        this.SimliWSURL = SimliWSURL
-        this.videoElement = videoElement
-        this.audioElement = audioElement
-        this.iceServers = iceServers
-        this.logger = new Logger(logLevel);
-
-        let resolveFn: () => void;
-        let rejectFn: () => void;
-
-        this.connectionPromise = new Promise<void>((resolve, reject) => {
-            resolveFn = resolve;
-            rejectFn = reject;
-        });
-
-        this.connectionResolve = resolveFn!;
-        this.connectionReject = rejectFn!;
-        this.persistent_events = new Map()
-        this.connectionTimeout = setTimeout(() => this.connectionReject("CONNECTION TIMED OUT"), this.CONNECTION_TIMEOUT_MS)
-
-        switch (this.transport) {
-            case "livekit":
-                this.connection = new LivekitTransport(this.SimliWSURL, this.session_token, videoElement, audioElement, this.logger, this.connectionReject)
-                break;
-            case "p2p":
-                if (!iceServers || iceServers.length == 0) {
-                    throw "Ice Servers Required for P2P Mode"
-                }
-                this.connection = new P2PTransport(this.SimliWSURL, this.session_token, true, iceServers, videoElement, audioElement, this.logger, this.connectionReject)
-                break
-            default:
-                throw new Error("Not Implemented Yet")
-
-        }
-        this.connection.on("start", () => {
-            this.connectionResolve()
-            clearTimeout(this.connectionTimeout)
-        })
-        this.connection.on("unknown", (message) => this.logger.debug("UNKOWN MESSAGE FROM SERVER: " + message))
-        this.connection.on("error", (message) => { this.failReason = message; this.connectionReject(message) })
-
-    }
-
-    private resetConnections(videoElement: HTMLVideoElement, audioElement: HTMLAudioElement, iceServers: RTCIceServer[] | null) {
-        this.failReason = null
-        let resolveFn: () => void;
-        let rejectFn: () => void;
-
-        this.connectionPromise = new Promise<void>((resolve, reject) => {
-            resolveFn = resolve;
-            rejectFn = reject;
-        });
-
-        this.connectionResolve = resolveFn!;
-        this.connectionReject = rejectFn!;
-        this.connectionTimeout = setTimeout(() => this.connectionReject("Connection Timed Out"), this.CONNECTION_TIMEOUT_MS)
-
-        switch (this.transport) {
-            case "livekit":
-                this.connection = new LivekitTransport(this.SimliWSURL, this.session_token, videoElement, audioElement, this.logger, this.connectionReject)
-                break;
-            case "p2p":
-                if (!iceServers || iceServers.length == 0) {
-                    throw "Ice Servers Required for P2P Mode"
-                }
-                this.connection = new P2PTransport(this.SimliWSURL, this.session_token, true, iceServers, videoElement, audioElement, this.logger, this.connectionReject)
-                break
-            default:
-                throw new Error("Not Implemented Yet")
-
-        }
-        this.connection.on("start", () => {
-            this.connectionResolve()
-            clearTimeout(this.connectionTimeout)
-        })
-        this.connection.on("error", (message) => { this.retryAttempt = this.MAX_RETRY_ATTEMPTS; this.connectionReject(message) })
-        this.connection.on("unknown", (message) => this.logger.debug("UNKOWN MESSAGE FROM SERVER: " + message))
-        // Re-register all user event handlers on the new connection
-        this.persistent_events.forEach((callbacks, event) => {
-            callbacks.forEach((callback) => {
-                this.connection.on(event as keyof SimliClientEvents, callback);
-            });
-        });
-    }
-    async start(): Promise<void> {
-        if (this.shouldStop) {
-            throw new Error("Disconnect Already Called, Can't reuse same SimliClient multiple times create a new SimliClient Object")
-        }
-        try {
-            await this.connection.connect()
-            await this.connectionPromise
-            this.retryAttempt = 0
-        } catch (error) {
-            if (this.failReason) {
-                throw error
-            }
-            if (this.retryAttempt >= this.MAX_RETRY_ATTEMPTS)
-                throw new Error("Too Many Retry Attempts Failed to connect")
-            if (this.shouldStop) {
-                this.shouldStop = false
-                throw new Error("Called Disconnect Before A Connecction succeeded")
-            }
-            this.logger.error("FAILED: " + error)
-            await this.connection.disconnect()
-            await new Promise(resolve => setTimeout(resolve, this.RETRY_DELAY));
-            this.retryAttempt += 1
-            if (this.retryAttempt > 2)
-                this.transport = "livekit"
-            this.resetConnections(this.videoElement, this.audioElement, this.iceServers)
-            await this.start()
-        }
-    }
-
-    async stop() {
-        this.shouldStop = true
-        await this.connection.disconnect()
-    }
-
-    listenToMediastreamTrack(stream: MediaStreamTrack) {
-        this.initializeAudioWorklet(this.audioContext, stream);
-    }
-
-    private initializeAudioWorklet(
-        audioContext: AudioContext,
-        stream: MediaStreamTrack,
-    ) {
-        audioContext.audioWorklet
-            .addModule(
-                URL.createObjectURL(
-                    new Blob([AudioProcessor(this.audioBufferSize)], {
-                        type: "application/javascript",
-                    }),
-                ),
-            )
-            .then(() => {
-                this.audioWorklet = new AudioWorkletNode(
-                    audioContext,
-                    "audio-processor",
-                );
-                this.sourceNode = audioContext.createMediaStreamSource(
-                    new MediaStream([stream]),
-                );
-                if (this.audioWorklet === null) {
-                    throw new Error("SIMLI: AudioWorklet not initialized");
-                }
-                this.sourceNode.connect(this.audioWorklet);
-                this.audioWorklet.port.onmessage = (event) => {
-                    if (event.data.type === "audioData") {
-                        this.connection.signalingConnection.sendAudioData(new Uint8Array(event.data.data.buffer));
-                    }
-                };
-            })
-    }
-
-
-    public ClearBuffer = () => {
-        this.connection.signalingConnection.sendSignal("SKIP");
-    };
-    sendAudioData(audioData: Uint8Array) {
-        this.connection.signalingConnection.sendAudioData(audioData);
-    }
-
-    sendAudioDataImmediate(audioData: Uint8Array) {
-        this.connection.signalingConnection.sendAudioDataImmediate(audioData);
-    }
-
-}
-
-export { SimliClient, generateSimliSessionToken, generateIceServers, Logger, LogLevel }
-export type { SimliSessionRequest };

lib/Config.ts

-
-interface SimliClientConfig {
-    faceID: string;
-    handleSilence: boolean;
-    maxSessionLength: number;
-    maxIdleTime: number;
-    enableSFU: boolean;
-    model: "fasttalk" | "artalk";
-}
-export type { SimliClientConfig }

lib/Events.ts

-interface SimliClientEvents {
-    start: () => void;
-    stop: () => void;
-    error: (detail: string) => void;
-    ack: () => void;
-    connection_info: (serialized_info: string) => void;
-    video_info: (serialized_info: string) => void;
-    destination: (serialized_info: string) => void;
-    speaking: () => void;
-    silent: () => void;
-    unknown: (message: string) => void;
-    startup_error: (message: string) => void
-}
-export type { SimliClientEvents }

lib/Signaling/BaseSignaling.ts

-import { Logger } from "../Logger";
-
-type ClientSignals = "DONE" | "SKIP"
-interface BaseSignaling {
-    logger: Logger
-    connect(connected: () => void): Promise<void>;
-    disconnect(): void;
-    sendSignal(data: ClientSignals): void;
-    sendAudioData(audioData: Uint8Array): void;
-    sendAudioDataImmediate(audioData: Uint8Array): void;
-}
-export type { BaseSignaling, ClientSignals }

lib/Signaling/WebSocketSignaling.ts

-import { Logger, LogLevel } from "../Logger"
-import { BaseSignaling, ClientSignals } from "./BaseSignaling"
-
-class WebSocketSignaling implements BaseSignaling {
-    wsURL: string | URL
-    wsConnection: WebSocket
-    logger: Logger
-    constructor(wsURL: string | URL, logger: Logger) {
-        this.wsURL = wsURL
-        this.wsConnection = new WebSocket(this.wsURL)
-        this.wsConnection.addEventListener("message", (message) => (this.logger.debug(message.data)))
-        this.logger = logger
-    }
-    async connect(connected: () => void): Promise<void> {
-        this.wsConnection.onopen = connected
-    }
-
-    disconnect(): void {
-        this.wsConnection.close()
-    }
-    private send(data: Uint8Array | string) {
-        if (this.wsConnection.readyState != WebSocket.OPEN) {
-            throw `Invalid State, WS Connection ${this.wsConnection.readyState.toString()}`
-        }
-        this.wsConnection.send(data)
-    }
-
-    sendOffer(offer: RTCSessionDescription): void {
-        this.send(JSON.stringify(offer))
-    }
-    sendSignal(data: ClientSignals): void {
-        this.send(data)
-    }
-
-    sendAudioData(audioData: Uint8Array) {
-        if (this.logger.getLevel() === LogLevel.DEBUG)
-            this.logger.debug("Sent Audio of length: " + (audioData.length / 32000).toString())
-        this.send(audioData);
-    }
-
-    sendAudioDataImmediate(audioData: Uint8Array) {
-        if (this.logger.getLevel() === LogLevel.DEBUG)
-            this.logger.debug("Sent Audio of length for immediate playback: " + (audioData.length / 32000).toString())
-        const asciiStr = "PLAY_IMMEDIATE";
-        const encoder = new TextEncoder(); // Default is utf-8
-        const strBytes = encoder.encode(asciiStr); // Uint8Array of " World!"
-        const buffer = new Uint8Array(strBytes.length + audioData.length);
-        buffer.set(strBytes, 0);
-        buffer.set(audioData, strBytes.length);
-        this.send(buffer);
-    }
-}
-export { WebSocketSignaling }

lib/Transports/BaseTransport.ts

-import { SimliClientEvents } from "../Events";
-import { Logger } from "../Logger";
-import { BaseSignaling } from "../Signaling/BaseSignaling";
-
-type EventCallback = (...args: any[]) => void;
-type EventMap = Map<string, Set<EventCallback>>;
-
-interface BaseTransport {
-    videoElementAnchor: HTMLVideoElement
-    audioElementAnchor: HTMLAudioElement
-    signalingConnection: BaseSignaling
-    session_token: string
-    events: EventMap;
-    logger: Logger;
-    connect(): Promise<void>;
-    disconnect(): void;
-    on<K extends keyof SimliClientEvents>(
-        event: K,
-        callback: SimliClientEvents[K]
-    ): void;
-
-    off<K extends keyof SimliClientEvents>(
-        event: K,
-        callback: SimliClientEvents[K],
-    ): void;
-
-    emit<K extends keyof SimliClientEvents>(
-        event: K,
-        ...args: Parameters<SimliClientEvents[K]>
-    ): void;
-
-}
-
-function register_destination(logger: Logger, serialized_info: string) {
-    const parsed = JSON.parse(serialized_info)
-    logger.destination = parsed.destination
-    logger.session_id = parsed.session_id
-}
-async function handleMessage(transport: BaseTransport, message: MessageEvent): Promise<void> {
-    const firstToken = (message.data as string).toUpperCase().split(" ")[0]
-    switch (firstToken) {
-        case "START": {
-            // SOFT IGNORE
-            break
-        }
-        case "ACK": {
-            transport.emit("ack")
-            break;
-        }
-        case "STOP": {
-            transport.disconnect();
-            transport.emit("stop")
-            break;
-        }
-        case "CLOSING":
-        case "RATE":
-        case "ERROR":
-        case "ERROR:": {
-            transport.disconnect()
-            transport.emit("error", message.data as string)
-        }
-        case "SPEAK": {
-            transport.emit("speaking");
-            break
-        }
-        case "SILENT": {
-            transport.emit("silent");
-            break
-        }
-        default: {
-            if (firstToken.includes("SDP") || firstToken.includes("LIVEKIT")) {
-                transport.emit("connection_info", message.data)
-            } else if (firstToken.includes("VIDEO_METADATA")) {
-                transport.emit("video_info", message.data)
-            } else if (firstToken.includes("ENDFRAME")) {
-                transport.emit("stop")
-                transport.disconnect()
-            } else if (firstToken.includes("DESTINATION")) {
-                transport.emit("destination", message.data)
-            } else {
-                transport.emit("unknown", message.data)
-            }
-        }
-
-    }
-}
-
-export { handleMessage, register_destination };
-export type {
-    BaseTransport, EventCallback, EventMap
-};

lib/Transports/LivekitTransport.ts

-import { RemoteParticipant, RemoteTrack, RemoteTrackPublication, Room, RoomEvent, RoomOptions, Track } from "livekit-client";
-import { WebSocketSignaling } from "../Signaling/WebSocketSignaling";
-import { SimliClientEvents } from "../Events";
-import { BaseTransport, EventCallback, EventMap, handleMessage, register_destination } from "./BaseTransport";
-import { Logger } from "../Logger";
-
-class LivekitTransport implements BaseTransport {
-    videoElementAnchor: HTMLVideoElement
-    audioElementAnchor: HTMLAudioElement
-    signalingConnection: WebSocketSignaling;
-    session_token: string;
-    pc: Room
-    logger: Logger
-    events: EventMap = new Map()
-    private websocketPromise: Promise<unknown>;
-    private websocketReject: ((reason: string) => void) | null = null;
-    constructor(
-        simliBaseWSURL: string,
-        session_token: string,
-        videoElementAnchor: HTMLVideoElement,
-        audioElementAnchor: HTMLAudioElement,
-        logger: Logger,
-        failSignal: (message: string) => void
-    ) {
-        this.logger = logger
-        this.on("startup_error", failSignal)
-
-        this.session_token = session_token
-        const wsURL = new URL(simliBaseWSURL + "/compose/webrtc/livekit")
-        wsURL.searchParams.set("session_token", session_token)
-        this.signalingConnection = new WebSocketSignaling(wsURL, this.logger)
-        this.on("destination", (serilized_info) => register_destination(this.logger, serilized_info))
-        this.websocketPromise = new Promise(
-            (resolve, reject) => {
-                this.websocketReject = reject;
-                this.signalingConnection.connect(() => {
-                    resolve("success")
-                    this.logger.debug("LK WebSocket Connected")
-                })
-            }
-        )
-        this.signalingConnection.wsConnection.onmessage = (message) => { handleMessage(this, message) }
-        this.signalingConnection.wsConnection.onerror = (evt) => {
-            this.emit("startup_error", "Websocket Failed");
-            if (this.websocketReject) {
-                this.websocketReject("Websocket Failed");
-                this.websocketReject = null; // Prevent multiple rejections
-            }
-        }
-        const options: RoomOptions = { adaptiveStream: true, dynacast: true }
-        this.pc = new Room(options);
-        this.on("connection_info", (serialized_info) => this.join_lk_room(serialized_info))
-        this.videoElementAnchor = videoElementAnchor
-        this.audioElementAnchor = audioElementAnchor
-
-    }
-    public on<K extends keyof SimliClientEvents>(
-        event: K,
-        callback: SimliClientEvents[K]
-    ): void {
-        if (!this.events.has(event)) {
-            this.events.set(event, new Set());
-        }
-        this.events.get(event)?.add(callback as EventCallback);
-        this.logger.debug("Registered Callback for Event: " + event)
-    }
-
-    public off<K extends keyof SimliClientEvents>(
-        event: K,
-        callback: SimliClientEvents[K],
-    ): void {
-        if (!this.events.has(event)) {
-            throw "Event Not Regsitered"
-        }
-        this.events.get(event)?.delete(callback as EventCallback);
-    }
-
-    emit<K extends keyof SimliClientEvents>(
-        event: K,
-        ...args: Parameters<SimliClientEvents[K]>
-    ): void {
-        this.logger.debug("Event: " + event)
-        this.events.get(event)?.forEach((callback) => {
-            callback(...args);
-        });
-    }
-
-    async connect() {
-        this.logger.info("Connecting")
-        this.setupConnectionStateHandler()
-        await this.websocketPromise
-    }
-
-    async disconnect() {
-        this.logger.info("Disconnecting")
-        try {
-            this.signalingConnection.sendSignal("DONE")
-        }
-        catch {
-            this.logger.error("FAILED TO SEND FINAL MESSAGE")
-        }
-        try {
-            this.signalingConnection.disconnect()
-        } catch {
-            this.logger.error("SIGNALING ALREADY DISCONNECTED")
-        }
-        try {
-            await this.pc.disconnect()
-        } catch {
-            this.logger.error("LOCAL PEER ALREADY CLOSED")
-        }
-
-    }
-
-    private async join_lk_room(serialized_info: string) {
-        const info = JSON.parse(serialized_info)
-        this.logger.debug(info)
-        if (info.livekit_url && info.livekit_token) {
-            await this.pc.connect(info.livekit_url, info.livekit_token);
-        } else {
-            this.disconnect()
-            this.emit("error", "Invalid Join Info, Contact Simli For Support")
-        }
-    }
-    private setupConnectionStateHandler() {
-        this.pc.on(RoomEvent.Disconnected, () => {
-            this.disconnect();
-        })
-
-        this.pc.on(RoomEvent.Connected, () => {
-        })
-        this.pc.on(RoomEvent.TrackSubscribed, (track: RemoteTrack,
-            publication: RemoteTrackPublication,
-            participant: RemoteParticipant,
-        ) => {
-            this.logger.debug("Track Received: " + track.kind)
-            if (track.kind === Track.Kind.Video) {
-                track.attach(this.videoElementAnchor)
-                this.videoElementAnchor.requestVideoFrameCallback(() => {
-                    this.emit("start");
-                });
-            } else if (track.kind === Track.Kind.Audio) {
-                track.attach(this.audioElementAnchor)
-            }
-        })
-    };
-
-
-}
-
-export { LivekitTransport }

lib/Transports/P2PTransport.ts

-import { WebSocketSignaling } from "../Signaling/WebSocketSignaling";
-import { SimliClientEvents } from "../Events";
-import { BaseTransport, EventCallback, EventMap, handleMessage, register_destination } from "./BaseTransport";
-import { Logger } from "../Logger";
-
-class P2PTransport implements BaseTransport {
-    videoElementAnchor: HTMLVideoElement
-    audioElementAnchor: HTMLAudioElement
-    signalingConnection: WebSocketSignaling;
-    session_token: string;
-    pc: RTCPeerConnection;
-    events: EventMap = new Map()
-    logger: Logger;
-    private iceCandidateCount: number;
-    private previousIceCandidateCount: number;
-    private iceTimeout: NodeJS.Timeout | null = null;
-    private websocketPromise: Promise<unknown>;
-    private websocketReject: ((reason: string) => void) | null = null;
-    constructor(
-        simliBaseWSURL: string,
-        session_token: string,
-        enableSFU: boolean,
-        iceServers: RTCIceServer[],
-        videoElementAnchor: HTMLVideoElement,
-        audioElementAnchor: HTMLAudioElement,
-        logger: Logger,
-        failSignal: (message: string) => void,
-    ) {
-        this.logger = logger
-        this.on("startup_error", failSignal)
-
-        this.session_token = session_token
-        const wsURL = new URL(simliBaseWSURL + "/compose/webrtc/p2p")
-        wsURL.searchParams.set("session_token", session_token)
-        wsURL.searchParams.set("enableSFU", String(enableSFU))
-        this.on("destination", (serilized_info) => register_destination(this.logger, serilized_info))
-        this.signalingConnection = new WebSocketSignaling(wsURL, this.logger)
-        this.websocketPromise = new Promise(
-            (resolve, reject) => {
-                this.websocketReject = reject;
-                this.signalingConnection.connect(() => {
-                    resolve("success")
-                    this.logger.debug("P2P WebSocket Connected")
-                })
-            }
-        )
-        this.signalingConnection.wsConnection.onmessage = (message) => { handleMessage(this, message) }
-        this.signalingConnection.wsConnection.onerror = (evt) => {
-            this.emit("startup_error", "Websocket Failed");
-            if (this.websocketReject) {
-                this.websocketReject("Websocket Failed");
-                this.websocketReject = null; // Prevent multiple rejections
-            }
-        }
-        this.on("connection_info", (serialized_info) => this.registerPeerInfo(serialized_info))
-        this.videoElementAnchor = videoElementAnchor
-        this.audioElementAnchor = audioElementAnchor
-        this.iceCandidateCount = 0
-        this.previousIceCandidateCount = 0
-        const config = {
-            sdpSemantics: "unified-plan",
-            iceServers: iceServers,
-        };
-        this.pc = new window.RTCPeerConnection(config);
-        this.pc.addTransceiver("audio", {
-            direction: "recvonly",
-        });
-        this.pc.addTransceiver("video", {
-            direction: "recvonly",
-        })
-    }
-    public on<K extends keyof SimliClientEvents>(
-        event: K,
-        callback: SimliClientEvents[K]
-    ): void {
-        if (!this.events.has(event)) {
-            this.events.set(event, new Set());
-        }
-        this.events.get(event)?.add(callback as EventCallback);
-    }
-
-    public off<K extends keyof SimliClientEvents>(
-        event: K,
-        callback: SimliClientEvents[K],
-    ): void {
-        this.events.get(event)?.delete(callback as EventCallback);
-    }
-
-    emit<K extends keyof SimliClientEvents>(
-        event: K,
-        ...args: Parameters<SimliClientEvents[K]>
-    ): void {
-        this.events.get(event)?.forEach((callback) => {
-            try {
-                callback(...args);
-            }
-            catch {
-                this.logger.error("CALLBACK FAILED: " + callback.name)
-            }
-        });
-
-    }
-    async connect() {
-
-        const offer = await this.pc.createOffer();
-        await this.pc.setLocalDescription(offer);
-        await this.waitForIceGathering();
-        this.setupPeerConnectionListeners()
-        await this.websocketPromise
-        if (this.pc.localDescription) {
-            this.signalingConnection.sendOffer(this.pc.localDescription)
-        }
-
-    }
-
-    async disconnect() {
-        this.logger.info("Disconnecting")
-        try {
-            this.signalingConnection.sendSignal("DONE")
-        }
-        catch {
-            this.logger.error("FAILED TO SEND FINAL MESSAGE")
-        }
-        try {
-            this.signalingConnection.disconnect()
-        } catch {
-            this.logger.error("SIGNALING ALREADY DISCONNECTED")
-        }
-        try {
-            this.pc.close()
-        } catch {
-            this.logger.error("LOCAL PEER ALREADY CLOSED")
-        }
-
-    }
-    private async registerPeerInfo(serialized_info: string) {
-        const info = JSON.parse(serialized_info)
-        if (info.sdp && info.type == "answer") {
-            await this.pc.setRemoteDescription(new RTCSessionDescription(info));
-        } else {
-            this.disconnect()
-            this.emit("error", "Invalid Join Info, Contact Simli For Support")
-        }
-    }
-
-    private async waitForIceGathering(): Promise<void> {
-        this.iceCandidateCount = 0;
-        this.previousIceCandidateCount = 0;
-        if (this.pc.iceGatheringState === "complete") {
-            return;
-        }
-
-        return new Promise<void>((resolve, reject) => {
-            if (!this.iceTimeout) {
-                this.iceTimeout = setTimeout(() => {
-                    reject(new Error("ICE gathering timeout"));
-                }, 10000);
-            }
-
-            const checkIceCandidates = () => {
-                if (
-                    this.pc.iceGatheringState === "complete" ||
-                    this.iceCandidateCount === this.previousIceCandidateCount
-                ) {
-                    if (this.iceTimeout) {
-                        clearTimeout(this.iceTimeout);
-                    }
-                    resolve();
-                } else {
-                    this.previousIceCandidateCount = this.iceCandidateCount;
-                    setTimeout(checkIceCandidates, 150);
-                }
-            };
-
-            checkIceCandidates();
-        });
-    }
-
-    private setupPeerConnectionListeners() {
-        this.pc.addEventListener("track", (evt) => {
-            if (evt.track.kind === "video") {
-                this.videoElementAnchor.srcObject = evt.streams[0];
-                this.videoElementAnchor.requestVideoFrameCallback(() => {
-                    this.emit("start")
-                });
-            } else if (evt.track.kind === "audio" && this.audioElementAnchor) {
-                this.audioElementAnchor.srcObject = evt.streams[0];
-            }
-        });
-
-        this.pc.onicecandidate = (event) => {
-            if (event.candidate !== null) {
-                this.iceCandidateCount += 1;
-            }
-        };
-    }
-}
-
-export { P2PTransport }

New alerts

Major refactor

Supply chain risk

Package has recently undergone a major refactor. It may be unstable or indicate significant internal changes. Use caution when updating to versions that include significant changes.

Improved metrics

Lines of code

1848

1.04%

Worsened metrics

Total package byte prevSize

74886

-0.03%

Number of medium supply chain risk alerts

5

25%

No dependency changes