@unith-ai/core-client - npm Package Compare versions

Comparing version 2.0.61 to 2.0.62

dist/index.d.ts

+// Generated by dts-bundle-generator v9.5.1
+
+export type AudioFormatConfig = {
+	sampleRate: number;
+	format: "pcm" | "wav" | "ulaw";
+};
+declare class AudioOutput {
+	readonly context: AudioContext;
+	readonly analyser: AnalyserNode;
+	readonly gain: GainNode;
+	readonly worklet: AudioWorkletNode;
+	private _isMuted;
+	static createAudioOutput({ sampleRate, format, }: AudioFormatConfig): Promise<AudioOutput>;
+	private ensureIOSCompatibility;
+	private constructor();
+	getOutputDevice(): Promise<MediaDeviceInfo | null>;
+	getAvailableOutputDevices(): Promise<MediaDeviceInfo[]>;
+	mute(): void;
+	unmute(): void;
+	toggleMute(): boolean;
+	close(): Promise<void>;
+}
+export type Environment = "production" | "staging" | "development";
+export declare enum EventType {
+	TEXT = "text",
+	CONVERSATION_END = "conversation_end",
+	JOIN = "join",
+	ERROR = "error",
+	TIME_OUT = "timeout",
+	UNITH_NLP_EXCEPTION = "unith_nlp_exception",
+	ANALYTICS = "analytics-userFeedback",
+	CHOICE = "choice",
+	TIMEOUT_WARNING = "timeout_warning",
+	KEEP_SESSION = "keep_session",
+	RESPONSE = "response",
+	STREAMING = "streaming",
+	PING = "ping",
+	PONG = "pong",
+	BINARY = "binary",
+	STOP = "stop"
+}
+export declare enum StreamingEventType {
+	VIDEO_FRAME = "video_frame",
+	AUDIO_FRAME = "audio_frame",
+	METADATA = "metadata",
+	ERROR = "error",
+	CACHE = "cache",
+	STOP = "stop"
+}
+export interface Message {
+	id: number;
+	timestamp: string;
+	speaker: string;
+	text: string;
+	isSent: boolean;
+	user_id: string;
+	username?: string;
+	event: EventType.TEXT | EventType.KEEP_SESSION | EventType.STOP;
+	session_id?: string;
+	visible: boolean;
+}
+export type JoinEventData = {
+	granted: boolean;
+};
+export type GlobalEventData = {
+	event: EventType;
+	user_id: string;
+	username: string;
+	type?: string;
+};
+export type BinaryEventData = {
+	data: ArrayBuffer;
+};
+export type SpeakerType = "user" | "ai";
+export type TextEventData = {
+	id: string;
+	timestamp: Date;
+	speaker: SpeakerType;
+	text: string;
+	isSent: boolean;
+	session_id: string;
+	visible: boolean;
+	is_last?: boolean;
+	part_order?: number;
+	input_message_id?: string;
+	suggestions: string[];
+	has_error?: boolean;
+	error_code?: number;
+	error_message?: string;
+	media?: string[];
+	video?: string;
+	stream_id?: string;
+};
+export type ConversationEndEventData = {
+	reason: string;
+	duration: number;
+};
+export type TimeoutEventData = {
+	remaining_time: number;
+};
+export type AnalyticsEventData = {
+	feedback_type: string;
+	rating: number;
+	comment?: string;
+};
+export type ChoiceEventData = {
+	choices: string[];
+	selected_choice?: string;
+};
+export type StreamingEventData = {
+	message?: string;
+	metadata_type?: string;
+	session_id?: string;
+	suggestions: string[];
+	event_timestamp_ms: number;
+	type: StreamingEventType;
+	video_url?: string;
+	text?: string;
+	error_type?: "resource_exhausted" | "deadline_exceeded" | "inactivity_timeout" | "canceled";
+	format?: "jpeg" | "wav";
+	frame_data?: string;
+	height?: number;
+	width?: string;
+	sample_rate?: string;
+	duration_ms?: number;
+	is_keyframe?: boolean;
+};
+export type PingEventData = {
+	event: EventType.PING;
+	timestamp: string;
+	id: string;
+};
+export type PongEventData = {
+	type: EventType.PONG;
+	timestamp: string;
+	id: string;
+};
+export type MessageEventData = {
+	timestamp: Date;
+	sender: SpeakerType;
+	text: string;
+	visible: boolean;
+};
+export type IncomingSocketEvent<T extends EventType = EventType> = T extends EventType.BINARY ? GlobalEventData & BinaryEventData : T extends EventType.JOIN ? GlobalEventData & JoinEventData : T extends EventType.TEXT ? GlobalEventData & TextEventData : T extends EventType.RESPONSE ? GlobalEventData & TextEventData : T extends EventType.CONVERSATION_END ? GlobalEventData & ConversationEndEventData : T extends EventType.STREAMING ? GlobalEventData & StreamingEventData : T extends EventType.TIME_OUT ? GlobalEventData & TimeoutEventData : T extends EventType.TIMEOUT_WARNING ? GlobalEventData & TimeoutEventData : T extends EventType.KEEP_SESSION ? GlobalEventData & JoinEventData : T extends EventType.ANALYTICS ? GlobalEventData & AnalyticsEventData : T extends EventType.PING ? GlobalEventData & PingEventData : T extends EventType.PONG ? GlobalEventData & PongEventData : T extends EventType.CHOICE ? GlobalEventData & ChoiceEventData : GlobalEventData;
+export type AnyIncomingSocketEvent = IncomingSocketEvent<EventType.JOIN> | IncomingSocketEvent<EventType.TEXT> | IncomingSocketEvent<EventType.RESPONSE> | IncomingSocketEvent<EventType.STREAMING> | IncomingSocketEvent<EventType.BINARY> | IncomingSocketEvent<EventType.CONVERSATION_END> | IncomingSocketEvent<EventType.TIME_OUT> | IncomingSocketEvent<EventType.TIMEOUT_WARNING> | IncomingSocketEvent<EventType.KEEP_SESSION> | IncomingSocketEvent<EventType.ANALYTICS> | IncomingSocketEvent<EventType.CHOICE> | IncomingSocketEvent<EventType.PING> | IncomingSocketEvent<EventType.PONG> | (GlobalEventData & {
+	event: Exclude<EventType, EventType.JOIN | EventType.TEXT | EventType.CONVERSATION_END | EventType.TIME_OUT | EventType.TIMEOUT_WARNING | EventType.ANALYTICS | EventType.CHOICE | EventType.RESPONSE | EventType.STREAMING | EventType.PING | EventType.KEEP_SESSION | EventType.BINARY>;
+});
+export type DisconnectionDetails = {
+	reason: "error" | "user";
+	message?: string;
+	context?: Event;
+};
+export type OnDisconnectCallback = (details: DisconnectionDetails) => void;
+export type OnMessageCallback = (event: AnyIncomingSocketEvent) => void;
+export type OnPingPongCallback = (event: PongEventData) => void;
+export declare function isJoinEvent(event: AnyIncomingSocketEvent): event is IncomingSocketEvent<EventType.JOIN>;
+export declare function isTextEvent(event: AnyIncomingSocketEvent): event is IncomingSocketEvent<EventType.TEXT>;
+export declare function isResponseEvent(event: AnyIncomingSocketEvent): event is IncomingSocketEvent<EventType.RESPONSE>;
+export declare function isStreamingEvent(event: AnyIncomingSocketEvent): event is IncomingSocketEvent<EventType.STREAMING>;
+export declare function isStreamingErrorEvent(event: AnyIncomingSocketEvent): event is IncomingSocketEvent<EventType.STREAMING>;
+export declare function isBinaryEvent(event: AnyIncomingSocketEvent): event is IncomingSocketEvent<EventType.BINARY>;
+export declare function isConversationEndEvent(event: AnyIncomingSocketEvent): event is IncomingSocketEvent<EventType.CONVERSATION_END>;
+export declare function isPongEvent(event: AnyIncomingSocketEvent): event is IncomingSocketEvent<EventType.PONG>;
+export declare function isTimeoutWarningEvent(event: AnyIncomingSocketEvent): event is IncomingSocketEvent<EventType.TIMEOUT_WARNING>;
+export declare function isTimeoutEvent(event: AnyIncomingSocketEvent): event is IncomingSocketEvent<EventType.TIME_OUT>;
+export declare function isKeepSessionEvent(event: AnyIncomingSocketEvent): event is IncomingSocketEvent<EventType.KEEP_SESSION>;
+export type MicrophoneEvents = {
+	onMicrophoneError: (prop: {
+		message: string;
+	}) => void;
+	onMicrophoneStatusChange: (prop: {
+		status: "ON" | "OFF" | "PROCESSING";
+	}) => void;
+	onMicrophoneSpeechRecognitionResult: (prop: {
+		transcript: string;
+	}) => void;
+	onMicrophonePartialSpeechRecognitionResult: (prop: {
+		transcript: string;
+	}) => void;
+};
+export type MicrophoneProvider = "azure" | "custom" | "eleven_labs";
+export type MicrophoneStatus = "ON" | "OFF" | "PROCESSING";
+export type ElevenLabsOptions = {
+	noiseSuppression: boolean;
+	vadSilenceThresholdSecs: number;
+	vadThreshold: number;
+	minSpeechDurationMs: number;
+	minSilenceDurationMs: number;
+	disableDynamicSpeechRecognition?: boolean;
+};
+export type MicTokenResponseType = {
+	token: string;
+	region: string;
+	generatedAt: number | null;
+};
+export type ApiErrorType = {
+	response?: {
+		data?: {
+			detail?: string;
+		};
+		status_code?: number;
+		status?: number;
+	};
+	name?: string;
+};
+export type TokenResponseType = {
+	access_token: string;
+	user_id: string;
+	token_type: string;
+};
+export type HeadType = {
+	alias: string;
+	icon_config: undefined;
+	lang_speech_recognition: string;
+	language: Language;
+	name: string;
+	operation_mode: "ttt" | "oc" | "doc_qa";
+	phrases: string[];
+	streaming_enabled: boolean;
+	allowed_iframe_origins: string[];
+	avatarSrc?: string;
+};
+export type ConnectHeadType = {
+	name: string;
+	phrases: string[];
+	language: Language;
+	avatar?: string;
+};
+export type AuthTokenType = {
+	exp: number;
+	username: string;
+	admin: boolean;
+};
+export type ApiAsrTokenType = {
+	token: string;
+	region: string;
+};
+export declare enum VideoTransitionType {
+	NONE = "none",
+	CROSSFADE = "crossfade",
+	FADEIN = "fadein",
+	FADEOUT = "fadeout"
+}
+export interface VideoFormatConfig {
+	width: number;
+	height: number;
+	frameRate: number;
+	format: "jpeg" | "webp" | "png";
+	backgroundColor?: string;
+	antialias?: boolean;
+	resolution?: number;
+	maxBufferSize?: number;
+	enableAdaptiveQuality?: boolean;
+}
+export type Language = "ar-AE" | "bg-BG" | "bs-BA" | "cs-CZ" | "de-DE" | "en-US" | "es-ES" | "fr-FR" | "hu-HU" | "id-ID" | "it-IT" | "ka-GE" | "kk-KZ" | "lt-LT" | "lv-LV" | "nl-NL" | "pl-PL" | "pt-PT" | "ro-RO" | "ru-RU" | "sk-SK" | "sr-RS" | "th-TH" | "uk-UA";
+export type Status = "connecting" | "connected" | "disconnecting" | "disconnected";
+export type DigitalHumanOptions = {
+	username: string;
+	apiKey: string;
+	environment: Environment;
+	mode: ConversationMode;
+	language?: Language;
+	allowWakeLock?: boolean;
+	fadeTransitionsType?: VideoTransitionType;
+	voiceInterruptions?: boolean;
+};
+export type HeadOptions = {
+	orgId: string;
+	headId: string;
+};
+export type VideoHtmlElement = {
+	element: HTMLElement;
+};
+export type KeepSessionEventData = {
+	granted: boolean;
+};
+export type ConversationOptions = DigitalHumanOptions & HeadOptions;
+export type MicrophoneType = {
+	microphoneProvider: MicrophoneProvider;
+	microphoneOptions: MicrophoneEvents;
+	elevenLabsOptions?: ElevenLabsOptions;
+};
+export type ConversationEvents = {
+	onStatusChange: (prop: {
+		status: Status;
+	}) => void;
+	onConnect: (prop: {
+		userId: string;
+		headInfo: ConnectHeadType;
+		microphoneAccess: boolean;
+	}) => void;
+	onDisconnect: OnDisconnectCallback;
+	onMessage: (prop: MessageEventData) => void;
+	onSuggestions: (prop: {
+		suggestions: string[];
+	}) => void;
+	onTimeoutWarning: () => void;
+	onTimeout: () => void;
+	onMuteStatusChange: (prop: {
+		isMuted: boolean;
+	}) => void;
+	onSpeakingStart: () => void;
+	onSpeakingEnd: () => void;
+	onStoppingEnd: () => void;
+	onStoppingStart: () => void;
+	onKeepSession: (prop: KeepSessionEventData) => void;
+	onError: (prop: {
+		message: string;
+		endConversation: boolean;
+		type: "toast" | "modal" | "silent";
+	}) => void;
+	onHighDemand: () => void;
+};
+export type ConversationMode = "chat" | "audio" | "video" | "default";
+export type Role = "user" | "ai";
+export type Mode = "speaking" | "listening" | "thinking" | "stopping";
+export type StreamingType = "jpg" | "vp8";
+export type QualityType = "low" | "standard" | "high" | "premium";
+export type WebsocketConnectionConfig = {
+	environment: Environment;
+	orgId: string;
+	headId: string;
+	token: string;
+	apiKey: string;
+	mode: ConversationMode;
+	language: Language;
+	onJoin?: () => void;
+	onError?: (error: string) => void;
+};
+declare class Connection {
+	readonly socket: WebSocket;
+	readonly userId: string;
+	static create(config: WebsocketConnectionConfig): Promise<Connection>;
+	private queue;
+	private disconnectionDetails;
+	private onDisconnectCallback;
+	private onMessageCallback;
+	private onPingPongCallback;
+	private constructor();
+	private disconnect;
+	close(): void;
+	sendMessage(message: Message): void;
+	sendPingEvent(message: PingEventData): void;
+	onPingPong(callback: OnPingPongCallback): void;
+	onMessage(callback: OnMessageCallback): void;
+	onDisconnect(callback: OnDisconnectCallback): void;
+}
+declare class IdleVideo {
+	readonly idleVideoSource: string;
+	readonly videoId: string;
+	constructor(idleVideoSource: string, videoId: string);
+	static getIdleVideo(apiUrl: string, orgId: string, headId: string): Promise<IdleVideo>;
+	getAvatarSrc(apiUrl: string, orgId: string, headId: string): Promise<string>;
+	private static getIdleVideoId;
+}
+declare class User {
+	readonly id: string;
+	username: string;
+	readonly password: string;
+	readonly orgId: string;
+	readonly headId: string;
+	readonly apiBase: string;
+	static loginUser(username: string, password: string, apiUrl: string, orgId: string, headId: string): Promise<User>;
+	private EXPIRATION_OFFSET;
+	accessToken: string;
+	tokenType: string;
+	sessionId: number;
+	private constructor();
+	getHeadDetails(apiKey: string): Promise<HeadType>;
+	getAuthToken(username: string, password: string): Promise<TokenResponseType>;
+	getProviderToken(auth_jwt: string, provider: "elevenlabs" | "azure"): Promise<ApiAsrTokenType>;
+	getAccessToken(): Promise<{
+		access_token: string;
+		user_id: string | null | undefined;
+		session_id: number;
+	}>;
+	getAsrToken(provider: "elevenlabs" | "azure"): Promise<{
+		token: string;
+		region: string;
+	}>;
+}
+export type IdleVideoConfig = {
+	src: string;
+	enabled: boolean;
+};
+export interface VideoEffectConfig {
+	background?: BackgroundConfig;
+}
+export interface BackgroundConfig {
+	type: "color" | "image";
+	value: number | string;
+}
+declare class VideoOutput {
+	private videoOutput;
+	private container;
+	private idleVideo;
+	private cachedVideo;
+	private idleVideoConfig;
+	private videoTransitionConfig;
+	private readonly CROSSFADE_DURATION;
+	private isTransitioning;
+	private isShowingIdleVideo;
+	private bufferCheckAnimationId;
+	private lastBufferCheckTime;
+	private sessionStarted;
+	private isRestoringIdleVideo;
+	private isShowingCachedVideo;
+	private onIdleVideoShown?;
+	private onIdleVideoHidden?;
+	private onSpeakingStartCallback;
+	private onSpeakingEndCallback;
+	static createVideoOutput(container: HTMLElement, config: VideoFormatConfig & {
+		effects?: VideoEffectConfig;
+		idleVideo: IdleVideoConfig;
+		transition: VideoTransitionType;
+	}): Promise<VideoOutput>;
+	private constructor();
+	private setupIdleVideo;
+	private setupCachedVideo;
+	onSpeakingStart(speakingStartCallback: () => void): void;
+	onSpeakingEnd(speakingEndCallback: () => void): void;
+	startStreaming(init?: boolean): void;
+	stopBufferMonitoring(): void;
+	getBufferLength(): number;
+	startBufferMonitoring(): void;
+	toggleCacheVideoMute(): void;
+	playCachedVideo(cachedVideoUrl: string): Promise<void>;
+	stopCachedVideo(): Promise<void>;
+	private crossfadeFromIdleToCached;
+	private crossfadeFromCachedToIdle;
+	private cleanupCachedVideo;
+	private hideIdleVideo;
+	private hideIdleVideoBeforeStream;
+	setEventCallbacks(callbacks: {
+		onIdleVideoShown?: () => void;
+		onIdleVideoHidden?: () => void;
+	}): void;
+	getStreamingStatus(): boolean;
+	isPlayingCachedVideo(): boolean;
+	showIdleVideo(): Promise<void>;
+	showIdleVideoAfterStream(): Promise<void>;
+	addFrame(frameData: Blob | ArrayBuffer, timeStamp: number, isKeyframe?: boolean): Promise<void>;
+	clearFrame(): void;
+	toggleStream(status: boolean): Promise<void>;
+	destroy(): void;
+}
+export declare const defaultElevenLabsOptions: ElevenLabsOptions;
+export type Options = ConversationOptions & ConversationEvents & MicrophoneType;
+export type HeadConfigOptions = HeadOptions & Partial<DigitalHumanOptions>;
+export type PartialOptions = HeadConfigOptions & Partial<MicrophoneType> & VideoHtmlElement & Partial<ConversationEvents>;
+export declare class Conversation {
+	private options;
+	private microphoneAccess;
+	private connection;
+	private idleVideo;
+	private wakeLock;
+	private user;
+	private audioOutput;
+	private videoOutput;
+	private headInfo;
+	private status;
+	protected volume: number;
+	private sessionStarted;
+	private isStoppingLastResponse;
+	private messageCounter;
+	private syncController;
+	private avController;
+	private monitor;
+	private microphone;
+	private videoFrameQueue;
+	private cachedResponseQueue;
+	private suggestionsQueue;
+	private static getFullOptions;
+	/**
+	 * Starts a digital human conversation.
+	 * @param options - The options for the conversation.
+	 * @returns A promise that resolves to a Conversation instance.
+	 */
+	static startDigitalHuman(options: PartialOptions): Promise<Conversation>;
+	/**
+	 * This retrieves the background video to use for widget mode & welcome screen
+	 * @param options PartialOptions
+	 * @returns Video link to use as widget background
+	 */
+	static getBackgroundVideo(options: {
+		orgId: string;
+		headId: string;
+		environment?: Environment;
+	}): Promise<string>;
+	constructor(options: Options, microphoneAccess: boolean, connection: Connection, idleVideo: IdleVideo, wakeLock: WakeLockSentinel | null, user: User, audioOutput: AudioOutput, videoOutput: VideoOutput, headInfo: HeadType);
+	private startLatencyMonitoring;
+	private handleIOSSilentMode;
+	private onOutputWorkletMessage;
+	private handleMessage;
+	private handleStreamError;
+	private handleStreamingEvent;
+	private handleVideoFrame;
+	private handleBinaryData;
+	private handleAudioFrame;
+	private onMessage;
+	private handleStopVideoEvents;
+	private endSessionWithDetails;
+	getUserId(): string;
+	private handleEndSession;
+	private updateStatus;
+	/**
+	 * To stop streaming response, we'll send a stop message to the BE. The BE will then stop sending audio and video frames, which will naturally end the response. This is more efficient and leads to a better user experience as it allows for a smoother transition when stopping the response.
+	 *
+	 * To stop cached response, we just stop the video and fade to idle.
+	 */
+	stopResponse(): Promise<void>;
+	toggleMute(): Promise<number>;
+	startSession(): Promise<Connection>;
+	toggleMicrophone(): Promise<void>;
+	getMicrophoneStatus(): MicrophoneStatus;
+	endSession(): Promise<void>;
+	sendMessage(text: string): void;
+	keepSession(): void;
+}
+
+export {};

package.json

 {
   "name": "@unith-ai/core-client",
-  "version": "2.0.61",
+  "version": "2.0.62",
   "description": "Core TypeScript SDK for building digital human experiences with Unith AI",
@@ -31,5 +31,6 @@ "main": "./dist/lib.js",
   "devDependencies": {
+    "dts-bundle-generator": "^9.5.1",
     "eslint": "^9.8.0",
     "microsoft-cognitiveservices-speech-sdk": "^1.45.0",
-    "rolldown": "1.0.0-rc.9",
+    "rolldown": "^1.0.0-rc.9",
     "typescript": "^5.5.4"
@@ -54,7 +55,7 @@ },
   "scripts": {
-    "build": "npm run build:esm && npm run build:cjs && npm run build:web",
+    "build": "npm run build:esm && npm run build:cjs && npm run build:web && npm run build:types",
     "build:esm": "rolldown src/index.ts -f es --file dist/lib.module.js --sourcemap --external microsoft-cognitiveservices-speech-sdk",
     "build:cjs": "rolldown src/index.ts -f cjs --file dist/lib.js --sourcemap --external microsoft-cognitiveservices-speech-sdk",
     "build:web": "rolldown src/index.ts -f iife --name UnithCoreClient --file dist/lib.web.js --sourcemap --external microsoft-cognitiveservices-speech-sdk",
-    "build:types": "tsc --project tsconfig.build.json --emitDeclarationOnly --declarationDir dist",
+    "build:types": "dts-bundle-generator --out-file dist/index.d.ts src/index.ts",
     "clean": "rm -rf ./dist",
@@ -61,0 +62,0 @@ "dev": "npm run clean && rolldown src/index.ts -f es --file dist/lib.module.js --sourcemap --external microsoft-cognitiveservices-speech-sdk --watch",

No alert changes

Improved metrics

Total package byte prevSize

7841790

0.23%

Number of package files

9

12.5%

Lines of code

73272

0.72%

Worsened metrics

Dev dependency count

5

25%

No dependency changes