@livekit/agents - npm Package Compare versions

Comparing version 0.3.5 to 0.4.0

CHANGELOG.md

 # @livekit/agents
 
+## 0.4.0
+
+### Minor Changes
+
+- OpenAI function calling: support arrays and optional fields in function call schema - [#140](https://github.com/livekit/agents-js/pull/140) ([@nbsp](https://github.com/nbsp))
+
+- add basic tokenizer implementations - [#140](https://github.com/livekit/agents-js/pull/140) ([@nbsp](https://github.com/nbsp))
+
+- add VoicePipelineAgent - [#138](https://github.com/livekit/agents-js/pull/138) ([@nbsp](https://github.com/nbsp))
+
+- add LLM and LLMStream baseclasses - [#140](https://github.com/livekit/agents-js/pull/140) ([@nbsp](https://github.com/nbsp))
+
+- add ChatContext - [#140](https://github.com/livekit/agents-js/pull/140) ([@nbsp](https://github.com/nbsp))
+
+- update TTS and STT baseclasses to match python - [#140](https://github.com/livekit/agents-js/pull/140) ([@nbsp](https://github.com/nbsp))
+
+### Patch Changes
+
+- make numIdleProcesses work - [#135](https://github.com/livekit/agents-js/pull/135) ([@nbsp](https://github.com/nbsp))
+
+- re-add ElevenLabs TTS plugin - [#140](https://github.com/livekit/agents-js/pull/140) ([@nbsp](https://github.com/nbsp))
+
+- add Deepgram text-to-speech plugin - [#140](https://github.com/livekit/agents-js/pull/140) ([@nbsp](https://github.com/nbsp))
+
+- throw an error when using CommonJS with tsx - [#139](https://github.com/livekit/agents-js/pull/139) ([@nbsp](https://github.com/nbsp))
+
+- add OpenAI LLM - [#140](https://github.com/livekit/agents-js/pull/140) ([@nbsp](https://github.com/nbsp))
+
+- add Silero VAD, overhaul VAD class - [#140](https://github.com/livekit/agents-js/pull/140) ([@nbsp](https://github.com/nbsp))
+
 ## 0.3.5
@@ -4,0 +34,0 @@

dist/audio.js

@@ -1,13 +0,1 @@
-var __classPrivateFieldSet = (this && this.__classPrivateFieldSet) || function (receiver, state, value, kind, f) {
-    if (kind === "m") throw new TypeError("Private method is not writable");
-    if (kind === "a" && !f) throw new TypeError("Private accessor was defined without a setter");
-    if (typeof state === "function" ? receiver !== state || !f : !state.has(receiver)) throw new TypeError("Cannot write private member to an object whose class did not declare it");
-    return (kind === "a" ? f.call(receiver, value) : f ? f.value = value : state.set(receiver, value)), value;
-};
-var __classPrivateFieldGet = (this && this.__classPrivateFieldGet) || function (receiver, state, kind, f) {
-    if (kind === "a" && !f) throw new TypeError("Private accessor was defined without a getter");
-    if (typeof state === "function" ? receiver !== state || !f : !state.has(receiver)) throw new TypeError("Cannot read private member from an object whose class did not declare it");
-    return kind === "m" ? f : kind === "a" ? f.call(receiver) : f ? f.value : state.get(receiver);
-};
-var _AudioByteStream_sampleRate, _AudioByteStream_numChannels, _AudioByteStream_bytesPerFrame, _AudioByteStream_buf, _AudioByteStream_logger;
 // SPDX-FileCopyrightText: 2024 LiveKit, Inc.
@@ -20,23 +8,23 @@ //
 export class AudioByteStream {
+    #sampleRate;
+    #numChannels;
+    #bytesPerFrame;
+    #buf;
+    #logger = log();
     constructor(sampleRate, numChannels, samplesPerChannel = null) {
-        _AudioByteStream_sampleRate.set(this, void 0);
-        _AudioByteStream_numChannels.set(this, void 0);
-        _AudioByteStream_bytesPerFrame.set(this, void 0);
-        _AudioByteStream_buf.set(this, void 0);
-        _AudioByteStream_logger.set(this, log());
-        __classPrivateFieldSet(this, _AudioByteStream_sampleRate, sampleRate, "f");
-        __classPrivateFieldSet(this, _AudioByteStream_numChannels, numChannels, "f");
+        this.#sampleRate = sampleRate;
+        this.#numChannels = numChannels;
         if (samplesPerChannel === null) {
             samplesPerChannel = Math.floor(sampleRate / 50); // 20ms by default
         }
-        __classPrivateFieldSet(this, _AudioByteStream_bytesPerFrame, numChannels * samplesPerChannel * 2, "f"); // 2 bytes per sample (Int16)
-        __classPrivateFieldSet(this, _AudioByteStream_buf, new Int8Array(), "f");
+        this.#bytesPerFrame = numChannels * samplesPerChannel * 2; // 2 bytes per sample (Int16)
+        this.#buf = new Int8Array();
     }
     write(data) {
-        __classPrivateFieldSet(this, _AudioByteStream_buf, new Int8Array([...__classPrivateFieldGet(this, _AudioByteStream_buf, "f"), ...new Int8Array(data)]), "f");
+        this.#buf = new Int8Array([...this.#buf, ...new Int8Array(data)]);
         const frames = [];
-        while (__classPrivateFieldGet(this, _AudioByteStream_buf, "f").length >= __classPrivateFieldGet(this, _AudioByteStream_bytesPerFrame, "f")) {
-            const frameData = __classPrivateFieldGet(this, _AudioByteStream_buf, "f").slice(0, __classPrivateFieldGet(this, _AudioByteStream_bytesPerFrame, "f"));
-            __classPrivateFieldSet(this, _AudioByteStream_buf, __classPrivateFieldGet(this, _AudioByteStream_buf, "f").slice(__classPrivateFieldGet(this, _AudioByteStream_bytesPerFrame, "f")), "f");
-            frames.push(new AudioFrame(new Int16Array(frameData.buffer), __classPrivateFieldGet(this, _AudioByteStream_sampleRate, "f"), __classPrivateFieldGet(this, _AudioByteStream_numChannels, "f"), frameData.length / 2));
+        while (this.#buf.length >= this.#bytesPerFrame) {
+            const frameData = this.#buf.slice(0, this.#bytesPerFrame);
+            this.#buf = this.#buf.slice(this.#bytesPerFrame);
+            frames.push(new AudioFrame(new Int16Array(frameData.buffer), this.#sampleRate, this.#numChannels, frameData.length / 2));
         }
@@ -46,12 +34,11 @@ return frames;
     flush() {
-        if (__classPrivateFieldGet(this, _AudioByteStream_buf, "f").length % (2 * __classPrivateFieldGet(this, _AudioByteStream_numChannels, "f")) !== 0) {
-            __classPrivateFieldGet(this, _AudioByteStream_logger, "f").warn('AudioByteStream: incomplete frame during flush, dropping');
+        if (this.#buf.length % (2 * this.#numChannels) !== 0) {
+            this.#logger.warn('AudioByteStream: incomplete frame during flush, dropping');
             return [];
         }
         return [
-            new AudioFrame(new Int16Array(__classPrivateFieldGet(this, _AudioByteStream_buf, "f").buffer), __classPrivateFieldGet(this, _AudioByteStream_sampleRate, "f"), __classPrivateFieldGet(this, _AudioByteStream_numChannels, "f"), __classPrivateFieldGet(this, _AudioByteStream_buf, "f").length / 2),
+            new AudioFrame(new Int16Array(this.#buf.buffer), this.#sampleRate, this.#numChannels, this.#buf.length / 2),
         ];
     }
 }
-_AudioByteStream_sampleRate = new WeakMap(), _AudioByteStream_numChannels = new WeakMap(), _AudioByteStream_bytesPerFrame = new WeakMap(), _AudioByteStream_buf = new WeakMap(), _AudioByteStream_logger = new WeakMap();
 //# sourceMappingURL=audio.js.map

dist/cli.js

@@ -1,12 +0,1 @@
-var __rest = (this && this.__rest) || function (s, e) {
-    var t = {};
-    for (var p in s) if (Object.prototype.hasOwnProperty.call(s, p) && e.indexOf(p) < 0)
-        t[p] = s[p];
-    if (s != null && typeof Object.getOwnPropertySymbols === "function")
-        for (var i = 0, p = Object.getOwnPropertySymbols(s); i < p.length; i++) {
-            if (e.indexOf(p[i]) < 0 && Object.prototype.propertyIsEnumerable.call(s, p[i]))
-                t[p[i]] = s[p[i]];
-        }
-    return t;
-};
 // SPDX-FileCopyrightText: 2024 LiveKit, Inc.
@@ -23,4 +12,4 @@ //
     // though `production` is defined in WorkerOptions, it will always be overriddden by CLI.
-    const _a = args.opts, { production: _ } = _a, opts = __rest(_a, ["production"]); // eslint-disable-line @typescript-eslint/no-unused-vars
-    const worker = new Worker(new WorkerOptions(Object.assign({ production: args.production }, opts)));
+    const { production: _, ...opts } = args.opts; // eslint-disable-line @typescript-eslint/no-unused-vars
+    const worker = new Worker(new WorkerOptions({ production: args.production, ...opts }));
     if (args.room) {
@@ -48,3 +37,3 @@ worker.event.once('worker_registered', () => {
     }
-    catch (_b) {
+    catch {
         logger.fatal('worker failed');
@@ -51,0 +40,0 @@ process.exit(1);

dist/http_server.d.ts

@@ -1,2 +0,2 @@
-/// <reference types="node" resolution-mode="require"/>
+/// <reference types="node" />
 import { type Server } from 'node:http';
@@ -3,0 +3,0 @@ export declare class HTTPServer {

dist/http_server.js

@@ -1,7 +0,1 @@
-var __classPrivateFieldGet = (this && this.__classPrivateFieldGet) || function (receiver, state, kind, f) {
-    if (kind === "a" && !f) throw new TypeError("Private accessor was defined without a getter");
-    if (typeof state === "function" ? receiver !== state || !f : !state.has(receiver)) throw new TypeError("Cannot read private member from an object whose class did not declare it");
-    return kind === "m" ? f : kind === "a" ? f.call(receiver) : f ? f.value : state.get(receiver);
-};
-var _HTTPServer_logger;
 // SPDX-FileCopyrightText: 2024 LiveKit, Inc.
@@ -17,4 +11,7 @@ //
 export class HTTPServer {
+    host;
+    port;
+    app;
+    #logger = log();
     constructor(host, port) {
-        _HTTPServer_logger.set(this, log());
         this.host = host;
@@ -39,3 +36,3 @@ this.port = port;
                 if (typeof address !== 'string') {
-                    __classPrivateFieldGet(this, _HTTPServer_logger, "f").info(`Server is listening on port ${address.port}`);
+                    this.#logger.info(`Server is listening on port ${address.port}`);
                 }
@@ -56,3 +53,2 @@ resolve();
 }
-_HTTPServer_logger = new WeakMap();
 //# sourceMappingURL=http_server.js.map

dist/index.d.ts

@@ -11,3 +11,5 @@ /**
 import * as multimodal from './multimodal/index.js';
+import * as pipeline from './pipeline/index.js';
 import * as stt from './stt/index.js';
+import * as tokenize from './tokenize/index.js';
 import * as tts from './tts/index.js';
@@ -22,6 +24,5 @@ export * from './vad.js';
 export * from './generator.js';
-export * from './tokenize.js';
 export * from './audio.js';
 export * from './transcription.js';
-export { cli, stt, tts, llm, multimodal };
+export { cli, stt, tts, llm, pipeline, multimodal, tokenize };
 //# sourceMappingURL=index.d.ts.map

dist/index.js

@@ -14,4 +14,17 @@ // SPDX-FileCopyrightText: 2024 LiveKit, Inc.
 import * as multimodal from './multimodal/index.js';
+import * as pipeline from './pipeline/index.js';
 import * as stt from './stt/index.js';
+import * as tokenize from './tokenize/index.js';
 import * as tts from './tts/index.js';
+const isCommonJS = () => {
+    try {
+        return !!require;
+    }
+    catch {
+        return false;
+    }
+};
+if (isCommonJS()) {
+    throw new ReferenceError('@livekit/agents cannot be used in a CommonJS environment. Please set `"type": "module"` in package.json.');
+}
 export * from './vad.js';
@@ -25,6 +38,5 @@ export * from './plugin.js';
 export * from './generator.js';
-export * from './tokenize.js';
 export * from './audio.js';
 export * from './transcription.js';
-export { cli, stt, tts, llm, multimodal };
+export { cli, stt, tts, llm, pipeline, multimodal, tokenize };
 //# sourceMappingURL=index.js.map

dist/ipc/job_executor.js

 export class JobExecutor {
-    constructor() {
-        this.PING_INTERVAL = 2.5 * 1000;
-        this.PING_TIMEOUT = 90 * 1000;
-        this.HIGH_PING_THRESHOLD = 0.5 * 1000;
-    }
+    PING_INTERVAL = 2.5 * 1000;
+    PING_TIMEOUT = 90 * 1000;
+    HIGH_PING_THRESHOLD = 0.5 * 1000;
 }
 //# sourceMappingURL=job_executor.js.map

dist/ipc/job_main.d.ts

@@ -1,2 +0,2 @@
-/// <reference types="node" resolution-mode="require"/>
+/// <reference types="node" />
 import type { ChildProcess } from 'node:child_process';
@@ -3,0 +3,0 @@ type StartArgs = {

dist/ipc/proc_job_executor.js

@@ -1,13 +0,1 @@
-var __classPrivateFieldSet = (this && this.__classPrivateFieldSet) || function (receiver, state, value, kind, f) {
-    if (kind === "m") throw new TypeError("Private method is not writable");
-    if (kind === "a" && !f) throw new TypeError("Private accessor was defined without a setter");
-    if (typeof state === "function" ? receiver !== state || !f : !state.has(receiver)) throw new TypeError("Cannot write private member to an object whose class did not declare it");
-    return (kind === "a" ? f.call(receiver, value) : f ? f.value = value : state.set(receiver, value)), value;
-};
-var __classPrivateFieldGet = (this && this.__classPrivateFieldGet) || function (receiver, state, kind, f) {
-    if (kind === "a" && !f) throw new TypeError("Private accessor was defined without a getter");
-    if (typeof state === "function" ? receiver !== state || !f : !state.has(receiver)) throw new TypeError("Cannot read private member from an object whose class did not declare it");
-    return kind === "m" ? f : kind === "a" ? f.call(receiver) : f ? f.value : state.get(receiver);
-};
-var _ProcJobExecutor_opts, _ProcJobExecutor_started, _ProcJobExecutor_closing, _ProcJobExecutor_runningJob, _ProcJobExecutor_proc, _ProcJobExecutor_pingInterval, _ProcJobExecutor_pongTimeout, _ProcJobExecutor_init, _ProcJobExecutor_join, _ProcJobExecutor_logger;
 import { once } from 'node:events';
@@ -18,53 +6,52 @@ import { log, loggerOptions } from '../log.js';
 export class ProcJobExecutor extends JobExecutor {
+    #opts;
+    #started = false;
+    #closing = false;
+    #runningJob = undefined;
+    #proc;
+    #pingInterval;
+    #pongTimeout;
+    #init = new Future();
+    #join = new Future();
+    #logger = log().child({ runningJob: this.#runningJob });
     constructor(agent, initializeTimeout, closeTimeout) {
         super();
-        _ProcJobExecutor_opts.set(this, void 0);
-        _ProcJobExecutor_started.set(this, false);
-        _ProcJobExecutor_closing.set(this, false);
-        _ProcJobExecutor_runningJob.set(this, undefined);
-        _ProcJobExecutor_proc.set(this, void 0);
-        _ProcJobExecutor_pingInterval.set(this, void 0);
-        _ProcJobExecutor_pongTimeout.set(this, void 0);
-        _ProcJobExecutor_init.set(this, new Future());
-        _ProcJobExecutor_join.set(this, new Future());
-        _ProcJobExecutor_logger.set(this, log().child({ runningJob: __classPrivateFieldGet(this, _ProcJobExecutor_runningJob, "f") }));
-        __classPrivateFieldSet(this, _ProcJobExecutor_opts, {
+        this.#opts = {
             agent,
             initializeTimeout,
             closeTimeout,
-        }, "f");
+        };
     }
     get started() {
-        return __classPrivateFieldGet(this, _ProcJobExecutor_started, "f");
+        return this.#started;
     }
     get runningJob() {
-        return __classPrivateFieldGet(this, _ProcJobExecutor_runningJob, "f");
+        return this.#runningJob;
     }
     async start() {
-        if (__classPrivateFieldGet(this, _ProcJobExecutor_started, "f")) {
+        if (this.#started) {
             throw new Error('runner already started');
         }
-        else if (__classPrivateFieldGet(this, _ProcJobExecutor_closing, "f")) {
+        else if (this.#closing) {
             throw new Error('runner is closed');
         }
-        __classPrivateFieldSet(this, _ProcJobExecutor_proc, await import('./job_main.js').then((m) => m.runProcess({
-            agentFile: __classPrivateFieldGet(this, _ProcJobExecutor_opts, "f").agent,
-        })), "f");
-        __classPrivateFieldSet(this, _ProcJobExecutor_started, true, "f");
+        this.#proc = await import('./job_main.js').then((m) => m.runProcess({
+            agentFile: this.#opts.agent,
+        }));
+        this.#started = true;
         this.run();
     }
     async run() {
-        await __classPrivateFieldGet(this, _ProcJobExecutor_init, "f").await;
-        __classPrivateFieldSet(this, _ProcJobExecutor_pingInterval, setInterval(() => {
-            __classPrivateFieldGet(this, _ProcJobExecutor_proc, "f").send({ case: 'pingRequest', value: { timestamp: Date.now() } });
-        }, this.PING_INTERVAL), "f");
-        __classPrivateFieldSet(this, _ProcJobExecutor_pongTimeout, setTimeout(() => {
-            __classPrivateFieldGet(this, _ProcJobExecutor_logger, "f").warn('job is unresponsive');
-            clearTimeout(__classPrivateFieldGet(this, _ProcJobExecutor_pongTimeout, "f"));
-            clearInterval(__classPrivateFieldGet(this, _ProcJobExecutor_pingInterval, "f"));
-            __classPrivateFieldGet(this, _ProcJobExecutor_proc, "f").kill();
-            __classPrivateFieldGet(this, _ProcJobExecutor_join, "f").resolve();
-        }, this.PING_TIMEOUT), "f");
+        await this.#init.await;
+        this.#pingInterval = setInterval(() => {
+            this.#proc.send({ case: 'pingRequest', value: { timestamp: Date.now() } });
+        }, this.PING_INTERVAL);
+        this.#pongTimeout = setTimeout(() => {
+            this.#logger.warn('job is unresponsive');
+            clearTimeout(this.#pongTimeout);
+            clearInterval(this.#pingInterval);
+            this.#proc.kill();
+            this.#join.resolve();
+        }, this.PING_TIMEOUT);
         const listener = (msg) => {
-            var _a;
             switch (msg.case) {
@@ -74,15 +61,15 @@ case 'pongResponse': {
                     if (delay > this.HIGH_PING_THRESHOLD) {
-                        __classPrivateFieldGet(this, _ProcJobExecutor_logger, "f").child({ delay }).warn('job executor is unresponsive');
+                        this.#logger.child({ delay }).warn('job executor is unresponsive');
                     }
-                    (_a = __classPrivateFieldGet(this, _ProcJobExecutor_pongTimeout, "f")) === null || _a === void 0 ? void 0 : _a.refresh();
+                    this.#pongTimeout?.refresh();
                     break;
                 }
                 case 'exiting': {
-                    __classPrivateFieldGet(this, _ProcJobExecutor_logger, "f").child({ reason: msg.value.reason }).debug('job exiting');
+                    this.#logger.child({ reason: msg.value.reason }).debug('job exiting');
                     break;
                 }
                 case 'done': {
-                    __classPrivateFieldSet(this, _ProcJobExecutor_closing, true, "f");
-                    __classPrivateFieldGet(this, _ProcJobExecutor_proc, "f").off('message', listener);
-                    __classPrivateFieldGet(this, _ProcJobExecutor_join, "f").resolve();
+                    this.#closing = true;
+                    this.#proc.off('message', listener);
+                    this.#join.resolve();
                     break;
@@ -92,18 +79,18 @@ }
         };
-        __classPrivateFieldGet(this, _ProcJobExecutor_proc, "f").on('message', listener);
-        __classPrivateFieldGet(this, _ProcJobExecutor_proc, "f").on('error', (err) => {
-            if (__classPrivateFieldGet(this, _ProcJobExecutor_closing, "f"))
+        this.#proc.on('message', listener);
+        this.#proc.on('error', (err) => {
+            if (this.#closing)
                 return;
-            __classPrivateFieldGet(this, _ProcJobExecutor_logger, "f").child({ err }).warn('job process exited unexpectedly');
-            clearTimeout(__classPrivateFieldGet(this, _ProcJobExecutor_pongTimeout, "f"));
-            clearInterval(__classPrivateFieldGet(this, _ProcJobExecutor_pingInterval, "f"));
-            __classPrivateFieldGet(this, _ProcJobExecutor_join, "f").resolve();
+            this.#logger.child({ err }).warn('job process exited unexpectedly');
+            clearTimeout(this.#pongTimeout);
+            clearInterval(this.#pingInterval);
+            this.#join.resolve();
         });
-        await __classPrivateFieldGet(this, _ProcJobExecutor_join, "f").await;
+        await this.#join.await;
     }
     async join() {
-        if (!__classPrivateFieldGet(this, _ProcJobExecutor_started, "f")) {
+        if (!this.#started) {
             throw new Error('runner not started');
         }
-        await __classPrivateFieldGet(this, _ProcJobExecutor_join, "f").await;
+        await this.#join.await;
     }
@@ -113,7 +100,7 @@ async initialize() {
             const err = new Error('runner initialization timed out');
-            __classPrivateFieldGet(this, _ProcJobExecutor_init, "f").reject(err);
+            this.#init.reject(err);
             throw err;
-        }, __classPrivateFieldGet(this, _ProcJobExecutor_opts, "f").initializeTimeout);
-        __classPrivateFieldGet(this, _ProcJobExecutor_proc, "f").send({ case: 'initializeRequest', value: { loggerOptions } });
-        await once(__classPrivateFieldGet(this, _ProcJobExecutor_proc, "f"), 'message').then(([msg]) => {
+        }, this.#opts.initializeTimeout);
+        this.#proc.send({ case: 'initializeRequest', value: { loggerOptions } });
+        await once(this.#proc, 'message').then(([msg]) => {
             clearTimeout(timer);
@@ -124,32 +111,31 @@ if (msg.case !== 'initializeResponse') {
         });
-        __classPrivateFieldGet(this, _ProcJobExecutor_init, "f").resolve();
+        this.#init.resolve();
     }
     async close() {
-        if (!__classPrivateFieldGet(this, _ProcJobExecutor_started, "f")) {
+        if (!this.#started) {
             return;
         }
-        __classPrivateFieldSet(this, _ProcJobExecutor_closing, true, "f");
-        if (!__classPrivateFieldGet(this, _ProcJobExecutor_runningJob, "f")) {
-            __classPrivateFieldGet(this, _ProcJobExecutor_proc, "f").kill();
-            __classPrivateFieldGet(this, _ProcJobExecutor_join, "f").resolve();
+        this.#closing = true;
+        if (!this.#runningJob) {
+            this.#proc.kill();
+            this.#join.resolve();
         }
-        __classPrivateFieldGet(this, _ProcJobExecutor_proc, "f").send({ case: 'shutdownRequest' });
+        this.#proc.send({ case: 'shutdownRequest' });
         const timer = setTimeout(() => {
-            __classPrivateFieldGet(this, _ProcJobExecutor_logger, "f").error('job shutdown is taking too much time');
-        }, __classPrivateFieldGet(this, _ProcJobExecutor_opts, "f").closeTimeout);
-        await __classPrivateFieldGet(this, _ProcJobExecutor_join, "f").await.then(() => {
+            this.#logger.error('job shutdown is taking too much time');
+        }, this.#opts.closeTimeout);
+        await this.#join.await.then(() => {
             clearTimeout(timer);
-            clearTimeout(__classPrivateFieldGet(this, _ProcJobExecutor_pongTimeout, "f"));
-            clearInterval(__classPrivateFieldGet(this, _ProcJobExecutor_pingInterval, "f"));
+            clearTimeout(this.#pongTimeout);
+            clearInterval(this.#pingInterval);
         });
     }
     async launchJob(info) {
-        if (__classPrivateFieldGet(this, _ProcJobExecutor_runningJob, "f")) {
+        if (this.#runningJob) {
             throw new Error('executor already has a running job');
         }
-        __classPrivateFieldSet(this, _ProcJobExecutor_runningJob, info, "f");
-        __classPrivateFieldGet(this, _ProcJobExecutor_proc, "f").send({ case: 'startJobRequest', value: { runningJob: info } });
+        this.#runningJob = info;
+        this.#proc.send({ case: 'startJobRequest', value: { runningJob: info } });
     }
 }
-_ProcJobExecutor_opts = new WeakMap(), _ProcJobExecutor_started = new WeakMap(), _ProcJobExecutor_closing = new WeakMap(), _ProcJobExecutor_runningJob = new WeakMap(), _ProcJobExecutor_proc = new WeakMap(), _ProcJobExecutor_pingInterval = new WeakMap(), _ProcJobExecutor_pongTimeout = new WeakMap(), _ProcJobExecutor_init = new WeakMap(), _ProcJobExecutor_join = new WeakMap(), _ProcJobExecutor_logger = new WeakMap();
 //# sourceMappingURL=proc_job_executor.js.map

dist/ipc/proc_pool.d.ts

@@ -1,2 +0,2 @@
-import { Mutex } from '@livekit/mutex';
+import { MultiMutex, Mutex } from '@livekit/mutex';
 import type { RunningJobInfo } from '../job.js';
@@ -15,6 +15,6 @@ import { Queue } from '../utils.js';
     initMutex: Mutex;
-    procMutex: Mutex;
+    procMutex: MultiMutex;
     procUnlock?: () => void;
     warmedProcQueue: Queue<JobExecutor>;
-    constructor(agent: string, _numIdleProcesses: number, initializeTimeout: number, closeTimeout: number);
+    constructor(agent: string, numIdleProcesses: number, initializeTimeout: number, closeTimeout: number);
     get processes(): JobExecutor[];
@@ -21,0 +21,0 @@ getByJobId(id: string): JobExecutor | null;

dist/ipc/proc_pool.js

 // SPDX-FileCopyrightText: 2024 LiveKit, Inc.
 //
 // SPDX-License-Identifier: Apache-2.0
-import { Mutex } from '@livekit/mutex';
+import { MultiMutex, Mutex } from '@livekit/mutex';
 import { Queue } from '../utils.js';
 import { ProcJobExecutor } from './proc_job_executor.js';
 export class ProcPool {
-    constructor(agent, _numIdleProcesses, initializeTimeout, closeTimeout) {
-        this.executors = [];
-        this.tasks = [];
-        this.started = false;
-        this.closed = false;
-        this.controller = new AbortController();
-        this.initMutex = new Mutex();
-        this.warmedProcQueue = new Queue();
+    agent;
+    initializeTimeout;
+    closeTimeout;
+    executors = [];
+    tasks = [];
+    started = false;
+    closed = false;
+    controller = new AbortController();
+    initMutex = new Mutex();
+    procMutex;
+    procUnlock;
+    warmedProcQueue = new Queue();
+    constructor(agent, numIdleProcesses, initializeTimeout, closeTimeout) {
         this.agent = agent;
-        this.procMutex = new Mutex();
+        this.procMutex = new MultiMutex(numIdleProcesses);
         this.initializeTimeout = initializeTimeout;
@@ -48,3 +53,3 @@ this.closeTimeout = closeTimeout;
             }
-            catch (_a) {
+            catch {
                 if (this.procUnlock) {
@@ -51,0 +56,0 @@ this.procUnlock();

dist/job.js

@@ -1,13 +0,1 @@
-var __classPrivateFieldSet = (this && this.__classPrivateFieldSet) || function (receiver, state, value, kind, f) {
-    if (kind === "m") throw new TypeError("Private method is not writable");
-    if (kind === "a" && !f) throw new TypeError("Private accessor was defined without a setter");
-    if (typeof state === "function" ? receiver !== state || !f : !state.has(receiver)) throw new TypeError("Cannot write private member to an object whose class did not declare it");
-    return (kind === "a" ? f.call(receiver, value) : f ? f.value = value : state.set(receiver, value)), value;
-};
-var __classPrivateFieldGet = (this && this.__classPrivateFieldGet) || function (receiver, state, kind, f) {
-    if (kind === "a" && !f) throw new TypeError("Private accessor was defined without a getter");
-    if (typeof state === "function" ? receiver !== state || !f : !state.has(receiver)) throw new TypeError("Cannot read private member from an object whose class did not declare it");
-    return kind === "m" ? f : kind === "a" ? f.call(receiver) : f ? f.value : state.get(receiver);
-};
-var _JobContext_proc, _JobContext_info, _JobContext_room, _JobContext_onConnect, _JobContext_onShutdown, _JobContext_participantEntrypoints, _JobContext_participantTasks, _JobContext_logger, _JobProcess_pid, _JobRequest_job, _JobRequest_onReject, _JobRequest_onAccept;
 import { ParticipantKind, RoomEvent, TrackKind } from '@livekit/rtc-node';
@@ -32,35 +20,35 @@ import { log } from './log.js';
 export class JobContext {
+    #proc;
+    #info;
+    #room;
+    #onConnect;
+    #onShutdown;
+    /** @internal */
+    shutdownCallbacks = [];
+    #participantEntrypoints = [];
+    #participantTasks = {};
+    #logger;
     constructor(proc, info, room, onConnect, onShutdown) {
-        _JobContext_proc.set(this, void 0);
-        _JobContext_info.set(this, void 0);
-        _JobContext_room.set(this, void 0);
-        _JobContext_onConnect.set(this, void 0);
-        _JobContext_onShutdown.set(this, void 0);
-        /** @internal */
-        this.shutdownCallbacks = [];
-        _JobContext_participantEntrypoints.set(this, []);
-        _JobContext_participantTasks.set(this, {});
-        _JobContext_logger.set(this, void 0);
-        __classPrivateFieldSet(this, _JobContext_proc, proc, "f");
-        __classPrivateFieldSet(this, _JobContext_info, info, "f");
-        __classPrivateFieldSet(this, _JobContext_room, room, "f");
-        __classPrivateFieldSet(this, _JobContext_onConnect, onConnect, "f");
-        __classPrivateFieldSet(this, _JobContext_onShutdown, onShutdown, "f");
+        this.#proc = proc;
+        this.#info = info;
+        this.#room = room;
+        this.#onConnect = onConnect;
+        this.#onShutdown = onShutdown;
         this.onParticipantConnected = this.onParticipantConnected.bind(this);
-        __classPrivateFieldGet(this, _JobContext_room, "f").on(RoomEvent.ParticipantConnected, this.onParticipantConnected);
-        __classPrivateFieldSet(this, _JobContext_logger, log().child({ info: __classPrivateFieldGet(this, _JobContext_info, "f") }), "f");
+        this.#room.on(RoomEvent.ParticipantConnected, this.onParticipantConnected);
+        this.#logger = log().child({ info: this.#info });
     }
     get proc() {
-        return __classPrivateFieldGet(this, _JobContext_proc, "f");
+        return this.#proc;
     }
     get job() {
-        return __classPrivateFieldGet(this, _JobContext_info, "f").job;
+        return this.#info.job;
     }
     /** @returns The room the agent was called into */
     get room() {
-        return __classPrivateFieldGet(this, _JobContext_room, "f");
+        return this.#room;
     }
     /** @returns The agent's participant if connected to the room, otherwise `undefined` */
     get agent() {
-        return __classPrivateFieldGet(this, _JobContext_room, "f").localParticipant;
+        return this.#room.localParticipant;
     }
@@ -72,6 +60,6 @@ /** Adds a promise to be awaited when {@link JobContext.shutdown | shutdown} is called. */
     async waitForParticipant(identity) {
-        if (!__classPrivateFieldGet(this, _JobContext_room, "f").isConnected) {
+        if (!this.#room.isConnected) {
             throw new Error('room is not connected');
         }
-        for (const p of __classPrivateFieldGet(this, _JobContext_room, "f").remoteParticipants.values()) {
+        for (const p of this.#room.remoteParticipants.values()) {
             if ((!identity || p.identity === identity) && p.info.kind != ParticipantKind.AGENT) {
@@ -94,7 +82,7 @@ return p;
             const clearHandlers = () => {
-                __classPrivateFieldGet(this, _JobContext_room, "f").off(RoomEvent.ParticipantConnected, onParticipantConnected);
-                __classPrivateFieldGet(this, _JobContext_room, "f").off(RoomEvent.Disconnected, onDisconnected);
+                this.#room.off(RoomEvent.ParticipantConnected, onParticipantConnected);
+                this.#room.off(RoomEvent.Disconnected, onDisconnected);
             };
-            __classPrivateFieldGet(this, _JobContext_room, "f").on(RoomEvent.ParticipantConnected, onParticipantConnected);
-            __classPrivateFieldGet(this, _JobContext_room, "f").on(RoomEvent.Disconnected, onDisconnected);
+            this.#room.on(RoomEvent.ParticipantConnected, onParticipantConnected);
+            this.#room.on(RoomEvent.Disconnected, onDisconnected);
         });
@@ -119,7 +107,7 @@ }
         };
-        await __classPrivateFieldGet(this, _JobContext_room, "f").connect(__classPrivateFieldGet(this, _JobContext_info, "f").url, __classPrivateFieldGet(this, _JobContext_info, "f").token, opts);
-        __classPrivateFieldGet(this, _JobContext_onConnect, "f").call(this);
-        __classPrivateFieldGet(this, _JobContext_room, "f").remoteParticipants.forEach(this.onParticipantConnected);
+        await this.#room.connect(this.#info.url, this.#info.token, opts);
+        this.#onConnect();
+        this.#room.remoteParticipants.forEach(this.onParticipantConnected);
         if ([AutoSubscribe.AUDIO_ONLY, AutoSubscribe.VIDEO_ONLY].includes(autoSubscribe)) {
-            __classPrivateFieldGet(this, _JobContext_room, "f").remoteParticipants.forEach((p) => {
+            this.#room.remoteParticipants.forEach((p) => {
                 p.trackPublications.forEach((pub) => {
@@ -140,14 +128,14 @@ if ((autoSubscribe === AutoSubscribe.AUDIO_ONLY && pub.kind === TrackKind.KIND_AUDIO) ||
     shutdown(reason = '') {
-        __classPrivateFieldGet(this, _JobContext_onShutdown, "f").call(this, reason);
+        this.#onShutdown(reason);
     }
     /** @internal */
     onParticipantConnected(p) {
-        for (const callback of __classPrivateFieldGet(this, _JobContext_participantEntrypoints, "f")) {
-            if (p.identity in __classPrivateFieldGet(this, _JobContext_participantTasks, "f") &&
-                __classPrivateFieldGet(this, _JobContext_participantTasks, "f")[p.identity].callback == callback) {
-                __classPrivateFieldGet(this, _JobContext_logger, "f").warn('a participant has joined before a prior prticipant task matching the same identity has finished:', p.identity);
+        for (const callback of this.#participantEntrypoints) {
+            if (p.identity in this.#participantTasks &&
+                this.#participantTasks[p.identity].callback == callback) {
+                this.#logger.warn('a participant has joined before a prior prticipant task matching the same identity has finished:', p.identity);
             }
             const result = callback(this, p);
-            result.finally(() => delete __classPrivateFieldGet(this, _JobContext_participantTasks, "f")[p.identity]);
-            __classPrivateFieldGet(this, _JobContext_participantTasks, "f")[p.identity] = { callback, result };
+            result.finally(() => delete this.#participantTasks[p.identity]);
+            this.#participantTasks[p.identity] = { callback, result };
         }
@@ -161,19 +149,15 @@ }
     addParticipantEntrypoint(callback) {
-        if (__classPrivateFieldGet(this, _JobContext_participantEntrypoints, "f").includes(callback)) {
+        if (this.#participantEntrypoints.includes(callback)) {
             throw new FunctionExistsError('entrypoints cannot be added more than once');
         }
-        __classPrivateFieldGet(this, _JobContext_participantEntrypoints, "f").push(callback);
+        this.#participantEntrypoints.push(callback);
     }
 }
-_JobContext_proc = new WeakMap(), _JobContext_info = new WeakMap(), _JobContext_room = new WeakMap(), _JobContext_onConnect = new WeakMap(), _JobContext_onShutdown = new WeakMap(), _JobContext_participantEntrypoints = new WeakMap(), _JobContext_participantTasks = new WeakMap(), _JobContext_logger = new WeakMap();
 export class JobProcess {
-    constructor() {
-        _JobProcess_pid.set(this, process.pid);
-        this.userData = {};
-    }
+    #pid = process.pid;
+    userData = {};
     get pid() {
-        return __classPrivateFieldGet(this, _JobProcess_pid, "f");
+        return this.#pid;
     }
 }
-_JobProcess_pid = new WeakMap();
 /**
@@ -188,34 +172,34 @@ * A request sent by the server to spawn a new agent job.
 export class JobRequest {
+    #job;
+    #onReject;
+    #onAccept;
     /** @internal */
     constructor(job, onReject, onAccept) {
-        _JobRequest_job.set(this, void 0);
-        _JobRequest_onReject.set(this, void 0);
-        _JobRequest_onAccept.set(this, void 0);
-        __classPrivateFieldSet(this, _JobRequest_job, job, "f");
-        __classPrivateFieldSet(this, _JobRequest_onReject, onReject, "f");
-        __classPrivateFieldSet(this, _JobRequest_onAccept, onAccept, "f");
+        this.#job = job;
+        this.#onReject = onReject;
+        this.#onAccept = onAccept;
     }
     /** @returns The ID of the job, set by the LiveKit server */
     get id() {
-        return __classPrivateFieldGet(this, _JobRequest_job, "f").id;
+        return this.#job.id;
     }
     /** @see {@link https://www.npmjs.com/package/@livekit/protocol | @livekit/protocol} */
     get job() {
-        return __classPrivateFieldGet(this, _JobRequest_job, "f");
+        return this.#job;
     }
     /** @see {@link https://www.npmjs.com/package/@livekit/protocol | @livekit/protocol} */
     get room() {
-        return __classPrivateFieldGet(this, _JobRequest_job, "f").room;
+        return this.#job.room;
     }
     /** @see {@link https://www.npmjs.com/package/@livekit/protocol | @livekit/protocol} */
     get publisher() {
-        return __classPrivateFieldGet(this, _JobRequest_job, "f").participant;
+        return this.#job.participant;
     }
     /** @returns The agent's name, as set in {@link WorkerOptions} */
     get agentName() {
-        return __classPrivateFieldGet(this, _JobRequest_job, "f").agentName;
+        return this.#job.agentName;
     }
     /** Rejects the job. */
     async reject() {
-        await __classPrivateFieldGet(this, _JobRequest_onReject, "f").call(this);
+        await this.#onReject();
     }
@@ -226,6 +210,5 @@ /** Accepts the job, launching it on an idle child process. */
             identity = 'agent-' + this.id;
-        __classPrivateFieldGet(this, _JobRequest_onAccept, "f").call(this, { name, identity, metadata });
+        this.#onAccept({ name, identity, metadata });
     }
 }
-_JobRequest_job = new WeakMap(), _JobRequest_onReject = new WeakMap(), _JobRequest_onAccept = new WeakMap();
 //# sourceMappingURL=job.js.map

dist/llm/function_context.d.ts

@@ -10,2 +10,18 @@ import { z } from 'zod';
 }
+/** A function that has been called but is not yet running */
+export interface FunctionCallInfo<P extends z.ZodTypeAny = any, R = any> {
+    name: string;
+    func: CallableFunction<P, R>;
+    toolCallId: string;
+    rawParams: string;
+    params: inferParameters<P>;
+    task?: PromiseLike<CallableFunctionResult>;
+}
+/** The result of a ran FunctionCallInfo. */
+export interface CallableFunctionResult {
+    name: string;
+    toolCallId: string;
+    result?: any;
+    error?: any;
+}
 /** An object containing callable functions and their names */
@@ -19,4 +35,6 @@ export type FunctionContext = {
     properties: Record<string, any>;
-    required_properties: string[];
+    required: string[];
 };
+/** @internal */
+export declare const oaiBuildFunctionInfo: (fncCtx: FunctionContext, toolCallId: string, fncName: string, rawArgs: string) => FunctionCallInfo;
 //# sourceMappingURL=function_context.d.ts.map

dist/llm/function_context.js

@@ -8,22 +8,45 @@ // SPDX-FileCopyrightText: 2024 LiveKit, Inc.
     const properties = {};
-    const required_properties = [];
-    for (const key in p.shape) {
-        const field = p.shape[key];
-        const description = field._def.description || undefined;
-        let type;
-        let enumValues;
-        if (field instanceof z.ZodEnum) {
-            enumValues = field._def.values;
-            type = typeof enumValues[0];
+    const requiredProperties = [];
+    const processZodType = (field) => {
+        const isOptional = field instanceof z.ZodOptional;
+        const nestedField = isOptional ? field._def.innerType : field;
+        const description = field._def.description;
+        if (nestedField instanceof z.ZodEnum) {
+            return {
+                type: typeof nestedField._def.values[0],
+                ...(description && { description }),
+                enum: nestedField._def.values,
+            };
         }
+        else if (nestedField instanceof z.ZodArray) {
+            const elementType = nestedField._def.type;
+            return {
+                type: 'array',
+                ...(description && { description }),
+                items: processZodType(elementType),
+            };
+        }
+        else if (nestedField instanceof z.ZodObject) {
+            const { properties, required } = oaiParams(nestedField);
+            return {
+                type: 'object',
+                ...(description && { description }),
+                properties,
+                required,
+            };
+        }
         else {
-            type = field._def.typeName.toLowerCase();
+            let type = nestedField._def.typeName.toLowerCase();
+            type = type.includes('zod') ? type.substring(3) : type;
+            return {
+                type,
+                ...(description && { description }),
+            };
         }
-        properties[key] = {
-            type: type.includes('zod') ? type.substring(3) : type,
-            description,
-            enum: enumValues,
-        };
-        if (!field._def.defaultValue) {
-            required_properties.push(key);
+    };
+    for (const key in p.shape) {
+        const field = p.shape[key];
+        properties[key] = processZodType(field);
+        if (!(field instanceof z.ZodOptional)) {
+            requiredProperties.push(key);
         }
@@ -35,5 +58,18 @@ }
         properties,
-        required_properties,
+        required: requiredProperties,
     };
 };
+/** @internal */
+export const oaiBuildFunctionInfo = (fncCtx, toolCallId, fncName, rawArgs) => {
+    if (!fncCtx[fncName]) {
+        throw new Error(`AI function ${fncName} not found`);
+    }
+    return {
+        name: fncName,
+        func: fncCtx[fncName],
+        toolCallId,
+        rawParams: rawArgs,
+        params: JSON.parse(rawArgs),
+    };
+};
 //# sourceMappingURL=function_context.js.map

dist/llm/index.d.ts

@@ -1,3 +0,4 @@
-import { type CallableFunction, type FunctionContext, type inferParameters, oaiParams } from './function_context.js';
-export { CallableFunction, FunctionContext, inferParameters, oaiParams };
+export { type CallableFunction, type FunctionCallInfo, type CallableFunctionResult, type FunctionContext, type inferParameters, oaiParams, oaiBuildFunctionInfo, } from './function_context.js';
+export { type ChatImage, type ChatAudio, type ChatContent, ChatRole, ChatMessage, ChatContext, } from './chat_context.js';
+export { type ChoiceDelta, type CompletionUsage, type Choice, type ChatChunk, LLM, LLMStream, } from './llm.js';
 //# sourceMappingURL=index.d.ts.map

dist/llm/index.js

 // SPDX-FileCopyrightText: 2024 LiveKit, Inc.
 //
 // SPDX-License-Identifier: Apache-2.0
-import { oaiParams, } from './function_context.js';
-export { oaiParams };
+export { oaiParams, oaiBuildFunctionInfo, } from './function_context.js';
+export { ChatRole, ChatMessage, ChatContext, } from './chat_context.js';
+export { LLM, LLMStream, } from './llm.js';
 //# sourceMappingURL=index.js.map

dist/multimodal/agent_playout.d.ts

@@ -1,2 +0,2 @@
-/// <reference types="node" resolution-mode="require"/>
+/// <reference types="node" />
 import type { AudioFrame } from '@livekit/rtc-node';
@@ -3,0 +3,0 @@ import { type AudioSource } from '@livekit/rtc-node';

dist/multimodal/agent_playout.js

@@ -1,20 +0,1 @@
-var __classPrivateFieldSet = (this && this.__classPrivateFieldSet) || function (receiver, state, value, kind, f) {
-    if (kind === "m") throw new TypeError("Private method is not writable");
-    if (kind === "a" && !f) throw new TypeError("Private accessor was defined without a setter");
-    if (typeof state === "function" ? receiver !== state || !f : !state.has(receiver)) throw new TypeError("Cannot write private member to an object whose class did not declare it");
-    return (kind === "a" ? f.call(receiver, value) : f ? f.value = value : state.set(receiver, value)), value;
-};
-var __classPrivateFieldGet = (this && this.__classPrivateFieldGet) || function (receiver, state, kind, f) {
-    if (kind === "a" && !f) throw new TypeError("Private accessor was defined without a getter");
-    if (typeof state === "function" ? receiver !== state || !f : !state.has(receiver)) throw new TypeError("Cannot read private member from an object whose class did not declare it");
-    return kind === "m" ? f : kind === "a" ? f.call(receiver) : f ? f.value : state.get(receiver);
-};
-var __asyncValues = (this && this.__asyncValues) || function (o) {
-    if (!Symbol.asyncIterator) throw new TypeError("Symbol.asyncIterator is not defined.");
-    var m = o[Symbol.asyncIterator], i;
-    return m ? m.call(o) : (o = typeof __values === "function" ? __values(o) : o[Symbol.iterator](), i = {}, verb("next"), verb("throw"), verb("return"), i[Symbol.asyncIterator] = function () { return this; }, i);
-    function verb(n) { i[n] = o[n] && function (v) { return new Promise(function (resolve, reject) { v = o[n](v), settle(resolve, reject, v.done, v.value); }); }; }
-    function settle(resolve, reject, d, v) { Promise.resolve(v).then(function(v) { resolve({ value: v, done: d }); }, reject); }
-};
-var _PlayoutHandle_audioSource, _PlayoutHandle_sampleRate, _PlayoutHandle_itemId, _PlayoutHandle_contentIndex, _PlayoutHandle_interrupted, _AgentPlayout_instances, _AgentPlayout_audioSource, _AgentPlayout_playoutTask, _AgentPlayout_sampleRate, _AgentPlayout_numChannels, _AgentPlayout_inFrameSize, _AgentPlayout_outFrameSize, _AgentPlayout_makePlayoutTask;
 import { EventEmitter } from 'node:events';
@@ -25,18 +6,28 @@ import { AudioByteStream } from '../audio.js';
 export class PlayoutHandle extends EventEmitter {
+    #audioSource;
+    #sampleRate;
+    #itemId;
+    #contentIndex;
+    /** @internal */
+    transcriptionFwd;
+    /** @internal */
+    doneFut;
+    /** @internal */
+    intFut;
+    /** @internal */
+    #interrupted;
+    /** @internal */
+    pushedDuration;
+    /** @internal */
+    totalPlayedTime; // Set when playout is done
     constructor(audioSource, sampleRate, itemId, contentIndex, transcriptionFwd) {
         super();
-        _PlayoutHandle_audioSource.set(this, void 0);
-        _PlayoutHandle_sampleRate.set(this, void 0);
-        _PlayoutHandle_itemId.set(this, void 0);
-        _PlayoutHandle_contentIndex.set(this, void 0);
-        /** @internal */
-        _PlayoutHandle_interrupted.set(this, void 0);
-        __classPrivateFieldSet(this, _PlayoutHandle_audioSource, audioSource, "f");
-        __classPrivateFieldSet(this, _PlayoutHandle_sampleRate, sampleRate, "f");
-        __classPrivateFieldSet(this, _PlayoutHandle_itemId, itemId, "f");
-        __classPrivateFieldSet(this, _PlayoutHandle_contentIndex, contentIndex, "f");
+        this.#audioSource = audioSource;
+        this.#sampleRate = sampleRate;
+        this.#itemId = itemId;
+        this.#contentIndex = contentIndex;
         this.transcriptionFwd = transcriptionFwd;
         this.doneFut = new Future();
         this.intFut = new Future();
-        __classPrivateFieldSet(this, _PlayoutHandle_interrupted, false, "f");
+        this.#interrupted = false;
         this.pushedDuration = 0;
@@ -46,9 +37,9 @@ this.totalPlayedTime = undefined;
     get itemId() {
-        return __classPrivateFieldGet(this, _PlayoutHandle_itemId, "f");
+        return this.#itemId;
     }
     get audioSamples() {
         if (this.totalPlayedTime !== undefined) {
-            return Math.floor(this.totalPlayedTime * __classPrivateFieldGet(this, _PlayoutHandle_sampleRate, "f"));
+            return Math.floor(this.totalPlayedTime * this.#sampleRate);
         }
-        return Math.floor((this.pushedDuration - __classPrivateFieldGet(this, _PlayoutHandle_audioSource, "f").queuedDuration) * (__classPrivateFieldGet(this, _PlayoutHandle_sampleRate, "f") / 1000));
+        return Math.floor((this.pushedDuration - this.#audioSource.queuedDuration) * (this.#sampleRate / 1000));
     }
@@ -59,9 +50,9 @@ get textChars() {
     get contentIndex() {
-        return __classPrivateFieldGet(this, _PlayoutHandle_contentIndex, "f");
+        return this.#contentIndex;
     }
     get interrupted() {
-        return __classPrivateFieldGet(this, _PlayoutHandle_interrupted, "f");
+        return this.#interrupted;
     }
     get done() {
-        return this.doneFut.done || __classPrivateFieldGet(this, _PlayoutHandle_interrupted, "f");
+        return this.doneFut.done || this.#interrupted;
     }
@@ -72,54 +63,46 @@ interrupt() {
         this.intFut.resolve();
-        __classPrivateFieldSet(this, _PlayoutHandle_interrupted, true, "f");
+        this.#interrupted = true;
     }
 }
-_PlayoutHandle_audioSource = new WeakMap(), _PlayoutHandle_sampleRate = new WeakMap(), _PlayoutHandle_itemId = new WeakMap(), _PlayoutHandle_contentIndex = new WeakMap(), _PlayoutHandle_interrupted = new WeakMap();
 export class AgentPlayout extends EventEmitter {
+    #audioSource;
+    #playoutTask;
+    #sampleRate;
+    #numChannels;
+    #inFrameSize;
+    #outFrameSize;
     constructor(audioSource, sampleRate, numChannels, inFrameSize, outFrameSize) {
         super();
-        _AgentPlayout_instances.add(this);
-        _AgentPlayout_audioSource.set(this, void 0);
-        _AgentPlayout_playoutTask.set(this, void 0);
-        _AgentPlayout_sampleRate.set(this, void 0);
-        _AgentPlayout_numChannels.set(this, void 0);
-        _AgentPlayout_inFrameSize.set(this, void 0);
-        _AgentPlayout_outFrameSize.set(this, void 0);
-        __classPrivateFieldSet(this, _AgentPlayout_audioSource, audioSource, "f");
-        __classPrivateFieldSet(this, _AgentPlayout_playoutTask, null, "f");
-        __classPrivateFieldSet(this, _AgentPlayout_sampleRate, sampleRate, "f");
-        __classPrivateFieldSet(this, _AgentPlayout_numChannels, numChannels, "f");
-        __classPrivateFieldSet(this, _AgentPlayout_inFrameSize, inFrameSize, "f");
-        __classPrivateFieldSet(this, _AgentPlayout_outFrameSize, outFrameSize, "f");
+        this.#audioSource = audioSource;
+        this.#playoutTask = null;
+        this.#sampleRate = sampleRate;
+        this.#numChannels = numChannels;
+        this.#inFrameSize = inFrameSize;
+        this.#outFrameSize = outFrameSize;
     }
     play(itemId, contentIndex, transcriptionFwd, textStream, audioStream) {
-        const handle = new PlayoutHandle(__classPrivateFieldGet(this, _AgentPlayout_audioSource, "f"), __classPrivateFieldGet(this, _AgentPlayout_sampleRate, "f"), itemId, contentIndex, transcriptionFwd);
-        __classPrivateFieldSet(this, _AgentPlayout_playoutTask, __classPrivateFieldGet(this, _AgentPlayout_instances, "m", _AgentPlayout_makePlayoutTask).call(this, __classPrivateFieldGet(this, _AgentPlayout_playoutTask, "f"), handle, textStream, audioStream), "f");
+        const handle = new PlayoutHandle(this.#audioSource, this.#sampleRate, itemId, contentIndex, transcriptionFwd);
+        this.#playoutTask = this.#makePlayoutTask(this.#playoutTask, handle, textStream, audioStream);
         return handle;
     }
-}
-_AgentPlayout_audioSource = new WeakMap(), _AgentPlayout_playoutTask = new WeakMap(), _AgentPlayout_sampleRate = new WeakMap(), _AgentPlayout_numChannels = new WeakMap(), _AgentPlayout_inFrameSize = new WeakMap(), _AgentPlayout_outFrameSize = new WeakMap(), _AgentPlayout_instances = new WeakSet(), _AgentPlayout_makePlayoutTask = function _AgentPlayout_makePlayoutTask(oldTask, handle, textStream, audioStream) {
-    return new CancellablePromise((resolve, reject, onCancel) => {
-        let cancelled = false;
-        onCancel(() => {
-            cancelled = true;
-        });
-        (async () => {
-            try {
-                if (oldTask) {
-                    await gracefullyCancel(oldTask);
-                }
-                let firstFrame = true;
-                const readText = () => new CancellablePromise((resolveText, rejectText, onCancelText) => {
-                    let cancelledText = false;
-                    onCancelText(() => {
-                        cancelledText = true;
-                    });
-                    (async () => {
-                        var _a, e_1, _b, _c;
-                        try {
+    #makePlayoutTask(oldTask, handle, textStream, audioStream) {
+        return new CancellablePromise((resolve, reject, onCancel) => {
+            let cancelled = false;
+            onCancel(() => {
+                cancelled = true;
+            });
+            (async () => {
+                try {
+                    if (oldTask) {
+                        await gracefullyCancel(oldTask);
+                    }
+                    let firstFrame = true;
+                    const readText = () => new CancellablePromise((resolveText, rejectText, onCancelText) => {
+                        let cancelledText = false;
+                        onCancelText(() => {
+                            cancelledText = true;
+                        });
+                        (async () => {
                             try {
-                                for (var _d = true, textStream_1 = __asyncValues(textStream), textStream_1_1; textStream_1_1 = await textStream_1.next(), _a = textStream_1_1.done, !_a; _d = true) {
-                                    _c = textStream_1_1.value;
-                                    _d = false;
-                                    const text = _c;
+                                for await (const text of textStream) {
                                     if (cancelledText || cancelled) {
@@ -130,32 +113,19 @@ break;
                                 }
+                                resolveText();
                             }
-                            catch (e_1_1) { e_1 = { error: e_1_1 }; }
-                            finally {
-                                try {
-                                    if (!_d && !_a && (_b = textStream_1.return)) await _b.call(textStream_1);
-                                }
-                                finally { if (e_1) throw e_1.error; }
+                            catch (error) {
+                                rejectText(error);
                             }
-                            resolveText();
-                        }
-                        catch (error) {
-                            rejectText(error);
-                        }
-                    })();
-                });
-                const capture = () => new CancellablePromise((resolveCapture, rejectCapture, onCancelCapture) => {
-                    let cancelledCapture = false;
-                    onCancelCapture(() => {
-                        cancelledCapture = true;
+                        })();
                     });
-                    (async () => {
-                        var _a, e_2, _b, _c;
-                        try {
-                            const samplesPerChannel = __classPrivateFieldGet(this, _AgentPlayout_outFrameSize, "f");
-                            const bstream = new AudioByteStream(__classPrivateFieldGet(this, _AgentPlayout_sampleRate, "f"), __classPrivateFieldGet(this, _AgentPlayout_numChannels, "f"), samplesPerChannel);
+                    const capture = () => new CancellablePromise((resolveCapture, rejectCapture, onCancelCapture) => {
+                        let cancelledCapture = false;
+                        onCancelCapture(() => {
+                            cancelledCapture = true;
+                        });
+                        (async () => {
                             try {
-                                for (var _d = true, audioStream_1 = __asyncValues(audioStream), audioStream_1_1; audioStream_1_1 = await audioStream_1.next(), _a = audioStream_1_1.done, !_a; _d = true) {
-                                    _c = audioStream_1_1.value;
-                                    _d = false;
-                                    const frame = _c;
+                                const samplesPerChannel = this.#outFrameSize;
+                                const bstream = new AudioByteStream(this.#sampleRate, this.#numChannels, samplesPerChannel);
+                                for await (const frame of audioStream) {
                                     if (cancelledCapture || cancelled) {
@@ -172,61 +142,54 @@ break;
                                         handle.pushedDuration += (f.samplesPerChannel / f.sampleRate) * 1000;
-                                        await __classPrivateFieldGet(this, _AgentPlayout_audioSource, "f").captureFrame(f);
+                                        await this.#audioSource.captureFrame(f);
                                     }
                                 }
-                            }
-                            catch (e_2_1) { e_2 = { error: e_2_1 }; }
-                            finally {
-                                try {
-                                    if (!_d && !_a && (_b = audioStream_1.return)) await _b.call(audioStream_1);
+                                if (!cancelledCapture && !cancelled) {
+                                    for (const f of bstream.flush()) {
+                                        handle.pushedDuration += (f.samplesPerChannel / f.sampleRate) * 1000;
+                                        await this.#audioSource.captureFrame(f);
+                                    }
+                                    handle.transcriptionFwd.markAudioComplete();
+                                    await this.#audioSource.waitForPlayout();
                                 }
-                                finally { if (e_2) throw e_2.error; }
+                                resolveCapture();
                             }
-                            if (!cancelledCapture && !cancelled) {
-                                for (const f of bstream.flush()) {
-                                    handle.pushedDuration += (f.samplesPerChannel / f.sampleRate) * 1000;
-                                    await __classPrivateFieldGet(this, _AgentPlayout_audioSource, "f").captureFrame(f);
-                                }
-                                handle.transcriptionFwd.markAudioComplete();
-                                await __classPrivateFieldGet(this, _AgentPlayout_audioSource, "f").waitForPlayout();
+                            catch (error) {
+                                rejectCapture(error);
                             }
-                            resolveCapture();
+                        })();
+                    });
+                    const readTextTask = readText();
+                    const captureTask = capture();
+                    try {
+                        await Promise.race([captureTask, handle.intFut.await]);
+                    }
+                    finally {
+                        if (!captureTask.isCancelled) {
+                            await gracefullyCancel(captureTask);
                         }
-                        catch (error) {
-                            rejectCapture(error);
+                        handle.totalPlayedTime = handle.pushedDuration - this.#audioSource.queuedDuration;
+                        if (handle.interrupted || captureTask.error) {
+                            this.#audioSource.clearQueue(); // make sure to remove any queued frames
                         }
-                    })();
-                });
-                const readTextTask = readText();
-                const captureTask = capture();
-                try {
-                    await Promise.race([captureTask, handle.intFut.await]);
-                }
-                finally {
-                    if (!captureTask.isCancelled) {
-                        await gracefullyCancel(captureTask);
-                    }
-                    handle.totalPlayedTime = handle.pushedDuration - __classPrivateFieldGet(this, _AgentPlayout_audioSource, "f").queuedDuration;
-                    if (handle.interrupted || captureTask.error) {
-                        __classPrivateFieldGet(this, _AgentPlayout_audioSource, "f").clearQueue(); // make sure to remove any queued frames
-                    }
-                    if (!readTextTask.isCancelled) {
-                        await gracefullyCancel(readTextTask);
-                    }
-                    if (!firstFrame) {
-                        if (!handle.interrupted) {
-                            handle.transcriptionFwd.markTextComplete();
+                        if (!readTextTask.isCancelled) {
+                            await gracefullyCancel(readTextTask);
                         }
-                        this.emit('playout_stopped', handle.interrupted);
+                        if (!firstFrame) {
+                            if (!handle.interrupted) {
+                                handle.transcriptionFwd.markTextComplete();
+                            }
+                            this.emit('playout_stopped', handle.interrupted);
+                        }
+                        handle.doneFut.resolve();
+                        await handle.transcriptionFwd.close(handle.interrupted);
                     }
-                    handle.doneFut.resolve();
-                    await handle.transcriptionFwd.close(handle.interrupted);
+                    resolve();
                 }
-                resolve();
-            }
-            catch (error) {
-                reject(error);
-            }
-        })();
-    });
-};
+                catch (error) {
+                    reject(error);
+                }
+            })();
+        });
+    }
+}
 //# sourceMappingURL=agent_playout.js.map

dist/multimodal/multimodal_agent.d.ts

@@ -1,2 +0,2 @@
-/// <reference types="node" resolution-mode="require"/>
+/// <reference types="node" />
 import type { RemoteAudioTrack, RemoteParticipant, Room } from '@livekit/rtc-node';
@@ -39,5 +39,6 @@ import { EventEmitter } from 'node:events';
     } | null;
-    constructor({ model, fncCtx, }: {
+    constructor({ model, chatCtx, fncCtx, }: {
         model: RealtimeModel;
-        fncCtx?: llm.FunctionContext | undefined;
+        chatCtx?: llm.ChatContext;
+        fncCtx?: llm.FunctionContext;
     });
@@ -44,0 +45,0 @@ get fncCtx(): llm.FunctionContext | undefined;

dist/multimodal/multimodal_agent.js

@@ -1,20 +0,1 @@
-var __classPrivateFieldSet = (this && this.__classPrivateFieldSet) || function (receiver, state, value, kind, f) {
-    if (kind === "m") throw new TypeError("Private method is not writable");
-    if (kind === "a" && !f) throw new TypeError("Private accessor was defined without a setter");
-    if (typeof state === "function" ? receiver !== state || !f : !state.has(receiver)) throw new TypeError("Cannot write private member to an object whose class did not declare it");
-    return (kind === "a" ? f.call(receiver, value) : f ? f.value = value : state.set(receiver, value)), value;
-};
-var __classPrivateFieldGet = (this && this.__classPrivateFieldGet) || function (receiver, state, kind, f) {
-    if (kind === "a" && !f) throw new TypeError("Private accessor was defined without a getter");
-    if (typeof state === "function" ? receiver !== state || !f : !state.has(receiver)) throw new TypeError("Cannot read private member from an object whose class did not declare it");
-    return kind === "m" ? f : kind === "a" ? f.call(receiver) : f ? f.value : state.get(receiver);
-};
-var __asyncValues = (this && this.__asyncValues) || function (o) {
-    if (!Symbol.asyncIterator) throw new TypeError("Symbol.asyncIterator is not defined.");
-    var m = o[Symbol.asyncIterator], i;
-    return m ? m.call(o) : (o = typeof __values === "function" ? __values(o) : o[Symbol.iterator](), i = {}, verb("next"), verb("throw"), verb("return"), i[Symbol.asyncIterator] = function () { return this; }, i);
-    function verb(n) { i[n] = o[n] && function (v) { return new Promise(function (resolve, reject) { v = o[n](v), settle(resolve, reject, v.done, v.value); }); }; }
-    function settle(resolve, reject, d, v) { Promise.resolve(v).then(function(v) { resolve({ value: v, done: d }); }, reject); }
-};
-var _MultimodalAgent_instances, _MultimodalAgent_participant, _MultimodalAgent_agentPublication, _MultimodalAgent_localTrackSid, _MultimodalAgent_localSource, _MultimodalAgent_agentPlayout, _MultimodalAgent_playingHandle, _MultimodalAgent_logger, _MultimodalAgent_session, _MultimodalAgent_fncCtx, _MultimodalAgent__started, _MultimodalAgent__pendingFunctionCalls, _MultimodalAgent__speaking, _MultimodalAgent_pendingFunctionCalls_get, _MultimodalAgent_pendingFunctionCalls_set, _MultimodalAgent_speaking_get, _MultimodalAgent_speaking_set, _MultimodalAgent_started_get, _MultimodalAgent_started_set, _MultimodalAgent_linkParticipant, _MultimodalAgent_subscribeToMicrophone, _MultimodalAgent_handleTrackSubscription, _MultimodalAgent_getLocalTrackSid, _MultimodalAgent_publishTranscription, _MultimodalAgent_updateState, _MultimodalAgent_setState;
 import { AudioSource, AudioStream, LocalAudioTrack, RoomEvent, TrackPublishOptions, TrackSource, } from '@livekit/rtc-node';
@@ -42,40 +23,62 @@ import { EventEmitter } from 'node:events';
 export class MultimodalAgent extends EventEmitter {
-    constructor({ model, fncCtx, }) {
+    model;
+    room = null;
+    linkedParticipant = null;
+    subscribedTrack = null;
+    readMicroTask = null;
+    constructor({ model, chatCtx, fncCtx, }) {
         super();
-        _MultimodalAgent_instances.add(this);
-        this.room = null;
-        this.linkedParticipant = null;
-        this.subscribedTrack = null;
-        this.readMicroTask = null;
-        _MultimodalAgent_participant.set(this, null);
-        _MultimodalAgent_agentPublication.set(this, null);
-        _MultimodalAgent_localTrackSid.set(this, null);
-        _MultimodalAgent_localSource.set(this, null);
-        _MultimodalAgent_agentPlayout.set(this, null);
-        _MultimodalAgent_playingHandle.set(this, undefined);
-        _MultimodalAgent_logger.set(this, log());
-        _MultimodalAgent_session.set(this, null);
-        _MultimodalAgent_fncCtx.set(this, undefined);
-        _MultimodalAgent__started.set(this, false);
-        _MultimodalAgent__pendingFunctionCalls.set(this, new Set());
-        _MultimodalAgent__speaking.set(this, false);
         this.model = model;
-        __classPrivateFieldSet(this, _MultimodalAgent_fncCtx, fncCtx, "f");
+        this.#chatCtx = chatCtx;
+        this.#fncCtx = fncCtx;
     }
+    #participant = null;
+    #agentPublication = null;
+    #localTrackSid = null;
+    #localSource = null;
+    #agentPlayout = null;
+    #playingHandle = undefined;
+    #logger = log();
+    #session = null;
+    #fncCtx = undefined;
+    #chatCtx = undefined;
+    #_started = false;
+    #_pendingFunctionCalls = new Set();
+    #_speaking = false;
     get fncCtx() {
-        return __classPrivateFieldGet(this, _MultimodalAgent_fncCtx, "f");
+        return this.#fncCtx;
     }
     set fncCtx(ctx) {
-        __classPrivateFieldSet(this, _MultimodalAgent_fncCtx, ctx, "f");
-        if (__classPrivateFieldGet(this, _MultimodalAgent_session, "f")) {
-            __classPrivateFieldGet(this, _MultimodalAgent_session, "f").fncCtx = ctx;
+        this.#fncCtx = ctx;
+        if (this.#session) {
+            this.#session.fncCtx = ctx;
         }
     }
+    get #pendingFunctionCalls() {
+        return this.#_pendingFunctionCalls;
+    }
+    set #pendingFunctionCalls(calls) {
+        this.#_pendingFunctionCalls = calls;
+        this.#updateState();
+    }
+    get #speaking() {
+        return this.#_speaking;
+    }
+    set #speaking(isSpeaking) {
+        this.#_speaking = isSpeaking;
+        this.#updateState();
+    }
+    get #started() {
+        return this.#_started;
+    }
+    set #started(started) {
+        this.#_started = started;
+        this.#updateState();
+    }
     start(room, participant = null) {
         return new Promise(async (resolve, reject) => {
-            var _a;
-            if (__classPrivateFieldGet(this, _MultimodalAgent_instances, "a", _MultimodalAgent_started_get)) {
+            if (this.#started) {
                 reject(new Error('MultimodalAgent already started'));
             }
-            __classPrivateFieldGet(this, _MultimodalAgent_instances, "m", _MultimodalAgent_updateState).call(this);
+            this.#updateState();
             room.on(RoomEvent.ParticipantConnected, (participant) => {
@@ -86,3 +89,3 @@ // automatically link to the first participant that connects, if not already linked
                 }
-                __classPrivateFieldGet(this, _MultimodalAgent_instances, "m", _MultimodalAgent_linkParticipant).call(this, participant.identity);
+                this.#linkParticipant(participant.identity);
             });
@@ -97,10 +100,10 @@ room.on(RoomEvent.TrackPublished, (trackPublication, participant) => {
             });
-            room.on(RoomEvent.TrackSubscribed, __classPrivateFieldGet(this, _MultimodalAgent_instances, "m", _MultimodalAgent_handleTrackSubscription).bind(this));
+            room.on(RoomEvent.TrackSubscribed, this.#handleTrackSubscription.bind(this));
             this.room = room;
-            __classPrivateFieldSet(this, _MultimodalAgent_participant, participant, "f");
-            __classPrivateFieldSet(this, _MultimodalAgent_localSource, new AudioSource(this.model.sampleRate, this.model.numChannels), "f");
-            __classPrivateFieldSet(this, _MultimodalAgent_agentPlayout, new AgentPlayout(__classPrivateFieldGet(this, _MultimodalAgent_localSource, "f"), this.model.sampleRate, this.model.numChannels, this.model.inFrameSize, this.model.outFrameSize), "f");
+            this.#participant = participant;
+            this.#localSource = new AudioSource(this.model.sampleRate, this.model.numChannels);
+            this.#agentPlayout = new AgentPlayout(this.#localSource, this.model.sampleRate, this.model.numChannels, this.model.inFrameSize, this.model.outFrameSize);
             const onPlayoutStarted = () => {
                 this.emit('agent_started_speaking');
-                __classPrivateFieldSet(this, _MultimodalAgent_instances, true, "a", _MultimodalAgent_speaking_set);
+                this.#speaking = true;
             };
@@ -110,22 +113,22 @@ // eslint-disable-next-line @typescript-eslint/no-unused-vars
                 this.emit('agent_stopped_speaking');
-                __classPrivateFieldSet(this, _MultimodalAgent_instances, false, "a", _MultimodalAgent_speaking_set);
+                this.#speaking = false;
             };
-            __classPrivateFieldGet(this, _MultimodalAgent_agentPlayout, "f").on('playout_started', onPlayoutStarted);
-            __classPrivateFieldGet(this, _MultimodalAgent_agentPlayout, "f").on('playout_stopped', onPlayoutStopped);
-            const track = LocalAudioTrack.createAudioTrack('assistant_voice', __classPrivateFieldGet(this, _MultimodalAgent_localSource, "f"));
+            this.#agentPlayout.on('playout_started', onPlayoutStarted);
+            this.#agentPlayout.on('playout_stopped', onPlayoutStopped);
+            const track = LocalAudioTrack.createAudioTrack('assistant_voice', this.#localSource);
             const options = new TrackPublishOptions();
             options.source = TrackSource.SOURCE_MICROPHONE;
-            __classPrivateFieldSet(this, _MultimodalAgent_agentPublication, (await ((_a = room.localParticipant) === null || _a === void 0 ? void 0 : _a.publishTrack(track, options))) || null, "f");
-            if (!__classPrivateFieldGet(this, _MultimodalAgent_agentPublication, "f")) {
-                __classPrivateFieldGet(this, _MultimodalAgent_logger, "f").error('Failed to publish track');
+            this.#agentPublication = (await room.localParticipant?.publishTrack(track, options)) || null;
+            if (!this.#agentPublication) {
+                this.#logger.error('Failed to publish track');
                 reject(new Error('Failed to publish track'));
                 return;
             }
-            await __classPrivateFieldGet(this, _MultimodalAgent_agentPublication, "f").waitForSubscription();
+            await this.#agentPublication.waitForSubscription();
             if (participant) {
                 if (typeof participant === 'string') {
-                    __classPrivateFieldGet(this, _MultimodalAgent_instances, "m", _MultimodalAgent_linkParticipant).call(this, participant);
+                    this.#linkParticipant(participant);
                 }
                 else {
-                    __classPrivateFieldGet(this, _MultimodalAgent_instances, "m", _MultimodalAgent_linkParticipant).call(this, participant.identity);
+                    this.#linkParticipant(participant.identity);
                 }
@@ -136,223 +139,193 @@ }
                 for (const participant of room.remoteParticipants.values()) {
-                    __classPrivateFieldGet(this, _MultimodalAgent_instances, "m", _MultimodalAgent_linkParticipant).call(this, participant.identity);
+                    this.#linkParticipant(participant.identity);
                     break;
                 }
             }
-            __classPrivateFieldSet(this, _MultimodalAgent_session, this.model.session({ fncCtx: __classPrivateFieldGet(this, _MultimodalAgent_fncCtx, "f") }), "f");
-            __classPrivateFieldSet(this, _MultimodalAgent_instances, true, "a", _MultimodalAgent_started_set);
+            this.#session = this.model.session({ fncCtx: this.#fncCtx, chatCtx: this.#chatCtx });
+            this.#started = true;
             // eslint-disable-next-line @typescript-eslint/no-explicit-any
-            __classPrivateFieldGet(this, _MultimodalAgent_session, "f").on('response_content_added', (message) => {
-                var _a;
+            this.#session.on('response_content_added', (message) => {
                 // openai.realtime.RealtimeContent
-                const trFwd = new BasicTranscriptionForwarder(this.room, this.room.localParticipant.identity, __classPrivateFieldGet(this, _MultimodalAgent_instances, "m", _MultimodalAgent_getLocalTrackSid).call(this), message.responseId);
-                const handle = (_a = __classPrivateFieldGet(this, _MultimodalAgent_agentPlayout, "f")) === null || _a === void 0 ? void 0 : _a.play(message.itemId, message.contentIndex, trFwd, message.textStream, message.audioStream);
-                __classPrivateFieldSet(this, _MultimodalAgent_playingHandle, handle, "f");
+                const trFwd = new BasicTranscriptionForwarder(this.room, this.room.localParticipant.identity, this.#getLocalTrackSid(), message.responseId);
+                const handle = this.#agentPlayout?.play(message.itemId, message.contentIndex, trFwd, message.textStream, message.audioStream);
+                this.#playingHandle = handle;
             });
             // eslint-disable-next-line @typescript-eslint/no-explicit-any
-            __classPrivateFieldGet(this, _MultimodalAgent_session, "f").on('input_speech_committed', (ev) => {
-                var _a, _b;
+            this.#session.on('input_speech_committed', (ev) => {
                 // openai.realtime.InputSpeechCommittedEvent
-                const participantIdentity = (_a = this.linkedParticipant) === null || _a === void 0 ? void 0 : _a.identity;
-                const trackSid = (_b = this.subscribedTrack) === null || _b === void 0 ? void 0 : _b.sid;
+                const participantIdentity = this.linkedParticipant?.identity;
+                const trackSid = this.subscribedTrack?.sid;
                 if (participantIdentity && trackSid) {
-                    __classPrivateFieldGet(this, _MultimodalAgent_instances, "m", _MultimodalAgent_publishTranscription).call(this, participantIdentity, trackSid, '…', false, ev.itemId);
+                    this.#publishTranscription(participantIdentity, trackSid, '…', false, ev.itemId);
                 }
                 else {
-                    __classPrivateFieldGet(this, _MultimodalAgent_logger, "f").error('Participant or track not set');
+                    this.#logger.error('Participant or track not set');
                 }
             });
             // eslint-disable-next-line @typescript-eslint/no-explicit-any
-            __classPrivateFieldGet(this, _MultimodalAgent_session, "f").on('input_speech_transcription_completed', (ev) => {
-                var _a, _b;
+            this.#session.on('input_speech_transcription_completed', (ev) => {
                 // openai.realtime.InputSpeechTranscriptionCompletedEvent
                 const transcription = ev.transcript;
-                const participantIdentity = (_a = this.linkedParticipant) === null || _a === void 0 ? void 0 : _a.identity;
-                const trackSid = (_b = this.subscribedTrack) === null || _b === void 0 ? void 0 : _b.sid;
+                const participantIdentity = this.linkedParticipant?.identity;
+                const trackSid = this.subscribedTrack?.sid;
                 if (participantIdentity && trackSid) {
-                    __classPrivateFieldGet(this, _MultimodalAgent_instances, "m", _MultimodalAgent_publishTranscription).call(this, participantIdentity, trackSid, transcription, true, ev.itemId);
+                    this.#publishTranscription(participantIdentity, trackSid, transcription, true, ev.itemId);
                 }
                 else {
-                    __classPrivateFieldGet(this, _MultimodalAgent_logger, "f").error('Participant or track not set');
+                    this.#logger.error('Participant or track not set');
                 }
             });
-            __classPrivateFieldGet(this, _MultimodalAgent_session, "f").on('input_speech_started', (ev) => {
-                var _a, _b;
-                if (__classPrivateFieldGet(this, _MultimodalAgent_playingHandle, "f") && !__classPrivateFieldGet(this, _MultimodalAgent_playingHandle, "f").done) {
-                    __classPrivateFieldGet(this, _MultimodalAgent_playingHandle, "f").interrupt();
-                    __classPrivateFieldGet(this, _MultimodalAgent_session, "f").conversation.item.truncate(__classPrivateFieldGet(this, _MultimodalAgent_playingHandle, "f").itemId, __classPrivateFieldGet(this, _MultimodalAgent_playingHandle, "f").contentIndex, Math.floor((__classPrivateFieldGet(this, _MultimodalAgent_playingHandle, "f").audioSamples / 24000) * 1000));
-                    __classPrivateFieldSet(this, _MultimodalAgent_playingHandle, undefined, "f");
+            this.#session.on('input_speech_started', (ev) => {
+                if (this.#playingHandle && !this.#playingHandle.done) {
+                    this.#playingHandle.interrupt();
+                    this.#session.conversation.item.truncate(this.#playingHandle.itemId, this.#playingHandle.contentIndex, Math.floor((this.#playingHandle.audioSamples / 24000) * 1000));
+                    this.#playingHandle = undefined;
                 }
-                const participantIdentity = (_a = this.linkedParticipant) === null || _a === void 0 ? void 0 : _a.identity;
-                const trackSid = (_b = this.subscribedTrack) === null || _b === void 0 ? void 0 : _b.sid;
+                const participantIdentity = this.linkedParticipant?.identity;
+                const trackSid = this.subscribedTrack?.sid;
                 if (participantIdentity && trackSid) {
-                    __classPrivateFieldGet(this, _MultimodalAgent_instances, "m", _MultimodalAgent_publishTranscription).call(this, participantIdentity, trackSid, '…', false, ev.itemId);
+                    this.#publishTranscription(participantIdentity, trackSid, '…', false, ev.itemId);
                 }
             });
             // eslint-disable-next-line @typescript-eslint/no-explicit-any
-            __classPrivateFieldGet(this, _MultimodalAgent_session, "f").on('function_call_started', (ev) => {
-                __classPrivateFieldGet(this, _MultimodalAgent_instances, "a", _MultimodalAgent_pendingFunctionCalls_get).add(ev.callId);
-                __classPrivateFieldGet(this, _MultimodalAgent_instances, "m", _MultimodalAgent_updateState).call(this);
+            this.#session.on('function_call_started', (ev) => {
+                this.#pendingFunctionCalls.add(ev.callId);
+                this.#updateState();
             });
             // eslint-disable-next-line @typescript-eslint/no-explicit-any
-            __classPrivateFieldGet(this, _MultimodalAgent_session, "f").on('function_call_completed', (ev) => {
-                __classPrivateFieldGet(this, _MultimodalAgent_instances, "a", _MultimodalAgent_pendingFunctionCalls_get).delete(ev.callId);
-                __classPrivateFieldGet(this, _MultimodalAgent_instances, "m", _MultimodalAgent_updateState).call(this);
+            this.#session.on('function_call_completed', (ev) => {
+                this.#pendingFunctionCalls.delete(ev.callId);
+                this.#updateState();
             });
             // eslint-disable-next-line @typescript-eslint/no-explicit-any
-            __classPrivateFieldGet(this, _MultimodalAgent_session, "f").on('function_call_failed', (ev) => {
-                __classPrivateFieldGet(this, _MultimodalAgent_instances, "a", _MultimodalAgent_pendingFunctionCalls_get).delete(ev.callId);
-                __classPrivateFieldGet(this, _MultimodalAgent_instances, "m", _MultimodalAgent_updateState).call(this);
+            this.#session.on('function_call_failed', (ev) => {
+                this.#pendingFunctionCalls.delete(ev.callId);
+                this.#updateState();
             });
-            resolve(__classPrivateFieldGet(this, _MultimodalAgent_session, "f"));
+            resolve(this.#session);
         });
     }
-}
-_MultimodalAgent_participant = new WeakMap(), _MultimodalAgent_agentPublication = new WeakMap(), _MultimodalAgent_localTrackSid = new WeakMap(), _MultimodalAgent_localSource = new WeakMap(), _MultimodalAgent_agentPlayout = new WeakMap(), _MultimodalAgent_playingHandle = new WeakMap(), _MultimodalAgent_logger = new WeakMap(), _MultimodalAgent_session = new WeakMap(), _MultimodalAgent_fncCtx = new WeakMap(), _MultimodalAgent__started = new WeakMap(), _MultimodalAgent__pendingFunctionCalls = new WeakMap(), _MultimodalAgent__speaking = new WeakMap(), _MultimodalAgent_instances = new WeakSet(), _MultimodalAgent_pendingFunctionCalls_get = function _MultimodalAgent_pendingFunctionCalls_get() {
-    return __classPrivateFieldGet(this, _MultimodalAgent__pendingFunctionCalls, "f");
-}, _MultimodalAgent_pendingFunctionCalls_set = function _MultimodalAgent_pendingFunctionCalls_set(calls) {
-    __classPrivateFieldSet(this, _MultimodalAgent__pendingFunctionCalls, calls, "f");
-    __classPrivateFieldGet(this, _MultimodalAgent_instances, "m", _MultimodalAgent_updateState).call(this);
-}, _MultimodalAgent_speaking_get = function _MultimodalAgent_speaking_get() {
-    return __classPrivateFieldGet(this, _MultimodalAgent__speaking, "f");
-}, _MultimodalAgent_speaking_set = function _MultimodalAgent_speaking_set(isSpeaking) {
-    __classPrivateFieldSet(this, _MultimodalAgent__speaking, isSpeaking, "f");
-    __classPrivateFieldGet(this, _MultimodalAgent_instances, "m", _MultimodalAgent_updateState).call(this);
-}, _MultimodalAgent_started_get = function _MultimodalAgent_started_get() {
-    return __classPrivateFieldGet(this, _MultimodalAgent__started, "f");
-}, _MultimodalAgent_started_set = function _MultimodalAgent_started_set(started) {
-    __classPrivateFieldSet(this, _MultimodalAgent__started, started, "f");
-    __classPrivateFieldGet(this, _MultimodalAgent_instances, "m", _MultimodalAgent_updateState).call(this);
-}, _MultimodalAgent_linkParticipant = function _MultimodalAgent_linkParticipant(participantIdentity) {
-    if (!this.room) {
-        __classPrivateFieldGet(this, _MultimodalAgent_logger, "f").error('Room is not set');
-        return;
+    #linkParticipant(participantIdentity) {
+        if (!this.room) {
+            this.#logger.error('Room is not set');
+            return;
+        }
+        this.linkedParticipant = this.room.remoteParticipants.get(participantIdentity) || null;
+        if (!this.linkedParticipant) {
+            this.#logger.error(`Participant with identity ${participantIdentity} not found`);
+            return;
+        }
+        if (this.linkedParticipant.trackPublications.size > 0) {
+            this.#subscribeToMicrophone();
+        }
+        // also check if already subscribed
+        for (const publication of this.linkedParticipant.trackPublications.values()) {
+            if (publication.source === TrackSource.SOURCE_MICROPHONE && publication.track) {
+                this.#handleTrackSubscription(publication.track, publication, this.linkedParticipant);
+                break;
+            }
+        }
     }
-    this.linkedParticipant = this.room.remoteParticipants.get(participantIdentity) || null;
-    if (!this.linkedParticipant) {
-        __classPrivateFieldGet(this, _MultimodalAgent_logger, "f").error(`Participant with identity ${participantIdentity} not found`);
-        return;
-    }
-    if (this.linkedParticipant.trackPublications.size > 0) {
-        __classPrivateFieldGet(this, _MultimodalAgent_instances, "m", _MultimodalAgent_subscribeToMicrophone).call(this);
-    }
-    // also check if already subscribed
-    for (const publication of this.linkedParticipant.trackPublications.values()) {
-        if (publication.source === TrackSource.SOURCE_MICROPHONE && publication.track) {
-            __classPrivateFieldGet(this, _MultimodalAgent_instances, "m", _MultimodalAgent_handleTrackSubscription).call(this, publication.track, publication, this.linkedParticipant);
-            break;
+    #subscribeToMicrophone() {
+        if (!this.linkedParticipant) {
+            this.#logger.error('Participant is not set');
+            return;
         }
+        let microphonePublication = undefined;
+        for (const publication of this.linkedParticipant.trackPublications.values()) {
+            if (publication.source === TrackSource.SOURCE_MICROPHONE) {
+                microphonePublication = publication;
+                break;
+            }
+        }
+        if (!microphonePublication) {
+            return;
+        }
+        if (!microphonePublication.subscribed) {
+            microphonePublication.setSubscribed(true);
+        }
     }
-}, _MultimodalAgent_subscribeToMicrophone = function _MultimodalAgent_subscribeToMicrophone() {
-    if (!this.linkedParticipant) {
-        __classPrivateFieldGet(this, _MultimodalAgent_logger, "f").error('Participant is not set');
-        return;
-    }
-    let microphonePublication = undefined;
-    for (const publication of this.linkedParticipant.trackPublications.values()) {
-        if (publication.source === TrackSource.SOURCE_MICROPHONE) {
-            microphonePublication = publication;
-            break;
+    #handleTrackSubscription(track, publication, participant) {
+        if (publication.source !== TrackSource.SOURCE_MICROPHONE ||
+            participant.identity !== this.linkedParticipant?.identity) {
+            return;
         }
-    }
-    if (!microphonePublication) {
-        return;
-    }
-    if (!microphonePublication.subscribed) {
-        microphonePublication.setSubscribed(true);
-    }
-}, _MultimodalAgent_handleTrackSubscription = function _MultimodalAgent_handleTrackSubscription(track, publication, participant) {
-    var _a;
-    if (publication.source !== TrackSource.SOURCE_MICROPHONE ||
-        participant.identity !== ((_a = this.linkedParticipant) === null || _a === void 0 ? void 0 : _a.identity)) {
-        return;
-    }
-    const readAudioStreamTask = async (audioStream) => {
-        var _a, e_1, _b, _c;
-        const bstream = new AudioByteStream(this.model.sampleRate, this.model.numChannels, this.model.inFrameSize);
-        try {
-            for (var _d = true, audioStream_1 = __asyncValues(audioStream), audioStream_1_1; audioStream_1_1 = await audioStream_1.next(), _a = audioStream_1_1.done, !_a; _d = true) {
-                _c = audioStream_1_1.value;
-                _d = false;
-                const frame = _c;
+        const readAudioStreamTask = async (audioStream) => {
+            const bstream = new AudioByteStream(this.model.sampleRate, this.model.numChannels, this.model.inFrameSize);
+            for await (const frame of audioStream) {
                 const audioData = frame.data;
                 for (const frame of bstream.write(audioData.buffer)) {
-                    __classPrivateFieldGet(this, _MultimodalAgent_session, "f").inputAudioBuffer.append(frame);
+                    this.#session.inputAudioBuffer.append(frame);
                 }
             }
+        };
+        this.subscribedTrack = track;
+        if (this.readMicroTask) {
+            this.readMicroTask.cancel();
         }
-        catch (e_1_1) { e_1 = { error: e_1_1 }; }
-        finally {
-            try {
-                if (!_d && !_a && (_b = audioStream_1.return)) await _b.call(audioStream_1);
-            }
-            finally { if (e_1) throw e_1.error; }
+        let cancel;
+        this.readMicroTask = {
+            promise: new Promise((resolve, reject) => {
+                cancel = () => {
+                    reject(new Error('Task cancelled'));
+                };
+                readAudioStreamTask(new AudioStream(track, this.model.sampleRate, this.model.numChannels))
+                    .then(resolve)
+                    .catch(reject);
+            }),
+            cancel: () => cancel(),
+        };
+    }
+    #getLocalTrackSid() {
+        if (!this.#localTrackSid && this.room && this.room.localParticipant) {
+            this.#localTrackSid = findMicroTrackId(this.room, this.room.localParticipant?.identity);
         }
-    };
-    this.subscribedTrack = track;
-    if (this.readMicroTask) {
-        this.readMicroTask.cancel();
+        return this.#localTrackSid;
     }
-    let cancel;
-    this.readMicroTask = {
-        promise: new Promise((resolve, reject) => {
-            cancel = () => {
-                reject(new Error('Task cancelled'));
-            };
-            readAudioStreamTask(new AudioStream(track, this.model.sampleRate, this.model.numChannels))
-                .then(resolve)
-                .catch(reject);
-        }),
-        cancel: () => cancel(),
-    };
-}, _MultimodalAgent_getLocalTrackSid = function _MultimodalAgent_getLocalTrackSid() {
-    var _a;
-    if (!__classPrivateFieldGet(this, _MultimodalAgent_localTrackSid, "f") && this.room && this.room.localParticipant) {
-        __classPrivateFieldSet(this, _MultimodalAgent_localTrackSid, findMicroTrackId(this.room, (_a = this.room.localParticipant) === null || _a === void 0 ? void 0 : _a.identity), "f");
+    #publishTranscription(participantIdentity, trackSid, text, isFinal, id) {
+        this.#logger.debug(`Publishing transcription ${participantIdentity} ${trackSid} ${text} ${isFinal} ${id}`);
+        if (!this.room?.localParticipant) {
+            this.#logger.error('Room or local participant not set');
+            return;
+        }
+        this.room.localParticipant.publishTranscription({
+            participantIdentity,
+            trackSid,
+            segments: [
+                {
+                    text,
+                    final: isFinal,
+                    id,
+                    startTime: BigInt(0),
+                    endTime: BigInt(0),
+                    language: '',
+                },
+            ],
+        });
     }
-    return __classPrivateFieldGet(this, _MultimodalAgent_localTrackSid, "f");
-}, _MultimodalAgent_publishTranscription = function _MultimodalAgent_publishTranscription(participantIdentity, trackSid, text, isFinal, id) {
-    var _a;
-    __classPrivateFieldGet(this, _MultimodalAgent_logger, "f").debug(`Publishing transcription ${participantIdentity} ${trackSid} ${text} ${isFinal} ${id}`);
-    if (!((_a = this.room) === null || _a === void 0 ? void 0 : _a.localParticipant)) {
-        __classPrivateFieldGet(this, _MultimodalAgent_logger, "f").error('Room or local participant not set');
-        return;
+    #updateState() {
+        let newState = 'initializing';
+        if (this.#pendingFunctionCalls.size > 0) {
+            newState = 'thinking';
+        }
+        else if (this.#speaking) {
+            newState = 'speaking';
+        }
+        else if (this.#started) {
+            newState = 'listening';
+        }
+        this.#setState(newState);
     }
-    this.room.localParticipant.publishTranscription({
-        participantIdentity,
-        trackSid,
-        segments: [
-            {
-                text,
-                final: isFinal,
-                id,
-                startTime: BigInt(0),
-                endTime: BigInt(0),
-                language: '',
-            },
-        ],
-    });
-}, _MultimodalAgent_updateState = function _MultimodalAgent_updateState() {
-    let newState = 'initializing';
-    if (__classPrivateFieldGet(this, _MultimodalAgent_instances, "a", _MultimodalAgent_pendingFunctionCalls_get).size > 0) {
-        newState = 'thinking';
-    }
-    else if (__classPrivateFieldGet(this, _MultimodalAgent_instances, "a", _MultimodalAgent_speaking_get)) {
-        newState = 'speaking';
-    }
-    else if (__classPrivateFieldGet(this, _MultimodalAgent_instances, "a", _MultimodalAgent_started_get)) {
-        newState = 'listening';
-    }
-    __classPrivateFieldGet(this, _MultimodalAgent_instances, "m", _MultimodalAgent_setState).call(this, newState);
-}, _MultimodalAgent_setState = function _MultimodalAgent_setState(state) {
-    var _a;
-    if (((_a = this.room) === null || _a === void 0 ? void 0 : _a.isConnected) && this.room.localParticipant) {
-        const currentState = this.room.localParticipant.attributes[AGENT_STATE_ATTRIBUTE];
-        if (currentState !== state) {
-            this.room.localParticipant.setAttributes({
-                [AGENT_STATE_ATTRIBUTE]: state,
-            });
-            __classPrivateFieldGet(this, _MultimodalAgent_logger, "f").debug(`${AGENT_STATE_ATTRIBUTE}: ${currentState} ->${state}`);
+    #setState(state) {
+        if (this.room?.isConnected && this.room.localParticipant) {
+            const currentState = this.room.localParticipant.attributes[AGENT_STATE_ATTRIBUTE];
+            if (currentState !== state) {
+                this.room.localParticipant.setAttributes({
+                    [AGENT_STATE_ATTRIBUTE]: state,
+                });
+                this.#logger.debug(`${AGENT_STATE_ATTRIBUTE}: ${currentState} ->${state}`);
+            }
         }
     }
-};
+}
 //# sourceMappingURL=multimodal_agent.js.map

dist/plugin.js

 // SPDX-FileCopyrightText: 2024 LiveKit, Inc.
 //
 // SPDX-License-Identifier: Apache-2.0
-var __classPrivateFieldSet = (this && this.__classPrivateFieldSet) || function (receiver, state, value, kind, f) {
-    if (kind === "m") throw new TypeError("Private method is not writable");
-    if (kind === "a" && !f) throw new TypeError("Private accessor was defined without a setter");
-    if (typeof state === "function" ? receiver !== state || !f : !state.has(receiver)) throw new TypeError("Cannot write private member to an object whose class did not declare it");
-    return (kind === "a" ? f.call(receiver, value) : f ? f.value = value : state.set(receiver, value)), value;
-};
-var __classPrivateFieldGet = (this && this.__classPrivateFieldGet) || function (receiver, state, kind, f) {
-    if (kind === "a" && !f) throw new TypeError("Private accessor was defined without a getter");
-    if (typeof state === "function" ? receiver !== state || !f : !state.has(receiver)) throw new TypeError("Cannot read private member from an object whose class did not declare it");
-    return kind === "m" ? f : kind === "a" ? f.call(receiver) : f ? f.value : state.get(receiver);
-};
-var _Plugin_title, _Plugin_version;
 export class Plugin {
+    registeredPlugins = [];
+    #title;
+    #version;
     constructor(title, version) {
-        this.registeredPlugins = [];
-        _Plugin_title.set(this, void 0);
-        _Plugin_version.set(this, void 0);
-        __classPrivateFieldSet(this, _Plugin_title, title, "f");
-        __classPrivateFieldSet(this, _Plugin_version, version, "f");
+        this.#title = title;
+        this.#version = version;
     }
@@ -28,9 +16,8 @@ static registerPlugins(plugin) {
     get title() {
-        return __classPrivateFieldGet(this, _Plugin_title, "f");
+        return this.#title;
     }
     get version() {
-        return __classPrivateFieldGet(this, _Plugin_version, "f");
+        return this.#version;
     }
 }
-_Plugin_title = new WeakMap(), _Plugin_version = new WeakMap();
 //# sourceMappingURL=plugin.js.map

dist/stt/index.d.ts

@@ -1,3 +0,2 @@
-export { STT, SpeechEvent, SpeechEventType, SpeechStream, type SpeechData } from './stt.js';
-export { StreamAdapter, StreamAdapterWrapper } from './stream_adapter.js';
+export { type SpeechEvent, type SpeechData, type STTCapabilities, SpeechEventType, STT, SpeechStream, } from './stt.js';
 //# sourceMappingURL=index.d.ts.map

dist/stt/index.js

 // SPDX-FileCopyrightText: 2024 LiveKit, Inc.
 //
 // SPDX-License-Identifier: Apache-2.0
-export { STT, SpeechEvent, SpeechEventType, SpeechStream } from './stt.js';
-export { StreamAdapter, StreamAdapterWrapper } from './stream_adapter.js';
+export { SpeechEventType, STT, SpeechStream, } from './stt.js';
 //# sourceMappingURL=index.js.map

dist/stt/stt.d.ts

 import type { AudioFrame } from '@livekit/rtc-node';
-import type { AudioBuffer } from '../utils.js';
+import { AsyncIterableQueue } from '../utils.js';
+/** Indicates start/middle/end of speech */
 export declare enum SpeechEventType {
@@ -7,3 +8,3 @@ /**
      * If the STT doesn't support this event, this will be emitted at the same time
-     * as the first INTERMIN_TRANSCRIPT.
+     * as the first INTERIM_TRANSCRIPT.
      */
@@ -26,2 +27,3 @@ START_OF_SPEECH = 0,
 }
+/** SpeechData contains metadata about this {@link SpeechEvent}. */
 export interface SpeechData {
@@ -34,31 +36,67 @@ language: string;
 }
-export declare class SpeechEvent {
+/** SpeechEvent is a packet of speech-to-text data. */
+export interface SpeechEvent {
     type: SpeechEventType;
     alternatives: SpeechData[];
-    constructor(type: SpeechEventType, alternatives?: SpeechData[]);
 }
-export declare abstract class SpeechStream implements IterableIterator<SpeechEvent> {
-    /**
-     * Push a frame to be recognised.
-     * It is recommended to push frames as soon as they are available.
-     */
-    abstract pushFrame(token: AudioFrame): void;
-    /**
-     * Close the stream.
-     *
-     * @param wait
-     *   Whether to wait for the STT to finish processing the remaining
-     *   frames before closing
-     */
-    abstract close(wait: boolean): Promise<void>;
-    abstract next(): IteratorResult<SpeechEvent>;
-    [Symbol.iterator](): SpeechStream;
+/**
+ * Describes the capabilities of the STT provider.
+ *
+ * @remarks
+ * At present, the framework only supports providers that have a streaming endpoint.
+ */
+export interface STTCapabilities {
+    streaming: boolean;
+    interimResults: boolean;
 }
+/**
+ * An instance of a speech-to-text adapter.
+ *
+ * @remarks
+ * This class is abstract, and as such cannot be used directly. Instead, use a provider plugin that
+ * exports its own child STT class, which inherits this class's methods.
+ */
 export declare abstract class STT {
     #private;
-    constructor(streamingSupported: boolean);
-    abstract recognize(buffer: AudioBuffer, language?: string): Promise<SpeechEvent>;
-    abstract stream(language: string | undefined): SpeechStream;
-    get streamingSupported(): boolean;
+    constructor(capabilities: STTCapabilities);
+    /** Returns this STT's capabilities */
+    get capabilities(): STTCapabilities;
+    /**
+     * Returns a {@link SpeechStream} that can be used to push audio frames and receive
+     * transcriptions
+     */
+    abstract stream(): SpeechStream;
 }
+/**
+ * An instance of a speech-to-text stream, as an asynchronous iterable iterator.
+ *
+ * @example Looping through frames
+ * ```ts
+ * for await (const event of stream) {
+ *   if (event.type === SpeechEventType.FINAL_TRANSCRIPT) {
+ *     console.log(event.alternatives[0].text)
+ *   }
+ * }
+ * ```
+ *
+ * @remarks
+ * This class is abstract, and as such cannot be used directly. Instead, use a provider plugin that
+ * exports its own child SpeechStream class, which inherits this class's methods.
+ */
+export declare abstract class SpeechStream implements AsyncIterableIterator<SpeechEvent> {
+    protected static readonly FLUSH_SENTINEL: unique symbol;
+    protected input: AsyncIterableQueue<AudioFrame | typeof SpeechStream.FLUSH_SENTINEL>;
+    protected queue: AsyncIterableQueue<SpeechEvent>;
+    protected closed: boolean;
+    /** Push an audio frame to the STT */
+    pushFrame(frame: AudioFrame): void;
+    /** Flush the STT, causing it to process all pending text */
+    flush(): void;
+    /** Mark the input as ended and forbid additional pushes */
+    endInput(): void;
+    next(): Promise<IteratorResult<SpeechEvent>>;
+    /** Close both the input and output of the STT stream */
+    close(): void;
+    [Symbol.asyncIterator](): SpeechStream;
+}
 //# sourceMappingURL=stt.d.ts.map

dist/stt/stt.js

@@ -1,13 +0,3 @@
-var __classPrivateFieldSet = (this && this.__classPrivateFieldSet) || function (receiver, state, value, kind, f) {
-    if (kind === "m") throw new TypeError("Private method is not writable");
-    if (kind === "a" && !f) throw new TypeError("Private accessor was defined without a setter");
-    if (typeof state === "function" ? receiver !== state || !f : !state.has(receiver)) throw new TypeError("Cannot write private member to an object whose class did not declare it");
-    return (kind === "a" ? f.call(receiver, value) : f ? f.value = value : state.set(receiver, value)), value;
-};
-var __classPrivateFieldGet = (this && this.__classPrivateFieldGet) || function (receiver, state, kind, f) {
-    if (kind === "a" && !f) throw new TypeError("Private accessor was defined without a getter");
-    if (typeof state === "function" ? receiver !== state || !f : !state.has(receiver)) throw new TypeError("Cannot read private member from an object whose class did not declare it");
-    return kind === "m" ? f : kind === "a" ? f.call(receiver) : f ? f.value : state.get(receiver);
-};
-var _STT_streamingSupported;
+import { AsyncIterableQueue } from '../utils.js';
+/** Indicates start/middle/end of speech */
 export var SpeechEventType;
@@ -18,3 +8,3 @@ (function (SpeechEventType) {
      * If the STT doesn't support this event, this will be emitted at the same time
-     * as the first INTERMIN_TRANSCRIPT.
+     * as the first INTERIM_TRANSCRIPT.
      */
@@ -37,23 +27,83 @@ SpeechEventType[SpeechEventType["START_OF_SPEECH"] = 0] = "START_OF_SPEECH";
 })(SpeechEventType || (SpeechEventType = {}));
-export class SpeechEvent {
-    constructor(type, alternatives = []) {
-        this.type = type;
-        this.alternatives = alternatives;
+/**
+ * An instance of a speech-to-text adapter.
+ *
+ * @remarks
+ * This class is abstract, and as such cannot be used directly. Instead, use a provider plugin that
+ * exports its own child STT class, which inherits this class's methods.
+ */
+export class STT {
+    #capabilities;
+    constructor(capabilities) {
+        this.#capabilities = capabilities;
     }
+    /** Returns this STT's capabilities */
+    get capabilities() {
+        return this.#capabilities;
+    }
 }
+/**
+ * An instance of a speech-to-text stream, as an asynchronous iterable iterator.
+ *
+ * @example Looping through frames
+ * ```ts
+ * for await (const event of stream) {
+ *   if (event.type === SpeechEventType.FINAL_TRANSCRIPT) {
+ *     console.log(event.alternatives[0].text)
+ *   }
+ * }
+ * ```
+ *
+ * @remarks
+ * This class is abstract, and as such cannot be used directly. Instead, use a provider plugin that
+ * exports its own child SpeechStream class, which inherits this class's methods.
+ */
 export class SpeechStream {
-    [Symbol.iterator]() {
-        return this;
+    static FLUSH_SENTINEL = Symbol('FLUSH_SENTINEL');
+    input = new AsyncIterableQueue();
+    queue = new AsyncIterableQueue();
+    closed = false;
+    /** Push an audio frame to the STT */
+    pushFrame(frame) {
+        if (this.input.closed) {
+            throw new Error('Input is closed');
+        }
+        if (this.closed) {
+            throw new Error('Stream is closed');
+        }
+        this.input.put(frame);
     }
-}
-export class STT {
-    constructor(streamingSupported) {
-        _STT_streamingSupported.set(this, void 0);
-        __classPrivateFieldSet(this, _STT_streamingSupported, streamingSupported, "f");
+    /** Flush the STT, causing it to process all pending text */
+    flush() {
+        if (this.input.closed) {
+            throw new Error('Input is closed');
+        }
+        if (this.closed) {
+            throw new Error('Stream is closed');
+        }
+        this.input.put(SpeechStream.FLUSH_SENTINEL);
     }
-    get streamingSupported() {
-        return __classPrivateFieldGet(this, _STT_streamingSupported, "f");
+    /** Mark the input as ended and forbid additional pushes */
+    endInput() {
+        if (this.input.closed) {
+            throw new Error('Input is closed');
+        }
+        if (this.closed) {
+            throw new Error('Stream is closed');
+        }
+        this.input.close();
     }
+    next() {
+        return this.queue.next();
+    }
+    /** Close both the input and output of the STT stream */
+    close() {
+        this.input.close();
+        this.queue.close();
+        this.closed = true;
+    }
+    [Symbol.asyncIterator]() {
+        return this;
+    }
 }
-_STT_streamingSupported = new WeakMap();
 //# sourceMappingURL=stt.js.map

dist/transcription.js

@@ -1,42 +0,27 @@
-var __classPrivateFieldSet = (this && this.__classPrivateFieldSet) || function (receiver, state, value, kind, f) {
-    if (kind === "m") throw new TypeError("Private method is not writable");
-    if (kind === "a" && !f) throw new TypeError("Private accessor was defined without a setter");
-    if (typeof state === "function" ? receiver !== state || !f : !state.has(receiver)) throw new TypeError("Cannot write private member to an object whose class did not declare it");
-    return (kind === "a" ? f.call(receiver, value) : f ? f.value = value : state.set(receiver, value)), value;
-};
-var __classPrivateFieldGet = (this && this.__classPrivateFieldGet) || function (receiver, state, kind, f) {
-    if (kind === "a" && !f) throw new TypeError("Private accessor was defined without a getter");
-    if (typeof state === "function" ? receiver !== state || !f : !state.has(receiver)) throw new TypeError("Cannot read private member from an object whose class did not declare it");
-    return kind === "m" ? f : kind === "a" ? f.call(receiver) : f ? f.value : state.get(receiver);
-};
-var _BasicTranscriptionForwarder_instances, _BasicTranscriptionForwarder_room, _BasicTranscriptionForwarder_participantIdentity, _BasicTranscriptionForwarder_trackSid, _BasicTranscriptionForwarder_currentText, _BasicTranscriptionForwarder_totalAudioDuration, _BasicTranscriptionForwarder_currentPlayoutTime, _BasicTranscriptionForwarder_DEFAULT_CHARS_PER_SECOND, _BasicTranscriptionForwarder_charsPerSecond, _BasicTranscriptionForwarder_messageId, _BasicTranscriptionForwarder_isRunning, _BasicTranscriptionForwarder_logger, _BasicTranscriptionForwarder_textIsComplete, _BasicTranscriptionForwarder_audioIsComplete, _BasicTranscriptionForwarder_adjustTimingIfBothFinished, _BasicTranscriptionForwarder_computeSleepInterval, _BasicTranscriptionForwarder_startPublishingLoop, _BasicTranscriptionForwarder_publishTranscription;
 import { log } from './log.js';
 export class BasicTranscriptionForwarder {
+    #room;
+    #participantIdentity;
+    #trackSid;
+    #currentText = '';
+    #totalAudioDuration = 0;
+    #currentPlayoutTime = 0;
+    #DEFAULT_CHARS_PER_SECOND = 16;
+    #charsPerSecond = this.#DEFAULT_CHARS_PER_SECOND;
+    #messageId;
+    #isRunning = false;
+    #logger = log();
+    currentCharacterIndex = 0;
     constructor(room, participantIdentity, trackSid, messageId) {
-        _BasicTranscriptionForwarder_instances.add(this);
-        _BasicTranscriptionForwarder_room.set(this, void 0);
-        _BasicTranscriptionForwarder_participantIdentity.set(this, void 0);
-        _BasicTranscriptionForwarder_trackSid.set(this, void 0);
-        _BasicTranscriptionForwarder_currentText.set(this, '');
-        _BasicTranscriptionForwarder_totalAudioDuration.set(this, 0);
-        _BasicTranscriptionForwarder_currentPlayoutTime.set(this, 0);
-        _BasicTranscriptionForwarder_DEFAULT_CHARS_PER_SECOND.set(this, 16);
-        _BasicTranscriptionForwarder_charsPerSecond.set(this, __classPrivateFieldGet(this, _BasicTranscriptionForwarder_DEFAULT_CHARS_PER_SECOND, "f"));
-        _BasicTranscriptionForwarder_messageId.set(this, void 0);
-        _BasicTranscriptionForwarder_isRunning.set(this, false);
-        _BasicTranscriptionForwarder_logger.set(this, log());
-        this.currentCharacterIndex = 0;
-        _BasicTranscriptionForwarder_textIsComplete.set(this, false);
-        _BasicTranscriptionForwarder_audioIsComplete.set(this, false);
-        __classPrivateFieldSet(this, _BasicTranscriptionForwarder_room, room, "f");
-        __classPrivateFieldSet(this, _BasicTranscriptionForwarder_participantIdentity, participantIdentity, "f");
-        __classPrivateFieldSet(this, _BasicTranscriptionForwarder_trackSid, trackSid, "f");
-        __classPrivateFieldSet(this, _BasicTranscriptionForwarder_messageId, messageId, "f");
+        this.#room = room;
+        this.#participantIdentity = participantIdentity;
+        this.#trackSid = trackSid;
+        this.#messageId = messageId;
     }
     start() {
-        if (!__classPrivateFieldGet(this, _BasicTranscriptionForwarder_isRunning, "f")) {
-            __classPrivateFieldSet(this, _BasicTranscriptionForwarder_isRunning, true, "f");
-            __classPrivateFieldGet(this, _BasicTranscriptionForwarder_instances, "m", _BasicTranscriptionForwarder_startPublishingLoop).call(this).catch((error) => {
-                __classPrivateFieldGet(this, _BasicTranscriptionForwarder_logger, "f").error('Error in publishing loop:', error);
-                __classPrivateFieldSet(this, _BasicTranscriptionForwarder_isRunning, false, "f");
+        if (!this.#isRunning) {
+            this.#isRunning = true;
+            this.#startPublishingLoop().catch((error) => {
+                this.#logger.error('Error in publishing loop:', error);
+                this.#isRunning = false;
             });
@@ -46,68 +31,72 @@ }
     pushAudio(frame) {
-        __classPrivateFieldSet(this, _BasicTranscriptionForwarder_totalAudioDuration, __classPrivateFieldGet(this, _BasicTranscriptionForwarder_totalAudioDuration, "f") + frame.samplesPerChannel / frame.sampleRate, "f");
+        this.#totalAudioDuration += frame.samplesPerChannel / frame.sampleRate;
     }
     pushText(text) {
-        __classPrivateFieldSet(this, _BasicTranscriptionForwarder_currentText, __classPrivateFieldGet(this, _BasicTranscriptionForwarder_currentText, "f") + text, "f");
+        this.#currentText += text;
     }
+    #textIsComplete = false;
+    #audioIsComplete = false;
     markTextComplete() {
-        __classPrivateFieldSet(this, _BasicTranscriptionForwarder_textIsComplete, true, "f");
-        __classPrivateFieldGet(this, _BasicTranscriptionForwarder_instances, "m", _BasicTranscriptionForwarder_adjustTimingIfBothFinished).call(this);
+        this.#textIsComplete = true;
+        this.#adjustTimingIfBothFinished();
     }
     markAudioComplete() {
-        __classPrivateFieldSet(this, _BasicTranscriptionForwarder_audioIsComplete, true, "f");
-        __classPrivateFieldGet(this, _BasicTranscriptionForwarder_instances, "m", _BasicTranscriptionForwarder_adjustTimingIfBothFinished).call(this);
+        this.#audioIsComplete = true;
+        this.#adjustTimingIfBothFinished();
     }
+    #adjustTimingIfBothFinished() {
+        if (this.#textIsComplete && this.#audioIsComplete) {
+            const actualDuration = this.#totalAudioDuration;
+            if (actualDuration > 0 && this.#currentText.length > 0) {
+                this.#charsPerSecond = this.#currentText.length / actualDuration;
+            }
+        }
+    }
+    #computeSleepInterval() {
+        return Math.min(Math.max(1 / this.#charsPerSecond, 0.0625), 0.5);
+    }
+    async #startPublishingLoop() {
+        this.#isRunning = true;
+        let sleepInterval = this.#computeSleepInterval();
+        let isComplete = false;
+        while (this.#isRunning && !isComplete) {
+            this.#currentPlayoutTime += sleepInterval;
+            this.currentCharacterIndex = Math.floor(this.#currentPlayoutTime * this.#charsPerSecond);
+            isComplete = this.#textIsComplete && this.currentCharacterIndex >= this.#currentText.length;
+            await this.#publishTranscription(false);
+            if (this.#isRunning && !isComplete) {
+                sleepInterval = this.#computeSleepInterval();
+                await new Promise((resolve) => setTimeout(resolve, sleepInterval * 1000));
+            }
+        }
+        if (this.#isRunning) {
+            this.close(false);
+        }
+    }
+    async #publishTranscription(final) {
+        const textToPublish = this.#currentText.slice(0, this.currentCharacterIndex);
+        await this.#room.localParticipant?.publishTranscription({
+            participantIdentity: this.#participantIdentity,
+            trackSid: this.#trackSid,
+            segments: [
+                {
+                    text: textToPublish,
+                    final: final,
+                    id: this.#messageId,
+                    startTime: BigInt(0),
+                    endTime: BigInt(0),
+                    language: '',
+                },
+            ],
+        });
+    }
     async close(interrupt) {
-        __classPrivateFieldSet(this, _BasicTranscriptionForwarder_isRunning, false, "f");
+        this.#isRunning = false;
         // Publish whatever we had as final
         if (!interrupt) {
-            this.currentCharacterIndex = __classPrivateFieldGet(this, _BasicTranscriptionForwarder_currentText, "f").length;
+            this.currentCharacterIndex = this.#currentText.length;
         }
-        await __classPrivateFieldGet(this, _BasicTranscriptionForwarder_instances, "m", _BasicTranscriptionForwarder_publishTranscription).call(this, true);
+        await this.#publishTranscription(true);
     }
 }
-_BasicTranscriptionForwarder_room = new WeakMap(), _BasicTranscriptionForwarder_participantIdentity = new WeakMap(), _BasicTranscriptionForwarder_trackSid = new WeakMap(), _BasicTranscriptionForwarder_currentText = new WeakMap(), _BasicTranscriptionForwarder_totalAudioDuration = new WeakMap(), _BasicTranscriptionForwarder_currentPlayoutTime = new WeakMap(), _BasicTranscriptionForwarder_DEFAULT_CHARS_PER_SECOND = new WeakMap(), _BasicTranscriptionForwarder_charsPerSecond = new WeakMap(), _BasicTranscriptionForwarder_messageId = new WeakMap(), _BasicTranscriptionForwarder_isRunning = new WeakMap(), _BasicTranscriptionForwarder_logger = new WeakMap(), _BasicTranscriptionForwarder_textIsComplete = new WeakMap(), _BasicTranscriptionForwarder_audioIsComplete = new WeakMap(), _BasicTranscriptionForwarder_instances = new WeakSet(), _BasicTranscriptionForwarder_adjustTimingIfBothFinished = function _BasicTranscriptionForwarder_adjustTimingIfBothFinished() {
-    if (__classPrivateFieldGet(this, _BasicTranscriptionForwarder_textIsComplete, "f") && __classPrivateFieldGet(this, _BasicTranscriptionForwarder_audioIsComplete, "f")) {
-        const actualDuration = __classPrivateFieldGet(this, _BasicTranscriptionForwarder_totalAudioDuration, "f");
-        if (actualDuration > 0 && __classPrivateFieldGet(this, _BasicTranscriptionForwarder_currentText, "f").length > 0) {
-            __classPrivateFieldSet(this, _BasicTranscriptionForwarder_charsPerSecond, __classPrivateFieldGet(this, _BasicTranscriptionForwarder_currentText, "f").length / actualDuration, "f");
-        }
-    }
-}, _BasicTranscriptionForwarder_computeSleepInterval = function _BasicTranscriptionForwarder_computeSleepInterval() {
-    return Math.min(Math.max(1 / __classPrivateFieldGet(this, _BasicTranscriptionForwarder_charsPerSecond, "f"), 0.0625), 0.5);
-}, _BasicTranscriptionForwarder_startPublishingLoop = async function _BasicTranscriptionForwarder_startPublishingLoop() {
-    __classPrivateFieldSet(this, _BasicTranscriptionForwarder_isRunning, true, "f");
-    let sleepInterval = __classPrivateFieldGet(this, _BasicTranscriptionForwarder_instances, "m", _BasicTranscriptionForwarder_computeSleepInterval).call(this);
-    let isComplete = false;
-    while (__classPrivateFieldGet(this, _BasicTranscriptionForwarder_isRunning, "f") && !isComplete) {
-        __classPrivateFieldSet(this, _BasicTranscriptionForwarder_currentPlayoutTime, __classPrivateFieldGet(this, _BasicTranscriptionForwarder_currentPlayoutTime, "f") + sleepInterval, "f");
-        this.currentCharacterIndex = Math.floor(__classPrivateFieldGet(this, _BasicTranscriptionForwarder_currentPlayoutTime, "f") * __classPrivateFieldGet(this, _BasicTranscriptionForwarder_charsPerSecond, "f"));
-        isComplete = __classPrivateFieldGet(this, _BasicTranscriptionForwarder_textIsComplete, "f") && this.currentCharacterIndex >= __classPrivateFieldGet(this, _BasicTranscriptionForwarder_currentText, "f").length;
-        await __classPrivateFieldGet(this, _BasicTranscriptionForwarder_instances, "m", _BasicTranscriptionForwarder_publishTranscription).call(this, false);
-        if (__classPrivateFieldGet(this, _BasicTranscriptionForwarder_isRunning, "f") && !isComplete) {
-            sleepInterval = __classPrivateFieldGet(this, _BasicTranscriptionForwarder_instances, "m", _BasicTranscriptionForwarder_computeSleepInterval).call(this);
-            await new Promise((resolve) => setTimeout(resolve, sleepInterval * 1000));
-        }
-    }
-    if (__classPrivateFieldGet(this, _BasicTranscriptionForwarder_isRunning, "f")) {
-        this.close(false);
-    }
-}, _BasicTranscriptionForwarder_publishTranscription = async function _BasicTranscriptionForwarder_publishTranscription(final) {
-    var _a;
-    const textToPublish = __classPrivateFieldGet(this, _BasicTranscriptionForwarder_currentText, "f").slice(0, this.currentCharacterIndex);
-    await ((_a = __classPrivateFieldGet(this, _BasicTranscriptionForwarder_room, "f").localParticipant) === null || _a === void 0 ? void 0 : _a.publishTranscription({
-        participantIdentity: __classPrivateFieldGet(this, _BasicTranscriptionForwarder_participantIdentity, "f"),
-        trackSid: __classPrivateFieldGet(this, _BasicTranscriptionForwarder_trackSid, "f"),
-        segments: [
-            {
-                text: textToPublish,
-                final: final,
-                id: __classPrivateFieldGet(this, _BasicTranscriptionForwarder_messageId, "f"),
-                startTime: BigInt(0),
-                endTime: BigInt(0),
-                language: '',
-            },
-        ],
-    }));
-};
 //# sourceMappingURL=transcription.js.map

dist/tts/index.d.ts

@@ -1,4 +0,2 @@
-import { StreamAdapter, StreamAdapterWrapper } from './stream_adapter.js';
-import { ChunkedStream, SynthesisEvent, SynthesisEventType, SynthesizeStream, type SynthesizedAudio, TTS } from './tts.js';
-export { TTS, SynthesisEvent, SynthesisEventType, SynthesizedAudio, SynthesizeStream, StreamAdapter, StreamAdapterWrapper, ChunkedStream, };
+export { type SynthesizedAudio, type TTSCapabilities, TTS, SynthesizeStream } from './tts.js';
 //# sourceMappingURL=index.d.ts.map

dist/tts/index.js

 // SPDX-FileCopyrightText: 2024 LiveKit, Inc.
 //
 // SPDX-License-Identifier: Apache-2.0
-import { StreamAdapter, StreamAdapterWrapper } from './stream_adapter.js';
-import { ChunkedStream, SynthesisEvent, SynthesisEventType, SynthesizeStream, TTS, } from './tts.js';
-export { TTS, SynthesisEvent, SynthesisEventType, SynthesizeStream, StreamAdapter, StreamAdapterWrapper, ChunkedStream, };
+export { TTS, SynthesizeStream } from './tts.js';
 //# sourceMappingURL=index.js.map

dist/tts/tts.d.ts

 import type { AudioFrame } from '@livekit/rtc-node';
+import { AsyncIterableQueue } from '../utils.js';
+/** SynthesizedAudio is a packet of speech synthesis as returned by the TTS. */
 export interface SynthesizedAudio {
-    text: string;
-    data: AudioFrame;
+    /** Request ID (one segment could be made up of multiple requests) */
+    requestId: string;
+    /** Segment ID, each segment is separated by a flush */
+    segmentId: string;
+    /** Synthesized audio frame */
+    frame: AudioFrame;
+    /** Current segment of the synthesized audio */
+    deltaText?: string;
 }
-export declare enum SynthesisEventType {
-    /**
-     * Indicate the start of synthesis.
-     * Retriggered after FINISHED.
-     */
-    STARTED = 0,
-    /**
-     * Indicate that audio data is available.
-     */
-    AUDIO = 1,
-    /**
-     * Indicate the end of synthesis. Does not necessarily mean stream is done.
-     */
-    FINISHED = 2
+/**
+ * Describes the capabilities of the TTS provider.
+ *
+ * @remarks
+ * At present, only `streaming` is supplied to this interface, and the framework only supports
+ * providers that do have a streaming endpoint.
+ */
+export interface TTSCapabilities {
+    streaming: boolean;
 }
-export declare class SynthesisEvent {
-    type: SynthesisEventType;
-    audio?: SynthesizedAudio;
-    constructor(type: SynthesisEventType, audio?: SynthesizedAudio | undefined);
-}
-export declare abstract class SynthesizeStream implements IterableIterator<SynthesisEvent> {
-    abstract pushText(token?: string): void;
-    markSegmentEnd(): void;
-    abstract close(wait: boolean): Promise<void>;
-    abstract next(): IteratorResult<SynthesisEvent>;
-    [Symbol.iterator](): SynthesizeStream;
-}
+/**
+ * An instance of a text-to-speech adapter.
+ *
+ * @remarks
+ * This class is abstract, and as such cannot be used directly. Instead, use a provider plugin that
+ * exports its own child TTS class, which inherits this class's methods.
+ */
 export declare abstract class TTS {
     #private;
-    constructor(streamingSupported: boolean);
-    abstract synthesize(text: string): Promise<ChunkedStream>;
+    constructor(sampleRate: number, numChannels: number, capabilities: TTSCapabilities);
+    /** Returns this TTS's capabilities */
+    get capabilities(): TTSCapabilities;
+    /** Returns the sample rate of audio frames returned by this TTS */
+    get sampleRate(): number;
+    /** Returns the channel count of audio frames returned by this TTS */
+    get numChannels(): number;
+    /**
+     * Returns a {@link SynthesizeStream} that can be used to push text and receive audio data
+     */
     abstract stream(): SynthesizeStream;
-    get streamingSupported(): boolean;
 }
-export declare abstract class ChunkedStream implements AsyncIterableIterator<SynthesizedAudio> {
-    collect(): Promise<AudioFrame>;
-    abstract close(): Promise<void>;
-    abstract next(): Promise<IteratorResult<SynthesizedAudio>>;
-    [Symbol.iterator](): ChunkedStream;
-    [Symbol.asyncIterator](): ChunkedStream;
+/**
+ * An instance of a text-to-speech stream, as an asynchronous iterable iterator.
+ *
+ * @example Looping through frames
+ * ```ts
+ * for await (const event of stream) {
+ *   await source.captureFrame(event.frame);
+ * }
+ * ```
+ *
+ * @remarks
+ * This class is abstract, and as such cannot be used directly. Instead, use a provider plugin that
+ * exports its own child SynthesizeStream class, which inherits this class's methods.
+ */
+export declare abstract class SynthesizeStream implements AsyncIterableIterator<SynthesizedAudio | typeof SynthesizeStream.END_OF_STREAM> {
+    protected static readonly FLUSH_SENTINEL: unique symbol;
+    static readonly END_OF_STREAM: unique symbol;
+    protected input: AsyncIterableQueue<string | typeof SynthesizeStream.FLUSH_SENTINEL>;
+    protected queue: AsyncIterableQueue<SynthesizedAudio | typeof SynthesizeStream.END_OF_STREAM>;
+    protected closed: boolean;
+    /** Push a string of text to the TTS */
+    pushText(text: string): void;
+    /** Flush the TTS, causing it to process all pending text */
+    flush(): void;
+    /** Mark the input as ended and forbid additional pushes */
+    endInput(): void;
+    next(): Promise<IteratorResult<SynthesizedAudio | typeof SynthesizeStream.END_OF_STREAM>>;
+    /** Close both the input and output of the TTS stream */
+    close(): void;
+    [Symbol.asyncIterator](): SynthesizeStream;
 }
 //# sourceMappingURL=tts.d.ts.map

dist/tts/tts.js

@@ -1,85 +0,90 @@
-var __classPrivateFieldSet = (this && this.__classPrivateFieldSet) || function (receiver, state, value, kind, f) {
-    if (kind === "m") throw new TypeError("Private method is not writable");
-    if (kind === "a" && !f) throw new TypeError("Private accessor was defined without a setter");
-    if (typeof state === "function" ? receiver !== state || !f : !state.has(receiver)) throw new TypeError("Cannot write private member to an object whose class did not declare it");
-    return (kind === "a" ? f.call(receiver, value) : f ? f.value = value : state.set(receiver, value)), value;
-};
-var __classPrivateFieldGet = (this && this.__classPrivateFieldGet) || function (receiver, state, kind, f) {
-    if (kind === "a" && !f) throw new TypeError("Private accessor was defined without a getter");
-    if (typeof state === "function" ? receiver !== state || !f : !state.has(receiver)) throw new TypeError("Cannot read private member from an object whose class did not declare it");
-    return kind === "m" ? f : kind === "a" ? f.call(receiver) : f ? f.value : state.get(receiver);
-};
-var __asyncValues = (this && this.__asyncValues) || function (o) {
-    if (!Symbol.asyncIterator) throw new TypeError("Symbol.asyncIterator is not defined.");
-    var m = o[Symbol.asyncIterator], i;
-    return m ? m.call(o) : (o = typeof __values === "function" ? __values(o) : o[Symbol.iterator](), i = {}, verb("next"), verb("throw"), verb("return"), i[Symbol.asyncIterator] = function () { return this; }, i);
-    function verb(n) { i[n] = o[n] && function (v) { return new Promise(function (resolve, reject) { v = o[n](v), settle(resolve, reject, v.done, v.value); }); }; }
-    function settle(resolve, reject, d, v) { Promise.resolve(v).then(function(v) { resolve({ value: v, done: d }); }, reject); }
-};
-var _TTS_streamingSupported;
-import { mergeFrames } from '../utils.js';
-export var SynthesisEventType;
-(function (SynthesisEventType) {
-    /**
-     * Indicate the start of synthesis.
-     * Retriggered after FINISHED.
-     */
-    SynthesisEventType[SynthesisEventType["STARTED"] = 0] = "STARTED";
-    /**
-     * Indicate that audio data is available.
-     */
-    SynthesisEventType[SynthesisEventType["AUDIO"] = 1] = "AUDIO";
-    /**
-     * Indicate the end of synthesis. Does not necessarily mean stream is done.
-     */
-    SynthesisEventType[SynthesisEventType["FINISHED"] = 2] = "FINISHED";
-})(SynthesisEventType || (SynthesisEventType = {}));
-export class SynthesisEvent {
-    constructor(type, audio = undefined) {
-        this.type = type;
-        this.audio = audio;
+import { AsyncIterableQueue } from '../utils.js';
+/**
+ * An instance of a text-to-speech adapter.
+ *
+ * @remarks
+ * This class is abstract, and as such cannot be used directly. Instead, use a provider plugin that
+ * exports its own child TTS class, which inherits this class's methods.
+ */
+export class TTS {
+    #capabilities;
+    #sampleRate;
+    #numChannels;
+    constructor(sampleRate, numChannels, capabilities) {
+        this.#capabilities = capabilities;
+        this.#sampleRate = sampleRate;
+        this.#numChannels = numChannels;
     }
-}
-export class SynthesizeStream {
-    markSegmentEnd() {
-        this.pushText(undefined);
+    /** Returns this TTS's capabilities */
+    get capabilities() {
+        return this.#capabilities;
     }
-    [Symbol.iterator]() {
-        return this;
+    /** Returns the sample rate of audio frames returned by this TTS */
+    get sampleRate() {
+        return this.#sampleRate;
     }
+    /** Returns the channel count of audio frames returned by this TTS */
+    get numChannels() {
+        return this.#numChannels;
+    }
 }
-export class TTS {
-    constructor(streamingSupported) {
-        _TTS_streamingSupported.set(this, void 0);
-        __classPrivateFieldSet(this, _TTS_streamingSupported, streamingSupported, "f");
+/**
+ * An instance of a text-to-speech stream, as an asynchronous iterable iterator.
+ *
+ * @example Looping through frames
+ * ```ts
+ * for await (const event of stream) {
+ *   await source.captureFrame(event.frame);
+ * }
+ * ```
+ *
+ * @remarks
+ * This class is abstract, and as such cannot be used directly. Instead, use a provider plugin that
+ * exports its own child SynthesizeStream class, which inherits this class's methods.
+ */
+export class SynthesizeStream {
+    static FLUSH_SENTINEL = Symbol('FLUSH_SENTINEL');
+    static END_OF_STREAM = Symbol('END_OF_STREAM');
+    input = new AsyncIterableQueue();
+    queue = new AsyncIterableQueue();
+    closed = false;
+    /** Push a string of text to the TTS */
+    pushText(text) {
+        if (this.input.closed) {
+            throw new Error('Input is closed');
+        }
+        if (this.closed) {
+            throw new Error('Stream is closed');
+        }
+        this.input.put(text);
     }
-    get streamingSupported() {
-        return __classPrivateFieldGet(this, _TTS_streamingSupported, "f");
+    /** Flush the TTS, causing it to process all pending text */
+    flush() {
+        if (this.input.closed) {
+            throw new Error('Input is closed');
+        }
+        if (this.closed) {
+            throw new Error('Stream is closed');
+        }
+        this.input.put(SynthesizeStream.FLUSH_SENTINEL);
     }
-}
-_TTS_streamingSupported = new WeakMap();
-export class ChunkedStream {
-    async collect() {
-        var _a, e_1, _b, _c;
-        const frames = [];
-        try {
-            for (var _d = true, _e = __asyncValues(this), _f; _f = await _e.next(), _a = _f.done, !_a; _d = true) {
-                _c = _f.value;
-                _d = false;
-                const ev = _c;
-                frames.push(ev.data);
-            }
+    /** Mark the input as ended and forbid additional pushes */
+    endInput() {
+        if (this.input.closed) {
+            throw new Error('Input is closed');
         }
-        catch (e_1_1) { e_1 = { error: e_1_1 }; }
-        finally {
-            try {
-                if (!_d && !_a && (_b = _e.return)) await _b.call(_e);
-            }
-            finally { if (e_1) throw e_1.error; }
+        if (this.closed) {
+            throw new Error('Stream is closed');
         }
-        return mergeFrames(frames);
+        this.input.close();
     }
-    [Symbol.iterator]() {
-        return this;
+    next() {
+        return this.queue.next();
     }
+    /** Close both the input and output of the TTS stream */
+    close() {
+        this.input.close();
+        this.queue.close();
+        this.closed = true;
+    }
     [Symbol.asyncIterator]() {
@@ -86,0 +91,0 @@ return this;

dist/utils.d.ts

@@ -48,11 +48,26 @@ import type { Room } from '@livekit/rtc-node';
 /** @internal */
-export declare class AsyncIterableQueue<T> implements AsyncIterable<T> {
-    private queue;
-    private closed;
-    private static readonly QUEUE_END_MARKER;
-    constructor();
+export declare class AsyncIterableQueue<T> implements AsyncIterableIterator<T> {
+    #private;
+    private static readonly CLOSE_SENTINEL;
+    get closed(): boolean;
     put(item: T): void;
     close(): void;
-    [Symbol.asyncIterator](): AsyncIterator<T>;
+    next(): Promise<IteratorResult<T>>;
+    [Symbol.asyncIterator](): AsyncIterableQueue<T>;
 }
+/** @internal */
+export declare class ExpFilter {
+    #private;
+    constructor(alpha: number, max?: number);
+    reset(alpha?: number): void;
+    apply(exp: number, sample: number): number;
+    get filtered(): number | undefined;
+    set alpha(alpha: number);
+}
+/** @internal */
+export declare class AudioEnergyFilter {
+    #private;
+    constructor(cooldownSeconds?: number);
+    pushFrame(frame: AudioFrame): boolean;
+}
 //# sourceMappingURL=utils.d.ts.map

dist/utils.js

@@ -1,13 +0,2 @@
-var __classPrivateFieldSet = (this && this.__classPrivateFieldSet) || function (receiver, state, value, kind, f) {
-    if (kind === "m") throw new TypeError("Private method is not writable");
-    if (kind === "a" && !f) throw new TypeError("Private accessor was defined without a setter");
-    if (typeof state === "function" ? receiver !== state || !f : !state.has(receiver)) throw new TypeError("Cannot write private member to an object whose class did not declare it");
-    return (kind === "a" ? f.call(receiver, value) : f ? f.value = value : state.set(receiver, value)), value;
-};
-var __classPrivateFieldGet = (this && this.__classPrivateFieldGet) || function (receiver, state, kind, f) {
-    if (kind === "a" && !f) throw new TypeError("Private accessor was defined without a getter");
-    if (typeof state === "function" ? receiver !== state || !f : !state.has(receiver)) throw new TypeError("Cannot read private member from an object whose class did not declare it");
-    return kind === "m" ? f : kind === "a" ? f.call(receiver) : f ? f.value : state.get(receiver);
-};
-var _Queue_limit, _Queue_events, _Future_await, _Future_resolvePromise, _Future_rejectPromise, _Future_done, _CancellablePromise_promise, _CancellablePromise_cancelFn, _CancellablePromise_isCancelled, _CancellablePromise_error;
+var _a;
 import { AudioFrame, TrackSource } from '@livekit/rtc-node';
@@ -48,5 +37,4 @@ import { EventEmitter, once } from 'node:events';
 export const findMicroTrackId = (room, identity) => {
-    var _a;
     let p = room.remoteParticipants.get(identity);
-    if (identity === ((_a = room.localParticipant) === null || _a === void 0 ? void 0 : _a.identity)) {
+    if (identity === room.localParticipant?.identity) {
         p = room.localParticipant;
@@ -72,92 +60,90 @@ }
 export class Queue {
+    /** @internal */
+    items = [];
+    #limit;
+    #events = new EventEmitter();
     constructor(limit) {
-        /** @internal */
-        this.items = [];
-        _Queue_limit.set(this, void 0);
-        _Queue_events.set(this, new EventEmitter());
-        __classPrivateFieldSet(this, _Queue_limit, limit, "f");
+        this.#limit = limit;
     }
     async get() {
         if (this.items.length === 0) {
-            await once(__classPrivateFieldGet(this, _Queue_events, "f"), 'put');
+            await once(this.#events, 'put');
         }
         const item = this.items.shift();
-        __classPrivateFieldGet(this, _Queue_events, "f").emit('get');
+        this.#events.emit('get');
         return item;
     }
     async put(item) {
-        if (__classPrivateFieldGet(this, _Queue_limit, "f") && this.items.length >= __classPrivateFieldGet(this, _Queue_limit, "f")) {
-            await once(__classPrivateFieldGet(this, _Queue_events, "f"), 'get');
+        if (this.#limit && this.items.length >= this.#limit) {
+            await once(this.#events, 'get');
         }
         this.items.push(item);
-        __classPrivateFieldGet(this, _Queue_events, "f").emit('put');
+        this.#events.emit('put');
     }
 }
-_Queue_limit = new WeakMap(), _Queue_events = new WeakMap();
 /** @internal */
 export class Future {
+    #await;
+    #resolvePromise;
+    #rejectPromise;
+    #done = false;
     constructor() {
-        _Future_await.set(this, void 0);
-        _Future_resolvePromise.set(this, void 0);
-        _Future_rejectPromise.set(this, void 0);
-        _Future_done.set(this, false);
-        __classPrivateFieldSet(this, _Future_await, new Promise((resolve, reject) => {
-            __classPrivateFieldSet(this, _Future_resolvePromise, resolve, "f");
-            __classPrivateFieldSet(this, _Future_rejectPromise, reject, "f");
-        }), "f");
+        this.#await = new Promise((resolve, reject) => {
+            this.#resolvePromise = resolve;
+            this.#rejectPromise = reject;
+        });
     }
     get await() {
-        return __classPrivateFieldGet(this, _Future_await, "f");
+        return this.#await;
     }
     get done() {
-        return __classPrivateFieldGet(this, _Future_done, "f");
+        return this.#done;
     }
     resolve() {
-        __classPrivateFieldSet(this, _Future_done, true, "f");
-        __classPrivateFieldGet(this, _Future_resolvePromise, "f").call(this);
+        this.#done = true;
+        this.#resolvePromise();
     }
     reject(error) {
-        __classPrivateFieldSet(this, _Future_done, true, "f");
-        __classPrivateFieldGet(this, _Future_rejectPromise, "f").call(this, error);
+        this.#done = true;
+        this.#rejectPromise(error);
     }
 }
-_Future_await = new WeakMap(), _Future_resolvePromise = new WeakMap(), _Future_rejectPromise = new WeakMap(), _Future_done = new WeakMap();
 /** @internal */
 export class CancellablePromise {
+    #promise;
+    #cancelFn;
+    #isCancelled = false;
+    #error = null;
     constructor(executor) {
-        _CancellablePromise_promise.set(this, void 0);
-        _CancellablePromise_cancelFn.set(this, void 0);
-        _CancellablePromise_isCancelled.set(this, false);
-        _CancellablePromise_error.set(this, null);
         let cancel;
-        __classPrivateFieldSet(this, _CancellablePromise_promise, new Promise((resolve, reject) => {
+        this.#promise = new Promise((resolve, reject) => {
             executor(resolve, (reason) => {
-                __classPrivateFieldSet(this, _CancellablePromise_error, reason instanceof Error ? reason : new Error(String(reason)), "f");
+                this.#error = reason instanceof Error ? reason : new Error(String(reason));
                 reject(reason);
             }, (cancelFn) => {
                 cancel = () => {
-                    __classPrivateFieldSet(this, _CancellablePromise_isCancelled, true, "f");
+                    this.#isCancelled = true;
                     cancelFn();
                 };
             });
-        }), "f");
-        __classPrivateFieldSet(this, _CancellablePromise_cancelFn, cancel, "f");
+        });
+        this.#cancelFn = cancel;
     }
     get isCancelled() {
-        return __classPrivateFieldGet(this, _CancellablePromise_isCancelled, "f");
+        return this.#isCancelled;
     }
     get error() {
-        return __classPrivateFieldGet(this, _CancellablePromise_error, "f");
+        return this.#error;
     }
     then(onfulfilled, onrejected) {
-        return __classPrivateFieldGet(this, _CancellablePromise_promise, "f").then(onfulfilled, onrejected);
+        return this.#promise.then(onfulfilled, onrejected);
     }
     catch(onrejected) {
-        return __classPrivateFieldGet(this, _CancellablePromise_promise, "f").catch(onrejected);
+        return this.#promise.catch(onrejected);
     }
     finally(onfinally) {
-        return __classPrivateFieldGet(this, _CancellablePromise_promise, "f").finally(onfinally);
+        return this.#promise.finally(onfinally);
     }
     cancel() {
-        __classPrivateFieldGet(this, _CancellablePromise_cancelFn, "f").call(this);
+        this.#cancelFn();
     }
@@ -170,3 +156,2 @@ static from(promise) {
 }
-_CancellablePromise_promise = new WeakMap(), _CancellablePromise_cancelFn = new WeakMap(), _CancellablePromise_isCancelled = new WeakMap(), _CancellablePromise_error = new WeakMap();
 /** @internal */
@@ -186,32 +171,91 @@ export async function gracefullyCancel(promise) {
 export class AsyncIterableQueue {
-    constructor() {
-        this.closed = false;
-        this.queue = new Queue();
+    static CLOSE_SENTINEL = Symbol('CLOSE_SENTINEL');
+    #queue = new Queue();
+    #closed = false;
+    get closed() {
+        return this.#closed;
     }
     put(item) {
-        if (this.closed) {
+        if (this.#closed) {
             throw new Error('Queue is closed');
         }
-        this.queue.put(item);
+        this.#queue.put(item);
     }
     close() {
-        this.closed = true;
-        this.queue.put(AsyncIterableQueue.QUEUE_END_MARKER);
+        this.#closed = true;
+        this.#queue.put(_a.CLOSE_SENTINEL);
     }
+    async next() {
+        if (this.#closed && this.#queue.items.length === 0) {
+            return { value: undefined, done: true };
+        }
+        const item = await this.#queue.get();
+        if (item === _a.CLOSE_SENTINEL && this.#closed) {
+            return { value: undefined, done: true };
+        }
+        return { value: item, done: false };
+    }
     [Symbol.asyncIterator]() {
-        return {
-            next: async () => {
-                if (this.closed && this.queue.items.length === 0) {
-                    return { value: undefined, done: true };
-                }
-                const item = await this.queue.get();
-                if (item === AsyncIterableQueue.QUEUE_END_MARKER && this.closed) {
-                    return { value: undefined, done: true };
-                }
-                return { value: item, done: false };
-            },
-        };
+        return this;
     }
 }
-AsyncIterableQueue.QUEUE_END_MARKER = Symbol('QUEUE_END_MARKER');
+_a = AsyncIterableQueue;
+/** @internal */
+export class ExpFilter {
+    #alpha;
+    #max;
+    #filtered = undefined;
+    constructor(alpha, max) {
+        this.#alpha = alpha;
+        this.#max = max;
+    }
+    reset(alpha) {
+        if (alpha) {
+            this.#alpha = alpha;
+        }
+        this.#filtered = undefined;
+    }
+    apply(exp, sample) {
+        if (this.#filtered) {
+            const a = this.#alpha ** exp;
+            this.#filtered = a * this.#filtered + (1 - a) * sample;
+        }
+        else {
+            this.#filtered = sample;
+        }
+        if (this.#max && this.#filtered > this.#max) {
+            this.#filtered = this.#max;
+        }
+        return this.#filtered;
+    }
+    get filtered() {
+        return this.#filtered;
+    }
+    set alpha(alpha) {
+        this.#alpha = alpha;
+    }
+}
+/** @internal */
+export class AudioEnergyFilter {
+    #cooldownSeconds;
+    #cooldown;
+    constructor(cooldownSeconds = 1) {
+        this.#cooldownSeconds = cooldownSeconds;
+        this.#cooldown = cooldownSeconds;
+    }
+    pushFrame(frame) {
+        const arr = Float32Array.from(frame.data, (x) => x / 32768);
+        const rms = (arr.map((x) => x ** 2).reduce((acc, x) => acc + x) / arr.length) ** 0.5;
+        if (rms > 0.004) {
+            this.#cooldown = this.#cooldownSeconds;
+            return true;
+        }
+        const durationSeconds = frame.samplesPerChannel / frame.sampleRate;
+        this.#cooldown -= durationSeconds;
+        if (this.#cooldown > 0) {
+            return true;
+        }
+        return false;
+    }
+}
 //# sourceMappingURL=utils.js.map

dist/vad.d.ts

 import type { AudioFrame } from '@livekit/rtc-node';
+import { AsyncIterableQueue } from './utils.js';
 export declare enum VADEventType {
-    START_OF_SPEECH = 1,
-    SPEAKING = 2,
-    END_OF_SPEECH = 3
+    START_OF_SPEECH = 0,
+    INFERENCE_DONE = 1,
+    END_OF_SPEECH = 2
 }
 export interface VADEvent {
+    /** Type of the VAD event (e.g., start of speech, end of speech, inference done). */
     type: VADEventType;
     /**
-     * Index of the samples of the event (when the event was fired)
+     * Index of the audio sample where the event occurred, relative to the inference sample rate.
      */
     samplesIndex: number;
+    /** Timestamp when the event was fired. */
+    timestamp: number;
+    /** Duration of the detected speech segment in seconds. */
+    speechDuration: number;
+    /** Duration of the silence segment preceding or following the speech, in seconds. */
+    silenceDuration: number;
     /**
-     * Duration of speech, in seconds
+     * List of audio frames associated with the speech.
+     *
+     * @remarks
+     * - For `start_of_speech` events, this contains the audio chunks that triggered the detection.
+     * - For `inference_done` events, this contains the audio chunks that were processed.
+     * - For `end_of_speech` events, this contains the complete user speech.
      */
-    duration: number;
-    speech: AudioFrame[];
+    frames: AudioFrame[];
+    /** Probability that speech is present (only for `INFERENCE_DONE` events). */
+    probability: number;
+    /** Time taken to perform the inference, in seconds (only for `INFERENCE_DONE` events). */
+    inferenceDuration: number;
+    /** Indicates whether speech was detected in the frames. */
+    speaking: boolean;
 }
+export interface VADCapabilities {
+    updateInterval: number;
+}
 export declare abstract class VAD {
+    #private;
+    constructor(capabilities: VADCapabilities);
+    get capabilities(): VADCapabilities;
     /**
      * Returns a {@link VADStream} that can be used to push audio frames and receive VAD events.
-     *
-     * @param options
      */
-    abstract stream({ minSpeakingDuration, minSilenceDuration, paddingDuration, sampleRate, maxBufferedSpeech, }: {
-        /**
-         * Minimum duration of speech required to trigger a {@link VADEventType.START_OF_SPEECH} event
-         */
-        minSpeakingDuration: number;
-        /**
-         * Milliseconds to wait before separating speech chunk.
-         * Not always precise, generally rounded to the nearest 40ms depending on VAD implementation
-         */
-        minSilenceDuration: number;
-        /**
-         * Number of frames to pad the start and end of speech with
-         */
-        paddingDuration: number;
-        /**
-         * Sample rate of inference/processing
-         */
-        sampleRate: number;
-        /**
-         * Number of seconds the buffer may keep until {@link VADEventType.END_OF_SPEECH} is triggered.
-         * It is recommended to set this to a positive value, as zero may OOM if the user doesn't stop
-         * speaking.
-         */
-        maxBufferedSpeech: number;
-    }): VADStream;
+    abstract stream(): VADStream;
 }
-export declare abstract class VADStream implements IterableIterator<VADEvent> {
-    abstract pushFrame(frame: AudioFrame): void;
-    abstract close(wait: boolean): Promise<void>;
-    abstract next(): IteratorResult<VADEvent>;
-    [Symbol.iterator](): VADStream;
+export declare abstract class VADStream implements AsyncIterableIterator<VADEvent> {
+    protected static readonly FLUSH_SENTINEL: unique symbol;
+    protected input: AsyncIterableQueue<AudioFrame | typeof VADStream.FLUSH_SENTINEL>;
+    protected queue: AsyncIterableQueue<VADEvent>;
+    protected closed: boolean;
+    pushFrame(frame: AudioFrame): void;
+    flush(): void;
+    endInput(): void;
+    next(): Promise<IteratorResult<VADEvent>>;
+    close(): void;
+    [Symbol.asyncIterator](): VADStream;
 }
 //# sourceMappingURL=vad.d.ts.map

dist/vad.js

@@ -0,11 +1,58 @@
+import { AsyncIterableQueue } from './utils.js';
 export var VADEventType;
 (function (VADEventType) {
-    VADEventType[VADEventType["START_OF_SPEECH"] = 1] = "START_OF_SPEECH";
-    VADEventType[VADEventType["SPEAKING"] = 2] = "SPEAKING";
-    VADEventType[VADEventType["END_OF_SPEECH"] = 3] = "END_OF_SPEECH";
+    VADEventType[VADEventType["START_OF_SPEECH"] = 0] = "START_OF_SPEECH";
+    VADEventType[VADEventType["INFERENCE_DONE"] = 1] = "INFERENCE_DONE";
+    VADEventType[VADEventType["END_OF_SPEECH"] = 2] = "END_OF_SPEECH";
 })(VADEventType || (VADEventType = {}));
 export class VAD {
+    #capabilities;
+    constructor(capabilities) {
+        this.#capabilities = capabilities;
+    }
+    get capabilities() {
+        return this.#capabilities;
+    }
 }
 export class VADStream {
-    [Symbol.iterator]() {
+    static FLUSH_SENTINEL = Symbol('FLUSH_SENTINEL');
+    input = new AsyncIterableQueue();
+    queue = new AsyncIterableQueue();
+    closed = false;
+    pushFrame(frame) {
+        if (this.input.closed) {
+            throw new Error('Input is closed');
+        }
+        if (this.closed) {
+            throw new Error('Stream is closed');
+        }
+        this.input.put(frame);
+    }
+    flush() {
+        if (this.input.closed) {
+            throw new Error('Input is closed');
+        }
+        if (this.closed) {
+            throw new Error('Stream is closed');
+        }
+        this.input.put(VADStream.FLUSH_SENTINEL);
+    }
+    endInput() {
+        if (this.input.closed) {
+            throw new Error('Input is closed');
+        }
+        if (this.closed) {
+            throw new Error('Stream is closed');
+        }
+        this.input.close();
+    }
+    next() {
+        return this.queue.next();
+    }
+    close() {
+        this.input.close();
+        this.queue.close();
+        this.closed = true;
+    }
+    [Symbol.asyncIterator]() {
         return this;
@@ -12,0 +59,0 @@ }

dist/worker.d.ts

@@ -1,2 +0,2 @@
-/// <reference types="node" resolution-mode="require"/>
+/// <reference types="node" />
 import type { TrackSource } from '@livekit/protocol';
@@ -3,0 +3,0 @@ import { JobType } from '@livekit/protocol';

dist/worker.js

@@ -1,13 +0,1 @@
-var __classPrivateFieldSet = (this && this.__classPrivateFieldSet) || function (receiver, state, value, kind, f) {
-    if (kind === "m") throw new TypeError("Private method is not writable");
-    if (kind === "a" && !f) throw new TypeError("Private accessor was defined without a setter");
-    if (typeof state === "function" ? receiver !== state || !f : !state.has(receiver)) throw new TypeError("Cannot write private member to an object whose class did not declare it");
-    return (kind === "a" ? f.call(receiver, value) : f ? f.value = value : state.set(receiver, value)), value;
-};
-var __classPrivateFieldGet = (this && this.__classPrivateFieldGet) || function (receiver, state, kind, f) {
-    if (kind === "a" && !f) throw new TypeError("Private accessor was defined without a getter");
-    if (typeof state === "function" ? receiver !== state || !f : !state.has(receiver)) throw new TypeError("Cannot read private member from an object whose class did not declare it");
-    return kind === "m" ? f : kind === "a" ? f.call(receiver) : f ? f.value : state.get(receiver);
-};
-var _Worker_instances, _Worker_opts, _Worker_procPool, _Worker_id, _Worker_closed, _Worker_draining, _Worker_connecting, _Worker_tasks, _Worker_pending, _Worker_close, _Worker_session, _Worker_httpServer, _Worker_logger, _Worker_runWS, _Worker_availability, _Worker_termination;
 import { JobType, ParticipantPermission, ServerMessage, WorkerMessage, WorkerStatus, } from '@livekit/protocol';
@@ -93,2 +81,8 @@ import { AccessToken, RoomServiceClient } from 'livekit-server-sdk';
 export class WorkerPermissions {
+    canPublish;
+    canSubscribe;
+    canPublishData;
+    canUpdateMetadata;
+    canPublishSources;
+    hidden;
     constructor(canPublish = true, canSubscribe = true, canPublishData = true, canUpdateMetadata = true, canPublishSources = [], hidden = false) {
@@ -113,2 +107,20 @@ this.canPublish = canPublish;
 export class WorkerOptions {
+    agent;
+    requestFunc;
+    loadFunc;
+    loadThreshold;
+    numIdleProcesses;
+    shutdownProcessTimeout;
+    initializeProcessTimeout;
+    permissions;
+    agentName;
+    workerType;
+    maxRetry;
+    wsURL;
+    apiKey;
+    apiSecret;
+    host;
+    port;
+    logLevel;
+    production;
     /** @param options */
@@ -140,7 +152,5 @@ constructor({ agent, requestFunc = defaultRequestFunc, loadFunc = defaultCpuLoad, loadThreshold = undefined, numIdleProcesses = undefined, shutdownProcessTimeout = 60 * 1000, initializeProcessTimeout = 10 * 1000, permissions = new WorkerPermissions(), agentName = '', workerType = JobType.JT_ROOM, maxRetry = MAX_RECONNECT_ATTEMPTS, wsURL = 'ws://localhost:7880', apiKey = undefined, apiSecret = undefined, host = 'localhost', port = undefined, logLevel = 'info', production = false, }) {
 class PendingAssignment {
-    constructor() {
-        this.promise = new Promise((resolve) => {
-            this.resolve = resolve; // this is how JavaScript lets you resolve promises externally
-        });
-    }
+    promise = new Promise((resolve) => {
+        this.resolve = resolve; // this is how JavaScript lets you resolve promises externally
+    });
     resolve(arg) {
@@ -160,18 +170,17 @@ arg; // useless call to counteract TypeScript E6133
 export class Worker {
+    #opts;
+    #procPool;
+    #id = 'unregistered';
+    #closed = true;
+    #draining = false;
+    #connecting = false;
+    #tasks = [];
+    #pending = {};
+    #close = new Future();
+    event = new EventEmitter();
+    #session = undefined;
+    #httpServer;
+    #logger = log().child({ version });
     /* @throws {@link MissingCredentialsError} if URL, API key or API secret are missing */
     constructor(opts) {
-        _Worker_instances.add(this);
-        _Worker_opts.set(this, void 0);
-        _Worker_procPool.set(this, void 0);
-        _Worker_id.set(this, 'unregistered');
-        _Worker_closed.set(this, true);
-        _Worker_draining.set(this, false);
-        _Worker_connecting.set(this, false);
-        _Worker_tasks.set(this, []);
-        _Worker_pending.set(this, {});
-        _Worker_close.set(this, new Future());
-        this.event = new EventEmitter();
-        _Worker_session.set(this, undefined);
-        _Worker_httpServer.set(this, void 0);
-        _Worker_logger.set(this, log().child({ version }));
         opts.wsURL = opts.wsURL || process.env.LIVEKIT_URL || '';
@@ -186,41 +195,41 @@ opts.apiKey = opts.apiKey || process.env.LIVEKIT_API_KEY || '';
             throw new MissingCredentialsError('API Secret is required: Set LIVEKIT_API_SECRET, run with --api-secret, or pass apiSecret in WorkerOptions');
-        __classPrivateFieldSet(this, _Worker_procPool, new ProcPool(opts.agent, opts.numIdleProcesses, opts.initializeProcessTimeout, opts.shutdownProcessTimeout), "f");
-        __classPrivateFieldSet(this, _Worker_opts, opts, "f");
-        __classPrivateFieldSet(this, _Worker_httpServer, new HTTPServer(opts.host, opts.port), "f");
+        this.#procPool = new ProcPool(opts.agent, opts.numIdleProcesses, opts.initializeProcessTimeout, opts.shutdownProcessTimeout);
+        this.#opts = opts;
+        this.#httpServer = new HTTPServer(opts.host, opts.port);
     }
     /* @throws {@link WorkerError} if worker failed to connect or already running */
     async run() {
-        if (!__classPrivateFieldGet(this, _Worker_closed, "f")) {
+        if (!this.#closed) {
             throw new WorkerError('worker is already running');
         }
-        __classPrivateFieldGet(this, _Worker_logger, "f").info('starting worker');
-        __classPrivateFieldSet(this, _Worker_closed, false, "f");
-        __classPrivateFieldGet(this, _Worker_procPool, "f").start();
+        this.#logger.info('starting worker');
+        this.#closed = false;
+        this.#procPool.start();
         const workerWS = async () => {
             let retries = 0;
-            __classPrivateFieldSet(this, _Worker_connecting, true, "f");
-            while (!__classPrivateFieldGet(this, _Worker_closed, "f")) {
-                const url = new URL(__classPrivateFieldGet(this, _Worker_opts, "f").wsURL);
+            this.#connecting = true;
+            while (!this.#closed) {
+                const url = new URL(this.#opts.wsURL);
                 url.protocol = url.protocol.replace('http', 'ws');
-                const token = new AccessToken(__classPrivateFieldGet(this, _Worker_opts, "f").apiKey, __classPrivateFieldGet(this, _Worker_opts, "f").apiSecret);
+                const token = new AccessToken(this.#opts.apiKey, this.#opts.apiSecret);
                 token.addGrant({ agent: true });
                 const jwt = await token.toJwt();
-                __classPrivateFieldSet(this, _Worker_session, new WebSocket(url + 'agent', {
+                this.#session = new WebSocket(url + 'agent', {
                     headers: { authorization: 'Bearer ' + jwt },
-                }), "f");
+                });
                 try {
                     await new Promise((resolve, reject) => {
-                        __classPrivateFieldGet(this, _Worker_session, "f").on('open', resolve);
-                        __classPrivateFieldGet(this, _Worker_session, "f").on('error', (error) => reject(error));
-                        __classPrivateFieldGet(this, _Worker_session, "f").on('close', (code) => reject(new Error(`WebSocket returned ${code}`)));
+                        this.#session.on('open', resolve);
+                        this.#session.on('error', (error) => reject(error));
+                        this.#session.on('close', (code) => reject(new Error(`WebSocket returned ${code}`)));
                     });
                     retries = 0;
-                    __classPrivateFieldGet(this, _Worker_logger, "f").debug('connected to LiveKit server');
-                    __classPrivateFieldGet(this, _Worker_instances, "m", _Worker_runWS).call(this, __classPrivateFieldGet(this, _Worker_session, "f"));
+                    this.#logger.debug('connected to LiveKit server');
+                    this.#runWS(this.#session);
                     return;
                 }
                 catch (e) {
-                    if (__classPrivateFieldGet(this, _Worker_closed, "f"))
+                    if (this.#closed)
                         return;
-                    if (retries >= __classPrivateFieldGet(this, _Worker_opts, "f").maxRetry) {
+                    if (retries >= this.#opts.maxRetry) {
                         throw new WorkerError(`failed to connect to LiveKit server after ${retries} attempts: ${e}`);
@@ -230,3 +239,3 @@ }
                     const delay = Math.min(retries * 2, 10);
-                    __classPrivateFieldGet(this, _Worker_logger, "f").warn(`failed to connect to LiveKit server, retrying in ${delay} seconds: ${e} (${retries}/${__classPrivateFieldGet(this, _Worker_opts, "f").maxRetry})`);
+                    this.#logger.warn(`failed to connect to LiveKit server, retrying in ${delay} seconds: ${e} (${retries}/${this.#opts.maxRetry})`);
                     await new Promise((resolve) => setTimeout(resolve, delay * 1000));
@@ -236,10 +245,10 @@ }
         };
-        await Promise.all([workerWS(), __classPrivateFieldGet(this, _Worker_httpServer, "f").run()]);
-        __classPrivateFieldGet(this, _Worker_close, "f").resolve();
+        await Promise.all([workerWS(), this.#httpServer.run()]);
+        this.#close.resolve();
     }
     get id() {
-        return __classPrivateFieldGet(this, _Worker_id, "f");
+        return this.#id;
     }
     get activeJobs() {
-        return __classPrivateFieldGet(this, _Worker_procPool, "f").processes
+        return this.#procPool.processes
             .filter((proc) => proc.runningJob)
@@ -250,7 +259,7 @@ .map((proc) => proc.runningJob);
     async drain(timeout) {
-        if (__classPrivateFieldGet(this, _Worker_draining, "f")) {
+        if (this.#draining) {
             return;
         }
-        __classPrivateFieldGet(this, _Worker_logger, "f").info('draining worker');
-        __classPrivateFieldSet(this, _Worker_draining, true, "f");
+        this.#logger.info('draining worker');
+        this.#draining = true;
         this.event.emit('worker_msg', new WorkerMessage({
@@ -265,3 +274,3 @@ message: {
         const joinJobs = async () => {
-            return Promise.all(__classPrivateFieldGet(this, _Worker_procPool, "f").processes.map((proc) => {
+            return Promise.all(this.#procPool.processes.map((proc) => {
                 if (!proc.runningJob) {
@@ -283,3 +292,3 @@ proc.close();
     async simulateJob(roomName, participantIdentity) {
-        const client = new RoomServiceClient(__classPrivateFieldGet(this, _Worker_opts, "f").wsURL, __classPrivateFieldGet(this, _Worker_opts, "f").apiKey, __classPrivateFieldGet(this, _Worker_opts, "f").apiSecret);
+        const client = new RoomServiceClient(this.#opts.wsURL, this.#opts.apiKey, this.#opts.apiSecret);
         const room = await client.createRoom({ name: roomName });
@@ -292,3 +301,3 @@ let participant = undefined;
             catch (e) {
-                __classPrivateFieldGet(this, _Worker_logger, "f").fatal(`participant with identity ${participantIdentity} not found in room ${roomName}`);
+                this.#logger.fatal(`participant with identity ${participantIdentity} not found in room ${roomName}`);
                 throw e;
@@ -308,206 +317,207 @@ }
     }
-    async close() {
-        var _a;
-        if (__classPrivateFieldGet(this, _Worker_closed, "f")) {
-            await __classPrivateFieldGet(this, _Worker_close, "f").await;
-            return;
-        }
-        __classPrivateFieldGet(this, _Worker_logger, "f").info('shutting down worker');
-        __classPrivateFieldSet(this, _Worker_closed, true, "f");
-        await __classPrivateFieldGet(this, _Worker_procPool, "f").close();
-        await __classPrivateFieldGet(this, _Worker_httpServer, "f").close();
-        await Promise.allSettled(__classPrivateFieldGet(this, _Worker_tasks, "f"));
-        (_a = __classPrivateFieldGet(this, _Worker_session, "f")) === null || _a === void 0 ? void 0 : _a.close();
-        await __classPrivateFieldGet(this, _Worker_close, "f").await;
-    }
-}
-_Worker_opts = new WeakMap(), _Worker_procPool = new WeakMap(), _Worker_id = new WeakMap(), _Worker_closed = new WeakMap(), _Worker_draining = new WeakMap(), _Worker_connecting = new WeakMap(), _Worker_tasks = new WeakMap(), _Worker_pending = new WeakMap(), _Worker_close = new WeakMap(), _Worker_session = new WeakMap(), _Worker_httpServer = new WeakMap(), _Worker_logger = new WeakMap(), _Worker_instances = new WeakSet(), _Worker_runWS = function _Worker_runWS(ws) {
-    let closingWS = false;
-    const send = (msg) => {
-        if (closingWS) {
-            this.event.off('worker_msg', send);
-            return;
-        }
-        ws.send(msg.toBinary());
-    };
-    this.event.on('worker_msg', send);
-    ws.addEventListener('close', () => {
-        closingWS = true;
-        __classPrivateFieldGet(this, _Worker_logger, "f").error('worker connection closed unexpectedly');
-        this.close();
-    });
-    ws.addEventListener('message', (event) => {
-        if (event.type !== 'message') {
-            __classPrivateFieldGet(this, _Worker_logger, "f").warn('unexpected message type: ' + event.type);
-            return;
-        }
-        const msg = new ServerMessage();
-        msg.fromBinary(event.data);
-        // register is the only valid first message, and it is only valid as the
-        // first message
-        if (__classPrivateFieldGet(this, _Worker_connecting, "f") && msg.message.case !== 'register') {
-            throw new WorkerError('expected register response as first message');
-        }
-        switch (msg.message.case) {
-            case 'register': {
-                __classPrivateFieldSet(this, _Worker_id, msg.message.value.workerId, "f");
-                __classPrivateFieldGet(this, _Worker_logger, "f")
-                    .child({ id: this.id, server_info: msg.message.value.serverInfo })
-                    .info('registered worker');
-                this.event.emit('worker_registered', msg.message.value.workerId, msg.message.value.serverInfo);
-                __classPrivateFieldSet(this, _Worker_connecting, false, "f");
-                break;
+    #runWS(ws) {
+        let closingWS = false;
+        const send = (msg) => {
+            if (closingWS) {
+                this.event.off('worker_msg', send);
+                return;
             }
-            case 'availability': {
-                if (!msg.message.value.job)
-                    return;
-                const task = __classPrivateFieldGet(this, _Worker_instances, "m", _Worker_availability).call(this, msg.message.value);
-                __classPrivateFieldGet(this, _Worker_tasks, "f").push(task);
-                task.finally(() => __classPrivateFieldGet(this, _Worker_tasks, "f").splice(__classPrivateFieldGet(this, _Worker_tasks, "f").indexOf(task)));
-                break;
+            ws.send(msg.toBinary());
+        };
+        this.event.on('worker_msg', send);
+        ws.addEventListener('close', () => {
+            closingWS = true;
+            this.#logger.error('worker connection closed unexpectedly');
+            this.close();
+        });
+        ws.addEventListener('message', (event) => {
+            if (event.type !== 'message') {
+                this.#logger.warn('unexpected message type: ' + event.type);
+                return;
             }
-            case 'assignment': {
-                if (!msg.message.value.job)
-                    return;
-                const job = msg.message.value.job;
-                if (job.id in __classPrivateFieldGet(this, _Worker_pending, "f")) {
-                    const task = __classPrivateFieldGet(this, _Worker_pending, "f")[job.id];
-                    delete __classPrivateFieldGet(this, _Worker_pending, "f")[job.id];
-                    task.resolve(msg.message.value);
+            const msg = new ServerMessage();
+            msg.fromBinary(event.data);
+            // register is the only valid first message, and it is only valid as the
+            // first message
+            if (this.#connecting && msg.message.case !== 'register') {
+                throw new WorkerError('expected register response as first message');
+            }
+            switch (msg.message.case) {
+                case 'register': {
+                    this.#id = msg.message.value.workerId;
+                    this.#logger
+                        .child({ id: this.id, server_info: msg.message.value.serverInfo })
+                        .info('registered worker');
+                    this.event.emit('worker_registered', msg.message.value.workerId, msg.message.value.serverInfo);
+                    this.#connecting = false;
+                    break;
                 }
-                else {
-                    __classPrivateFieldGet(this, _Worker_logger, "f").child({ job }).warn('received assignment for unknown job ' + job.id);
+                case 'availability': {
+                    if (!msg.message.value.job)
+                        return;
+                    const task = this.#availability(msg.message.value);
+                    this.#tasks.push(task);
+                    task.finally(() => this.#tasks.splice(this.#tasks.indexOf(task)));
+                    break;
                 }
-                break;
-            }
-            case 'termination': {
-                const task = __classPrivateFieldGet(this, _Worker_instances, "m", _Worker_termination).call(this, msg.message.value);
-                __classPrivateFieldGet(this, _Worker_tasks, "f").push(task);
-                task.finally(() => __classPrivateFieldGet(this, _Worker_tasks, "f").splice(__classPrivateFieldGet(this, _Worker_tasks, "f").indexOf(task)));
-                break;
-            }
-        }
-    });
-    this.event.emit('worker_msg', new WorkerMessage({
-        message: {
-            case: 'register',
-            value: {
-                type: __classPrivateFieldGet(this, _Worker_opts, "f").workerType,
-                agentName: __classPrivateFieldGet(this, _Worker_opts, "f").agentName,
-                allowedPermissions: new ParticipantPermission({
-                    canPublish: __classPrivateFieldGet(this, _Worker_opts, "f").permissions.canPublish,
-                    canSubscribe: __classPrivateFieldGet(this, _Worker_opts, "f").permissions.canSubscribe,
-                    canPublishData: __classPrivateFieldGet(this, _Worker_opts, "f").permissions.canPublishData,
-                    canUpdateMetadata: __classPrivateFieldGet(this, _Worker_opts, "f").permissions.canUpdateMetadata,
-                    hidden: __classPrivateFieldGet(this, _Worker_opts, "f").permissions.hidden,
-                    agent: true,
-                }),
-                version,
-            },
-        },
-    }));
-    let currentStatus = WorkerStatus.WS_AVAILABLE;
-    const loadMonitor = setInterval(() => {
-        if (closingWS)
-            clearInterval(loadMonitor);
-        const oldStatus = currentStatus;
-        __classPrivateFieldGet(this, _Worker_opts, "f").loadFunc().then((currentLoad) => {
-            const isFull = currentLoad >= __classPrivateFieldGet(this, _Worker_opts, "f").loadThreshold;
-            const currentlyAvailable = !isFull;
-            currentStatus = currentlyAvailable ? WorkerStatus.WS_AVAILABLE : WorkerStatus.WS_FULL;
-            if (oldStatus != currentStatus) {
-                const extra = { load: currentLoad, loadThreshold: __classPrivateFieldGet(this, _Worker_opts, "f").loadThreshold };
-                if (isFull) {
-                    __classPrivateFieldGet(this, _Worker_logger, "f").child(extra).info('worker is at full capacity, marking as unavailable');
+                case 'assignment': {
+                    if (!msg.message.value.job)
+                        return;
+                    const job = msg.message.value.job;
+                    if (job.id in this.#pending) {
+                        const task = this.#pending[job.id];
+                        delete this.#pending[job.id];
+                        task.resolve(msg.message.value);
+                    }
+                    else {
+                        this.#logger.child({ job }).warn('received assignment for unknown job ' + job.id);
+                    }
+                    break;
                 }
-                else {
-                    __classPrivateFieldGet(this, _Worker_logger, "f").child(extra).info('worker is below capacity, marking as available');
+                case 'termination': {
+                    const task = this.#termination(msg.message.value);
+                    this.#tasks.push(task);
+                    task.finally(() => this.#tasks.splice(this.#tasks.indexOf(task)));
+                    break;
                 }
             }
-            this.event.emit('worker_msg', new WorkerMessage({
-                message: {
-                    case: 'updateWorker',
-                    value: {
-                        load: currentLoad,
-                        status: currentStatus,
-                    },
-                },
-            }));
         });
-    }, UPDATE_LOAD_INTERVAL);
-}, _Worker_availability = async function _Worker_availability(msg) {
-    let answered = false;
-    const onReject = async () => {
-        answered = true;
         this.event.emit('worker_msg', new WorkerMessage({
             message: {
-                case: 'availability',
+                case: 'register',
                 value: {
-                    jobId: msg.job.id,
-                    available: false,
+                    type: this.#opts.workerType,
+                    agentName: this.#opts.agentName,
+                    allowedPermissions: new ParticipantPermission({
+                        canPublish: this.#opts.permissions.canPublish,
+                        canSubscribe: this.#opts.permissions.canSubscribe,
+                        canPublishData: this.#opts.permissions.canPublishData,
+                        canUpdateMetadata: this.#opts.permissions.canUpdateMetadata,
+                        hidden: this.#opts.permissions.hidden,
+                        agent: true,
+                    }),
+                    version,
                 },
             },
         }));
-    };
-    const onAccept = async (args) => {
-        answered = true;
-        this.event.emit('worker_msg', new WorkerMessage({
-            message: {
-                case: 'availability',
-                value: {
-                    jobId: msg.job.id,
-                    available: true,
-                    participantIdentity: args.identity,
-                    participantName: args.name,
-                    participantMetadata: args.metadata,
+        let currentStatus = WorkerStatus.WS_AVAILABLE;
+        const loadMonitor = setInterval(() => {
+            if (closingWS)
+                clearInterval(loadMonitor);
+            const oldStatus = currentStatus;
+            this.#opts.loadFunc().then((currentLoad) => {
+                const isFull = currentLoad >= this.#opts.loadThreshold;
+                const currentlyAvailable = !isFull;
+                currentStatus = currentlyAvailable ? WorkerStatus.WS_AVAILABLE : WorkerStatus.WS_FULL;
+                if (oldStatus != currentStatus) {
+                    const extra = { load: currentLoad, loadThreshold: this.#opts.loadThreshold };
+                    if (isFull) {
+                        this.#logger.child(extra).info('worker is at full capacity, marking as unavailable');
+                    }
+                    else {
+                        this.#logger.child(extra).info('worker is below capacity, marking as available');
+                    }
+                }
+                this.event.emit('worker_msg', new WorkerMessage({
+                    message: {
+                        case: 'updateWorker',
+                        value: {
+                            load: currentLoad,
+                            status: currentStatus,
+                        },
+                    },
+                }));
+            });
+        }, UPDATE_LOAD_INTERVAL);
+    }
+    async #availability(msg) {
+        let answered = false;
+        const onReject = async () => {
+            answered = true;
+            this.event.emit('worker_msg', new WorkerMessage({
+                message: {
+                    case: 'availability',
+                    value: {
+                        jobId: msg.job.id,
+                        available: false,
+                    },
                 },
-            },
-        }));
-        __classPrivateFieldGet(this, _Worker_pending, "f")[req.id] = new PendingAssignment();
-        const timer = setTimeout(() => {
-            __classPrivateFieldGet(this, _Worker_logger, "f").child({ req }).warn(`assignment for job ${req.id} timed out`);
+            }));
+        };
+        const onAccept = async (args) => {
+            answered = true;
+            this.event.emit('worker_msg', new WorkerMessage({
+                message: {
+                    case: 'availability',
+                    value: {
+                        jobId: msg.job.id,
+                        available: true,
+                        participantIdentity: args.identity,
+                        participantName: args.name,
+                        participantMetadata: args.metadata,
+                    },
+                },
+            }));
+            this.#pending[req.id] = new PendingAssignment();
+            const timer = setTimeout(() => {
+                this.#logger.child({ req }).warn(`assignment for job ${req.id} timed out`);
+                return;
+            }, ASSIGNMENT_TIMEOUT);
+            const asgn = await this.#pending[req.id].promise.then(async (asgn) => {
+                clearTimeout(timer);
+                return asgn;
+            });
+            await this.#procPool.launchJob({
+                acceptArguments: args,
+                job: msg.job,
+                url: asgn.url || this.#opts.wsURL,
+                token: asgn.token,
+            });
+        };
+        const req = new JobRequest(msg.job, onReject, onAccept);
+        this.#logger
+            .child({ job: msg.job, resuming: msg.resuming, agentName: this.#opts.agentName })
+            .info('received job request');
+        const jobRequestTask = async () => {
+            try {
+                await this.#opts.requestFunc(req);
+            }
+            catch (e) {
+                this.#logger
+                    .child({ job: msg.job, resuming: msg.resuming, agentName: this.#opts.agentName })
+                    .info('jobRequestFunc failed');
+                await onReject();
+            }
+            if (!answered) {
+                this.#logger
+                    .child({ job: msg.job, resuming: msg.resuming, agentName: this.#opts.agentName })
+                    .info('no answer was given inside the jobRequestFunc, automatically rejecting the job');
+            }
+        };
+        const task = jobRequestTask();
+        this.#tasks.push(task);
+        task.finally(() => this.#tasks.splice(this.#tasks.indexOf(task)));
+    }
+    async #termination(msg) {
+        const proc = this.#procPool.getByJobId(msg.jobId);
+        if (proc === null) {
+            // safe to ignore
             return;
-        }, ASSIGNMENT_TIMEOUT);
-        const asgn = await __classPrivateFieldGet(this, _Worker_pending, "f")[req.id].promise.then(async (asgn) => {
-            clearTimeout(timer);
-            return asgn;
-        });
-        await __classPrivateFieldGet(this, _Worker_procPool, "f").launchJob({
-            acceptArguments: args,
-            job: msg.job,
-            url: asgn.url || __classPrivateFieldGet(this, _Worker_opts, "f").wsURL,
-            token: asgn.token,
-        });
-    };
-    const req = new JobRequest(msg.job, onReject, onAccept);
-    __classPrivateFieldGet(this, _Worker_logger, "f")
-        .child({ job: msg.job, resuming: msg.resuming, agentName: __classPrivateFieldGet(this, _Worker_opts, "f").agentName })
-        .info('received job request');
-    const jobRequestTask = async () => {
-        try {
-            await __classPrivateFieldGet(this, _Worker_opts, "f").requestFunc(req);
         }
-        catch (e) {
-            __classPrivateFieldGet(this, _Worker_logger, "f")
-                .child({ job: msg.job, resuming: msg.resuming, agentName: __classPrivateFieldGet(this, _Worker_opts, "f").agentName })
-                .info('jobRequestFunc failed');
-            await onReject();
+        await proc.close();
+    }
+    async close() {
+        if (this.#closed) {
+            await this.#close.await;
+            return;
         }
-        if (!answered) {
-            __classPrivateFieldGet(this, _Worker_logger, "f")
-                .child({ job: msg.job, resuming: msg.resuming, agentName: __classPrivateFieldGet(this, _Worker_opts, "f").agentName })
-                .info('no answer was given inside the jobRequestFunc, automatically rejecting the job');
-        }
-    };
-    const task = jobRequestTask();
-    __classPrivateFieldGet(this, _Worker_tasks, "f").push(task);
-    task.finally(() => __classPrivateFieldGet(this, _Worker_tasks, "f").splice(__classPrivateFieldGet(this, _Worker_tasks, "f").indexOf(task)));
-}, _Worker_termination = async function _Worker_termination(msg) {
-    const proc = __classPrivateFieldGet(this, _Worker_procPool, "f").getByJobId(msg.jobId);
-    if (proc === null) {
-        // safe to ignore
-        return;
+        this.#logger.info('shutting down worker');
+        this.#closed = true;
+        await this.#procPool.close();
+        await this.#httpServer.close();
+        await Promise.allSettled(this.#tasks);
+        this.#session?.close();
+        await this.#close.await;
     }
-    await proc.close();
-};
+}
 //# sourceMappingURL=worker.js.map

package.json

 {
   "name": "@livekit/agents",
-  "version": "0.3.5",
+  "version": "0.4.0",
   "description": "LiveKit Agents - Node.js",
@@ -16,5 +16,6 @@ "main": "dist/index.js",
   "dependencies": {
-    "@livekit/mutex": "^1.0.0",
+    "@livekit/mutex": "^1.1.0",
     "@livekit/protocol": "^1.21.0",
-    "@livekit/rtc-node": "^0.11.0",
+    "@livekit/rtc-node": "^0.11.1",
+    "@livekit/typed-emitter": "^3.0.0",
     "commander": "^12.0.0",
@@ -21,0 +22,0 @@ "livekit-server-sdk": "^2.6.1",

src/index.ts

@@ -15,5 +15,20 @@ // SPDX-FileCopyrightText: 2024 LiveKit, Inc.
 import * as multimodal from './multimodal/index.js';
+import * as pipeline from './pipeline/index.js';
 import * as stt from './stt/index.js';
+import * as tokenize from './tokenize/index.js';
 import * as tts from './tts/index.js';
 
+const isCommonJS = (): boolean => {
+  try {
+    return !!require;
+  } catch {
+    return false;
+  }
+};
+if (isCommonJS()) {
+  throw new ReferenceError(
+    '@livekit/agents cannot be used in a CommonJS environment. Please set `"type": "module"` in package.json.',
+  );
+}
+
 export * from './vad.js';
@@ -27,6 +42,5 @@ export * from './plugin.js';
 export * from './generator.js';
-export * from './tokenize.js';
 export * from './audio.js';
 export * from './transcription.js';
 
-export { cli, stt, tts, llm, multimodal };
+export { cli, stt, tts, llm, pipeline, multimodal, tokenize };

src/ipc/proc_pool.ts

 // SPDX-FileCopyrightText: 2024 LiveKit, Inc.
 //
 // SPDX-License-Identifier: Apache-2.0
-import { Mutex } from '@livekit/mutex';
+import { MultiMutex, Mutex } from '@livekit/mutex';
 import type { RunningJobInfo } from '../job.js';
@@ -20,3 +20,3 @@ import { Queue } from '../utils.js';
   initMutex = new Mutex();
-  procMutex: Mutex;
+  procMutex: MultiMutex;
   procUnlock?: () => void;
@@ -27,3 +27,3 @@ warmedProcQueue = new Queue<JobExecutor>();
     agent: string,
-    _numIdleProcesses: number,
+    numIdleProcesses: number,
     initializeTimeout: number,
@@ -33,3 +33,3 @@ closeTimeout: number,
     this.agent = agent;
-    this.procMutex = new Mutex();
+    this.procMutex = new MultiMutex(numIdleProcesses);
     this.initializeTimeout = initializeTimeout;
@@ -36,0 +36,0 @@ this.closeTimeout = closeTimeout;

src/llm/function_context.ts

@@ -21,2 +21,20 @@ // SPDX-FileCopyrightText: 2024 LiveKit, Inc.
 
+/** A function that has been called but is not yet running */
+export interface FunctionCallInfo<P extends z.ZodTypeAny = any, R = any> {
+  name: string;
+  func: CallableFunction<P, R>;
+  toolCallId: string;
+  rawParams: string;
+  params: inferParameters<P>;
+  task?: PromiseLike<CallableFunctionResult>;
+}
+
+/** The result of a ran FunctionCallInfo. */
+export interface CallableFunctionResult {
+  name: string;
+  toolCallId: string;
+  result?: any;
+  error?: any;
+}
+
 /** An object containing callable functions and their names */
@@ -30,25 +48,46 @@ export type FunctionContext = {
   const properties: Record<string, any> = {};
-  const required_properties: string[] = [];
+  const requiredProperties: string[] = [];
 
-  for (const key in p.shape) {
-    const field = p.shape[key];
-    const description = field._def.description || undefined;
-    let type: string;
-    let enumValues: any[] | undefined;
+  const processZodType = (field: z.ZodTypeAny): any => {
+    const isOptional = field instanceof z.ZodOptional;
+    const nestedField = isOptional ? field._def.innerType : field;
+    const description = field._def.description;
 
-    if (field instanceof z.ZodEnum) {
-      enumValues = field._def.values;
-      type = typeof enumValues![0];
+    if (nestedField instanceof z.ZodEnum) {
+      return {
+        type: typeof nestedField._def.values[0],
+        ...(description && { description }),
+        enum: nestedField._def.values,
+      };
+    } else if (nestedField instanceof z.ZodArray) {
+      const elementType = nestedField._def.type;
+      return {
+        type: 'array',
+        ...(description && { description }),
+        items: processZodType(elementType),
+      };
+    } else if (nestedField instanceof z.ZodObject) {
+      const { properties, required } = oaiParams(nestedField);
+      return {
+        type: 'object',
+        ...(description && { description }),
+        properties,
+        required,
+      };
     } else {
-      type = field._def.typeName.toLowerCase();
+      let type = nestedField._def.typeName.toLowerCase();
+      type = type.includes('zod') ? type.substring(3) : type;
+      return {
+        type,
+        ...(description && { description }),
+      };
     }
+  };
 
-    properties[key] = {
-      type: type.includes('zod') ? type.substring(3) : type,
-      description,
-      enum: enumValues,
-    };
+  for (const key in p.shape) {
+    const field = p.shape[key];
+    properties[key] = processZodType(field);
 
-    if (!field._def.defaultValue) {
-      required_properties.push(key);
+    if (!(field instanceof z.ZodOptional)) {
+      requiredProperties.push(key);
     }
@@ -61,4 +100,24 @@ }
     properties,
-    required_properties,
+    required: requiredProperties,
   };
 };
+
+/** @internal */
+export const oaiBuildFunctionInfo = (
+  fncCtx: FunctionContext,
+  toolCallId: string,
+  fncName: string,
+  rawArgs: string,
+): FunctionCallInfo => {
+  if (!fncCtx[fncName]) {
+    throw new Error(`AI function ${fncName} not found`);
+  }
+
+  return {
+    name: fncName,
+    func: fncCtx[fncName],
+    toolCallId,
+    rawParams: rawArgs,
+    params: JSON.parse(rawArgs),
+  };
+};

src/llm/index.ts

 // SPDX-FileCopyrightText: 2024 LiveKit, Inc.
 //
 // SPDX-License-Identifier: Apache-2.0
-import {
+export {
   type CallableFunction,
+  type FunctionCallInfo,
+  type CallableFunctionResult,
   type FunctionContext,
   type inferParameters,
   oaiParams,
+  oaiBuildFunctionInfo,
 } from './function_context.js';
 
-export { CallableFunction, FunctionContext, inferParameters, oaiParams };
+export {
+  type ChatImage,
+  type ChatAudio,
+  type ChatContent,
+  ChatRole,
+  ChatMessage,
+  ChatContext,
+} from './chat_context.js';
+
+export {
+  type ChoiceDelta,
+  type CompletionUsage,
+  type Choice,
+  type ChatChunk,
+  LLM,
+  LLMStream,
+} from './llm.js';

src/multimodal/multimodal_agent.ts

@@ -67,9 +67,12 @@ // SPDX-FileCopyrightText: 2024 LiveKit, Inc.
     model,
+    chatCtx,
     fncCtx,
   }: {
     model: RealtimeModel;
-    fncCtx?: llm.FunctionContext | undefined;
+    chatCtx?: llm.ChatContext;
+    fncCtx?: llm.FunctionContext;
   }) {
     super();
     this.model = model;
+    this.#chatCtx = chatCtx;
     this.#fncCtx = fncCtx;
@@ -87,2 +90,3 @@ }
   #fncCtx: llm.FunctionContext | undefined = undefined;
+  #chatCtx: llm.ChatContext | undefined = undefined;
 
@@ -214,3 +218,3 @@ #_started: boolean = false;
 
-      this.#session = this.model.session({ fncCtx: this.#fncCtx });
+      this.#session = this.model.session({ fncCtx: this.#fncCtx, chatCtx: this.#chatCtx });
       this.#started = true;
@@ -217,0 +221,0 @@

src/stt/index.ts

@@ -5,3 +5,9 @@ // SPDX-FileCopyrightText: 2024 LiveKit, Inc.
 
-export { STT, SpeechEvent, SpeechEventType, SpeechStream, type SpeechData } from './stt.js';
-export { StreamAdapter, StreamAdapterWrapper } from './stream_adapter.js';
+export {
+  type SpeechEvent,
+  type SpeechData,
+  type STTCapabilities,
+  SpeechEventType,
+  STT,
+  SpeechStream,
+} from './stt.js';

src/stt/stt.ts

@@ -5,4 +5,5 @@ // SPDX-FileCopyrightText: 2024 LiveKit, Inc.
 import type { AudioFrame } from '@livekit/rtc-node';
-import type { AudioBuffer } from '../utils.js';
+import { AsyncIterableQueue } from '../utils.js';
 
+/** Indicates start/middle/end of speech */
 export enum SpeechEventType {
@@ -12,3 +13,3 @@ /**
    * If the STT doesn't support this event, this will be emitted at the same time
-   * as the first INTERMIN_TRANSCRIPT.
+   * as the first INTERIM_TRANSCRIPT.
    */
@@ -32,2 +33,3 @@ START_OF_SPEECH = 0,
 
+/** SpeechData contains metadata about this {@link SpeechEvent}. */
 export interface SpeechData {
@@ -41,49 +43,114 @@ language: string;
 
-export class SpeechEvent {
+/** SpeechEvent is a packet of speech-to-text data. */
+export interface SpeechEvent {
   type: SpeechEventType;
   alternatives: SpeechData[];
+}
 
-  constructor(type: SpeechEventType, alternatives: SpeechData[] = []) {
-    this.type = type;
-    this.alternatives = alternatives;
-  }
+/**
+ * Describes the capabilities of the STT provider.
+ *
+ * @remarks
+ * At present, the framework only supports providers that have a streaming endpoint.
+ */
+export interface STTCapabilities {
+  streaming: boolean;
+  interimResults: boolean;
 }
 
-export abstract class SpeechStream implements IterableIterator<SpeechEvent> {
-  /**
-   * Push a frame to be recognised.
-   * It is recommended to push frames as soon as they are available.
-   */
-  abstract pushFrame(token: AudioFrame): void;
+/**
+ * An instance of a speech-to-text adapter.
+ *
+ * @remarks
+ * This class is abstract, and as such cannot be used directly. Instead, use a provider plugin that
+ * exports its own child STT class, which inherits this class's methods.
+ */
+export abstract class STT {
+  #capabilities: STTCapabilities;
 
+  constructor(capabilities: STTCapabilities) {
+    this.#capabilities = capabilities;
+  }
+
+  /** Returns this STT's capabilities */
+  get capabilities(): STTCapabilities {
+    return this.#capabilities;
+  }
+
   /**
-   * Close the stream.
-   *
-   * @param wait
-   *   Whether to wait for the STT to finish processing the remaining
-   *   frames before closing
+   * Returns a {@link SpeechStream} that can be used to push audio frames and receive
+   * transcriptions
    */
-  abstract close(wait: boolean): Promise<void>;
+  abstract stream(): SpeechStream;
+}
 
-  abstract next(): IteratorResult<SpeechEvent>;
+/**
+ * An instance of a speech-to-text stream, as an asynchronous iterable iterator.
+ *
+ * @example Looping through frames
+ * ```ts
+ * for await (const event of stream) {
+ *   if (event.type === SpeechEventType.FINAL_TRANSCRIPT) {
+ *     console.log(event.alternatives[0].text)
+ *   }
+ * }
+ * ```
+ *
+ * @remarks
+ * This class is abstract, and as such cannot be used directly. Instead, use a provider plugin that
+ * exports its own child SpeechStream class, which inherits this class's methods.
+ */
+export abstract class SpeechStream implements AsyncIterableIterator<SpeechEvent> {
+  protected static readonly FLUSH_SENTINEL = Symbol('FLUSH_SENTINEL');
+  protected input = new AsyncIterableQueue<AudioFrame | typeof SpeechStream.FLUSH_SENTINEL>();
+  protected queue = new AsyncIterableQueue<SpeechEvent>();
+  protected closed = false;
 
-  [Symbol.iterator](): SpeechStream {
-    return this;
+  /** Push an audio frame to the STT */
+  pushFrame(frame: AudioFrame) {
+    if (this.input.closed) {
+      throw new Error('Input is closed');
+    }
+    if (this.closed) {
+      throw new Error('Stream is closed');
+    }
+    this.input.put(frame);
   }
-}
 
-export abstract class STT {
-  #streamingSupported: boolean;
+  /** Flush the STT, causing it to process all pending text */
+  flush() {
+    if (this.input.closed) {
+      throw new Error('Input is closed');
+    }
+    if (this.closed) {
+      throw new Error('Stream is closed');
+    }
+    this.input.put(SpeechStream.FLUSH_SENTINEL);
+  }
 
-  constructor(streamingSupported: boolean) {
-    this.#streamingSupported = streamingSupported;
+  /** Mark the input as ended and forbid additional pushes */
+  endInput() {
+    if (this.input.closed) {
+      throw new Error('Input is closed');
+    }
+    if (this.closed) {
+      throw new Error('Stream is closed');
+    }
+    this.input.close();
   }
 
-  abstract recognize(buffer: AudioBuffer, language?: string): Promise<SpeechEvent>;
+  next(): Promise<IteratorResult<SpeechEvent>> {
+    return this.queue.next();
+  }
 
-  abstract stream(language: string | undefined): SpeechStream;
+  /** Close both the input and output of the STT stream */
+  close() {
+    this.input.close();
+    this.queue.close();
+    this.closed = true;
+  }
 
-  get streamingSupported(): boolean {
-    return this.#streamingSupported;
+  [Symbol.asyncIterator](): SpeechStream {
+    return this;
   }
 }

src/tts/index.ts

 // SPDX-FileCopyrightText: 2024 LiveKit, Inc.
 //
 // SPDX-License-Identifier: Apache-2.0
-import { StreamAdapter, StreamAdapterWrapper } from './stream_adapter.js';
-import {
-  ChunkedStream,
-  SynthesisEvent,
-  SynthesisEventType,
-  SynthesizeStream,
-  type SynthesizedAudio,
-  TTS,
-} from './tts.js';
-
-export {
-  TTS,
-  SynthesisEvent,
-  SynthesisEventType,
-  SynthesizedAudio,
-  SynthesizeStream,
-  StreamAdapter,
-  StreamAdapterWrapper,
-  ChunkedStream,
-};
+export { type SynthesizedAudio, type TTSCapabilities, TTS, SynthesizeStream } from './tts.js';

src/tts/tts.ts

@@ -5,85 +5,138 @@ // SPDX-FileCopyrightText: 2024 LiveKit, Inc.
 import type { AudioFrame } from '@livekit/rtc-node';
-import { mergeFrames } from '../utils.js';
+import { AsyncIterableQueue } from '../utils.js';
 
+/** SynthesizedAudio is a packet of speech synthesis as returned by the TTS. */
 export interface SynthesizedAudio {
-  text: string;
-  data: AudioFrame;
+  /** Request ID (one segment could be made up of multiple requests) */
+  requestId: string;
+  /** Segment ID, each segment is separated by a flush */
+  segmentId: string;
+  /** Synthesized audio frame */
+  frame: AudioFrame;
+  /** Current segment of the synthesized audio */
+  deltaText?: string;
 }
 
-export enum SynthesisEventType {
-  /**
-   * Indicate the start of synthesis.
-   * Retriggered after FINISHED.
-   */
-  STARTED = 0,
-  /**
-   * Indicate that audio data is available.
-   */
-  AUDIO = 1,
-  /**
-   * Indicate the end of synthesis. Does not necessarily mean stream is done.
-   */
-  FINISHED = 2,
+/**
+ * Describes the capabilities of the TTS provider.
+ *
+ * @remarks
+ * At present, only `streaming` is supplied to this interface, and the framework only supports
+ * providers that do have a streaming endpoint.
+ */
+export interface TTSCapabilities {
+  streaming: boolean;
 }
 
-export class SynthesisEvent {
-  type: SynthesisEventType;
-  audio?: SynthesizedAudio;
+/**
+ * An instance of a text-to-speech adapter.
+ *
+ * @remarks
+ * This class is abstract, and as such cannot be used directly. Instead, use a provider plugin that
+ * exports its own child TTS class, which inherits this class's methods.
+ */
+export abstract class TTS {
+  #capabilities: TTSCapabilities;
+  #sampleRate: number;
+  #numChannels: number;
 
-  constructor(type: SynthesisEventType, audio: SynthesizedAudio | undefined = undefined) {
-    this.type = type;
-    this.audio = audio;
+  constructor(sampleRate: number, numChannels: number, capabilities: TTSCapabilities) {
+    this.#capabilities = capabilities;
+    this.#sampleRate = sampleRate;
+    this.#numChannels = numChannels;
   }
-}
 
-export abstract class SynthesizeStream implements IterableIterator<SynthesisEvent> {
-  abstract pushText(token?: string): void;
+  /** Returns this TTS's capabilities */
+  get capabilities(): TTSCapabilities {
+    return this.#capabilities;
+  }
 
-  markSegmentEnd() {
-    this.pushText(undefined);
+  /** Returns the sample rate of audio frames returned by this TTS */
+  get sampleRate(): number {
+    return this.#sampleRate;
   }
 
-  abstract close(wait: boolean): Promise<void>;
-  abstract next(): IteratorResult<SynthesisEvent>;
+  /** Returns the channel count of audio frames returned by this TTS */
+  get numChannels(): number {
+    return this.#numChannels;
+  }
 
-  [Symbol.iterator](): SynthesizeStream {
-    return this;
-  }
+  /**
+   * Returns a {@link SynthesizeStream} that can be used to push text and receive audio data
+   */
+  abstract stream(): SynthesizeStream;
 }
 
-export abstract class TTS {
-  #streamingSupported: boolean;
+/**
+ * An instance of a text-to-speech stream, as an asynchronous iterable iterator.
+ *
+ * @example Looping through frames
+ * ```ts
+ * for await (const event of stream) {
+ *   await source.captureFrame(event.frame);
+ * }
+ * ```
+ *
+ * @remarks
+ * This class is abstract, and as such cannot be used directly. Instead, use a provider plugin that
+ * exports its own child SynthesizeStream class, which inherits this class's methods.
+ */
+export abstract class SynthesizeStream
+  implements AsyncIterableIterator<SynthesizedAudio | typeof SynthesizeStream.END_OF_STREAM>
+{
+  protected static readonly FLUSH_SENTINEL = Symbol('FLUSH_SENTINEL');
+  static readonly END_OF_STREAM = Symbol('END_OF_STREAM');
+  protected input = new AsyncIterableQueue<string | typeof SynthesizeStream.FLUSH_SENTINEL>();
+  protected queue = new AsyncIterableQueue<
+    SynthesizedAudio | typeof SynthesizeStream.END_OF_STREAM
+  >();
+  protected closed = false;
 
-  constructor(streamingSupported: boolean) {
-    this.#streamingSupported = streamingSupported;
+  /** Push a string of text to the TTS */
+  pushText(text: string) {
+    if (this.input.closed) {
+      throw new Error('Input is closed');
+    }
+    if (this.closed) {
+      throw new Error('Stream is closed');
+    }
+    this.input.put(text);
   }
 
-  abstract synthesize(text: string): Promise<ChunkedStream>;
-
-  abstract stream(): SynthesizeStream;
-
-  get streamingSupported(): boolean {
-    return this.#streamingSupported;
+  /** Flush the TTS, causing it to process all pending text */
+  flush() {
+    if (this.input.closed) {
+      throw new Error('Input is closed');
+    }
+    if (this.closed) {
+      throw new Error('Stream is closed');
+    }
+    this.input.put(SynthesizeStream.FLUSH_SENTINEL);
   }
-}
 
-export abstract class ChunkedStream implements AsyncIterableIterator<SynthesizedAudio> {
-  async collect(): Promise<AudioFrame> {
-    const frames = [];
-    for await (const ev of this) {
-      frames.push(ev.data);
+  /** Mark the input as ended and forbid additional pushes */
+  endInput() {
+    if (this.input.closed) {
+      throw new Error('Input is closed');
     }
-    return mergeFrames(frames);
+    if (this.closed) {
+      throw new Error('Stream is closed');
+    }
+    this.input.close();
   }
 
-  abstract close(): Promise<void>;
-  abstract next(): Promise<IteratorResult<SynthesizedAudio>>;
+  next(): Promise<IteratorResult<SynthesizedAudio | typeof SynthesizeStream.END_OF_STREAM>> {
+    return this.queue.next();
+  }
 
-  [Symbol.iterator](): ChunkedStream {
-    return this;
+  /** Close both the input and output of the TTS stream */
+  close() {
+    this.input.close();
+    this.queue.close();
+    this.closed = true;
   }
 
-  [Symbol.asyncIterator](): ChunkedStream {
+  [Symbol.asyncIterator](): SynthesizeStream {
     return this;
   }
 }

src/utils.ts

@@ -228,37 +228,107 @@ // SPDX-FileCopyrightText: 2024 LiveKit, Inc.
 /** @internal */
-export class AsyncIterableQueue<T> implements AsyncIterable<T> {
-  private queue: Queue<T | typeof AsyncIterableQueue.QUEUE_END_MARKER>;
-  private closed = false;
-  private static readonly QUEUE_END_MARKER = Symbol('QUEUE_END_MARKER');
+export class AsyncIterableQueue<T> implements AsyncIterableIterator<T> {
+  private static readonly CLOSE_SENTINEL = Symbol('CLOSE_SENTINEL');
+  #queue = new Queue<T | typeof AsyncIterableQueue.CLOSE_SENTINEL>();
+  #closed = false;
 
-  constructor() {
-    this.queue = new Queue<T | typeof AsyncIterableQueue.QUEUE_END_MARKER>();
+  get closed(): boolean {
+    return this.#closed;
   }
 
   put(item: T): void {
-    if (this.closed) {
+    if (this.#closed) {
       throw new Error('Queue is closed');
     }
-    this.queue.put(item);
+    this.#queue.put(item);
   }
 
   close(): void {
-    this.closed = true;
-    this.queue.put(AsyncIterableQueue.QUEUE_END_MARKER);
+    this.#closed = true;
+    this.#queue.put(AsyncIterableQueue.CLOSE_SENTINEL);
   }
 
-  [Symbol.asyncIterator](): AsyncIterator<T> {
-    return {
-      next: async (): Promise<IteratorResult<T>> => {
-        if (this.closed && this.queue.items.length === 0) {
-          return { value: undefined, done: true };
-        }
-        const item = await this.queue.get();
-        if (item === AsyncIterableQueue.QUEUE_END_MARKER && this.closed) {
-          return { value: undefined, done: true };
-        }
-        return { value: item as T, done: false };
-      },
-    };
+  async next(): Promise<IteratorResult<T>> {
+    if (this.#closed && this.#queue.items.length === 0) {
+      return { value: undefined, done: true };
+    }
+    const item = await this.#queue.get();
+    if (item === AsyncIterableQueue.CLOSE_SENTINEL && this.#closed) {
+      return { value: undefined, done: true };
+    }
+    return { value: item as T, done: false };
   }
+
+  [Symbol.asyncIterator](): AsyncIterableQueue<T> {
+    return this;
+  }
 }
+
+/** @internal */
+export class ExpFilter {
+  #alpha: number;
+  #max?: number;
+  #filtered?: number = undefined;
+
+  constructor(alpha: number, max?: number) {
+    this.#alpha = alpha;
+    this.#max = max;
+  }
+
+  reset(alpha?: number) {
+    if (alpha) {
+      this.#alpha = alpha;
+    }
+    this.#filtered = undefined;
+  }
+
+  apply(exp: number, sample: number): number {
+    if (this.#filtered) {
+      const a = this.#alpha ** exp;
+      this.#filtered = a * this.#filtered + (1 - a) * sample;
+    } else {
+      this.#filtered = sample;
+    }
+
+    if (this.#max && this.#filtered > this.#max) {
+      this.#filtered = this.#max;
+    }
+
+    return this.#filtered;
+  }
+
+  get filtered(): number | undefined {
+    return this.#filtered;
+  }
+
+  set alpha(alpha: number) {
+    this.#alpha = alpha;
+  }
+}
+
+/** @internal */
+export class AudioEnergyFilter {
+  #cooldownSeconds: number;
+  #cooldown: number;
+
+  constructor(cooldownSeconds = 1) {
+    this.#cooldownSeconds = cooldownSeconds;
+    this.#cooldown = cooldownSeconds;
+  }
+
+  pushFrame(frame: AudioFrame): boolean {
+    const arr = Float32Array.from(frame.data, (x) => x / 32768);
+    const rms = (arr.map((x) => x ** 2).reduce((acc, x) => acc + x) / arr.length) ** 0.5;
+    if (rms > 0.004) {
+      this.#cooldown = this.#cooldownSeconds;
+      return true;
+    }
+
+    const durationSeconds = frame.samplesPerChannel / frame.sampleRate;
+    this.#cooldown -= durationSeconds;
+    if (this.#cooldown > 0) {
+      return true;
+    }
+
+    return false;
+  }
+}

src/vad.ts

@@ -5,68 +5,109 @@ // SPDX-FileCopyrightText: 2024 LiveKit, Inc.
 import type { AudioFrame } from '@livekit/rtc-node';
+import { AsyncIterableQueue } from './utils.js';
 
 export enum VADEventType {
-  START_OF_SPEECH = 1,
-  SPEAKING = 2,
-  END_OF_SPEECH = 3,
+  START_OF_SPEECH,
+  INFERENCE_DONE,
+  END_OF_SPEECH,
 }
 
 export interface VADEvent {
+  /** Type of the VAD event (e.g., start of speech, end of speech, inference done). */
   type: VADEventType;
   /**
-   * Index of the samples of the event (when the event was fired)
+   * Index of the audio sample where the event occurred, relative to the inference sample rate.
    */
   samplesIndex: number;
+  /** Timestamp when the event was fired. */
+  timestamp: number;
+  /** Duration of the detected speech segment in seconds. */
+  speechDuration: number;
+  /** Duration of the silence segment preceding or following the speech, in seconds. */
+  silenceDuration: number;
   /**
-   * Duration of speech, in seconds
+   * List of audio frames associated with the speech.
+   *
+   * @remarks
+   * - For `start_of_speech` events, this contains the audio chunks that triggered the detection.
+   * - For `inference_done` events, this contains the audio chunks that were processed.
+   * - For `end_of_speech` events, this contains the complete user speech.
    */
-  duration: number;
-  speech: AudioFrame[];
+  frames: AudioFrame[];
+  /** Probability that speech is present (only for `INFERENCE_DONE` events). */
+  probability: number;
+  /** Time taken to perform the inference, in seconds (only for `INFERENCE_DONE` events). */
+  inferenceDuration: number;
+  /** Indicates whether speech was detected in the frames. */
+  speaking: boolean;
 }
 
+export interface VADCapabilities {
+  updateInterval: number;
+}
+
 export abstract class VAD {
+  #capabilities: VADCapabilities;
+  constructor(capabilities: VADCapabilities) {
+    this.#capabilities = capabilities;
+  }
+
+  get capabilities(): VADCapabilities {
+    return this.#capabilities;
+  }
+
   /**
    * Returns a {@link VADStream} that can be used to push audio frames and receive VAD events.
-   *
-   * @param options
    */
-  abstract stream({
-    minSpeakingDuration,
-    minSilenceDuration,
-    paddingDuration,
-    sampleRate,
-    maxBufferedSpeech,
-  }: {
-    /**
-     * Minimum duration of speech required to trigger a {@link VADEventType.START_OF_SPEECH} event
-     */
-    minSpeakingDuration: number;
-    /**
-     * Milliseconds to wait before separating speech chunk.
-     * Not always precise, generally rounded to the nearest 40ms depending on VAD implementation
-     */
-    minSilenceDuration: number;
-    /**
-     * Number of frames to pad the start and end of speech with
-     */
-    paddingDuration: number;
-    /**
-     * Sample rate of inference/processing
-     */
-    sampleRate: number;
-    /**
-     * Number of seconds the buffer may keep until {@link VADEventType.END_OF_SPEECH} is triggered.
-     * It is recommended to set this to a positive value, as zero may OOM if the user doesn't stop
-     * speaking.
-     */
-    maxBufferedSpeech: number;
-  }): VADStream;
+  abstract stream(): VADStream;
 }
 
-export abstract class VADStream implements IterableIterator<VADEvent> {
-  abstract pushFrame(frame: AudioFrame): void;
-  abstract close(wait: boolean): Promise<void>;
-  abstract next(): IteratorResult<VADEvent>;
-  [Symbol.iterator](): VADStream {
+export abstract class VADStream implements AsyncIterableIterator<VADEvent> {
+  protected static readonly FLUSH_SENTINEL = Symbol('FLUSH_SENTINEL');
+  protected input = new AsyncIterableQueue<AudioFrame | typeof VADStream.FLUSH_SENTINEL>();
+  protected queue = new AsyncIterableQueue<VADEvent>();
+  protected closed = false;
+
+  pushFrame(frame: AudioFrame) {
+    if (this.input.closed) {
+      throw new Error('Input is closed');
+    }
+    if (this.closed) {
+      throw new Error('Stream is closed');
+    }
+    this.input.put(frame);
+  }
+
+  flush() {
+    if (this.input.closed) {
+      throw new Error('Input is closed');
+    }
+    if (this.closed) {
+      throw new Error('Stream is closed');
+    }
+    this.input.put(VADStream.FLUSH_SENTINEL);
+  }
+
+  endInput() {
+    if (this.input.closed) {
+      throw new Error('Input is closed');
+    }
+    if (this.closed) {
+      throw new Error('Stream is closed');
+    }
+    this.input.close();
+  }
+
+  next(): Promise<IteratorResult<VADEvent>> {
+    return this.queue.next();
+  }
+
+  close() {
+    this.input.close();
+    this.queue.close();
+    this.closed = true;
+  }
+
+  [Symbol.asyncIterator](): VADStream {
     return this;
   }
 }

No alert changes

Improved metrics

Total package byte prevSize

777546

increased by59.49%

Number of package files

232

increased by47.77%

Lines of code

12353

increased by82.33%

Worsened metrics

Dependency count

10

increased by11.11%

Dependency changes

• + Added@livekit/typed-emitter@^3.0.0

• Updated@livekit/mutex@^1.1.0

• Updated@livekit/rtc-node@^0.11.1