@livekit/agents - npm Package Compare versions

Comparing version 0.3.2 to 0.3.3

CHANGELOG.md

 # @livekit/agents
 
+## 0.3.3
+
+### Patch Changes
+
+- Fix subscription timing - [#110](https://github.com/livekit/agents-js/pull/110) ([@nbsp](https://github.com/nbsp))
+
+- fix usage on Windows by importing using URLs, not paths - [#110](https://github.com/livekit/agents-js/pull/110) ([@nbsp](https://github.com/nbsp))
+
 ## 0.3.2
@@ -4,0 +12,0 @@

dist/http_server.d.ts

@@ -1,3 +0,3 @@
-/// <reference types="node" />
-import { type Server } from 'http';
+/// <reference types="node" resolution-mode="require"/>
+import { type Server } from 'node:http';
 export declare class HTTPServer {
@@ -4,0 +4,0 @@ #private;

dist/http_server.js

@@ -10,3 +10,3 @@ var __classPrivateFieldGet = (this && this.__classPrivateFieldGet) || function (receiver, state, kind, f) {
 // SPDX-License-Identifier: Apache-2.0
-import { createServer } from 'http';
+import { createServer } from 'node:http';
 import { log } from './log.js';
@@ -13,0 +13,0 @@ const healthCheck = async (res) => {

dist/ipc/job_main.d.ts

@@ -1,3 +0,3 @@
-/// <reference types="node" />
-import type { ChildProcess } from 'child_process';
+/// <reference types="node" resolution-mode="require"/>
+import type { ChildProcess } from 'node:child_process';
 type StartArgs = {
@@ -4,0 +4,0 @@ agentFile: string;

dist/ipc/job_main.js

@@ -5,5 +5,5 @@ // SPDX-FileCopyrightText: 2024 LiveKit, Inc.
 import { Room, RoomEvent } from '@livekit/rtc-node';
-import { fork } from 'child_process';
-import { EventEmitter, once } from 'events';
-import { fileURLToPath } from 'url';
+import { fork } from 'node:child_process';
+import { EventEmitter, once } from 'node:events';
+import { pathToFileURL } from 'node:url';
 import { isAgent } from '../generator.js';
@@ -16,3 +16,3 @@ import { JobContext } from '../job.js';
 export const runProcess = (args) => {
-    return fork(fileURLToPath(import.meta.url), [args.agentFile]);
+    return fork(new URL(import.meta.url), [args.agentFile]);
 };
@@ -63,3 +63,3 @@ const startJob = (proc, func, info, closeEvent, logger) => {
     const moduleFile = process.argv[2];
-    const agent = await import(moduleFile).then((module) => {
+    const agent = await import(pathToFileURL(moduleFile).href).then((module) => {
         const agent = module.default;
@@ -66,0 +66,0 @@ if (agent === undefined || !isAgent(agent)) {

dist/ipc/proc_job_executor.js

@@ -13,3 +13,3 @@ var __classPrivateFieldSet = (this && this.__classPrivateFieldSet) || function (receiver, state, value, kind, f) {
 var _ProcJobExecutor_opts, _ProcJobExecutor_started, _ProcJobExecutor_closing, _ProcJobExecutor_runningJob, _ProcJobExecutor_proc, _ProcJobExecutor_pingInterval, _ProcJobExecutor_pongTimeout, _ProcJobExecutor_init, _ProcJobExecutor_join, _ProcJobExecutor_logger;
-import { once } from 'events';
+import { once } from 'node:events';
 import { log, loggerOptions } from '../log.js';
@@ -16,0 +16,0 @@ import { Future } from '../utils.js';

dist/multimodal/agent_playout.d.ts

@@ -1,5 +0,5 @@
-/// <reference types="node" />
+/// <reference types="node" resolution-mode="require"/>
 import type { AudioFrame } from '@livekit/rtc-node';
 import { type AudioSource } from '@livekit/rtc-node';
-import { EventEmitter } from 'events';
+import { EventEmitter } from 'node:events';
 import type { TranscriptionForwarder } from '../transcription.js';
@@ -6,0 +6,0 @@ import { type AsyncIterableQueue, Future } from '../utils.js';

dist/multimodal/agent_playout.js

@@ -20,3 +20,3 @@ var __classPrivateFieldSet = (this && this.__classPrivateFieldSet) || function (receiver, state, value, kind, f) {
 var _PlayoutHandle_audioSource, _PlayoutHandle_sampleRate, _PlayoutHandle_itemId, _PlayoutHandle_contentIndex, _PlayoutHandle_interrupted, _AgentPlayout_instances, _AgentPlayout_audioSource, _AgentPlayout_playoutTask, _AgentPlayout_sampleRate, _AgentPlayout_numChannels, _AgentPlayout_inFrameSize, _AgentPlayout_outFrameSize, _AgentPlayout_makePlayoutTask;
-import { EventEmitter } from 'events';
+import { EventEmitter } from 'node:events';
 import { AudioByteStream } from '../audio.js';
@@ -23,0 +23,0 @@ import { CancellablePromise, Future, gracefullyCancel } from '../utils.js';

dist/multimodal/multimodal_agent.d.ts

@@ -1,4 +0,4 @@
-/// <reference types="node" />
+/// <reference types="node" resolution-mode="require"/>
 import type { RemoteAudioTrack, RemoteParticipant, Room } from '@livekit/rtc-node';
-import { EventEmitter } from 'events';
+import { EventEmitter } from 'node:events';
 import type * as llm from '../llm/index.js';
@@ -5,0 +5,0 @@ /**

dist/multimodal/multimodal_agent.js

@@ -19,5 +19,5 @@ var __classPrivateFieldSet = (this && this.__classPrivateFieldSet) || function (receiver, state, value, kind, f) {
 };
-var _MultimodalAgent_instances, _MultimodalAgent_participant, _MultimodalAgent_agentPublication, _MultimodalAgent_localTrackSid, _MultimodalAgent_localSource, _MultimodalAgent_agentPlayout, _MultimodalAgent_playingHandle, _MultimodalAgent_logger, _MultimodalAgent_session, _MultimodalAgent_fncCtx, _MultimodalAgent__started, _MultimodalAgent__pendingFunctionCalls, _MultimodalAgent__speaking, _MultimodalAgent_pendingFunctionCalls_get, _MultimodalAgent_pendingFunctionCalls_set, _MultimodalAgent_speaking_get, _MultimodalAgent_speaking_set, _MultimodalAgent_started_get, _MultimodalAgent_started_set, _MultimodalAgent_linkParticipant, _MultimodalAgent_subscribeToMicrophone, _MultimodalAgent_getLocalTrackSid, _MultimodalAgent_publishTranscription, _MultimodalAgent_updateState, _MultimodalAgent_setState;
+var _MultimodalAgent_instances, _MultimodalAgent_participant, _MultimodalAgent_agentPublication, _MultimodalAgent_localTrackSid, _MultimodalAgent_localSource, _MultimodalAgent_agentPlayout, _MultimodalAgent_playingHandle, _MultimodalAgent_logger, _MultimodalAgent_session, _MultimodalAgent_fncCtx, _MultimodalAgent__started, _MultimodalAgent__pendingFunctionCalls, _MultimodalAgent__speaking, _MultimodalAgent_pendingFunctionCalls_get, _MultimodalAgent_pendingFunctionCalls_set, _MultimodalAgent_speaking_get, _MultimodalAgent_speaking_set, _MultimodalAgent_started_get, _MultimodalAgent_started_set, _MultimodalAgent_linkParticipant, _MultimodalAgent_subscribeToMicrophone, _MultimodalAgent_handleTrackSubscription, _MultimodalAgent_getLocalTrackSid, _MultimodalAgent_publishTranscription, _MultimodalAgent_updateState, _MultimodalAgent_setState;
 import { AudioSource, AudioStream, LocalAudioTrack, RoomEvent, TrackPublishOptions, TrackSource, } from '@livekit/rtc-node';
-import { EventEmitter } from 'events';
+import { EventEmitter } from 'node:events';
 import { AudioByteStream } from '../audio.js';
@@ -82,3 +82,4 @@ import { log } from '../log.js';
             room.on(RoomEvent.ParticipantConnected, (participant) => {
-                if (!this.linkedParticipant) {
+                // automatically link to the first participant that connects, if not already linked
+                if (this.linkedParticipant) {
                     return;
@@ -88,2 +89,7 @@ }
             });
+            room.on(RoomEvent.TrackPublished, () => {
+                // in case we are connected before the participant has published, we'd need to re-subscribe
+                __classPrivateFieldGet(this, _MultimodalAgent_instances, "m", _MultimodalAgent_subscribeToMicrophone).call(this);
+            });
+            room.on(RoomEvent.TrackSubscribed, __classPrivateFieldGet(this, _MultimodalAgent_instances, "m", _MultimodalAgent_handleTrackSubscription).bind(this));
             this.room = room;
@@ -226,8 +232,33 @@ __classPrivateFieldSet(this, _MultimodalAgent_participant, participant, "f");
     }
-    else {
-        this.room.on(RoomEvent.TrackPublished, () => {
-            __classPrivateFieldGet(this, _MultimodalAgent_instances, "m", _MultimodalAgent_subscribeToMicrophone).call(this);
-        });
+    // also check if already subscribed
+    for (const publication of this.linkedParticipant.trackPublications.values()) {
+        if (publication.source === TrackSource.SOURCE_MICROPHONE && publication.track) {
+            __classPrivateFieldGet(this, _MultimodalAgent_instances, "m", _MultimodalAgent_handleTrackSubscription).call(this, publication.track, publication, this.linkedParticipant);
+            break;
+        }
     }
 }, _MultimodalAgent_subscribeToMicrophone = function _MultimodalAgent_subscribeToMicrophone() {
+    if (!this.linkedParticipant) {
+        __classPrivateFieldGet(this, _MultimodalAgent_logger, "f").error('Participant is not set');
+        return;
+    }
+    let microphonePublication = undefined;
+    for (const publication of this.linkedParticipant.trackPublications.values()) {
+        if (publication.source === TrackSource.SOURCE_MICROPHONE) {
+            microphonePublication = publication;
+            break;
+        }
+    }
+    if (!microphonePublication) {
+        return;
+    }
+    if (!microphonePublication.subscribed) {
+        microphonePublication.setSubscribed(true);
+    }
+}, _MultimodalAgent_handleTrackSubscription = function _MultimodalAgent_handleTrackSubscription(track, publication, participant) {
+    var _a;
+    if (publication.source !== TrackSource.SOURCE_MICROPHONE ||
+        participant.identity !== ((_a = this.linkedParticipant) === null || _a === void 0 ? void 0 : _a.identity)) {
+        return;
+    }
     const readAudioStreamTask = async (audioStream) => {
@@ -255,33 +286,18 @@ var _a, e_1, _b, _c;
     };
-    if (!this.linkedParticipant) {
-        __classPrivateFieldGet(this, _MultimodalAgent_logger, "f").error('Participant is not set');
-        return;
+    this.subscribedTrack = track;
+    if (this.readMicroTask) {
+        this.readMicroTask.cancel();
     }
-    for (const publication of this.linkedParticipant.trackPublications.values()) {
-        if (publication.source !== TrackSource.SOURCE_MICROPHONE) {
-            continue;
-        }
-        if (!publication.subscribed) {
-            publication.setSubscribed(true);
-        }
-        const track = publication.track;
-        if (track && track !== this.subscribedTrack) {
-            this.subscribedTrack = track;
-            if (this.readMicroTask) {
-                this.readMicroTask.cancel();
-            }
-            let cancel;
-            this.readMicroTask = {
-                promise: new Promise((resolve, reject) => {
-                    cancel = () => {
-                        reject(new Error('Task cancelled'));
-                    };
-                    readAudioStreamTask(new AudioStream(track, this.model.sampleRate, this.model.numChannels))
-                        .then(resolve)
-                        .catch(reject);
-                }),
-                cancel: () => cancel(),
+    let cancel;
+    this.readMicroTask = {
+        promise: new Promise((resolve, reject) => {
+            cancel = () => {
+                reject(new Error('Task cancelled'));
             };
-        }
-    }
+            readAudioStreamTask(new AudioStream(track, this.model.sampleRate, this.model.numChannels))
+                .then(resolve)
+                .catch(reject);
+        }),
+        cancel: () => cancel(),
+    };
 }, _MultimodalAgent_getLocalTrackSid = function _MultimodalAgent_getLocalTrackSid() {
@@ -288,0 +304,0 @@ var _a;

dist/utils.js

@@ -14,3 +14,3 @@ var __classPrivateFieldSet = (this && this.__classPrivateFieldSet) || function (receiver, state, value, kind, f) {
 import { AudioFrame, TrackSource } from '@livekit/rtc-node';
-import { EventEmitter, once } from 'events';
+import { EventEmitter, once } from 'node:events';
 /**
@@ -17,0 +17,0 @@ * Merge one or more {@link AudioFrame}s into a single one.

dist/worker.d.ts

@@ -1,5 +0,5 @@
-/// <reference types="node" />
+/// <reference types="node" resolution-mode="require"/>
 import type { TrackSource } from '@livekit/protocol';
 import { JobType } from '@livekit/protocol';
-import { EventEmitter } from 'events';
+import { EventEmitter } from 'node:events';
 import type { JobProcess, RunningJobInfo } from './job.js';
@@ -6,0 +6,0 @@ import { JobRequest } from './job.js';

dist/worker.js

@@ -14,5 +14,5 @@ var __classPrivateFieldSet = (this && this.__classPrivateFieldSet) || function (receiver, state, value, kind, f) {
 import { JobType, ParticipantPermission, ServerMessage, WorkerMessage, WorkerStatus, } from '@livekit/protocol';
-import { EventEmitter } from 'events';
 import { AccessToken, RoomServiceClient } from 'livekit-server-sdk';
-import os from 'os';
+import { EventEmitter } from 'node:events';
+import os from 'node:os';
 import { WebSocket } from 'ws';
@@ -19,0 +19,0 @@ import { HTTPServer } from './http_server.js';

package.json

 {
   "name": "@livekit/agents",
-  "version": "0.3.2",
+  "version": "0.3.3",
   "description": "LiveKit Agents - Node.js",
@@ -5,0 +5,0 @@ "main": "dist/index.js",

src/cli.ts

@@ -5,3 +5,3 @@ // SPDX-FileCopyrightText: 2024 LiveKit, Inc.
 import { Command, Option } from 'commander';
-import type { EventEmitter } from 'events';
+import type { EventEmitter } from 'node:events';
 import { initializeLogger, log } from './log.js';
@@ -8,0 +8,0 @@ import { version } from './version.js';

src/http_server.ts

 // SPDX-FileCopyrightText: 2024 LiveKit, Inc.
 //
 // SPDX-License-Identifier: Apache-2.0
-import { type IncomingMessage, type Server, type ServerResponse, createServer } from 'http';
+import { type IncomingMessage, type Server, type ServerResponse, createServer } from 'node:http';
 import { log } from './log.js';
@@ -6,0 +6,0 @@

src/ipc/job_main.ts

@@ -5,7 +5,7 @@ // SPDX-FileCopyrightText: 2024 LiveKit, Inc.
 import { Room, RoomEvent } from '@livekit/rtc-node';
-import type { ChildProcess } from 'child_process';
-import { fork } from 'child_process';
-import { EventEmitter, once } from 'events';
+import type { ChildProcess } from 'node:child_process';
+import { fork } from 'node:child_process';
+import { EventEmitter, once } from 'node:events';
+import { pathToFileURL } from 'node:url';
 import type { Logger } from 'pino';
-import { fileURLToPath } from 'url';
 import { type Agent, isAgent } from '../generator.js';
@@ -32,3 +32,3 @@ import type { RunningJobInfo } from '../job.js';
 export const runProcess = (args: StartArgs): ChildProcess => {
-  return fork(fileURLToPath(import.meta.url), [args.agentFile]);
+  return fork(new URL(import.meta.url), [args.agentFile]);
 };
@@ -99,3 +99,3 @@
   const moduleFile = process.argv[2];
-  const agent: Agent = await import(moduleFile).then((module) => {
+  const agent: Agent = await import(pathToFileURL(moduleFile).href).then((module) => {
     const agent = module.default;
@@ -102,0 +102,0 @@ if (agent === undefined || !isAgent(agent)) {

src/ipc/proc_job_executor.ts

 // SPDX-FileCopyrightText: 2024 LiveKit, Inc.
 //
 // SPDX-License-Identifier: Apache-2.0
-import type { ChildProcess } from 'child_process';
-import { once } from 'events';
+import type { ChildProcess } from 'node:child_process';
+import { once } from 'node:events';
 import type { RunningJobInfo } from '../job.js';
@@ -7,0 +7,0 @@ import { log, loggerOptions } from '../log.js';

src/multimodal/agent_playout.ts

@@ -6,3 +6,3 @@ // SPDX-FileCopyrightText: 2024 LiveKit, Inc.
 import { type AudioSource } from '@livekit/rtc-node';
-import { EventEmitter } from 'events';
+import { EventEmitter } from 'node:events';
 import { AudioByteStream } from '../audio.js';
@@ -9,0 +9,0 @@ import type { TranscriptionForwarder } from '../transcription.js';

src/multimodal/multimodal_agent.ts

@@ -8,2 +8,4 @@ // SPDX-FileCopyrightText: 2024 LiveKit, Inc.
   RemoteParticipant,
+  RemoteTrack,
+  RemoteTrackPublication,
   Room,
@@ -19,3 +21,3 @@ } from '@livekit/rtc-node';
 } from '@livekit/rtc-node';
-import { EventEmitter } from 'events';
+import { EventEmitter } from 'node:events';
 import { AudioByteStream } from '../audio.js';
@@ -140,8 +142,13 @@ import type * as llm from '../llm/index.js';
       room.on(RoomEvent.ParticipantConnected, (participant: RemoteParticipant) => {
-        if (!this.linkedParticipant) {
+        // automatically link to the first participant that connects, if not already linked
+        if (this.linkedParticipant) {
           return;
         }
-
         this.#linkParticipant(participant.identity);
       });
+      room.on(RoomEvent.TrackPublished, () => {
+        // in case we are connected before the participant has published, we'd need to re-subscribe
+        this.#subscribeToMicrophone();
+      });
+      room.on(RoomEvent.TrackSubscribed, this.#handleTrackSubscription.bind(this));
 
@@ -303,10 +310,46 @@ this.room = room;
       this.#subscribeToMicrophone();
-    } else {
-      this.room.on(RoomEvent.TrackPublished, () => {
-        this.#subscribeToMicrophone();
-      });
     }
+
+    // also check if already subscribed
+    for (const publication of this.linkedParticipant.trackPublications.values()) {
+      if (publication.source === TrackSource.SOURCE_MICROPHONE && publication.track) {
+        this.#handleTrackSubscription(publication.track, publication, this.linkedParticipant);
+        break;
+      }
+    }
   }
 
   #subscribeToMicrophone(): void {
+    if (!this.linkedParticipant) {
+      this.#logger.error('Participant is not set');
+      return;
+    }
+
+    let microphonePublication: RemoteTrackPublication | undefined = undefined;
+    for (const publication of this.linkedParticipant.trackPublications.values()) {
+      if (publication.source === TrackSource.SOURCE_MICROPHONE) {
+        microphonePublication = publication;
+        break;
+      }
+    }
+    if (!microphonePublication) {
+      return;
+    }
+
+    if (!microphonePublication.subscribed) {
+      microphonePublication.setSubscribed(true);
+    }
+  }
+
+  #handleTrackSubscription(
+    track: RemoteTrack,
+    publication: RemoteTrackPublication,
+    participant: RemoteParticipant,
+  ) {
+    if (
+      publication.source !== TrackSource.SOURCE_MICROPHONE ||
+      participant.identity !== this.linkedParticipant?.identity
+    ) {
+      return;
+    }
     const readAudioStreamTask = async (audioStream: AudioStream) => {
@@ -326,42 +369,20 @@ const bstream = new AudioByteStream(
     };
+    this.subscribedTrack = track;
 
-    if (!this.linkedParticipant) {
-      this.#logger.error('Participant is not set');
-      return;
+    if (this.readMicroTask) {
+      this.readMicroTask.cancel();
     }
 
-    for (const publication of this.linkedParticipant.trackPublications.values()) {
-      if (publication.source !== TrackSource.SOURCE_MICROPHONE) {
-        continue;
-      }
-
-      if (!publication.subscribed) {
-        publication.setSubscribed(true);
-      }
-
-      const track = publication.track;
-
-      if (track && track !== this.subscribedTrack) {
-        this.subscribedTrack = track;
-
-        if (this.readMicroTask) {
-          this.readMicroTask.cancel();
-        }
-
-        let cancel: () => void;
-        this.readMicroTask = {
-          promise: new Promise<void>((resolve, reject) => {
-            cancel = () => {
-              reject(new Error('Task cancelled'));
-            };
-            readAudioStreamTask(
-              new AudioStream(track, this.model.sampleRate, this.model.numChannels),
-            )
-              .then(resolve)
-              .catch(reject);
-          }),
-          cancel: () => cancel(),
+    let cancel: () => void;
+    this.readMicroTask = {
+      promise: new Promise<void>((resolve, reject) => {
+        cancel = () => {
+          reject(new Error('Task cancelled'));
         };
-      }
-    }
+        readAudioStreamTask(new AudioStream(track, this.model.sampleRate, this.model.numChannels))
+          .then(resolve)
+          .catch(reject);
+      }),
+      cancel: () => cancel(),
+    };
   }
@@ -368,0 +389,0 @@

src/utils.ts

@@ -11,3 +11,3 @@ // SPDX-FileCopyrightText: 2024 LiveKit, Inc.
 import { AudioFrame, TrackSource } from '@livekit/rtc-node';
-import { EventEmitter, once } from 'events';
+import { EventEmitter, once } from 'node:events';
 
@@ -14,0 +14,0 @@ /** Union of a single and a list of {@link AudioFrame}s */

src/worker.ts

@@ -18,5 +18,5 @@ // SPDX-FileCopyrightText: 2024 LiveKit, Inc.
 } from '@livekit/protocol';
-import { EventEmitter } from 'events';
 import { AccessToken, RoomServiceClient } from 'livekit-server-sdk';
-import os from 'os';
+import { EventEmitter } from 'node:events';
+import os from 'node:os';
 import { WebSocket } from 'ws';
@@ -23,0 +23,0 @@ import { HTTPServer } from './http_server.js';

Fixed alerts

Network access

Supply chain risk

This module accesses the network.

Found 1 instance in 1 package

Shell access

Supply chain risk

This module accesses the system shell. Accessing the system shell increases the risk of executing arbitrary code.

Found 1 instance in 1 package

Improved metrics

Total package byte prevSize

478212

increased by18.81%

Number of package files

157

increased by0.64%

Lines of code

6818

increased by0.58%

Number of medium supply chain risk alerts

0

decreased by-100%

No dependency changes