@empiricalrun/llm - npm Package Compare versions

Comparing version 0.4.2 to 0.4.3

CHANGELOG.md

 # @empiricalrun/llm
 
+## 0.4.3
+
+### Patch Changes
+
+- 3c714f7: feat: add master agent support
+- d1a1ff2: fix: restrict usage of llm basis token usage
+
 ## 0.4.2
@@ -4,0 +11,0 @@

dist/index.d.ts

@@ -7,12 +7,26 @@ import { LangfuseGenerationClient, LangfuseSpanClient, LangfuseTraceClient } from "langfuse";
 type TraceClient = LangfuseTraceClient | LangfuseSpanClient;
-export declare function getLLMResult({ traceName, messages, trace, tools, provider, providerApiKey, model, modelParameters, }: {
-    traceName?: string;
-    messages: OpenAI.Chat.Completions.ChatCompletionMessageParam[];
-    trace?: TraceClient;
-    tools?: OpenAI.Chat.Completions.ChatCompletionTool[];
-    provider: LLMProvider;
-    providerApiKey?: string;
-    model: LLMModel;
-    modelParameters?: LangfuseGenerationClient["generation"]["arguments"]["modelParameters"];
-}): Promise<OpenAI.Chat.Completions.ChatCompletionMessage | undefined>;
+export declare class LLM {
+    private _trace;
+    private _provider;
+    private _providerApiKey;
+    private _traceName;
+    private _usedTokens;
+    private _defaultModel;
+    private _maxTokens;
+    constructor({ trace, provider, providerApiKey, traceName, maxTokens, defaultModel, }: {
+        trace: TraceClient;
+        traceName?: string;
+        provider: LLMProvider;
+        providerApiKey?: string;
+        maxTokens?: number;
+        defaultModel?: LLMModel;
+    });
+    createChatCompletion({ messages, modelParameters, model, tools, trace, }: {
+        tools?: OpenAI.Chat.Completions.ChatCompletionTool[];
+        messages: OpenAI.Chat.Completions.ChatCompletionMessageParam[];
+        modelParameters?: LangfuseGenerationClient["generation"]["arguments"]["modelParameters"];
+        model?: LLMModel;
+        trace?: TraceClient;
+    }): Promise<OpenAI.Chat.Completions.ChatCompletionMessage | undefined>;
+}
 export type { TraceClient };
@@ -19,0 +33,0 @@ export { flushAllTraces, getPrompt, langfuseInstance };

dist/index.js

@@ -6,3 +6,3 @@ "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.langfuseInstance = exports.getPrompt = exports.flushAllTraces = exports.getLLMResult = void 0;
+exports.langfuseInstance = exports.getPrompt = exports.flushAllTraces = exports.LLM = void 0;
 const async_retry_1 = __importDefault(require("async-retry"));
@@ -16,34 +16,60 @@ const openai_1 = __importDefault(require("openai"));
 Object.defineProperty(exports, "langfuseInstance", { enumerable: true, get: function () { return trace_1.langfuseInstance; } });
-async function getLLMResult({ traceName = "get-llm-result", messages, trace, tools, provider, providerApiKey, model, modelParameters, }) {
-    const openai = new openai_1.default({
-        apiKey: providerApiKey,
-        baseURL: "https://ai.empirical.run/v1/",
-        defaultHeaders: (0, portkey_ai_1.createHeaders)({
-            provider,
-        }),
-    });
-    const generation = trace?.generation({
-        name: traceName,
-        model,
-        modelParameters: modelParameters || {},
-        input: messages,
-    });
-    const completion = await (0, async_retry_1.default)(async () => await openai.chat.completions.create({
-        messages,
-        model,
-        tools,
-        ...modelParameters,
-        max_tokens: 1000,
-        stream: false,
-    }), {
-        retries: 5,
-        factor: 3,
-        minTimeout: 1000,
-        maxTimeout: 60000,
-        randomize: true,
-    });
-    const output = completion.choices[0]?.message;
-    generation?.end({ output });
-    return output;
+class LLM {
+    _trace;
+    _provider;
+    _providerApiKey;
+    _traceName;
+    _usedTokens = 0;
+    _defaultModel;
+    _maxTokens; // limit the max tokens to avoid infinite or high number of roundtrips to llm
+    constructor({ trace, provider, providerApiKey, traceName = "get-llm-result", maxTokens, defaultModel, }) {
+        this._trace = trace;
+        this._provider = provider;
+        this._providerApiKey = providerApiKey;
+        this._traceName = traceName;
+        this._maxTokens = maxTokens ?? 1000000;
+        this._defaultModel = defaultModel;
+    }
+    async createChatCompletion({ messages, modelParameters, model, tools, trace, }) {
+        if (this._usedTokens >= this._maxTokens) {
+            throw new Error(`Exceeded max tokens limit of ${this._maxTokens} tokens. Please try again later.`);
+        }
+        const openai = new openai_1.default({
+            apiKey: this._providerApiKey,
+            baseURL: "https://ai.empirical.run/v1/",
+            defaultHeaders: (0, portkey_ai_1.createHeaders)({
+                provider: this._provider,
+            }),
+        });
+        // if model is not provided, use the default model
+        model = model || this._defaultModel;
+        const generation = (trace || this._trace)?.generation({
+            name: this._traceName,
+            model,
+            modelParameters,
+            input: messages,
+        });
+        const completion = await (0, async_retry_1.default)(async () => await openai.chat.completions.create({
+            messages,
+            model,
+            tools,
+            ...modelParameters,
+            max_tokens: 1000,
+            stream: false,
+        }), {
+            retries: 5,
+            factor: 3,
+            minTimeout: 1000,
+            maxTimeout: 60000,
+            randomize: true,
+        });
+        const output = completion.choices[0]?.message;
+        generation?.end({ output });
+        if (!completion.usage?.total_tokens) {
+            console.warn("No usage.total_tokens in completion");
+        }
+        this._usedTokens += completion.usage?.total_tokens || 0;
+        return output;
+    }
 }
-exports.getLLMResult = getLLMResult;
+exports.LLM = LLM;

dist/prompts/provider/index.d.ts

 import OpenAI from "openai";
-export declare function getPrompt(name: string, vars: any): Promise<OpenAI.Chat.Completions.ChatCompletionMessageParam[]>;
+export declare function getPrompt(name: string, vars: any, version?: number | undefined): Promise<OpenAI.Chat.Completions.ChatCompletionMessageParam[]>;
 //# sourceMappingURL=index.d.ts.map

dist/prompts/provider/index.js

@@ -11,4 +11,4 @@ "use strict";
 });
-async function getPrompt(name, vars) {
-    const prompt = await langfuse.getPrompt(name, undefined, { type: "chat" });
+async function getPrompt(name, vars, version) {
+    const prompt = await langfuse.getPrompt(name, version, { type: "chat" });
     const compiledPrompt = prompt.compile(vars);
@@ -15,0 +15,0 @@ return compiledPrompt;

package.json

 {
   "name": "@empiricalrun/llm",
-  "version": "0.4.2",
+  "version": "0.4.3",
   "main": "dist/index.js",
@@ -5,0 +5,0 @@ "publishConfig": {

No alert changes

Improved metrics

Total package byte prevSize

12295

increased by19.59%

Lines of code

184

increased by27.78%

No dependency changes