You're Invited:Meet the Socket Team at BlackHat and DEF CON in Las Vegas, Aug 4-6.RSVP
Socket
Book a DemoInstallSign in
Socket

@empiricalrun/llm

Package Overview
Dependencies
Maintainers
1
Versions
100
Alerts
File Explorer

Advanced tools

Socket logo

Install Socket

Detect and block malicious and high-risk dependencies

Install

@empiricalrun/llm - npm Package Compare versions

Comparing version

to
0.18.0

14

CHANGELOG.md
# @empiricalrun/llm
## 0.18.0
### Minor Changes
- 1b08d58: feat: tool response interface supports images for claude
### Patch Changes
- 82acf53: feat: enable tools to contribute to cost calculations
- 1177d63: chore: bump up gemini sdk to 0.13.0
- 7712b2e: chore: move more types to shared-types package
- fb32af6: feat: add tokens and cost to chat state
- 1b9087e: feat: improve feature flags ui, upgrade gemini-pro
## 0.17.3

@@ -4,0 +18,0 @@

12

dist/chat/claude/index.d.ts

@@ -6,9 +6,9 @@ import type { Anthropic } from "@anthropic-ai/sdk";

import type { IChatModel } from "../types";
export type AnthropicMessageTypeV2 = {
export type AnthropicMessageType = {
userMessage?: Anthropic.MessageParam;
assistantMessage?: Pick<Anthropic.Beta.BetaMessage, "content" | "role">;
assistantMessage?: Pick<Anthropic.Beta.BetaMessage, "content" | "role" | "usage">;
correspondingToolCallNames?: string[];
};
export declare class ClaudeChatModel implements IChatModel<AnthropicMessageTypeV2> {
messages: Array<AnthropicMessageTypeV2>;
export declare class ClaudeChatModel implements IChatModel<AnthropicMessageType> {
messages: Array<AnthropicMessageType>;
tokensUsedSoFar: {

@@ -19,3 +19,3 @@ input: number;

askUserForInput: boolean;
constructor(messages: AnthropicMessageTypeV2[]);
constructor(messages: AnthropicMessageType[]);
getLLMResponse({ systemPrompt, tools, selectedModel, trace, }: {

@@ -36,3 +36,3 @@ systemPrompt: string;

pushUserMessage(userPrompt: string): void;
pushMessage(response: AnthropicMessageTypeV2): void;
pushMessage(response: AnthropicMessageType): void;
getUsageSummary(): string;

@@ -39,0 +39,0 @@ getPendingToolCalls(): PendingToolCall[];

@@ -59,2 +59,3 @@ "use strict";

outputTokens: output_tokens,
model: selectedModel,
});

@@ -164,8 +165,41 @@ generation?.end({

pushToolResultsMessage(toolCalls, toolResults) {
const toolResultBlocks = toolCalls.map((call, index) => ({
type: "tool_result",
tool_use_id: call.id,
content: toolResults[index].result,
is_error: toolResults[index].isError,
}));
const toolResultBlocks = toolCalls.map((call, index) => {
const toolResult = toolResults[index];
if (!toolResult)
throw new Error("Tool result not found");
let content;
const result = toolResult.result;
if (typeof result === "string") {
content = result;
}
else {
content = result.map((r) => {
if (r.type === "text") {
return {
type: "text",
text: r.text,
};
}
else if (r.type === "image/png") {
return {
type: "image",
source: {
type: "base64",
data: r.base64Data,
media_type: "image/png",
},
};
}
else {
throw new Error("Invalid tool result type");
}
});
}
return {
type: "tool_result",
tool_use_id: call.id,
content,
is_error: toolResult.isError,
};
});
const message = {

@@ -172,0 +206,0 @@ role: "user",

import Anthropic from "@anthropic-ai/sdk";
export type Models = "claude-3-7-sonnet-20250219" | "claude-3-5-sonnet-20241022" | "claude-sonnet-4-20250514" | "claude-opus-4-20250514";
export type ClaudeModels = "claude-3-7-sonnet-20250219" | "claude-3-5-sonnet-20241022" | "claude-sonnet-4-20250514" | "claude-opus-4-20250514";
export declare function createClaudeMessage({ systemPrompt, messages, tools, model, withStrReplaceEditor, }: {

@@ -7,3 +7,3 @@ systemPrompt: string;

tools: Array<Anthropic.Beta.BetaToolUnion>;
model: Models;
model: ClaudeModels;
withStrReplaceEditor?: boolean;

@@ -13,5 +13,6 @@ }): Promise<(Anthropic.Beta.Messages.BetaMessage & {

}) | undefined>;
export declare function calculateTokenCosts({ inputTokens, outputTokens, }: {
export declare function calculateTokenCosts({ inputTokens, outputTokens, model, }: {
inputTokens: number;
outputTokens: number;
model: ClaudeModels;
}): {

@@ -18,0 +19,0 @@ input: number;

@@ -77,6 +77,10 @@ "use strict";

}
function calculateTokenCosts({ inputTokens, outputTokens, }) {
function calculateTokenCosts({ inputTokens, outputTokens, model, }) {
// Pricing: https://www.anthropic.com/pricing#api
const inputRateUSDPerMillion = 3.0;
const outputRateUSDPerMillion = 15.0;
let inputRateUSDPerMillion = 3.0;
let outputRateUSDPerMillion = 15.0;
if (model.includes("opus")) {
inputRateUSDPerMillion = 15.0;
outputRateUSDPerMillion = 75.0;
}
const inputCostUSD = (inputTokens * inputRateUSDPerMillion) / 1_000_000;

@@ -83,0 +87,0 @@ const outputCostUSD = (outputTokens * outputRateUSDPerMillion) / 1_000_000;

@@ -17,2 +17,3 @@ import { PendingToolCall, ToolResult } from "@empiricalrun/shared-types";

tools: Array<OpenAI.Chat.Completions.ChatCompletionTool>;
selectedModel: string;
trace: TraceClient | undefined;

@@ -19,0 +20,0 @@ }): Promise<GeminiMessageType | undefined>;

@@ -30,7 +30,8 @@ "use strict";

async getLLMResponse(params) {
const { systemPrompt, tools, trace } = params;
const { systemPrompt, tools, selectedModel, trace } = params;
const model = selectedModel;
const generation = trace?.generation({
name: "chat_agent",
input: this.messages[this.messages.length - 1],
model: "gemini-2.5-pro-preview-03-25",
model,
});

@@ -42,3 +43,3 @@ try {

const chat = genAI.chats.create({
model: "gemini-2.5-pro-preview-03-25",
model,
config: {

@@ -45,0 +46,0 @@ systemInstruction: systemPrompt,

export { convertOpenAISchemaToAnthropic, zodToOpenAITool } from "./tools";
export { getProviderForModel } from "./tools";
export { claudeToCanonical, geminiToCanonical, openaiToCanonical, } from "./transform";
export type { IChatModel, SupportedChatModels } from "./types";
export type { IChatModel } from "./types";
export type { CanonicalMessage, PendingToolCall, ToolResult, } from "@empiricalrun/shared-types";
import { CanonicalMessage } from "@empiricalrun/shared-types";
import { CanonicalMessage, SupportedChatModels } from "@empiricalrun/shared-types";
export { SUPPORTED_CHAT_MODELS } from "./types";
import type { IChatModel, SupportedChatModels } from "./types";
import type { IChatModel } from "./types";
export declare function createChatModel(messages: CanonicalMessage[], selectedModel: SupportedChatModels): IChatModel<CanonicalMessage>;
//# sourceMappingURL=index.d.ts.map

@@ -19,5 +19,5 @@ "use strict";

const modelMap = {
claude: (messages) => new transform_2.ClaudeCanonicalChatModel(messages),
google: (messages) => new transform_2.GeminiCanonicalChatModel(messages),
openai: (messages) => new transform_2.OpenAICanonicalChatModel(messages),
claude: (model, messages) => new transform_2.ClaudeCanonicalChatModel(model, messages),
google: (model, messages) => new transform_2.GeminiCanonicalChatModel(model, messages),
openai: (model, messages) => new transform_2.OpenAICanonicalChatModel(model, messages),
};

@@ -33,3 +33,3 @@ function createChatModel(messages, selectedModel) {

}
return fn(messages);
return fn(selectedModel, messages);
}

@@ -158,2 +158,5 @@ "use strict";

}
if (Array.isArray(result.result)) {
throw new Error(`Array type is unsupported in tool result [toolCallId: ${call.id}]`);
}
return {

@@ -160,0 +163,0 @@ call_id: call.id,

import Anthropic from "@anthropic-ai/sdk";
import { SupportedChatModels } from "@empiricalrun/shared-types";
import { FunctionDeclaration } from "@google/genai";
import OpenAI from "openai";
import { SupportedChatModels } from "../types";
export declare function convertOpenAISchemaToAnthropic(openaiSchema: OpenAI.Chat.Completions.ChatCompletionTool): Anthropic.Tool;

@@ -6,0 +6,0 @@ export { zodToOpenAITool } from "./zod-schema";

@@ -1,9 +0,10 @@

import { CanonicalMessage, PendingToolCall, ToolResult } from "@empiricalrun/shared-types";
import { CanonicalMessage, PendingToolCall, SupportedChatModels, ToolResult } from "@empiricalrun/shared-types";
import { IChatModel, LLMResponseArgs } from "../types";
export declare class BaseCanonicalChatModel<T> {
private readonly internalModel;
private readonly canonicalToInternal;
private readonly internalToCanonical;
model: SupportedChatModels;
messages: CanonicalMessage[];
constructor(messages: CanonicalMessage[], internalModel: IChatModel<T>, canonicalToInternal: (msg: CanonicalMessage) => T, internalToCanonical: (msg: T, msgArray: CanonicalMessage[]) => CanonicalMessage);
constructor(model: SupportedChatModels, messages: CanonicalMessage[], internalModel: IChatModel<T>);
canonicalToInternal(msg: CanonicalMessage): T;
internalToCanonical(msg: T): CanonicalMessage;
get askUserForInput(): boolean;

@@ -16,9 +17,9 @@ getUsageSummary(): string;

getLLMResponse(params: LLMResponseArgs): Promise<CanonicalMessage | undefined>;
private updateMessagesWhilePreservingArtifacts;
private updateMessagesWhilePreservingToolResults;
private updateMyMessages;
pushMessage(message: CanonicalMessage): void;
pushUserMessage(userPrompt: string): void;
pushToolResultsMessage(toolUse: PendingToolCall[], toolResult: ToolResult[]): void;
pushToolResultsMessage(toolCalls: PendingToolCall[], toolResults: ToolResult[]): void;
getPendingToolCalls(): PendingToolCall[];
}
//# sourceMappingURL=base.d.ts.map

@@ -6,11 +6,15 @@ "use strict";

internalModel;
canonicalToInternal;
internalToCanonical;
model;
messages;
constructor(messages, internalModel, canonicalToInternal, internalToCanonical) {
constructor(model, messages, internalModel) {
this.internalModel = internalModel;
this.canonicalToInternal = canonicalToInternal;
this.internalToCanonical = internalToCanonical;
this.model = model;
this.messages = messages;
}
canonicalToInternal(msg) {
throw new Error("Not implemented", { cause: msg });
}
internalToCanonical(msg) {
throw new Error("Not implemented", { cause: msg });
}
get askUserForInput() {

@@ -30,13 +34,29 @@ return this.internalModel.askUserForInput;

}
const canonical = this.internalToCanonical(response, this.messages);
const canonical = this.internalToCanonical(response);
return canonical;
}
updateMessagesWhilePreservingArtifacts(toolCall, toolResult) {
updateMessagesWhilePreservingToolResults(toolCalls, toolResults) {
const latestMessage = this.internalModel.messages[this.internalModel.messages.length - 1];
const m = this.internalToCanonical(latestMessage, this.messages);
if (toolResult.length && toolCall && m.role === "tool") {
m.parts = toolResult.map((toolResult, index) => ({
const m = this.internalToCanonical(latestMessage);
let hasAnyUsageInfo = false;
let totalUsage = {
tokens: { input: 0, output: 0 },
cost: { input: 0, output: 0 },
};
if (toolResults.length && toolCalls && m.role === "tool") {
// For tool results, we overwrite with the original tool result which has artifacts
m.parts = toolResults.map((toolResult, index) => ({
...m.parts[index],
toolResult,
}));
// We also add the usage info to the total usage, to contribute to cost calculations
toolResults.forEach((toolResult) => {
if (toolResult.usage) {
hasAnyUsageInfo = true;
totalUsage.tokens.input += toolResult.usage.tokens.input;
totalUsage.tokens.output += toolResult.usage.tokens.output;
totalUsage.cost.input += toolResult.usage.cost.input;
totalUsage.cost.output += toolResult.usage.cost.output;
}
});
}

@@ -46,2 +66,3 @@ this.messages.push({

id: this.messages.length + 1,
usage: hasAnyUsageInfo ? totalUsage : undefined,
});

@@ -51,3 +72,3 @@ }

const latestMessage = this.internalModel.messages[this.internalModel.messages.length - 1];
const m = this.internalToCanonical(latestMessage, this.messages);
const m = this.internalToCanonical(latestMessage);
this.messages.push({

@@ -67,5 +88,5 @@ ...m,

}
pushToolResultsMessage(toolUse, toolResult) {
this.internalModel.pushToolResultsMessage(toolUse, toolResult);
this.updateMessagesWhilePreservingArtifacts(toolUse, toolResult);
pushToolResultsMessage(toolCalls, toolResults) {
this.internalModel.pushToolResultsMessage(toolCalls, toolResults);
this.updateMessagesWhilePreservingToolResults(toolCalls, toolResults);
}

@@ -72,0 +93,0 @@ getPendingToolCalls() {

@@ -1,9 +0,12 @@

import { CanonicalMessage } from "@empiricalrun/shared-types";
import type { AnthropicMessageTypeV2 } from "../claude";
import { CanonicalMessage, SupportedChatModels } from "@empiricalrun/shared-types";
import type { AnthropicMessageType } from "../claude";
import { ClaudeModels } from "../claude/utils";
import { BaseCanonicalChatModel } from "./base";
export declare class ClaudeCanonicalChatModel extends BaseCanonicalChatModel<AnthropicMessageTypeV2> {
constructor(messages: CanonicalMessage[]);
export declare class ClaudeCanonicalChatModel extends BaseCanonicalChatModel<AnthropicMessageType> {
constructor(model: SupportedChatModels, messages: CanonicalMessage[]);
canonicalToInternal(msg: CanonicalMessage): AnthropicMessageType;
internalToCanonical(msg: AnthropicMessageType): CanonicalMessage;
}
export declare function claudeToCanonical(msg: AnthropicMessageTypeV2): CanonicalMessage;
export declare function canonicalToClaude(msg: CanonicalMessage): AnthropicMessageTypeV2;
export declare function claudeToCanonical(model: ClaudeModels, msg: AnthropicMessageType): CanonicalMessage;
export declare function canonicalToClaude(msg: CanonicalMessage): AnthropicMessageType;
//# sourceMappingURL=claude.d.ts.map

@@ -7,13 +7,20 @@ "use strict";

const claude_1 = require("../claude");
const utils_1 = require("../claude/utils");
const base_1 = require("./base");
function isTextBlock(block) {
return block && block.type === "text" && typeof block.text === "string";
}
class ClaudeCanonicalChatModel extends base_1.BaseCanonicalChatModel {
constructor(messages) {
super(messages, new claude_1.ClaudeChatModel(messages.map(canonicalToClaude)), canonicalToClaude, claudeToCanonical);
constructor(model, messages) {
super(model, messages, new claude_1.ClaudeChatModel(messages.map(canonicalToClaude)));
}
canonicalToInternal(msg) {
return canonicalToClaude(msg);
}
internalToCanonical(msg) {
return claudeToCanonical(this.model, msg);
}
}
exports.ClaudeCanonicalChatModel = ClaudeCanonicalChatModel;
function isTextBlock(block) {
return block && block.type === "text" && typeof block.text === "string";
}
function claudeToCanonical(msg) {
function claudeToCanonical(model, msg) {
let timestamp = new Date().toISOString();

@@ -31,9 +38,38 @@ if (msg.userMessage) {

if (b.type === "tool_result") {
if (!b.content) {
throw new Error("Invalid message type in claudeToCanonical");
}
let toolResult;
if (Array.isArray(b.content)) {
toolResult = {
isError: b.is_error ?? false,
result: b.content.map((c) => {
if (c.type === "text") {
return { type: "text", text: c.text };
}
else if (c.type === "image") {
return {
type: "image/png",
base64Data: c.source.data,
};
}
else {
throw new Error("Invalid message type in claudeToCanonical");
}
}),
};
}
else if (typeof b.content === "string") {
toolResult = {
isError: b.is_error ?? false,
result: b.content,
};
}
else {
throw new Error("Invalid message type in claudeToCanonical");
}
return {
toolCallId: b.tool_use_id,
toolName: correspondingToolCallNames[idx],
toolResult: {
isError: b.is_error ?? false,
result: b.content,
},
toolResult,
};

@@ -87,2 +123,8 @@ }

}
const tokens = msg.assistantMessage.usage;
const cost = (0, utils_1.calculateTokenCosts)({
model,
inputTokens: tokens.input_tokens,
outputTokens: tokens.output_tokens,
});
return {

@@ -93,2 +135,6 @@ id: 1,

parts,
usage: {
tokens: { input: tokens.input_tokens, output: tokens.output_tokens },
cost: { input: cost.input, output: cost.output },
},
};

@@ -130,2 +176,11 @@ }

}),
usage: {
input_tokens: msg.usage?.tokens.input ?? 0,
output_tokens: msg.usage?.tokens.output ?? 0,
cache_creation: null,
cache_creation_input_tokens: 0,
cache_read_input_tokens: 0,
server_tool_use: null,
service_tier: "standard",
},
},

@@ -140,8 +195,3 @@ };

.filter((p) => "toolResult" in p)
.map((p) => ({
type: "tool_result",
tool_use_id: p.toolCallId,
content: p.toolResult.result,
is_error: p.toolResult.isError,
})),
.map((p) => convertCanonicalToolResultToClaude(p)),
},

@@ -157,1 +207,37 @@ correspondingToolCallNames: msg.parts

}
function convertCanonicalToolResultToClaude(toolResultPart) {
const toolResult = toolResultPart.toolResult;
if (Array.isArray(toolResult.result)) {
return {
type: "tool_result",
tool_use_id: toolResultPart.toolCallId,
is_error: toolResult.isError,
content: toolResult.result.map((r) => {
if (r.type === "text") {
return { type: "text", text: r.text };
}
else if (r.type === "image/png") {
return {
type: "image",
source: {
type: "base64",
data: r.base64Data,
media_type: "image/png",
},
};
}
else {
throw new Error("Invalid message type in convertCanonicalToolResultToClaude");
}
}),
};
}
else {
return {
type: "tool_result",
tool_use_id: toolResultPart.toolCallId,
content: toolResult.result,
is_error: toolResult.isError,
};
}
}

@@ -1,6 +0,8 @@

import { CanonicalMessage } from "@empiricalrun/shared-types";
import { CanonicalMessage, SupportedChatModels } from "@empiricalrun/shared-types";
import { GeminiMessageType } from "../gemini/utils";
import { BaseCanonicalChatModel } from "./base";
export declare class GeminiCanonicalChatModel extends BaseCanonicalChatModel<GeminiMessageType> {
constructor(messages: CanonicalMessage[]);
constructor(model: SupportedChatModels, messages: CanonicalMessage[]);
canonicalToInternal(msg: CanonicalMessage): GeminiMessageType;
internalToCanonical(msg: GeminiMessageType): CanonicalMessage;
}

@@ -7,0 +9,0 @@ export declare function geminiToCanonical(msg: GeminiMessageType): CanonicalMessage;

@@ -9,5 +9,11 @@ "use strict";

class GeminiCanonicalChatModel extends base_1.BaseCanonicalChatModel {
constructor(messages) {
super(messages, new gemini_1.GeminiChatModel(messages.map(canonicalToGemini)), canonicalToGemini, geminiToCanonical);
constructor(model, messages) {
super(model, messages, new gemini_1.GeminiChatModel(messages.map(canonicalToGemini)));
}
canonicalToInternal(msg) {
return canonicalToGemini(msg);
}
internalToCanonical(msg) {
return geminiToCanonical(msg);
}
}

@@ -14,0 +20,0 @@ exports.GeminiCanonicalChatModel = GeminiCanonicalChatModel;

@@ -1,6 +0,8 @@

import { CanonicalMessage } from "@empiricalrun/shared-types";
import { CanonicalMessage, SupportedChatModels } from "@empiricalrun/shared-types";
import { OpenAIMessageType } from "../openai";
import { BaseCanonicalChatModel } from "./base";
export declare class OpenAICanonicalChatModel extends BaseCanonicalChatModel<OpenAIMessageType> {
constructor(messages: CanonicalMessage[]);
constructor(model: SupportedChatModels, messages: CanonicalMessage[]);
canonicalToInternal(msg: CanonicalMessage): OpenAIMessageType;
internalToCanonical(msg: OpenAIMessageType): CanonicalMessage;
}

@@ -7,0 +9,0 @@ export declare function openaiToCanonical(msg: OpenAIMessageType): CanonicalMessage;

@@ -9,5 +9,11 @@ "use strict";

class OpenAICanonicalChatModel extends base_1.BaseCanonicalChatModel {
constructor(messages) {
super(messages, new openai_1.OpenAIChatModel(messages.map(canonicalToOpenAI)), canonicalToOpenAI, openaiToCanonical);
constructor(model, messages) {
super(model, messages, new openai_1.OpenAIChatModel(messages.map(canonicalToOpenAI)));
}
canonicalToInternal(msg) {
return canonicalToOpenAI(msg);
}
internalToCanonical(msg) {
return openaiToCanonical(msg);
}
}

@@ -143,2 +149,5 @@ exports.OpenAICanonicalChatModel = OpenAICanonicalChatModel;

const output = isError ? `Error: ${result}` : result;
if (Array.isArray(output)) {
throw new Error(`Array type is unsupported in tool result [toolCallId: ${part.toolCallId}]`);
}
return {

@@ -145,0 +154,0 @@ call_id: part.toolCallId,

@@ -25,3 +25,2 @@ import { ModelInfo, PendingToolCall, ToolResult } from "@empiricalrun/shared-types";

export declare const SUPPORTED_CHAT_MODELS: readonly ModelInfo[];
export type SupportedChatModels = (typeof SUPPORTED_CHAT_MODELS)[number]["id"];
//# sourceMappingURL=types.d.ts.map
"use strict";
Object.defineProperty(exports, "__esModule", { value: true });
exports.SUPPORTED_CHAT_MODELS = void 0;
// TODO: This needs to be kept in sync with the SupportedChatModels type
// in shared-types/chat-agent.ts
exports.SUPPORTED_CHAT_MODELS = [
{
id: "gemini-2.5-pro-preview-03-25",
id: "gemini-2.5-pro-preview-06-05",
label: "Gemini 2.5 Pro",

@@ -8,0 +10,0 @@ provider: "google",

{
"name": "@empiricalrun/llm",
"version": "0.17.3",
"version": "0.18.0",
"main": "dist/index.js",

@@ -38,3 +38,3 @@ "exports": {

"@anthropic-ai/sdk": "^0.52.0",
"@google/genai": "^0.9.0",
"@google/genai": "^0.13.0",
"async-retry": "^1.3.3",

@@ -52,3 +52,3 @@ "handlebars": "^4.7.8",

"@types/async-retry": "^1.4.8",
"@empiricalrun/shared-types": "0.4.1"
"@empiricalrun/shared-types": "0.5.0"
},

@@ -55,0 +55,0 @@ "scripts": {

Sorry, the diff of this file is not supported yet

Sorry, the diff of this file is not supported yet

Sorry, the diff of this file is not supported yet

Sorry, the diff of this file is not supported yet

Sorry, the diff of this file is not supported yet

Sorry, the diff of this file is not supported yet

Sorry, the diff of this file is not supported yet

Sorry, the diff of this file is not supported yet

Sorry, the diff of this file is not supported yet

Sorry, the diff of this file is not supported yet

Sorry, the diff of this file is not supported yet