llm-api - npm Package Compare versions

Comparing version 1.3.5 to 1.4.0

dist/src/models/groq.d.ts

+import { Groq } from 'groq-sdk';
+import { ChatRequestMessage, ModelConfig, ModelRequestOptions, ChatResponse, GroqConfig } from '../types';
+import { CompletionApi } from './interface';
+import { getTikTokenTokensFromPrompt } from './tokenizer';
+export declare class GroqChatApi implements CompletionApi {
+    client: Groq;
+    modelConfig: ModelConfig;
+    constructor(config?: GroqConfig, modelConfig?: ModelConfig);
+    getTokensFromPrompt: typeof getTikTokenTokensFromPrompt;
+    chatCompletion(initialMessages: ChatRequestMessage[], requestOptions?: ModelRequestOptions | undefined): Promise<ChatResponse>;
+    textCompletion(prompt: string, requestOptions?: Partial<ModelRequestOptions>): Promise<ChatResponse>;
+}
+//# sourceMappingURL=groq.d.ts.map

dist/src/models/groq.d.ts.map

+{"version":3,"file":"groq.d.ts","sourceRoot":"","sources":["../../../src/models/groq.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,IAAI,EAAE,MAAM,UAAU,CAAC;AAWhC,OAAO,EACL,kBAAkB,EAClB,WAAW,EACX,mBAAmB,EACnB,YAAY,EACZ,UAAU,EACX,MAAM,UAAU,CAAC;AAIlB,OAAO,EAAE,aAAa,EAAE,MAAM,aAAa,CAAC;AAC5C,OAAO,EAAE,2BAA2B,EAAE,MAAM,aAAa,CAAC;AAS1D,qBAAa,WAAY,YAAW,aAAa;IAC/C,MAAM,EAAE,IAAI,CAAC;IACb,WAAW,EAAE,WAAW,CAAC;gBAEb,MAAM,CAAC,EAAE,UAAU,EAAE,WAAW,CAAC,EAAE,WAAW;IAK1D,mBAAmB,qCAA+B;IAI5C,cAAc,CAClB,eAAe,EAAE,kBAAkB,EAAE,EACrC,cAAc,CAAC,EAAE,mBAAmB,GAAG,SAAS,GAC/C,OAAO,CAAC,YAAY,CAAC;IA+HxB,cAAc,CACZ,MAAM,EAAE,MAAM,EACd,cAAc,+BAAqC,GAClD,OAAO,CAAC,YAAY,CAAC;CAIzB"}

dist/src/models/groq.js

+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.GroqChatApi = void 0;
+const groq_sdk_1 = require("groq-sdk");
+const lodash_1 = require("lodash");
+const config_1 = require("../config");
+const utils_1 = require("../utils");
+const errors_1 = require("./errors");
+const tokenizer_1 = require("./tokenizer");
+const RequestDefaults = {
+    retries: config_1.CompletionDefaultRetries,
+    timeout: config_1.CompletionDefaultTimeout,
+    minimumResponseTokens: config_1.MinimumResponseTokens,
+    maximumResponseTokens: config_1.MaximumResponseTokens,
+};
+class GroqChatApi {
+    client;
+    modelConfig;
+    constructor(config, modelConfig) {
+        this.client = new groq_sdk_1.Groq(config);
+        this.modelConfig = modelConfig ?? {};
+    }
+    getTokensFromPrompt = tokenizer_1.getTikTokenTokensFromPrompt;
+    async chatCompletion(initialMessages, requestOptions) {
+        const finalRequestOptions = (0, lodash_1.defaults)(requestOptions, RequestDefaults);
+        const messages = (0, lodash_1.compact)([
+            finalRequestOptions.systemMessage
+                ? {
+                    role: 'system',
+                    content: typeof finalRequestOptions.systemMessage === 'string'
+                        ? finalRequestOptions.systemMessage
+                        : finalRequestOptions.systemMessage(),
+                }
+                : null,
+            ...initialMessages,
+            finalRequestOptions.responsePrefix
+                ? {
+                    role: 'assistant',
+                    content: finalRequestOptions.responsePrefix,
+                }
+                : null,
+        ]);
+        utils_1.debug.log(`🔼 completion requested: ${JSON.stringify(messages)}, config: ${JSON.stringify(this.modelConfig)}, options: ${JSON.stringify(finalRequestOptions)}`);
+        const maxPromptTokens = this.modelConfig.contextSize
+            ? this.modelConfig.contextSize - finalRequestOptions.minimumResponseTokens
+            : 100_000;
+        const messageTokens = this.getTokensFromPrompt(messages.map((m) => m.content ?? ''));
+        if (messageTokens > maxPromptTokens) {
+            throw new errors_1.TokenError('Prompt too big, not enough tokens to meet minimum response', messageTokens - maxPromptTokens);
+        }
+        let completion = '';
+        const completionBody = {
+            stop: finalRequestOptions.stop,
+            temperature: this.modelConfig.temperature,
+            top_p: this.modelConfig.topP,
+            model: this.modelConfig.model ?? config_1.DefaultGroqModel,
+            max_tokens: finalRequestOptions.maximumResponseTokens,
+            messages: messages
+                .filter((m) => (m.role === 'user' || m.role === 'assistant') && m.content)
+                .map((m) => ({
+                role: m.role,
+                content: m.content ?? '',
+            })),
+        };
+        const completionOptions = {
+            timeout: finalRequestOptions.timeout,
+            maxRetries: finalRequestOptions.retries,
+        };
+        if (this.modelConfig.stream) {
+            const stream = await this.client.chat.completions.create({ ...completionBody, stream: true }, completionOptions);
+            if (finalRequestOptions?.responsePrefix) {
+                finalRequestOptions?.events?.emit('data', finalRequestOptions.responsePrefix);
+            }
+            for await (const part of stream) {
+                const text = part.choices[0]?.delta?.content ?? '';
+                utils_1.debug.write(text);
+                completion += text;
+                finalRequestOptions?.events?.emit('data', text);
+            }
+            utils_1.debug.write('\n[STREAM] response end\n');
+        }
+        else {
+            const response = await this.client.chat.completions.create({ ...completionBody, stream: false }, completionOptions);
+            completion = response.choices[0].message.content ?? '';
+            utils_1.debug.log('🔽 completion received', completion);
+        }
+        const content = finalRequestOptions.responsePrefix
+            ? finalRequestOptions.responsePrefix + completion
+            : completion;
+        if (!content) {
+            throw new Error('Completion response malformed');
+        }
+        const receivedMessage = {
+            role: 'assistant',
+            content,
+        };
+        return {
+            message: receivedMessage,
+            content,
+            respond: (message, opt) => this.chatCompletion([
+                ...messages,
+                receivedMessage,
+                typeof message === 'string'
+                    ? { role: 'user', content: message }
+                    : message,
+            ], opt ?? requestOptions),
+        };
+    }
+    textCompletion(prompt, requestOptions = {}) {
+        const messages = [{ role: 'user', content: prompt }];
+        return this.chatCompletion(messages, requestOptions);
+    }
+}
+exports.GroqChatApi = GroqChatApi;

dist/playground.js

 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
 const src_1 = require("./src");
+const groq_1 = require("./src/models/groq");
 (async function go() {
@@ -9,3 +10,3 @@ let client;
             apiKey: process.env.OPENAI_KEY ?? 'YOUR_client_KEY',
-        }, { stream: true, contextSize: 4096, model: 'gpt-4-1106-preview' });
+        }, { stream: true, contextSize: 4096 });
         const resfn = await client?.textCompletion('Hello', {
@@ -31,3 +32,3 @@ callFunction: 'print',
             apiKey: process.env.ANTHROPIC_KEY ?? 'YOUR_client_KEY',
-        }, { stream: true, temperature: 0, model: 'claude-3-sonnet-20240229' });
+        }, { stream: true, temperature: 0 });
     }
@@ -41,2 +42,7 @@ else if (process.env.AWS_BEDROCK_ACCESS_KEY &&
     }
+    else if (process.env.GROQ_KEY) {
+        client = new groq_1.GroqChatApi({
+            apiKey: process.env.GROQ_KEY ?? 'YOUR_client_KEY',
+        }, { stream: true, temperature: 0 });
+    }
     const res0 = await client?.textCompletion('Hello', {
@@ -50,3 +56,3 @@ systemMessage: 'You will respond to all human messages in JSON',
     const res1 = await client?.textCompletion('Hello', {
-        maximumResponseTokens: 1,
+        maximumResponseTokens: 2,
     });
@@ -53,0 +59,0 @@ console.info('Response 1: ', res1);

dist/src/config.d.ts

@@ -5,5 +5,6 @@ export declare const CompletionDefaultRetries = 3;
 export declare const MaximumResponseTokens = 4096;
-export declare const DefaultOpenAIModel = "gpt-3.5-turbo";
-export declare const DefaultAnthropicModel = "claude-instant-1";
+export declare const DefaultOpenAIModel = "gpt-4-1106-preview";
+export declare const DefaultAnthropicModel = "claude-3-sonnet-20240229";
+export declare const DefaultGroqModel = "mixtral-8x7b-32768";
 export declare const DefaultAzureVersion = "2023-09-01-preview";
 //# sourceMappingURL=config.d.ts.map

dist/src/config.js

 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.DefaultAzureVersion = exports.DefaultAnthropicModel = exports.DefaultOpenAIModel = exports.MaximumResponseTokens = exports.MinimumResponseTokens = exports.CompletionDefaultTimeout = exports.CompletionDefaultRetries = void 0;
+exports.DefaultAzureVersion = exports.DefaultGroqModel = exports.DefaultAnthropicModel = exports.DefaultOpenAIModel = exports.MaximumResponseTokens = exports.MinimumResponseTokens = exports.CompletionDefaultTimeout = exports.CompletionDefaultRetries = void 0;
 exports.CompletionDefaultRetries = 3;
@@ -8,4 +8,5 @@ exports.CompletionDefaultTimeout = 300_000;
 exports.MaximumResponseTokens = 4_096;
-exports.DefaultOpenAIModel = 'gpt-3.5-turbo';
-exports.DefaultAnthropicModel = 'claude-instant-1';
+exports.DefaultOpenAIModel = 'gpt-4-1106-preview';
+exports.DefaultAnthropicModel = 'claude-3-sonnet-20240229';
+exports.DefaultGroqModel = 'mixtral-8x7b-32768';
 exports.DefaultAzureVersion = '2023-09-01-preview';

dist/src/types.d.ts

 /// <reference types="node" />
 import { ClientOptions as AnthropicClientOptions } from '@anthropic-ai/sdk';
 import EventEmitter from 'events';
+import { ClientOptions as GroqClientOptions } from 'groq-sdk';
 import { ClientOptions as OpenAIClientOptions } from 'openai';
 import { JsonValue } from 'type-fest';
+export type GroqConfig = GroqClientOptions;
 export type AnthropicConfig = AnthropicClientOptions;
@@ -7,0 +9,0 @@ export type OpenAIConfig = OpenAIClientOptions & {

package.json

 {
   "name": "llm-api",
   "description": "Fully typed chat APIs for OpenAI and Azure's chat models - with token checking and retries",
-  "version": "1.3.5",
+  "version": "1.4.0",
   "packageManager": "yarn@3.4.1",
@@ -48,2 +48,3 @@ "main": "dist/src/index.js",
     "debug": "^4.3.4",
+    "groq-sdk": "^0.3.2",
     "js-tiktoken": "^1.0.10",
@@ -50,0 +51,0 @@ "jsonic": "^1.0.1",

New alerts

Environment variable access

Supply chain risk

Package accesses environment variables, which may be a sign of credential stuffing or data theft.

Found 6 instances in 1 package

Fixed alerts

Environment variable access

Supply chain risk

Package accesses environment variables, which may be a sign of credential stuffing or data theft.

Found 5 instances in 1 package

Improved metrics

Total package byte prevSize

64135

12.16%

Number of package files

42

7.69%

Lines of code

1079

14.42%

Worsened metrics

Dependency count

11

10%

Number of low supply chain risk alerts

13

18.18%

Dependency changes

• + Added

  groq-sdk@^0.3.2

• + Added

  groq-sdk@0.3.3
  (transitive)