New Case Study:See how Anthropic automated 95% of dependency reviews with Socket.Learn More

ollama-ai-provider

Advanced tools

Socket logo

Install Socket

Detect and block malicious and high-risk dependencies

Install

ollama-ai-provider - npm Package Compare versions

Comparing version 0.4.0 to 0.5.0

@@ -30,8 +30,8 @@ "use strict";

// src/ollama-facade.ts
var import_provider_utils5 = require("@ai-sdk/provider-utils");
var import_provider_utils6 = require("@ai-sdk/provider-utils");
// src/ollama-chat-language-model.ts
var import_provider3 = require("@ai-sdk/provider");
var import_provider_utils4 = require("@ai-sdk/provider-utils");
var import_zod2 = require("zod");
var import_provider_utils5 = require("@ai-sdk/provider-utils");
var import_zod3 = require("zod");

@@ -41,8 +41,46 @@ // src/convert-to-ollama-chat-messages.ts

var import_provider_utils = require("@ai-sdk/provider-utils");
function convertToOllamaChatMessages(prompt) {
// src/generate-tool/inject-tools-schema-into-system.ts
var DEFAULT_SCHEMA_PREFIX = "You have access to the following tools:";
var DEFAULT_SCHEMA_SUFFIX = `To use a tool, you MUST answer with a JSON object with the following structure:
[
{
"name": <name of the called tool>,
"arguments": <arguments for the tool matching the above JSON schema>
}
]`;
function injectToolsSchemaIntoSystem({
schemaPrefix = DEFAULT_SCHEMA_PREFIX,
schemaSuffix = DEFAULT_SCHEMA_SUFFIX,
system,
tools
}) {
if (!tools) {
return system;
}
return [
system,
system === null ? null : "",
// add a newline if system is not null
schemaPrefix,
JSON.stringify(tools),
schemaSuffix
].filter((line) => line !== null).join("\n");
}
// src/convert-to-ollama-chat-messages.ts
function convertToOllamaChatMessages(prompt, tools) {
const messages = [];
let hasSystem = false;
for (const { content, role } of prompt) {
switch (role) {
case "system": {
messages.push({ content, role: "system" });
messages.push({
content: injectToolsSchemaIntoSystem({
system: content,
tools
}),
role: "system"
});
hasSystem = true;
break;

@@ -91,10 +129,57 @@ }

}
if (!hasSystem && tools) {
messages.unshift({
content: injectToolsSchemaIntoSystem({
system: "",
tools
}),
role: "system"
});
}
return messages;
}
// src/generate-tool/infer-tool-calls-from-response.ts
var import_provider_utils2 = require("@ai-sdk/provider-utils");
var import_zod = require("zod");
function inferToolCallsFromResponse(response) {
try {
const tool = JSON.parse(response.message.content);
const parsedTools = toolResponseSchema.parse(tool);
return {
...response,
finish_reason: "tool-calls",
message: {
content: "",
role: "assistant",
tool_calls: parsedTools.map((parsedTool) => ({
function: {
arguments: JSON.stringify(parsedTool.arguments),
name: parsedTool.name
},
id: (0, import_provider_utils2.generateId)(),
type: "function"
}))
}
};
} catch (e) {
return {
...response,
finish_reason: "stop"
};
}
}
var toolResponseSchema = import_zod.z.array(
import_zod.z.object({
arguments: import_zod.z.record(import_zod.z.unknown()),
name: import_zod.z.string()
})
);
// src/map-ollama-finish-reason.ts
function mapOllamaFinishReason(finishReason) {
switch (finishReason) {
case "stop": {
return "stop";
case "stop":
case "tool-calls": {
return finishReason;
}

@@ -108,13 +193,13 @@ default: {

// src/ollama-error.ts
var import_provider_utils2 = require("@ai-sdk/provider-utils");
var import_zod = require("zod");
var ollamaErrorDataSchema = import_zod.z.object({
error: import_zod.z.object({
code: import_zod.z.string().nullable(),
message: import_zod.z.string(),
param: import_zod.z.any().nullable(),
type: import_zod.z.string()
var import_provider_utils3 = require("@ai-sdk/provider-utils");
var import_zod2 = require("zod");
var ollamaErrorDataSchema = import_zod2.z.object({
error: import_zod2.z.object({
code: import_zod2.z.string().nullable(),
message: import_zod2.z.string(),
param: import_zod2.z.any().nullable(),
type: import_zod2.z.string()
})
});
var ollamaFailedResponseHandler = (0, import_provider_utils2.createJsonErrorResponseHandler)({
var ollamaFailedResponseHandler = (0, import_provider_utils3.createJsonErrorResponseHandler)({
errorSchema: ollamaErrorDataSchema,

@@ -126,5 +211,5 @@ errorToMessage: (data) => data.error.message

var import_provider2 = require("@ai-sdk/provider");
var import_provider_utils3 = require("@ai-sdk/provider-utils");
var import_provider_utils4 = require("@ai-sdk/provider-utils");
var createJsonStreamResponseHandler = (chunkSchema) => async ({ response }) => {
const responseHeaders = (0, import_provider_utils3.extractResponseHeaders)(response);
const responseHeaders = (0, import_provider_utils4.extractResponseHeaders)(response);
if (response.body === null) {

@@ -139,3 +224,3 @@ throw new import_provider2.EmptyResponseBodyError({});

controller.enqueue(
(0, import_provider_utils3.safeParseJSON)({
(0, import_provider_utils4.safeParseJSON)({
schema: chunkSchema,

@@ -150,2 +235,7 @@ text: data

};
function removeUndefined(object) {
return Object.fromEntries(
Object.entries(object).filter(([, v]) => v !== void 0)
);
}

@@ -174,8 +264,8 @@ // src/ollama-chat-language-model.ts

}) {
var _a;
const type = mode.type;
const warnings = [];
const baseArguments = {
messages: convertToOllamaChatMessages(prompt),
model: this.modelId,
options: {
options: removeUndefined({
frequency_penalty: frequencyPenalty,

@@ -196,9 +286,19 @@ mirostat: this.settings.mirostat,

top_p: topP
}
})
};
switch (type) {
case "regular": {
const tools = ((_a = mode.tools) == null ? void 0 : _a.length) ? mode.tools : void 0;
return {
args: {
...baseArguments
...baseArguments,
messages: convertToOllamaChatMessages(prompt, tools),
tools: tools == null ? void 0 : tools.map((tool) => ({
function: {
description: tool.description,
name: tool.name,
parameters: tool.parameters
},
type: "function"
}))
},

@@ -212,3 +312,4 @@ warnings

...baseArguments,
format: "json"
format: "json",
messages: convertToOllamaChatMessages(prompt)
},

@@ -219,5 +320,24 @@ warnings

case "object-tool": {
throw new import_provider3.UnsupportedFunctionalityError({
functionality: "object-tool mode"
});
return {
args: {
...baseArguments,
format: "json",
messages: convertToOllamaChatMessages(prompt, [mode.tool]),
tool_choice: {
function: { name: mode.tool.name },
type: "function"
},
tools: [
{
function: {
description: mode.tool.description,
name: mode.tool.name,
parameters: mode.tool.parameters
},
type: "function"
}
]
},
warnings
};
}

@@ -236,4 +356,5 @@ case "object-grammar": {

async doGenerate(options) {
var _a, _b;
const { args, warnings } = this.getArguments(options);
const { responseHeaders, value: response } = await (0, import_provider_utils4.postJsonToApi)({
const { responseHeaders, value } = await (0, import_provider_utils5.postJsonToApi)({
abortSignal: options.abortSignal,

@@ -246,16 +367,23 @@ body: {

headers: this.config.headers(),
successfulResponseHandler: (0, import_provider_utils4.createJsonResponseHandler)(
ollamaChatChunkSchema
successfulResponseHandler: (0, import_provider_utils5.createJsonResponseHandler)(
ollamaChatResponseSchema
),
url: `${this.config.baseURL}/chat`
});
const response = inferToolCallsFromResponse(value);
const { messages: rawPrompt, ...rawSettings } = args;
return {
finishReason: "stop",
finishReason: mapOllamaFinishReason(response.finish_reason),
rawCall: { rawPrompt, rawSettings },
rawResponse: { headers: responseHeaders },
text: response.message.content,
text: (_a = response.message.content) != null ? _a : void 0,
toolCalls: (_b = response.message.tool_calls) == null ? void 0 : _b.map((toolCall) => ({
args: toolCall.function.arguments,
toolCallId: (0, import_provider_utils5.generateId)(),
toolCallType: "function",
toolName: toolCall.function.name
})),
usage: {
completionTokens: Number.NaN,
promptTokens: response.prompt_eval_count || Number.NaN
completionTokens: response.eval_count || 0,
promptTokens: response.prompt_eval_count || 0
},

@@ -267,3 +395,3 @@ warnings

const { args, warnings } = this.getArguments(options);
const { responseHeaders, value: response } = await (0, import_provider_utils4.postJsonToApi)({
const { responseHeaders, value: response } = await (0, import_provider_utils5.postJsonToApi)({
abortSignal: options.abortSignal,

@@ -323,37 +451,46 @@ body: args,

};
var ollamaChatChunkSchema = import_zod2.z.object({
created_at: import_zod2.z.string(),
done: import_zod2.z.literal(true),
eval_count: import_zod2.z.number(),
eval_duration: import_zod2.z.number(),
load_duration: import_zod2.z.number().optional(),
message: import_zod2.z.object({
content: import_zod2.z.string(),
role: import_zod2.z.string()
var ollamaChatResponseSchema = import_zod3.z.object({
created_at: import_zod3.z.string(),
done: import_zod3.z.literal(true),
eval_count: import_zod3.z.number(),
eval_duration: import_zod3.z.number(),
finish_reason: import_zod3.z.string().optional().nullable(),
load_duration: import_zod3.z.number().optional(),
message: import_zod3.z.object({
content: import_zod3.z.string(),
role: import_zod3.z.string(),
tool_calls: import_zod3.z.array(
import_zod3.z.object({
function: import_zod3.z.object({
arguments: import_zod3.z.string(),
name: import_zod3.z.string()
})
})
).optional().nullable()
}),
model: import_zod2.z.string(),
prompt_eval_count: import_zod2.z.number().optional(),
prompt_eval_duration: import_zod2.z.number().optional(),
total_duration: import_zod2.z.number()
model: import_zod3.z.string(),
prompt_eval_count: import_zod3.z.number().optional(),
prompt_eval_duration: import_zod3.z.number().optional(),
total_duration: import_zod3.z.number()
});
var ollamaChatStreamChunkSchema = import_zod2.z.discriminatedUnion("done", [
import_zod2.z.object({
created_at: import_zod2.z.string(),
done: import_zod2.z.literal(false),
message: import_zod2.z.object({
content: import_zod2.z.string(),
role: import_zod2.z.string()
var ollamaChatStreamChunkSchema = import_zod3.z.discriminatedUnion("done", [
import_zod3.z.object({
created_at: import_zod3.z.string(),
done: import_zod3.z.literal(false),
message: import_zod3.z.object({
content: import_zod3.z.string(),
role: import_zod3.z.string()
}),
model: import_zod2.z.string()
model: import_zod3.z.string()
}),
import_zod2.z.object({
created_at: import_zod2.z.string(),
done: import_zod2.z.literal(true),
eval_count: import_zod2.z.number(),
eval_duration: import_zod2.z.number(),
load_duration: import_zod2.z.number().optional(),
model: import_zod2.z.string(),
prompt_eval_count: import_zod2.z.number().optional(),
prompt_eval_duration: import_zod2.z.number().optional(),
total_duration: import_zod2.z.number()
import_zod3.z.object({
created_at: import_zod3.z.string(),
done: import_zod3.z.literal(true),
eval_count: import_zod3.z.number(),
eval_duration: import_zod3.z.number(),
load_duration: import_zod3.z.number().optional(),
model: import_zod3.z.string(),
prompt_eval_count: import_zod3.z.number().optional(),
prompt_eval_duration: import_zod3.z.number().optional(),
total_duration: import_zod3.z.number()
})

@@ -366,4 +503,4 @@ ]);

var _a, _b;
this.baseURL = (_a = (0, import_provider_utils5.withoutTrailingSlash)(options.baseURL)) != null ? _a : "http://127.0.0.1:11434/api";
this.generateId = (_b = options.generateId) != null ? _b : import_provider_utils5.generateId;
this.baseURL = (_a = (0, import_provider_utils6.withoutTrailingSlash)(options.baseURL)) != null ? _a : "http://127.0.0.1:11434/api";
this.generateId = (_b = options.generateId) != null ? _b : import_provider_utils6.generateId;
this.headers = options.headers;

@@ -370,0 +507,0 @@ }

{
"name": "ollama-ai-provider",
"version": "0.4.0",
"version": "0.5.0",
"description": "Vercel AI Provider for running LLMs locally using Ollama",

@@ -26,3 +26,3 @@ "main": "./dist/index.js",

"@edge-runtime/vm": "^3.2.0",
"@types/node": "^18.19.31",
"@types/node": "^18.19.33",
"@typescript-eslint/eslint-plugin": "^7.8.0",

@@ -29,0 +29,0 @@ "@typescript-eslint/parser": "^7.8.0",

@@ -61,11 +61,18 @@ # ollama-ai-provider

| Model | Image input | Object generation | Tool usage | Tool streaming |
|------------|--------------------|--------------------|------------|----------------|
| llama2 | :x: | :white_check_mark: | :x: | :x: |
| llama3 | :x: | :white_check_mark: | :x: | :x: |
| llava | :white_check_mark: | :white_check_mark: | :x: | :x: |
| mistral | :x: | :white_check_mark: | :x: | :x: |
| mixtral | :x: | :white_check_mark: | :x: | :x: |
| openhermes | :x: | :white_check_mark: | :x: | :x: |
| phi3 | :x: | :white_check_mark: | :x: | :x: |
| Model | Image input | Object generation | Tool usage | Tool streaming |
|------------|--------------------|--------------------|--------------------|----------------|
| llama2 | :x: | :white_check_mark: | :x: | :x: |
| llama3 | :x: | :white_check_mark: | :x: | :x: |
| llava | :white_check_mark: | :white_check_mark: | :x: | :x: |
| mistral | :x: | :white_check_mark: | :x: | :x: |
| mixtral | :x: | :white_check_mark: | :white_check_mark: | :x: |
| openhermes | :x: | :white_check_mark: | :white_check_mark: | :x: |
| phi3 | :x: | :white_check_mark: | :x: | :x: |
### Caveats
* Some models have been found to be slow when streaming objects. See https://github.com/ollama/ollama/issues/3851
* The use of tools is not supported by the Ollama API and has been simulated with system prompt injection, so the behavior
depending on the model can be erratic.
* This library is highly experimental and can change constantly. All releases will be of type MAJOR following the
0.MAJOR.MINOR scheme. Only bugs and model updates will be released as MINOR.

Sorry, the diff of this file is not supported yet

Sorry, the diff of this file is not supported yet

Sorry, the diff of this file is not supported yet