ollama-ai-provider
Advanced tools
Comparing version 0.4.0 to 0.5.0
@@ -30,8 +30,8 @@ "use strict"; | ||
// src/ollama-facade.ts | ||
var import_provider_utils5 = require("@ai-sdk/provider-utils"); | ||
var import_provider_utils6 = require("@ai-sdk/provider-utils"); | ||
// src/ollama-chat-language-model.ts | ||
var import_provider3 = require("@ai-sdk/provider"); | ||
var import_provider_utils4 = require("@ai-sdk/provider-utils"); | ||
var import_zod2 = require("zod"); | ||
var import_provider_utils5 = require("@ai-sdk/provider-utils"); | ||
var import_zod3 = require("zod"); | ||
@@ -41,8 +41,46 @@ // src/convert-to-ollama-chat-messages.ts | ||
var import_provider_utils = require("@ai-sdk/provider-utils"); | ||
function convertToOllamaChatMessages(prompt) { | ||
// src/generate-tool/inject-tools-schema-into-system.ts | ||
var DEFAULT_SCHEMA_PREFIX = "You have access to the following tools:"; | ||
var DEFAULT_SCHEMA_SUFFIX = `To use a tool, you MUST answer with a JSON object with the following structure: | ||
[ | ||
{ | ||
"name": <name of the called tool>, | ||
"arguments": <arguments for the tool matching the above JSON schema> | ||
} | ||
]`; | ||
function injectToolsSchemaIntoSystem({ | ||
schemaPrefix = DEFAULT_SCHEMA_PREFIX, | ||
schemaSuffix = DEFAULT_SCHEMA_SUFFIX, | ||
system, | ||
tools | ||
}) { | ||
if (!tools) { | ||
return system; | ||
} | ||
return [ | ||
system, | ||
system === null ? null : "", | ||
// add a newline if system is not null | ||
schemaPrefix, | ||
JSON.stringify(tools), | ||
schemaSuffix | ||
].filter((line) => line !== null).join("\n"); | ||
} | ||
// src/convert-to-ollama-chat-messages.ts | ||
function convertToOllamaChatMessages(prompt, tools) { | ||
const messages = []; | ||
let hasSystem = false; | ||
for (const { content, role } of prompt) { | ||
switch (role) { | ||
case "system": { | ||
messages.push({ content, role: "system" }); | ||
messages.push({ | ||
content: injectToolsSchemaIntoSystem({ | ||
system: content, | ||
tools | ||
}), | ||
role: "system" | ||
}); | ||
hasSystem = true; | ||
break; | ||
@@ -91,10 +129,57 @@ } | ||
} | ||
if (!hasSystem && tools) { | ||
messages.unshift({ | ||
content: injectToolsSchemaIntoSystem({ | ||
system: "", | ||
tools | ||
}), | ||
role: "system" | ||
}); | ||
} | ||
return messages; | ||
} | ||
// src/generate-tool/infer-tool-calls-from-response.ts | ||
var import_provider_utils2 = require("@ai-sdk/provider-utils"); | ||
var import_zod = require("zod"); | ||
function inferToolCallsFromResponse(response) { | ||
try { | ||
const tool = JSON.parse(response.message.content); | ||
const parsedTools = toolResponseSchema.parse(tool); | ||
return { | ||
...response, | ||
finish_reason: "tool-calls", | ||
message: { | ||
content: "", | ||
role: "assistant", | ||
tool_calls: parsedTools.map((parsedTool) => ({ | ||
function: { | ||
arguments: JSON.stringify(parsedTool.arguments), | ||
name: parsedTool.name | ||
}, | ||
id: (0, import_provider_utils2.generateId)(), | ||
type: "function" | ||
})) | ||
} | ||
}; | ||
} catch (e) { | ||
return { | ||
...response, | ||
finish_reason: "stop" | ||
}; | ||
} | ||
} | ||
var toolResponseSchema = import_zod.z.array( | ||
import_zod.z.object({ | ||
arguments: import_zod.z.record(import_zod.z.unknown()), | ||
name: import_zod.z.string() | ||
}) | ||
); | ||
// src/map-ollama-finish-reason.ts | ||
function mapOllamaFinishReason(finishReason) { | ||
switch (finishReason) { | ||
case "stop": { | ||
return "stop"; | ||
case "stop": | ||
case "tool-calls": { | ||
return finishReason; | ||
} | ||
@@ -108,13 +193,13 @@ default: { | ||
// src/ollama-error.ts | ||
var import_provider_utils2 = require("@ai-sdk/provider-utils"); | ||
var import_zod = require("zod"); | ||
var ollamaErrorDataSchema = import_zod.z.object({ | ||
error: import_zod.z.object({ | ||
code: import_zod.z.string().nullable(), | ||
message: import_zod.z.string(), | ||
param: import_zod.z.any().nullable(), | ||
type: import_zod.z.string() | ||
var import_provider_utils3 = require("@ai-sdk/provider-utils"); | ||
var import_zod2 = require("zod"); | ||
var ollamaErrorDataSchema = import_zod2.z.object({ | ||
error: import_zod2.z.object({ | ||
code: import_zod2.z.string().nullable(), | ||
message: import_zod2.z.string(), | ||
param: import_zod2.z.any().nullable(), | ||
type: import_zod2.z.string() | ||
}) | ||
}); | ||
var ollamaFailedResponseHandler = (0, import_provider_utils2.createJsonErrorResponseHandler)({ | ||
var ollamaFailedResponseHandler = (0, import_provider_utils3.createJsonErrorResponseHandler)({ | ||
errorSchema: ollamaErrorDataSchema, | ||
@@ -126,5 +211,5 @@ errorToMessage: (data) => data.error.message | ||
var import_provider2 = require("@ai-sdk/provider"); | ||
var import_provider_utils3 = require("@ai-sdk/provider-utils"); | ||
var import_provider_utils4 = require("@ai-sdk/provider-utils"); | ||
var createJsonStreamResponseHandler = (chunkSchema) => async ({ response }) => { | ||
const responseHeaders = (0, import_provider_utils3.extractResponseHeaders)(response); | ||
const responseHeaders = (0, import_provider_utils4.extractResponseHeaders)(response); | ||
if (response.body === null) { | ||
@@ -139,3 +224,3 @@ throw new import_provider2.EmptyResponseBodyError({}); | ||
controller.enqueue( | ||
(0, import_provider_utils3.safeParseJSON)({ | ||
(0, import_provider_utils4.safeParseJSON)({ | ||
schema: chunkSchema, | ||
@@ -150,2 +235,7 @@ text: data | ||
}; | ||
function removeUndefined(object) { | ||
return Object.fromEntries( | ||
Object.entries(object).filter(([, v]) => v !== void 0) | ||
); | ||
} | ||
@@ -174,8 +264,8 @@ // src/ollama-chat-language-model.ts | ||
}) { | ||
var _a; | ||
const type = mode.type; | ||
const warnings = []; | ||
const baseArguments = { | ||
messages: convertToOllamaChatMessages(prompt), | ||
model: this.modelId, | ||
options: { | ||
options: removeUndefined({ | ||
frequency_penalty: frequencyPenalty, | ||
@@ -196,9 +286,19 @@ mirostat: this.settings.mirostat, | ||
top_p: topP | ||
} | ||
}) | ||
}; | ||
switch (type) { | ||
case "regular": { | ||
const tools = ((_a = mode.tools) == null ? void 0 : _a.length) ? mode.tools : void 0; | ||
return { | ||
args: { | ||
...baseArguments | ||
...baseArguments, | ||
messages: convertToOllamaChatMessages(prompt, tools), | ||
tools: tools == null ? void 0 : tools.map((tool) => ({ | ||
function: { | ||
description: tool.description, | ||
name: tool.name, | ||
parameters: tool.parameters | ||
}, | ||
type: "function" | ||
})) | ||
}, | ||
@@ -212,3 +312,4 @@ warnings | ||
...baseArguments, | ||
format: "json" | ||
format: "json", | ||
messages: convertToOllamaChatMessages(prompt) | ||
}, | ||
@@ -219,5 +320,24 @@ warnings | ||
case "object-tool": { | ||
throw new import_provider3.UnsupportedFunctionalityError({ | ||
functionality: "object-tool mode" | ||
}); | ||
return { | ||
args: { | ||
...baseArguments, | ||
format: "json", | ||
messages: convertToOllamaChatMessages(prompt, [mode.tool]), | ||
tool_choice: { | ||
function: { name: mode.tool.name }, | ||
type: "function" | ||
}, | ||
tools: [ | ||
{ | ||
function: { | ||
description: mode.tool.description, | ||
name: mode.tool.name, | ||
parameters: mode.tool.parameters | ||
}, | ||
type: "function" | ||
} | ||
] | ||
}, | ||
warnings | ||
}; | ||
} | ||
@@ -236,4 +356,5 @@ case "object-grammar": { | ||
async doGenerate(options) { | ||
var _a, _b; | ||
const { args, warnings } = this.getArguments(options); | ||
const { responseHeaders, value: response } = await (0, import_provider_utils4.postJsonToApi)({ | ||
const { responseHeaders, value } = await (0, import_provider_utils5.postJsonToApi)({ | ||
abortSignal: options.abortSignal, | ||
@@ -246,16 +367,23 @@ body: { | ||
headers: this.config.headers(), | ||
successfulResponseHandler: (0, import_provider_utils4.createJsonResponseHandler)( | ||
ollamaChatChunkSchema | ||
successfulResponseHandler: (0, import_provider_utils5.createJsonResponseHandler)( | ||
ollamaChatResponseSchema | ||
), | ||
url: `${this.config.baseURL}/chat` | ||
}); | ||
const response = inferToolCallsFromResponse(value); | ||
const { messages: rawPrompt, ...rawSettings } = args; | ||
return { | ||
finishReason: "stop", | ||
finishReason: mapOllamaFinishReason(response.finish_reason), | ||
rawCall: { rawPrompt, rawSettings }, | ||
rawResponse: { headers: responseHeaders }, | ||
text: response.message.content, | ||
text: (_a = response.message.content) != null ? _a : void 0, | ||
toolCalls: (_b = response.message.tool_calls) == null ? void 0 : _b.map((toolCall) => ({ | ||
args: toolCall.function.arguments, | ||
toolCallId: (0, import_provider_utils5.generateId)(), | ||
toolCallType: "function", | ||
toolName: toolCall.function.name | ||
})), | ||
usage: { | ||
completionTokens: Number.NaN, | ||
promptTokens: response.prompt_eval_count || Number.NaN | ||
completionTokens: response.eval_count || 0, | ||
promptTokens: response.prompt_eval_count || 0 | ||
}, | ||
@@ -267,3 +395,3 @@ warnings | ||
const { args, warnings } = this.getArguments(options); | ||
const { responseHeaders, value: response } = await (0, import_provider_utils4.postJsonToApi)({ | ||
const { responseHeaders, value: response } = await (0, import_provider_utils5.postJsonToApi)({ | ||
abortSignal: options.abortSignal, | ||
@@ -323,37 +451,46 @@ body: args, | ||
}; | ||
var ollamaChatChunkSchema = import_zod2.z.object({ | ||
created_at: import_zod2.z.string(), | ||
done: import_zod2.z.literal(true), | ||
eval_count: import_zod2.z.number(), | ||
eval_duration: import_zod2.z.number(), | ||
load_duration: import_zod2.z.number().optional(), | ||
message: import_zod2.z.object({ | ||
content: import_zod2.z.string(), | ||
role: import_zod2.z.string() | ||
var ollamaChatResponseSchema = import_zod3.z.object({ | ||
created_at: import_zod3.z.string(), | ||
done: import_zod3.z.literal(true), | ||
eval_count: import_zod3.z.number(), | ||
eval_duration: import_zod3.z.number(), | ||
finish_reason: import_zod3.z.string().optional().nullable(), | ||
load_duration: import_zod3.z.number().optional(), | ||
message: import_zod3.z.object({ | ||
content: import_zod3.z.string(), | ||
role: import_zod3.z.string(), | ||
tool_calls: import_zod3.z.array( | ||
import_zod3.z.object({ | ||
function: import_zod3.z.object({ | ||
arguments: import_zod3.z.string(), | ||
name: import_zod3.z.string() | ||
}) | ||
}) | ||
).optional().nullable() | ||
}), | ||
model: import_zod2.z.string(), | ||
prompt_eval_count: import_zod2.z.number().optional(), | ||
prompt_eval_duration: import_zod2.z.number().optional(), | ||
total_duration: import_zod2.z.number() | ||
model: import_zod3.z.string(), | ||
prompt_eval_count: import_zod3.z.number().optional(), | ||
prompt_eval_duration: import_zod3.z.number().optional(), | ||
total_duration: import_zod3.z.number() | ||
}); | ||
var ollamaChatStreamChunkSchema = import_zod2.z.discriminatedUnion("done", [ | ||
import_zod2.z.object({ | ||
created_at: import_zod2.z.string(), | ||
done: import_zod2.z.literal(false), | ||
message: import_zod2.z.object({ | ||
content: import_zod2.z.string(), | ||
role: import_zod2.z.string() | ||
var ollamaChatStreamChunkSchema = import_zod3.z.discriminatedUnion("done", [ | ||
import_zod3.z.object({ | ||
created_at: import_zod3.z.string(), | ||
done: import_zod3.z.literal(false), | ||
message: import_zod3.z.object({ | ||
content: import_zod3.z.string(), | ||
role: import_zod3.z.string() | ||
}), | ||
model: import_zod2.z.string() | ||
model: import_zod3.z.string() | ||
}), | ||
import_zod2.z.object({ | ||
created_at: import_zod2.z.string(), | ||
done: import_zod2.z.literal(true), | ||
eval_count: import_zod2.z.number(), | ||
eval_duration: import_zod2.z.number(), | ||
load_duration: import_zod2.z.number().optional(), | ||
model: import_zod2.z.string(), | ||
prompt_eval_count: import_zod2.z.number().optional(), | ||
prompt_eval_duration: import_zod2.z.number().optional(), | ||
total_duration: import_zod2.z.number() | ||
import_zod3.z.object({ | ||
created_at: import_zod3.z.string(), | ||
done: import_zod3.z.literal(true), | ||
eval_count: import_zod3.z.number(), | ||
eval_duration: import_zod3.z.number(), | ||
load_duration: import_zod3.z.number().optional(), | ||
model: import_zod3.z.string(), | ||
prompt_eval_count: import_zod3.z.number().optional(), | ||
prompt_eval_duration: import_zod3.z.number().optional(), | ||
total_duration: import_zod3.z.number() | ||
}) | ||
@@ -366,4 +503,4 @@ ]); | ||
var _a, _b; | ||
this.baseURL = (_a = (0, import_provider_utils5.withoutTrailingSlash)(options.baseURL)) != null ? _a : "http://127.0.0.1:11434/api"; | ||
this.generateId = (_b = options.generateId) != null ? _b : import_provider_utils5.generateId; | ||
this.baseURL = (_a = (0, import_provider_utils6.withoutTrailingSlash)(options.baseURL)) != null ? _a : "http://127.0.0.1:11434/api"; | ||
this.generateId = (_b = options.generateId) != null ? _b : import_provider_utils6.generateId; | ||
this.headers = options.headers; | ||
@@ -370,0 +507,0 @@ } |
{ | ||
"name": "ollama-ai-provider", | ||
"version": "0.4.0", | ||
"version": "0.5.0", | ||
"description": "Vercel AI Provider for running LLMs locally using Ollama", | ||
@@ -26,3 +26,3 @@ "main": "./dist/index.js", | ||
"@edge-runtime/vm": "^3.2.0", | ||
"@types/node": "^18.19.31", | ||
"@types/node": "^18.19.33", | ||
"@typescript-eslint/eslint-plugin": "^7.8.0", | ||
@@ -29,0 +29,0 @@ "@typescript-eslint/parser": "^7.8.0", |
@@ -61,11 +61,18 @@ # ollama-ai-provider | ||
| Model | Image input | Object generation | Tool usage | Tool streaming | | ||
|------------|--------------------|--------------------|------------|----------------| | ||
| llama2 | :x: | :white_check_mark: | :x: | :x: | | ||
| llama3 | :x: | :white_check_mark: | :x: | :x: | | ||
| llava | :white_check_mark: | :white_check_mark: | :x: | :x: | | ||
| mistral | :x: | :white_check_mark: | :x: | :x: | | ||
| mixtral | :x: | :white_check_mark: | :x: | :x: | | ||
| openhermes | :x: | :white_check_mark: | :x: | :x: | | ||
| phi3 | :x: | :white_check_mark: | :x: | :x: | | ||
| Model | Image input | Object generation | Tool usage | Tool streaming | | ||
|------------|--------------------|--------------------|--------------------|----------------| | ||
| llama2 | :x: | :white_check_mark: | :x: | :x: | | ||
| llama3 | :x: | :white_check_mark: | :x: | :x: | | ||
| llava | :white_check_mark: | :white_check_mark: | :x: | :x: | | ||
| mistral | :x: | :white_check_mark: | :x: | :x: | | ||
| mixtral | :x: | :white_check_mark: | :white_check_mark: | :x: | | ||
| openhermes | :x: | :white_check_mark: | :white_check_mark: | :x: | | ||
| phi3 | :x: | :white_check_mark: | :x: | :x: | | ||
### Caveats | ||
* Some models have been found to be slow when streaming objects. See https://github.com/ollama/ollama/issues/3851 | ||
* The use of tools is not supported by the Ollama API and has been simulated with system prompt injection, so the behavior | ||
depending on the model can be erratic. | ||
* This library is highly experimental and can change constantly. All releases will be of type MAJOR following the | ||
0.MAJOR.MINOR scheme. Only bugs and model updates will be released as MINOR. |
Sorry, the diff of this file is not supported yet
Sorry, the diff of this file is not supported yet
Sorry, the diff of this file is not supported yet
99796
1102
78