Huge News!Announcing our $40M Series B led by Abstract Ventures.Learn More →
Socket
Sign inDemoInstall
Socket
Sign inDemoInstall

@empiricalrun/llm

Package Overview
Dependencies
Maintainers
0
Versions
44
Alerts
File Explorer

Advanced tools

License
Socket logo

Install Socket

Detect and block malicious and high-risk dependencies

Install

@empiricalrun/llm - npm Package Compare versions

Comparing version 0.9.13 to 0.9.14

dist/vision/image.d.ts
dist/vision/image.d.ts.map
dist/vision/image.js
dist/vision/types.d.ts
dist/vision/types.d.ts.map
dist/vision/types.js
test-data/grapevine-feed/small-user-name.png

6

CHANGELOG.md
# @empiricalrun/llm
## 0.9.14
### Patch Changes
- b1343f9: feat: use sharp instead of canvas for image manipulation
## 0.9.13

@@ -4,0 +10,0 @@

23

dist/vision/bbox/index.d.ts

@@ -1,26 +0,5 @@

interface BoundingBox {
corners: {
min: {
x: number;
y: number;
};
max: {
x: number;
y: number;
};
};
center: {
x: number;
y: number;
};
container: {
width: number;
height: number;
};
annotatedImage?: string;
}
import { BoundingBox } from "../types";
export declare function getBoundingBox(base64Image: string, elementDescription: string, options?: {
debug?: true;
}): Promise<BoundingBox>;
export {};
//# sourceMappingURL=index.d.ts.map

33

dist/vision/bbox/index.js
"use strict";
var __importDefault = (this && this.__importDefault) || function (mod) {
return (mod && mod.__esModule) ? mod : { "default": mod };
};
Object.defineProperty(exports, "__esModule", { value: true });
exports.getBoundingBox = void 0;
const canvas_1 = require("canvas");
const image_size_1 = __importDefault(require("image-size"));
const __1 = require("../..");
const image_1 = require("../image");
const utils_1 = require("../utils");

@@ -61,3 +57,3 @@ async function getBoundingBox(base64Image, elementDescription, options = {}) {

try {
bbox.annotatedImage = await annotatedImage(base64Image, bbox);
bbox.annotatedImage = await (0, image_1.drawBoxOnImage)(base64Image, bbox);
}

@@ -74,7 +70,3 @@ catch (e) {

// Docs: https://ai.google.dev/gemini-api/docs/vision?lang=python#bbox
const dimensions = (0, image_size_1.default)(Buffer.from(base64Image, "base64"));
const { width, height } = dimensions;
if (!width || !height) {
throw new Error("Failed to get image dimensions");
}
const { width, height } = (0, image_1.dimensions)(base64Image);
const [y_min, x_min, y_max, x_max] = coordinates;

@@ -111,20 +103,1 @@ function sy(value) {

}
async function annotatedImage(base64Image, bbox) {
const { corners, center } = bbox;
const width = corners.max.x - corners.min.x;
const height = corners.max.y - corners.min.y;
const imgBuffer = Buffer.from(base64Image, "base64");
const dimensions = (0, image_size_1.default)(imgBuffer);
const canvas = (0, canvas_1.createCanvas)(dimensions.width, dimensions.height);
const ctx = canvas.getContext("2d");
const image = await (0, canvas_1.loadImage)(imgBuffer);
ctx.drawImage(image, 0, 0);
ctx.strokeStyle = "red";
ctx.lineWidth = 2;
ctx.strokeRect(corners.min.x, corners.min.y, width, height);
ctx.fillStyle = "red";
ctx.beginPath();
ctx.arc(center.x, center.y, 5, 0, 2 * Math.PI);
ctx.fill();
return canvas.toDataURL();
}

14

dist/vision/point/index.d.ts

@@ -1,16 +0,4 @@

export type Point = {
x: number;
y: number;
};
export interface Coordinates {
x: number;
y: number;
container: {
width: number;
height: number;
};
annotatedImage: string;
}
import { Coordinates, Point } from "../types";
export declare function getCoordinatesFor(prompt: string, base64Image: string): Promise<Coordinates>;
export declare function extractTapCoordinateFromString(inputString: string): Point;
//# sourceMappingURL=index.d.ts.map

58

dist/vision/point/index.js
"use strict";
var __importDefault = (this && this.__importDefault) || function (mod) {
return (mod && mod.__esModule) ? mod : { "default": mod };
};
Object.defineProperty(exports, "__esModule", { value: true });
exports.extractTapCoordinateFromString = exports.getCoordinatesFor = void 0;
const canvas_1 = require("canvas");
const image_size_1 = __importDefault(require("image-size"));
const image_1 = require("../image");
const API_BASE_URL = process.env.VISION_MODEL_ENDPOINT;
async function getCoordinatesFor(prompt, base64Image) {
const llmResponse = await getCoordinateForElement(base64Image, prompt);
const coordinates = extractTapCoordinateFromString(llmResponse);
return drawRedDotAtPoint(base64Image, coordinates);
const llmResponse = await getLlmResponse(base64Image, prompt);
const pointFromLlm = extractTapCoordinateFromString(llmResponse);
const scaledPoint = scaleForImage(pointFromLlm, base64Image);
const annotatedImage = await (0, image_1.drawRedDotAtPoint)(base64Image, scaledPoint);
const dims = (0, image_1.dimensions)(base64Image);
return {
x: scaledPoint.x,
y: scaledPoint.y,
container: {
width: dims.width,
height: dims.height,
},
annotatedImage,
};
}
exports.getCoordinatesFor = getCoordinatesFor;
function scaleForImage(coordinates, base64Image) {
const dimensions = (0, image_size_1.default)(Buffer.from(base64Image, "base64"));
const { width, height } = dimensions;
if (!width || !height) {
throw new Error("Failed to get image dimensions");
}
const { width, height } = (0, image_1.dimensions)(base64Image);
function sy(value) {
// Molmo returns coordinates as percentages
return (value * height) / 100;

@@ -57,29 +61,3 @@ }

exports.extractTapCoordinateFromString = extractTapCoordinateFromString;
async function drawRedDotAtPoint(base64Image, coordinate) {
const point = scaleForImage(coordinate, base64Image);
const imgBuffer = Buffer.from(base64Image, "base64");
const dimensions = (0, image_size_1.default)(imgBuffer);
if (!dimensions.width || !dimensions.height) {
throw new Error("Failed to get image dimensions");
}
const { width, height } = dimensions;
const canvas = (0, canvas_1.createCanvas)(dimensions.width, dimensions.height);
const ctx = canvas.getContext("2d");
const image = await (0, canvas_1.loadImage)(imgBuffer);
ctx.drawImage(image, 0, 0);
ctx.fillStyle = "red";
ctx.beginPath();
ctx.arc(point.x, point.y, 5, 0, 2 * Math.PI);
ctx.fill();
return {
x: point.x,
y: point.y,
container: {
width,
height,
},
annotatedImage: canvas.toDataURL().split(",")[1],
};
}
async function getCoordinateForElement(base64Image, prompt) {
async function getLlmResponse(base64Image, prompt) {
const url = `${API_BASE_URL}/predict`;

@@ -86,0 +64,0 @@ const response = await fetch(url, {

6

package.json
{
"name": "@empiricalrun/llm",
"version": "0.9.13",
"version": "0.9.14",
"main": "dist/index.js",

@@ -38,2 +38,3 @@ "exports": {

"portkey-ai": "^1.3.2",
"sharp": "^0.33.5",
"zod": "^3.23.8"

@@ -44,5 +45,2 @@ },

},
"peerDependencies": {
"canvas": "^2.11.2"
},
"scripts": {

@@ -49,0 +47,0 @@ "dev": "tsc --build --watch",

6

README.md

@@ -64,7 +64,1 @@ # llm

```
To use this, you need to install the canvas npm package, which is a peer dependency.
```sh
npm install canvas
```
dist/vision/bbox/index.d.ts.map

Sorry, the diff of this file is not supported yet

dist/vision/point/index.d.ts.map

Sorry, the diff of this file is not supported yet

SocketSocket SOC 2 Logo

Product

  • Package Alerts
  • Integrations
  • Docs
  • Pricing
  • FAQ
  • Roadmap
  • Changelog

About

Packages

npm

Go

Maven

PyPI

Rubygems

Stay in touch

Get open source security insights delivered straight into your inbox.

  • Terms
  • Privacy
  • Security

Made with ⚡️ by Socket Inc