@open-evals/metrics - npm Package Compare versions

Comparing version 0.1.3 to 0.1.4

README.md

+# @open-evals/metrics
+
+Pre-built evaluation metrics for RAG and LLM applications.
+
+## Installation
+
+```bash
+pnpm add @open-evals/metrics
+```
+
+## Features
+
+- **Faithfulness**: Measures how grounded responses are in the provided context
+- **Factual Correctness**: Evaluates factual accuracy against reference answers
+- **Answer Similarity**: Semantic similarity between generated and reference answers
+- **Context Recall**: Measures retrieval quality against reference answers
+- **Noise Sensitivity**: Tests robustness against irrelevant context
+- **Evalite Integration**: Convert metrics to Evalite scorers

dist/index.d.ts

@@ -0,4 +1,4 @@
+import { EmbeddingMetric, LLMMetric, MetricScore, SingleTurnSample } from "@ai-sdk-eval/core";
 import { EmbeddingModel, LanguageModel } from "ai";
 import * as evalite0 from "evalite";
-import { EmbeddingMetric, LLMMetric, MetricScore, SingleTurnSample } from "@ai-sdk-eval/core";
 import { Metric, MetricScore as MetricScore$1, SingleTurnSample as SingleTurnSample$1 } from "@open-evals/core";
@@ -5,0 +5,0 @@

dist/index.js

@@ -0,1 +1,2 @@
+import { EmbeddingMetric, LLMMetric } from "@ai-sdk-eval/core";
 import { cosineSimilarity, embedMany, generateObject } from "ai";
@@ -5,67 +6,2 @@ import { z } from "zod";
 
-//#region ../core/dist/index.js
-function isSingleTurnSample(sample) {
-	return "query" in sample && "response" in sample;
-}
-function isMultiTurnSample(sample) {
-	return "messages" in sample;
-}
-var Metric = class Metric$1 {
-	name;
-	description;
-	constructor(config) {
-		this.name = config.name;
-		this.description = config.description;
-	}
-	/**
-	* Evaluate a single-turn sample
-	* Override this method for single-turn metrics
-	*/
-	async evaluateSingleTurn(sample) {
-		throw new Error(`Metric ${this.name} does not support single-turn evaluation`);
-	}
-	/**
-	* Evaluate a multi-turn sample
-	* Override this method for multi-turn metrics
-	*/
-	async evaluateMultiTurn(sample) {
-		throw new Error(`Metric ${this.name} does not support multi-turn evaluation`);
-	}
-	async evaluate(sample) {
-		if (isSingleTurnSample(sample)) return this.evaluateSingleTurn(sample);
-		else if (isMultiTurnSample(sample)) return this.evaluateMultiTurn(sample);
-		throw new Error(`Metric ${this.name} does not support this sample type`);
-	}
-	supports(sample) {
-		try {
-			if (isSingleTurnSample(sample)) return this.evaluateSingleTurn !== Metric$1.prototype.evaluateSingleTurn;
-			else if (isMultiTurnSample(sample)) return this.evaluateMultiTurn !== Metric$1.prototype.evaluateMultiTurn;
-			return false;
-		} catch {
-			return false;
-		}
-	}
-};
-/**
-* Base class for metrics that use LLMs for evaluation
-*/
-var LLMMetric = class extends Metric {
-	model;
-	constructor(config) {
-		super(config);
-		this.model = config.model;
-	}
-};
-/**
-* Base class for metrics that use embeddings for evaluation
-*/
-var EmbeddingMetric = class extends Metric {
-	model;
-	constructor(config) {
-		super(config);
-		this.model = config.model;
-	}
-};
-
-//#endregion
 //#region src/metrics/faithfulness.ts
@@ -72,0 +8,0 @@ /**

package.json

 {
   "name": "@open-evals/metrics",
-  "version": "0.1.3",
+  "version": "0.1.4",
   "description": "Metrics for Open Evals",
@@ -33,3 +33,3 @@ "author": {
     "evalite": "^0.16.1",
-    "@open-evals/core": "0.1.2"
+    "@open-evals/core": "0.1.3"
   },
@@ -36,0 +36,0 @@ "devDependencies": {

Fixed alerts

No README

Quality

Package does not have a README. This may indicate a failed publish or a low quality package.

Improved metrics

Number of package files

5

25%

Number of medium quality alerts

1

-50%

Number of lines in readme file

19

Infinity%

Worsened metrics

Total package byte prevSize

49235

-2.06%

Lines of code

1125

-5.3%

Dependency changes

• + Added

  @open-evals/core@0.1.3
  (transitive)

• - Removed

  @open-evals/core@0.1.2
  (transitive)

• Updated

  @open-evals/core@0.1.3