@open-evals/metrics - npm Package Compare versions

Comparing version 0.1.0 to 0.1.1

dist/index.d.ts

@@ -1,3 +0,3 @@
-import { LanguageModel } from "ai";
-import { LLMMetric, MetricScore, SingleTurnSample } from "@ai-sdk-eval/core";
+import { EmbeddingModel, LanguageModel } from "ai";
+import { EmbeddingMetric, LLMMetric, MetricScore, SingleTurnSample } from "@ai-sdk-eval/core";
 
@@ -29,3 +29,2 @@ //#region src/metrics/faithfulness.d.ts
 declare class Faithfulness extends LLMMetric<'faithfulness'> {
-  private model;
   constructor(options: FaithfulnessOptions);
@@ -83,3 +82,2 @@ /**
 declare class FactualCorrectness extends LLMMetric<'factual_correctness'> {
-  private model;
   private mode;
@@ -113,2 +111,143 @@ private beta;
 //#endregion
+//#region src/metrics/answer-similarity.d.ts
+/**
+ * Options for configuring the AnswerSimilarity metric
+ */
+interface AnswerSimilarityOptions {
+  /** The embedding model to use for semantic similarity calculation */
+  model: EmbeddingModel;
+  /**
+   * Optional threshold for binary output.
+   * If provided, scores >= threshold will be 1, otherwise 0.
+   */
+  threshold?: number;
+}
+/**
+ * AnswerSimilarity metric scores the semantic similarity between
+ * a ground truth (reference) answer and a generated answer (response).
+ *
+ * The metric works by:
+ * 1. Embedding both the reference and response using the provided embedding model
+ * 2. Normalizing the embeddings to unit vectors
+ * 3. Computing cosine similarity between the normalized embeddings
+ * 4. Optionally applying a threshold for binary output
+ *
+ * This metric requires:
+ * - response: The model's answer
+ * - reference: The ground truth/reference answer
+ *
+ * Score ranges from -1 to 1 (or 0 to 1 for binary output with threshold),
+ * where 1 means perfect semantic similarity.
+ *
+ * Based on the SAS paper: https://arxiv.org/pdf/2108.06130.pdf
+ */
+declare class AnswerSimilarity extends EmbeddingMetric<'answer_similarity'> {
+  protected threshold?: number;
+  constructor(options: AnswerSimilarityOptions);
+  /**
+   * Compute the semantic similarity score
+   */
+  protected computeScore(reference: string, response: string): Promise<number>;
+  /**
+   * Evaluate a single-turn sample
+   */
+  evaluateSingleTurn(sample: SingleTurnSample): Promise<MetricScore>;
+}
+//#endregion
+//#region src/metrics/context-recall.d.ts
+/**
+ * Options for configuring the ContextRecall metric
+ */
+interface ContextRecallOptions {
+  /** The language model to use for evaluation */
+  model: LanguageModel;
+}
+/**
+ * ContextRecall metric estimates the recall of the retrieved context by analyzing
+ * how much of the reference answer can be attributed to the retrieved contexts.
+ *
+ * The metric works by:
+ * 1. Taking each sentence in the reference answer
+ * 2. Classifying whether each sentence can be attributed to the retrieved contexts
+ * 3. Computing recall as the proportion of attributed sentences
+ *
+ * This metric requires:
+ * - query: The user's question
+ * - retrievedContexts: The source documents retrieved
+ * - reference: The ground truth answer
+ *
+ * Score ranges from 0 to 1, where 1 means all reference answer content
+ * can be attributed to the retrieved contexts (perfect recall).
+ */
+declare class ContextRecall extends LLMMetric<'context_recall'> {
+  constructor(options: ContextRecallOptions);
+  /**
+   * Classify each statement in the answer against the context
+   */
+  private classifyStatements;
+  /**
+   * Compute the context recall score
+   */
+  private computeScore;
+  /**
+   * Evaluate a single-turn sample
+   */
+  evaluateSingleTurn(sample: SingleTurnSample): Promise<MetricScore>;
+}
+//#endregion
+//#region src/metrics/noise-sensitivity.d.ts
+/**
+ * Options for configuring the NoiseSensitivity metric
+ */
+interface NoiseSensitivityOptions {
+  /** The language model to use for evaluation */
+  model: LanguageModel;
+  /** Mode for noise sensitivity evaluation: 'relevant' or 'irrelevant' */
+  mode?: 'relevant' | 'irrelevant';
+}
+/**
+ * NoiseSensitivity metric measures the extent to which the generated answer is influenced
+ * by noise or irrelevant information in the retrieved contexts.
+ *
+ * The metric operates in two modes:
+ * - **relevant**: Measures how many incorrect statements come from relevant retrieved contexts
+ * - **irrelevant**: Measures how many incorrect statements come from irrelevant retrieved contexts
+ *
+ * The metric works by:
+ * 1. Decomposing both the reference answer and the generated response into statements
+ * 2. For each retrieved context, evaluating which statements are supported
+ * 3. Identifying which contexts are relevant (support reference statements)
+ * 4. Computing the proportion of incorrect statements that are faithful to relevant/irrelevant contexts
+ *
+ * This metric requires:
+ * - query: The user's question
+ * - response: The model's answer
+ * - reference: The ground truth answer
+ * - retrievedContexts: The source documents used to generate the response
+ *
+ * Score ranges from 0 to 1, where:
+ * - In 'relevant' mode: Higher scores indicate more incorrect statements from relevant contexts
+ * - In 'irrelevant' mode: Higher scores indicate more incorrect statements from irrelevant contexts
+ */
+declare class NoiseSensitivity extends LLMMetric<'noise_sensitivity'> {
+  private mode;
+  constructor(options: NoiseSensitivityOptions);
+  /**
+   * Generate atomic statements from the answer
+   */
+  private generateStatements;
+  /**
+   * Evaluate faithfulness of statements against context
+   */
+  private evaluateStatements;
+  /**
+   * Compute the noise sensitivity score based on the mode
+   */
+  private computeScore;
+  /**
+   * Evaluate a single-turn sample
+   */
+  evaluateSingleTurn(sample: SingleTurnSample): Promise<MetricScore>;
+}
+//#endregion
 //#region src/utils.d.ts
@@ -126,2 +265,2 @@ /**
 //#endregion
-export { FactualCorrectness, type FactualCorrectnessOptions, Faithfulness, type FaithfulnessOptions, fbetaScore };
+export { AnswerSimilarity, type AnswerSimilarityOptions, ContextRecall, type ContextRecallOptions, FactualCorrectness, type FactualCorrectnessOptions, Faithfulness, type FaithfulnessOptions, NoiseSensitivity, type NoiseSensitivityOptions, fbetaScore };

dist/index.js

@@ -1,2 +0,2 @@
-import { generateObject } from "ai";
+import { cosineSimilarity, embedMany, generateObject } from "ai";
 import { z } from "zod";
@@ -50,3 +50,19 @@
 */
-var LLMMetric = class extends Metric {};
+var LLMMetric = class extends Metric {
+	model;
+	constructor(config) {
+		super(config);
+		this.model = config.model;
+	}
+};
+/**
+* Base class for metrics that use embeddings for evaluation
+*/
+var EmbeddingMetric = class extends Metric {
+	model;
+	constructor(config) {
+		super(config);
+		this.model = config.model;
+	}
+};
 
@@ -58,7 +74,7 @@ //#endregion
 */
-const StatementGeneratorOutputSchema = z.object({ statements: z.array(z.string()).describe("The generated statements") });
+const StatementGeneratorOutputSchema$1 = z.object({ statements: z.array(z.string()).describe("The generated statements") });
 /**
 * Schema for a single faithfulness verdict
 */
-const StatementFaithfulnessAnswerSchema$1 = z.object({
+const StatementFaithfulnessAnswerSchema$2 = z.object({
 	statement: z.string().describe("the original statement, word-by-word"),
@@ -71,3 +87,3 @@ reason: z.string().describe("the reason of the verdict"),
 */
-const NLIStatementOutputSchema$1 = z.object({ statements: z.array(StatementFaithfulnessAnswerSchema$1) });
+const NLIStatementOutputSchema$2 = z.object({ statements: z.array(StatementFaithfulnessAnswerSchema$2) });
 /**
@@ -89,9 +105,8 @@ * Faithfulness metric evaluates how grounded the model's response is in the provided context.
 var Faithfulness = class extends LLMMetric {
-	model;
 	constructor(options) {
 		super({
 			name: "faithfulness",
-			description: "Evaluates the faithfulness of the model's response to the retrieved contexts"
+			description: "Evaluates the faithfulness of the model's response to the retrieved contexts",
+			model: options.model
 		});
-		this.model = options.model;
 	}
@@ -104,3 +119,3 @@ /**
 			model: this.model,
-			schema: StatementGeneratorOutputSchema,
+			schema: StatementGeneratorOutputSchema$1,
 			prompt: `Given a question and an answer, analyze the complexity of each sentence in the answer. Break down each sentence into one or more fully understandable statements. Ensure that no pronouns are used in any statement. Format the outputs in JSON.
@@ -133,3 +148,3 @@
 			model: this.model,
-			schema: NLIStatementOutputSchema$1,
+			schema: NLIStatementOutputSchema$2,
 			prompt: `Your task is to judge the faithfulness of a series of statements based on a given context. For each statement you must return verdict as 1 if the statement can be directly inferred based on the context or 0 if the statement can not be directly inferred based on the context.
@@ -257,3 +272,3 @@
 */
-const StatementFaithfulnessAnswerSchema = z.object({
+const StatementFaithfulnessAnswerSchema$1 = z.object({
 	statement: z.string().describe("the original statement, word-by-word"),
@@ -266,3 +281,3 @@ reason: z.string().describe("the reason of the verdict"),
 */
-const NLIStatementOutputSchema = z.object({ statements: z.array(StatementFaithfulnessAnswerSchema) });
+const NLIStatementOutputSchema$1 = z.object({ statements: z.array(StatementFaithfulnessAnswerSchema$1) });
 /**
@@ -285,3 +300,2 @@ * FactualCorrectness metric evaluates the factual correctness of responses
 var FactualCorrectness = class extends LLMMetric {
-	model;
 	mode;
@@ -295,5 +309,5 @@ beta;
 			name: "factual_correctness",
-			description: "Evaluates the factual correctness of responses against reference answers"
+			description: "Evaluates the factual correctness of responses against reference answers",
+			model: options.model
 		});
-		this.model = options.model;
 		this.mode = options.mode ?? "f1";
@@ -403,3 +417,3 @@ this.beta = options.beta ?? 1;
 			model: this.model,
-			schema: NLIStatementOutputSchema,
+			schema: NLIStatementOutputSchema$1,
 			prompt: `Your task is to judge the faithfulness of a series of statements based on a given context. For each statement you must return verdict as 1 if the statement can be directly inferred based on the context or 0 if the statement can not be directly inferred based on the context.
@@ -494,2 +508,445 @@
 //#endregion
-export { FactualCorrectness, Faithfulness, fbetaScore };
+//#region src/metrics/answer-similarity.ts
+/**
+* AnswerSimilarity metric scores the semantic similarity between
+* a ground truth (reference) answer and a generated answer (response).
+*
+* The metric works by:
+* 1. Embedding both the reference and response using the provided embedding model
+* 2. Normalizing the embeddings to unit vectors
+* 3. Computing cosine similarity between the normalized embeddings
+* 4. Optionally applying a threshold for binary output
+*
+* This metric requires:
+* - response: The model's answer
+* - reference: The ground truth/reference answer
+*
+* Score ranges from -1 to 1 (or 0 to 1 for binary output with threshold),
+* where 1 means perfect semantic similarity.
+*
+* Based on the SAS paper: https://arxiv.org/pdf/2108.06130.pdf
+*/
+var AnswerSimilarity = class extends EmbeddingMetric {
+	threshold;
+	constructor(options) {
+		super({
+			name: "answer_similarity",
+			description: "Scores the semantic similarity of ground truth with generated answer using embeddings",
+			model: options.model
+		});
+		this.threshold = options.threshold;
+	}
+	/**
+	* Compute the semantic similarity score
+	*/
+	async computeScore(reference, response) {
+		const groundTruth = reference || " ";
+		const answer = response || " ";
+		const { embeddings } = await embedMany({
+			model: this.model,
+			values: [groundTruth, answer]
+		});
+		const [embedding1, embedding2] = embeddings;
+		let score = cosineSimilarity(embedding1, embedding2);
+		if (this.threshold !== void 0) score = score >= this.threshold ? 1 : 0;
+		return score;
+	}
+	/**
+	* Evaluate a single-turn sample
+	*/
+	async evaluateSingleTurn(sample) {
+		if (!sample.reference) throw new Error("AnswerSimilarity metric requires reference to be present");
+		try {
+			const score = await this.computeScore(sample.reference, sample.response);
+			const roundedScore = Math.round(score * 1e4) / 1e4;
+			return {
+				name: this.name,
+				score: roundedScore,
+				reason: this.threshold ? `Answer similarity ${roundedScore >= this.threshold ? "meets" : "does not meet"} threshold of ${this.threshold}` : `Answer similarity score: ${roundedScore.toFixed(4)}`
+			};
+		} catch (error) {
+			throw new Error(`Failed to evaluate answer similarity: ${error instanceof Error ? error.message : String(error)}`);
+		}
+	}
+};
+
+//#endregion
+//#region src/metrics/context-recall.ts
+/**
+* Schema for a single context recall classification
+*/
+const ContextRecallClassificationSchema = z.object({
+	statement: z.string().describe("The statement from the answer"),
+	reason: z.string().describe("The reason for the attribution decision"),
+	attributed: z.number().int().min(0).max(1).describe("Whether the statement can be attributed to the context (0 or 1)")
+});
+/**
+* Schema for context recall classifications output
+*/
+const ContextRecallClassificationsSchema = z.object({ classifications: z.array(ContextRecallClassificationSchema) });
+/**
+* ContextRecall metric estimates the recall of the retrieved context by analyzing
+* how much of the reference answer can be attributed to the retrieved contexts.
+*
+* The metric works by:
+* 1. Taking each sentence in the reference answer
+* 2. Classifying whether each sentence can be attributed to the retrieved contexts
+* 3. Computing recall as the proportion of attributed sentences
+*
+* This metric requires:
+* - query: The user's question
+* - retrievedContexts: The source documents retrieved
+* - reference: The ground truth answer
+*
+* Score ranges from 0 to 1, where 1 means all reference answer content
+* can be attributed to the retrieved contexts (perfect recall).
+*/
+var ContextRecall = class extends LLMMetric {
+	constructor(options) {
+		super({
+			name: "context_recall",
+			description: "Estimates context recall by analyzing how much of the reference answer can be attributed to retrieved contexts",
+			model: options.model
+		});
+	}
+	/**
+	* Classify each statement in the answer against the context
+	*/
+	async classifyStatements(question, context, answer) {
+		return (await generateObject({
+			model: this.model,
+			schema: ContextRecallClassificationsSchema,
+			prompt: `Given a context, and an answer, analyze each sentence in the answer and classify if the sentence can be attributed to the given context or not. Use only 'Yes' (1) or 'No' (0) as a binary classification. Output json with reason.
+
+Example:
+Question: "What can you tell me about albert Albert Einstein?"
+Context: "Albert Einstein (14 March 1879 - 18 April 1955) was a German-born theoretical physicist, widely held to be one of the greatest and most influential scientists of all time. Best known for developing the theory of relativity, he also made important contributions to quantum mechanics, and was thus a central figure in the revolutionary reshaping of the scientific understanding of nature that modern physics accomplished in the first decades of the twentieth century. His mass-energy equivalence formula E = mc2, which arises from relativity theory, has been called 'the world's most famous equation'. He received the 1921 Nobel Prize in Physics 'for his services to theoretical physics, and especially for his discovery of the law of the photoelectric effect', a pivotal step in the development of quantum theory. His work is also known for its influence on the philosophy of science. In a 1999 poll of 130 leading physicists worldwide by the British journal Physics World, Einstein was ranked the greatest physicist of all time. His intellectual achievements and originality have made Einstein synonymous with genius."
+Answer: "Albert Einstein, born on 14 March 1879, was a German-born theoretical physicist, widely held to be one of the greatest and most influential scientists of all time. He received the 1921 Nobel Prize in Physics for his services to theoretical physics. He published 4 papers in 1905. Einstein moved to Switzerland in 1895."
+
+Expected Output:
+{
+  "classifications": [
+    {
+      "statement": "Albert Einstein, born on 14 March 1879, was a German-born theoretical physicist, widely held to be one of the greatest and most influential scientists of all time.",
+      "reason": "The date of birth of Einstein is mentioned clearly in the context.",
+      "attributed": 1
+    },
+    {
+      "statement": "He received the 1921 Nobel Prize in Physics for his services to theoretical physics.",
+      "reason": "The exact sentence is present in the given context.",
+      "attributed": 1
+    },
+    {
+      "statement": "He published 4 papers in 1905.",
+      "reason": "There is no mention about papers he wrote in the given context.",
+      "attributed": 0
+    },
+    {
+      "statement": "Einstein moved to Switzerland in 1895.",
+      "reason": "There is no supporting evidence for this in the given context.",
+      "attributed": 0
+    }
+  ]
+}
+
+Now analyze this:
+Question: "${question}"
+Context: "${context}"
+Answer: "${answer}"`
+		})).object;
+	}
+	/**
+	* Compute the context recall score
+	*/
+	computeScore(classifications) {
+		if (classifications.length === 0) return NaN;
+		return classifications.filter((c) => c.attributed === 1).length / classifications.length;
+	}
+	/**
+	* Evaluate a single-turn sample
+	*/
+	async evaluateSingleTurn(sample) {
+		if (!sample.retrievedContexts || sample.retrievedContexts.length === 0) throw new Error("ContextRecall metric requires retrievedContexts to be present");
+		if (!sample.reference) throw new Error("ContextRecall metric requires reference to be present");
+		try {
+			const context = sample.retrievedContexts.join("\n");
+			const classifications = await this.classifyStatements(sample.query, context, sample.reference);
+			if (classifications.classifications.length === 0) return {
+				name: this.name,
+				score: 0,
+				reason: "No statements were found in the reference answer"
+			};
+			const score = this.computeScore(classifications.classifications);
+			if (isNaN(score)) return {
+				name: this.name,
+				score: 0,
+				reason: "The LLM did not return a valid classification"
+			};
+			const roundedScore = Math.round(score * 1e4) / 1e4;
+			return {
+				name: this.name,
+				score: roundedScore,
+				reason: `${classifications.classifications.filter((c) => c.attributed === 1).length} out of ${classifications.classifications.length} statements from the reference answer were attributed to the retrieved contexts`,
+				metadata: { classifications: classifications.classifications }
+			};
+		} catch (error) {
+			throw new Error(`Failed to evaluate context recall: ${error instanceof Error ? error.message : String(error)}`);
+		}
+	}
+};
+
+//#endregion
+//#region src/metrics/noise-sensitivity.ts
+/**
+* Schema for statement generation output
+*/
+const StatementGeneratorOutputSchema = z.object({ statements: z.array(z.string()).describe("The generated statements") });
+/**
+* Schema for a single faithfulness verdict
+*/
+const StatementFaithfulnessAnswerSchema = z.object({
+	statement: z.string().describe("the original statement, word-by-word"),
+	reason: z.string().describe("the reason of the verdict"),
+	verdict: z.number().int().min(0).max(1).describe("the verdict (0/1) of the faithfulness")
+});
+/**
+* Schema for NLI statement output
+*/
+const NLIStatementOutputSchema = z.object({ statements: z.array(StatementFaithfulnessAnswerSchema) });
+/**
+* NoiseSensitivity metric measures the extent to which the generated answer is influenced
+* by noise or irrelevant information in the retrieved contexts.
+*
+* The metric operates in two modes:
+* - **relevant**: Measures how many incorrect statements come from relevant retrieved contexts
+* - **irrelevant**: Measures how many incorrect statements come from irrelevant retrieved contexts
+*
+* The metric works by:
+* 1. Decomposing both the reference answer and the generated response into statements
+* 2. For each retrieved context, evaluating which statements are supported
+* 3. Identifying which contexts are relevant (support reference statements)
+* 4. Computing the proportion of incorrect statements that are faithful to relevant/irrelevant contexts
+*
+* This metric requires:
+* - query: The user's question
+* - response: The model's answer
+* - reference: The ground truth answer
+* - retrievedContexts: The source documents used to generate the response
+*
+* Score ranges from 0 to 1, where:
+* - In 'relevant' mode: Higher scores indicate more incorrect statements from relevant contexts
+* - In 'irrelevant' mode: Higher scores indicate more incorrect statements from irrelevant contexts
+*/
+var NoiseSensitivity = class extends LLMMetric {
+	mode;
+	constructor(options) {
+		super({
+			name: "noise_sensitivity",
+			description: `Measures the extent to which the generated answer is influenced by noise in retrieved contexts (mode: ${options.mode ?? "relevant"})`,
+			model: options.model
+		});
+		this.mode = options.mode ?? "relevant";
+		if (this.mode !== "relevant" && this.mode !== "irrelevant") throw new Error(`Invalid argument passed for 'mode': ${this.mode}. Must be 'relevant' or 'irrelevant'.`);
+	}
+	/**
+	* Generate atomic statements from the answer
+	*/
+	async generateStatements(question, answer) {
+		return (await generateObject({
+			model: this.model,
+			schema: StatementGeneratorOutputSchema,
+			prompt: `Given a question and an answer, analyze the complexity of each sentence in the answer. Break down each sentence into one or more fully understandable statements. Ensure that no pronouns are used in any statement. Format the outputs in JSON.
+
+Example:
+Question: "Who was Albert Einstein and what is he best known for?"
+Answer: "He was a German-born theoretical physicist, widely acknowledged to be one of the greatest and most influential physicists of all time. He was best known for developing the theory of relativity, he also made important contributions to the development of the theory of quantum mechanics."
+
+Expected Output:
+{
+  "statements": [
+    "Albert Einstein was a German-born theoretical physicist.",
+    "Albert Einstein is recognized as one of the greatest and most influential physicists of all time.",
+    "Albert Einstein was best known for developing the theory of relativity.",
+    "Albert Einstein also made important contributions to the development of the theory of quantum mechanics."
+  ]
+}
+
+Now analyze this:
+Question: "${question}"
+Answer: "${answer}"`
+		})).object;
+	}
+	/**
+	* Evaluate faithfulness of statements against context
+	*/
+	async evaluateStatements(context, statements) {
+		return (await generateObject({
+			model: this.model,
+			schema: NLIStatementOutputSchema,
+			prompt: `Your task is to judge the faithfulness of a series of statements based on a given context. For each statement you must return verdict as 1 if the statement can be directly inferred based on the context or 0 if the statement can not be directly inferred based on the context.
+
+Example 1:
+Context: "John is a student at XYZ University. He is pursuing a degree in Computer Science. He is enrolled in several courses this semester, including Data Structures, Algorithms, and Database Management. John is a diligent student and spends a significant amount of time studying and completing assignments. He often stays late in the library to work on his projects."
+
+Statements:
+1. "John is majoring in Biology."
+2. "John is taking a course on Artificial Intelligence."
+3. "John is a dedicated student."
+4. "John has a part-time job."
+
+Expected Output:
+{
+  "statements": [
+    {
+      "statement": "John is majoring in Biology.",
+      "reason": "John's major is explicitly mentioned as Computer Science. There is no information suggesting he is majoring in Biology.",
+      "verdict": 0
+    },
+    {
+      "statement": "John is taking a course on Artificial Intelligence.",
+      "reason": "The context mentions the courses John is currently enrolled in, and Artificial Intelligence is not mentioned. Therefore, it cannot be deduced that John is taking a course on AI.",
+      "verdict": 0
+    },
+    {
+      "statement": "John is a dedicated student.",
+      "reason": "The context states that he spends a significant amount of time studying and completing assignments. Additionally, it mentions that he often stays late in the library to work on his projects, which implies dedication.",
+      "verdict": 1
+    },
+    {
+      "statement": "John has a part-time job.",
+      "reason": "There is no information given in the context about John having a part-time job.",
+      "verdict": 0
+    }
+  ]
+}
+
+Example 2:
+Context: "Photosynthesis is a process used by plants, algae, and certain bacteria to convert light energy into chemical energy."
+
+Statements:
+1. "Albert Einstein was a genius."
+
+Expected Output:
+{
+  "statements": [
+    {
+      "statement": "Albert Einstein was a genius.",
+      "reason": "The context and statement are unrelated",
+      "verdict": 0
+    }
+  ]
+}
+
+Now evaluate these:
+Context: "${context}"
+
+Statements:
+${statements.map((s, i) => `${i + 1}. "${s}"`).join("\n")}`
+		})).object;
+	}
+	/**
+	* Compute the noise sensitivity score based on the mode
+	*/
+	computeScore(answers) {
+		const { retrieved2GroundTruth, retrieved2Answer, groundTruth2Answer } = answers;
+		const incorrect = groundTruth2Answer.map((v) => !v);
+		if (retrieved2Answer.length === 0 || incorrect.every((v) => !v)) return 0;
+		const numContexts = retrieved2GroundTruth.length;
+		const relevantRetrieved = new Array(numContexts).fill(false);
+		for (let ctxIdx = 0; ctxIdx < numContexts; ctxIdx++) {
+			const contextVerdicts = retrieved2GroundTruth[ctxIdx];
+			if (!contextVerdicts) continue;
+			for (let stmtIdx = 0; stmtIdx < contextVerdicts.length; stmtIdx++) if (contextVerdicts[stmtIdx]) {
+				relevantRetrieved[ctxIdx] = true;
+				break;
+			}
+		}
+		const relevantFaithful = retrieved2Answer.map((answerVerdicts) => {
+			const maxIdx = Math.min(answerVerdicts.length, relevantRetrieved.length);
+			for (let ctxIdx = 0; ctxIdx < maxIdx; ctxIdx++) if (relevantRetrieved[ctxIdx] && answerVerdicts[ctxIdx]) return true;
+			return false;
+		});
+		if (this.mode === "irrelevant") {
+			const irrelevantFaithful = retrieved2Answer.map((answerVerdicts) => {
+				const maxIdx = Math.min(answerVerdicts.length, relevantRetrieved.length);
+				for (let ctxIdx = 0; ctxIdx < maxIdx; ctxIdx++) if (!relevantRetrieved[ctxIdx] && answerVerdicts[ctxIdx]) return true;
+				return false;
+			});
+			const minLength = Math.min(irrelevantFaithful.length, relevantFaithful.length);
+			const exclusiveIrrelevantFaithful = new Array(minLength).fill(false).map((_, idx) => irrelevantFaithful[idx] && !relevantFaithful[idx]);
+			const countLength = Math.min(exclusiveIrrelevantFaithful.length, incorrect.length);
+			if (countLength === 0) return 0;
+			let count = 0;
+			for (let i = 0; i < countLength; i++) if (exclusiveIrrelevantFaithful[i] && incorrect[i]) count++;
+			return count / incorrect.length;
+		} else {
+			const countLength = Math.min(relevantFaithful.length, incorrect.length);
+			if (countLength === 0) return 0;
+			let count = 0;
+			for (let i = 0; i < countLength; i++) if (relevantFaithful[i] && incorrect[i]) count++;
+			return count / incorrect.length;
+		}
+	}
+	/**
+	* Evaluate a single-turn sample
+	*/
+	async evaluateSingleTurn(sample) {
+		if (!sample.retrievedContexts || sample.retrievedContexts.length === 0) throw new Error("NoiseSensitivity metric requires retrievedContexts to be present");
+		if (!sample.reference) throw new Error("NoiseSensitivity metric requires reference to be present");
+		try {
+			const [groundTruthStatements, answerStatements] = await Promise.all([this.generateStatements(sample.query, sample.reference), this.generateStatements(sample.query, sample.response)]);
+			if (groundTruthStatements.statements.length === 0) return {
+				name: this.name,
+				score: 0,
+				reason: "No statements were generated from the reference answer"
+			};
+			if (answerStatements.statements.length === 0) return {
+				name: this.name,
+				score: 0,
+				reason: "No statements were generated from the response"
+			};
+			const gtVerdictsPromises = sample.retrievedContexts.map((ctx) => this.evaluateStatements(ctx, groundTruthStatements.statements));
+			const ansVerdictsPromises = sample.retrievedContexts.map((ctx) => this.evaluateStatements(ctx, answerStatements.statements));
+			const [gtVerdictsList, ansVerdictsList] = await Promise.all([Promise.all(gtVerdictsPromises), Promise.all(ansVerdictsPromises)]);
+			const groundTruth2AnswerVerdicts = await this.evaluateStatements(sample.reference, answerStatements.statements);
+			const expectedGtLength = groundTruthStatements.statements.length;
+			const expectedAnsLength = answerStatements.statements.length;
+			for (let i = 0; i < gtVerdictsList.length; i++) {
+				const actualLength = gtVerdictsList[i].statements.length;
+				if (actualLength !== expectedGtLength) throw new Error(`LLM returned ${actualLength} verdicts for ground truth statements against context ${i}, expected ${expectedGtLength}. This indicates an issue with the evaluation model's response.`);
+			}
+			for (let i = 0; i < ansVerdictsList.length; i++) {
+				const actualLength = ansVerdictsList[i].statements.length;
+				if (actualLength !== expectedAnsLength) throw new Error(`LLM returned ${actualLength} verdicts for answer statements against context ${i}, expected ${expectedAnsLength}. This indicates an issue with the evaluation model's response.`);
+			}
+			if (groundTruth2AnswerVerdicts.statements.length !== expectedAnsLength) throw new Error(`LLM returned ${groundTruth2AnswerVerdicts.statements.length} verdicts for answer statements against reference, expected ${expectedAnsLength}. This indicates an issue with the evaluation model's response.`);
+			const retrieved2GroundTruth = gtVerdictsList.map((verdicts) => verdicts.statements.map((v) => v.verdict === 1));
+			const retrieved2Answer = answerStatements.statements.map((_, stmtIdx) => ansVerdictsList.map((verdicts) => verdicts.statements[stmtIdx].verdict === 1));
+			const groundTruth2Answer = groundTruth2AnswerVerdicts.statements.map((v) => v.verdict === 1);
+			const score = this.computeScore({
+				retrieved2GroundTruth,
+				retrieved2Answer,
+				groundTruth2Answer
+			});
+			const roundedScore = Math.round(score * 1e4) / 1e4;
+			const incorrectCount = groundTruth2Answer.filter((v) => !v).length;
+			const modeDescription = this.mode === "relevant" ? "relevant retrieved contexts" : "irrelevant retrieved contexts";
+			return {
+				name: this.name,
+				score: roundedScore,
+				reason: `${Math.round(score * incorrectCount)} out of ${incorrectCount} incorrect statements were faithful to ${modeDescription}`,
+				metadata: {
+					mode: this.mode,
+					groundTruthStatements: groundTruthStatements.statements,
+					answerStatements: answerStatements.statements,
+					incorrectStatementsCount: incorrectCount
+				}
+			};
+		} catch (error) {
+			throw new Error(`Failed to evaluate noise sensitivity: ${error instanceof Error ? error.message : String(error)}`);
+		}
+	}
+};
+
+//#endregion
+export { AnswerSimilarity, ContextRecall, FactualCorrectness, Faithfulness, NoiseSensitivity, fbetaScore };

package.json

 {
   "name": "@open-evals/metrics",
-  "version": "0.1.0",
+  "version": "0.1.1",
   "description": "Metrics for Open Evals",
@@ -5,0 +5,0 @@ "author": {

New alerts

Long strings

Supply chain risk

Contains long string literals, which may be a sign of obfuscated or packed code.

Improved metrics

Total package byte prevSize

49562

112.33%

Lines of code

1162

101.39%

Worsened metrics

Number of low supply chain risk alerts

6

100%

No dependency changes