@looptail/sdk - npm Package Compare versions

+106

dist/evals.d.ts

		import type { Looptail } from './index.js';
		import type { TrailEvent } from './chain.js';
		export declare const DEFAULT_JUDGE = "anthropic:claude-opus-4-8";
		export declare const JUDGE_SYSTEM: string;
		export declare const JUDGE_SCHEMA: {
		readonly type: "object";
		readonly properties: {
		readonly criteria: {
		readonly type: "array";
		readonly items: {
		readonly type: "object";
		readonly properties: {
		readonly index: {
		readonly type: "integer";
		};
		readonly score: {
		readonly type: "number";
		};
		readonly note: {
		readonly type: "string";
		};
		};
		readonly required: readonly ["index", "score", "note"];
		readonly additionalProperties: false;
		};
		};
		readonly overall_score: {
		readonly type: "number";
		};
		readonly reasoning: {
		readonly type: "string";
		};
		};
		readonly required: readonly ["criteria", "overall_score", "reasoning"];
		readonly additionalProperties: false;
		};
		export interface Rubric {
		name: string;
		criteria: string[];
		version: number;
		description: string;
		passThreshold: number;
		}
		/** Load a rubric from a JSON file (name, criteria[], version?, description?,
		* pass_threshold?). */
		export declare function loadRubric(path: string): Rubric;
		export interface Verdict {
		score: number;
		pass: boolean;
		reasoning: string;
		criteriaScores: {
		index: number;
		score: number;
		}[];
		judge: string;
		}
		export interface Judge {
		name: string;
		judge(rubric: Rubric, event: TrailEvent): Promise<Verdict>;
		}
		export declare function judgePrompt(rubric: Rubric, event: Partial<TrailEvent>): string;
		export declare function verdictFrom(data: {
		overall_score: number;
		reasoning?: string;
		criteria?: {
		index: number;
		score: number;
		}[];
		}, rubric: Rubric, judgeName: string): Verdict;
		export declare class JudgeError extends Error {
		}
		/** LLM judge on the Anthropic API. Needs `@anthropic-ai/sdk` installed. */
		export declare class AnthropicJudge implements Judge {
		private readonly client;
		private readonly model;
		readonly name: string;
		private constructor();
		static create(model?: string): Promise<AnthropicJudge>;
		judge(rubric: Rubric, event: TrailEvent): Promise<Verdict>;
		}
		/** LLM judge on the OpenAI API. Needs `openai` installed. */
		export declare class OpenAIJudge implements Judge {
		private readonly client;
		private readonly model;
		readonly name: string;
		private constructor();
		static create(model: string): Promise<OpenAIJudge>;
		judge(rubric: Rubric, event: TrailEvent): Promise<Verdict>;
		}
		/** Build a judge from a "provider:model" spec, e.g. "anthropic:claude-opus-4-8". */
		export declare function judgeFromSpec(spec: string): Promise<Judge>;
		export interface EvalSummary {
		event: string;
		evaluation: string;
		score: number;
		pass: boolean;
		}
		export interface RunOptions {
		kinds?: string[];
		limit?: number;
		skipEvaluated?: boolean;
		onResult?: (summary: EvalSummary) => void;
		}
		/** Score recorded events and append signed `evaluate` events. Idempotent by
		* default: events already scored by this rubric name+version are skipped. */
		export declare function run(rubric: Rubric, client: Looptail, judge: Judge, opts?: RunOptions): Promise<EvalSummary[]>;

+254

dist/evals.js

		/**
		* Continuous evaluators — the Understand phase, in TypeScript.
		*
		* Score recorded loop events against a rubric with an LLM judge on a model you
		* choose, and write the verdicts back into the trail as signed `evaluate`
		* events. Mirrors the Python `looptail.evals` (prompt, schema, event shape)
		* byte-for-byte, so a JS verdict and a Python one agree — and so the hosted
		* evaluator produces the same records.
		*
		* import { Looptail, loadRubric, judgeFromSpec, run } from '@looptail/sdk';
		*
		* const lt = new Looptail({ app: 'support-agent' });
		* const judge = await judgeFromSpec('anthropic:claude-opus-4-8');
		* await run(loadRubric('rubrics/refund-policy.json'), lt, judge);
		*
		* The provider SDKs (`@anthropic-ai/sdk`, `openai`) are optional peer deps,
		* imported only when a judge is built — so importing this module never
		* requires them.
		*/
		import { readFileSync } from 'node:fs';
		/** Import an optional peer dependency by name. The variable specifier keeps
		* TypeScript from resolving (and requiring) the module at build time — it's
		* only needed at runtime when a judge is actually built. */
		async function optionalImport(name) {
		return import(name);
		}
		export const DEFAULT_JUDGE = 'anthropic:claude-opus-4-8';
		export const JUDGE_SYSTEM = 'You are a rubric evaluator for production AI systems. You are given a ' +
		'recorded decision (function call, inputs, output) and a rubric written by ' +
		'the team that owns the system. Score how well the recorded decision meets ' +
		'EACH criterion from 0.0 (clear violation) to 1.0 (clearly met), using the ' +
		'recorded evidence only — do not invent context. Be strict: if the record ' +
		'does not contain enough evidence that a criterion is met, score it low and ' +
		'say what evidence is missing.';
		export const JUDGE_SCHEMA = {
		type: 'object',
		properties: {
		criteria: {
		type: 'array',
		items: {
		type: 'object',
		properties: {
		index: { type: 'integer' },
		score: { type: 'number' },
		note: { type: 'string' },
		},
		required: ['index', 'score', 'note'],
		additionalProperties: false,
		},
		},
		overall_score: { type: 'number' },
		reasoning: { type: 'string' },
		},
		required: ['criteria', 'overall_score', 'reasoning'],
		additionalProperties: false,
		};
		/** Load a rubric from a JSON file (name, criteria[], version?, description?,
		* pass_threshold?). */
		export function loadRubric(path) {
		const data = JSON.parse(readFileSync(path, 'utf8'));
		if (typeof data.name !== 'string')
		throw new Error(`rubric ${path}: missing "name"`);
		if (!Array.isArray(data.criteria) \|\| data.criteria.length === 0) {
		throw new Error(`rubric ${path}: "criteria" must be a non-empty array`);
		}
		const version = Number(data.version ?? 1);
		if (!Number.isInteger(version) \|\| version < 1) {
		throw new Error(`rubric ${path}: "version" must be a positive integer`);
		}
		const passThreshold = Number(data.pass_threshold ?? 0.8);
		if (!(passThreshold >= 0 && passThreshold <= 1)) {
		throw new Error(`rubric ${path}: "pass_threshold" must be between 0 and 1`);
		}
		return {
		name: data.name,
		criteria: data.criteria.map(String),
		version,
		description: String(data.description ?? ''),
		passThreshold,
		};
		}
		/** Distinguish "the optional peer dep isn't installed" (actionable) from a
		* real load failure (surface it, don't mask it). */
		function missingDepError(err, pkg) {
		const code = err?.code;
		if (code === 'ERR_MODULE_NOT_FOUND' \|\| code === 'MODULE_NOT_FOUND') {
		return new Error(`the ${pkg} judge needs the official SDK: npm i ${pkg}`);
		}
		return err instanceof Error ? err : new Error(String(err));
		}
		export function judgePrompt(rubric, event) {
		const criteria = rubric.criteria.map((c, i) => `${i}. ${c}`).join('\n');
		const record = JSON.stringify({ kind: event.kind, ts: event.ts, body: event.body }, null, 2);
		return (`## Rubric: ${rubric.name} (v${rubric.version})\n${rubric.description}\n\n` +
		`## Criteria\n${criteria}\n\n` +
		`## Recorded decision\n\`\`\`json\n${record}\n\`\`\`\n\n` +
		'Score each criterion by its index, then give an overall score (the ' +
		'weakest criterion should weigh heavily) and a two-sentence reasoning.');
		}
		export function verdictFrom(data, rubric, judgeName) {
		const score = Number(data.overall_score);
		return {
		score,
		// derive pass from the ROUNDED score that gets published/signed, so a
		// verifier recomputes the same pass — and so the local and hosted
		// evaluators agree (both round-then-compare)
		pass: Number(score.toFixed(3)) >= rubric.passThreshold,
		// slice by code points, not UTF-16 units, to match Python's [:500]
		reasoning: [...String(data.reasoning ?? '')].slice(0, 500).join(''),
		criteriaScores: (data.criteria ?? []).map((c) => {
		const s = Number(c.score);
		return { index: Number(c.index), score: Number.isFinite(s) ? s : 0 };
		}),
		judge: judgeName,
		};
		}
		export class JudgeError extends Error {
		}
		/** LLM judge on the Anthropic API. Needs `@anthropic-ai/sdk` installed. */
		export class AnthropicJudge {
		client;
		model;
		name;
		constructor(client, model) {
		this.client = client;
		this.model = model;
		this.name = `anthropic:${model}`;
		}
		static async create(model = 'claude-opus-4-8') {
		let Anthropic;
		try {
		Anthropic = (await optionalImport('@anthropic-ai/sdk')).default;
		}
		catch (err) {
		throw missingDepError(err, '@anthropic-ai/sdk');
		}
		return new AnthropicJudge(new Anthropic(), model);
		}
		async judge(rubric, event) {
		const res = (await this.client.messages.create({
		model: this.model,
		max_tokens: 2048,
		system: JUDGE_SYSTEM,
		messages: [{ role: 'user', content: judgePrompt(rubric, event) }],
		output_config: { format: { type: 'json_schema', schema: JUDGE_SCHEMA } },
		}));
		if (res.stop_reason !== 'end_turn')
		throw new JudgeError(`judge stopped: ${res.stop_reason}`);
		const text = res.content?.find((b) => b.type === 'text')?.text ?? '{}';
		return verdictFrom(JSON.parse(text), rubric, this.name);
		}
		}
		/** LLM judge on the OpenAI API. Needs `openai` installed. */
		export class OpenAIJudge {
		client;
		model;
		name;
		constructor(client, model) {
		this.client = client;
		this.model = model;
		this.name = `openai:${model}`;
		}
		static async create(model) {
		let OpenAI;
		try {
		OpenAI = (await optionalImport('openai')).default;
		}
		catch (err) {
		throw missingDepError(err, 'openai');
		}
		return new OpenAIJudge(new OpenAI(), model);
		}
		async judge(rubric, event) {
		const res = (await this.client.chat.completions.create({
		model: this.model,
		messages: [
		{ role: 'system', content: JUDGE_SYSTEM },
		{ role: 'user', content: judgePrompt(rubric, event) },
		],
		response_format: {
		type: 'json_schema',
		json_schema: { name: 'verdict', strict: true, schema: JUDGE_SCHEMA },
		},
		}));
		const content = res.choices?.[0]?.message?.content;
		if (!content)
		throw new JudgeError('judge returned no content');
		return verdictFrom(JSON.parse(content), rubric, this.name);
		}
		}
		/** Build a judge from a "provider:model" spec, e.g. "anthropic:claude-opus-4-8". */
		export async function judgeFromSpec(spec) {
		const idx = spec.indexOf(':');
		const provider = idx < 0 ? spec : spec.slice(0, idx);
		const model = idx < 0 ? '' : spec.slice(idx + 1);
		if (provider === 'anthropic')
		return AnthropicJudge.create(model \|\| 'claude-opus-4-8');
		if (provider === 'openai') {
		if (!model)
		throw new Error('openai judge needs a model, e.g. openai:<model-id>');
		return OpenAIJudge.create(model);
		}
		throw new Error(`unknown judge provider ${JSON.stringify(provider)} (use anthropic: or openai:)`);
		}
		function alreadyEvaluated(events, rubric) {
		const done = new Set();
		for (const e of events) {
		if (e.kind !== 'evaluate')
		continue;
		const body = e.body;
		if (body.rubric === rubric.name && body.rubric_version === rubric.version && e.ref) {
		done.add(e.ref);
		}
		}
		return done;
		}
		/** Score recorded events and append signed `evaluate` events. Idempotent by
		* default: events already scored by this rubric name+version are skipped. */
		export async function run(rubric, client, judge, opts = {}) {
		const kinds = opts.kinds ?? ['observe'];
		const events = client.store.readAll();
		const done = opts.skipEvaluated === false ? new Set() : alreadyEvaluated(events, rubric);
		let targets = events.filter((e) => kinds.includes(e.kind) && !done.has(e.id));
		if (opts.limit !== undefined)
		targets = targets.slice(0, opts.limit);
		const results = [];
		for (const event of targets) {
		const started = performance.now();
		const verdict = await judge.judge(rubric, event);
		const evalId = client.event('evaluate', {
		rubric: rubric.name,
		rubric_version: rubric.version,
		score: verdict.score.toFixed(3), // spec §5: floats travel as strings
		pass: verdict.pass,
		reasoning: verdict.reasoning,
		judge: verdict.judge,
		// per-criterion scores feed failure clustering (cluster())
		criteria: verdict.criteriaScores.map((c, i) => ({
		index: Number.isInteger(c.index) ? c.index : i,
		score: Number(c.score).toFixed(3),
		})),
		duration_ms: Math.round(performance.now() - started),
		}, event.id);
		const summary = {
		event: event.id,
		evaluation: evalId,
		score: verdict.score,
		pass: verdict.pass,
		};
		results.push(summary);
		opts.onResult?.(summary);
		}
		return results;
		}

+89

dist/improve.d.ts

		import type { Judge, Rubric } from './evals.js';
		import type { Looptail } from './index.js';
		export declare const DEFAULT_PROPOSER = "anthropic:claude-opus-4-8";
		export declare const PROPOSER_SYSTEM: string;
		export declare const PROPOSER_SCHEMA: {
		readonly type: "object";
		readonly properties: {
		readonly new_prompt: {
		readonly type: "string";
		};
		readonly rationale: {
		readonly type: "string";
		};
		};
		readonly required: readonly ["new_prompt", "rationale"];
		readonly additionalProperties: false;
		};
		export declare class GateError extends Error {
		}
		export interface Proposal {
		newPrompt: string;
		rationale: string;
		}
		export interface Proposer {
		name: string;
		propose(issue: Record<string, unknown>, currentPrompt: string, examples: unknown[]): Promise<Proposal>;
		}
		/** LLM proposer on the Anthropic API. Needs `@anthropic-ai/sdk` installed. */
		export declare class AnthropicProposer implements Proposer {
		private readonly client;
		private readonly model;
		readonly name: string;
		private constructor();
		static create(model?: string): Promise<AnthropicProposer>;
		propose(issue: Record<string, unknown>, currentPrompt: string, examples: unknown[]): Promise<Proposal>;
		}
		/** Build a proposer from a "provider:model" spec. */
		export declare function proposerFromSpec(spec: string): Promise<Proposer>;
		export interface ProposeResult {
		proposal: string;
		event: string;
		proposalFile: string;
		rationale: string;
		}
		/** Ask the proposer for a patched prompt; record it as an improve event. The
		* proposal is written under `<trail-dir>/proposals/` — the prompt file itself
		* is untouched until `approve({ apply: true })`. */
		export declare function propose(client: Looptail, issueKey: string, promptFile: string, proposer: Proposer, maxExamples?: number): Promise<ProposeResult>;
		/** A replay runner: your code, `(promptText, caseInput) => output`. */
		export type Runner = (promptText: string, caseInput: unknown) => unknown \| Promise<unknown>;
		/** Import a runner from "module:export" (a module path/specifier + a named or
		* default export). */
		export declare function loadRunner(spec: string): Promise<Runner>;
		/** A regression set: JSONL, one case per line. Lines with an `input` field use
		* it; other lines are treated as the input itself. */
		export declare function loadCases(path: string): {
		input: unknown;
		}[];
		export interface ReplayResult {
		event: string;
		total: number;
		passed: number;
		failed: number;
		pass: boolean;
		}
		/** Run the regression set under the proposed prompt, judge every output
		* against the rubric, and record the results as an improve event. */
		export declare function replay(client: Looptail, proposalId: string, runner: Runner, rubric: Rubric, judge: Judge, cases: {
		input: unknown;
		}[]): Promise<ReplayResult>;
		/** Record the intent to roll the proposal to a slice of traffic. Gated on a
		* passing replay; the rollout is your deploy's job — the trail records that it
		* happened, at what percentage, and on what evidence. */
		export declare function canary(client: Looptail, proposalId: string, percent: number, force?: boolean): {
		event: string;
		percent: number;
		forced: boolean;
		};
		/** The approval gate: sign off on a proposal, optionally applying it. Refuses
		* without a passing replay unless forced — and a forced approval says so in
		* the event body, permanently. */
		export declare function approve(client: Looptail, proposalId: string, decidedBy: string, opts?: {
		apply?: boolean;
		force?: boolean;
		}): {
		event: string;
		applied: boolean;
		forced: boolean;
		};

+318

dist/improve.js

		/**
		* The Improve phase — propose, replay, canary, approve — in TypeScript.
		*
		* A tracked issue (from `cluster`) becomes a patch proposal, the proposal is
		* replayed against a regression set, optionally canaried, and finally
		* approved — with every step recorded in the trail as a signed `improve` or
		* `approve` event. A faithful port of Python `looptail.improve`; the event
		* shapes match, so a JS-driven improvement and a Python-driven one read the
		* same.
		*
		* Approval is gated: without a passing replay on record, `approve` and
		* `canary` refuse (`force` overrides, and the override is recorded).
		* Proposers are LLMs behind the same `provider:model` spec as judges; replay
		* runners are your own code.
		*/
		import { existsSync, mkdirSync, readFileSync, writeFileSync } from 'node:fs';
		import { dirname, join } from 'node:path';
		import { nowIso, ulid } from './chain.js';
		import { latestIssues } from './issues.js';
		export const DEFAULT_PROPOSER = 'anthropic:claude-opus-4-8';
		const MAX_PREVIEW = 1000;
		export const PROPOSER_SYSTEM = 'You are a prompt engineer fixing a recurring failure in a production AI ' +
		'system. You are given the current prompt, the failing rubric criterion, ' +
		'and recorded examples of failures. Rewrite the prompt to fix the recurring ' +
		'failure while preserving everything that already works. Make the smallest ' +
		'change that plausibly fixes the failure — do not restructure or rewrite ' +
		'sections the failures do not implicate.';
		export const PROPOSER_SCHEMA = {
		type: 'object',
		properties: {
		new_prompt: { type: 'string' },
		rationale: { type: 'string' },
		},
		required: ['new_prompt', 'rationale'],
		additionalProperties: false,
		};
		export class GateError extends Error {
		}
		function preview(value, limit = MAX_PREVIEW) {
		let text;
		try {
		text = typeof value === 'string' ? value : (JSON.stringify(value) ?? String(value));
		}
		catch {
		text = String(value);
		}
		// measure and cut by code point, like Python's len()/[:limit] — never split a
		// surrogate pair (this text is fed to the proposer/judge, not the trail).
		const cps = [...text];
		return cps.length <= limit ? text : cps.slice(0, limit).join('') + '…';
		}
		/** str(PurePosixPath(p)) — mirror Python's path normalization so a JS-written
		* prompt_file field reads identically to a Python-written one. Strips a leading
		* "./", collapses "//" and "." segments, drops a trailing "/", and — unlike
		* node:path normalize — preserves ".." and the root, exactly as pathlib does. */
		function posixPath(p) {
		const abs = p.startsWith('/');
		const body = p.split('/').filter((s) => s !== '' && s !== '.').join('/');
		if (abs)
		return '/' + body;
		return body === '' ? '.' : body;
		}
		/** PurePosixPath(p).suffix — the final component's extension, but "" when the
		* name has no dot, starts with a dot, or ends with a dot (pathlib's rule:
		* 0 < rfind(".") < len-1). node:path extname disagrees on a trailing dot. */
		function pathSuffix(p) {
		const name = p.split('/').pop() ?? '';
		const i = name.lastIndexOf('.');
		return i > 0 && i < name.length - 1 ? name.slice(i) : '';
		}
		function proposerPrompt(issue, currentPrompt, examples) {
		const rendered = JSON.stringify(examples, null, 2);
		return (`## Tracked issue\n${issue.title ?? issue.key ?? ''}\n` +
		`(key ${issue.key}, ${issue.count} recorded failures)\n\n` +
		`## Current prompt\n\`\`\`\n${currentPrompt}\n\`\`\`\n\n` +
		`## Recorded failures\n\`\`\`json\n${rendered}\n\`\`\`\n\n` +
		'Return the full revised prompt and a short rationale explaining what you ' +
		'changed and why it fixes these failures.');
		}
		async function optionalImport(name) {
		return import(name);
		}
		/** LLM proposer on the Anthropic API. Needs `@anthropic-ai/sdk` installed. */
		export class AnthropicProposer {
		client;
		model;
		name;
		constructor(client, model) {
		this.client = client;
		this.model = model;
		this.name = `anthropic:${model}`;
		}
		static async create(model = 'claude-opus-4-8') {
		let Anthropic;
		try {
		Anthropic = (await optionalImport('@anthropic-ai/sdk')).default;
		}
		catch (err) {
		const code = err?.code;
		if (code === 'ERR_MODULE_NOT_FOUND' \|\| code === 'MODULE_NOT_FOUND') {
		throw new Error('the anthropic proposer needs the official SDK: npm i @anthropic-ai/sdk');
		}
		throw err instanceof Error ? err : new Error(String(err));
		}
		return new AnthropicProposer(new Anthropic(), model);
		}
		async propose(issue, currentPrompt, examples) {
		const res = (await this.client.messages.create({
		model: this.model,
		max_tokens: 8192,
		system: PROPOSER_SYSTEM,
		messages: [{ role: 'user', content: proposerPrompt(issue, currentPrompt, examples) }],
		output_config: { format: { type: 'json_schema', schema: PROPOSER_SCHEMA } },
		}));
		if (res.stop_reason !== 'end_turn')
		throw new Error(`proposer stopped with ${res.stop_reason}`);
		const data = JSON.parse(res.content?.find((b) => b.type === 'text')?.text ?? '{}');
		return { newPrompt: String(data.new_prompt), rationale: String(data.rationale) };
		}
		}
		/** Build a proposer from a "provider:model" spec. */
		export async function proposerFromSpec(spec) {
		const idx = spec.indexOf(':');
		const provider = idx < 0 ? spec : spec.slice(0, idx);
		const model = idx < 0 ? '' : spec.slice(idx + 1);
		if (provider === 'anthropic')
		return AnthropicProposer.create(model \|\| 'claude-opus-4-8');
		throw new Error(`unknown proposer provider ${JSON.stringify(provider)} (use anthropic:<model>)`);
		}
		function gatherExamples(client, issue, limit = 5) {
		const byId = new Map(client.store.readAll().map((e) => [e.id, e]));
		const examples = [];
		const refs = (issue.examples ?? []).slice(-limit);
		for (const ref of refs) {
		const verdict = byId.get(ref);
		if (!verdict)
		continue;
		const judged = byId.get(verdict.ref ?? '');
		const vbody = verdict.body;
		examples.push({
		verdict: { score: vbody.score, reasoning: vbody.reasoning },
		recorded: preview(judged?.body ?? {}),
		});
		}
		return examples;
		}
		function improveEvents(client, phase, proposalId) {
		return client.store.readAll().filter((e) => {
		if (e.kind !== 'improve')
		return false;
		const body = e.body;
		if (body.phase !== phase)
		return false;
		if (proposalId !== undefined && body.proposal !== proposalId)
		return false;
		return true;
		});
		}
		function requireProposal(client, proposalId) {
		const proposals = improveEvents(client, 'propose', proposalId);
		if (proposals.length === 0)
		throw new Error(`no proposal ${JSON.stringify(proposalId)} recorded in this trail`);
		return proposals[proposals.length - 1];
		}
		function passingReplay(client, proposalId) {
		const replays = improveEvents(client, 'replay', proposalId);
		const last = replays[replays.length - 1];
		return last && last.body.pass === true ? last : null;
		}
		/** Ask the proposer for a patched prompt; record it as an improve event. The
		* proposal is written under `<trail-dir>/proposals/` — the prompt file itself
		* is untouched until `approve({ apply: true })`. */
		export async function propose(client, issueKey, promptFile, proposer, maxExamples = 5) {
		const issue = latestIssues(client)[issueKey];
		if (!issue) {
		throw new Error(`no issue ${JSON.stringify(issueKey)} in this trail — run cluster() first`);
		}
		// normalize once, the way Python's Path(prompt_file) does, so the stored
		// prompt_file (and the approve event that reads it back) matches byte-for-byte.
		const promptPath = posixPath(promptFile);
		const current = readFileSync(promptPath, 'utf8');
		const examples = gatherExamples(client, issue, maxExamples);
		const proposal = await proposer.propose(issue, current, examples);
		const proposalId = `prop_${ulid()}`;
		const dir = join(dirname(client.store.path), 'proposals');
		mkdirSync(dir, { recursive: true });
		const proposalFile = join(dir, `${proposalId}${pathSuffix(promptPath) \|\| '.txt'}`);
		writeFileSync(proposalFile, proposal.newPrompt);
		const event = client.event('improve', {
		phase: 'propose',
		proposal: proposalId,
		issue: issueKey,
		prompt_file: promptPath,
		proposal_file: proposalFile,
		rationale: [...proposal.rationale].slice(0, 500).join(''),
		proposer: proposer.name,
		examples_used: examples.length,
		});
		return { proposal: proposalId, event, proposalFile, rationale: proposal.rationale };
		}
		/** Import a runner from "module:export" (a module path/specifier + a named or
		* default export). */
		export async function loadRunner(spec) {
		const at = spec.lastIndexOf(':');
		const moduleName = at < 0 ? spec : spec.slice(0, at);
		const attr = at < 0 ? 'default' : spec.slice(at + 1);
		if (!moduleName)
		throw new Error(`runner must look like module:export, got ${JSON.stringify(spec)}`);
		const mod = (await import(moduleName));
		const fn = mod[attr];
		if (typeof fn !== 'function')
		throw new Error(`${JSON.stringify(spec)} did not resolve to a callable`);
		return fn;
		}
		/** A regression set: JSONL, one case per line. Lines with an `input` field use
		* it; other lines are treated as the input itself. */
		export function loadCases(path) {
		const cases = [];
		for (const line of readFileSync(path, 'utf8').split('\n')) {
		if (!line.trim())
		continue;
		const data = JSON.parse(line);
		cases.push(data && typeof data === 'object' && 'input' in data ? data : { input: data });
		}
		if (cases.length === 0)
		throw new Error(`regression set ${path} is empty`);
		return cases;
		}
		/** Run the regression set under the proposed prompt, judge every output
		* against the rubric, and record the results as an improve event. */
		export async function replay(client, proposalId, runner, rubric, judge, cases) {
		const proposed = requireProposal(client, proposalId);
		const promptText = readFileSync(proposed.body.proposal_file, 'utf8');
		const started = performance.now();
		let passed = 0;
		let failed = 0;
		const scores = [];
		for (const c of cases) {
		const output = await runner(promptText, c.input);
		// judged exactly like live traffic: a recorded decision against the rubric
		const synthetic = { kind: 'replay', ts: nowIso(), body: { input: preview(c.input), output: preview(output) } };
		const verdict = await judge.judge(rubric, synthetic);
		scores.push(verdict.score.toFixed(3));
		if (verdict.pass)
		passed++;
		else
		failed++;
		}
		const event = client.event('improve', {
		phase: 'replay',
		proposal: proposalId,
		rubric: rubric.name,
		rubric_version: rubric.version,
		judge: judge.name,
		total: cases.length,
		passed,
		failed,
		pass: failed === 0,
		scores,
		duration_ms: Math.round(performance.now() - started),
		}, proposed.id);
		return { event, total: cases.length, passed, failed, pass: failed === 0 };
		}
		/** Record the intent to roll the proposal to a slice of traffic. Gated on a
		* passing replay; the rollout is your deploy's job — the trail records that it
		* happened, at what percentage, and on what evidence. */
		export function canary(client, proposalId, percent, force = false) {
		const proposed = requireProposal(client, proposalId);
		if (!(percent >= 1 && percent <= 100))
		throw new Error('percent must be 1-100');
		const replayed = passingReplay(client, proposalId);
		if (replayed === null && !force) {
		throw new GateError(`no passing replay recorded for ${proposalId} — run replay() first (or force to override)`);
		}
		const body = {
		phase: 'canary',
		proposal: proposalId,
		issue: proposed.body.issue,
		percent,
		};
		if (replayed === null)
		body.forced = true;
		const event = client.event('improve', body, (replayed ?? proposed).id);
		return { event, percent, forced: replayed === null };
		}
		/** The approval gate: sign off on a proposal, optionally applying it. Refuses
		* without a passing replay unless forced — and a forced approval says so in
		* the event body, permanently. */
		export function approve(client, proposalId, decidedBy, opts = {}) {
		const proposed = requireProposal(client, proposalId);
		const replayed = passingReplay(client, proposalId);
		if (replayed === null && !opts.force) {
		throw new GateError(`no passing replay recorded for ${proposalId} — run replay() first (or force to override)`);
		}
		const pbody = proposed.body;
		const promptFile = pbody.prompt_file;
		if (opts.apply) {
		if (!existsSync(pbody.proposal_file))
		throw new Error(`proposal file ${pbody.proposal_file} is missing`);
		writeFileSync(promptFile, readFileSync(pbody.proposal_file, 'utf8'));
		}
		const body = {
		proposal: proposalId,
		issue: proposed.body.issue,
		decided_by: decidedBy,
		applied: opts.apply === true,
		prompt_file: promptFile,
		};
		if (replayed !== null) {
		const rb = replayed.body;
		body.replay = { passed: rb.passed, failed: rb.failed, total: rb.total };
		}
		else {
		body.forced = true;
		}
		const event = client.event('approve', body, (replayed ?? proposed).id);
		return { event, applied: opts.apply === true, forced: replayed === null };
		}

+24

dist/instrument.d.ts

		/**
		* Auto-instrumentation for LLM provider clients.
		*
		* Wrap a client once; every API call becomes a signed `observe` event with the
		* model, truncated input/output previews, token usage, and duration:
		*
		* import { Looptail, instrumentAnthropic, instrumentOpenAI } from '@looptail/sdk';
		* import Anthropic from '@anthropic-ai/sdk';
		* import OpenAI from 'openai';
		*
		* const lt = new Looptail({ app: 'support-agent' });
		* const anthropic = instrumentAnthropic(new Anthropic(), lt);
		* const openai = instrumentOpenAI(new OpenAI(), lt);
		*
		* The wrappers mutate the given client in place (and return it), so existing
		* call sites keep working. Failures are recorded and re-thrown —
		* instrumentation never swallows errors. Mirrors Python `looptail.instrument`.
		*/
		import type { Looptail } from './index.js';
		/** Instrument an Anthropic client's `messages.create`. */
		export declare function instrumentAnthropic<T>(client: T, lt: Looptail): T;
		/** Instrument an OpenAI client's `chat.completions.create` (and
		* `responses.create` when present). */
		export declare function instrumentOpenAI<T>(client: T, lt: Looptail): T;

+108

dist/instrument.js

		const PREVIEW = 1000;
		const MARK = Symbol.for('looptail.instrumented');
		function preview(value) {
		let text;
		try {
		text = typeof value === 'string' ? value : (JSON.stringify(value) ?? String(value));
		}
		catch {
		text = String(value);
		}
		return text.length <= PREVIEW ? text : text.slice(0, PREVIEW) + '…';
		}
		function usageInts(usage, fields) {
		const out = {};
		if (usage && typeof usage === 'object') {
		for (const field of fields) {
		const value = usage[field];
		if (Number.isInteger(value))
		out[field] = value;
		}
		}
		return out;
		}
		/** Wrap a `create(args)` call, recording an observe event around it. Works for
		* sync-returning and promise-returning create methods. */
		function wrapCreate(lt, original, provider, api, extract) {
		const wrapped = function (...args) {
		const started = performance.now();
		const params = (args[0] ?? {});
		const base = {
		provider,
		api,
		model: String(params.model ?? ''),
		input: preview(params.messages ?? params.input ?? args),
		};
		const finish = (extra) => {
		Object.assign(base, extra, { duration_ms: Math.round(performance.now() - started) });
		lt.event('observe', base);
		};
		let result;
		try {
		result = original.apply(this, args);
		}
		catch (err) {
		finish({ error: err instanceof Error ? `${err.name}: ${err.message}` : preview(err) });
		throw err;
		}
		if (result instanceof Promise) {
		return result.then((response) => {
		finish(extract(response));
		return response;
		}, (err) => {
		finish({ error: err instanceof Error ? `${err.name}: ${err.message}` : preview(err) });
		throw err;
		});
		}
		finish(extract(result));
		return result;
		};
		wrapped[MARK] = true;
		return wrapped;
		}
		function instrumented(fn) {
		return typeof fn === 'function' && fn[MARK] === true;
		}
		/** Instrument an Anthropic client's `messages.create`. */
		export function instrumentAnthropic(client, lt) {
		const c = client;
		if (instrumented(c.messages.create))
		return client;
		const extract = (response) => {
		const r = (response ?? {});
		return {
		output: preview(r.content),
		stop_reason: String(r.stop_reason ?? ''),
		...usageInts(r.usage, ['input_tokens', 'output_tokens']),
		};
		};
		c.messages.create = wrapCreate(lt, c.messages.create.bind(c.messages), 'anthropic', 'messages.create', extract);
		return client;
		}
		/** Instrument an OpenAI client's `chat.completions.create` (and
		* `responses.create` when present). */
		export function instrumentOpenAI(client, lt) {
		const extract = (response) => {
		const r = (response ?? {});
		const out = {};
		if (r.choices && r.choices.length) {
		out.output = preview(r.choices[0].message ?? r.choices[0]);
		out.stop_reason = String(r.choices[0].finish_reason ?? '');
		}
		else if ('output_text' in r) {
		out.output = preview(r.output_text);
		}
		return {
		...out,
		...usageInts(r.usage, ['prompt_tokens', 'completion_tokens', 'input_tokens', 'output_tokens']),
		};
		};
		const c = client;
		if (!instrumented(c.chat.completions.create)) {
		c.chat.completions.create = wrapCreate(lt, c.chat.completions.create.bind(c.chat.completions), 'openai', 'chat.completions.create', extract);
		}
		if (c.responses && typeof c.responses.create === 'function' && !instrumented(c.responses.create)) {
		c.responses.create = wrapCreate(lt, c.responses.create.bind(c.responses), 'openai', 'responses.create', extract);
		}
		return client;
		}

+19

dist/issues.d.ts

		import type { TrailEvent } from './chain.js';
		import type { Looptail } from './index.js';
		export interface Issue {
		key: string;
		rubric: string;
		rubricVersion: number;
		criterionIndex: number;
		count: number;
		exampleRefs: string[];
		title: string;
		}
		/** The lowest-scoring criterion index from a verdict's body, or -1 when the
		* verdict carries no per-criterion detail. */
		export declare function weakestCriterion(evaluation: Partial<TrailEvent>): number;
		/** Group failing evaluate events into issues and record them. Cumulative
		* across runs; failures already attached to an issue are not re-counted. */
		export declare function cluster(client: Looptail, maxExamples?: number): Issue[];
		/** The most recent issue body per key (what the Improve loop consumes). */
		export declare function latestIssues(client: Looptail): Record<string, Record<string, unknown>>;

+110

dist/issues.js

		/**
		* Failure clustering — the connective tissue between Understand and Improve.
		*
		* Groups failing `evaluate` verdicts into tracked `issue` events by (rubric,
		* version, weakest criterion). Deterministic and explainable on purpose: no
		* embeddings, nothing opaque to argue with in a postmortem. A byte-for-byte
		* port of Python `looptail.issues`, so the issue events match.
		*
		* import { Looptail, cluster } from '@looptail/sdk';
		* const issues = cluster(new Looptail({ app: 'support-agent' }));
		*/
		import { compareCodePoints } from './chain.js';
		/** The lowest-scoring criterion index from a verdict's body, or -1 when the
		* verdict carries no per-criterion detail. */
		export function weakestCriterion(evaluation) {
		const body = (evaluation.body ?? {});
		const criteria = body.criteria ?? [];
		let bestIndex = -1;
		let bestScore = null;
		for (const entry of criteria) {
		const score = Number(entry.score);
		const index = Number(entry.index);
		if (!Number.isFinite(score) \|\| !Number.isInteger(index))
		continue;
		if (bestScore === null \|\| score < bestScore) {
		bestIndex = index;
		bestScore = score;
		}
		}
		return bestIndex;
		}
		/** Group failing evaluate events into issues and record them. Cumulative
		* across runs; failures already attached to an issue are not re-counted. */
		export function cluster(client, maxExamples = 5) {
		const events = client.store.readAll();
		const attached = new Set();
		const latestIssue = new Map();
		for (const event of events) {
		if (event.kind !== 'issue')
		continue;
		const body = event.body;
		latestIssue.set(String(body.key ?? ''), body);
		for (const ref of body.all_refs ?? [])
		attached.add(ref);
		}
		const groups = new Map();
		for (const event of events) {
		if (event.kind !== 'evaluate')
		continue;
		const body = event.body;
		if (body.pass !== false)
		continue;
		if (attached.has(event.id))
		continue;
		const rubric = String(body.rubric ?? 'unknown');
		const version = Number(body.rubric_version ?? 0);
		const criterion = weakestCriterion(event);
		const key = `${rubric}/v${version}/criterion-${criterion}`;
		(groups.get(key) ?? groups.set(key, []).get(key)).push(event);
		}
		const issues = [];
		// code-point order (not JS default UTF-16-unit order) so JS and Python
		// append issue events in the same sequence for the same trail
		for (const key of [...groups.keys()].sort(compareCodePoints)) {
		const failures = groups.get(key);
		const prior = latestIssue.get(key) ?? {};
		const priorRefs = (prior.all_refs ?? []).slice();
		const newRefs = failures.map((f) => f.id);
		const allRefs = priorRefs.concat(newRefs);
		const firstBody = failures[0].body;
		const rubric = String(firstBody.rubric ?? 'unknown');
		const version = Number(firstBody.rubric_version ?? 0);
		const criterionIndex = weakestCriterion(failures[0]);
		const title = prior.title \|\|
		`${rubric} v${version}: criterion ${criterionIndex} failing`;
		const issue = {
		key,
		rubric,
		rubricVersion: version,
		criterionIndex,
		count: allRefs.length,
		exampleRefs: allRefs.slice(-maxExamples),
		title,
		};
		client.event('issue', {
		key: issue.key,
		title: issue.title,
		rubric: issue.rubric,
		rubric_version: issue.rubricVersion,
		criterion_index: issue.criterionIndex,
		count: issue.count,
		examples: issue.exampleRefs,
		all_refs: allRefs,
		new_failures: newRefs.length,
		});
		issues.push(issue);
		}
		return issues;
		}
		/** The most recent issue body per key (what the Improve loop consumes). */
		export function latestIssues(client) {
		const out = {};
		for (const event of client.store.readAll()) {
		if (event.kind === 'issue') {
		const body = event.body;
		out[String(body.key ?? '')] = body;
		}
		}
		return out;
		}

+7

-0

dist/index.d.ts

		@@ -20,2 +20,9 @@ /**
		export { TrailStore } from './store.js';
		export { loadRubric, judgeFromSpec, run, AnthropicJudge, OpenAIJudge, JudgeError, judgePrompt, verdictFrom, DEFAULT_JUDGE, JUDGE_SYSTEM, JUDGE_SCHEMA, } from './evals.js';
		export type { Rubric, Verdict, Judge, EvalSummary, RunOptions } from './evals.js';
		export { cluster, latestIssues, weakestCriterion } from './issues.js';
		export type { Issue } from './issues.js';
		export { instrumentAnthropic, instrumentOpenAI } from './instrument.js';
		export { propose, replay, canary, approve, proposerFromSpec, loadRunner, loadCases, AnthropicProposer, GateError, DEFAULT_PROPOSER, PROPOSER_SYSTEM, PROPOSER_SCHEMA, } from './improve.js';
		export type { Proposal, Proposer, ProposeResult, ReplayResult, Runner, } from './improve.js';
		export interface LooptailOptions {
		@@ -22,0 +29,0 @@ /** Application/agent name; trail file is .looptail/<app>.jsonl */

+5

-0

dist/index.js

		@@ -20,2 +20,7 @@ /**
		export { TrailStore } from './store.js';
		// Understand phase + instrumentation (TypeScript parity with the Python SDK)
		export { loadRubric, judgeFromSpec, run, AnthropicJudge, OpenAIJudge, JudgeError, judgePrompt, verdictFrom, DEFAULT_JUDGE, JUDGE_SYSTEM, JUDGE_SCHEMA, } from './evals.js';
		export { cluster, latestIssues, weakestCriterion } from './issues.js';
		export { instrumentAnthropic, instrumentOpenAI } from './instrument.js';
		export { propose, replay, canary, approve, proposerFromSpec, loadRunner, loadCases, AnthropicProposer, GateError, DEFAULT_PROPOSER, PROPOSER_SYSTEM, PROPOSER_SCHEMA, } from './improve.js';
		const MAX_REPR = 2048;
		@@ -22,0 +27,0 @@ function safeRepr(value) {

+13

-1

package.json

		{
		"name": "@looptail/sdk",
		"version": "0.2.1",
		"version": "0.3.0",
		"description": "Record every AI decision as a signed, append-only trail. Every loop leaves a tail.",
		@@ -41,2 +41,14 @@ "license": "Apache-2.0",
		},
		"peerDependencies": {
		"@anthropic-ai/sdk": ">=0.30",
		"openai": ">=4.50"
		},
		"peerDependenciesMeta": {
		"@anthropic-ai/sdk": {
		"optional": true
		},
		"openai": {
		"optional": true
		}
		},
		"devDependencies": {
		@@ -43,0 +55,0 @@ "typescript": "^5.5.0"

+24

-0

README.md

		@@ -35,2 +35,26 @@ # @looptail/sdk

		## The whole loop, in TypeScript

		```ts
		import { Looptail, instrumentAnthropic, loadRubric, judgeFromSpec, run, cluster } from '@looptail/sdk';
		import Anthropic from '@anthropic-ai/sdk';

		const lt = new Looptail({ app: 'support-agent' });

		// Observe — every provider call becomes a signed observe event
		const anthropic = instrumentAnthropic(new Anthropic(), lt);

		// Understand — score recorded events against your rubric, cluster failures
		const judge = await judgeFromSpec('anthropic:claude-opus-4-8'); // or openai:<model>
		await run(loadRubric('rubrics/refund-policy.json'), lt, judge);
		cluster(lt);
		```

		The whole loop — Observe, Understand, and Improve (`propose` / `replay` /
		`canary` / `approve`) — is at full parity with the Python SDK: same rubric
		prompt, judge schema, and event shapes, so a JS verdict, a Python verdict, and
		a hosted verdict all agree. The provider SDKs (`@anthropic-ai/sdk`, `openai`)
		are optional peer deps, only needed when you build a judge or proposer. From
		the terminal: `npx @looptail/cli evals run` / `issues cluster` / `improve`.

		## How it works
		@@ -37,0 +61,0 @@

@looptail/sdk - npm Package Compare versions

New alerts

Improved metrics

Worsened metrics