langsmith - npm Package Compare versions

Comparing version 0.1.21 to 0.1.22

dist/client.d.ts

 import { AsyncCallerParams } from "./utils/async_caller.js";
 import { DataType, Dataset, DatasetDiffInfo, DatasetShareSchema, Example, ExampleUpdate, Feedback, FeedbackConfig, FeedbackIngestToken, KVMap, LangChainBaseMessage, Run, RunCreate, RunUpdate, ScoreType, TimeDelta, TracerSession, TracerSessionResult, ValueType } from "./schemas.js";
-import { RunEvaluator } from "./evaluation/evaluator.js";
+import { EvaluationResult, EvaluationResults, RunEvaluator } from "./evaluation/evaluator.js";
 interface ClientConfig {
@@ -105,3 +105,2 @@ apiUrl?: string;
      * The values to include in the response.
-     *
      */
@@ -409,3 +408,3 @@ select?: string[];
     }): Promise<Feedback>;
-    createFeedback(runId: string, key: string, { score, value, correction, comment, sourceInfo, feedbackSourceType, sourceRunId, feedbackId, feedbackConfig, }: {
+    createFeedback(runId: string | null, key: string, { score, value, correction, comment, sourceInfo, feedbackSourceType, sourceRunId, feedbackId, feedbackConfig, projectId, }: {
         score?: ScoreType;
@@ -421,2 +420,3 @@ value?: ValueType;
         eager?: boolean;
+        projectId?: string;
     }): Promise<Feedback>;
@@ -461,3 +461,7 @@ updateFeedback(feedbackId: string, { score, value, correction, comment, }: {
     listPresignedFeedbackTokens(runId: string): AsyncIterable<FeedbackIngestToken>;
+    _selectEvalResults(results: EvaluationResult | EvaluationResults): Array<EvaluationResult>;
+    logEvaluationFeedback(evaluatorResponse: EvaluationResult | EvaluationResults, run?: Run, sourceInfo?: {
+        [key: string]: any;
+    }): Promise<EvaluationResult[]>;
 }
 export {};

dist/evaluation/evaluator.d.ts

@@ -1,2 +0,3 @@
-import { Example, Run, ScoreType, ValueType } from "../schemas.js";
+import { Example, FeedbackConfig, Run, ScoreType, ValueType } from "../schemas.js";
+import { RunTreeConfig } from "../run_trees.js";
 /**
@@ -16,27 +17,2 @@ * Represents a categorical class.
 /**
- * Configuration for feedback.
- */
-export type FeedbackConfig = {
-    /**
-     * The type of feedback.
-     * - "continuous": Feedback with a continuous numeric.
-     * - "categorical": Feedback with a categorical value (classes)
-     * - "freeform": Feedback with a freeform text value (notes).
-     */
-    type: "continuous" | "categorical" | "freeform";
-    /**
-     * The minimum value for continuous feedback.
-     */
-    min?: number;
-    /**
-     * The maximum value for continuous feedback.
-     */
-    max?: number;
-    /**
-     * The categories for categorical feedback.
-     * Each category can be a string or an object with additional properties.
-     */
-    categories?: (Category | Record<string, unknown>)[];
-};
-/**
  * Represents the result of an evaluation.
@@ -87,4 +63,32 @@ */
 };
+/**
+ * Batch evaluation results, if your evaluator wishes
+ * to return multiple scores.
+ */
+export type EvaluationResults = {
+    /**
+     * The evaluation results.
+     */
+    results: Array<EvaluationResult>;
+};
 export interface RunEvaluator {
-    evaluateRun(run: Run, example?: Example): Promise<EvaluationResult>;
+    evaluateRun(run: Run, example?: Example, options?: Partial<RunTreeConfig>): Promise<EvaluationResult>;
 }
+export type RunEvaluatorLike = ((run: Run, example?: Example) => Promise<EvaluationResult | EvaluationResults>) | ((run: Run, example?: Example) => EvaluationResult | EvaluationResults);
+/**
+ * Wraps an evaluator function + implements the RunEvaluator interface.
+ */
+export declare class DynamicRunEvaluator<Func extends (...args: any[]) => any> implements RunEvaluator {
+    func: Func;
+    constructor(evaluator: Func);
+    private coerceEvaluationResults;
+    private coerceEvaluationResult;
+    /**
+     * Evaluates a run with an optional example and returns the evaluation result.
+     * @param run The run to evaluate.
+     * @param example The optional example to use for evaluation.
+     * @returns A promise that extracts to the evaluation result.
+     */
+    evaluateRun(run: Run, example?: Example, options?: Partial<RunTreeConfig>): Promise<EvaluationResult>;
+}
+export declare function runEvaluator(func: RunEvaluatorLike): RunEvaluator;

dist/evaluation/evaluator.js

@@ -1,1 +0,83 @@
-export {};
+import { v4 as uuidv4 } from "uuid";
+import { wrapFunctionAndEnsureTraceable } from "../traceable.js";
+/**
+ * Wraps an evaluator function + implements the RunEvaluator interface.
+ */
+export class DynamicRunEvaluator {
+    constructor(evaluator) {
+        Object.defineProperty(this, "func", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        const wrappedFunc = (input) => {
+            const runAndExample = input.langSmithRunAndExample;
+            return evaluator(...Object.values(runAndExample));
+        };
+        this.func = wrappedFunc;
+    }
+    coerceEvaluationResults(results, sourceRunId) {
+        if ("results" in results) {
+            throw new Error("EvaluationResults not supported yet.");
+        }
+        return this.coerceEvaluationResult(results, sourceRunId, true);
+    }
+    coerceEvaluationResult(result, sourceRunId, allowNoKey = false) {
+        if ("key" in result) {
+            if (!result.sourceRunId) {
+                result.sourceRunId = sourceRunId;
+            }
+            return result;
+        }
+        if (!("key" in result)) {
+            if (allowNoKey) {
+                result["key"] = this.func.name;
+            }
+        }
+        return {
+            sourceRunId,
+            ...result,
+        };
+    }
+    /**
+     * Evaluates a run with an optional example and returns the evaluation result.
+     * @param run The run to evaluate.
+     * @param example The optional example to use for evaluation.
+     * @returns A promise that extracts to the evaluation result.
+     */
+    async evaluateRun(run, example, options) {
+        const sourceRunId = uuidv4();
+        const metadata = {
+            targetRunId: run.id,
+        };
+        if ("session_id" in run) {
+            metadata["experiment"] = run.session_id;
+        }
+        const wrappedTraceableFunc = wrapFunctionAndEnsureTraceable(this.func, options || {}, "evaluator");
+        // Pass data via `langSmithRunAndExample` key to avoid conflicts with other
+        // inputs. This key is extracted in the wrapped function, with `run` and
+        // `example` passed to evaluator function as arguments.
+        const langSmithRunAndExample = {
+            run,
+            example,
+        };
+        const result = (await wrappedTraceableFunc({ langSmithRunAndExample }, {
+            metadata,
+        }));
+        // Check the one required property of EvaluationResult since 'instanceof' is not possible
+        if ("key" in result) {
+            if (!result.sourceRunId) {
+                result.sourceRunId = sourceRunId;
+            }
+            return result;
+        }
+        if (typeof result !== "object") {
+            throw new Error("Evaluator function must return an object.");
+        }
+        return this.coerceEvaluationResults(result, sourceRunId);
+    }
+}
+export function runEvaluator(func) {
+    return new DynamicRunEvaluator(func);
+}

dist/evaluation/index.d.ts

 export { RunEvaluator, EvaluationResult } from "./evaluator.js";
 export { StringEvaluator, GradingFunctionParams, GradingFunctionResult, } from "./string_evaluator.js";
+export { evaluate, type EvaluateOptions } from "./_runner.js";

dist/evaluation/index.js

 export { StringEvaluator, } from "./string_evaluator.js";
+export { evaluate } from "./_runner.js";

dist/index.d.ts

 export { Client } from "./client.js";
 export type { Dataset, Example, TracerSession, Run, Feedback, } from "./schemas.js";
 export { RunTree, type RunTreeConfig } from "./run_trees.js";
-export declare const __version__ = "0.1.21";
+export declare const __version__ = "0.1.22";

dist/index.js

 export { Client } from "./client.js";
 export { RunTree } from "./run_trees.js";
 // Update using yarn bump-version
-export const __version__ = "0.1.21";
+export const __version__ = "0.1.22";

dist/run_trees.d.ts

@@ -23,2 +23,3 @@ import { BaseRun, KVMap } from "./schemas.js";
     client?: Client;
+    on_end?: (runTree: RunTree) => void;
 }
@@ -25,0 +26,0 @@ export interface RunnableConfigLike {

dist/schemas.d.ts

@@ -8,2 +8,5 @@ export interface TracerSession {
     name?: string;
+    /** Extra metadata for the project. */
+    extra?: KVMap;
+    reference_dataset_id?: string;
 }
@@ -19,3 +22,2 @@ export interface TracerSessionResult extends TracerSession {
     feedback_stats?: Record<string, unknown>;
-    reference_dataset_id?: string;
     run_facets?: KVMap[];
@@ -83,2 +85,10 @@ }
 }
+type S3URL = {
+    ROOT: {
+        /** A pre-signed URL */
+        presigned_url: string;
+        /** The S3 path to the object in storage */
+        s3_url: string;
+    };
+};
 /**
@@ -117,2 +127,6 @@ * Describes properties of a run when loaded from the database.
     in_dataset?: boolean;
+    /** The output S3 URLs */
+    outputs_s3_urls?: S3URL;
+    /** The input S3 URLs */
+    inputs_s3_urls?: S3URL;
 }
@@ -242,2 +256,5 @@ export interface RunCreate extends BaseRun {
      * The type of feedback.
+     * - "continuous": Feedback with a continuous numeric.
+     * - "categorical": Feedback with a categorical value (classes)
+     * - "freeform": Feedback with a freeform text value (notes).
      */
@@ -254,2 +271,5 @@ type: "continuous" | "categorical" | "freeform";
     /**
+     * The categories for categorical feedback.
+     * Each category can be a string or an object with additional properties.
+     *
      * If feedback is categorical, this defines the valid categories the server will accept.
@@ -265,1 +285,2 @@ * Not applicable to continuous or freeform feedback types.
 }
+export {};

dist/traceable.d.ts

@@ -51,2 +51,3 @@ import { RunTree, RunTreeConfig, RunnableConfigLike } from "./run_trees.js";
     argsConfigPath?: [number] | [number, string];
+    tracingEnabled?: boolean;
 }): TraceableFunction<Func>;
@@ -61,2 +62,3 @@ /**
 export declare function isTraceableFunction(x: unknown): x is TraceableFunction<any>;
+export declare function wrapFunctionAndEnsureTraceable<Func extends (...args: any[]) => any>(target: Func, options: Partial<RunTreeConfig>, name?: string): TraceableFunction<Func>;
 export {};

dist/traceable.js

@@ -15,6 +15,17 @@ import { AsyncLocalStorage } from "async_hooks";
     typeof x[Symbol.asyncIterator] === "function";
-const getTracingRunTree = (runTree) => {
-    const tracingEnabled = getEnvironmentVariable("LANGSMITH_TRACING_V2") === "true" ||
-        getEnvironmentVariable("LANGCHAIN_TRACING_V2") === "true";
-    if (!tracingEnabled) {
+const tracingIsEnabled = (tracingEnabled) => {
+    if (tracingEnabled !== undefined) {
+        return tracingEnabled;
+    }
+    const envVars = [
+        "LANGSMITH_TRACING_V2",
+        "LANGCHAIN_TRACING_V2",
+        "LANGSMITH_TRACING",
+        "LANGCHAIN_TRACING",
+    ];
+    return Boolean(envVars.find((envVar) => getEnvironmentVariable(envVar) === "true"));
+};
+const getTracingRunTree = (runTree, tracingEnabled) => {
+    const tracingEnabled_ = tracingIsEnabled(tracingEnabled);
+    if (!tracingEnabled_) {
         return undefined;
@@ -40,3 +51,3 @@ }
 export function traceable(wrappedFunc, config) {
-    const { aggregator, argsConfigPath, ...runTreeConfig } = config ?? {};
+    const { aggregator, argsConfigPath, tracingEnabled, ...runTreeConfig } = config ?? {};
     const traceableFunc = (...args) => {
@@ -107,3 +118,3 @@ let currentRunTree;
         }
-        currentRunTree = getTracingRunTree(currentRunTree);
+        currentRunTree = getTracingRunTree(currentRunTree, tracingEnabled);
         let inputs;
@@ -176,2 +187,11 @@ const firstInput = rawInputs[0];
                         }
+                        const onEnd = config?.on_end;
+                        if (onEnd) {
+                            if (!currentRunTree) {
+                                console.warn("Can not call 'on_end' if currentRunTree is undefined");
+                            }
+                            else {
+                                onEnd(currentRunTree);
+                            }
+                        }
                         await postRunPromise;
@@ -229,2 +249,11 @@ await currentRunTree?.patchRun();
                                 }
+                                const onEnd = config?.on_end;
+                                if (onEnd) {
+                                    if (!currentRunTree) {
+                                        console.warn("Can not call 'on_end' if currentRunTree is undefined");
+                                    }
+                                    else {
+                                        onEnd(currentRunTree);
+                                    }
+                                }
                                 await postRunPromise;
@@ -239,2 +268,11 @@ await currentRunTree?.patchRun();
                             await currentRunTree?.end(isKVMap(rawOutput) ? rawOutput : { outputs: rawOutput });
+                            const onEnd = config?.on_end;
+                            if (onEnd) {
+                                if (!currentRunTree) {
+                                    console.warn("Can not call 'on_end' if currentRunTree is undefined");
+                                }
+                                else {
+                                    onEnd(currentRunTree);
+                                }
+                            }
                             await postRunPromise;
@@ -252,2 +290,11 @@ await currentRunTree?.patchRun();
                     await currentRunTree?.end(undefined, String(error));
+                    const onEnd = config?.on_end;
+                    if (onEnd) {
+                        if (!currentRunTree) {
+                            console.warn("Can not call 'on_end' if currentRunTree is undefined");
+                        }
+                        else {
+                            onEnd(currentRunTree);
+                        }
+                    }
                     await postRunPromise;
@@ -306,1 +353,10 @@ await currentRunTree?.patchRun();
 }
+export function wrapFunctionAndEnsureTraceable(target, options, name = "target") {
+    if (typeof target === "function") {
+        return traceable(target, {
+            ...options,
+            name,
+        });
+    }
+    throw new Error("Target must be runnable function");
+}

package.json

 {
   "name": "langsmith",
-  "version": "0.1.21",
+  "version": "0.1.22",
   "description": "Client library to connect to the LangSmith LLM Tracing and Evaluation Platform.",
@@ -5,0 +5,0 @@ "packageManager": "yarn@1.22.19",

README.md

@@ -54,3 +54,3 @@ # LangSmith Client SDK
 ```typescript
-process.env["LANGCHAIN_TRACING_V2"] = "true";
+process.env["LANGSMITH_TRACING"] = "true";
 process.env["LANGCHAIN_ENDPOINT"] = "https://api.smith.langchain.com";
@@ -57,0 +57,0 @@ process.env["LANGCHAIN_API_KEY"] = "<YOUR-LANGSMITH-API-KEY>";

New alerts

License Policy Violation

License

This package is not allowed per your license policy. Review the package's license to ensure compliance.

Found 1 instance in 1 package

Fixed alerts

License Policy Violation

License

This package is not allowed per your license policy. Review the package's license to ensure compliance.

Found 1 instance in 1 package

Improved metrics

Total package byte prevSize

393685

increased by34.36%

Number of package files

88

increased by20.55%

Lines of code

10871

increased by50.92%

No dependency changes