langsmith - npm Package Compare versions

Comparing version 0.1.23 to 0.1.24-rc.0

dist/client.d.ts

 import { AsyncCallerParams } from "./utils/async_caller.js";
-import { DataType, Dataset, DatasetDiffInfo, DatasetShareSchema, Example, ExampleUpdate, Feedback, FeedbackConfig, FeedbackIngestToken, KVMap, LangChainBaseMessage, Run, RunCreate, RunUpdate, ScoreType, TimeDelta, TracerSession, TracerSessionResult, ValueType } from "./schemas.js";
+import { ComparativeExperiment, DataType, Dataset, DatasetDiffInfo, DatasetShareSchema, Example, ExampleUpdate, Feedback, FeedbackConfig, FeedbackIngestToken, KVMap, LangChainBaseMessage, Run, RunCreate, RunUpdate, ScoreType, TimeDelta, TracerSession, TracerSessionResult, ValueType } from "./schemas.js";
 import { EvaluationResult, EvaluationResults, RunEvaluator } from "./evaluation/evaluator.js";
@@ -185,3 +185,3 @@ interface ClientConfig {
     };
-    private getHostUrl;
+    getHostUrl(): string;
     private get headers();
@@ -338,2 +338,6 @@ private processInputs;
     }): Promise<TracerSessionResult>;
+    getProjectUrl({ projectId, projectName, }: {
+        projectId?: string;
+        projectName?: string;
+    }): Promise<string>;
     private _getTenantId;
@@ -414,3 +418,3 @@ listProjects({ projectIds, name, nameContains, referenceDatasetId, referenceDatasetName, referenceFree, }?: {
     }): Promise<Feedback>;
-    createFeedback(runId: string | null, key: string, { score, value, correction, comment, sourceInfo, feedbackSourceType, sourceRunId, feedbackId, feedbackConfig, projectId, }: {
+    createFeedback(runId: string | null, key: string, { score, value, correction, comment, sourceInfo, feedbackSourceType, sourceRunId, feedbackId, feedbackConfig, projectId, comparativeExperimentId, }: {
         score?: ScoreType;
@@ -427,2 +431,3 @@ value?: ValueType;
         projectId?: string;
+        comparativeExperimentId?: string;
     }): Promise<Feedback>;
@@ -461,2 +466,11 @@ updateFeedback(feedbackId: string, { score, value, correction, comment, }: {
     }): Promise<FeedbackIngestToken>;
+    createComparativeExperiment({ name, experimentIds, referenceDatasetId, createdAt, description, metadata, id, }: {
+        name: string;
+        experimentIds: Array<string>;
+        referenceDatasetId?: string;
+        createdAt?: Date;
+        description?: string;
+        metadata?: Record<string, unknown>;
+        id?: string;
+    }): Promise<ComparativeExperiment>;
     /**
@@ -463,0 +477,0 @@ * Retrieves a list of presigned feedback tokens for a given run ID.

dist/evaluation/_runner.d.ts

 import { Client } from "../index.js";
 import { Example, KVMap, Run, TracerSession } from "../schemas.js";
 import { EvaluationResult, EvaluationResults, RunEvaluator } from "./evaluator.js";
-type TargetT = ((input: KVMap, config?: KVMap) => Promise<KVMap>) | ((input: KVMap, config?: KVMap) => KVMap) | {
-    invoke: (input: KVMap, config?: KVMap) => KVMap;
+type TargetT<TInput = any, TOutput = KVMap> = ((input: TInput, config?: KVMap) => Promise<TOutput>) | ((input: TInput, config?: KVMap) => TOutput) | {
+    invoke: (input: TInput, config?: KVMap) => TOutput;
 } | {
-    invoke: (input: KVMap, config?: KVMap) => Promise<KVMap>;
+    invoke: (input: TInput, config?: KVMap) => Promise<TOutput>;
 };
-type TargetNoInvoke = ((input: KVMap, config?: KVMap) => Promise<KVMap>) | ((input: KVMap, config?: KVMap) => KVMap);
+type TargetNoInvoke<TInput = any, TOutput = KVMap> = ((input: TInput, config?: KVMap) => Promise<TOutput>) | ((input: TInput, config?: KVMap) => TOutput);
 type DataT = string | AsyncIterable<Example> | Example[];
@@ -11,0 +11,0 @@ type SummaryEvaluatorT = ((runs: Array<Run>, examples: Array<Example>) => Promise<EvaluationResult | EvaluationResults>) | ((runs: Array<Run>, examples: Array<Example>) => EvaluationResult | EvaluationResults);

dist/evaluation/_runner.js

@@ -469,6 +469,26 @@ import { Client } from "../index.js";
         const modifiedAt = examples.map((ex) => ex.modified_at);
-        const maxModifiedAt = modifiedAt.length > 0
-            ? new Date(Math.max(...modifiedAt.map((date) => new Date(date).getTime())))
-            : undefined;
-        return maxModifiedAt?.toISOString();
+        // Python might return microseconds, which we need
+        // to account for when comparing dates.
+        const modifiedAtTime = modifiedAt.map((date) => {
+            function getMiliseconds(isoString) {
+                const time = isoString.split("T").at(1);
+                if (!time)
+                    return "";
+                const regex = /[0-9]{2}:[0-9]{2}:[0-9]{2}.([0-9]+)/;
+                const strMiliseconds = time.match(regex)?.[1];
+                return strMiliseconds ?? "";
+            }
+            const jsDate = new Date(date);
+            let source = getMiliseconds(date);
+            let parsed = getMiliseconds(jsDate.toISOString());
+            const length = Math.max(source.length, parsed.length);
+            source = source.padEnd(length, "0");
+            parsed = parsed.padEnd(length, "0");
+            const microseconds = (Number.parseInt(source, 10) - Number.parseInt(parsed, 10)) / 1000;
+            const time = jsDate.getTime() + microseconds;
+            return { date, time };
+        });
+        if (modifiedAtTime.length === 0)
+            return undefined;
+        return modifiedAtTime.reduce((max, current) => (current.time > max.time ? current : max), modifiedAtTime[0]).date;
     }
@@ -572,5 +592,3 @@ async _end() {
     if (_isCallable(target)) {
-        manager = await manager.withPredictions(convertInvokeToTopLevel(target), {
-            maxConcurrency: fields.maxConcurrency,
-        });
+        manager = await manager.withPredictions(convertInvokeToTopLevel(target), { maxConcurrency: fields.maxConcurrency });
     }
@@ -577,0 +595,0 @@ if (fields.evaluators) {

dist/evaluation/evaluator.js

 import { v4 as uuidv4 } from "uuid";
-import { wrapFunctionAndEnsureTraceable } from "../traceable.js";
+import { traceable } from "../traceable.js";
 /**
@@ -14,7 +14,6 @@ * Wraps an evaluator function + implements the RunEvaluator interface.
         });
-        const wrappedFunc = (input) => {
-            const runAndExample = input.langSmithRunAndExample;
-            return evaluator(...Object.values(runAndExample));
-        };
-        this.func = wrappedFunc;
+        this.func = ((input) => {
+            const { run, example } = input.langSmithRunAndExample;
+            return evaluator(run, example);
+        });
     }
@@ -58,13 +57,11 @@ coerceEvaluationResults(results, sourceRunId) {
         }
-        const wrappedTraceableFunc = wrapFunctionAndEnsureTraceable(this.func, options || {}, "evaluator");
+        if (typeof this.func !== "function") {
+            throw new Error("Target must be runnable function");
+        }
+        const wrappedTraceableFunc = traceable(this.func, { project_name: "evaluators", name: "evaluator", ...options });
+        const result = (await wrappedTraceableFunc(
         // Pass data via `langSmithRunAndExample` key to avoid conflicts with other
         // inputs. This key is extracted in the wrapped function, with `run` and
         // `example` passed to evaluator function as arguments.
-        const langSmithRunAndExample = {
-            run,
-            example,
-        };
-        const result = (await wrappedTraceableFunc({ langSmithRunAndExample }, {
-            metadata,
-        }));
+        { langSmithRunAndExample: { run, example } }, { metadata }));
         // Check the one required property of EvaluationResult since 'instanceof' is not possible
@@ -71,0 +68,0 @@ if ("key" in result) {

dist/evaluation/index.d.ts

 export { RunEvaluator, EvaluationResult } from "./evaluator.js";
 export { StringEvaluator, GradingFunctionParams, GradingFunctionResult, } from "./string_evaluator.js";
 export { evaluate, type EvaluateOptions } from "./_runner.js";
+export { evaluateComparative } from "./evaluate_comparative.js";

dist/evaluation/index.js

 export { StringEvaluator, } from "./string_evaluator.js";
 export { evaluate } from "./_runner.js";
+export { evaluateComparative } from "./evaluate_comparative.js";

dist/index.d.ts

 export { Client } from "./client.js";
 export type { Dataset, Example, TracerSession, Run, Feedback, } from "./schemas.js";
 export { RunTree, type RunTreeConfig } from "./run_trees.js";
-export declare const __version__ = "0.1.23";
+export declare const __version__ = "0.1.24-rc.0";

dist/index.js

 export { Client } from "./client.js";
 export { RunTree } from "./run_trees.js";
 // Update using yarn bump-version
-export const __version__ = "0.1.23";
+export const __version__ = "0.1.24-rc.0";

dist/run_trees.js

@@ -315,2 +315,3 @@ import * as uuid from "uuid";
             error: this.error,
+            inputs: this.inputs,
             outputs: this.outputs,
@@ -317,0 +318,0 @@ parent_run_id: this.parent_run?.id,

dist/schemas.d.ts

@@ -279,2 +279,19 @@ export interface TracerSession {
 }
+export interface ComparisonEvaluationResult {
+    key: string;
+    scores: Record<string, ScoreType>;
+    source_run_id?: string;
+}
+export interface ComparativeExperiment {
+    id: string;
+    name: string;
+    description: string;
+    tenant_id: string;
+    created_at: string;
+    modified_at: string;
+    reference_dataset_id: string;
+    extra?: Record<string, unknown>;
+    experiments_info?: Array<Record<string, unknown>>;
+    feedback_stats?: Record<string, unknown>;
+}
 export {};

dist/traceable.js

@@ -16,2 +16,18 @@ import { AsyncLocalStorage } from "async_hooks";
     typeof x[Symbol.asyncIterator] === "function";
+const GeneratorFunction = function* () { }.constructor;
+const isIteratorLike = (x) => x != null &&
+    typeof x === "object" &&
+    "next" in x &&
+    typeof x.next === "function";
+const isGenerator = (x) => 
+// eslint-disable-next-line no-instanceof/no-instanceof
+x != null && typeof x === "function" && x instanceof GeneratorFunction;
+const isThenable = (x) => x != null &&
+    typeof x === "object" &&
+    "then" in x &&
+    typeof x.then === "function";
+const isReadableStream = (x) => x != null &&
+    typeof x === "object" &&
+    "getReader" in x &&
+    typeof x.getReader === "function";
 const tracingIsEnabled = (tracingEnabled) => {
@@ -56,2 +72,137 @@ if (tracingEnabled !== undefined) {
 };
+// idea: store the state of the promise outside
+// but only when the promise is "consumed"
+const getSerializablePromise = (arg) => {
+    const proxyState = { current: undefined };
+    const promiseProxy = new Proxy(arg, {
+        get(target, prop, receiver) {
+            if (prop === "then") {
+                const boundThen = arg[prop].bind(arg);
+                return (resolve, reject = (x) => {
+                    throw x;
+                }) => {
+                    return boundThen((value) => {
+                        proxyState.current = ["resolve", value];
+                        return resolve(value);
+                    }, (error) => {
+                        proxyState.current = ["reject", error];
+                        return reject(error);
+                    });
+                };
+            }
+            if (prop === "catch") {
+                const boundCatch = arg[prop].bind(arg);
+                return (reject) => {
+                    return boundCatch((error) => {
+                        proxyState.current = ["reject", error];
+                        return reject(error);
+                    });
+                };
+            }
+            if (prop === "toJSON") {
+                return () => {
+                    if (!proxyState.current)
+                        return undefined;
+                    const [type, value] = proxyState.current ?? [];
+                    if (type === "resolve")
+                        return value;
+                    return { error: value };
+                };
+            }
+            return Reflect.get(target, prop, receiver);
+        },
+    });
+    return promiseProxy;
+};
+const convertSerializableArg = (arg) => {
+    if (isReadableStream(arg)) {
+        const proxyState = [];
+        const transform = new TransformStream({
+            start: () => void 0,
+            transform: (chunk, controller) => {
+                proxyState.push(chunk);
+                controller.enqueue(chunk);
+            },
+            flush: () => void 0,
+        });
+        const pipeThrough = arg.pipeThrough(transform);
+        Object.assign(pipeThrough, { toJSON: () => proxyState });
+        return pipeThrough;
+    }
+    if (isAsyncIterable(arg)) {
+        const proxyState = { current: [] };
+        return new Proxy(arg, {
+            get(target, prop, receiver) {
+                if (prop === Symbol.asyncIterator) {
+                    return () => {
+                        const boundIterator = arg[Symbol.asyncIterator].bind(arg);
+                        const iterator = boundIterator();
+                        return new Proxy(iterator, {
+                            get(target, prop, receiver) {
+                                if (prop === "next" || prop === "return" || prop === "throw") {
+                                    const bound = iterator.next.bind(iterator);
+                                    return (...args) => {
+                                        // @ts-expect-error TS cannot infer the argument types for the bound function
+                                        const wrapped = getSerializablePromise(bound(...args));
+                                        proxyState.current.push(wrapped);
+                                        return wrapped;
+                                    };
+                                }
+                                if (prop === "return" || prop === "throw") {
+                                    return iterator.next.bind(iterator);
+                                }
+                                return Reflect.get(target, prop, receiver);
+                            },
+                        });
+                    };
+                }
+                if (prop === "toJSON") {
+                    return () => {
+                        const onlyNexts = proxyState.current;
+                        const serialized = onlyNexts.map((next) => next.toJSON());
+                        const chunks = serialized.reduce((memo, next) => {
+                            if (next?.value)
+                                memo.push(next.value);
+                            return memo;
+                        }, []);
+                        return chunks;
+                    };
+                }
+                return Reflect.get(target, prop, receiver);
+            },
+        });
+    }
+    if (!Array.isArray(arg) && isIteratorLike(arg)) {
+        const proxyState = [];
+        return new Proxy(arg, {
+            get(target, prop, receiver) {
+                if (prop === "next" || prop === "return" || prop === "throw") {
+                    const bound = arg[prop]?.bind(arg);
+                    return (...args) => {
+                        // @ts-expect-error TS cannot infer the argument types for the bound function
+                        const next = bound?.(...args);
+                        if (next != null)
+                            proxyState.push(next);
+                        return next;
+                    };
+                }
+                if (prop === "toJSON") {
+                    return () => {
+                        const chunks = proxyState.reduce((memo, next) => {
+                            if (next.value)
+                                memo.push(next.value);
+                            return memo;
+                        }, []);
+                        return chunks;
+                    };
+                }
+                return Reflect.get(target, prop, receiver);
+            },
+        });
+    }
+    if (isThenable(arg)) {
+        return getSerializablePromise(arg);
+    }
+    return arg;
+};
 /**
@@ -120,4 +271,9 @@ * Higher-order function that takes function as input and returns a
         }
+        // TODO: deal with possible nested promises and async iterables
+        const processedArgs = args;
+        for (let i = 0; i < processedArgs.length; i++) {
+            processedArgs[i] = convertSerializableArg(processedArgs[i]);
+        }
         const [currentRunTree, rawInputs] = (() => {
-            const [firstArg, ...restArgs] = args;
+            const [firstArg, ...restArgs] = processedArgs;
             // used for handoff between LangChain.JS and traceable functions
@@ -150,8 +306,8 @@ if (isRunnableConfigLike(firstArg)) {
                 return [
-                    getTracingRunTree(prevRunFromStore.createChild(ensuredConfig), args),
-                    args,
+                    getTracingRunTree(prevRunFromStore.createChild(ensuredConfig), processedArgs),
+                    processedArgs,
                 ];
             }
-            const currentRunTree = getTracingRunTree(new RunTree(ensuredConfig), args);
-            return [currentRunTree, args];
+            const currentRunTree = getTracingRunTree(new RunTree(ensuredConfig), processedArgs);
+            return [currentRunTree, processedArgs];
         })();
@@ -212,2 +368,13 @@ return asyncLocalStorage.run(currentRunTree, () => {
             }
+            function gatherAll(iterator) {
+                const chunks = [];
+                // eslint-disable-next-line no-constant-condition
+                while (true) {
+                    const next = iterator.next();
+                    chunks.push(next);
+                    if (next.done)
+                        break;
+                }
+                return chunks;
+            }
             let returnValue;
@@ -231,12 +398,27 @@ try {
                     }
-                    else {
-                        try {
-                            await currentRunTree?.end(handleRunOutputs(rawOutput));
-                            await handleEnd();
-                        }
-                        finally {
-                            // eslint-disable-next-line no-unsafe-finally
-                            return rawOutput;
-                        }
+                    if (isGenerator(wrappedFunc) && isIteratorLike(rawOutput)) {
+                        const chunks = gatherAll(rawOutput);
+                        await currentRunTree?.end(handleRunOutputs(await handleChunks(chunks.reduce((memo, { value, done }) => {
+                            if (!done || typeof value !== "undefined") {
+                                memo.push(value);
+                            }
+                            return memo;
+                        }, []))));
+                        await handleEnd();
+                        return (function* () {
+                            for (const ret of chunks) {
+                                if (ret.done)
+                                    return ret.value;
+                                yield ret.value;
+                            }
+                        })();
                     }
+                    try {
+                        await currentRunTree?.end(handleRunOutputs(rawOutput));
+                        await handleEnd();
+                    }
+                    finally {
+                        // eslint-disable-next-line no-unsafe-finally
+                        return rawOutput;
+                    }
                 }, async (error) => {
@@ -290,7 +472,11 @@ await currentRunTree?.end(undefined, String(error));
 function isKVMap(x) {
-    return (typeof x === "object" &&
-        x != null &&
-        !Array.isArray(x) &&
-        // eslint-disable-next-line no-instanceof/no-instanceof
-        !(x instanceof Date));
+    if (typeof x !== "object" || x == null) {
+        return false;
+    }
+    const prototype = Object.getPrototypeOf(x);
+    return ((prototype === null ||
+        prototype === Object.prototype ||
+        Object.getPrototypeOf(prototype) === null) &&
+        !(Symbol.toStringTag in x) &&
+        !(Symbol.iterator in x));
 }
@@ -297,0 +483,0 @@ export function wrapFunctionAndEnsureTraceable(target, options, name = "target") {

package.json

 {
   "name": "langsmith",
-  "version": "0.1.23",
+  "version": "0.1.24-rc.0",
   "description": "Client library to connect to the LangSmith LLM Tracing and Evaluation Platform.",
@@ -5,0 +5,0 @@ "packageManager": "yarn@1.22.19",

New alerts

License Policy Violation

License

This package is not allowed per your license policy. Review the package's license to ensure compliance.

Found 1 instance in 1 package

Fixed alerts

License Policy Violation

License

This package is not allowed per your license policy. Review the package's license to ensure compliance.

Found 1 instance in 1 package

Improved metrics

Total package byte prevSize

436996

increased by10.94%

Number of package files

94

increased by6.82%

Lines of code

11912

increased by9.12%

Worsened metrics

Number of medium supply chain risk alerts

64

increased by3.23%

No dependency changes