polyfact - npm Package Compare versions

Comparing version 0.1.72 to 0.1.73

auth.js

@@ -32,3 +32,3 @@ import axios, { AxiosError } from "axios";
             token = data.session?.access_token || "";
-            if (!token) {
+            if (!token || !data.session?.refresh_token) {
                 window.localStorage.removeItem("polyfact_refresh_token");
@@ -49,3 +49,3 @@ return {};
         options: {
-            redirectTo: window?.location,
+            redirectTo: `${window?.location}`,
             skipBrowserRedirect: !browserRedirect,
@@ -75,5 +75,5 @@ },
     await co.deresolve();
-    console.log("login", input);
     if (typeof input === "object" && input.provider === "firebase") {
-        return signInWithOAuthToken(input.token, "firebase", co, projectOptions);
+        signInWithOAuthToken(input.token, "firebase", co, projectOptions);
+        return;
     }
@@ -80,0 +80,0 @@ const provider = typeof input === "string" ? input : input.provider;

chats/index.js

@@ -83,3 +83,8 @@ import axios, { AxiosError } from "axios";
             }
-            const result = generate(message, { ...options, web: false, chatId }, await this.clientOptions).pipeInto(resultStream);
+            const result = generate(message, {
+                ...this.options,
+                ...options,
+                web: false,
+                chatId,
+            }, await this.clientOptions).pipeInto(resultStream);
             result.on("data", (d) => {
@@ -86,0 +91,0 @@ aiMessage = aiMessage.concat(d);

dataloader/index.d.ts

@@ -1,22 +0,8 @@
-/// <reference types="node" />
-import { Buffer } from "buffer";
 import { Memory } from "../memory";
+import { FileInput } from "../utils";
 import { InputClientOptions } from "../clientOpts";
-interface MinimalStream {
-    on(event: string | symbol, listener: (...args: any[]) => void): this;
-}
-interface FetchReadableStream {
-    getReader(): {
-        read(): Promise<{
-            done: boolean;
-            value?: Uint8Array | undefined;
-        }>;
-    };
-}
-type LoaderFileInput = MinimalStream | Buffer | FetchReadableStream;
 export type LoaderFunction = (memory: Memory, clientOptions: InputClientOptions) => Promise<void>;
-export declare function TextFileLoader(file: LoaderFileInput, maxTokenPerChunk?: number): LoaderFunction;
+export declare function TextFileLoader(file: FileInput, maxTokenPerChunk?: number): LoaderFunction;
 export declare function StringLoader(str: string, maxTokenPerChunk?: number): LoaderFunction;
-export declare function AudioLoader(file: LoaderFileInput, maxTokenPerChunk?: number): LoaderFunction;
+export declare function AudioLoader(file: FileInput, maxTokenPerChunk?: number): LoaderFunction;
 export declare function loaderToMemory(loaders: LoaderFunction | LoaderFunction[], clientOptions?: InputClientOptions): Promise<Memory>;
-export {};

dataloader/index.js

@@ -1,43 +0,9 @@
-import { Buffer } from "buffer";
 import { Memory } from "../memory";
 import { transcribe } from "../transcribe";
 import { splitString } from "../split";
-function stream2buffer(stream) {
-    return new Promise((resolve, reject) => {
-        const buf = [];
-        stream.on("data", (chunk) => buf.push(chunk));
-        stream.on("end", () => resolve(Buffer.concat(buf)));
-        stream.on("error", (err) => reject(err));
-    });
-}
-async function fetchStream2buffer(stream) {
-    const reader = stream.getReader();
-    const chunks = [];
-    let done = false;
-    let value;
-    while (!done) {
-        // eslint-disable-next-line no-await-in-loop
-        ({ done, value } = await reader.read());
-        if (value) {
-            chunks.push(value);
-        }
-    }
-    return Buffer.concat(chunks);
-}
-async function loaderInputToBuffer(input) {
-    if (input instanceof Buffer) {
-        return input;
-    }
-    if ("on" in input) {
-        return stream2buffer(input);
-    }
-    if ("getReader" in input) {
-        return fetchStream2buffer(input);
-    }
-    return null;
-}
-// eslint-disable-next-line consistent-return
+import { fileInputToBuffer } from "../utils";
 async function batchify(array, size, callback) {
     if (array.length < size) {
-        return callback(array);
+        await callback(array);
+        return;
     }
@@ -49,3 +15,3 @@ await callback(array.slice(0, size));
     return async function loadPdfIntoMemory(memory, _clientOptions = {}) {
-        const fileBuffer = await loaderInputToBuffer(file);
+        const fileBuffer = await fileInputToBuffer(file);
         const splittedFile = splitString(fileBuffer.toString("utf8"), maxTokenPerChunk);
@@ -69,3 +35,3 @@ async function addBatchIntoMemory(batches) {
     return async function loadAudioIntoMemory(memory, clientOptions = {}) {
-        const fileBuffer = await loaderInputToBuffer(file);
+        const fileBuffer = await fileInputToBuffer(file);
         const transcription = await transcribe(fileBuffer, clientOptions);
@@ -72,0 +38,0 @@ const transcriptions = splitString(transcription, maxTokenPerChunk);

generate.d.ts

@@ -15,3 +15,3 @@ /// <reference types="node" />
 export type GenerationSimpleOptions = {
-    provider?: "openai" | "cohere" | "llama" | "";
+    provider?: "openai" | "cohere" | "llama" | "" | undefined;
     model?: string;
@@ -24,3 +24,3 @@ stop?: string[];
     chatId: string;
-}, {}];
+}];
 export type MemoryOptions = [
@@ -35,4 +35,3 @@ {
         data: [LoaderFunction] | LoaderFunction;
-    },
-    {}
+    }
 ];
@@ -43,10 +42,10 @@ export type SystemPromptOptions = [{
     systemPrompt: string;
-}, {}];
+}];
 export type GenerationWithWebOptions = GenerationSimpleOptions & NeverN<ChatOptions> & NeverN<MemoryOptions> & NeverN<SystemPromptOptions> & {
     web: true;
 };
-export type GenerationWithoutWebOptions = GenerationSimpleOptions & ExclusiveN<ChatOptions> & ExclusiveN<MemoryOptions> & ExclusiveN<SystemPromptOptions> & {
+export type GenerationWithoutWebOptions = GenerationSimpleOptions & ExclusiveN<ChatOptions> & ExclusiveN<MemoryOptions> & ExclusiveN<SystemPromptOptions>;
+export type GenerationOptions = GenerationWithWebOptions | (GenerationWithoutWebOptions & {
     web?: false;
-};
-export type GenerationOptions = GenerationWithWebOptions | GenerationWithoutWebOptions;
+});
 export type GenerationCompleteOptions = GenerationSimpleOptions & AndN<ChatOptions> & AndN<MemoryOptions> & AndN<SystemPromptOptions> & {
@@ -53,0 +52,0 @@ web?: boolean;

generate.js

@@ -11,13 +11,2 @@ var _a;
 }
-const PartialResultType = t.partial({
-    ressources: t.array(t.type({ id: t.string, content: t.string, similarity: t.number })),
-});
-const Required = t.type({
-    result: t.string,
-    token_usage: t.type({
-        input: t.number,
-        output: t.number,
-    }),
-});
-const GenerationAPIResponse = t.intersection([Required, PartialResultType]);
 async function getMemoryIds(dataMemory, genOptionsMemoryId, genOptionsMemory) {
@@ -140,4 +129,11 @@ const memoryIds = [
 }
+const GenerateDataType = t.type({
+    data: t.string,
+});
 export function generate(task, options, clientOptions) {
     return stream(task, options, clientOptions, (data, resultStream) => {
+        if (!GenerateDataType.is(data)) {
+            resultStream.emit("error", "Invalid data");
+            return;
+        }
         if (data.data === "") {
@@ -144,0 +140,0 @@ resultStream.push(null);

hooks/useAgent.js

@@ -50,5 +50,3 @@ import { useState } from "react";
     };
-    const start = async (question, 
-    // eslint-disable-next-line @typescript-eslint/no-empty-function
-    progress = () => { }) => {
+    const start = async (question, progress = () => { }) => {
         console.info("Starting...");
@@ -55,0 +53,0 @@ let history = `

package.json

 {
     "name": "polyfact",
-    "version": "0.1.72",
+    "version": "0.1.73",
     "main": "index.js",
@@ -5,0 +5,0 @@ "types": "index.d.ts",

probabilistic_helpers/generateWithType.js

 import { generate } from "../generate";
+// The ts.io types are way too complex for me to write, I didn't want to spend 2 days fixing this so I
+// decided to bypass the typechecker and throw an error at runtime if the type is not supported.
+// eslint-disable-next-line @typescript-eslint/no-explicit-any
 function typePartial2String(entries, indent, partial) {
@@ -14,2 +17,3 @@ const leftpad = Array(2 * (indent + 1))
 }
+// eslint-disable-next-line @typescript-eslint/no-explicit-any
 function internalTsio2String(type, indent) {
@@ -35,2 +39,3 @@ const leftpad = Array(2 * indent)
     if (type._tag === "UnionType") {
+        // eslint-disable-next-line @typescript-eslint/no-explicit-any
         return type.types.map((t) => internalTsio2String(t, indent + 1)).join(" | ");
@@ -37,0 +42,0 @@ }

transcribe.d.ts

 /// <reference types="node" />
 import { Readable } from "readable-stream";
-import { Buffer } from "buffer";
+import { FileInput } from "./utils";
 import { InputClientOptions } from "./clientOpts";
-interface MinimalStream {
-    on(event: string | symbol, listener: (...args: any[]) => void): this;
-}
-export declare function transcribe(file: Buffer | MinimalStream, clientOptions?: InputClientOptions, supabaseClient?: {
+export declare function transcribe(file: FileInput, clientOptions?: InputClientOptions, supabaseClient?: {
     supabaseUrl: string;
@@ -16,2 +13,1 @@ supabaseKey: string;
 export default function client(clientOptions?: InputClientOptions): TranscribeClient;
-export {};

transcribe.js

 import axios, { AxiosError } from "axios";
 import { createClient } from "@supabase/supabase-js";
 import * as t from "polyfact-io-ts";
-import { Buffer } from "buffer";
+import { fileInputToBuffer } from "./utils";
 import { defaultOptions, supabaseDefaultClient } from "./clientOpts";
@@ -10,10 +10,2 @@ import { ApiError } from "./helpers/error";
 });
-function stream2buffer(stream) {
-    return new Promise((resolve, reject) => {
-        const buf = [];
-        stream.on("data", (chunk) => buf.push(chunk));
-        stream.on("end", () => resolve(Buffer.concat(buf)));
-        stream.on("error", (err) => reject(err));
-    });
-}
 function randomString() {
@@ -26,9 +18,3 @@ const a = () => Math.floor(Math.random() * 1e16).toString(36);
         const { token, endpoint } = await defaultOptions(clientOptions);
-        let buf;
-        if (file instanceof Buffer) {
-            buf = file;
-        }
-        else {
-            buf = await stream2buffer(file);
-        }
+        const buf = await fileInputToBuffer(file);
         const supa = createClient(supabaseClient.supabaseUrl, supabaseClient.supabaseKey, {
@@ -35,0 +21,0 @@ auth: { persistSession: false },

utils.d.ts

@@ -0,2 +1,4 @@
+/// <reference types="node" />
 import { Mutex } from "async-mutex";
+import { Buffer } from "buffer";
 export declare const POLYFACT_TOKEN: string | undefined;
@@ -27,2 +29,16 @@ export declare const POLYFACT_ENDPOINT: string | undefined;
 }
+export type OnFn<R> = ((t: "data", listener: (chunk: Buffer) => void) => R) & ((t: "error", listener: (err: Error) => void) => R) & ((t: string, listener: (...args: unknown[]) => void) => R);
+export interface MinimalStream {
+    on: OnFn<this>;
+}
+export interface FetchReadableStream {
+    getReader(): {
+        read(): Promise<{
+            done: boolean;
+            value?: Uint8Array | undefined;
+        }>;
+    };
+}
+export type FileInput = MinimalStream | Buffer | FetchReadableStream;
+export declare function fileInputToBuffer(input: FileInput): Promise<Buffer>;
 export {};

utils.js

 import { Mutex } from "async-mutex";
+import { Buffer } from "buffer";
 let token;
@@ -10,2 +11,9 @@ let endpoint;
 export const POLYFACT_ENDPOINT = endpoint;
+function splitPromiseResolver() {
+    let resolver;
+    const promise = new Promise((resolve) => {
+        resolver = resolve;
+    });
+    return [promise, resolver]; // eslint-disable-line @typescript-eslint/no-non-null-assertion
+}
 export class MutablePromise {
@@ -17,7 +25,3 @@ constructor() {
         this.mutex = new Mutex();
-        let resolver;
-        this.promiseResult = new Promise((resolve, reject) => {
-            resolver = resolve;
-        });
-        this.resolver = resolver;
+        [this.promiseResult, this.resolver] = splitPromiseResolver();
     }
@@ -46,7 +50,3 @@ set(value) {
         await this.mutex.runExclusive(() => {
-            let resolver;
-            this.promiseResult = new Promise((resolve) => {
-                resolver = resolve;
-            });
-            this.resolver = resolver;
+            [this.promiseResult, this.resolver] = splitPromiseResolver();
             Object.assign(this.result, {
@@ -80,1 +80,35 @@ status: "pending",
 }
+function stream2buffer(stream) {
+    return new Promise((resolve, reject) => {
+        const buf = [];
+        stream.on("data", (chunk) => buf.push(chunk));
+        stream.on("end", () => resolve(Buffer.concat(buf)));
+        stream.on("error", (err) => reject(err));
+    });
+}
+async function fetchStream2buffer(stream) {
+    const reader = stream.getReader();
+    const chunks = [];
+    let done = false;
+    let value;
+    while (!done) {
+        // eslint-disable-next-line no-await-in-loop
+        ({ done, value } = await reader.read());
+        if (value) {
+            chunks.push(value);
+        }
+    }
+    return Buffer.concat(chunks);
+}
+export async function fileInputToBuffer(input) {
+    if (input instanceof Buffer) {
+        return input;
+    }
+    if ("on" in input) {
+        return stream2buffer(input);
+    }
+    if ("getReader" in input) {
+        return fetchStream2buffer(input);
+    }
+    return null;
+}

No alert changes

Worsened metrics

Total package byte prevSize

2352226

Lines of code

1880

decreased by-0.69%

No dependency changes