@llamaindex/core - npm Package Compare versions

Comparing version 0.2.1 to 0.2.2

dist/embeddings/index.d.ts

 import { Tokenizers } from '@llamaindex/env';
 import { MessageContentDetail } from '../llms/index.js';
-import { TransformComponent } from '../schema/index.js';
+import { TransformComponent, BaseNode, ImageType } from '../schema/index.js';
 
@@ -36,3 +36,3 @@ declare const DEFAULT_SIMILARITY_TOP_K = 2;
     embedInfo?: EmbeddingInfo;
-    constructor();
+    protected constructor(transformFn?: (nodes: BaseNode[], options?: BaseEmbeddingOptions) => Promise<BaseNode[]>);
     similarity(embedding1: number[], embedding2: number[], mode?: SimilarityType): number;
@@ -56,4 +56,15 @@ abstract getTextEmbedding(text: string): Promise<number[]>;
 
+declare abstract class MultiModalEmbedding extends BaseEmbedding {
+    abstract getImageEmbedding(images: ImageType): Promise<number[]>;
+    protected constructor();
+    /**
+     * Optionally override this method to retrieve multiple image embeddings in a single request
+     * @param images
+     */
+    getImageEmbeddings(images: ImageType[]): Promise<number[][]>;
+    getQueryEmbedding(query: MessageContentDetail): Promise<number[] | null>;
+}
+
 declare function truncateMaxTokens(tokenizer: Tokenizers, value: string, maxTokens: number): string;
 
-export { BaseEmbedding, type BaseEmbeddingOptions, DEFAULT_SIMILARITY_TOP_K, type EmbeddingInfo, SimilarityType, batchEmbeddings, similarity, truncateMaxTokens };
+export { BaseEmbedding, type BaseEmbeddingOptions, DEFAULT_SIMILARITY_TOP_K, type EmbeddingInfo, MultiModalEmbedding, SimilarityType, batchEmbeddings, similarity, truncateMaxTokens };

dist/embeddings/index.js

@@ -1,3 +0,3 @@
-import { TransformComponent, MetadataMode } from '../schema/index.js';
-import { extractSingleText } from '../utils/index.js';
+import { TransformComponent, MetadataMode, splitNodesByType, ModalityType } from '../schema/index.js';
+import { extractSingleText, extractImage } from '../utils/index.js';
 import { tokenizers } from '@llamaindex/env';
@@ -74,23 +74,36 @@
 class BaseEmbedding extends TransformComponent {
-    constructor(){
-        super(async (nodes, options)=>{
-            const texts = nodes.map((node)=>node.getContent(MetadataMode.EMBED));
-            const embeddings = await this.getTextEmbeddingsBatch(texts, options);
-            for(let i = 0; i < nodes.length; i++){
-                nodes[i].embedding = embeddings[i];
-            }
-            return nodes;
-        });
-        this.embedBatchSize = DEFAULT_EMBED_BATCH_SIZE;
-        /**
+    constructor(transformFn){
+        if (transformFn) {
+            super(transformFn);
+            this.embedBatchSize = DEFAULT_EMBED_BATCH_SIZE;
+            /**
    * Optionally override this method to retrieve multiple embeddings in a single request
    * @param texts
    */ this.getTextEmbeddings = async (texts)=>{
-            const embeddings = [];
-            for (const text of texts){
-                const embedding = await this.getTextEmbedding(text);
-                embeddings.push(embedding);
-            }
-            return embeddings;
-        };
+                const embeddings = [];
+                for (const text of texts){
+                    const embedding = await this.getTextEmbedding(text);
+                    embeddings.push(embedding);
+                }
+                return embeddings;
+            };
+        } else {
+            super(async (nodes, options)=>{
+                const texts = nodes.map((node)=>node.getContent(MetadataMode.EMBED));
+                const embeddings = await this.getTextEmbeddingsBatch(texts, options);
+                for(let i = 0; i < nodes.length; i++){
+                    nodes[i].embedding = embeddings[i];
+                }
+                return nodes;
+            });
+            this.embedBatchSize = DEFAULT_EMBED_BATCH_SIZE;
+            this.getTextEmbeddings = async (texts)=>{
+                const embeddings = [];
+                for (const text of texts){
+                    const embedding = await this.getTextEmbedding(text);
+                    embeddings.push(embedding);
+                }
+                return embeddings;
+            };
+        }
     }
@@ -140,2 +153,40 @@ similarity(embedding1, embedding2, mode = SimilarityType.DEFAULT) {
 
-export { BaseEmbedding, DEFAULT_SIMILARITY_TOP_K, SimilarityType, batchEmbeddings, similarity, truncateMaxTokens };
+/*
+ * Base class for Multi Modal embeddings.
+ */ class MultiModalEmbedding extends BaseEmbedding {
+    constructor(){
+        super(async (nodes, options)=>{
+            const nodeMap = splitNodesByType(nodes);
+            const imageNodes = nodeMap[ModalityType.IMAGE] ?? [];
+            const textNodes = nodeMap[ModalityType.TEXT] ?? [];
+            const embeddings = await batchEmbeddings(textNodes.map((node)=>node.getContent(MetadataMode.EMBED)), this.getTextEmbeddings.bind(this), this.embedBatchSize, options);
+            for(let i = 0; i < textNodes.length; i++){
+                textNodes[i].embedding = embeddings[i];
+            }
+            const imageEmbeddings = await batchEmbeddings(imageNodes.map((n)=>n.image), this.getImageEmbeddings.bind(this), this.embedBatchSize, options);
+            for(let i = 0; i < imageNodes.length; i++){
+                imageNodes[i].embedding = imageEmbeddings[i];
+            }
+            return nodes;
+        });
+    }
+    /**
+   * Optionally override this method to retrieve multiple image embeddings in a single request
+   * @param images
+   */ async getImageEmbeddings(images) {
+        return Promise.all(images.map((imgFilePath)=>this.getImageEmbedding(imgFilePath)));
+    }
+    async getQueryEmbedding(query) {
+        const image = extractImage(query);
+        if (image) {
+            return await this.getImageEmbedding(image);
+        }
+        const text = extractSingleText(query);
+        if (text) {
+            return await this.getTextEmbedding(text);
+        }
+        return null;
+    }
+}
+
+export { BaseEmbedding, DEFAULT_SIMILARITY_TOP_K, MultiModalEmbedding, SimilarityType, batchEmbeddings, similarity, truncateMaxTokens };

package.json

 {
   "name": "@llamaindex/core",
   "type": "module",
-  "version": "0.2.1",
+  "version": "0.2.2",
   "description": "LlamaIndex Core Module",
@@ -160,2 +160,30 @@ "exports": {
       }
+    },
+    "./memory": {
+      "require": {
+        "types": "./dist/memory/index.d.cts",
+        "default": "./dist/memory/index.cjs"
+      },
+      "import": {
+        "types": "./dist/memory/index.d.ts",
+        "default": "./dist/memory/index.js"
+      },
+      "default": {
+        "types": "./dist/memory/index.d.ts",
+        "default": "./dist/memory/index.js"
+      }
+    },
+    "./storage/chat-store": {
+      "require": {
+        "types": "./dist/storage/chat-store/index.d.cts",
+        "default": "./dist/storage/chat-store/index.cjs"
+      },
+      "import": {
+        "types": "./dist/storage/chat-store/index.d.ts",
+        "default": "./dist/storage/chat-store/index.js"
+      },
+      "default": {
+        "types": "./dist/storage/chat-store/index.d.ts",
+        "default": "./dist/storage/chat-store/index.js"
+      }
     }
@@ -162,0 +190,0 @@ },

No alert changes

Improved metrics

Total package byte prevSize

364741

10.25%

Number of package files

54

17.39%

Lines of code

7945

9.1%

No dependency changes