llamaindex - npm Package Compare versions

Comparing version 0.0.17 to 0.0.18

CHANGELOG.md

 # llamaindex
 
+## 0.0.18
+
+### Patch Changes
+
+- 824c13c: Breaking: allow documents to be reimported with hash checking.
+- 18b8915: Update storage exports (thanks @TomPenguin)
+- ade9d8f: Bug fix: use session in OpenAI Embeddings (thanks @swk777)
+- 824c13c: Breaking: removed nodeId and docId. Just use id\_
+
 ## 0.0.17
@@ -4,0 +13,0 @@

dist/index.d.ts

@@ -1,4 +0,4 @@
-import OpenAI$1, { ClientOptions } from 'openai';
+import Anthropic$1, { ClientOptions } from '@anthropic-ai/sdk';
+import OpenAI$1, { ClientOptions as ClientOptions$1 } from 'openai';
 import Replicate from 'replicate';
-import Anthropic$1, { ClientOptions as ClientOptions$1 } from '@anthropic-ai/sdk';
 
@@ -35,2 +35,8 @@ declare enum NodeRelationship {
 declare abstract class BaseNode {
+    /**
+     * The unique ID of the Node/Document. The trailing underscore is here
+     * to avoid collisions with the id keyword in Python.
+     *
+     * Set to a UUID by default.
+     */
     id_: string;
@@ -48,3 +54,2 @@ embedding?: number[];
     abstract setContent(value: any): void;
-    get nodeId(): string;
     get sourceNode(): RelatedNodeInfo | undefined;
@@ -55,2 +60,3 @@ get prevNode(): RelatedNodeInfo | undefined;
     get childNodes(): RelatedNodeInfo[] | undefined;
+    abstract generateHash(): string;
     getEmbedding(): number[];
@@ -73,3 +79,8 @@ asRelatedNodeInfo(): RelatedNodeInfo;
     constructor(init?: Partial<TextNode>);
-    generateHash(): void;
+    /**
+     * Generate a hash of the text node.
+     * The ID is not part of the hash as it can change independent of content.
+     * @returns
+     */
+    generateHash(): string;
     getType(): ObjectType;
@@ -96,3 +107,2 @@ getContent(metadataMode?: MetadataMode): string;
     getType(): ObjectType;
-    get docId(): string;
 }
@@ -159,5 +169,17 @@ declare function jsonToNode(json: any): TextNode;
 
+declare class AnthropicSession {
+    anthropic: Anthropic$1;
+    constructor(options?: ClientOptions);
+}
+
+interface AzureOpenAIConfig {
+    apiKey?: string;
+    endpoint?: string;
+    apiVersion?: string;
+    deploymentName?: string;
+}
+
 declare class OpenAISession {
     openai: OpenAI$1;
-    constructor(options?: ClientOptions & {
+    constructor(options?: ClientOptions$1 & {
         azure?: boolean;
@@ -173,14 +195,2 @@ });
 
-declare class AnthropicSession {
-    anthropic: Anthropic$1;
-    constructor(options?: ClientOptions$1);
-}
-
-interface AzureOpenAIConfig {
-    apiKey?: string;
-    endpoint?: string;
-    apiVersion?: string;
-    deploymentName?: string;
-}
-
 type MessageType = "user" | "assistant" | "system" | "generic" | "function";
@@ -967,3 +977,3 @@ interface ChatMessage {
 interface RefDocInfo {
-    docIds: string[];
+    nodeIds: string[];
     extraInfo: Record<string, any>;
@@ -974,5 +984,5 @@ }
     abstract docs(): Promise<Record<string, BaseNode>>;
-    abstract addDocuments(docs: BaseNode[], allowUpdate: boolean): void;
+    abstract addDocuments(docs: BaseNode[], allowUpdate: boolean): Promise<void>;
     abstract getDocument(docId: string, raiseError: boolean): Promise<BaseNode | undefined>;
-    abstract deleteDocument(docId: string, raiseError: boolean): void;
+    abstract deleteDocument(docId: string, raiseError: boolean): Promise<void>;
     abstract documentExists(docId: string): Promise<boolean>;
@@ -1029,3 +1039,2 @@ abstract setDocumentHash(docId: string, docHash: string): void;
     nodesDict: Record<string, BaseNode>;
-    docStore: Record<string, Document>;
     type: IndexStructType;
@@ -1035,2 +1044,3 @@ getSummary(): string;
     toJson(): Record<string, unknown>;
+    delete(nodeId: string): void;
 }
@@ -1078,2 +1088,9 @@ declare function jsonToIndexStruct(json: any): IndexStruct;
     }): BaseQueryEngine;
+    /**
+     * Insert a document into the index.
+     * @param document
+     */
+    insert(document: Document): Promise<void>;
+    abstract insertNodes(nodes: BaseNode[]): Promise<void>;
+    abstract deleteRefDoc(refDocId: string, deleteFromDocStore?: boolean): Promise<void>;
 }
@@ -1119,5 +1136,6 @@ interface VectorIndexOptions {
     }): BaseQueryEngine;
-    static _buildIndexFromNodes(nodes: BaseNode[], docStore: BaseDocumentStore, indexStruct?: IndexList): Promise<IndexList>;
-    protected _insert(nodes: BaseNode[]): void;
-    protected _deleteNode(nodeId: string): void;
+    static buildIndexFromNodes(nodes: BaseNode[], docStore: BaseDocumentStore, indexStruct?: IndexList): Promise<IndexList>;
+    insertNodes(nodes: BaseNode[]): Promise<void>;
+    deleteRefDoc(refDocId: string, deleteFromDocStore?: boolean): Promise<void>;
+    deleteNodes(nodeIds: string[], deleteFromDocStore: boolean): Promise<void>;
     getRefDocInfo(): Promise<Record<string, RefDocInfo>>;
@@ -1199,3 +1217,3 @@ }
      */
-    static buildIndexFromNodes(nodes: BaseNode[], serviceContext: ServiceContext, vectorStore: VectorStore, docStore: BaseDocumentStore): Promise<IndexDict>;
+    static buildIndexFromNodes(nodes: BaseNode[], serviceContext: ServiceContext, vectorStore: VectorStore, docStore: BaseDocumentStore, indexDict?: IndexDict): Promise<IndexDict>;
     /**
@@ -1217,2 +1235,4 @@ * High level API: split documents, get embeddings, and build index.
     }): BaseQueryEngine;
+    insertNodes(nodes: BaseNode[]): Promise<void>;
+    deleteRefDoc(refDocId: string, deleteFromDocStore?: boolean): Promise<void>;
 }
@@ -1261,2 +1281,103 @@
 
-export { ALL_AVAILABLE_LLAMADEUCE_MODELS, ALL_AVAILABLE_OPENAI_MODELS, Anthropic, BaseEmbedding, BaseIndex, BaseIndexInit, BaseNode, BaseOutputParser, BaseQueryEngine, BaseQuestionGenerator, BaseReader, BaseRetriever, BaseTool, CallbackManager, ChatEngine, ChatMessage, ChatResponse, CompactAndRefine, CompleteFileSystem, CompletionResponse, CondenseQuestionChatEngine, ContextChatEngine, DEFAULT_CHUNK_OVERLAP, DEFAULT_CHUNK_OVERLAP_RATIO, DEFAULT_CHUNK_SIZE, DEFAULT_COLLECTION, DEFAULT_CONTEXT_WINDOW, DEFAULT_DOC_STORE_PERSIST_FILENAME, DEFAULT_EMBEDDING_DIM, DEFAULT_FS, DEFAULT_GRAPH_STORE_PERSIST_FILENAME, DEFAULT_INDEX_STORE_PERSIST_FILENAME, DEFAULT_NAMESPACE, DEFAULT_NUM_OUTPUTS, DEFAULT_PADDING, DEFAULT_PERSIST_DIR, DEFAULT_SIMILARITY_TOP_K, DEFAULT_VECTOR_STORE_PERSIST_FILENAME, DeuceChatStrategy, Document, Event, EventTag, EventType, ExactMatchFilter, GPT4_MODELS, GenericFileSystem, InMemoryFileSystem, IndexDict, IndexList, IndexNode, IndexStruct, IndexStructType, LLM, LLMQuestionGenerator, ListIndex, ListIndexLLMRetriever, ListIndexRetriever, ListRetrieverMode, LlamaDeuce, MessageType, MetadataFilters, MetadataInfo, MetadataMode, NodeParser, NodeRelationship, NodeWithEmbedding, NodeWithScore, ObjectType, OpenAI, OpenAIEmbedding, PDFReader, QueryEngineTool, Refine, RelatedNodeInfo, RelatedNodeType, Response, ResponseSynthesizer, RetrievalCallbackResponse, RetrieverQueryEngine, SentenceSplitter, ServiceContext, ServiceContextOptions, SimilarityType, SimpleChatEngine, SimpleDirectoryReader, SimpleDirectoryReaderLoadDataProps, SimpleNodeParser, SimplePrompt, SimpleResponseBuilder, StorageContext, StreamCallbackResponse, StreamToken, StructuredOutput, SubQuestion, SubQuestionOutputParser, SubQuestionQueryEngine, TURBO_MODELS, TextFileReader, TextNode, ToolMetadata, TreeSummarize, VectorIndexConstructorProps, VectorIndexOptions, VectorIndexRetriever, VectorStore, VectorStoreIndex, VectorStoreInfo, VectorStoreQuery, VectorStoreQueryMode, VectorStoreQueryResult, VectorStoreQuerySpec, WalkableFileSystem, buildToolsText, contextSystemPrompt, defaultChoiceSelectPrompt, defaultCondenseQuestionPrompt, defaultRefinePrompt, defaultSubQuestionPrompt, defaultSummaryPrompt, defaultTextQaPrompt, exists, getNodeFS, getNodesFromDocument, getResponseBuilder, getTextSplitsFromDocument, getTopKEmbeddings, getTopKEmbeddingsLearner, getTopKMMREmbeddings, globalsHelper, jsonToIndexStruct, jsonToNode, messagesToHistoryStr, serviceContextFromDefaults, serviceContextFromServiceContext, similarity, storageContextFromDefaults, walk };
+type StoredValue = Record<string, any> | null;
+declare abstract class BaseKVStore {
+    abstract put(key: string, val: Record<string, any>, collection?: string): Promise<void>;
+    abstract get(key: string, collection?: string): Promise<StoredValue>;
+    abstract getAll(collection?: string): Promise<Record<string, StoredValue>>;
+    abstract delete(key: string, collection?: string): Promise<boolean>;
+}
+declare abstract class BaseInMemoryKVStore extends BaseKVStore {
+    abstract persist(persistPath: string, fs?: GenericFileSystem): void;
+    static fromPersistPath(persistPath: string): BaseInMemoryKVStore;
+}
+
+declare class KVDocumentStore extends BaseDocumentStore {
+    private kvstore;
+    private nodeCollection;
+    private refDocCollection;
+    private metadataCollection;
+    constructor(kvstore: BaseKVStore, namespace?: string);
+    docs(): Promise<Record<string, BaseNode>>;
+    addDocuments(docs: BaseNode[], allowUpdate?: boolean): Promise<void>;
+    getDocument(docId: string, raiseError?: boolean): Promise<BaseNode | undefined>;
+    getRefDocInfo(refDocId: string): Promise<RefDocInfo | undefined>;
+    getAllRefDocInfo(): Promise<Record<string, RefDocInfo> | undefined>;
+    refDocExists(refDocId: string): Promise<boolean>;
+    documentExists(docId: string): Promise<boolean>;
+    private removeRefDocNode;
+    deleteDocument(docId: string, raiseError?: boolean, removeRefDocNode?: boolean): Promise<void>;
+    deleteRefDoc(refDocId: string, raiseError?: boolean): Promise<void>;
+    setDocumentHash(docId: string, docHash: string): Promise<void>;
+    getDocumentHash(docId: string): Promise<string | undefined>;
+}
+
+type DataType = Record<string, Record<string, any>>;
+declare class SimpleKVStore extends BaseKVStore {
+    private data;
+    private persistPath;
+    private fs;
+    constructor(data?: DataType);
+    put(key: string, val: any, collection?: string): Promise<void>;
+    get(key: string, collection?: string): Promise<any>;
+    getAll(collection?: string): Promise<DataType>;
+    delete(key: string, collection?: string): Promise<boolean>;
+    persist(persistPath: string, fs?: GenericFileSystem): Promise<void>;
+    static fromPersistPath(persistPath: string, fs?: GenericFileSystem): Promise<SimpleKVStore>;
+    toDict(): DataType;
+    static fromDict(saveDict: DataType): SimpleKVStore;
+}
+
+type SaveDict = Record<string, any>;
+declare class SimpleDocumentStore extends KVDocumentStore {
+    private kvStore;
+    constructor(kvStore?: SimpleKVStore, namespace?: string);
+    static fromPersistDir(persistDir?: string, namespace?: string, fsModule?: GenericFileSystem): Promise<SimpleDocumentStore>;
+    static fromPersistPath(persistPath: string, namespace?: string, fs?: GenericFileSystem): Promise<SimpleDocumentStore>;
+    persist(persistPath?: string, fs?: GenericFileSystem): Promise<void>;
+    static fromDict(saveDict: SaveDict, namespace?: string): SimpleDocumentStore;
+    toDict(): SaveDict;
+}
+
+declare class KVIndexStore extends BaseIndexStore {
+    private _kvStore;
+    private _collection;
+    constructor(kvStore: BaseKVStore, namespace?: string);
+    addIndexStruct(indexStruct: IndexStruct): Promise<void>;
+    deleteIndexStruct(key: string): Promise<void>;
+    getIndexStruct(structId?: string): Promise<IndexStruct | undefined>;
+    getIndexStructs(): Promise<IndexStruct[]>;
+}
+
+declare class SimpleIndexStore extends KVIndexStore {
+    private kvStore;
+    constructor(kvStore?: BaseInMemoryKVStore);
+    static fromPersistDir(persistDir?: string, fs?: GenericFileSystem): Promise<SimpleIndexStore>;
+    static fromPersistPath(persistPath: string, fs?: GenericFileSystem): Promise<SimpleIndexStore>;
+    persist(persistPath?: string, fs?: GenericFileSystem): Promise<void>;
+    static fromDict(saveDict: DataType): SimpleIndexStore;
+    toDict(): Record<string, unknown>;
+}
+
+declare class SimpleVectorStoreData {
+    embeddingDict: Record<string, number[]>;
+    textIdToRefDocId: Record<string, string>;
+}
+declare class SimpleVectorStore implements VectorStore {
+    storesText: boolean;
+    private data;
+    private fs;
+    private persistPath;
+    constructor(data?: SimpleVectorStoreData, fs?: GenericFileSystem);
+    static fromPersistDir(persistDir?: string, fs?: GenericFileSystem): Promise<SimpleVectorStore>;
+    get client(): any;
+    get(textId: string): Promise<number[]>;
+    add(embeddingResults: NodeWithEmbedding[]): Promise<string[]>;
+    delete(refDocId: string): Promise<void>;
+    query(query: VectorStoreQuery): Promise<VectorStoreQueryResult>;
+    persist(persistPath?: string, fs?: GenericFileSystem): Promise<void>;
+    static fromPersistPath(persistPath: string, fs?: GenericFileSystem): Promise<SimpleVectorStore>;
+    static fromDict(saveDict: SimpleVectorStoreData): SimpleVectorStore;
+    toDict(): SimpleVectorStoreData;
+}
+
+export { ALL_AVAILABLE_LLAMADEUCE_MODELS, ALL_AVAILABLE_OPENAI_MODELS, Anthropic, BaseEmbedding, BaseIndex, BaseIndexInit, BaseNode, BaseOutputParser, BaseQueryEngine, BaseQuestionGenerator, BaseReader, BaseRetriever, BaseTool, CallbackManager, ChatEngine, ChatMessage, ChatResponse, CompactAndRefine, CompleteFileSystem, CompletionResponse, CondenseQuestionChatEngine, ContextChatEngine, DEFAULT_CHUNK_OVERLAP, DEFAULT_CHUNK_OVERLAP_RATIO, DEFAULT_CHUNK_SIZE, DEFAULT_COLLECTION, DEFAULT_CONTEXT_WINDOW, DEFAULT_DOC_STORE_PERSIST_FILENAME, DEFAULT_EMBEDDING_DIM, DEFAULT_FS, DEFAULT_GRAPH_STORE_PERSIST_FILENAME, DEFAULT_INDEX_STORE_PERSIST_FILENAME, DEFAULT_NAMESPACE, DEFAULT_NUM_OUTPUTS, DEFAULT_PADDING, DEFAULT_PERSIST_DIR, DEFAULT_SIMILARITY_TOP_K, DEFAULT_VECTOR_STORE_PERSIST_FILENAME, DeuceChatStrategy, Document, Event, EventTag, EventType, ExactMatchFilter, GPT4_MODELS, GenericFileSystem, InMemoryFileSystem, IndexDict, IndexList, IndexNode, IndexStruct, IndexStructType, LLM, LLMQuestionGenerator, ListIndex, ListIndexLLMRetriever, ListIndexRetriever, ListRetrieverMode, LlamaDeuce, MessageType, MetadataFilters, MetadataInfo, MetadataMode, NodeParser, NodeRelationship, NodeWithEmbedding, NodeWithScore, ObjectType, OpenAI, OpenAIEmbedding, PDFReader, QueryEngineTool, Refine, RelatedNodeInfo, RelatedNodeType, Response, ResponseSynthesizer, RetrievalCallbackResponse, RetrieverQueryEngine, SentenceSplitter, ServiceContext, ServiceContextOptions, SimilarityType, SimpleChatEngine, SimpleDirectoryReader, SimpleDirectoryReaderLoadDataProps, SimpleDocumentStore, SimpleIndexStore, SimpleNodeParser, SimplePrompt, SimpleResponseBuilder, SimpleVectorStore, StorageContext, StreamCallbackResponse, StreamToken, StructuredOutput, SubQuestion, SubQuestionOutputParser, SubQuestionQueryEngine, TURBO_MODELS, TextFileReader, TextNode, ToolMetadata, TreeSummarize, VectorIndexConstructorProps, VectorIndexOptions, VectorIndexRetriever, VectorStore, VectorStoreIndex, VectorStoreInfo, VectorStoreQuery, VectorStoreQueryMode, VectorStoreQueryResult, VectorStoreQuerySpec, WalkableFileSystem, buildToolsText, contextSystemPrompt, defaultChoiceSelectPrompt, defaultCondenseQuestionPrompt, defaultRefinePrompt, defaultSubQuestionPrompt, defaultSummaryPrompt, defaultTextQaPrompt, exists, getNodeFS, getNodesFromDocument, getResponseBuilder, getTextSplitsFromDocument, getTopKEmbeddings, getTopKEmbeddingsLearner, getTopKMMREmbeddings, globalsHelper, jsonToIndexStruct, jsonToNode, messagesToHistoryStr, serviceContextFromDefaults, serviceContextFromServiceContext, similarity, storageContextFromDefaults, walk };

package.json

 {
   "name": "llamaindex",
-  "version": "0.0.17",
+  "version": "0.0.18",
   "dependencies": {
@@ -5,0 +5,0 @@ "@anthropic-ai/sdk": "^0.5.9",

src/Embedding.ts

@@ -262,7 +262,9 @@ import { DEFAULT_SIMILARITY_TOP_K } from "./constants";
       this.apiKey = init?.apiKey ?? undefined;
-      this.session = getOpenAISession({
-        apiKey: this.apiKey,
-        maxRetries: this.maxRetries,
-        timeout: this.timeout,
-      });
+      this.session =
+        init?.session ??
+        getOpenAISession({
+          apiKey: this.apiKey,
+          maxRetries: this.maxRetries,
+          timeout: this.timeout,
+        });
     }
@@ -269,0 +271,0 @@ }

src/index.ts

@@ -27,5 +27,2 @@ export * from "./ChatEngine";
 
-export * from "./storage/constants";
-export * from "./storage/FileSystem";
-export * from "./storage/StorageContext";
-export * from "./storage/vectorStore/types";
+export * from "./storage";

src/indices/BaseIndex.ts

@@ -1,11 +0,11 @@
-import { Document, BaseNode, jsonToNode } from "../Node";
 import { v4 as uuidv4 } from "uuid";
+import { BaseNode, Document, jsonToNode } from "../Node";
+import { BaseQueryEngine } from "../QueryEngine";
+import { ResponseSynthesizer } from "../ResponseSynthesizer";
 import { BaseRetriever } from "../Retriever";
 import { ServiceContext } from "../ServiceContext";
+import { BaseDocumentStore } from "../storage/docStore/types";
+import { BaseIndexStore } from "../storage/indexStore/types";
 import { StorageContext } from "../storage/StorageContext";
-import { BaseDocumentStore } from "../storage/docStore/types";
 import { VectorStore } from "../storage/vectorStore/types";
-import { BaseIndexStore } from "../storage/indexStore/types";
-import { BaseQueryEngine } from "../QueryEngine";
-import { ResponseSynthesizer } from "../ResponseSynthesizer";
 
@@ -46,3 +46,2 @@ /**
   nodesDict: Record<string, BaseNode> = {};
-  docStore: Record<string, Document> = {}; // FIXME: this should be implemented in storageContext
   type: IndexStructType = IndexStructType.SIMPLE_DICT;
@@ -69,2 +68,6 @@
   }
+
+  delete(nodeId: string) {
+    delete this.nodesDict[nodeId];
+  }
 }
@@ -153,2 +156,20 @@
   }): BaseQueryEngine;
+
+  /**
+   * Insert a document into the index.
+   * @param document
+   */
+  async insert(document: Document) {
+    const nodes = this.serviceContext.nodeParser.getNodesFromDocuments([
+      document,
+    ]);
+    await this.insertNodes(nodes);
+    this.docStore.setDocumentHash(document.id_, document.hash);
+  }
+
+  abstract insertNodes(nodes: BaseNode[]): Promise<void>;
+  abstract deleteRefDoc(
+    refDocId: string,
+    deleteFromDocStore?: boolean,
+  ): Promise<void>;
 }
@@ -155,0 +176,0 @@

src/indices/list/ListIndex.ts

@@ -0,14 +1,10 @@
+import _ from "lodash";
 import { BaseNode, Document } from "../../Node";
-import { BaseIndex, BaseIndexInit, IndexList } from "../BaseIndex";
 import { BaseQueryEngine, RetrieverQueryEngine } from "../../QueryEngine";
 import {
-  StorageContext,
-  storageContextFromDefaults,
-} from "../../storage/StorageContext";
+  CompactAndRefine,
+  ResponseSynthesizer,
+} from "../../ResponseSynthesizer";
 import { BaseRetriever } from "../../Retriever";
 import {
-  ListIndexRetriever,
-  ListIndexLLMRetriever,
-} from "./ListIndexRetriever";
-import {
   ServiceContext,
@@ -18,8 +14,16 @@ serviceContextFromDefaults,
 import { BaseDocumentStore, RefDocInfo } from "../../storage/docStore/types";
-import _ from "lodash";
 import {
-  ResponseSynthesizer,
-  CompactAndRefine,
-} from "../../ResponseSynthesizer";
-import { IndexStructType } from "../BaseIndex";
+  StorageContext,
+  storageContextFromDefaults,
+} from "../../storage/StorageContext";
+import {
+  BaseIndex,
+  BaseIndexInit,
+  IndexList,
+  IndexStructType,
+} from "../BaseIndex";
+import {
+  ListIndexLLMRetriever,
+  ListIndexRetriever,
+} from "./ListIndexRetriever";
 
@@ -61,3 +65,3 @@ export enum ListRetrieverMode {
       throw new Error(
-        "Cannot initialize index with both indexStruct and indexStore"
+        "Cannot initialize index with both indexStruct and indexStore",
       );
@@ -72,3 +76,3 @@ }
       indexStruct = (await indexStore.getIndexStruct(
-        options.indexId
+        options.indexId,
       )) as IndexList;
@@ -82,3 +86,3 @@ } else {
       throw new Error(
-        "Attempting to initialize ListIndex with non-list indexStruct"
+        "Attempting to initialize ListIndex with non-list indexStruct",
       );
@@ -90,3 +94,3 @@ }
         throw new Error(
-          "Cannot initialize VectorStoreIndex with both nodes and indexStruct"
+          "Cannot initialize VectorStoreIndex with both nodes and indexStruct",
         );
@@ -97,8 +101,8 @@ }
         throw new Error(
-          "Cannot initialize VectorStoreIndex without nodes or indexStruct"
+          "Cannot initialize VectorStoreIndex without nodes or indexStruct",
         );
       }
-      indexStruct = await ListIndex._buildIndexFromNodes(
+      indexStruct = await ListIndex.buildIndexFromNodes(
         options.nodes,
-        storageContext.docStore
+        storageContext.docStore,
       );
@@ -123,3 +127,3 @@
       serviceContext?: ServiceContext;
-    } = {}
+    } = {},
   ): Promise<ListIndex> {
@@ -179,6 +183,6 @@ let { storageContext, serviceContext } = args;
 
-  static async _buildIndexFromNodes(
+  static async buildIndexFromNodes(
     nodes: BaseNode[],
     docStore: BaseDocumentStore,
-    indexStruct?: IndexList
+    indexStruct?: IndexList,
   ): Promise<IndexList> {
@@ -195,3 +199,3 @@ indexStruct = indexStruct || new IndexList();
 
-  protected _insert(nodes: BaseNode[]): void {
+  async insertNodes(nodes: BaseNode[]): Promise<void> {
     for (const node of nodes) {
@@ -202,6 +206,33 @@ this.indexStruct.addNode(node);
 
-  protected _deleteNode(nodeId: string): void {
+  async deleteRefDoc(
+    refDocId: string,
+    deleteFromDocStore?: boolean,
+  ): Promise<void> {
+    const refDocInfo = await this.docStore.getRefDocInfo(refDocId);
+
+    if (!refDocInfo) {
+      return;
+    }
+
+    await this.deleteNodes(refDocInfo.nodeIds, false);
+
+    if (deleteFromDocStore) {
+      await this.docStore.deleteRefDoc(refDocId, false);
+    }
+
+    return;
+  }
+
+  async deleteNodes(nodeIds: string[], deleteFromDocStore: boolean) {
     this.indexStruct.nodes = this.indexStruct.nodes.filter(
-      (existingNodeId: string) => existingNodeId !== nodeId
+      (existingNodeId: string) => !nodeIds.includes(existingNodeId),
     );
+
+    if (deleteFromDocStore) {
+      for (const nodeId of nodeIds) {
+        await this.docStore.deleteDocument(nodeId, false);
+      }
+    }
+
+    await this.storageContext.indexStore.addIndexStruct(this.indexStruct);
   }
@@ -208,0 +239,0 @@

src/indices/vectorStore/VectorStoreIndex.ts

 import {
+  BaseNode,
   Document,
-  BaseNode,
   MetadataMode,
@@ -8,3 +8,4 @@ NodeWithEmbedding,
 import { BaseQueryEngine, RetrieverQueryEngine } from "../../QueryEngine";
-import { VectorIndexRetriever } from "./VectorIndexRetriever";
+import { ResponseSynthesizer } from "../../ResponseSynthesizer";
+import { BaseRetriever } from "../../Retriever";
 import {
@@ -14,2 +15,3 @@ ServiceContext,
 } from "../../ServiceContext";
+import { BaseDocumentStore } from "../../storage/docStore/types";
 import {
@@ -23,9 +25,7 @@ StorageContext,
   IndexDict,
+  IndexStructType,
   VectorIndexConstructorProps,
   VectorIndexOptions,
-  IndexStructType
 } from "../BaseIndex";
-import { BaseRetriever } from "../../Retriever";
-import { ResponseSynthesizer } from "../../ResponseSynthesizer";
-import { BaseDocumentStore } from "../../storage/docStore/types";
+import { VectorIndexRetriever } from "./VectorIndexRetriever";
 
@@ -61,7 +61,7 @@ /**
     let indexStructs = (await indexStore.getIndexStructs()) as IndexDict[];
-    let indexStruct: IndexDict | null;
+    let indexStruct: IndexDict | undefined;
 
     if (options.indexStruct && indexStructs.length > 0) {
       throw new Error(
-        "Cannot initialize index with both indexStruct and indexStore"
+        "Cannot initialize index with both indexStruct and indexStore",
       );
@@ -76,6 +76,6 @@ }
       indexStruct = (await indexStore.getIndexStruct(
-        options.indexId
+        options.indexId,
       )) as IndexDict;
     } else {
-      indexStruct = null;
+      indexStruct = undefined;
     }
@@ -86,28 +86,24 @@
       throw new Error(
-        "Attempting to initialize VectorStoreIndex with non-vector indexStruct"
+        "Attempting to initialize VectorStoreIndex with non-vector indexStruct",
       );
     }
 
-    if (indexStruct) {
-      if (options.nodes) {
-        throw new Error(
-          "Cannot initialize VectorStoreIndex with both nodes and indexStruct"
-        );
-      }
-    } else {
-      if (!options.nodes) {
-        throw new Error(
-          "Cannot initialize VectorStoreIndex without nodes or indexStruct"
-        );
-      }
-      indexStruct = await VectorStoreIndex.buildIndexFromNodes(
-        options.nodes,
-        serviceContext,
-        vectorStore,
-        docStore
+    if (!indexStruct && !options.nodes) {
+      throw new Error(
+        "Cannot initialize VectorStoreIndex without nodes or indexStruct",
       );
-
-      await indexStore.addIndexStruct(indexStruct);
     }
 
+    const nodes = options.nodes ?? [];
+
+    indexStruct = await VectorStoreIndex.buildIndexFromNodes(
+      nodes,
+      serviceContext,
+      vectorStore,
+      docStore,
+      indexStruct,
+    );
+
+    await indexStore.addIndexStruct(indexStruct);
+
     return new VectorStoreIndex({
@@ -132,3 +128,3 @@ storageContext,
     serviceContext: ServiceContext,
-    logProgress = false
+    logProgress = false,
   ) {
@@ -143,3 +139,3 @@ const nodesWithEmbeddings: NodeWithEmbedding[] = [];
       const embedding = await serviceContext.embedModel.getTextEmbedding(
-        node.getContent(MetadataMode.EMBED)
+        node.getContent(MetadataMode.EMBED),
       );
@@ -163,10 +159,23 @@ nodesWithEmbeddings.push({ node, embedding });
     vectorStore: VectorStore,
-    docStore: BaseDocumentStore
+    docStore: BaseDocumentStore,
+    indexDict?: IndexDict,
   ): Promise<IndexDict> {
+    indexDict = indexDict ?? new IndexDict();
+
+    // Check if the index already has nodes with the same hash
+    const newNodes = nodes.filter((node) =>
+      Object.entries(indexDict!.nodesDict).reduce((acc, [key, value]) => {
+        if (value.hash === node.hash) {
+          acc = false;
+        }
+        return acc;
+      }, true),
+    );
+
     const embeddingResults = await this.getNodeEmbeddingResults(
-      nodes,
-      serviceContext
+      newNodes,
+      serviceContext,
     );
 
-    vectorStore.add(embeddingResults);
+    await vectorStore.add(embeddingResults);
 
@@ -176,7 +185,6 @@ if (!vectorStore.storesText) {
         embeddingResults.map((result) => result.node),
-        true
+        true,
       );
     }
 
-    const indexDict = new IndexDict();
     for (const { node } of embeddingResults) {
@@ -201,3 +209,3 @@ indexDict.addNode(node);
       serviceContext?: ServiceContext;
-    } = {}
+    } = {},
   ): Promise<VectorStoreIndex> {
@@ -231,4 +239,55 @@ let { storageContext, serviceContext } = args;
     const { retriever, responseSynthesizer } = options ?? {};
-    return new RetrieverQueryEngine(retriever ?? this.asRetriever(), responseSynthesizer);
+    return new RetrieverQueryEngine(
+      retriever ?? this.asRetriever(),
+      responseSynthesizer,
+    );
   }
+
+  async insertNodes(nodes: BaseNode[]): Promise<void> {
+    const embeddingResults = await VectorStoreIndex.getNodeEmbeddingResults(
+      nodes,
+      this.serviceContext,
+    );
+
+    const newIds = await this.vectorStore.add(embeddingResults);
+
+    if (!this.vectorStore.storesText) {
+      for (let i = 0; i < nodes.length; ++i) {
+        this.indexStruct.addNode(nodes[i], newIds[i]);
+        this.docStore.addDocuments([nodes[i]], true);
+      }
+    } else {
+      for (let i = 0; i < nodes.length; ++i) {
+        if (nodes[i].getType() === "INDEX") {
+          this.indexStruct.addNode(nodes[i], newIds[i]);
+          this.docStore.addDocuments([nodes[i]], true);
+        }
+      }
+    }
+
+    await this.storageContext.indexStore.addIndexStruct(this.indexStruct);
+  }
+
+  async deleteRefDoc(
+    refDocId: string,
+    deleteFromDocStore: boolean = true,
+  ): Promise<void> {
+    this.vectorStore.delete(refDocId);
+
+    if (!this.vectorStore.storesText) {
+      const refDocInfo = await this.docStore.getRefDocInfo(refDocId);
+
+      if (refDocInfo) {
+        for (const nodeId of refDocInfo.nodeIds) {
+          this.indexStruct.delete(nodeId);
+        }
+      }
+
+      await this.storageContext.indexStore.addIndexStruct(this.indexStruct);
+    }
+
+    if (deleteFromDocStore) {
+      await this.docStore.deleteDocument(refDocId, false);
+    }
+  }
 }

src/llm/LLM.ts

@@ -0,10 +1,8 @@
+import OpenAILLM from "openai";
 import { CallbackManager, Event } from "../callbacks/CallbackManager";
 import { handleOpenAIStream } from "../callbacks/utility/handleOpenAIStream";
-import { OpenAISession, getOpenAISession } from "./openai";
-import OpenAILLM from "openai";
-import { ReplicateSession } from "./replicate";
 import {
+  AnthropicSession,
   ANTHROPIC_AI_PROMPT,
   ANTHROPIC_HUMAN_PROMPT,
-  AnthropicSession,
   getAnthropicSession,
@@ -14,7 +12,9 @@ } from "./anthropic";
   AzureOpenAIConfig,
+  getAzureBaseUrl,
   getAzureConfigFromEnv,
   getAzureModel,
-  getAzureBaseUrl,
   shouldUseAzure,
 } from "./azure";
+import { getOpenAISession, OpenAISession } from "./openai";
+import { ReplicateSession } from "./replicate";
 
@@ -112,3 +112,3 @@ export type MessageType =
         throw new Error(
-          "Azure API key is required for OpenAI Azure models. Please set the AZURE_OPENAI_KEY environment variable."
+          "Azure API key is required for OpenAI Azure models. Please set the AZURE_OPENAI_KEY environment variable.",
         );
@@ -143,3 +143,3 @@ }
   mapMessageType(
-    messageType: MessageType
+    messageType: MessageType,
   ): "user" | "assistant" | "system" | "function" {
@@ -162,3 +162,3 @@ switch (messageType) {
     messages: ChatMessage[],
-    parentEvent?: Event
+    parentEvent?: Event,
   ): Promise<ChatResponse> {
@@ -203,3 +203,3 @@ const baseRequestParams: OpenAILLM.Chat.CompletionCreateParams = {
     prompt: string,
-    parentEvent?: Event
+    parentEvent?: Event,
   ): Promise<CompletionResponse> {
@@ -329,3 +329,3 @@ return this.chat([{ content: prompt, role: "user" }], parentEvent);
     messages: ChatMessage[],
-    opts?: { withBos?: boolean; replicate4Bit?: boolean }
+    opts?: { withBos?: boolean; replicate4Bit?: boolean },
   ) {
@@ -363,3 +363,3 @@ const { withBos = false, replicate4Bit = false } = opts ?? {};
           throw new Error(
-            "LlamaDeuce: if there is a system message, the second message must be a user message."
+            "LlamaDeuce: if there is a system message, the second message must be a user message.",
           );
@@ -394,3 +394,3 @@ }
     messages: ChatMessage[],
-    _parentEvent?: Event
+    _parentEvent?: Event,
   ): Promise<ChatResponse> {
@@ -419,3 +419,3 @@ const api = ALL_AVAILABLE_LLAMADEUCE_MODELS[this.model]
       api,
-      replicateOptions
+      replicateOptions,
     );
@@ -433,3 +433,3 @@ return {
     prompt: string,
-    parentEvent?: Event
+    parentEvent?: Event,
   ): Promise<CompletionResponse> {
@@ -496,3 +496,3 @@ return this.chat([{ content: prompt, role: "user" }], parentEvent);
     messages: ChatMessage[],
-    parentEvent?: Event | undefined
+    parentEvent?: Event | undefined,
   ): Promise<ChatResponse> {
@@ -515,3 +515,3 @@ const response = await this.session.anthropic.completions.create({
     prompt: string,
-    parentEvent?: Event | undefined
+    parentEvent?: Event | undefined,
   ): Promise<CompletionResponse> {
@@ -518,0 +518,0 @@ return this.chat([{ content: prompt, role: "user" }], parentEvent);

src/Node.ts

@@ -0,1 +1,2 @@
+import crypto from "crypto"; // TODO Node dependency
 import { v4 as uuidv4 } from "uuid";
@@ -38,2 +39,8 @@
 export abstract class BaseNode {
+  /**
+   * The unique ID of the Node/Document. The trailing underscore is here
+   * to avoid collisions with the id keyword in Python.
+   *
+   * Set to a UUID by default.
+   */
   id_: string = uuidv4();
@@ -59,6 +66,2 @@ embedding?: number[];
 
-  get nodeId(): string {
-    return this.id_;
-  }
-
   get sourceNode(): RelatedNodeInfo | undefined {
@@ -79,3 +82,3 @@ const relationship = this.relationships[NodeRelationship.SOURCE];
       throw new Error(
-        "Previous object must be a single RelatedNodeInfo object"
+        "Previous object must be a single RelatedNodeInfo object",
       );
@@ -112,3 +115,3 @@ }
       throw new Error(
-        "Child object must be a an array of RelatedNodeInfo objects"
+        "Child object must be a an array of RelatedNodeInfo objects",
       );
@@ -120,2 +123,4 @@ }
 
+  abstract generateHash(): string;
+
   getEmbedding(): number[] {
@@ -131,3 +136,3 @@ if (this.embedding === undefined) {
     return {
-      nodeId: this.nodeId,
+      nodeId: this.id_,
       metadata: this.metadata,
@@ -161,6 +166,23 @@ hash: this.hash,
     Object.assign(this, init);
+
+    if (new.target === TextNode) {
+      // Don't generate the hash repeatedly so only do it if this is
+      // constructing the derived class
+      this.hash = this.generateHash();
+    }
   }
 
+  /**
+   * Generate a hash of the text node.
+   * The ID is not part of the hash as it can change independent of content.
+   * @returns
+   */
   generateHash() {
-    throw new Error("Not implemented");
+    const hashFunction = crypto.createHash("sha256");
+    hashFunction.update(`type=${this.getType()}`);
+    hashFunction.update(
+      `startCharIdx=${this.startCharIdx} endCharIdx=${this.endCharIdx}`,
+    );
+    hashFunction.update(this.getContent(MetadataMode.ALL));
+    return hashFunction.digest("base64");
   }
@@ -200,2 +222,4 @@
     this.text = value;
+
+    this.hash = this.generateHash();
   }
@@ -226,2 +250,6 @@
     Object.assign(this, init);
+
+    if (new.target === IndexNode) {
+      this.hash = this.generateHash();
+    }
   }
@@ -241,2 +269,6 @@
     Object.assign(this, init);
+
+    if (new.target === Document) {
+      this.hash = this.generateHash();
+    }
   }
@@ -247,6 +279,2 @@
   }
-
-  get docId() {
-    return this.id_;
-  }
 }
@@ -253,0 +281,0 @@

src/storage/docStore/KVDocumentStore.ts

@@ -1,7 +0,7 @@
-import { BaseNode, Document, ObjectType, TextNode } from "../../Node";
+import _, * as lodash from "lodash";
+import { BaseNode, ObjectType } from "../../Node";
+import { DEFAULT_NAMESPACE } from "../constants";
+import { BaseKVStore } from "../kvStore/types";
 import { BaseDocumentStore, RefDocInfo } from "./types";
-import { BaseKVStore } from "../kvStore/types";
-import _, * as lodash from "lodash";
 import { docToJson, jsonToDoc } from "./utils";
-import { DEFAULT_NAMESPACE } from "../constants";
 
@@ -35,3 +35,3 @@ type DocMetaData = { docHash: string; refDocId?: string };
     docs: BaseNode[],
-    allowUpdate: boolean = true
+    allowUpdate: boolean = true,
   ): Promise<void> {
@@ -45,3 +45,3 @@ for (var idx = 0; idx < docs.length; idx++) {
         throw new Error(
-          `doc_id ${doc.id_} already exists. Set allow_update to True to overwrite.`
+          `doc_id ${doc.id_} already exists. Set allow_update to True to overwrite.`,
         );
@@ -56,6 +56,6 @@ }
         let refDocInfo = (await this.getRefDocInfo(doc.sourceNode.nodeId)) || {
-          docIds: [],
+          nodeIds: [],
           extraInfo: {},
         };
-        refDocInfo.docIds.push(doc.id_);
+        refDocInfo.nodeIds.push(doc.id_);
         if (_.isEmpty(refDocInfo.extraInfo)) {
@@ -67,3 +67,3 @@ refDocInfo.extraInfo = {};
           refDocInfo,
-          this.refDocCollection
+          this.refDocCollection,
         );
@@ -79,3 +79,3 @@ metadata.refDocId = doc.sourceNode.nodeId!;
     docId: string,
-    raiseError: boolean = true
+    raiseError: boolean = true,
   ): Promise<BaseNode | undefined> {
@@ -139,3 +139,3 @@ let json = await this.kvstore.get(docId, this.nodeCollection);
     raiseError: boolean = true,
-    removeRefDocNode: boolean = true
+    removeRefDocNode: boolean = true,
   ): Promise<void> {
@@ -156,3 +156,3 @@ if (removeRefDocNode) {
     refDocId: string,
-    raiseError: boolean = true
+    raiseError: boolean = true,
   ): Promise<void> {
@@ -168,3 +168,3 @@ let refDocInfo = await this.getRefDocInfo(refDocId);
 
-    for (let docId of refDocInfo.docIds) {
+    for (let docId of refDocInfo.nodeIds) {
       await this.deleteDocument(docId, false, false);
@@ -171,0 +171,0 @@ }

src/storage/docStore/types.ts

 import { BaseNode } from "../../Node";
-import { GenericFileSystem } from "../FileSystem";
 import {
+  DEFAULT_DOC_STORE_PERSIST_FILENAME,
   DEFAULT_PERSIST_DIR,
-  DEFAULT_DOC_STORE_PERSIST_FILENAME,
 } from "../constants";
+import { GenericFileSystem } from "../FileSystem";
 
@@ -11,3 +11,3 @@ const defaultPersistPath = `${DEFAULT_PERSIST_DIR}/${DEFAULT_DOC_STORE_PERSIST_FILENAME}`;
 export interface RefDocInfo {
-  docIds: string[];
+  nodeIds: string[];
   extraInfo: Record<string, any>;
@@ -20,3 +20,3 @@ }
     persistPath: string = defaultPersistPath,
-    fs?: GenericFileSystem
+    fs?: GenericFileSystem,
   ): void {
@@ -29,10 +29,10 @@ // Persist the docstore to a file.
 
-  abstract addDocuments(docs: BaseNode[], allowUpdate: boolean): void;
+  abstract addDocuments(docs: BaseNode[], allowUpdate: boolean): Promise<void>;
 
   abstract getDocument(
     docId: string,
-    raiseError: boolean
+    raiseError: boolean,
   ): Promise<BaseNode | undefined>;
 
-  abstract deleteDocument(docId: string, raiseError: boolean): void;
+  abstract deleteDocument(docId: string, raiseError: boolean): Promise<void>;
 
@@ -56,3 +56,3 @@ abstract documentExists(docId: string): Promise<boolean>;
     return Promise.all(
-      nodeIds.map((nodeId) => this.getNode(nodeId, raiseError))
+      nodeIds.map((nodeId) => this.getNode(nodeId, raiseError)),
     );
@@ -59,0 +59,0 @@ }

New alerts

License Policy Violation

License

This package is not allowed per your license policy. Review the package's license to ensure compliance.

Found 1 instance in 1 package

Fixed alerts

License Policy Violation

License

This package is not allowed per your license policy. Review the package's license to ensure compliance.

Found 1 instance in 1 package

Improved metrics

Total package byte prevSize

556416

increased by4.38%

Number of package files

71

increased by1.43%

Lines of code

13857

increased by3.37%

No dependency changes