@open-evals/rag - npm Package Compare versions

Comparing version 0.1.2 to 0.1.3

dist/index.d.ts

@@ -15,3 +15,3 @@ //#region src/types.d.ts
     documentId: string;
-  } & Record<string, unknown>;
+  };
 }
@@ -22,2 +22,7 @@ interface Splitter {
 type LengthFunction = (text: string) => number;
+interface BaseChunkOptions {
+  chunkSize?: number;
+  chunkOverlap?: number;
+  lengthFunction?: LengthFunction;
+}
 //#endregion
@@ -71,3 +76,9 @@ //#region src/splitters/base.d.ts
 //#endregion
-export { type Chunk, type Document, type LengthFunction, RecursiveCharacterSplitter, type Splitter };
+//#region src/splitters/markdown-splitter.d.ts
+declare class MarkdownSplitter extends RecursiveCharacterSplitter {
+  private static readonly MARKDOWN_SEPARATORS;
+  constructor(options?: BaseChunkOptions);
+}
+//#endregion
+export { type Chunk, type Document, type LengthFunction, MarkdownSplitter, RecursiveCharacterSplitter, type Splitter };
 //# sourceMappingURL=index.d.ts.map

dist/index.d.ts.map

@@ -1,1 +0,1 @@
-{"version":3,"file":"index.d.ts","names":[],"sources":["../src/types.ts","../src/splitters/base.ts","../src/splitters/recursive-character.ts"],"sourcesContent":[],"mappings":";UAAiB,QAAA;EAAjB,EAAA,EAAiB,MAAA;EAMjB,OAAiB,EAAA,MAAA;EAWjB,QAAiB,EAdL,MAcK,CAAA,MAAA,EAAA,OAAA,CAAA;;AACC,UAZD,KAAA,CAYC;YAAqB;SAAV,EAAA,MAAA;UAAkC,EAAA;SAAf,EAAA,MAAA;IAAA,GAAA,EAAA,MAAA;IAGhD,KAAY,EAAA,MAAA;;MAPN;;ACZgB,UDeL,QAAA,CCfK;EAAA,KAAA,CAAA,QAAA,EDgBJ,QChBI,CAAA,EDgBO,SChBP,CDgBiB,KChBjB,CAAA,GDgB0B,cChB1B,CDgByC,KChBzC,CAAA;;AAGqB,KDgB/B,cAAA,GChB+B,CAAA,IAAA,EAAA,MAAA,EAAA,GAAA,MAAA;;;ADL1B,uBCEK,YAAA,YAAwB,QDClC,CAAA;EAGZ,SAAiB,cAQX,ECXqB,cDWrB;EAGN,WAAiB,CAAA,QAAA,EAAA;IAAA,cAAA,CAAA,ECZ0B,cDY1B;;WACsB,KAAA,CAAA,QAAA,ECRZ,QDQY,CAAA,ECRD,cDQC,CCRc,KDQd,CAAA;;;;AAZtB,UEHA,iCAAA,CFWX;EAGN,SAAiB,CAAA,EAAA,MAAA;EAAA,YAAA,CAAA,EAAA,MAAA;YACC,CAAA,EAAA,MAAA,EAAA;eAAqB,CAAA,EAAA,OAAA,GAAA,OAAA,GAAA,KAAA;kBAAV,CAAA,EAAA,OAAA;gBAAkC,CAAA,EET5C,cFS4C;;AAAf,cEInC,0BAAA,SAAmC,YAAA,CFJA;EAGhD,iBAAY,SAAA;;;;ECnBZ,iBAAsB,gBAAA;EAAA,WAAA,CAAA,OAAA,CAAA,EC2BC,iCD3BD;QACK,QAAA,ECoCF,QDpCE,CAAA,ECoCS,cDpCT,CCoCwB,KDpCxB,CAAA;UAEgB,eAAA;UAKhB,WAAA;UAA0B,SAAA;UAAf,mBAAA;UARQ,kBAAA;EAAA,QAAA,uBAAA;;;;ECC9C,QAAiB,gBAAA;EAmBjB,QAAa,eAAA;EAAA,QAAA,oBAAA;UAOU,gBAAA;UAUE,mBAAA;UAA0B,aAAA;UAAf,YAAA;UAjBY,qBAAA;EAAA,QAAA,eAAA"}
+{"version":3,"file":"index.d.ts","names":[],"sources":["../src/types.ts","../src/splitters/base.ts","../src/splitters/recursive-character.ts","../src/splitters/markdown-splitter.ts"],"sourcesContent":[],"mappings":";UAAiB,QAAA;EAAjB,EAAA,EAAiB,MAAA;EAMjB,OAAiB,EAAA,MAAA;EAWjB,QAAiB,EAdL,MAcK,CAAA,MAAA,EAAA,OAAA,CAAA;;AACC,UAZD,KAAA,CAYC;YAAqB;SAAV,EAAA,MAAA;UAAkC,EAAA;SAAf,EAAA,MAAA;IAAA,GAAA,EAAA,MAAA;IAGhD,KAAY,EAAA,MAAA;IAEZ,UAAiB,EAAA,MAAA;;;UANA,QAAA;ECfjB,KAAsB,CAAA,QAAA,EDgBJ,QChBI,CAAA,EDgBO,SChBP,CDgBiB,KChBjB,CAAA,GDgB0B,cChB1B,CDgByC,KChBzC,CAAA;;AACK,KDkBf,cAAA,GClBe,CAAA,IAAA,EAAA,MAAA,EAAA,GAAA,MAAA;AAEgB,UDkB1B,gBAAA,CClB0B;WAKhB,CAAA,EAAA,MAAA;cAA0B,CAAA,EAAA,MAAA;gBAAf,CAAA,EDgBnB,cChBmB;;;;ADVrB,uBCEK,YAAA,YAAwB,QDClC,CAAA;EAGZ,SAAiB,cAAA,ECHU,cDGV;EAWjB,WAAiB,CAAA,QAAA,EAAA;IAAA,cAAA,CAAA,ECZ0B,cDY1B;;WACsB,KAAA,CAAA,QAAA,ECRZ,QDQY,CAAA,ECRD,cDQC,CCRc,KDQd,CAAA;;;;AAZtB,UEHA,iCAAA,CFGA;EAWjB,SAAiB,CAAA,EAAA,MAAA;EAAA,YAAA,CAAA,EAAA,MAAA;YACC,CAAA,EAAA,MAAA,EAAA;eAAqB,CAAA,EAAA,OAAA,GAAA,OAAA,GAAA,KAAA;kBAAV,CAAA,EAAA,OAAA;gBAAkC,CAAA,EET5C,cFS4C;;AAAf,cEInC,0BAAA,SAAmC,YAAA,CFJA;EAGhD,iBAAY,SAAA;EAEZ,iBAAiB,YAAA;;;;ECrBjB,WAAsB,CAAA,OAAA,CAAA,EC2BC,iCD3BD;EAAA,KAAA,CAAA,QAAA,ECqCG,QDrCH,CAAA,ECqCc,cDrCd,CCqC6B,KDrC7B,CAAA;UACK,eAAA;UAEgB,WAAA;UAKhB,SAAA;UAA0B,mBAAA;UAAf,kBAAA;UARQ,uBAAA;EAAA,QAAA,gBAAA;;;;ECC9C,QAAiB,eAAA;EAmBjB,QAAa,oBAAA;EAAA,QAAA,gBAAA;UAOU,mBAAA;UAUE,aAAA;UAA0B,YAAA;UAAf,qBAAA;UAjBY,eAAA;EAAA,QAAA,oBAAA;;;;AFhB/B,cGHJ,gBAAA,SAAyB,0BAAA,CHGrB;EAWjB,wBAAiB,mBAAA;EAAA,WAAA,CAAA,OAAA,CAAA,EGMM,gBHNN"}

dist/index.js

@@ -20,3 +20,3 @@ //#region src/splitters/base.ts
 		this.chunkSize = options.chunkSize ?? 1e3;
-		this.chunkOverlap = options.chunkOverlap ?? 200;
+		this.chunkOverlap = options.chunkOverlap ?? 0;
 		this.separators = options.separators ?? [
@@ -208,3 +208,26 @@ "\n\n",
 //#endregion
-export { RecursiveCharacterSplitter };
+//#region src/splitters/markdown-splitter.ts
+var MarkdownSplitter = class MarkdownSplitter extends RecursiveCharacterSplitter {
+	static MARKDOWN_SEPARATORS = [
+		"\n#{1,6} ",
+		"```\n",
+		"\n\\*\\*\\*+\n",
+		"\n---+\n",
+		"\n___+\n",
+		"\n\n",
+		"\n",
+		" ",
+		""
+	];
+	constructor(options = {}) {
+		super({
+			...options,
+			separators: MarkdownSplitter.MARKDOWN_SEPARATORS,
+			isSeparatorRegex: true
+		});
+	}
+};
+
+//#endregion
+export { MarkdownSplitter, RecursiveCharacterSplitter };
 //# sourceMappingURL=index.js.map

dist/index.js.map

@@ -1,1 +0,1 @@
-{"version":3,"file":"index.js","names":["chunks: ChunkPosition[]","result: string[]","separator","text","finalChunks: string[]","goodSplits: string[]","chunks: string[]","currentChunk: string[]"],"sources":["../src/splitters/base.ts","../src/splitters/recursive-character.ts"],"sourcesContent":["import type { Splitter, Document, Chunk, LengthFunction } from '../types'\n\nexport abstract class BaseSplitter implements Splitter {\n  readonly lengthFunction: LengthFunction\n\n  constructor(options?: { lengthFunction?: LengthFunction }) {\n    this.lengthFunction =\n      options?.lengthFunction || ((text: string) => text.length)\n  }\n\n  abstract split(document: Document): AsyncGenerator<Chunk>\n}\n","import { BaseSplitter } from './base'\nimport type { Document, Chunk, LengthFunction } from '../types'\n\nexport interface RecursiveCharacterSplitterOptions {\n  chunkSize?: number\n  chunkOverlap?: number\n  separators?: string[]\n  keepSeparator?: boolean | 'start' | 'end'\n  isSeparatorRegex?: boolean\n  lengthFunction?: LengthFunction\n}\n\ninterface SeparatorResult {\n  separator: string\n  remainingSeparators: string[]\n}\n\ninterface ChunkPosition {\n  text: string\n  start: number\n}\n\nexport class RecursiveCharacterSplitter extends BaseSplitter {\n  private readonly chunkSize: number\n  private readonly chunkOverlap: number\n  private readonly separators: string[]\n  private readonly keepSeparator: boolean | 'start' | 'end'\n  private readonly isSeparatorRegex: boolean\n\n  constructor(options: RecursiveCharacterSplitterOptions = {}) {\n    super({ lengthFunction: options.lengthFunction })\n    this.chunkSize = options.chunkSize ?? 1000\n    this.chunkOverlap = options.chunkOverlap ?? 200\n    this.separators = options.separators ?? ['\\n\\n', '\\n', ' ', '']\n    this.keepSeparator = options.keepSeparator ?? true\n    this.isSeparatorRegex = options.isSeparatorRegex ?? false\n    this.validateOptions()\n  }\n\n  async *split(document: Document): AsyncGenerator<Chunk> {\n    const chunksWithPositions = this.splitText(document.content)\n\n    for (let i = 0; i < chunksWithPositions.length; i++) {\n      const chunk = chunksWithPositions[i]\n      yield this.createChunk(document, chunk, i)\n    }\n  }\n\n  private validateOptions(): void {\n    if (this.chunkSize <= 0) {\n      throw new Error('chunkSize must be greater than 0')\n    }\n    if (this.chunkOverlap < 0) {\n      throw new Error('chunkOverlap must be greater than or equal to 0')\n    }\n    if (this.chunkOverlap >= this.chunkSize) {\n      throw new Error('chunkOverlap must be less than chunkSize')\n    }\n  }\n\n  private createChunk(\n    document: Document,\n    chunk: ChunkPosition,\n    index: number\n  ): Chunk {\n    const start = chunk.start\n    const end = chunk.start + chunk.text.length\n\n    return {\n      id: `${document.id}-chunk-${index}`,\n      content: chunk.text,\n      metadata: {\n        start,\n        end,\n        index,\n        documentId: document.id,\n      },\n    }\n  }\n\n  private splitText(text: string): ChunkPosition[] {\n    const splits = this.splitTextRecursive(text, this.separators)\n    return this.trackChunkPositions(splits, text)\n  }\n\n  private trackChunkPositions(splits: string[], text: string): ChunkPosition[] {\n    const chunks: ChunkPosition[] = []\n    let currentPosition = 0\n\n    for (const split of splits) {\n      const index = text.indexOf(split, currentPosition)\n      if (index !== -1) {\n        chunks.push({ text: split, start: index })\n        currentPosition = index + split.length\n      } else {\n        chunks.push({ text: split, start: currentPosition })\n        currentPosition += split.length\n      }\n    }\n\n    return chunks\n  }\n\n  private splitTextRecursive(text: string, separators: string[]): string[] {\n    const { separator, remainingSeparators } = this.findApplicableSeparator(\n      text,\n      separators\n    )\n\n    const splits = this.splitBySeparator(text, separator)\n\n    return this.processSplits(splits, separator, remainingSeparators)\n  }\n\n  private findApplicableSeparator(\n    text: string,\n    separators: string[]\n  ): SeparatorResult {\n    for (let i = 0; i < separators.length; i++) {\n      const separator = separators[i]\n\n      if (separator === '') {\n        return {\n          separator,\n          remainingSeparators: separators.slice(i + 1),\n        }\n      }\n\n      const exists = this.isSeparatorRegex\n        ? new RegExp(separator).test(text)\n        : text.includes(separator)\n\n      if (exists) {\n        return {\n          separator,\n          remainingSeparators: separators.slice(i + 1),\n        }\n      }\n    }\n\n    return {\n      separator: separators[separators.length - 1],\n      remainingSeparators: [],\n    }\n  }\n\n  private splitBySeparator(text: string, separator: string): string[] {\n    if (separator === '') {\n      return text.split('')\n    }\n    return this.splitWithSeparator(text, separator)\n  }\n\n  private splitWithSeparator(text: string, separator: string): string[] {\n    if (!separator) {\n      return text.split('').filter((s) => s !== '')\n    }\n\n    const sepPattern = this.isSeparatorRegex\n      ? separator\n      : separator.replace(/[.*+?^${}()|[\\]\\\\]/g, '\\\\$&')\n\n    if (!this.keepSeparator) {\n      return text.split(new RegExp(sepPattern)).filter((s) => s !== '')\n    }\n\n    if (!text) {\n      return []\n    }\n\n    const splits = text.split(new RegExp(`(${sepPattern})`))\n    const result: string[] = []\n\n    if (this.keepSeparator === 'end' || this.keepSeparator === true) {\n      for (let i = 0; i < splits.length - 1; i += 2) {\n        if (i + 1 < splits.length) {\n          const chunk = splits[i] + (splits[i + 1] || '')\n          if (chunk) result.push(chunk)\n        }\n      }\n      if (splits.length % 2 === 1 && splits[splits.length - 1]) {\n        result.push(splits[splits.length - 1])\n      }\n    } else if (this.keepSeparator === 'start') {\n      if (splits[0]) result.push(splits[0])\n\n      for (let i = 1; i < splits.length - 1; i += 2) {\n        const separator = splits[i]\n        const text = splits[i + 1]\n        if (separator && text) {\n          result.push(separator + text)\n        } else if (separator) {\n          result.push(separator)\n        }\n      }\n    }\n\n    return result.filter((s) => s !== '')\n  }\n\n  private processSplits(\n    splits: string[],\n    separator: string,\n    remainingSeparators: string[]\n  ): string[] {\n    const finalChunks: string[] = []\n    const goodSplits: string[] = []\n\n    for (const split of splits) {\n      if (this.isUnderChunkSize(split)) {\n        goodSplits.push(split)\n      } else {\n        this.flushGoodSplits(goodSplits, separator, finalChunks)\n        this.handleOversizedSplit(split, remainingSeparators, finalChunks)\n      }\n    }\n\n    this.flushGoodSplits(goodSplits, separator, finalChunks)\n\n    return finalChunks\n  }\n\n  private isUnderChunkSize(text: string): boolean {\n    return this.lengthFunction(text) < this.chunkSize\n  }\n\n  private flushGoodSplits(\n    goodSplits: string[],\n    separator: string,\n    output: string[]\n  ): void {\n    if (goodSplits.length === 0) return\n\n    const mergedSeparator = this.keepSeparator\n      ? ''\n      : (separator === '' ? '' : ' ')\n    const merged = this.mergeSmallSplits(goodSplits, mergedSeparator)\n    output.push(...merged)\n    goodSplits.length = 0\n  }\n\n  private handleOversizedSplit(\n    split: string,\n    remainingSeparators: string[],\n    output: string[]\n  ): void {\n    if (remainingSeparators.length > 0) {\n      const recursiveSplits = this.splitTextRecursive(\n        split,\n        remainingSeparators\n      )\n      output.push(...recursiveSplits)\n    } else if (split !== '') {\n      output.push(split)\n    }\n  }\n\n  private mergeSmallSplits(splits: string[], separator: string): string[] {\n    const separatorLength = this.lengthFunction(separator)\n    const chunks: string[] = []\n    const currentChunk: string[] = []\n    let currentLength = 0\n\n    for (const split of splits) {\n      const splitLength = this.lengthFunction(split)\n\n      if (\n        this.shouldFinalizeChunk(\n          currentChunk,\n          currentLength,\n          splitLength,\n          separatorLength\n        )\n      ) {\n        chunks.push(this.finalizeChunk(currentChunk, separator))\n        currentLength = this.applyOverlap(\n          currentChunk,\n          currentLength,\n          splitLength,\n          separatorLength\n        )\n      }\n\n      this.addSplitToChunk(currentChunk, split)\n      currentLength += this.calculateAddedLength(\n        currentChunk.length,\n        splitLength,\n        separatorLength\n      )\n    }\n\n    if (currentChunk.length > 0) {\n      chunks.push(this.finalizeChunk(currentChunk, separator))\n    }\n\n    return chunks\n  }\n\n  private shouldFinalizeChunk(\n    currentChunk: string[],\n    currentLength: number,\n    splitLength: number,\n    separatorLength: number\n  ): boolean {\n    if (currentChunk.length === 0) return false\n\n    const potentialLength =\n      currentLength +\n      splitLength +\n      (currentChunk.length > 0 ? separatorLength : 0)\n\n    return potentialLength > this.chunkSize\n  }\n\n  private finalizeChunk(chunk: string[], separator: string): string {\n    return chunk.join(separator)\n  }\n\n  private applyOverlap(\n    currentChunk: string[],\n    currentLength: number,\n    nextSplitLength: number,\n    separatorLength: number\n  ): number {\n    let length = currentLength\n\n    while (\n      this.shouldRemoveFromChunk(\n        currentChunk,\n        length,\n        nextSplitLength,\n        separatorLength\n      )\n    ) {\n      const removed = currentChunk.shift()!\n      length -= this.lengthFunction(removed)\n      if (currentChunk.length > 0) {\n        length -= separatorLength\n      }\n    }\n\n    return length\n  }\n\n  private shouldRemoveFromChunk(\n    currentChunk: string[],\n    currentLength: number,\n    nextSplitLength: number,\n    separatorLength: number\n  ): boolean {\n    if (currentChunk.length === 0) return false\n\n    const exceedsOverlap = currentLength > this.chunkOverlap\n    const wouldExceedWithNext =\n      currentLength + nextSplitLength + separatorLength > this.chunkSize &&\n      currentLength > 0\n\n    return exceedsOverlap || wouldExceedWithNext\n  }\n\n  private addSplitToChunk(chunk: string[], split: string): void {\n    chunk.push(split)\n  }\n\n  private calculateAddedLength(\n    chunkLength: number,\n    splitLength: number,\n    separatorLength: number\n  ): number {\n    return splitLength + (chunkLength > 1 ? separatorLength : 0)\n  }\n}\n"],"mappings":";AAEA,IAAsB,eAAtB,MAAuD;CACrD,AAAS;CAET,YAAY,SAA+C;AACzD,OAAK,iBACH,SAAS,oBAAoB,SAAiB,KAAK;;;;;;ACezD,IAAa,6BAAb,cAAgD,aAAa;CAC3D,AAAiB;CACjB,AAAiB;CACjB,AAAiB;CACjB,AAAiB;CACjB,AAAiB;CAEjB,YAAY,UAA6C,EAAE,EAAE;AAC3D,QAAM,EAAE,gBAAgB,QAAQ,gBAAgB,CAAC;AACjD,OAAK,YAAY,QAAQ,aAAa;AACtC,OAAK,eAAe,QAAQ,gBAAgB;AAC5C,OAAK,aAAa,QAAQ,cAAc;GAAC;GAAQ;GAAM;GAAK;GAAG;AAC/D,OAAK,gBAAgB,QAAQ,iBAAiB;AAC9C,OAAK,mBAAmB,QAAQ,oBAAoB;AACpD,OAAK,iBAAiB;;CAGxB,OAAO,MAAM,UAA2C;EACtD,MAAM,sBAAsB,KAAK,UAAU,SAAS,QAAQ;AAE5D,OAAK,IAAI,IAAI,GAAG,IAAI,oBAAoB,QAAQ,KAAK;GACnD,MAAM,QAAQ,oBAAoB;AAClC,SAAM,KAAK,YAAY,UAAU,OAAO,EAAE;;;CAI9C,AAAQ,kBAAwB;AAC9B,MAAI,KAAK,aAAa,EACpB,OAAM,IAAI,MAAM,mCAAmC;AAErD,MAAI,KAAK,eAAe,EACtB,OAAM,IAAI,MAAM,kDAAkD;AAEpE,MAAI,KAAK,gBAAgB,KAAK,UAC5B,OAAM,IAAI,MAAM,2CAA2C;;CAI/D,AAAQ,YACN,UACA,OACA,OACO;EACP,MAAM,QAAQ,MAAM;EACpB,MAAM,MAAM,MAAM,QAAQ,MAAM,KAAK;AAErC,SAAO;GACL,IAAI,GAAG,SAAS,GAAG,SAAS;GAC5B,SAAS,MAAM;GACf,UAAU;IACR;IACA;IACA;IACA,YAAY,SAAS;IACtB;GACF;;CAGH,AAAQ,UAAU,MAA+B;EAC/C,MAAM,SAAS,KAAK,mBAAmB,MAAM,KAAK,WAAW;AAC7D,SAAO,KAAK,oBAAoB,QAAQ,KAAK;;CAG/C,AAAQ,oBAAoB,QAAkB,MAA+B;EAC3E,MAAMA,SAA0B,EAAE;EAClC,IAAI,kBAAkB;AAEtB,OAAK,MAAM,SAAS,QAAQ;GAC1B,MAAM,QAAQ,KAAK,QAAQ,OAAO,gBAAgB;AAClD,OAAI,UAAU,IAAI;AAChB,WAAO,KAAK;KAAE,MAAM;KAAO,OAAO;KAAO,CAAC;AAC1C,sBAAkB,QAAQ,MAAM;UAC3B;AACL,WAAO,KAAK;KAAE,MAAM;KAAO,OAAO;KAAiB,CAAC;AACpD,uBAAmB,MAAM;;;AAI7B,SAAO;;CAGT,AAAQ,mBAAmB,MAAc,YAAgC;EACvE,MAAM,EAAE,WAAW,wBAAwB,KAAK,wBAC9C,MACA,WACD;EAED,MAAM,SAAS,KAAK,iBAAiB,MAAM,UAAU;AAErD,SAAO,KAAK,cAAc,QAAQ,WAAW,oBAAoB;;CAGnE,AAAQ,wBACN,MACA,YACiB;AACjB,OAAK,IAAI,IAAI,GAAG,IAAI,WAAW,QAAQ,KAAK;GAC1C,MAAM,YAAY,WAAW;AAE7B,OAAI,cAAc,GAChB,QAAO;IACL;IACA,qBAAqB,WAAW,MAAM,IAAI,EAAE;IAC7C;AAOH,OAJe,KAAK,mBAChB,IAAI,OAAO,UAAU,CAAC,KAAK,KAAK,GAChC,KAAK,SAAS,UAAU,CAG1B,QAAO;IACL;IACA,qBAAqB,WAAW,MAAM,IAAI,EAAE;IAC7C;;AAIL,SAAO;GACL,WAAW,WAAW,WAAW,SAAS;GAC1C,qBAAqB,EAAE;GACxB;;CAGH,AAAQ,iBAAiB,MAAc,WAA6B;AAClE,MAAI,cAAc,GAChB,QAAO,KAAK,MAAM,GAAG;AAEvB,SAAO,KAAK,mBAAmB,MAAM,UAAU;;CAGjD,AAAQ,mBAAmB,MAAc,WAA6B;AACpE,MAAI,CAAC,UACH,QAAO,KAAK,MAAM,GAAG,CAAC,QAAQ,MAAM,MAAM,GAAG;EAG/C,MAAM,aAAa,KAAK,mBACpB,YACA,UAAU,QAAQ,uBAAuB,OAAO;AAEpD,MAAI,CAAC,KAAK,cACR,QAAO,KAAK,MAAM,IAAI,OAAO,WAAW,CAAC,CAAC,QAAQ,MAAM,MAAM,GAAG;AAGnE,MAAI,CAAC,KACH,QAAO,EAAE;EAGX,MAAM,SAAS,KAAK,sBAAM,IAAI,OAAO,IAAI,WAAW,GAAG,CAAC;EACxD,MAAMC,SAAmB,EAAE;AAE3B,MAAI,KAAK,kBAAkB,SAAS,KAAK,kBAAkB,MAAM;AAC/D,QAAK,IAAI,IAAI,GAAG,IAAI,OAAO,SAAS,GAAG,KAAK,EAC1C,KAAI,IAAI,IAAI,OAAO,QAAQ;IACzB,MAAM,QAAQ,OAAO,MAAM,OAAO,IAAI,MAAM;AAC5C,QAAI,MAAO,QAAO,KAAK,MAAM;;AAGjC,OAAI,OAAO,SAAS,MAAM,KAAK,OAAO,OAAO,SAAS,GACpD,QAAO,KAAK,OAAO,OAAO,SAAS,GAAG;aAE/B,KAAK,kBAAkB,SAAS;AACzC,OAAI,OAAO,GAAI,QAAO,KAAK,OAAO,GAAG;AAErC,QAAK,IAAI,IAAI,GAAG,IAAI,OAAO,SAAS,GAAG,KAAK,GAAG;IAC7C,MAAMC,cAAY,OAAO;IACzB,MAAMC,SAAO,OAAO,IAAI;AACxB,QAAID,eAAaC,OACf,QAAO,KAAKD,cAAYC,OAAK;aACpBD,YACT,QAAO,KAAKA,YAAU;;;AAK5B,SAAO,OAAO,QAAQ,MAAM,MAAM,GAAG;;CAGvC,AAAQ,cACN,QACA,WACA,qBACU;EACV,MAAME,cAAwB,EAAE;EAChC,MAAMC,aAAuB,EAAE;AAE/B,OAAK,MAAM,SAAS,OAClB,KAAI,KAAK,iBAAiB,MAAM,CAC9B,YAAW,KAAK,MAAM;OACjB;AACL,QAAK,gBAAgB,YAAY,WAAW,YAAY;AACxD,QAAK,qBAAqB,OAAO,qBAAqB,YAAY;;AAItE,OAAK,gBAAgB,YAAY,WAAW,YAAY;AAExD,SAAO;;CAGT,AAAQ,iBAAiB,MAAuB;AAC9C,SAAO,KAAK,eAAe,KAAK,GAAG,KAAK;;CAG1C,AAAQ,gBACN,YACA,WACA,QACM;AACN,MAAI,WAAW,WAAW,EAAG;EAE7B,MAAM,kBAAkB,KAAK,gBACzB,KACC,cAAc,KAAK,KAAK;EAC7B,MAAM,SAAS,KAAK,iBAAiB,YAAY,gBAAgB;AACjE,SAAO,KAAK,GAAG,OAAO;AACtB,aAAW,SAAS;;CAGtB,AAAQ,qBACN,OACA,qBACA,QACM;AACN,MAAI,oBAAoB,SAAS,GAAG;GAClC,MAAM,kBAAkB,KAAK,mBAC3B,OACA,oBACD;AACD,UAAO,KAAK,GAAG,gBAAgB;aACtB,UAAU,GACnB,QAAO,KAAK,MAAM;;CAItB,AAAQ,iBAAiB,QAAkB,WAA6B;EACtE,MAAM,kBAAkB,KAAK,eAAe,UAAU;EACtD,MAAMC,SAAmB,EAAE;EAC3B,MAAMC,eAAyB,EAAE;EACjC,IAAI,gBAAgB;AAEpB,OAAK,MAAM,SAAS,QAAQ;GAC1B,MAAM,cAAc,KAAK,eAAe,MAAM;AAE9C,OACE,KAAK,oBACH,cACA,eACA,aACA,gBACD,EACD;AACA,WAAO,KAAK,KAAK,cAAc,cAAc,UAAU,CAAC;AACxD,oBAAgB,KAAK,aACnB,cACA,eACA,aACA,gBACD;;AAGH,QAAK,gBAAgB,cAAc,MAAM;AACzC,oBAAiB,KAAK,qBACpB,aAAa,QACb,aACA,gBACD;;AAGH,MAAI,aAAa,SAAS,EACxB,QAAO,KAAK,KAAK,cAAc,cAAc,UAAU,CAAC;AAG1D,SAAO;;CAGT,AAAQ,oBACN,cACA,eACA,aACA,iBACS;AACT,MAAI,aAAa,WAAW,EAAG,QAAO;AAOtC,SAJE,gBACA,eACC,aAAa,SAAS,IAAI,kBAAkB,KAEtB,KAAK;;CAGhC,AAAQ,cAAc,OAAiB,WAA2B;AAChE,SAAO,MAAM,KAAK,UAAU;;CAG9B,AAAQ,aACN,cACA,eACA,iBACA,iBACQ;EACR,IAAI,SAAS;AAEb,SACE,KAAK,sBACH,cACA,QACA,iBACA,gBACD,EACD;GACA,MAAM,UAAU,aAAa,OAAO;AACpC,aAAU,KAAK,eAAe,QAAQ;AACtC,OAAI,aAAa,SAAS,EACxB,WAAU;;AAId,SAAO;;CAGT,AAAQ,sBACN,cACA,eACA,iBACA,iBACS;AACT,MAAI,aAAa,WAAW,EAAG,QAAO;EAEtC,MAAM,iBAAiB,gBAAgB,KAAK;EAC5C,MAAM,sBACJ,gBAAgB,kBAAkB,kBAAkB,KAAK,aACzD,gBAAgB;AAElB,SAAO,kBAAkB;;CAG3B,AAAQ,gBAAgB,OAAiB,OAAqB;AAC5D,QAAM,KAAK,MAAM;;CAGnB,AAAQ,qBACN,aACA,aACA,iBACQ;AACR,SAAO,eAAe,cAAc,IAAI,kBAAkB"}
+{"version":3,"file":"index.js","names":["chunks: ChunkPosition[]","result: string[]","separator","text","finalChunks: string[]","goodSplits: string[]","chunks: string[]","currentChunk: string[]"],"sources":["../src/splitters/base.ts","../src/splitters/recursive-character.ts","../src/splitters/markdown-splitter.ts"],"sourcesContent":["import type { Splitter, Document, Chunk, LengthFunction } from '../types'\n\nexport abstract class BaseSplitter implements Splitter {\n  readonly lengthFunction: LengthFunction\n\n  constructor(options?: { lengthFunction?: LengthFunction }) {\n    this.lengthFunction =\n      options?.lengthFunction || ((text: string) => text.length)\n  }\n\n  abstract split(document: Document): AsyncGenerator<Chunk>\n}\n","import { BaseSplitter } from './base'\nimport type { Document, Chunk, LengthFunction } from '../types'\n\nexport interface RecursiveCharacterSplitterOptions {\n  chunkSize?: number\n  chunkOverlap?: number\n  separators?: string[]\n  keepSeparator?: boolean | 'start' | 'end'\n  isSeparatorRegex?: boolean\n  lengthFunction?: LengthFunction\n}\n\ninterface SeparatorResult {\n  separator: string\n  remainingSeparators: string[]\n}\n\ninterface ChunkPosition {\n  text: string\n  start: number\n}\n\nexport class RecursiveCharacterSplitter extends BaseSplitter {\n  private readonly chunkSize: number\n  private readonly chunkOverlap: number\n  private readonly separators: string[]\n  private readonly keepSeparator: boolean | 'start' | 'end'\n  private readonly isSeparatorRegex: boolean\n\n  constructor(options: RecursiveCharacterSplitterOptions = {}) {\n    super({ lengthFunction: options.lengthFunction })\n    this.chunkSize = options.chunkSize ?? 1000\n    this.chunkOverlap = options.chunkOverlap ?? 0\n    this.separators = options.separators ?? ['\\n\\n', '\\n', ' ', '']\n    this.keepSeparator = options.keepSeparator ?? true\n    this.isSeparatorRegex = options.isSeparatorRegex ?? false\n    this.validateOptions()\n  }\n\n  async *split(document: Document): AsyncGenerator<Chunk> {\n    const chunksWithPositions = this.splitText(document.content)\n\n    for (let i = 0; i < chunksWithPositions.length; i++) {\n      const chunk = chunksWithPositions[i]\n      yield this.createChunk(document, chunk, i)\n    }\n  }\n\n  private validateOptions(): void {\n    if (this.chunkSize <= 0) {\n      throw new Error('chunkSize must be greater than 0')\n    }\n    if (this.chunkOverlap < 0) {\n      throw new Error('chunkOverlap must be greater than or equal to 0')\n    }\n    if (this.chunkOverlap >= this.chunkSize) {\n      throw new Error('chunkOverlap must be less than chunkSize')\n    }\n  }\n\n  private createChunk(\n    document: Document,\n    chunk: ChunkPosition,\n    index: number\n  ): Chunk {\n    const start = chunk.start\n    const end = chunk.start + chunk.text.length\n\n    return {\n      id: `${document.id}-chunk-${index}`,\n      content: chunk.text,\n      metadata: {\n        start,\n        end,\n        index,\n        documentId: document.id,\n      },\n    }\n  }\n\n  private splitText(text: string): ChunkPosition[] {\n    const splits = this.splitTextRecursive(text, this.separators)\n    return this.trackChunkPositions(splits, text)\n  }\n\n  private trackChunkPositions(splits: string[], text: string): ChunkPosition[] {\n    const chunks: ChunkPosition[] = []\n    let currentPosition = 0\n\n    for (const split of splits) {\n      const index = text.indexOf(split, currentPosition)\n      if (index !== -1) {\n        chunks.push({ text: split, start: index })\n        currentPosition = index + split.length\n      } else {\n        chunks.push({ text: split, start: currentPosition })\n        currentPosition += split.length\n      }\n    }\n\n    return chunks\n  }\n\n  private splitTextRecursive(text: string, separators: string[]): string[] {\n    const { separator, remainingSeparators } = this.findApplicableSeparator(\n      text,\n      separators\n    )\n\n    const splits = this.splitBySeparator(text, separator)\n\n    return this.processSplits(splits, separator, remainingSeparators)\n  }\n\n  private findApplicableSeparator(\n    text: string,\n    separators: string[]\n  ): SeparatorResult {\n    for (let i = 0; i < separators.length; i++) {\n      const separator = separators[i]\n\n      if (separator === '') {\n        return {\n          separator,\n          remainingSeparators: separators.slice(i + 1),\n        }\n      }\n\n      const exists = this.isSeparatorRegex\n        ? new RegExp(separator).test(text)\n        : text.includes(separator)\n\n      if (exists) {\n        return {\n          separator,\n          remainingSeparators: separators.slice(i + 1),\n        }\n      }\n    }\n\n    return {\n      separator: separators[separators.length - 1],\n      remainingSeparators: [],\n    }\n  }\n\n  private splitBySeparator(text: string, separator: string): string[] {\n    if (separator === '') {\n      return text.split('')\n    }\n    return this.splitWithSeparator(text, separator)\n  }\n\n  private splitWithSeparator(text: string, separator: string): string[] {\n    if (!separator) {\n      return text.split('').filter((s) => s !== '')\n    }\n\n    const sepPattern = this.isSeparatorRegex\n      ? separator\n      : separator.replace(/[.*+?^${}()|[\\]\\\\]/g, '\\\\$&')\n\n    if (!this.keepSeparator) {\n      return text.split(new RegExp(sepPattern)).filter((s) => s !== '')\n    }\n\n    if (!text) {\n      return []\n    }\n\n    const splits = text.split(new RegExp(`(${sepPattern})`))\n    const result: string[] = []\n\n    if (this.keepSeparator === 'end' || this.keepSeparator === true) {\n      for (let i = 0; i < splits.length - 1; i += 2) {\n        if (i + 1 < splits.length) {\n          const chunk = splits[i] + (splits[i + 1] || '')\n          if (chunk) result.push(chunk)\n        }\n      }\n      if (splits.length % 2 === 1 && splits[splits.length - 1]) {\n        result.push(splits[splits.length - 1])\n      }\n    } else if (this.keepSeparator === 'start') {\n      if (splits[0]) result.push(splits[0])\n\n      for (let i = 1; i < splits.length - 1; i += 2) {\n        const separator = splits[i]\n        const text = splits[i + 1]\n        if (separator && text) {\n          result.push(separator + text)\n        } else if (separator) {\n          result.push(separator)\n        }\n      }\n    }\n\n    return result.filter((s) => s !== '')\n  }\n\n  private processSplits(\n    splits: string[],\n    separator: string,\n    remainingSeparators: string[]\n  ): string[] {\n    const finalChunks: string[] = []\n    const goodSplits: string[] = []\n\n    for (const split of splits) {\n      if (this.isUnderChunkSize(split)) {\n        goodSplits.push(split)\n      } else {\n        this.flushGoodSplits(goodSplits, separator, finalChunks)\n        this.handleOversizedSplit(split, remainingSeparators, finalChunks)\n      }\n    }\n\n    this.flushGoodSplits(goodSplits, separator, finalChunks)\n\n    return finalChunks\n  }\n\n  private isUnderChunkSize(text: string): boolean {\n    return this.lengthFunction(text) < this.chunkSize\n  }\n\n  private flushGoodSplits(\n    goodSplits: string[],\n    separator: string,\n    output: string[]\n  ): void {\n    if (goodSplits.length === 0) return\n\n    const mergedSeparator = this.keepSeparator\n      ? ''\n      : separator === ''\n      ? ''\n      : ' '\n    const merged = this.mergeSmallSplits(goodSplits, mergedSeparator)\n    output.push(...merged)\n    goodSplits.length = 0\n  }\n\n  private handleOversizedSplit(\n    split: string,\n    remainingSeparators: string[],\n    output: string[]\n  ): void {\n    if (remainingSeparators.length > 0) {\n      const recursiveSplits = this.splitTextRecursive(\n        split,\n        remainingSeparators\n      )\n      output.push(...recursiveSplits)\n    } else if (split !== '') {\n      output.push(split)\n    }\n  }\n\n  private mergeSmallSplits(splits: string[], separator: string): string[] {\n    const separatorLength = this.lengthFunction(separator)\n    const chunks: string[] = []\n    const currentChunk: string[] = []\n    let currentLength = 0\n\n    for (const split of splits) {\n      const splitLength = this.lengthFunction(split)\n\n      if (\n        this.shouldFinalizeChunk(\n          currentChunk,\n          currentLength,\n          splitLength,\n          separatorLength\n        )\n      ) {\n        chunks.push(this.finalizeChunk(currentChunk, separator))\n        currentLength = this.applyOverlap(\n          currentChunk,\n          currentLength,\n          splitLength,\n          separatorLength\n        )\n      }\n\n      this.addSplitToChunk(currentChunk, split)\n      currentLength += this.calculateAddedLength(\n        currentChunk.length,\n        splitLength,\n        separatorLength\n      )\n    }\n\n    if (currentChunk.length > 0) {\n      chunks.push(this.finalizeChunk(currentChunk, separator))\n    }\n\n    return chunks\n  }\n\n  private shouldFinalizeChunk(\n    currentChunk: string[],\n    currentLength: number,\n    splitLength: number,\n    separatorLength: number\n  ): boolean {\n    if (currentChunk.length === 0) return false\n\n    const potentialLength =\n      currentLength +\n      splitLength +\n      (currentChunk.length > 0 ? separatorLength : 0)\n\n    return potentialLength > this.chunkSize\n  }\n\n  private finalizeChunk(chunk: string[], separator: string): string {\n    return chunk.join(separator)\n  }\n\n  private applyOverlap(\n    currentChunk: string[],\n    currentLength: number,\n    nextSplitLength: number,\n    separatorLength: number\n  ): number {\n    let length = currentLength\n\n    while (\n      this.shouldRemoveFromChunk(\n        currentChunk,\n        length,\n        nextSplitLength,\n        separatorLength\n      )\n    ) {\n      const removed = currentChunk.shift()!\n      length -= this.lengthFunction(removed)\n      if (currentChunk.length > 0) {\n        length -= separatorLength\n      }\n    }\n\n    return length\n  }\n\n  private shouldRemoveFromChunk(\n    currentChunk: string[],\n    currentLength: number,\n    nextSplitLength: number,\n    separatorLength: number\n  ): boolean {\n    if (currentChunk.length === 0) return false\n\n    const exceedsOverlap = currentLength > this.chunkOverlap\n    const wouldExceedWithNext =\n      currentLength + nextSplitLength + separatorLength > this.chunkSize &&\n      currentLength > 0\n\n    return exceedsOverlap || wouldExceedWithNext\n  }\n\n  private addSplitToChunk(chunk: string[], split: string): void {\n    chunk.push(split)\n  }\n\n  private calculateAddedLength(\n    chunkLength: number,\n    splitLength: number,\n    separatorLength: number\n  ): number {\n    return splitLength + (chunkLength > 1 ? separatorLength : 0)\n  }\n}\n","import type { BaseChunkOptions } from '../types'\nimport { RecursiveCharacterSplitter } from './recursive-character'\n\nexport class MarkdownSplitter extends RecursiveCharacterSplitter {\n  private static readonly MARKDOWN_SEPARATORS = [\n    // Headers\n    '\\n#{1,6} ',\n    // Code blocks\n    '```\\n',\n    // Horizontal rules\n    '\\n\\\\*\\\\*\\\\*+\\n',\n    '\\n---+\\n',\n    '\\n___+\\n',\n    // Paragraph breaks\n    '\\n\\n',\n    // Single newlines\n    '\\n',\n    // Spaces\n    ' ',\n    // Empty string\n    '',\n  ]\n\n  constructor(options: BaseChunkOptions = {}) {\n    super({\n      ...options,\n      separators: MarkdownSplitter.MARKDOWN_SEPARATORS,\n      isSeparatorRegex: true,\n    })\n  }\n}\n"],"mappings":";AAEA,IAAsB,eAAtB,MAAuD;CACrD,AAAS;CAET,YAAY,SAA+C;AACzD,OAAK,iBACH,SAAS,oBAAoB,SAAiB,KAAK;;;;;;ACezD,IAAa,6BAAb,cAAgD,aAAa;CAC3D,AAAiB;CACjB,AAAiB;CACjB,AAAiB;CACjB,AAAiB;CACjB,AAAiB;CAEjB,YAAY,UAA6C,EAAE,EAAE;AAC3D,QAAM,EAAE,gBAAgB,QAAQ,gBAAgB,CAAC;AACjD,OAAK,YAAY,QAAQ,aAAa;AACtC,OAAK,eAAe,QAAQ,gBAAgB;AAC5C,OAAK,aAAa,QAAQ,cAAc;GAAC;GAAQ;GAAM;GAAK;GAAG;AAC/D,OAAK,gBAAgB,QAAQ,iBAAiB;AAC9C,OAAK,mBAAmB,QAAQ,oBAAoB;AACpD,OAAK,iBAAiB;;CAGxB,OAAO,MAAM,UAA2C;EACtD,MAAM,sBAAsB,KAAK,UAAU,SAAS,QAAQ;AAE5D,OAAK,IAAI,IAAI,GAAG,IAAI,oBAAoB,QAAQ,KAAK;GACnD,MAAM,QAAQ,oBAAoB;AAClC,SAAM,KAAK,YAAY,UAAU,OAAO,EAAE;;;CAI9C,AAAQ,kBAAwB;AAC9B,MAAI,KAAK,aAAa,EACpB,OAAM,IAAI,MAAM,mCAAmC;AAErD,MAAI,KAAK,eAAe,EACtB,OAAM,IAAI,MAAM,kDAAkD;AAEpE,MAAI,KAAK,gBAAgB,KAAK,UAC5B,OAAM,IAAI,MAAM,2CAA2C;;CAI/D,AAAQ,YACN,UACA,OACA,OACO;EACP,MAAM,QAAQ,MAAM;EACpB,MAAM,MAAM,MAAM,QAAQ,MAAM,KAAK;AAErC,SAAO;GACL,IAAI,GAAG,SAAS,GAAG,SAAS;GAC5B,SAAS,MAAM;GACf,UAAU;IACR;IACA;IACA;IACA,YAAY,SAAS;IACtB;GACF;;CAGH,AAAQ,UAAU,MAA+B;EAC/C,MAAM,SAAS,KAAK,mBAAmB,MAAM,KAAK,WAAW;AAC7D,SAAO,KAAK,oBAAoB,QAAQ,KAAK;;CAG/C,AAAQ,oBAAoB,QAAkB,MAA+B;EAC3E,MAAMA,SAA0B,EAAE;EAClC,IAAI,kBAAkB;AAEtB,OAAK,MAAM,SAAS,QAAQ;GAC1B,MAAM,QAAQ,KAAK,QAAQ,OAAO,gBAAgB;AAClD,OAAI,UAAU,IAAI;AAChB,WAAO,KAAK;KAAE,MAAM;KAAO,OAAO;KAAO,CAAC;AAC1C,sBAAkB,QAAQ,MAAM;UAC3B;AACL,WAAO,KAAK;KAAE,MAAM;KAAO,OAAO;KAAiB,CAAC;AACpD,uBAAmB,MAAM;;;AAI7B,SAAO;;CAGT,AAAQ,mBAAmB,MAAc,YAAgC;EACvE,MAAM,EAAE,WAAW,wBAAwB,KAAK,wBAC9C,MACA,WACD;EAED,MAAM,SAAS,KAAK,iBAAiB,MAAM,UAAU;AAErD,SAAO,KAAK,cAAc,QAAQ,WAAW,oBAAoB;;CAGnE,AAAQ,wBACN,MACA,YACiB;AACjB,OAAK,IAAI,IAAI,GAAG,IAAI,WAAW,QAAQ,KAAK;GAC1C,MAAM,YAAY,WAAW;AAE7B,OAAI,cAAc,GAChB,QAAO;IACL;IACA,qBAAqB,WAAW,MAAM,IAAI,EAAE;IAC7C;AAOH,OAJe,KAAK,mBAChB,IAAI,OAAO,UAAU,CAAC,KAAK,KAAK,GAChC,KAAK,SAAS,UAAU,CAG1B,QAAO;IACL;IACA,qBAAqB,WAAW,MAAM,IAAI,EAAE;IAC7C;;AAIL,SAAO;GACL,WAAW,WAAW,WAAW,SAAS;GAC1C,qBAAqB,EAAE;GACxB;;CAGH,AAAQ,iBAAiB,MAAc,WAA6B;AAClE,MAAI,cAAc,GAChB,QAAO,KAAK,MAAM,GAAG;AAEvB,SAAO,KAAK,mBAAmB,MAAM,UAAU;;CAGjD,AAAQ,mBAAmB,MAAc,WAA6B;AACpE,MAAI,CAAC,UACH,QAAO,KAAK,MAAM,GAAG,CAAC,QAAQ,MAAM,MAAM,GAAG;EAG/C,MAAM,aAAa,KAAK,mBACpB,YACA,UAAU,QAAQ,uBAAuB,OAAO;AAEpD,MAAI,CAAC,KAAK,cACR,QAAO,KAAK,MAAM,IAAI,OAAO,WAAW,CAAC,CAAC,QAAQ,MAAM,MAAM,GAAG;AAGnE,MAAI,CAAC,KACH,QAAO,EAAE;EAGX,MAAM,SAAS,KAAK,sBAAM,IAAI,OAAO,IAAI,WAAW,GAAG,CAAC;EACxD,MAAMC,SAAmB,EAAE;AAE3B,MAAI,KAAK,kBAAkB,SAAS,KAAK,kBAAkB,MAAM;AAC/D,QAAK,IAAI,IAAI,GAAG,IAAI,OAAO,SAAS,GAAG,KAAK,EAC1C,KAAI,IAAI,IAAI,OAAO,QAAQ;IACzB,MAAM,QAAQ,OAAO,MAAM,OAAO,IAAI,MAAM;AAC5C,QAAI,MAAO,QAAO,KAAK,MAAM;;AAGjC,OAAI,OAAO,SAAS,MAAM,KAAK,OAAO,OAAO,SAAS,GACpD,QAAO,KAAK,OAAO,OAAO,SAAS,GAAG;aAE/B,KAAK,kBAAkB,SAAS;AACzC,OAAI,OAAO,GAAI,QAAO,KAAK,OAAO,GAAG;AAErC,QAAK,IAAI,IAAI,GAAG,IAAI,OAAO,SAAS,GAAG,KAAK,GAAG;IAC7C,MAAMC,cAAY,OAAO;IACzB,MAAMC,SAAO,OAAO,IAAI;AACxB,QAAID,eAAaC,OACf,QAAO,KAAKD,cAAYC,OAAK;aACpBD,YACT,QAAO,KAAKA,YAAU;;;AAK5B,SAAO,OAAO,QAAQ,MAAM,MAAM,GAAG;;CAGvC,AAAQ,cACN,QACA,WACA,qBACU;EACV,MAAME,cAAwB,EAAE;EAChC,MAAMC,aAAuB,EAAE;AAE/B,OAAK,MAAM,SAAS,OAClB,KAAI,KAAK,iBAAiB,MAAM,CAC9B,YAAW,KAAK,MAAM;OACjB;AACL,QAAK,gBAAgB,YAAY,WAAW,YAAY;AACxD,QAAK,qBAAqB,OAAO,qBAAqB,YAAY;;AAItE,OAAK,gBAAgB,YAAY,WAAW,YAAY;AAExD,SAAO;;CAGT,AAAQ,iBAAiB,MAAuB;AAC9C,SAAO,KAAK,eAAe,KAAK,GAAG,KAAK;;CAG1C,AAAQ,gBACN,YACA,WACA,QACM;AACN,MAAI,WAAW,WAAW,EAAG;EAE7B,MAAM,kBAAkB,KAAK,gBACzB,KACA,cAAc,KACd,KACA;EACJ,MAAM,SAAS,KAAK,iBAAiB,YAAY,gBAAgB;AACjE,SAAO,KAAK,GAAG,OAAO;AACtB,aAAW,SAAS;;CAGtB,AAAQ,qBACN,OACA,qBACA,QACM;AACN,MAAI,oBAAoB,SAAS,GAAG;GAClC,MAAM,kBAAkB,KAAK,mBAC3B,OACA,oBACD;AACD,UAAO,KAAK,GAAG,gBAAgB;aACtB,UAAU,GACnB,QAAO,KAAK,MAAM;;CAItB,AAAQ,iBAAiB,QAAkB,WAA6B;EACtE,MAAM,kBAAkB,KAAK,eAAe,UAAU;EACtD,MAAMC,SAAmB,EAAE;EAC3B,MAAMC,eAAyB,EAAE;EACjC,IAAI,gBAAgB;AAEpB,OAAK,MAAM,SAAS,QAAQ;GAC1B,MAAM,cAAc,KAAK,eAAe,MAAM;AAE9C,OACE,KAAK,oBACH,cACA,eACA,aACA,gBACD,EACD;AACA,WAAO,KAAK,KAAK,cAAc,cAAc,UAAU,CAAC;AACxD,oBAAgB,KAAK,aACnB,cACA,eACA,aACA,gBACD;;AAGH,QAAK,gBAAgB,cAAc,MAAM;AACzC,oBAAiB,KAAK,qBACpB,aAAa,QACb,aACA,gBACD;;AAGH,MAAI,aAAa,SAAS,EACxB,QAAO,KAAK,KAAK,cAAc,cAAc,UAAU,CAAC;AAG1D,SAAO;;CAGT,AAAQ,oBACN,cACA,eACA,aACA,iBACS;AACT,MAAI,aAAa,WAAW,EAAG,QAAO;AAOtC,SAJE,gBACA,eACC,aAAa,SAAS,IAAI,kBAAkB,KAEtB,KAAK;;CAGhC,AAAQ,cAAc,OAAiB,WAA2B;AAChE,SAAO,MAAM,KAAK,UAAU;;CAG9B,AAAQ,aACN,cACA,eACA,iBACA,iBACQ;EACR,IAAI,SAAS;AAEb,SACE,KAAK,sBACH,cACA,QACA,iBACA,gBACD,EACD;GACA,MAAM,UAAU,aAAa,OAAO;AACpC,aAAU,KAAK,eAAe,QAAQ;AACtC,OAAI,aAAa,SAAS,EACxB,WAAU;;AAId,SAAO;;CAGT,AAAQ,sBACN,cACA,eACA,iBACA,iBACS;AACT,MAAI,aAAa,WAAW,EAAG,QAAO;EAEtC,MAAM,iBAAiB,gBAAgB,KAAK;EAC5C,MAAM,sBACJ,gBAAgB,kBAAkB,kBAAkB,KAAK,aACzD,gBAAgB;AAElB,SAAO,kBAAkB;;CAG3B,AAAQ,gBAAgB,OAAiB,OAAqB;AAC5D,QAAM,KAAK,MAAM;;CAGnB,AAAQ,qBACN,aACA,aACA,iBACQ;AACR,SAAO,eAAe,cAAc,IAAI,kBAAkB;;;;;;AChX9D,IAAa,mBAAb,MAAa,yBAAyB,2BAA2B;CAC/D,OAAwB,sBAAsB;EAE5C;EAEA;EAEA;EACA;EACA;EAEA;EAEA;EAEA;EAEA;EACD;CAED,YAAY,UAA4B,EAAE,EAAE;AAC1C,QAAM;GACJ,GAAG;GACH,YAAY,iBAAiB;GAC7B,kBAAkB;GACnB,CAAC"}

package.json

 {
   "name": "@open-evals/rag",
-  "version": "0.1.2",
+  "version": "0.1.3",
   "description": "RAG utilities for Open Evals",
@@ -5,0 +5,0 @@ "author": {

No alert changes

Improved metrics

Total package byte prevSize

30259

7.08%

Lines of code

307

12.04%

No dependency changes