@orama/orama - npm Package Compare versions

Comparing version 2.0.5 to 2.0.6

dist/components/sorter.js

@@ -80,2 +80,10 @@ import { createError } from '../errors.js';
     const s = sorter.sorts[prop];
+    // This happen during a document updating
+    // Because we re-use the same internalId
+    // We need to clean-up the data structure before re-inserting
+    // to avoid duplicates in the orderedDocs array
+    // See: https://github.com/oramasearch/orama/issues/629
+    if (s.orderedDocsToRemove.has(internalId)) {
+        ensureOrderedDocsAreDeletedByProperty(sorter, prop);
+    }
     s.docs.set(internalId, s.orderedDocs.length);
@@ -82,0 +90,0 @@ s.orderedDocs.push([

dist/components/tokenizer/index.d.ts

 import { Stemmer, Tokenizer, DefaultTokenizerConfig } from '../../types.js';
 import { Language } from './languages.js';
-interface DefaultTokenizer extends Tokenizer {
+export interface DefaultTokenizer extends Tokenizer {
     language: Language;
@@ -13,3 +13,3 @@ stemmer?: Stemmer;
 }
+export declare function normalizeToken(this: DefaultTokenizer, prop: string, token: string): string;
 export declare function createTokenizer(config?: DefaultTokenizerConfig): Promise<DefaultTokenizer>;
-export {};

dist/components/tokenizer/index.js

@@ -5,3 +5,3 @@ import { createError } from '../../errors.js';
 import { stemmer as english } from './english-stemmer.js';
-function normalizeToken(prop, token) {
+export function normalizeToken(prop, token) {
     var _this_stopWords;
@@ -8,0 +8,0 @@ const key = `${this.language}:${prop}:${token}`;

dist/internals.d.ts

 export { boundedLevenshtein } from './components/levenshtein.js';
 export { formatBytes, formatNanoseconds, getNanosecondsTime, uniqueId, convertDistanceToMeters, safeArrayPush } from './utils.js';
 export { BM25 } from './components/algorithms.js';
+export { normalizeToken } from './components/tokenizer/index.js';

dist/internals.js

 export { boundedLevenshtein } from './components/levenshtein.js';
 export { formatBytes, formatNanoseconds, getNanosecondsTime, uniqueId, convertDistanceToMeters, safeArrayPush } from './utils.js';
 export { BM25 } from './components/algorithms.js';
+export { normalizeToken } from './components/tokenizer/index.js';
 
 //# sourceMappingURL=internals.js.map

dist/types.d.ts

@@ -8,2 +8,3 @@ import { MODE_FULLTEXT_SEARCH, MODE_HYBRID_SEARCH, MODE_VECTOR_SEARCH } from './constants.js';
 import { Point } from './trees/bkd.js';
+export type { DefaultTokenizer } from './components/tokenizer/index.js';
 export type Nullable<T> = T | null;
@@ -942,2 +943,1 @@ export type SingleOrArray<T> = T | T[];
 export type OramaPlugin = OramaPluginSync | OramaPluginAsync;
-export {};

package.json

 {
   "name": "@orama/orama",
-  "version": "2.0.5",
+  "version": "2.0.6",
   "type": "module",
@@ -91,4 +91,4 @@ "description": "Next generation full-text and vector search engine, written in TypeScript",
     "vite": "^4.1.4",
-    "@orama/stemmers": "2.0.5",
-    "@orama/stopwords": "2.0.5"
+    "@orama/stemmers": "2.0.6",
+    "@orama/stopwords": "2.0.6"
   },
@@ -95,0 +95,0 @@ "engines": {

README.md

@@ -27,3 +27,3 @@ # Orama
 - [BM25](https://docs.oramasearch.com/open-source/usage/search/bm25-algorithm)
-- [Stemming and tokenization in 28 languages](https://docs.oramasearch.com/open-source/text-analysis/stemming)
+- [Stemming and tokenization in 29 languages](https://docs.oramasearch.com/open-source/text-analysis/stemming)
 - [Plugin System](https://docs.oramasearch.com/open-source/plugins/introduction)
@@ -30,0 +30,0 @@

No alert changes

Improved metrics

Total package byte prevSize

709081

increased by0.21%

Lines of code

7034

increased by0.14%

No dependency changes