cspell-dictionary - npm Package Compare versions

Comparing version 8.12.1 to 8.13.0

dist/index.d.ts

@@ -0,3 +1,11 @@
+import { enableLogging as cacheDictionaryEnableLogging, getLog as cacheDictionaryGetLog } from './SpellingDictionary/CachingDictionary.js';
 export type { CachingDictionary, FindOptions, FindResult, HasOptions, SearchOptions, SpellingDictionary, SpellingDictionaryCollection, SpellingDictionaryOptions, SuggestionCollector, SuggestionResult, SuggestOptions, } from './SpellingDictionary/index.js';
 export { createCachingDictionary, createCollection, createFailedToLoadDictionary, createFlagWordsDictionary, createForbiddenWordsDictionary, createIgnoreWordsDictionary, createInlineSpellingDictionary, createSpellingDictionary, createSpellingDictionaryFromTrieFile, createSuggestDictionary, createSuggestOptions, } from './SpellingDictionary/index.js';
+/**
+ * Debugging utilities.
+ */
+export declare const _debug: {
+    cacheDictionaryEnableLogging: typeof cacheDictionaryEnableLogging;
+    cacheDictionaryGetLog: typeof cacheDictionaryGetLog;
+};
 //# sourceMappingURL=index.d.ts.map

dist/index.js

@@ -0,2 +1,10 @@
+import { enableLogging as cacheDictionaryEnableLogging, getLog as cacheDictionaryGetLog, } from './SpellingDictionary/CachingDictionary.js';
 export { createCachingDictionary, createCollection, createFailedToLoadDictionary, createFlagWordsDictionary, createForbiddenWordsDictionary, createIgnoreWordsDictionary, createInlineSpellingDictionary, createSpellingDictionary, createSpellingDictionaryFromTrieFile, createSuggestDictionary, createSuggestOptions, } from './SpellingDictionary/index.js';
+/**
+ * Debugging utilities.
+ */
+export const _debug = {
+    cacheDictionaryEnableLogging,
+    cacheDictionaryGetLog,
+};
 //# sourceMappingURL=index.js.map

dist/perf/has.perf.js

@@ -5,22 +5,37 @@ import assert from 'node:assert';
 import { suite } from 'perf-insight';
+import { createCachingDictionary } from '../SpellingDictionary/CachingDictionary.js';
 import { createSpellingDictionary } from '../SpellingDictionary/createSpellingDictionary.js';
 import { createCollection } from '../SpellingDictionary/SpellingDictionaryCollection.js';
 suite('dictionary has', async (test) => {
-    const words = genWords(10_000);
+    const words1 = genWords(10_000);
     const words2 = genWords(1000);
     const words3 = genWords(1000);
-    const iTrie = buildITrieFromWords(words);
-    const dict = createSpellingDictionary(words, 'test', import.meta.url);
+    const words = words1;
+    const iTrie = buildITrieFromWords(words1);
+    const dict = createSpellingDictionary(words1, 'test', import.meta.url);
     const dict2 = createSpellingDictionary(words2, 'test2', import.meta.url);
     const dict3 = createSpellingDictionary(words3, 'test3', import.meta.url);
     const dictCol = createCollection([dict, dict2, dict3], 'test-collection');
+    const dictColRev = createCollection([dict3, dict2, dict], 'test-collection-reverse');
+    const cacheDictSingle = createCachingDictionary(dict, {});
+    const cacheDictCol = createCachingDictionary(dictCol, {});
+    const dictSet = new Set(words);
+    test('Set has 100k words', () => {
+        checkWords(dictSet, words);
+    });
     test('dictionary has 100k words', () => {
         checkWords(dict, words);
     });
-    test('dictionary has 100k words (2nd time)', () => {
-        checkWords(dict, words);
-    });
     test('collection has 100k words', () => {
         checkWords(dictCol, words);
     });
+    test('collection reverse has 100k words', () => {
+        checkWords(dictColRev, words);
+    });
+    test('cache dictionary has 100k words', () => {
+        checkWords(cacheDictSingle, words);
+    });
+    test('cache collection has 100k words', () => {
+        checkWords(cacheDictCol, words);
+    });
     test('iTrie has 100k words', () => {
@@ -47,22 +62,96 @@ checkWords(iTrie, words);
     const dictCol = createCollection([dict, dict2, dict3], 'test-collection');
-    test('dictionary has 100k words', () => {
+    const dictSet = new Set(words);
+    test('Set has not 100k words', () => {
+        checkWords(dictSet, missingWords, false);
+    });
+    test('dictionary has not 100k words', () => {
         checkWords(dict, missingWords, false);
     });
-    test('dictionary has 100k words (2nd time)', () => {
+    test('collection has not 100k words', () => {
+        checkWords(dictCol, missingWords, false);
+    });
+    test('iTrie has not 100k words', () => {
+        checkWords(iTrie, missingWords, false);
+    });
+    test('iTrie.hasWord has not 100k words', () => {
+        const dict = { has: (word) => iTrie.hasWord(word, true) };
         checkWords(dict, missingWords, false);
     });
+    test('iTrie.data has not 100k words', () => {
+        checkWords(iTrie.data, missingWords, false);
+    });
+});
+suite('dictionary has sampling', async (test) => {
+    const words1 = genWords(10_000);
+    const words2 = genWords(1000);
+    const words3 = genWords(1000);
+    const sampleIdx = genSamples(100_000, words1.length);
+    const wordsSample = sampleIdx.map((i) => words1[i]);
+    const iTrie = buildITrieFromWords(words1);
+    const dict = createSpellingDictionary(words1, 'test', import.meta.url);
+    const dict2 = createSpellingDictionary(words2, 'test2', import.meta.url);
+    const dict3 = createSpellingDictionary(words3, 'test3', import.meta.url);
+    const dictCol = createCollection([dict, dict2, dict3], 'test-collection');
+    const dictColRev = createCollection([dict3, dict2, dict], 'test-collection-reverse');
+    const cacheDictSingle = createCachingDictionary(dict, {});
+    const cacheDictCol = createCachingDictionary(dictCol, {});
+    const dictSet = new Set(words1);
+    test('Set has 100k words', () => {
+        checkWords(dictSet, wordsSample);
+    });
+    test('dictionary has 100k words', () => {
+        checkWords(dict, wordsSample);
+    });
     test('collection has 100k words', () => {
-        checkWords(dictCol, missingWords, false);
+        checkWords(dictCol, wordsSample);
     });
+    test('collection reverse has 100k words', () => {
+        checkWords(dictColRev, wordsSample);
+    });
+    test('cache dictionary has 100k words', () => {
+        checkWords(cacheDictSingle, wordsSample);
+    });
+    test('cache collection has 100k words', () => {
+        checkWords(cacheDictCol, wordsSample);
+    });
     test('iTrie has 100k words', () => {
-        checkWords(iTrie, missingWords, false);
+        checkWords(iTrie, wordsSample);
     });
     test('iTrie.hasWord has 100k words', () => {
         const dict = { has: (word) => iTrie.hasWord(word, true) };
-        checkWords(dict, missingWords, false);
+        checkWords(dict, wordsSample);
     });
     test('iTrie.data has 100k words', () => {
-        checkWords(iTrie.data, missingWords, false);
+        checkWords(iTrie.data, wordsSample);
     });
 });
+suite('dictionary isForbidden sampling', async (test) => {
+    const words1 = genWords(10_000);
+    const words2 = genWords(1000);
+    const words3 = genWords(1000);
+    const sampleIdx = genSamples(100_000, words1.length);
+    const wordsSample = sampleIdx.map((i) => words1[i]);
+    const dict = createSpellingDictionary(words1, 'test', import.meta.url);
+    const dict2 = createSpellingDictionary(words2, 'test2', import.meta.url);
+    const dict3 = createSpellingDictionary(words3, 'test3', import.meta.url);
+    const dictCol = createCollection([dict, dict2, dict3], 'test-collection');
+    const dictColRev = createCollection([dict3, dict2, dict], 'test-collection-reverse');
+    const cacheDictSingle = createCachingDictionary(dict, {});
+    const cacheDictCol = createCachingDictionary(dictCol, {});
+    test('dictionary isForbidden 100k words', () => {
+        checkForForbiddenWords(dict, wordsSample);
+    });
+    test('collection isForbidden 100k words', () => {
+        checkForForbiddenWords(dictCol, wordsSample);
+    });
+    test('collection reverse isForbidden 100k words', () => {
+        checkForForbiddenWords(dictColRev, wordsSample);
+    });
+    test('cache dictionary isForbidden 100k words', () => {
+        checkForForbiddenWords(cacheDictSingle, wordsSample);
+    });
+    test('cache collection isForbidden 100k words', () => {
+        checkForForbiddenWords(cacheDictCol, wordsSample);
+    });
+});
 function checkWords(dict, words, expected = true, totalChecks = 100_000) {
@@ -81,2 +170,12 @@ let has = true;
 }
+function checkForForbiddenWords(dict, words, totalChecks = 100_000) {
+    let result = true;
+    const len = words.length;
+    for (let i = 0; i < totalChecks; ++i) {
+        const word = words[i % len];
+        const r = !dict.isForbidden(word);
+        result = r && result;
+    }
+    assert(result, 'All words should not be forbidden');
+}
 function genWords(count, includeForbidden = true) {
@@ -107,2 +206,13 @@ const setOfWords = new Set(loremIpsum({ count }).split(' '));
 }
+function genSamples(count, max, depth = 3) {
+    const r = Array(count);
+    for (let j = 0; j < count; ++j) {
+        let n = Math.random() * max;
+        for (let i = 1; i < depth; ++i) {
+            n = Math.random() * n;
+        }
+        r[j] = Math.floor(n);
+    }
+    return r;
+}
 //# sourceMappingURL=has.perf.js.map

dist/SpellingDictionary/CachingDictionary.d.ts

@@ -24,2 +24,13 @@ import type { CacheStats } from '../util/AutoCache.js';
 }
+interface LogEntryBase extends SearchOptions {
+    time: number;
+    method: 'has';
+    word: string;
+    value?: unknown;
+}
+interface LogEntryHas extends LogEntryBase {
+    method: 'has';
+    value: boolean;
+}
+export type LogEntry = LogEntryHas;
 /**
@@ -32,3 +43,5 @@ * create a caching dictionary
 export declare function createCachingDictionary(dict: SpellingDictionary | SpellingDictionaryCollection, options: SearchOptions): CachingDictionary;
+export declare function enableLogging(enabled?: boolean): void;
+export declare function getLog(): LogEntryBase[];
 export {};
 //# sourceMappingURL=CachingDictionary.d.ts.map

dist/SpellingDictionary/CachingDictionary.js

@@ -5,2 +5,5 @@ import { autoCache, extractStats } from '../util/AutoCache.js';
 const DefaultAutoCacheSize = 1000;
+let logRequests = false;
+const log = [];
+const startTime = performance.now();
 class CachedDict {
@@ -17,3 +20,11 @@ dict;
     }
-    has = autoCache((word) => this.dict.has(word, this.options), DefaultAutoCacheSize);
+    #has = autoCache((word) => this.dict.has(word, this.options), DefaultAutoCacheSize);
+    has = logRequests
+        ? (word) => {
+            const time = performance.now() - startTime;
+            const value = this.#has(word);
+            log.push({ time, method: 'has', word, value });
+            return value;
+        }
+        : this.#has;
     isNoSuggestWord = autoCache((word) => this.dict.isNoSuggestWord(word, this.options), DefaultAutoCacheSize);
@@ -26,3 +37,3 @@ isForbidden = autoCache((word) => this.dict.isForbidden(word), DefaultAutoCacheSize);
             id: this.id,
-            has: extractStats(this.has),
+            has: extractStats(this.#has),
             isNoSuggestWord: extractStats(this.isNoSuggestWord),
@@ -55,2 +66,8 @@ isForbidden: extractStats(this.isForbidden),
 }
+export function enableLogging(enabled = !logRequests) {
+    logRequests = enabled;
+}
+export function getLog() {
+    return log;
+}
 //# sourceMappingURL=CachingDictionary.js.map

dist/SpellingDictionary/SpellingDictionaryCollection.js

 import { CASE_INSENSITIVE_PREFIX, CompoundWordsMethod } from 'cspell-trie-lib';
-import { genSequence } from 'gensequence';
 import { isDefined } from '../util/util.js';
@@ -101,3 +100,3 @@ import * as Defaults from './defaults.js';
 function isWordInAnyDictionary(dicts, word, options) {
-    return genSequence(dicts).first((dict) => dict.has(word, options));
+    return dicts.find((dict) => dict.has(word, options));
 }
@@ -115,6 +114,6 @@ function findInAnyDictionary(dicts, word, options) {
 function isNoSuggestWordInAnyDictionary(dicts, word, options) {
-    return genSequence(dicts).first((dict) => dict.isNoSuggestWord(word, options));
+    return dicts.find((dict) => dict.isNoSuggestWord(word, options));
 }
 function isWordForbiddenInAnyDictionary(dicts, word, ignoreCase) {
-    return genSequence(dicts).first((dict) => dict.isForbidden(word, ignoreCase));
+    return dicts.find((dict) => dict.isForbidden(word, ignoreCase));
 }
@@ -121,0 +120,0 @@ export function isSpellingDictionaryCollection(dict) {

dist/SpellingDictionary/SpellingDictionaryFromTrie.d.ts

@@ -9,3 +9,2 @@ import type { ITrie, SuggestionCollector, SuggestionResult } from 'cspell-trie-lib';
     readonly source: string;
-    static readonly cachedWordsLimit = 50000;
     private _size;
@@ -30,3 +29,2 @@ readonly knownWords: Set<string>;
     isForbidden(word: string, _ignoreCaseAndAccents?: boolean): boolean;
-    private _isForbidden;
     suggest(word: string, suggestOptions?: SuggestOptions): SuggestionResult[];
@@ -33,0 +31,0 @@ private _suggest;

dist/SpellingDictionary/SpellingDictionaryFromTrie.js

 import { CompoundWordsMethod, decodeTrie, suggestionCollector } from 'cspell-trie-lib';
-import { autoCache, createCache01 } from '../util/AutoCache.js';
 import { clean } from '../util/clean.js';
@@ -14,3 +13,2 @@ import { createMapper, createRepMapper } from '../util/repMap.js';
     source;
-    static cachedWordsLimit = 50_000;
     _size = 0;
@@ -72,3 +70,3 @@ knownWords = new Set();
     }
-    _find = findCache((word, useCompounds, ignoreCase) => this.findAnyForm(word, useCompounds, ignoreCase));
+    _find = (word, useCompounds, ignoreCase) => this.findAnyForm(word, useCompounds, ignoreCase);
     findAnyForm(word, useCompounds, ignoreCase) {
@@ -111,7 +109,4 @@ const outerForms = outerWordForms(word, this.remapWord || ((word) => [this.mapWord(word)]));
     isForbidden(word, _ignoreCaseAndAccents) {
-        return this._isForbidden(word);
+        return this.trie.isForbiddenWord(word);
     }
-    _isForbidden = autoCache((word) => {
-        return this.trie.isForbiddenWord(word);
-    });
     suggest(word, suggestOptions = {}) {
@@ -160,15 +155,2 @@ return this._suggest(word, suggestOptions);
 }
-function findCache(fn, size = 2000) {
-    const cache = createCache01(size);
-    function find(word, useCompounds, ignoreCase) {
-        const r = cache.get(word);
-        if (r !== undefined && r.useCompounds === useCompounds && r.ignoreCase === ignoreCase) {
-            return r.findResult;
-        }
-        const findResult = fn(word, useCompounds, ignoreCase);
-        cache.set(word, { useCompounds, ignoreCase, findResult });
-        return findResult;
-    }
-    return find;
-}
 function* outerWordForms(word, mapWord) {
@@ -178,17 +160,21 @@ // Only generate the needed forms.
     let w = word;
+    const ww = w;
     yield w;
     sent.add(w);
     w = word.normalize('NFC');
-    if (!sent.has(w))
+    if (w !== ww) {
         yield w;
-    sent.add(w);
+        sent.add(w);
+    }
     w = word.normalize('NFD');
-    if (!sent.has(w))
+    if (w !== ww && !sent.has(w)) {
         yield w;
-    sent.add(w);
-    for (const f of [...sent]) {
+        sent.add(w);
+    }
+    for (const f of sent) {
         for (const m of mapWord(f)) {
-            if (!sent.has(m))
+            if (m !== ww && !sent.has(m)) {
                 yield m;
-            sent.add(m);
+                sent.add(m);
+            }
         }
@@ -195,0 +181,0 @@ }

dist/util/AutoCache.d.ts

@@ -9,7 +9,4 @@ interface AutoCache<R> extends CacheStats {
 }
-declare class Cache01<R> implements CacheStats {
+declare abstract class Cache01<R> implements CacheStats {
     readonly maxSize: number;
-    private count;
-    private cache0;
-    private cache1;
     hits: number;
@@ -19,4 +16,4 @@ misses: number;
     constructor(maxSize: number);
-    get(key: string): R | undefined;
-    set(key: string, value: R): this;
+    abstract get(key: string): R | undefined;
+    abstract set(key: string, value: R): this;
 }
@@ -23,0 +20,0 @@ export declare function createCache01<R>(size: number): Cache01<R>;

dist/util/AutoCache.js

 const CACHE_SIZE = 100;
 class Cache01 {
     maxSize;
-    count = 0;
-    cache0 = Object.create(null);
-    cache1 = Object.create(null);
     hits = 0;
@@ -13,15 +10,24 @@ misses = 0;
     }
+}
+class Cache01Map extends Cache01 {
+    count = 0;
+    cache0 = new Map();
+    cache1 = new Map();
+    constructor(maxSize) {
+        super(maxSize);
+    }
     get(key) {
         const cache0 = this.cache0;
         const cache1 = this.cache1;
-        if (key in cache0) {
+        let found = cache0.get(key);
+        if (found !== undefined) {
             ++this.hits;
-            return cache0[key];
+            return found;
         }
-        if (key in cache1) {
+        found = cache1.get(key);
+        if (found !== undefined) {
             ++this.hits;
             ++this.count;
-            const r = cache1[key];
-            cache0[key] = r;
-            return r;
+            cache0.set(key, found);
+            return found;
         }
@@ -33,4 +39,6 @@ ++this.misses;
         if (this.count >= this.maxSize) {
+            const c = this.cache1;
             this.cache1 = this.cache0;
-            this.cache0 = Object.create(null);
+            this.cache0 = c;
+            c.clear();
             this.swaps++;
@@ -40,3 +48,3 @@ this.count = 0;
         ++this.count;
-        this.cache0[key] = value;
+        this.cache0.set(key, value);
         return this;
@@ -46,3 +54,3 @@ }
 export function createCache01(size) {
-    return new Cache01(size);
+    return new Cache01Map(size);
 }
@@ -49,0 +57,0 @@ export function autoCache(fn, size = CACHE_SIZE) {

package.json

 {
   "name": "cspell-dictionary",
-  "version": "8.12.1",
+  "version": "8.13.0",
   "description": "A spelling dictionary library useful for checking words and getting suggestions.",
@@ -53,12 +53,12 @@ "type": "module",
   "dependencies": {
-    "@cspell/cspell-pipe": "8.12.1",
-    "@cspell/cspell-types": "8.12.1",
-    "cspell-trie-lib": "8.12.1",
-    "fast-equals": "^5.0.1",
-    "gensequence": "^7.0.0"
+    "@cspell/cspell-pipe": "8.13.0",
+    "@cspell/cspell-types": "8.13.0",
+    "cspell-trie-lib": "8.13.0",
+    "fast-equals": "^5.0.1"
   },
   "devDependencies": {
+    "gensequence": "^7.0.0",
     "lorem-ipsum": "^2.0.8"
   },
-  "gitHead": "2b9d2fd6fb3a228886016f897c40e8b926e063ab"
+  "gitHead": "2fd3fb430cc96a8a50543f57d96b288219a11923"
 }

New alerts

License Policy Violation

License

This package is not allowed per your license policy. Review the package's license to ensure compliance.

Found 1 instance in 1 package

Fixed alerts

License Policy Violation

License

This package is not allowed per your license policy. Review the package's license to ensure compliance.

Found 1 instance in 1 package

Improved metrics

Total package byte prevSize

105645

increased by7%

Dependency count

4

decreased by-20%

Number of package files

73

increased by2.82%

Lines of code

2755

increased by7.49%

Worsened metrics

Dev dependency count

2

increased by100%

Dependency changes

• + Added@cspell/cspell-pipe@8.13.0(transitive)

• + Added@cspell/cspell-types@8.13.0(transitive)

• + Addedcspell-trie-lib@8.13.0(transitive)

• - Removedgensequence@^7.0.0

• - Removed@cspell/cspell-pipe@8.12.1(transitive)

• - Removed@cspell/cspell-types@8.12.1(transitive)

• - Removedcspell-trie-lib@8.12.1(transitive)

• Updated@cspell/cspell-pipe@8.13.0

• Updated@cspell/cspell-types@8.13.0

• Updatedcspell-trie-lib@8.13.0