cspell-trie-lib - npm Package Compare versions

Comparing version 4.1.8 to 5.0.1-alpha.3

CHANGELOG.md

@@ -0,1 +1,14 @@
+# Change Log
+
+All notable changes to this project will be documented in this file.
+See [Conventional Commits](https://conventionalcommits.org) for commit guidelines.
+
+## [5.0.1-alpha.0](https://github.com/streetsidesoftware/cspell/compare/cspell-trie-lib@4.1.8...cspell-trie-lib@5.0.1-alpha.0) (2020-02-20)
+
+**Note:** Version bump only for package cspell-trie-lib
+
+
+
+
+
 # Release Notes
@@ -2,0 +15,0 @@

dist/lib/bufferLines.d.ts

@@ -1,3 +0,2 @@
-import { IterableLike } from './IterableLike';
-export declare function buffer<T>(iter: IterableLike<T>, bufferSize: number): IterableIterator<T[]>;
-export declare function bufferLines(iter: IterableLike<string>, bufferSize: number, eol: string): IterableIterator<string>;
+export declare function buffer<T>(iter: Iterable<T>, bufferSize: number): IterableIterator<T[]>;
+export declare function bufferLines(iter: Iterable<string>, bufferSize: number, eol: string): IterableIterator<string>;

dist/lib/consolidate.d.ts

@@ -1,2 +0,2 @@
-import { TrieNode } from './TrieNode';
+import { TrieRoot } from './TrieNode';
 /**
@@ -6,2 +6,2 @@ * Consolidate to DAWG
  */
-export declare function consolidate(root: TrieNode): TrieNode;
+export declare function consolidate(root: TrieRoot): TrieRoot;

dist/lib/consolidate.js

 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
 const TrieNode_1 = require("./TrieNode");
+const util_1 = require("./util");
 /**
@@ -89,6 +90,5 @@ * Consolidate to DAWG
     cached.set(eow, count++);
-    root = process(root);
-    return root;
+    return util_1.trieNodeToRoot(process(root), root);
 }
 exports.consolidate = consolidate;
 //# sourceMappingURL=consolidate.js.map

dist/lib/importExport.d.ts

 import { Sequence } from 'gensequence';
-import { TrieNode } from './TrieNode';
+import { TrieRoot } from './TrieNode';
 export interface ExportOptions {
@@ -14,3 +14,3 @@ base?: number;
  */
-export declare function serializeTrie(root: TrieNode, options?: ExportOptions | number): Sequence<string>;
-export declare function importTrie(lines: Iterable<string> | IterableIterator<string>): TrieNode;
+export declare function serializeTrie(root: TrieRoot, options?: ExportOptions | number): Sequence<string>;
+export declare function importTrie(lines: Iterable<string> | IterableIterator<string>): TrieRoot;

dist/lib/importExportV1.d.ts

@@ -1,2 +0,2 @@
-import { TrieNode } from './TrieNode';
+import { TrieRoot } from './TrieNode';
 import { Sequence } from 'gensequence';
@@ -14,3 +14,3 @@ export declare const DATA = "*";
  */
-export declare function serializeTrie(root: TrieNode, options?: ExportOptions | number): Sequence<string>;
-export declare function importTrie(linesX: Iterable<string> | IterableIterator<string>): TrieNode;
+export declare function serializeTrie(root: TrieRoot, options?: ExportOptions | number): Sequence<string>;
+export declare function importTrie(linesX: Iterable<string> | IterableIterator<string>): TrieRoot;

dist/lib/importExportV1.js

@@ -6,2 +6,3 @@ "use strict";
 const convertToTrieRefNodes_1 = require("./convertToTrieRefNodes");
+const util_1 = require("./util");
 const EOW = '*';
@@ -139,5 +140,5 @@ exports.DATA = EOW;
     }, { lines: 0, nodes: [], root: {} });
-    return n.root;
+    return util_1.trieNodeToRoot(n.root, {});
 }
 exports.importTrie = importTrie;
 //# sourceMappingURL=importExportV1.js.map

dist/lib/importExportV2.d.ts

@@ -1,2 +0,2 @@
-import { TrieNode } from './TrieNode';
+import { TrieRoot } from './TrieNode';
 import { Sequence } from 'gensequence';
@@ -14,3 +14,3 @@ export declare const DATA = "__DATA__";
  */
-export declare function serializeTrie(root: TrieNode, options?: ExportOptions | number): Sequence<string>;
-export declare function importTrie(linesX: Iterable<string> | IterableIterator<string>): TrieNode;
+export declare function serializeTrie(root: TrieRoot, options?: ExportOptions | number): Sequence<string>;
+export declare function importTrie(linesX: Iterable<string> | IterableIterator<string>): TrieRoot;

dist/lib/importExportV2.js

@@ -5,2 +5,3 @@ "use strict";
 const gensequence_1 = require("gensequence");
+const util_1 = require("./util");
 const EOW = '*';
@@ -12,3 +13,3 @@ exports.DATA = '__DATA__';
         const refNode = node;
-        refNode.s = (_a = refNode.s, (_a !== null && _a !== void 0 ? _a : k));
+        refNode.s = (_a = refNode.s) !== null && _a !== void 0 ? _a : k;
         return refNode;
@@ -170,6 +171,6 @@ }
         return { root, nodes };
-    }, { nodes: [], root: { s: '' } });
-    return n.root;
+    }, { nodes: [], root: { s: '', c: new Map() } });
+    return util_1.trieNodeToRoot(n.root, {});
 }
 exports.importTrie = importTrie;
 //# sourceMappingURL=importExportV2.js.map

dist/lib/importExportV3.d.ts

@@ -1,4 +0,3 @@
-import { TrieNode } from './TrieNode';
+import { TrieRoot } from './TrieNode';
 import { Sequence } from 'gensequence';
-import { IterableLike } from './IterableLike';
 export declare const DATA = "__DATA__";
@@ -10,5 +9,5 @@ export interface ExportOptions {
 /**
- * Serialize a TrieNode.
+ * Serialize a TrieRoot.
  */
-export declare function serializeTrie(root: TrieNode, options?: ExportOptions | number): Sequence<string>;
-export declare function importTrie(linesX: IterableLike<string>): TrieNode;
+export declare function serializeTrie(root: TrieRoot, options?: ExportOptions | number): Sequence<string>;
+export declare function importTrie(linesX: Iterable<string>): TrieRoot;

dist/lib/importExportV3.js

@@ -6,2 +6,3 @@ "use strict";
 const bufferLines_1 = require("./bufferLines");
+const util_1 = require("./util");
 const EOW = '$';
@@ -38,3 +39,3 @@ const BACK = '<';
 /**
- * Serialize a TrieNode.
+ * Serialize a TrieRoot.
  */
@@ -111,2 +112,3 @@ function serializeTrie(root, options = 16) {
 function importTrie(linesX) {
+    const root = util_1.trieNodeToRoot({}, {});
     let radix = 16;
@@ -142,3 +144,2 @@ const comment = /^\s*#/;
     readHeader(iter);
-    const root = {};
     const n = gensequence_1.genSequence(iter)
@@ -191,3 +192,3 @@ .concatMap(a => a.split(''))
         const node = top.node;
-        node.c = (_a = node.c, (_a !== null && _a !== void 0 ? _a : new Map()));
+        node.c = (_a = node.c) !== null && _a !== void 0 ? _a : new Map();
         const n = { f: undefined, c: undefined, n: nodes.length };
@@ -239,3 +240,3 @@ node.c.set(s, n);
         var _a, _b;
-        const parser = (_b = (_a = acc.parser, (_a !== null && _a !== void 0 ? _a : parsers.get(s))), (_b !== null && _b !== void 0 ? _b : parseCharacter));
+        const parser = (_b = (_a = acc.parser) !== null && _a !== void 0 ? _a : parsers.get(s)) !== null && _b !== void 0 ? _b : parseCharacter;
         return parser(acc, s);
@@ -242,0 +243,0 @@ }

dist/lib/index.d.ts

@@ -9,1 +9,2 @@ export * from './trie';
 export { SuggestionResult, MaxCost, suggestionCollector, SuggestionCollector, CompoundWordsMethod } from './suggest';
+export { parseDictionaryLines, parseDictionary } from './SimpleDictionaryParser';

dist/lib/index.js

@@ -16,2 +16,5 @@ "use strict";
 exports.CompoundWordsMethod = suggest_1.CompoundWordsMethod;
+var SimpleDictionaryParser_1 = require("./SimpleDictionaryParser");
+exports.parseDictionaryLines = SimpleDictionaryParser_1.parseDictionaryLines;
+exports.parseDictionary = SimpleDictionaryParser_1.parseDictionary;
 //# sourceMappingURL=index.js.map

dist/lib/suggest.d.ts

@@ -1,2 +0,2 @@
-import { TrieNode } from './TrieNode';
+import { TrieRoot } from './TrieNode';
 import { CompoundWordsMethod } from './walker';
@@ -10,7 +10,7 @@ export { CompoundWordsMethod, JOIN_SEPARATOR, WORD_SEPARATOR } from './walker';
 }
-export interface SuggestionIterator extends Generator<SuggestionResult, any, MaxCost | undefined> {
+export interface SuggestionIterator extends Generator<SuggestionResult, undefined, MaxCost | undefined> {
 }
-export declare function suggest(root: TrieNode, word: string, maxNumSuggestions?: number, compoundMethod?: CompoundWordsMethod, numChanges?: number): SuggestionResult[];
-export declare function genSuggestions(root: TrieNode, word: string, compoundMethod?: CompoundWordsMethod): SuggestionIterator;
-export declare function genCompoundableSuggestions(root: TrieNode, word: string, compoundMethod: CompoundWordsMethod): SuggestionIterator;
+export declare function suggest(root: TrieRoot, word: string, maxNumSuggestions?: number, compoundMethod?: CompoundWordsMethod, numChanges?: number): SuggestionResult[];
+export declare function genSuggestions(root: TrieRoot, word: string, compoundMethod?: CompoundWordsMethod): SuggestionIterator;
+export declare function genCompoundableSuggestions(root: TrieRoot, word: string, compoundMethod: CompoundWordsMethod): SuggestionIterator;
 export declare function compSuggestionResults(a: SuggestionResult, b: SuggestionResult): number;
@@ -24,3 +24,4 @@ export interface SuggestionCollector {
     readonly maxNumSuggestions: number;
+    readonly includesTies: boolean;
 }
-export declare function suggestionCollector(wordToMatch: string, maxNumSuggestions: number, filter?: (word: string) => boolean, changeLimit?: number): SuggestionCollector;
+export declare function suggestionCollector(wordToMatch: string, maxNumSuggestions: number, filter?: (word: string, cost: number) => boolean, changeLimit?: number, includeTies?: boolean): SuggestionCollector;

dist/lib/suggest.js

@@ -20,3 +20,4 @@ "use strict";
 const regexSeparator = new RegExp(regexQuote(walker_1.JOIN_SEPARATOR) + '|' + regexQuote(walker_1.WORD_SEPARATOR), 'g');
-const wordLengthCost = [0, 50, 25, 0];
+const wordLengthCost = [0, 50, 25, 5, 0];
+const extraWordsCost = 5;
 function suggest(root, word, maxNumSuggestions = defaultMaxNumberSuggestions, compoundMethod = walker_1.CompoundWordsMethod.NONE, numChanges = maxNumChanges) {
@@ -30,2 +31,3 @@ const collector = suggestionCollector(word, maxNumSuggestions, undefined, numChanges);
     yield* genCompoundableSuggestions(root, word, compoundMethod);
+    return undefined;
 }
@@ -42,6 +44,6 @@ exports.genSuggestions = genSuggestions;
     const mx = x.length - 1;
-    const specialCosts = {
-        [walker_1.WORD_SEPARATOR]: insertSpaceCost,
-        [walker_1.JOIN_SEPARATOR]: insertSpaceCost,
-    };
+    const specialCosts = new Map([
+        [walker_1.WORD_SEPARATOR, insertSpaceCost],
+        [walker_1.JOIN_SEPARATOR, insertSpaceCost],
+    ]);
     let costLimit = Math.min(bc * word.length / 2, bc * maxNumChanges);
@@ -56,3 +58,3 @@ let a = 0;
     stack[0] = { a, b };
-    let hint = word.slice(a);
+    const hint = word;
     const i = walker_1.hintedWalker(root, compoundMethod, hint);
@@ -102,3 +104,3 @@ let goDeeper = true;
         const c = bc - d;
-        const ci = c + (specialCosts[w] || 0);
+        const ci = c + (specialCosts.get(w) || 0);
         // Setup first column
@@ -153,6 +155,4 @@ matrix[d] = matrix[d] || [];
         // Adjust the range between a and b
-        for (; b > a && matrix[d][b] > costLimit; b -= 1) {
-        }
-        for (; a < b && matrix[d][a] > costLimit; a += 1) {
-        }
+        for (; b > a && matrix[d][b] > costLimit; b -= 1) { }
+        for (; a < b && matrix[d][a] > costLimit; a += 1) { }
         b = Math.min(b + 1, mx);
@@ -167,6 +167,4 @@ stack[d] = { a, b };
         goDeeper = (min <= costLimit);
-        hint = word.slice(a, b);
     }
-    // console.log(`tag size: ${historyTags.size}, history size: ${history.length}`);
-    // console.log(history.map((r, i) => `${i} ${r.cost} ${r.word}`).join('\n'));
+    return undefined;
 }
@@ -181,5 +179,6 @@ exports.genCompoundableSuggestions = genCompoundableSuggestions;
 exports.compSuggestionResults = compSuggestionResults;
-function suggestionCollector(wordToMatch, maxNumSuggestions, filter = () => true, changeLimit = maxNumChanges) {
+function suggestionCollector(wordToMatch, maxNumSuggestions, filter = () => true, changeLimit = maxNumChanges, includeTies = false) {
     const sugs = new Map();
     let maxCost = Math.min(baseCost * wordToMatch.length / 2, baseCost * changeLimit);
+    maxNumSuggestions = Math.max(maxNumSuggestions, 0) || 0;
     function dropMax() {
@@ -191,6 +190,8 @@ if (sugs.size < 2) {
         const sorted = [...sugs.values()].sort(compSuggestionResults);
-        const toRemove = sorted.pop();
-        const maxSug = sorted.pop();
-        sugs.delete(toRemove.word);
-        maxCost = maxSug.cost;
+        let i = maxNumSuggestions - 1;
+        maxCost = sorted[i].cost;
+        for (; i < sorted.length && sorted[i].cost <= maxCost; ++i) { }
+        for (; i < sorted.length; ++i) {
+            sugs.delete(sorted[i].word);
+        }
     }
@@ -201,3 +202,3 @@ function adjustCost(sug) {
             .map(w => wordLengthCost[w.length] || 0)
-            .reduce((a, b) => a + b, 0);
+            .reduce((a, b) => a + b, 0) + (words.length - 1) * extraWordsCost;
         return { word: sug.word, cost: sug.cost + extraCost };
@@ -207,3 +208,3 @@ }
         const { word, cost } = adjustCost(suggestion);
-        if (cost <= maxCost && filter(suggestion.word)) {
+        if (cost <= maxCost && filter(suggestion.word, cost)) {
             if (sugs.has(word)) {
@@ -215,3 +216,3 @@ const known = sugs.get(word);
                 sugs.set(word, { word, cost });
-                if (sugs.size > maxNumSuggestions) {
+                if (cost < maxCost && sugs.size > maxNumSuggestions) {
                     dropMax();
@@ -231,9 +232,17 @@ }
     }
+    function suggestions() {
+        const sorted = [...sugs.values()].sort(compSuggestionResults);
+        if (!includeTies && sorted.length > maxNumSuggestions) {
+            sorted.length = maxNumSuggestions;
+        }
+        return sorted;
+    }
     return {
         collect,
         add: function (suggestion) { collector(suggestion); return this; },
-        get suggestions() { return [...sugs.values()].sort(compSuggestionResults); },
+        get suggestions() { return suggestions(); },
         get maxCost() { return maxCost; },
         get word() { return wordToMatch; },
         get maxNumSuggestions() { return maxNumSuggestions; },
+        includesTies: includeTies,
     };
@@ -240,0 +249,0 @@ }

dist/lib/trie.d.ts

 import { Sequence } from 'gensequence';
-import { TrieNode } from './TrieNode';
+import { TrieNode, TrieOptions, TrieRoot, PartialTrieOptions } from './TrieNode';
 import { SuggestionCollector, SuggestionResult, CompoundWordsMethod } from './suggest';
 import { WalkerIterator } from './walker';
-export interface TrieOptions {
-    compoundCharacter: string;
-    stripCaseAndAccentsPrefix: string;
-    forbiddenWordPrefix: string;
-}
-export declare const defaultTrieOptions: TrieOptions;
-declare type Optional<T> = {
-    [key in keyof T]?: T[key];
-};
-declare type PartialTrieOptions = Optional<TrieOptions> | undefined;
+export { COMPOUND_FIX, OPTIONAL_COMPOUND_FIX, CASE_INSENSITIVE_PREFIX, FORBID_PREFIX, } from './constants';
+export { TrieOptions, PartialTrieOptions } from './TrieNode';
+export { defaultTrieOptions } from './constants';
+/** @deprecated */
+export declare const COMPOUND = "+";
+/** @deprecated */
+export declare const OPTIONAL_COMPOUND = "*";
+/** @deprecated */
+export declare const NORMALIZED = "~";
+/** @deprecated */
+export declare const FORBID = "!";
 export declare class Trie {
-    readonly root: TrieNode;
+    readonly root: TrieRoot;
     private count?;
     private _options;
-    constructor(root: TrieNode, count?: number | undefined, options?: PartialTrieOptions);
+    readonly isLegacy: boolean;
+    private hasForbidden;
+    constructor(root: TrieRoot, count?: number | undefined);
     /**
@@ -24,8 +27,22 @@ * Number of words in the Trie
     size(): number;
+    isSizeKnown(): boolean;
     get options(): TrieOptions;
     find(text: string, minCompoundLength?: boolean | number): TrieNode | undefined;
-    findCompound(text: string, minCompoundLength?: number, minLength?: number): TrieNode | undefined;
+    findCompound(text: string, minCompoundLength?: number): TrieNode | undefined;
     findExact(text: string): TrieNode | undefined;
-    has(word: string, minCompoundLength?: boolean | number): boolean;
+    has(word: string, minLegacyCompoundLength?: boolean | number): boolean;
     /**
+     * Determine if a word is in the dictionary.
+     * @param word - the exact word to search for - must be normalized - for non-case sensitive
+     *      searches, word must be lower case with accents removed.
+     * @param caseSensitive - false means searching a dictionary where the words were normalized to lower case and accents removed.
+     * @returns true if the word was found and is not forbidden.
+     */
+    hasWord(word: string, caseSensitive: boolean): boolean;
+    /**
+     * Determine if a word is in the forbidden word list.
+     * @param word word to lookup.
+     */
+    isForbiddenWord(word: string): boolean;
+    /**
      * Provides an ordered sequence of words with the prefix of text.
@@ -64,4 +81,5 @@ */
     insert(word: string): this;
+    private getSuggestRoot;
+    private calcIsLegacy;
     static create(words: Iterable<string> | IterableIterator<string>, options?: PartialTrieOptions): Trie;
 }
-export {};

dist/lib/trie.js

@@ -7,17 +7,27 @@ "use strict";
 const walker_1 = require("./walker");
-const _defaultTrieOptions = {
-    compoundCharacter: '+',
-    stripCaseAndAccentsPrefix: '~',
-    forbiddenWordPrefix: '!'
-};
-exports.defaultTrieOptions = Object.freeze(_defaultTrieOptions);
-function mergeOptionalWithDefaults(options) {
-    const { compoundCharacter = exports.defaultTrieOptions.compoundCharacter, stripCaseAndAccentsPrefix = exports.defaultTrieOptions.stripCaseAndAccentsPrefix, forbiddenWordPrefix = exports.defaultTrieOptions.forbiddenWordPrefix, } = options || {};
-    return { compoundCharacter, stripCaseAndAccentsPrefix, forbiddenWordPrefix };
-}
+const constants_1 = require("./constants");
+const find_1 = require("./find");
+var constants_2 = require("./constants");
+exports.COMPOUND_FIX = constants_2.COMPOUND_FIX;
+exports.OPTIONAL_COMPOUND_FIX = constants_2.OPTIONAL_COMPOUND_FIX;
+exports.CASE_INSENSITIVE_PREFIX = constants_2.CASE_INSENSITIVE_PREFIX;
+exports.FORBID_PREFIX = constants_2.FORBID_PREFIX;
+var constants_3 = require("./constants");
+exports.defaultTrieOptions = constants_3.defaultTrieOptions;
+/** @deprecated */
+exports.COMPOUND = constants_1.COMPOUND_FIX;
+/** @deprecated */
+exports.OPTIONAL_COMPOUND = constants_1.OPTIONAL_COMPOUND_FIX;
+/** @deprecated */
+exports.NORMALIZED = constants_1.CASE_INSENSITIVE_PREFIX;
+/** @deprecated */
+exports.FORBID = constants_1.FORBID_PREFIX;
+const defaultLegacyMinCompoundLength = 3;
 class Trie {
-    constructor(root, count, options) {
+    constructor(root, count) {
         this.root = root;
         this.count = count;
-        this._options = mergeOptionalWithDefaults(options);
+        this._options = util_1.mergeOptionalWithDefaults(root);
+        this.isLegacy = this.calcIsLegacy();
+        this.hasForbidden = !!root.c.get(root.forbiddenWordPrefix);
     }
@@ -29,5 +39,8 @@ /**
         var _a;
-        this.count = (_a = this.count, (_a !== null && _a !== void 0 ? _a : util_1.countWords(this.root)));
+        this.count = (_a = this.count) !== null && _a !== void 0 ? _a : util_1.countWords(this.root);
         return this.count;
     }
+    isSizeKnown() {
+        return this.count !== undefined;
+    }
     get options() {
@@ -40,31 +53,41 @@ return this._options;
     }
-    findCompound(text, minCompoundLength = 3, minLength = 0) {
-        let n = this.root;
-        let p;
-        let q;
-        for (p = 0; n && n.c && p < text.length; p = q) {
-            n = n.c.get(text[p]);
-            q = p + 1;
-            if (n && n.f && q < text.length && q >= minCompoundLength) {
-                const r = this.findCompound(text.slice(q), minCompoundLength, minCompoundLength);
-                if (r && r.f) {
-                    return r;
-                }
-            }
-        }
-        return p === text.length && p >= minLength ? n : undefined;
+    findCompound(text, minCompoundLength = defaultLegacyMinCompoundLength) {
+        const r = find_1.findLegacyCompoundNode(this.root, text, minCompoundLength);
+        return r.node;
     }
     findExact(text) {
-        let n = this.root;
-        let p;
-        for (p = 0; n && n.c && p < text.length; ++p) {
-            n = n.c.get(text[p]);
+        const r = find_1.findCompoundNode(this.root, text, this.options.compoundCharacter);
+        return r.node;
+    }
+    has(word, minLegacyCompoundLength) {
+        const f = find_1.findCompoundWord(this.root, word, this.options.compoundCharacter);
+        if (!!f.found)
+            return true;
+        if (minLegacyCompoundLength) {
+            const len = minLegacyCompoundLength !== true ? minLegacyCompoundLength : defaultLegacyMinCompoundLength;
+            return !!find_1.findLegacyCompoundWord(this.root, word, len).found;
         }
-        return p === text.length ? n : undefined;
+        return false;
     }
-    has(word, minCompoundLength) {
-        const n = this.find(word, minCompoundLength);
-        return !!n && util_1.isWordTerminationNode(n);
+    /**
+     * Determine if a word is in the dictionary.
+     * @param word - the exact word to search for - must be normalized - for non-case sensitive
+     *      searches, word must be lower case with accents removed.
+     * @param caseSensitive - false means searching a dictionary where the words were normalized to lower case and accents removed.
+     * @returns true if the word was found and is not forbidden.
+     */
+    hasWord(word, caseSensitive) {
+        var _a;
+        const root = !caseSensitive ? ((_a = this.root.c) === null || _a === void 0 ? void 0 : _a.get(this.options.stripCaseAndAccentsPrefix)) || this.root : this.root;
+        const f = find_1.findCompoundWord(root, word, this.options.compoundCharacter);
+        return !!f.found;
     }
     /**
+     * Determine if a word is in the forbidden word list.
+     * @param word word to lookup.
+     */
+    isForbiddenWord(word) {
+        return this.hasForbidden && find_1.isForbiddenWord(this.root, word, this.options.forbiddenWordPrefix);
+    }
+    /**
      * Provides an ordered sequence of words with the prefix of text.
@@ -74,4 +97,6 @@ */
         const n = this.find(text);
+        const compoundChar = this.options.compoundCharacter;
+        const subNodes = util_1.iteratorTrieWords(n || {}).filter(w => w[w.length - 1] !== compoundChar).map(suffix => text + suffix);
         return gensequence_1.genSequence(n && util_1.isWordTerminationNode(n) ? [text] : [])
-            .concat(n ? util_1.iteratorTrieWords(n).map(suffix => text + suffix) : []);
+            .concat(subNodes);
     }
@@ -95,3 +120,4 @@ /**
     suggestWithCost(text, maxNumSuggestions, compoundMethod, numChanges) {
-        return suggest_1.suggest(this.root, text, maxNumSuggestions, compoundMethod, numChanges);
+        return suggest_1.suggest(this.getSuggestRoot(true), text, maxNumSuggestions, compoundMethod, numChanges)
+            .filter(sug => !this.isForbiddenWord(sug.word));
     }
@@ -104,3 +130,15 @@ /**
     genSuggestions(collector, compoundMethod) {
-        collector.collect(suggest_1.genSuggestions(this.root, collector.word, compoundMethod));
+        const filter = (sug) => !this.isForbiddenWord(sug.word);
+        const suggestions = suggest_1.genSuggestions(this.getSuggestRoot(true), collector.word, compoundMethod);
+        function* filteredSuggestions() {
+            let maxCost = collector.maxCost;
+            let ir;
+            while (!(ir = suggestions.next(maxCost)).done) {
+                if (ir.value !== undefined && filter(ir.value)) {
+                    maxCost = yield ir.value;
+                }
+            }
+            return undefined;
+        }
+        collector.collect(filteredSuggestions());
     }
@@ -124,6 +162,21 @@ /**
     }
+    getSuggestRoot(caseSensitive) {
+        var _a;
+        const root = !caseSensitive && ((_a = this.root.c) === null || _a === void 0 ? void 0 : _a.get(this._options.stripCaseAndAccentsPrefix)) || this.root;
+        if (!root.c)
+            return Object.assign({ c: new Map() }, this._options);
+        const blockNodes = new Set([
+            this._options.forbiddenWordPrefix,
+            this._options.stripCaseAndAccentsPrefix,
+        ]);
+        return Object.assign({ c: new Map([...root.c].filter(([k]) => !blockNodes.has(k))) }, this._options);
+    }
+    calcIsLegacy() {
+        const c = this.root.c;
+        return !((c === null || c === void 0 ? void 0 : c.get(this._options.compoundCharacter)) || (c === null || c === void 0 ? void 0 : c.get(this._options.stripCaseAndAccentsPrefix)) || (c === null || c === void 0 ? void 0 : c.get(this._options.forbiddenWordPrefix)));
+    }
     static create(words, options) {
-        const root = util_1.createTriFromList(words);
+        const root = util_1.createTriFromList(words, options);
         util_1.orderTrie(root);
-        return new Trie(root, undefined, options);
+        return new Trie(root, undefined);
     }
@@ -130,0 +183,0 @@ }

dist/lib/TrieBuilder.d.ts

@@ -1,3 +0,15 @@
-import { Trie } from './trie';
-export declare function buildTrie(words: Iterable<string>): Trie;
+import { Trie, PartialTrieOptions, TrieOptions } from './trie';
+/**
+ * Builds an optimized Trie from a Iterable<string>. It attempts to reduce the size of the trie
+ * by finding common endings.
+ * @param words Iterable set of words -- no processing is done on the words, they are inserted as is.
+ * @param trieOptions options for the Trie
+ */
+export declare function buildTrie(words: Iterable<string>, trieOptions?: PartialTrieOptions): Trie;
+/**
+ * Builds a Trie from a Iterable<string>. NO attempt a reducing the size of the Trie is done.
+ * @param words Iterable set of words -- no processing is done on the words, they are inserted as is.
+ * @param trieOptions options for the Trie
+ */
+export declare function buildTrieFast(words: Iterable<string>, trieOptions?: PartialTrieOptions): Trie;
 export declare class TrieBuilder {
@@ -12,3 +24,4 @@ private count;
     private tails;
-    constructor(words?: Iterable<string>);
+    trieOptions: TrieOptions;
+    constructor(words?: Iterable<string>, trieOptions?: PartialTrieOptions);
     private set _root(value);
@@ -15,0 +28,0 @@ private get _root();

dist/lib/TrieBuilder.js

@@ -5,8 +5,25 @@ "use strict";
 const consolidate_1 = require("./consolidate");
-function buildTrie(words) {
-    return new TrieBuilder(words).build();
+const util_1 = require("./util");
+/**
+ * Builds an optimized Trie from a Iterable<string>. It attempts to reduce the size of the trie
+ * by finding common endings.
+ * @param words Iterable set of words -- no processing is done on the words, they are inserted as is.
+ * @param trieOptions options for the Trie
+ */
+function buildTrie(words, trieOptions) {
+    return new TrieBuilder(words, trieOptions).build();
 }
 exports.buildTrie = buildTrie;
+/**
+ * Builds a Trie from a Iterable<string>. NO attempt a reducing the size of the Trie is done.
+ * @param words Iterable set of words -- no processing is done on the words, they are inserted as is.
+ * @param trieOptions options for the Trie
+ */
+function buildTrieFast(words, trieOptions) {
+    const root = util_1.createTriFromList(words, trieOptions);
+    return new trie_1.Trie(root, undefined);
+}
+exports.buildTrieFast = buildTrieFast;
 class TrieBuilder {
-    constructor(words) {
+    constructor(words, trieOptions) {
         this.count = 0;
@@ -23,2 +40,3 @@ this.signatures = new Map();
         this.cached.set(this._eow, this.count++);
+        this.trieOptions = Object.freeze(util_1.mergeOptionalWithDefaults(trieOptions));
         if (words) {
@@ -32,3 +50,3 @@ this.insert(words);
     get _root() {
-        return this.lastPath[0].n;
+        return util_1.trieNodeToRoot(this.lastPath[0].n, this.trieOptions);
     }
@@ -86,3 +104,3 @@ signature(n) {
             return;
-        const t = (_a = this.transforms.get(src), (_a !== null && _a !== void 0 ? _a : new Map()));
+        const t = (_a = this.transforms.get(src)) !== null && _a !== void 0 ? _a : new Map();
         t.set(s, result);
@@ -95,3 +113,3 @@ this.transforms.set(src, t);
             if (!node.c || Object.isFrozen(node)) {
-                node = Object.assign(Object.assign({}, node), { c: new Map((_b = node.c, (_b !== null && _b !== void 0 ? _b : []))) });
+                node = Object.assign(Object.assign({}, node), { c: new Map((_b = node.c) !== null && _b !== void 0 ? _b : []) });
             }
@@ -147,7 +165,6 @@ node.c.set(head, child);
     insertWord(word) {
-        var _a;
         let d = 1;
         for (const s of word.split('')) {
             const p = this.lastPath[d];
-            if (((_a = p) === null || _a === void 0 ? void 0 : _a.s) !== s)
+            if ((p === null || p === void 0 ? void 0 : p.s) !== s)
                 break;
@@ -186,3 +203,3 @@ d++;
     reset() {
-        this._root = { f: undefined, c: undefined };
+        this._root = util_1.createTrieRoot(this.trieOptions);
         this.cached.clear();
@@ -189,0 +206,0 @@ this.signatures.clear();

dist/lib/TrieNode.d.ts

@@ -8,1 +8,10 @@ export declare const FLAG_WORD = 1;
 }
+export interface TrieOptions {
+    compoundCharacter: string;
+    stripCaseAndAccentsPrefix: string;
+    forbiddenWordPrefix: string;
+}
+export declare type PartialTrieOptions = Partial<TrieOptions> | undefined;
+export interface TrieRoot extends TrieOptions {
+    c: ChildMap;
+}

dist/lib/util.d.ts

 import { Sequence } from 'gensequence';
-import { TrieNode } from './TrieNode';
+import { TrieNode, TrieRoot, PartialTrieOptions, TrieOptions } from './TrieNode';
 import { YieldResult } from './walker';
@@ -20,4 +20,5 @@ export { YieldResult } from './walker';
 export declare function iteratorTrieWords(node: TrieNode): Sequence<string>;
-export declare function createRoot(): TrieNode;
-export declare function createTriFromList(words: Iterable<string>): TrieNode;
+export declare function mergeOptionalWithDefaults(options: PartialTrieOptions): TrieOptions;
+export declare function createTrieRoot(options: PartialTrieOptions): TrieRoot;
+export declare function createTriFromList(words: Iterable<string>, options?: PartialTrieOptions): TrieRoot;
 export declare function has(node: TrieNode, word: string): boolean;
@@ -28,1 +29,12 @@ export declare function findNode(node: TrieNode, prefix: string): TrieNode | undefined;
 export declare function isCircular(root: TrieNode): boolean;
+/**
+ * Creates a new object of type T based upon the field values from `value`.
+ * n[k] = value[k] ?? default[k] where k must be a field in default.
+ * Note: it will remove fields not in defaultValue!
+ * @param value
+ * @param defaultValue
+ */
+export declare function mergeDefaults<T>(value: Partial<T> | undefined, defaultValue: T): T;
+export declare function trieNodeToRoot(node: TrieNode, options: PartialTrieOptions): TrieRoot;
+export declare const normalizeWord: (text: string) => string;
+export declare const normalizeWordToLowercase: (text: string) => string;

dist/lib/util.js

@@ -6,2 +6,3 @@ "use strict";
 const walker_1 = require("./walker");
+const constants_1 = require("./constants");
 function insert(text, node = {}) {
@@ -54,8 +55,13 @@ if (text.length) {
 exports.iteratorTrieWords = iteratorTrieWords;
-function createRoot() {
-    return {};
+function mergeOptionalWithDefaults(options) {
+    return mergeDefaults(options, constants_1.defaultTrieOptions);
 }
-exports.createRoot = createRoot;
-function createTriFromList(words) {
-    const root = createRoot();
+exports.mergeOptionalWithDefaults = mergeOptionalWithDefaults;
+function createTrieRoot(options) {
+    const fullOptions = mergeOptionalWithDefaults(options);
+    return Object.assign(Object.assign({}, fullOptions), { c: new Map() });
+}
+exports.createTrieRoot = createTrieRoot;
+function createTriFromList(words, options) {
+    const root = createTrieRoot(options);
     for (const word of words) {
@@ -155,2 +161,29 @@ if (word.length) {
 exports.isCircular = isCircular;
+/**
+ * Creates a new object of type T based upon the field values from `value`.
+ * n[k] = value[k] ?? default[k] where k must be a field in default.
+ * Note: it will remove fields not in defaultValue!
+ * @param value
+ * @param defaultValue
+ */
+function mergeDefaults(value, defaultValue) {
+    const result = Object.assign({}, defaultValue);
+    const allowedKeys = new Set(Object.keys(defaultValue));
+    if (value) {
+        for (const [k, v] of Object.entries(value)) {
+            if (allowedKeys.has(k)) {
+                result[k] = v !== null && v !== void 0 ? v : result[k];
+            }
+        }
+    }
+    return result;
+}
+exports.mergeDefaults = mergeDefaults;
+function trieNodeToRoot(node, options) {
+    const newOptions = mergeOptionalWithDefaults(options);
+    return Object.assign(Object.assign({}, newOptions), { c: node.c || new Map() });
+}
+exports.trieNodeToRoot = trieNodeToRoot;
+exports.normalizeWord = (text) => text.normalize();
+exports.normalizeWordToLowercase = (text) => text.toLowerCase().normalize('NFD').replace(/[\u0300-\u036f]/g, '');
 //# sourceMappingURL=util.js.map

dist/lib/walker.d.ts

@@ -1,2 +0,2 @@
-import { TrieNode } from './TrieNode';
+import { TrieNode, TrieRoot } from './TrieNode';
 export declare const JOIN_SEPARATOR: string;
@@ -39,2 +39,2 @@ export declare const WORD_SEPARATOR: string;
  */
-export declare function hintedWalker(root: TrieNode, compoundingMethod: CompoundWordsMethod | undefined, hint: string): HintedWalkerIterator;
+export declare function hintedWalker(root: TrieRoot, compoundingMethod: CompoundWordsMethod | undefined, hint: string): HintedWalkerIterator;

dist/lib/walker.js

@@ -63,17 +63,33 @@ "use strict";
 function* hintedWalker(root, compoundingMethod = CompoundWordsMethod.NONE, hint) {
+    const baseRoot = { c: new Map([...root.c].filter(n => n[0] !== root.compoundCharacter)) };
     const roots = {
         [CompoundWordsMethod.NONE]: [],
-        [CompoundWordsMethod.JOIN_WORDS]: [[exports.JOIN_SEPARATOR, root]],
-        [CompoundWordsMethod.SEPARATE_WORDS]: [[exports.WORD_SEPARATOR, root]],
+        [CompoundWordsMethod.JOIN_WORDS]: [[exports.JOIN_SEPARATOR, baseRoot]],
+        [CompoundWordsMethod.SEPARATE_WORDS]: [[exports.WORD_SEPARATOR, baseRoot]],
     };
-    const hints = new Set(hint.slice(0, 5));
-    function* children(n) {
+    const compoundCharacter = root.compoundCharacter;
+    function* children(n, hintOffset) {
         if (n.c) {
+            const h = hint.slice(hintOffset, hintOffset + 3) + hint.slice(Math.max(0, hintOffset - 2), hintOffset);
+            const hints = new Set(h);
             // First yield the hints
-            yield* [...hints].filter(a => n.c.has(a)).map(a => [a, n.c.get(a)]);
+            yield* [...hints]
+                .filter(a => n.c.has(a))
+                .map(letter => ({ letter, node: n.c.get(letter), hintOffset: hintOffset + 1 }));
+            // We don't want to suggest the compound character.
+            hints.add(compoundCharacter);
             // Then yield everything else.
-            yield* [...n.c].filter(a => !hints.has(a[0]));
+            yield* [...n.c]
+                .filter(a => !hints.has(a[0]))
+                .map(([letter, node]) => ({ letter, node, hintOffset: hintOffset + 1 }));
+            if (n.c.has(compoundCharacter)) {
+                const compoundRoot = root.c.get(compoundCharacter);
+                if (compoundRoot) {
+                    yield* children(compoundRoot, hintOffset);
+                }
+            }
         }
         if (n.f) {
-            yield* roots[compoundingMethod];
+            yield* [...roots[compoundingMethod]]
+                .map(([letter, node]) => ({ letter, node, hintOffset }));
         }
@@ -84,7 +100,7 @@ }
     let baseText = '';
-    stack[depth] = children(root);
+    stack[depth] = children(baseRoot, depth);
     let ir;
     while (depth >= 0) {
         while (!(ir = stack[depth].next()).done) {
-            const [char, node] = ir.value;
+            const { letter: char, node, hintOffset } = ir.value;
             const text = baseText + char;
@@ -95,3 +111,3 @@ const hinting = (yield { text, node, depth });
                 baseText = text;
-                stack[depth] = children(node);
+                stack[depth] = children(node, hintOffset);
             }
@@ -98,0 +114,0 @@ }

package.json

 {
   "name": "cspell-trie-lib",
-  "version": "4.1.8",
+  "version": "5.0.1-alpha.3",
   "description": "Trie Data Structure to support cspell.",
@@ -36,3 +36,3 @@ "main": "dist/index.js",
   "dependencies": {
-    "gensequence": "^3.0.3"
+    "gensequence": "^3.1.1"
   },
@@ -60,3 +60,3 @@ "nyc": {
   },
-  "gitHead": "cbe103ba540d0ec7af38876372722802f80f4098"
+  "gitHead": "efceae2801b83dd00d976926ddaf29069ac97a23"
 }

New alerts

License Policy Violation

License

This package is not allowed per your license policy. Review the package's license to ensure compliance.

Found 1 instance in 1 package

No v1

Quality

Package is not semver >=1. This means it is not stable and does not support ^ ranges.

Found 1 instance in 1 package

Fixed alerts

License Policy Violation

License

This package is not allowed per your license policy. Review the package's license to ensure compliance.

Found 1 instance in 1 package

Improved metrics

Total package byte prevSize

104589

increased by33.2%

Number of package files

48

increased by14.29%

Lines of code

2706

increased by27.76%

Worsened metrics

Number of low quality alerts

1

increased byInfinity%

Dependency changes

• Updatedgensequence@^3.1.1