hunspell-reader - npm Package Compare versions

Comparing version 3.1.4 to 3.1.5

dist/aff.d.ts

@@ -126,3 +126,3 @@ import { Converter } from './converter';
      */
-    isCompondForbidden?: boolean;
+    isCompoundForbidden?: boolean;
     /**
@@ -154,3 +154,3 @@ * WARN flag
      * This flag signs forbidden word form. Because affixed forms are also forbidden, we can subtract a subset from set of the
-     * accepted affixed and compound words. Note: usefull to forbid erroneous words, generated by the compounding mechanism.
+     * accepted affixed and compound words. Note: useful to forbid erroneous words, generated by the compounding mechanism.
      */
@@ -178,2 +178,3 @@ isForbiddenWord?: boolean;
     rulesApplied: string;
+    /** prefix + base + suffix == word */
     base: string;
@@ -184,2 +185,4 @@ suffix: string;
 }
+/** The `word` field in a Converted AffWord has been converted using the OCONV mapping */
+export declare type ConvertedAffWord = AffWord;
 export declare class Aff {
@@ -196,5 +199,6 @@ affInfo: AffInfo;
      * Takes a line from a hunspell.dic file and applies the rules found in the aff file.
+     * For performance reasons, only the `word` field is mapped with OCONV.
      * @param {string} line - the line from the .dic file.
      */
-    applyRulesToDicEntry(line: string, maxDepth?: number): AffWord[];
+    applyRulesToDicEntry(line: string, maxDepth?: number): ConvertedAffWord[];
     /**
@@ -217,2 +221,8 @@ * @internal
 export declare function flagsToString(flags: AffWordFlags): string;
-export declare function asAffWord(word: string, rules?: string): AffWord;
+export declare function asAffWord(word: string, rules?: string, flags?: AffWordFlags): AffWord;
+export declare function compareAff(a: AffWord, b: AffWord): 1 | -1 | 0;
+/**
+ * Returns a filter function that will filter adjacent AffWords
+ * It compares the word and the flags.
+ */
+export declare function filterAff(): (t: AffWord) => boolean;

dist/aff.js

@@ -8,3 +8,2 @@ "use strict";
 const util_1 = require("./util");
-// cSpell:enableCompoundWords
 const log = false;
@@ -28,2 +27,3 @@ const DefaultMaxDepth = 5;
      * Takes a line from a hunspell.dic file and applies the rules found in the aff file.
+     * For performance reasons, only the `word` field is mapped with OCONV.
      * @param {string} line - the line from the .dic file.
@@ -36,6 +36,6 @@ */
         const results = this.applyRulesToWord(asAffWord(word, rules), maxSuffixDepth)
-            .map(affWord => (Object.assign(Object.assign({}, affWord), { word: this._oConv.convert(affWord.word) })))
-            .filter(util_1.uniqueFilter(10000, a => a.word));
-        results.sort((a, b) => a.word < b.word ? -1 : 1);
-        return results;
+            .map(affWord => (Object.assign(Object.assign({}, affWord), { word: this._oConv.convert(affWord.word) })));
+        results.sort(compareAff);
+        const filtered = results.filter(filterAff());
+        return filtered;
     }
@@ -68,7 +68,7 @@ /**
         }
-        const combineableRules = affixRules
+        const combinableRules = affixRules
             .filter(rule => rule.type === 'SFX')
             .filter(rule => rule.combinable === true)
             .map(({ id }) => id);
-        const combinableSfx = this.joinRules(combineableRules);
+        const combinableSfx = this.joinRules(combinableRules);
         const r = affixRules
@@ -160,2 +160,7 @@ .map(affix => this.applyAffixToWord(affix, affWord, combinableSfx))
 exports.Aff = Aff;
+function signature(aff) {
+    const { word, flags } = aff;
+    const sig = Object.keys(flags).map(f => flagToStringMap[f]).sort().join('|');
+    return word + '|' + sig;
+}
 function processRules(affInfo) {
@@ -183,3 +188,3 @@ const sfxRules = gensequence_1.genSequence(affInfo.SFX || []).map(([, sfx]) => sfx).map(sfx => ({ id: sfx.id, type: 'sfx', sfx }));
     COMPOUNDPERMITFLAG: { isCompoundPermitted: true },
-    COMPOUNDFORBIDFLAG: { isCompondForbidden: true },
+    COMPOUNDFORBIDFLAG: { isCompoundForbidden: true },
     ONLYINCOMPOUND: { isOnlyAllowedInCompound: true },
@@ -199,3 +204,3 @@ };
     isNeedAffix: 'A',
-    isCompondForbidden: '-',
+    isCompoundForbidden: '-',
 };
@@ -227,3 +232,3 @@ function logAffWord(affWord, message) {
 exports.flagsToString = flagsToString;
-function asAffWord(word, rules = '') {
+function asAffWord(word, rules = '', flags = {}) {
     return {
@@ -235,8 +240,25 @@ word,
         rulesApplied: '',
-        rules: rules || '',
-        flags: {},
-        dic: word + '/' + rules
+        rules,
+        flags,
+        dic: rules ? word + '/' + rules : word,
     };
 }
 exports.asAffWord = asAffWord;
+function compareAff(a, b) {
+    if (a.word !== b.word) {
+        return a.word < b.word ? -1 : 1;
+    }
+    const sigA = signature(a);
+    const sigB = signature(b);
+    return sigA < sigB ? -1 : sigA > sigB ? 1 : 0;
+}
+exports.compareAff = compareAff;
+/**
+ * Returns a filter function that will filter adjacent AffWords
+ * It compares the word and the flags.
+ */
+function filterAff() {
+    return util_1.filterOrderedList((a, b) => a.word !== b.word || signature(a) !== signature(b));
+}
+exports.filterAff = filterAff;
 //# sourceMappingURL=aff.js.map

dist/IterableHunspellReader.d.ts

@@ -6,3 +6,5 @@ import { Aff } from './aff';
 export interface HunspellSrcData {
+    /** The Aff rules to use with the dictionary entries */
     aff: Aff;
+    /** the hunspell dictionary entries complete with affix flags */
     dic: string[];
@@ -17,2 +19,4 @@ }
     get maxDepth(): number;
+    /** the number of .dic entries */
+    get size(): number;
     /**
@@ -27,2 +31,6 @@ * @internal
     iterateWords(): Iterable<string>;
+    /**
+     * Iterator for all the words in the dictionary. The words are in the order found in the .dic after the
+     * transformations have been applied. No filtering is done based upon the AFF flags.
+     */
     [Symbol.iterator](): Sequence<string>;
@@ -33,5 +41,5 @@ /**
      * @param tapPreApplyRules -- optional function to be called before rules are applied to a word.
-     *                            It is mostly used for monitoring progress.
+     *                            It is mostly used for monitoring progress in combination with `size`.
      */
-    seqAffWords(tapPreApplyRules?: (w: string) => any, maxDepth?: number): Sequence<import("./aff").AffWord>;
+    seqAffWords(tapPreApplyRules?: (w: string, index: number) => any, maxDepth?: number): Sequence<import("./aff").AffWord>;
     /**
@@ -47,1 +55,2 @@ * @internal
 }
+export declare function createMatchingWordsFilter(): (t: string) => boolean;

dist/IterableHunspellReader.js

@@ -16,2 +16,3 @@ "use strict";
 const iconv_lite_1 = require("iconv-lite");
+const util_1 = require("./util");
 const defaultEncoding = 'UTF-8';
@@ -32,2 +33,6 @@ class IterableHunspellReader {
     }
+    /** the number of .dic entries */
+    get size() {
+        return this.src.dic.length;
+    }
     /**
@@ -52,2 +57,6 @@ * @internal
     }
+    /**
+     * Iterator for all the words in the dictionary. The words are in the order found in the .dic after the
+     * transformations have been applied. No filtering is done based upon the AFF flags.
+     */
     [Symbol.iterator]() { return this.seqWords(); }
@@ -58,7 +67,8 @@ /**
      * @param tapPreApplyRules -- optional function to be called before rules are applied to a word.
-     *                            It is mostly used for monitoring progress.
+     *                            It is mostly used for monitoring progress in combination with `size`.
      */
     seqAffWords(tapPreApplyRules, maxDepth) {
         const seq = gensequence_1.genSequence(this.src.dic);
-        const dicWords = tapPreApplyRules ? seq.map(a => (tapPreApplyRules(a), a)) : seq;
+        let count = 0;
+        const dicWords = tapPreApplyRules ? seq.map(a => (tapPreApplyRules(a, count++), a)) : seq;
         return dicWords
@@ -72,3 +82,3 @@ .filter(a => !!a.trim())
     seqWords() {
-        return this.seqAffWords().map(w => w.word);
+        return this.seqAffWords().map(w => w.word).filter(createMatchingWordsFilter());
     }
@@ -94,2 +104,6 @@ /**
 exports.IterableHunspellReader = IterableHunspellReader;
+function createMatchingWordsFilter() {
+    return util_1.filterOrderedList((a, b) => a !== b);
+}
+exports.createMatchingWordsFilter = createMatchingWordsFilter;
 //# sourceMappingURL=IterableHunspellReader.js.map

dist/util.d.ts

@@ -5,1 +5,6 @@ export declare function hrTimeToSeconds([seconds, nanoseconds]: number[]): number;
 export declare function batch<T>(i: Iterable<T>, size: number): Iterable<T[]>;
+/**
+ * Generate a filter function that will remove adjacent values that compare to falsy;
+ * @param compare function to evaluate if two values are considered the same.
+ */
+export declare function filterOrderedList<T>(compare: (a: T, b: T) => boolean | number): (t: T) => boolean;

dist/util.js

@@ -43,2 +43,15 @@ "use strict";
 exports.batch = batch;
+/**
+ * Generate a filter function that will remove adjacent values that compare to falsy;
+ * @param compare function to evaluate if two values are considered the same.
+ */
+function filterOrderedList(compare) {
+    let last;
+    return function (t) {
+        const r = last === undefined ? (last !== t) : !!compare(last, t);
+        last = r ? t : last;
+        return r;
+    };
+}
+exports.filterOrderedList = filterOrderedList;
 //# sourceMappingURL=util.js.map

package.json

 {
   "name": "hunspell-reader",
-  "version": "3.1.4",
+  "version": "3.1.5",
   "description": "A library for reading Hunspell Dictionary Files",
@@ -5,0 +5,0 @@ "bin": "bin.js",

New alerts

License Policy Violation

License

This package is not allowed per your license policy. Review the package's license to ensure compliance.

Found 1 instance in 1 package

Fixed alerts

License Policy Violation

License

This package is not allowed per your license policy. Review the package's license to ensure compliance.

Found 1 instance in 1 package

Improved metrics

Total package byte prevSize

52796

increased by6.19%

Lines of code

1274

increased by6.08%

No dependency changes