hunspell-reader
Advanced tools
Comparing version 5.9.1-alpha.1 to 5.9.1
@@ -216,3 +216,3 @@ import { Converter } from './converter'; | ||
export declare function asAffWord(word: string, rules?: string, flags?: AffWordFlags): AffWord; | ||
export declare function compareAff(a: AffWord, b: AffWord): 1 | -1 | 0; | ||
export declare function compareAff(a: AffWord, b: AffWord): 0 | 1 | -1; | ||
/** | ||
@@ -219,0 +219,0 @@ * Returns a filter function that will filter adjacent AffWords |
"use strict"; | ||
var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) { | ||
if (k2 === undefined) k2 = k; | ||
Object.defineProperty(o, k2, { enumerable: true, get: function() { return m[k]; } }); | ||
}) : (function(o, m, k, k2) { | ||
if (k2 === undefined) k2 = k; | ||
o[k2] = m[k]; | ||
})); | ||
var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) { | ||
Object.defineProperty(o, "default", { enumerable: true, value: v }); | ||
}) : function(o, v) { | ||
o["default"] = v; | ||
}); | ||
var __importStar = (this && this.__importStar) || function (mod) { | ||
if (mod && mod.__esModule) return mod; | ||
var result = {}; | ||
if (mod != null) for (var k in mod) if (k !== "default" && Object.prototype.hasOwnProperty.call(mod, k)) __createBinding(result, mod, k); | ||
__setModuleDefault(result, mod); | ||
return result; | ||
}; | ||
Object.defineProperty(exports, "__esModule", { value: true }); | ||
exports.debug = exports.filterAff = exports.compareAff = exports.asAffWord = exports.flagsToString = exports.affWordToColoredString = exports.logAffWord = exports.processRules = exports.Aff = void 0; | ||
const util = require("util"); | ||
const util = __importStar(require("util")); | ||
const converter_1 = require("./converter"); | ||
const gensequence_1 = require("gensequence"); | ||
const GS = require("gensequence"); | ||
const GS = __importStar(require("gensequence")); | ||
const util_1 = require("./util"); | ||
@@ -34,3 +53,6 @@ const log = false; | ||
const [word, rules = ''] = lineLeft.split('/', 2); | ||
const results = this.applyRulesToWord(asAffWord(word, rules), maxSuffixDepth).map((affWord) => (Object.assign(Object.assign({}, affWord), { word: this._oConv.convert(affWord.word) }))); | ||
const results = this.applyRulesToWord(asAffWord(word, rules), maxSuffixDepth).map((affWord) => ({ | ||
...affWord, | ||
word: this._oConv.convert(affWord.word), | ||
})); | ||
results.sort(compareAff); | ||
@@ -50,3 +72,3 @@ const filtered = results.filter(filterAff()); | ||
rulesApplied: [acc.rulesApplied, rule.id].join(' '), | ||
flags: Object.assign(Object.assign({}, acc.flags), rule.flags), | ||
flags: { ...acc.flags, ...rule.flags }, | ||
}), { rulesApplied: affWord.rulesApplied, flags: affWord.flags }); | ||
@@ -56,3 +78,3 @@ const rules = this.joinRules(allRules.filter((rule) => !rule.flags).map((rule) => rule.id)); | ||
const wordWithFlags = { word, flags, rulesApplied, rules: '', base, suffix, prefix, dic }; | ||
return [wordWithFlags, ...this.applyAffixesToWord(affixRules, Object.assign(Object.assign({}, wordWithFlags), { rules }), remainingDepth)] | ||
return [wordWithFlags, ...this.applyAffixesToWord(affixRules, { ...wordWithFlags, rules }, remainingDepth)] | ||
.filter(({ flags }) => !flags.isNeedAffix) | ||
@@ -82,3 +104,3 @@ .map((affWord) => logAffWord(affWord, 'applyRulesToWord')); | ||
const matchingSubstitutions = [...affix.substitutionSets.values()].filter((sub) => sub.match.test(word)); | ||
const partialAffWord = Object.assign(Object.assign({}, affWord), { flags, rules: combineRules }); | ||
const partialAffWord = { ...affWord, flags, rules: combineRules }; | ||
return matchingSubstitutions | ||
@@ -223,3 +245,3 @@ .map((sub) => sub.substitutions) | ||
return util | ||
.inspect(Object.assign(Object.assign({}, affWord), { flags: flagsToString(affWord.flags) }), { showHidden: false, depth: 5, colors: true }) | ||
.inspect({ ...affWord, flags: flagsToString(affWord.flags) }, { showHidden: false, depth: 5, colors: true }) | ||
.replace(/(\s|\n|\r)+/g, ' '); | ||
@@ -273,3 +295,3 @@ } | ||
function removeNeedAffix(flags) { | ||
const newFlags = Object.assign({}, flags); | ||
const newFlags = { ...flags }; | ||
delete newFlags.isNeedAffix; | ||
@@ -276,0 +298,0 @@ return newFlags; |
"use strict"; | ||
var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) { | ||
function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); } | ||
return new (P || (P = Promise))(function (resolve, reject) { | ||
function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } } | ||
function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } } | ||
function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); } | ||
step((generator = generator.apply(thisArg, _arguments || [])).next()); | ||
}); | ||
}; | ||
Object.defineProperty(exports, "__esModule", { value: true }); | ||
@@ -216,12 +207,10 @@ exports.testing = exports.parseAffFileToAff = exports.parseAff = exports.parseAffFile = void 0; | ||
}; | ||
function parseAffFile(filename, encoding = UTF8) { | ||
return __awaiter(this, void 0, void 0, function* () { | ||
const buffer = yield (0, fs_extra_1.readFile)(filename); | ||
const file = (0, iconv_lite_1.decode)(buffer, encoding); | ||
const affInfo = parseAff(file, encoding); | ||
if (affInfo.SET && affInfo.SET.toLowerCase() !== encoding.toLowerCase()) { | ||
return parseAff((0, iconv_lite_1.decode)(buffer, affInfo.SET.toLowerCase()), affInfo.SET); | ||
} | ||
return affInfo; | ||
}); | ||
async function parseAffFile(filename, encoding = UTF8) { | ||
const buffer = await (0, fs_extra_1.readFile)(filename); | ||
const file = (0, iconv_lite_1.decode)(buffer, encoding); | ||
const affInfo = parseAff(file, encoding); | ||
if (affInfo.SET && affInfo.SET.toLowerCase() !== encoding.toLowerCase()) { | ||
return parseAff((0, iconv_lite_1.decode)(buffer, affInfo.SET.toLowerCase()), affInfo.SET); | ||
} | ||
return affInfo; | ||
} | ||
@@ -228,0 +217,0 @@ exports.parseAffFile = parseAffFile; |
@@ -1,2 +0,1 @@ | ||
#!/usr/bin/env node | ||
export {}; |
177
dist/app.js
@@ -1,11 +0,20 @@ | ||
#!/usr/bin/env node | ||
"use strict"; | ||
var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) { | ||
function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); } | ||
return new (P || (P = Promise))(function (resolve, reject) { | ||
function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } } | ||
function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } } | ||
function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); } | ||
step((generator = generator.apply(thisArg, _arguments || [])).next()); | ||
}); | ||
var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) { | ||
if (k2 === undefined) k2 = k; | ||
Object.defineProperty(o, k2, { enumerable: true, get: function() { return m[k]; } }); | ||
}) : (function(o, m, k, k2) { | ||
if (k2 === undefined) k2 = k; | ||
o[k2] = m[k]; | ||
})); | ||
var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) { | ||
Object.defineProperty(o, "default", { enumerable: true, value: v }); | ||
}) : function(o, v) { | ||
o["default"] = v; | ||
}); | ||
var __importStar = (this && this.__importStar) || function (mod) { | ||
if (mod && mod.__esModule) return mod; | ||
var result = {}; | ||
if (mod != null) for (var k in mod) if (k !== "default" && Object.prototype.hasOwnProperty.call(mod, k)) __createBinding(result, mod, k); | ||
__setModuleDefault(result, mod); | ||
return result; | ||
}; | ||
@@ -16,3 +25,3 @@ Object.defineProperty(exports, "__esModule", { value: true }); | ||
const IterableHunspellReader_1 = require("./IterableHunspellReader"); | ||
const fs = require("fs-extra"); | ||
const fs = __importStar(require("fs-extra")); | ||
const util_1 = require("./util"); | ||
@@ -57,9 +66,9 @@ const gensequence_1 = require("gensequence"); | ||
function affWordToInfix(aff) { | ||
return Object.assign(Object.assign({}, aff), { word: aff.prefix + '<' + aff.base + '>' + aff.suffix }); | ||
return { ...aff, word: aff.prefix + '<' + aff.base + '>' + aff.suffix }; | ||
} | ||
function mapWord(map) { | ||
return (aff) => (Object.assign(Object.assign({}, aff), { word: map(aff.word) })); | ||
return (aff) => ({ ...aff, word: map(aff.word) }); | ||
} | ||
function appendRules(aff) { | ||
return Object.assign(Object.assign({}, aff), { word: aff.word + '\t[' + aff.rulesApplied + ' ]\t' + '(' + aff.dic + ')' }); | ||
return { ...aff, word: aff.word + '\t[' + aff.rulesApplied + ' ]\t' + '(' + aff.dic + ')' }; | ||
} | ||
@@ -110,76 +119,74 @@ function writeSeqToFile(seq, outFile) { | ||
} | ||
function actionPrime(hunspellDicFilename, options) { | ||
return __awaiter(this, void 0, void 0, function* () { | ||
displayHelp = false; | ||
const { sort = false, unique = false, output: outputFile, lower_case: lowerCase = false, transform = true, infix = false, rules = false, progress: showProgress = false, max_depth, forbidden = false, only_forbidden: onlyForbidden = false, partial_compounds: partialCompoundsAllowed = false, } = options; | ||
logStream = outputFile ? process.stdout : process.stderr; | ||
const log = notify; | ||
log('Write words'); | ||
log(`Sort: ${yesNo(sort)}`); | ||
log(`Unique: ${yesNo(unique)}`); | ||
const baseFile = hunspellDicFilename.replace(/\.(dic|aff)$/, ''); | ||
const dicFile = baseFile + '.dic'; | ||
const affFile = baseFile + '.aff'; | ||
log(`Dic file: ${dicFile}`); | ||
log(`Aff file: ${affFile}`); | ||
log(`Generating Words...`); | ||
const reader = yield IterableHunspellReader_1.IterableHunspellReader.createFromFiles(affFile, dicFile); | ||
if (max_depth && Number.parseInt(max_depth) >= 0) { | ||
reader.maxDepth = Number.parseInt(max_depth); | ||
async function actionPrime(hunspellDicFilename, options) { | ||
displayHelp = false; | ||
const { sort = false, unique = false, output: outputFile, lower_case: lowerCase = false, transform = true, infix = false, rules = false, progress: showProgress = false, max_depth, forbidden = false, only_forbidden: onlyForbidden = false, partial_compounds: partialCompoundsAllowed = false, } = options; | ||
logStream = outputFile ? process.stdout : process.stderr; | ||
const log = notify; | ||
log('Write words'); | ||
log(`Sort: ${yesNo(sort)}`); | ||
log(`Unique: ${yesNo(unique)}`); | ||
const baseFile = hunspellDicFilename.replace(/\.(dic|aff)$/, ''); | ||
const dicFile = baseFile + '.dic'; | ||
const affFile = baseFile + '.aff'; | ||
log(`Dic file: ${dicFile}`); | ||
log(`Aff file: ${affFile}`); | ||
log(`Generating Words...`); | ||
const reader = await IterableHunspellReader_1.IterableHunspellReader.createFromFiles(affFile, dicFile); | ||
if (max_depth && Number.parseInt(max_depth) >= 0) { | ||
reader.maxDepth = Number.parseInt(max_depth); | ||
} | ||
const transformers = []; | ||
const filters = []; | ||
if (!forbidden && !onlyForbidden) | ||
filters.push((aff) => !aff.flags.isForbiddenWord); | ||
if (onlyForbidden) | ||
filters.push((aff) => !!aff.flags.isForbiddenWord); | ||
if (!partialCompoundsAllowed) | ||
filters.push((aff) => !aff.flags.isOnlyAllowedInCompound); | ||
if (infix) { | ||
transformers.push(affWordToInfix); | ||
} | ||
if (lowerCase) { | ||
transformers.push(mapWord((a) => a.toLowerCase())); | ||
} | ||
if (rules) { | ||
transformers.push(appendRules); | ||
} | ||
transformers.push(mapWord((a) => a.trim())); | ||
const dicSize = reader.dic.length; | ||
let current = 0; | ||
const calcProgress = () => '\r' + current + ' / ' + dicSize; | ||
const reportProgressRate = 253; | ||
const callback = showProgress | ||
? () => { | ||
current++; | ||
!(current % reportProgressRate) && process.stderr.write(calcProgress(), 'utf-8'); | ||
} | ||
const transformers = []; | ||
const filters = []; | ||
if (!forbidden && !onlyForbidden) | ||
filters.push((aff) => !aff.flags.isForbiddenWord); | ||
if (onlyForbidden) | ||
filters.push((aff) => !!aff.flags.isForbiddenWord); | ||
if (!partialCompoundsAllowed) | ||
filters.push((aff) => !aff.flags.isOnlyAllowedInCompound); | ||
if (infix) { | ||
transformers.push(affWordToInfix); | ||
} | ||
if (lowerCase) { | ||
transformers.push(mapWord((a) => a.toLowerCase())); | ||
} | ||
if (rules) { | ||
transformers.push(appendRules); | ||
} | ||
transformers.push(mapWord((a) => a.trim())); | ||
const dicSize = reader.dic.length; | ||
let current = 0; | ||
const calcProgress = () => '\r' + current + ' / ' + dicSize; | ||
const reportProgressRate = 253; | ||
const callback = showProgress | ||
? () => { | ||
current++; | ||
!(current % reportProgressRate) && process.stderr.write(calcProgress(), 'utf-8'); | ||
} | ||
: () => { }; | ||
const seqWords = transform ? reader.seqAffWords(callback) : reader.seqRootWords().map(aff_1.asAffWord); | ||
const filterUnique = unique ? (0, util_1.uniqueFilter)(uniqueHistorySize) : (_) => true; | ||
const applyTransformers = (aff) => transformers.reduce((aff, fn) => fn(aff), aff); | ||
const applyFilters = (aff) => filters.reduce((cur, fn) => cur && fn(aff), true); | ||
const allWords = seqWords | ||
.filter(applyFilters) | ||
.map(applyTransformers) | ||
.map((a) => a.word) | ||
.filter((a) => !!a) | ||
.filter(filterUnique) | ||
.map((a) => a + '\n'); | ||
const words = options.number ? allWords.take(Number.parseInt(options.number)) : allWords; | ||
if (sort) { | ||
log('Sorting...'); | ||
const data = words.toArray().sort().join(''); | ||
const fd = outputFile ? fs.openSync(outputFile, 'w') : 1; | ||
fs.writeSync(fd, data); | ||
} | ||
else { | ||
yield writeSeqToFile(words, outputFile); | ||
} | ||
if (showProgress) { | ||
console.error(calcProgress()); | ||
} | ||
log('Done.'); | ||
}); | ||
: () => { }; | ||
const seqWords = transform ? reader.seqAffWords(callback) : reader.seqRootWords().map(aff_1.asAffWord); | ||
const filterUnique = unique ? (0, util_1.uniqueFilter)(uniqueHistorySize) : (_) => true; | ||
const applyTransformers = (aff) => transformers.reduce((aff, fn) => fn(aff), aff); | ||
const applyFilters = (aff) => filters.reduce((cur, fn) => cur && fn(aff), true); | ||
const allWords = seqWords | ||
.filter(applyFilters) | ||
.map(applyTransformers) | ||
.map((a) => a.word) | ||
.filter((a) => !!a) | ||
.filter(filterUnique) | ||
.map((a) => a + '\n'); | ||
const words = options.number ? allWords.take(Number.parseInt(options.number)) : allWords; | ||
if (sort) { | ||
log('Sorting...'); | ||
const data = words.toArray().sort().join(''); | ||
const fd = outputFile ? fs.openSync(outputFile, 'w') : 1; | ||
fs.writeSync(fd, data); | ||
} | ||
else { | ||
await writeSeqToFile(words, outputFile); | ||
} | ||
if (showProgress) { | ||
console.error(calcProgress()); | ||
} | ||
log('Done.'); | ||
} | ||
//# sourceMappingURL=app.js.map |
"use strict"; | ||
var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) { | ||
function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); } | ||
return new (P || (P = Promise))(function (resolve, reject) { | ||
function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } } | ||
function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } } | ||
function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); } | ||
step((generator = generator.apply(thisArg, _arguments || [])).next()); | ||
}); | ||
var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) { | ||
if (k2 === undefined) k2 = k; | ||
Object.defineProperty(o, k2, { enumerable: true, get: function() { return m[k]; } }); | ||
}) : (function(o, m, k, k2) { | ||
if (k2 === undefined) k2 = k; | ||
o[k2] = m[k]; | ||
})); | ||
var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) { | ||
Object.defineProperty(o, "default", { enumerable: true, value: v }); | ||
}) : function(o, v) { | ||
o["default"] = v; | ||
}); | ||
var __importStar = (this && this.__importStar) || function (mod) { | ||
if (mod && mod.__esModule) return mod; | ||
var result = {}; | ||
if (mod != null) for (var k in mod) if (k !== "default" && Object.prototype.hasOwnProperty.call(mod, k)) __createBinding(result, mod, k); | ||
__setModuleDefault(result, mod); | ||
return result; | ||
}; | ||
@@ -15,3 +25,3 @@ Object.defineProperty(exports, "__esModule", { value: true }); | ||
const gensequence_1 = require("gensequence"); | ||
const fs = require("fs-extra"); | ||
const fs = __importStar(require("fs-extra")); | ||
const iconv_lite_1 = require("iconv-lite"); | ||
@@ -111,14 +121,12 @@ const util_1 = require("./util"); | ||
} | ||
static createFromFiles(affFile, dicFile) { | ||
return __awaiter(this, void 0, void 0, function* () { | ||
const aff = yield (0, affReader_1.parseAffFileToAff)(affFile, defaultEncoding); | ||
const buffer = yield fs.readFile(dicFile); | ||
const dicFileContent = (0, iconv_lite_1.decode)(buffer, aff.affInfo.SET); | ||
const dic = dicFileContent | ||
.split('\n') | ||
.slice(1) // The first entry is the count of entries. | ||
.map((a) => a.trim()) | ||
.filter((line) => !!line); | ||
return new IterableHunspellReader({ aff, dic }); | ||
}); | ||
static async createFromFiles(affFile, dicFile) { | ||
const aff = await (0, affReader_1.parseAffFileToAff)(affFile, defaultEncoding); | ||
const buffer = await fs.readFile(dicFile); | ||
const dicFileContent = (0, iconv_lite_1.decode)(buffer, aff.affInfo.SET); | ||
const dic = dicFileContent | ||
.split('\n') | ||
.slice(1) // The first entry is the count of entries. | ||
.map((a) => a.trim()) | ||
.filter((line) => !!line); | ||
return new IterableHunspellReader({ aff, dic }); | ||
} | ||
@@ -125,0 +133,0 @@ } |
"use strict"; | ||
var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) { | ||
if (k2 === undefined) k2 = k; | ||
Object.defineProperty(o, k2, { enumerable: true, get: function() { return m[k]; } }); | ||
}) : (function(o, m, k, k2) { | ||
if (k2 === undefined) k2 = k; | ||
o[k2] = m[k]; | ||
})); | ||
var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) { | ||
Object.defineProperty(o, "default", { enumerable: true, value: v }); | ||
}) : function(o, v) { | ||
o["default"] = v; | ||
}); | ||
var __importStar = (this && this.__importStar) || function (mod) { | ||
if (mod && mod.__esModule) return mod; | ||
var result = {}; | ||
if (mod != null) for (var k in mod) if (k !== "default" && Object.prototype.hasOwnProperty.call(mod, k)) __createBinding(result, mod, k); | ||
__setModuleDefault(result, mod); | ||
return result; | ||
}; | ||
Object.defineProperty(exports, "__esModule", { value: true }); | ||
exports.iterableToStream = void 0; | ||
const stream = require("stream"); | ||
const stream = __importStar(require("stream")); | ||
/** | ||
@@ -6,0 +25,0 @@ * Transform an iterable into a node readable stream. |
@@ -5,3 +5,3 @@ "use strict"; | ||
const counters = new Map(); | ||
let isLoggingOn = true; | ||
const isLoggingOn = true; | ||
function incCounter(name, count = 1) { | ||
@@ -8,0 +8,0 @@ counters.set(name, (counters.get(name) || 0) + count); |
{ | ||
"name": "hunspell-reader", | ||
"version": "5.9.1-alpha.1", | ||
"version": "5.9.1", | ||
"description": "A library for reading Hunspell Dictionary Files", | ||
@@ -23,3 +23,3 @@ "bin": "bin.js", | ||
"lint": "prettier -w \"**/*.{md,yaml,yml,json,ts}\"", | ||
"clean": "rimraf dist coverage", | ||
"clean": "rimraf dist coverage .tsbuildinfo", | ||
"coverage": "jest --coverage", | ||
@@ -67,2 +67,6 @@ "watch": "tsc -w" | ||
}, | ||
"ignorePatterns": [ | ||
"dist/**", | ||
"node_modules/**" | ||
], | ||
"rules": {} | ||
@@ -73,3 +77,3 @@ }, | ||
}, | ||
"gitHead": "e79d890b347f53e49a05d683ec83c23db4c59b00" | ||
"gitHead": "c0d5026dadde1aaaa32c23d417de19c2b237f97e" | ||
} |
No v1
QualityPackage is not semver >=1. This means it is not stable and does not support ^ ranges.
Found 1 instance in 1 package
56315
1408
0