cspell-tools - npm Package Compare versions

Comparing version 1.2.1 to 1.3.1

CHANGELOG.md

 # Release Notes
 
+## 1.3.1
+* Add command line tool for building trie files.
+
 ## 1.2.1
@@ -4,0 +7,0 @@ * Hyphenated words are now treated as two words.

dist/app.js

@@ -30,2 +30,20 @@ #!/usr/bin/env node
 });
+program
+    .command('compile-trie <src...>')
+    .description('compile words lists into simple dictionary files.')
+    .option('-o, --output <path>', 'Specify the output directory, otherwise files are written back to the same location.')
+    .option('-n, --no-compress', 'By default the files are Gzipped, this will turn that off.')
+    .action((src, options) => {
+    console.log('Compile:\n output: %s\n compress: %s\n files:\n  %s \n\n', options.output || 'default', options.compress ? 'true' : 'false', src.join('\n  '));
+    const ext = '.trie' + (options.compress ? '.gz' : '');
+    Rx.Observable.from(src)
+        .flatMap(src => globRx(src))
+        .flatMap(s => s)
+        .map(s => [s, options.output ? path.join(options.output, path.basename(s, '.txt') + ext) : s + ext])
+        .concatMap(([src, dst]) => {
+        console.log('Process "%s" to "%s"', src, dst);
+        return compiler_1.compileTrie(src, dst).then(() => src);
+    })
+        .forEach(name => console.log(`Complete.`));
+});
 program.parse(process.argv);
@@ -32,0 +50,0 @@ if (!process.argv.slice(2).length) {

dist/compiler/wordListCompiler.d.ts

@@ -6,2 +6,3 @@ /// <reference types="node" />
 import { Observable } from 'rxjs/Rx';
+import * as Trie from 'cspell-trie';
 export declare function normalizeWords(lines: Rx.Observable<string>): Observable<string>;
@@ -11,1 +12,4 @@ export declare function lineToWords(line: string): Sequence<string>;
 export declare function compileWordList(filename: string, destFilename: string): Promise<fs.WriteStream>;
+export declare function normalizeWordsToTrie(words: Rx.Observable<string>): Promise<Trie.TrieNode>;
+export declare function compileWordListToTrieFile(words: Rx.Observable<string>, destFilename: string): Promise<void>;
+export declare function compileTrie(filename: string, destFilename: string): Promise<void>;

dist/compiler/wordListCompiler.js

 "use strict";
+const Rx = require("rxjs/Rx");
 const XRegExp = require("xregexp");
@@ -6,5 +7,7 @@ const gensequence_1 = require("gensequence");
 const fileReader_1 = require("./fileReader");
-const fileWriter_1 = require("./fileWriter");
+const cspell_lib_1 = require("cspell-lib");
 const path = require("path");
 const fs_promise_1 = require("fs-promise");
+const rxjs_from_iterable_1 = require("rxjs-from-iterable");
+const Trie = require("cspell-trie");
 const regNonWordOrSpace = XRegExp("[^\\p{L}' ]+", 'gi');
@@ -56,6 +59,28 @@ const regExpSpaceOrDash = /(?:\s+)|(?:-+)/g;
             .join('');
-        return fileWriter_1.writeToFile(destFilename, data);
+        return cspell_lib_1.writeToFile(destFilename, data);
     });
 }
 exports.compileWordList = compileWordList;
+function normalizeWordsToTrie(words) {
+    const result = normalizeWords(words)
+        .reduce((node, word) => Trie.insert(word, node), {})
+        .toPromise();
+    return result;
+}
+exports.normalizeWordsToTrie = normalizeWordsToTrie;
+function compileWordListToTrieFile(words, destFilename) {
+    const destDir = path.dirname(destFilename);
+    const dir = fs_promise_1.mkdirp(destDir);
+    const root = normalizeWordsToTrie(words);
+    const data = Rx.Observable.zip(dir, root, (_, b) => b)
+        .map(node => Trie.serializeTrie(node))
+        .flatMap(seq => rxjs_from_iterable_1.observableFromIterable(seq));
+    return cspell_lib_1.writeToFileRxP(destFilename, data.bufferCount(1024).map(a => a.join('')));
+}
+exports.compileWordListToTrieFile = compileWordListToTrieFile;
+function compileTrie(filename, destFilename) {
+    const words = fileReader_1.lineReaderRx(filename);
+    return compileWordListToTrieFile(words, destFilename);
+}
+exports.compileTrie = compileTrie;
 //# sourceMappingURL=wordListCompiler.js.map

package.json

 {
   "name": "cspell-tools",
-  "version": "1.2.1",
+  "version": "1.3.1",
   "description": "Tools to assist with the development of cSpell",
@@ -52,3 +52,4 @@ "main": "dist/index.js",
     "cspell-lib": "^1.0.1",
-    "fs-promise": "^1.0.0",
+    "cspell-trie": "^1.1.0",
+    "fs-promise": "^2.0.0",
     "gensequence": "^1.0.0",
@@ -59,2 +60,3 @@ "glob": "^7.1.1",
     "rxjs": "^5.1.0",
+    "rxjs-from-iterable": "^1.0.5",
     "rxjs-stream": "^1.0.4",
@@ -61,0 +63,0 @@ "tsmerge": "^1.0.4",

src/app.ts

 #!/usr/bin/env node
 
-import { compileWordList } from './compiler';
+import { compileWordList, compileTrie } from './compiler';
 import * as path from 'path';
@@ -41,2 +41,26 @@ import * as program from 'commander';
 
+program
+    .command('compile-trie <src...>')
+    .description('compile words lists into simple dictionary files.')
+    .option('-o, --output <path>', 'Specify the output directory, otherwise files are written back to the same location.')
+    .option('-n, --no-compress', 'By default the files are Gzipped, this will turn that off.')
+    .action((src: string[], options: { output?: string, compress: boolean }) => {
+        console.log('Compile:\n output: %s\n compress: %s\n files:\n  %s \n\n',
+            options.output || 'default',
+            options.compress ? 'true' : 'false',
+            src.join('\n  ') );
+
+        const ext = '.trie' + (options.compress ? '.gz' : '');
+
+        Rx.Observable.from(src)
+            .flatMap(src => globRx(src))
+            .flatMap(s => s)
+            .map(s => [s, options.output ? path.join(options.output, path.basename(s, '.txt') + ext) : s + ext])
+            .concatMap(([src, dst]) => {
+                console.log('Process "%s" to "%s"', src, dst);
+                return compileTrie(src, dst).then(() => src);
+            })
+            .forEach(name => console.log(`Complete.`));
+    });
+
 program.parse(process.argv);
@@ -43,0 +67,0 @@

src/compiler/fileWriter.test.ts

@@ -19,3 +19,2 @@ import { expect } from 'chai';
             })
-            .do(() => console.log('HERE'))
             .concatMap(() => fileReader.textFileStreamRx(filename))
@@ -22,0 +21,0 @@ .reduce((a, b) => a + b)

src/compiler/wordListCompiler.test.ts

@@ -6,5 +6,8 @@ // cSpell:ignore jpegs outing dirs lcode
 import { expect } from 'chai';
-import { lineToWords, compileWordList } from './wordListCompiler';
+import { lineToWords, compileWordList, compileTrie } from './wordListCompiler';
+import { normalizeWords, normalizeWordsToTrie } from './wordListCompiler';
+import * as fsp from 'fs-promise';
+import * as Trie from 'cspell-trie';
 import * as path from 'path';
-import * as fs from 'fs';
+import * as Rx from 'rxjs/Rx';
 
@@ -59,4 +62,5 @@ describe('Validate the wordListCompiler', function() {
                 s.on('error', () => reject());
-            }).then(() => {
-                const output = fs.readFileSync(destName, 'UTF-8');
+            })
+            .then(() => fsp.readFile(destName, 'UTF-8'))
+            .then(output => {
                 expect(output).to.be.equal(citiesResult);
@@ -66,2 +70,29 @@ });
     });
+
+    it('tests normalized to a trie', () => {
+        const words = citiesResult.split('\n');
+        const nWords = normalizeWords(Rx.Observable.from(words)).toArray().toPromise();
+        const tWords = normalizeWordsToTrie(Rx.Observable.from(words))
+            .then(node => Trie.iteratorTrieWords(node))
+            .then(seq => [...seq]);
+        return Promise.all([nWords, tWords])
+            .then(([nWords, tWords]) => {
+                expect(tWords.sort()).to.be.deep.equal([...(new Set(nWords.sort()))]);
+            });
+    });
+
+    it('test reading and normalizing to a trie file', () => {
+        const sourceName = path.join(__dirname, '..', '..', 'Samples', 'cities.txt');
+        const destName = path.join(__dirname, '..', '..', 'temp', 'cities.trie');
+        return compileTrie(sourceName, destName)
+        .then(() => fsp.readFile(destName, 'UTF-8'))
+        .then(output => output.split('\n'))
+        .then(words => {
+            return Trie.importTrieRx(Rx.Observable.from(words)).take(1).toPromise()
+            .then(node => {
+                expect([...Trie.iteratorTrieWords(node)].sort()).to.be.deep
+                    .equal(citiesResult.split('\n').filter(a => !!a).sort());
+            });
+        });
+    });
 });
@@ -68,0 +99,0 @@

src/compiler/wordListCompiler.ts

@@ -7,6 +7,8 @@ import * as Rx from 'rxjs/Rx';
 import { lineReaderRx } from './fileReader';
-import { writeToFile } from './fileWriter';
+import { writeToFile, writeToFileRxP} from 'cspell-lib';
 import { Observable } from 'rxjs/Rx';
 import * as path from 'path';
 import { mkdirp } from 'fs-promise';
+import { observableFromIterable } from 'rxjs-from-iterable';
+import * as Trie from 'cspell-trie';
 
@@ -68,1 +70,29 @@ const regNonWordOrSpace = XRegExp("[^\\p{L}' ]+", 'gi');
 }
+
+
+export function normalizeWordsToTrie(words: Rx.Observable<string>): Promise<Trie.TrieNode> {
+    const result = normalizeWords(words)
+        .reduce((node, word) => Trie.insert(word, node), {} as Trie.TrieNode)
+        .toPromise();
+    return result;
+}
+
+export function compileWordListToTrieFile(words: Rx.Observable<string>, destFilename: string): Promise<void> {
+    const destDir = path.dirname(destFilename);
+    const dir = mkdirp(destDir);
+    const root = normalizeWordsToTrie(words);
+
+    const data = Rx.Observable.zip(dir, root, (_: void, b: Trie.TrieNode) =>
+    b
+    )
+        .map(node => Trie.serializeTrie(node))
+        .flatMap(seq => observableFromIterable(seq));
+
+    return writeToFileRxP(destFilename, data.bufferCount(1024).map(a => a.join('')));
+}
+
+
+export function compileTrie(filename: string, destFilename: string): Promise<void> {
+    const words = lineReaderRx(filename);
+    return compileWordListToTrieFile(words, destFilename);
+}

New alerts

License Policy Violation

License

This package is not allowed per your license policy. Review the package's license to ensure compliance.

Found 1 instance in 1 package

Fixed alerts

License Policy Violation

License

This package is not allowed per your license policy. Review the package's license to ensure compliance.

Found 1 instance in 1 package

Major refactor

Supply chain risk

Package has recently undergone a major refactor. It may be unstable or indicate significant internal changes. Use caution when updating to versions that include significant changes.

Found 1 instance in 1 package

Improved metrics

Total package byte prevSize

58066

increased by16.43%

Lines of code

1051

increased by12.65%

Number of medium supply chain risk alerts

0

decreased by-100%

Worsened metrics

Dependency count

13

increased by18.18%

Dependency changes

• + Addedcspell-trie@^1.1.0

• + Addedrxjs-from-iterable@^1.0.5

• + Addedcspell-trie@1.6.12(transitive)

• + Addedfs-extra@2.1.24.0.3(transitive)

• + Addedfs-promise@2.0.3(transitive)

• + Addedgensequence@2.3.0(transitive)

• + Addedjsonfile@4.0.0(transitive)

• + Addedrxjs-from-iterable@1.0.5(transitive)

• + Addeduniversalify@0.1.2(transitive)

• - Removedfs-extra@1.0.0(transitive)

• - Removedfs-promise@1.0.0(transitive)

• - Removedklaw@1.3.1(transitive)

• Updatedfs-promise@^2.0.0