segment-dict - npm Package Compare versions

Comparing version 1.0.0 to 1.0.1

lib/loader/jieba/index.js

@@ -6,16 +6,9 @@ "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
-const fs = require("fs");
-const crlf_normalize_1 = require("crlf-normalize");
-const stream_1 = require("../../stream");
-function parse(input) {
-    return crlf_normalize_1.lineSplit(input)
-        .reduce(function (a, input) {
-        if (input.trim() !== '') {
-            let row = parseLine(input);
-            a.push(row);
-        }
-        return a;
-    }, []);
-}
-exports.parse = parse;
+const line_1 = require("../../fs/line");
+const stream_1 = require("../../fs/stream");
+/**
+ * 云计算
+ * 蓝翔 nz
+ * 区块链 10 nz
+*/
 function parseLine(input) {
@@ -25,57 +18,42 @@ let [str, n, s] = input
         .split(/\s+/g);
-    return [str, parseInt(n), s];
+    if (n === '') {
+        n = undefined;
+    }
+    if (s === '') {
+        s = undefined;
+    }
+    if (typeof s == 'undefined' || s == '') {
+        if (typeof n == 'string' && !/^\d+(?:\.\d+)?$/.test(n)) {
+            [n, s] = [undefined, n];
+        }
+    }
+    if (typeof n == 'string') {
+        // @ts-ignore
+        n = parseInt(n);
+    }
+    if (!str) {
+        throw new ReferenceError(`${input}`);
+    }
+    return [str, n, s];
 }
 exports.parseLine = parseLine;
-function loadSync(file) {
-    let input = fs.readFileSync(file);
-    return parse(input.toString());
-}
-exports.loadSync = loadSync;
 function load(file) {
-    return new Promise(function (resolve, reject) {
-        loadStream(file, function (err, dict, stream) {
-            if (err) {
-                reject(err);
-            }
-            else {
-                resolve(dict);
-            }
-        });
+    return line_1.wrapStreamToPromise(loadStream(file))
+        .then(function (stream) {
+        return stream.value;
     });
 }
 exports.load = load;
-function loadStream(file, cb) {
-    let stream = stream_1.default.createReadStream(file, {
-        trailing: true,
-        allowEmptyLine: false,
+function loadStream(file, callback) {
+    let stream = stream_1.default(file, {
+        callback,
+        mapper(line) {
+            if (line) {
+                return parseLine(line);
+            }
+        },
     });
-    let i = 0;
-    let dict = [];
-    try {
-        stream.on('data', function (line, ...argv) {
-            //console.log('data', i++, line);
-            dict.push(parseLine(line));
-        });
-        stream.on('close', function (line, ...argv) {
-            if (typeof line == 'string') {
-                //console.log('close', i++, line, argv);
-                dict.push(parseLine(line));
-            }
-        });
-        stream.on('end', function () {
-            if (cb) {
-                cb(null, dict, stream);
-            }
-        });
-    }
-    catch (e) {
-        stream.emit('error', e, dict);
-        if (cb) {
-            cb(e, dict, stream);
-        }
-    }
     return stream;
 }
 exports.loadStream = loadStream;
-const self = require("./index");
-exports.default = self;
+exports.default = load;

lib/loader/jieba/index.ts

@@ -5,5 +5,5 @@ /**
 
-import * as fs from 'fs';
-import crlf, { LF, lineSplit } from 'crlf-normalize';
-import ReadlineStream from '../../stream';
+import { wrapStreamToPromise, IStreamLineWithValue } from '../../fs/line';
+import * as Promise from 'bluebird';
+import createLoadStream, { ICallback } from '../../fs/stream';
 
@@ -13,19 +13,7 @@ export type IDictRow = [string, number, string];
 
-export function parse(input: string): IDict
-{
-	return lineSplit(input)
-		.reduce(function (a, input)
-		{
-			if (input.trim() !== '')
-			{
-				let row = parseLine(input);
-
-				a.push(row)
-			}
-
-			return a;
-		}, [] as IDict) as IDict
-		;
-}
-
+/**
+ * 云计算
+ * 蓝翔 nz
+ * 区块链 10 nz
+*/
 export function parseLine(input: string): IDictRow
@@ -38,15 +26,31 @@ {
 
-	return [str, parseInt(n), s];
-}
+	if (n === '')
+	{
+		n = undefined;
+	}
+	if (s === '')
+	{
+		s = undefined;
+	}
 
-export function loadSync(file: string): IDict
-{
-	let input = fs.readFileSync(file);
+	if (typeof s == 'undefined' || s == '')
+	{
+		if (typeof n == 'string' && !/^\d+(?:\.\d+)?$/.test(n))
+		{
+			[n, s] = [undefined, n];
+		}
+	}
 
-	return parse(input.toString());
-}
+	if (typeof n == 'string')
+	{
+		// @ts-ignore
+		n = parseInt(n);
+	}
 
-export interface ICallback extends Function
-{
-	(err: Error, dict?: IDict, stream?: ReadlineStream): void
+	if (!str)
+	{
+		throw new ReferenceError(`${input}`);
+	}
+
+	return [str, n as any as number, s];
 }
@@ -56,69 +60,29 @@
 {
-	return new Promise(function (resolve, reject)
-	{
-		loadStream(file, function (err, dict, stream)
+	return wrapStreamToPromise(loadStream(file))
+		.then(function (stream: IStreamLineWithValue<IDict>)
 		{
-			if (err)
-			{
-				reject(err);
-			}
-			else
-			{
-				resolve(dict);
-			}
+			return stream.value;
 		})
-	})
+		;
 }
 
-export function loadStream(file: string, cb?: ICallback)
+export function loadStream(file: string, callback?: ICallback<IDict>)
 {
-	let stream = ReadlineStream.createReadStream(file, {
-		trailing: true,
-		allowEmptyLine: false,
-	});
+	let stream = createLoadStream<IDict>(file, {
 
-	let i = 0;
+		callback,
 
-	let dict: IDict = [];
-
-	try
-	{
-		stream.on('data', function (line, ...argv)
+		mapper(line)
 		{
-			//console.log('data', i++, line);
-			dict.push(parseLine(line as string));
-		});
-
-		stream.on('close', function (line, ...argv)
-		{
-
-			if (typeof line == 'string')
+			if (line)
 			{
-				//console.log('close', i++, line, argv);
-				dict.push(parseLine(line as string));
+				return parseLine(line);
 			}
-		});
+		},
 
-		stream.on('end', function ()
-		{
-			if (cb)
-			{
-				cb(null, dict, stream);
-			}
-		});
-	}
-	catch (e)
-	{
-		stream.emit('error', e, dict);
+	});
 
-		if (cb)
-		{
-			cb(e, dict, stream);
-		}
-	}
-
 	return stream;
 }
 
-import * as self from './index';
-export default self;
+export default load;

package.json

 {
   "name": "segment-dict",
-  "version": "1.0.0",
-  "description": "",
+  "version": "1.0.1",
+  "description": "segment data",
+  "keywords": [
+    "NLP",
+    "PoS tagging",
+    "analyzer",
+    "chinese",
+    "chinese segmentation",
+    "dict",
+    "jieba",
+    "machine learning",
+    "morphological analysis",
+    "natural language processing",
+    "node-novel",
+    "novel",
+    "optimizer",
+    "segment",
+    "segmentation",
+    "split",
+    "text mining",
+    "tokenizer",
+    "word",
+    "word segmentation",
+    "中文",
+    "中文分词",
+    "分词"
+  ],
+  "homepage": "https://github.com/bluelovers/node-segment-dict#readme",
+  "bugs": {
+    "url": "https://github.com/bluelovers/node-segment-dict/issues"
+  },
+  "repository": {
+    "type": "git",
+    "url": "git+https://github.com/bluelovers/node-segment-dict.git"
+  },
+  "license": "ISC",
+  "author": "",
   "main": "index.js",
   "directories": {
-    "lib": "lib"
+    "lib": "lib",
+    "dict": "dict",
+    "test": "test"
   },
+  "scripts": {
+    "test": "echo \"Error: no test specified\" && exit 1"
+  },
   "dependencies": {
+    "bluebird": "^3.5.1",
+    "split2": "^2.2.0",
+    "stream-pipe": "^1.0.1",
+    "through2": "^2.0.3"
+  },
+  "devDependencies": {
+    "@types/bluebird": "^3.5.20",
+    "@types/node": "^9.4.7",
     "core-decorators": "^0.20.0",
     "crlf-normalize": "^1.0.1"
-  },
-  "devDependencies": {
-    "@types/node": "^9.4.7"
-  },
-  "scripts": {
-    "test": "echo \"Error: no test specified\" && exit 1"
-  },
-  "author": "",
-  "license": "ISC"
+  }
 }

New alerts

License Policy Violation

License

This package is not allowed per your license policy. Review the package's license to ensure compliance.

Found 1 instance in 1 package

Major refactor

Supply chain risk

Package has recently undergone a major refactor. It may be unstable or indicate significant internal changes. Use caution when updating to versions that include significant changes.

Found 1 instance in 1 package

Dynamic require

Supply chain risk

Dynamic require can indicate the package is performing dangerous or unsafe dynamic code execution.

Found 1 instance in 1 package

Fixed alerts

License Policy Violation

License

This package is not allowed per your license policy. Review the package's license to ensure compliance.

Found 1 instance in 1 package

No README

Quality

Package does not have a README. This may indicate a failed publish or a low quality package.

Found 1 instance in 1 package

No bug tracker

Maintenance

Package does not have a linked bug tracker in package.json.

Found 1 instance in 1 package

No repository

Supply chain risk

Package does not have a linked source code repository. Without this field, a package will have no reference to the location of the source code use to generate the package.

Found 1 instance in 1 package

No website

Quality

Package does not have a website.

Found 1 instance in 1 package

Improved metrics

Lines of code

823

increased by92.29%

Number of low maintenance alerts

1

decreased by-50%

Number of low quality alerts

1

decreased by-50%

Number of medium quality alerts

0

decreased by-100%

Number of lines in readme file

26

increased byInfinity%

Worsened metrics

Total package byte prevSize

38172672

decreased by-0.08%

Dependency count

4

increased by100%

Dev dependency count

4

increased by300%

Number of package files

44

decreased by-2.22%

Number of medium supply chain risk alerts

1

increased byInfinity%

Dependency changes

• + Addedbluebird@^3.5.1

• + Addedsplit2@^2.2.0

• + Addedstream-pipe@^1.0.1

• + Addedthrough2@^2.0.3

• + Addedbluebird@3.7.2(transitive)

• + Addedcore-util-is@1.0.3(transitive)

• + Addedinherits@2.0.4(transitive)

• + Addedisarray@1.0.0(transitive)

• + Addedprocess-nextick-args@2.0.1(transitive)

• + Addedreadable-stream@2.3.8(transitive)

• + Addedsafe-buffer@5.1.2(transitive)

• + Addedsplit2@2.2.0(transitive)

• + Addedstream-pipe@1.0.4(transitive)

• + Addedstring_decoder@1.1.1(transitive)

• + Addedthrough2@2.0.5(transitive)

• + Addedutil-deprecate@1.0.2(transitive)

• + Addedxtend@4.0.2(transitive)

• - Removedcore-decorators@^0.20.0

• - Removedcrlf-normalize@^1.0.1

• - Removed@types/node@22.10.0(transitive)

• - Removedcore-decorators@0.20.0(transitive)

• - Removedcrlf-normalize@1.0.20(transitive)

• - Removedts-toolbelt@9.6.0(transitive)

• - Removedts-type@3.0.1(transitive)

• - Removedtypedarray-dts@1.0.0(transitive)

• - Removedundici-types@6.20.0(transitive)