transliteration - npm Package Compare versions

Comparing version 2.1.11 to 2.2.0

dist/node/src/common/slugify.js

 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
+exports.Slugify = exports.defaultOptions = void 0;
 const transliterate_1 = require("./transliterate");
 const utils_1 = require("./utils");
 // Slugify
-exports.defaultOptions = Object.assign(Object.assign({}, utils_1.deepClone(transliterate_1.defaultOptions)), { allowedChars: 'a-zA-Z0-9-_.~', lowercase: true, separator: '-', uppercase: false });
+exports.defaultOptions = Object.assign(Object.assign({}, utils_1.deepClone(transliterate_1.defaultOptions)), { allowedChars: 'a-zA-Z0-9-_.~', lowercase: true, separator: '-', uppercase: false, fixChineseSpacing: true });
 class Slugify extends transliterate_1.Transliterate {
@@ -8,0 +9,0 @@ get options() {

dist/node/src/common/transliterate.d.ts

@@ -20,3 +20,3 @@ import { IntervalArray, OptionReplaceArray, OptionReplaceCombined, OptionsTransliterate } from '../types';
      */
-    codeMapReplace(str: string, ignoreRanges: IntervalArray, unknown?: string): string;
+    codeMapReplace(str: string, ignoreRanges: IntervalArray | undefined, opt: OptionsTransliterate): string;
     /**
@@ -23,0 +23,0 @@ * Convert the object version of the 'replace' option into tuple array one

dist/node/src/common/transliterate.js

 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
+exports.Transliterate = exports.defaultOptions = void 0;
 const charmap_1 = require("../../data/charmap");
@@ -11,2 +12,3 @@ const utils_1 = require("./utils");
     unknown: '',
+    fixChineseSpacing: true,
 };
@@ -40,10 +42,14 @@ class Transliterate {
      */
-    codeMapReplace(str, ignoreRanges, unknown) {
+    codeMapReplace(str, ignoreRanges = [], opt) {
         let index = 0;
         let result = '';
+        const strContainsChinese = opt.fixChineseSpacing && utils_1.hasChinese(str);
+        let lastCharHasChinese = false;
         for (let i = 0; i < str.length; i++) {
             // Get current character, taking surrogates in consideration
-            const char = /[\uD800-\uDBFF]/.test(str[i]) && /[\uDC00-\uDFFF]/.test(str[i + 1]) ?
-                str[i] + str[i + 1] : str[i];
+            const char = /[\uD800-\uDBFF]/.test(str[i]) && /[\uDC00-\uDFFF]/.test(str[i + 1])
+                ? str[i] + str[i + 1]
+                : str[i];
             let s;
+            let ignoreFixingChinese = false;
             switch (true) {
@@ -55,11 +61,18 @@ // current character is in ignored list
                     s = char;
+                    // if it's the first character of an ignored string, then leave ignoreFixingChinese to true
+                    if (!ignoreRanges.find((range) => range[1] >= index && range[0] === index)) {
+                        ignoreFixingChinese = true;
+                    }
                     break;
                 default:
-                    s = this.map[char] || unknown;
+                    s = this.map[char] || opt.unknown || '';
             }
             // fix Chinese spacing issue
-            if (utils_1.isChinese(char) && this.map[char]) {
-                if (str[index - 1] && !utils_1.isChinese(s) && !/\s/.test(result[result.length - 1])) {
+            if (strContainsChinese) {
+                if (lastCharHasChinese &&
+                    !ignoreFixingChinese &&
+                    !utils_1.hasPunctuationOrSpace(s)) {
                     s = ' ' + s;
                 }
+                lastCharHasChinese = !!s && utils_1.hasChinese(char);
             }
@@ -107,3 +120,3 @@ result += s;
                     break;
-                case (typeof item[0] === 'string') && item[0].length > 0:
+                case typeof item[0] === 'string' && item[0].length > 0:
                     item[0] = RegExp(utils_1.escapeRegExp(item[0]), 'g');
@@ -132,3 +145,5 @@ break;
                 /* istanbul ignore else */
-                if (Object.prototype.hasOwnProperty.call(data, from) && from.length < 3 && from <= '\udbff\udfff') {
+                if (Object.prototype.hasOwnProperty.call(data, from) &&
+                    from.length < 3 &&
+                    from <= '\udbff\udfff') {
                     this.map[from] = data[from];
@@ -155,7 +170,6 @@ }
         // ignore
-        let ignoreRanges = [];
-        if (opt.ignore && opt.ignore.length > 0) {
-            ignoreRanges = utils_1.findStrOccurrences(source, opt.ignore);
-        }
-        str = this.codeMapReplace(str, ignoreRanges, opt.unknown);
+        const ignoreRanges = opt.ignore && opt.ignore.length > 0
+            ? utils_1.findStrOccurrences(str, opt.ignore)
+            : [];
+        str = this.codeMapReplace(str, ignoreRanges, opt);
         // trim spaces at the beginning and ending of the string
@@ -162,0 +176,0 @@ if (opt.trim) {

dist/node/src/common/utils.d.ts

@@ -7,4 +7,11 @@ import { IntervalArray } from '../types';
 export declare function escapeRegExp(str?: string): string;
-export declare function isChinese(char: string): boolean;
 /**
+ * Check if a character is Chinese
+ */
+export declare function hasChinese(char: string): boolean;
+/**
+ * Check if a character is a punctuation
+ */
+export declare function hasPunctuationOrSpace(char: string): boolean;
+/**
  * Deep clone a variable
@@ -11,0 +18,0 @@ * @param obj Object to clone

dist/node/src/common/utils.js

 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
+exports.regexpReplaceCustom = exports.inRange = exports.findStrOccurrences = exports.deepClone = exports.hasPunctuationOrSpace = exports.hasChinese = exports.escapeRegExp = void 0;
 /**
@@ -11,8 +12,17 @@ * Escape regular expression string
 exports.escapeRegExp = escapeRegExp;
-function isChinese(char) {
-    const c = char.charCodeAt(0);
-    return (c >= 0x4e00 && c <= 0x9fff) || (c >= 0xf900 && c <= 0xfaff);
+/**
+ * Check if a character is Chinese
+ */
+function hasChinese(char) {
+    return /[\u2E80-\u2E99\u2E9B-\u2EF3\u2F00-\u2FD5\u3005\u3007\u3021-\u3029\u3038-\u303B\u3400-\u4DBF\u4E00-\u9FFC\uF900-\uFA6D\uFA70-\uFAD9]|\uD81B[\uDFF0\uDFF1]|[\uD840-\uD868\uD86A-\uD86C\uD86F-\uD872\uD874-\uD879\uD880-\uD883][\uDC00-\uDFFF]|\uD869[\uDC00-\uDEDD\uDF00-\uDFFF]|\uD86D[\uDC00-\uDF34\uDF40-\uDFFF]|\uD86E[\uDC00-\uDC1D\uDC20-\uDFFF]|\uD873[\uDC00-\uDEA1\uDEB0-\uDFFF]|\uD87A[\uDC00-\uDFE0]|\uD87E[\uDC00-\uDE1D]|\uD884[\uDC00-\uDF4A]/.test(char);
 }
-exports.isChinese = isChinese;
+exports.hasChinese = hasChinese;
 /**
+ * Check if a character is a punctuation
+ */
+function hasPunctuationOrSpace(char) {
+    return /[\s!-#%-\*,-\/:;\?@\[-\]_\{\}\xA1\xA7\xAB\xB6\xB7\xBB\xBF\u037E\u0387\u055A-\u055F\u0589\u058A\u05BE\u05C0\u05C3\u05C6\u05F3\u05F4\u0609\u060A\u060C\u060D\u061B\u061E\u061F\u066A-\u066D\u06D4\u0700-\u070D\u07F7-\u07F9\u0830-\u083E\u085E\u0964\u0965\u0970\u09FD\u0A76\u0AF0\u0C77\u0C84\u0DF4\u0E4F\u0E5A\u0E5B\u0F04-\u0F12\u0F14\u0F3A-\u0F3D\u0F85\u0FD0-\u0FD4\u0FD9\u0FDA\u104A-\u104F\u10FB\u1360-\u1368\u1400\u166E\u169B\u169C\u16EB-\u16ED\u1735\u1736\u17D4-\u17D6\u17D8-\u17DA\u1800-\u180A\u1944\u1945\u1A1E\u1A1F\u1AA0-\u1AA6\u1AA8-\u1AAD\u1B5A-\u1B60\u1BFC-\u1BFF\u1C3B-\u1C3F\u1C7E\u1C7F\u1CC0-\u1CC7\u1CD3\u2010-\u2027\u2030-\u2043\u2045-\u2051\u2053-\u205E\u207D\u207E\u208D\u208E\u2308-\u230B\u2329\u232A\u2768-\u2775\u27C5\u27C6\u27E6-\u27EF\u2983-\u2998\u29D8-\u29DB\u29FC\u29FD\u2CF9-\u2CFC\u2CFE\u2CFF\u2D70\u2E00-\u2E2E\u2E30-\u2E4F\u2E52\u3001-\u3003\u3008-\u3011\u3014-\u301F\u3030\u303D\u30A0\u30FB\uA4FE\uA4FF\uA60D-\uA60F\uA673\uA67E\uA6F2-\uA6F7\uA874-\uA877\uA8CE\uA8CF\uA8F8-\uA8FA\uA8FC\uA92E\uA92F\uA95F\uA9C1-\uA9CD\uA9DE\uA9DF\uAA5C-\uAA5F\uAADE\uAADF\uAAF0\uAAF1\uABEB\uFD3E\uFD3F\uFE10-\uFE19\uFE30-\uFE52\uFE54-\uFE61\uFE63\uFE68\uFE6A\uFE6B\uFF01-\uFF03\uFF05-\uFF0A\uFF0C-\uFF0F\uFF1A\uFF1B\uFF1F\uFF20\uFF3B-\uFF3D\uFF3F\uFF5B\uFF5D\uFF5F-\uFF65]|\uD800[\uDD00-\uDD02\uDF9F\uDFD0]|\uD801\uDD6F|\uD802[\uDC57\uDD1F\uDD3F\uDE50-\uDE58\uDE7F\uDEF0-\uDEF6\uDF39-\uDF3F\uDF99-\uDF9C]|\uD803[\uDEAD\uDF55-\uDF59]|\uD804[\uDC47-\uDC4D\uDCBB\uDCBC\uDCBE-\uDCC1\uDD40-\uDD43\uDD74\uDD75\uDDC5-\uDDC8\uDDCD\uDDDB\uDDDD-\uDDDF\uDE38-\uDE3D\uDEA9]|\uD805[\uDC4B-\uDC4F\uDC5A\uDC5B\uDC5D\uDCC6\uDDC1-\uDDD7\uDE41-\uDE43\uDE60-\uDE6C\uDF3C-\uDF3E]|\uD806[\uDC3B\uDD44-\uDD46\uDDE2\uDE3F-\uDE46\uDE9A-\uDE9C\uDE9E-\uDEA2]|\uD807[\uDC41-\uDC45\uDC70\uDC71\uDEF7\uDEF8\uDFFF]|\uD809[\uDC70-\uDC74]|\uD81A[\uDE6E\uDE6F\uDEF5\uDF37-\uDF3B\uDF44]|\uD81B[\uDE97-\uDE9A\uDFE2]|\uD82F\uDC9F|\uD836[\uDE87-\uDE8B]|\uD83A[\uDD5E\uDD5F]/.test(char);
+}
+exports.hasPunctuationOrSpace = hasPunctuationOrSpace;
+/**
  * Deep clone a variable
@@ -19,0 +29,0 @@ * @param obj Object to clone

dist/node/src/node/index.js

 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
+exports.slugify = exports.transliterate = void 0;
 const slugify_1 = require("../common/slugify");
@@ -4,0 +5,0 @@ const transliterate_1 = require("../common/transliterate");

dist/node/src/types/index.d.ts

@@ -36,2 +36,7 @@ import { Charmap } from '../../data/charmap';
     unknown?: string;
+    /**
+     * Fix Chinese spacing. For example, `你好` is transliterated to `Ni Hao` instead of `NiHao`. If you don't need to transliterate Chinese characters, set it to false to false to improve performance.
+     * @default true
+     */
+    fixChineseSpacing?: boolean;
 }
@@ -61,2 +66,6 @@ export interface OptionsSlugify extends OptionsTransliterate {
     allowedChars?: string;
+    /**
+     * Fix Chinese spacing. For example, `你好` is transliterated to `Ni Hao` instead of `NiHao`. If you don't need to transliterate Chinese characters, set it to false to false to improve performance.
+     */
+    fixChineseSpacing?: boolean;
 }
@@ -63,0 +72,0 @@ export declare type Options = OptionsTransliterate | OptionsSlugify;

package.json

 {
   "name": "transliteration",
-  "version": "2.1.11",
+  "version": "2.2.0",
   "description": "Unicode to ACSII transliteration / slugify module for node.js, browser, Web Worker, ReactNative and CLI.",
@@ -55,38 +55,38 @@ "main": "dist/node/src/node/index.js",
   "devDependencies": {
-    "@babel/core": "7.9.6",
-    "@babel/plugin-proposal-class-properties": "7.8.3",
-    "@babel/plugin-proposal-object-rest-spread": "7.9.6",
-    "@babel/preset-env": "7.9.6",
-    "@babel/preset-typescript": "7.9.0",
+    "@babel/core": "7.12.3",
+    "@babel/plugin-proposal-class-properties": "7.12.1",
+    "@babel/plugin-proposal-object-rest-spread": "7.12.1",
+    "@babel/preset-env": "7.12.1",
+    "@babel/preset-typescript": "7.12.1",
     "@types/execa": "2.0.0",
     "@types/tape": "4.13.0",
-    "@types/yargs": "15.0.4",
-    "codecov": "3.6.5",
+    "@types/yargs": "15.0.9",
+    "codecov": "3.8.0",
     "core-js": "3.6.5",
     "coveralls": "3.1.0",
-    "eslint-config-prettier": "^6.11.0",
-    "execa": "4.0.0",
+    "eslint-config-prettier": "^6.14.0",
+    "execa": "4.0.3",
     "json5": "2.1.3",
-    "nyc": "15.0.1",
-    "prettier": "^2.0.5",
+    "nyc": "15.1.0",
+    "prettier": "^2.1.2",
     "rimraf": "3.0.2",
-    "rollup": "2.7.6",
-    "rollup-plugin-babel": "5.0.0-alpha.0",
+    "rollup": "2.32.1",
+    "rollup-plugin-babel": "5.0.0-alpha.2",
     "rollup-plugin-commonjs": "10.1.0",
     "rollup-plugin-hashbang": "2.2.2",
-    "rollup-plugin-sourcemaps": "0.6.1",
-    "rollup-plugin-terser": "5.3.0",
-    "rollup-plugin-typescript2": "0.27.0",
+    "rollup-plugin-sourcemaps": "0.6.3",
+    "rollup-plugin-terser": "7.0.2",
+    "rollup-plugin-typescript2": "0.28.0",
     "rollup-plugin-uglify": "6.0.4",
     "tap-spec": "5.0.0",
-    "tape": "5.0.0",
-    "ts-loader": "7.0.2",
-    "ts-node": "8.9.1",
-    "tslint": "6.1.2",
+    "tape": "5.0.1",
+    "ts-loader": "8.0.7",
+    "ts-node": "9.0.0",
+    "tslint": "6.1.3",
     "tslint-config-prettier": "1.18.0",
-    "typescript": "3.8.3"
+    "typescript": "4.0.3"
   },
   "dependencies": {
-    "yargs": "^15.3.1"
+    "yargs": "^16.1.0"
   }
 }

README.md

@@ -43,3 +43,7 @@ <p align="center"><img src="http://dzcpy.github.io/transliteration/transliteration.png" alt="Transliteration"></p>
 <!-- UMD build -->
-<script async defer src="https://cdn.jsdelivr.net/npm/transliteration@2.1.8/dist/browser/bundle.umd.min.js"></script>
+<script
+  async
+  defer
+  src="https://cdn.jsdelivr.net/npm/transliteration@2.1.8/dist/browser/bundle.umd.min.js"
+></script>
 <script>
@@ -87,3 +91,3 @@ console.log(transliterate('你好'));
 
-__Options:__ (optional)
+**Options:** (optional)
 
@@ -119,2 +123,7 @@ ```javascript
   unknown?: string;
+  /**
+   * Fix Chinese spacing. For example, `你好` is transliterated to `Ni Hao` instead of `NiHao`. If you don't need to transliterate Chinese characters, set it to false to false to improve performance.
+   * @default true
+   */
+  fixChineseSpacing?: boolean;
 }
@@ -135,8 +144,8 @@ ```
 // annyeonghaseyo, segye
-tr('你好，世界', { replace: {你: 'You'}, ignore: ['好'] });
-// You 好, Shi Jie
+tr('你好，世界', { replace: { 你: 'You' }, ignore: ['好'] });
+// You 好,Shi Jie
 tr('你好，世界', { replace: [['你', 'You']], ignore: ['好'] });
- // You 好, Shi Jie (option in array form)
+// You 好,Shi Jie (option in array form)
 tr.config({ replace: [['你', 'You']], ignore: ['好'] });
-tr('你好，世界') // You 好, Shi Jie
+tr('你好，世界'); // You 好,Shi Jie
 console.log(tr.config());
@@ -153,3 +162,3 @@ // { replace: [['你', 'You']], ignore: ['好'] }
 
-__Options:__ (optional)
+**Options:** (optional)
 
@@ -206,2 +215,6 @@ ```javascript
   allowedChars?: string;
+  /**
+   * Fix Chinese spacing. For example, `你好` is transliterated to `Ni Hao` instead of `NiHao`. If you don't need to transliterate Chinese characters, set it to false to false to improve performance.
+   */
+  fixChineseSpacing?: boolean;
 ```
@@ -214,5 +227,14 @@
 // Ni_Hao_Shi_Jie
-slugify('你好，世界', { replace: {你好: 'Hello', 世界: 'world'}, separator: '_' });
+slugify('你好，世界', {
+  replace: { 你好: 'Hello', 世界: 'world' },
+  separator: '_',
+});
 // hello_world
-slugify('你好，世界', { replace: [['你好', 'Hello'], ['世界', 'world']], separator: '_' }); // replace option in array form)
+slugify('你好，世界', {
+  replace: [
+    ['你好', 'Hello'],
+    ['世界', 'world'],
+  ],
+  separator: '_',
+}); // replace option in array form)
 // hello_world
@@ -241,3 +263,2 @@ slugify('你好，世界', { ignore: ['你好'] });
 // {}
-
 ```
@@ -273,3 +294,3 @@
   -U, --unknown    Placeholder for unknown characters                         [string] [default: ""]
-  -l, --lowercase  Peturns result in lowercase                             [boolean] [default: true]
+  -l, --lowercase  Returns result in lowercase                             [boolean] [default: true]
   -u, --uppercase  Returns result in uppercase                            [boolean] [default: false]
@@ -291,36 +312,2 @@ -s, --separator  Separator of the slug                                     [string] [default: "-"]
 
-## Change log
-
-### 2.1.0
-
-* Add `transliterate` as a global variable for browser builds. Keep `transl` for backward compatibility.
-
-### 2.0.0
-
-* **CDN file structure changed**: [https://www.jsdelivr.com/package/npm/transliteration](https://www.jsdelivr.com/package/npm/transliteration)
-* The entire module had been refactored in Typescript, with big performance improvements as well as a reduced package size.
-* Better code quality. 100% unit tested.
-* `bower` support was dropped. Please use CDN or `webpack`/`rollup`.
-* As according to RFC 3986, more characters(`/a-zA-Z0-9-_.~/`) are kept as allowed characters in the result for `slugify`, and it is configurable.
-* Added `uppercase` as an option for `slugify`, if is set to `true` then the generated slug will be converted to uppercase letters.
-* Unknown characters will be transliterated as empty string by default, instead of a meaningless `[?]`.
-
-
-### 1.6.6
-
-* Added support for `TypeScript`. #77
-
-### 1.5.0
-
-* Minimum node requirement: 6.0+
-
-### 1.0.0
-
-* Code had been entirely refactored since version 1.0.0. Be careful when you plan to upgrade from v0.1.x or v0.2.x to v1.0.x
-* The `options` parameter of `transliterate` now is an `Object` (In 0.1.x it's a string `unknown`).
-* Added `transliterate.config` and `slugify.config`.
-* Unknown string will be transliterated as `[?]` instead of `?`.
-* In the browser, global variables have been changed to `window.transl` and `windnow.slugify`. Other global variables are removed.
-
-
 ## Caveats
@@ -330,9 +317,9 @@
 
-* __Chinese:__ Polyphonic characters are not always transliterated correctly. Alternative: `pinyin`.
+- **Chinese:** Polyphonic characters are not always transliterated correctly. Alternative: `pinyin`.
 
-* __Japanese:__ Most Japanese Kanji characters are transliterated into Chinese Pinyin because of the overlapped code map in Unicode. Also there are many polyphonic characters in Japanese which makes it impossible to transliterate Japanese Kanji correctly without tokenizing the sentence. Consider using `kuroshiro` for a better Kanji -> Romaji conversion.
+- **Japanese:** Most Japanese Kanji characters are transliterated into Chinese Pinyin because of the overlapped code map in Unicode. Also there are many polyphonic characters in Japanese which makes it impossible to transliterate Japanese Kanji correctly without tokenizing the sentence. Consider using `kuroshiro` for a better Kanji -> Romaji conversion.
 
-* __Thai:__ Currently it is not working. If you know how to fix it, please comment on [this](https://github.com/dzcpy/transliteration/issues/67) issue.
+- **Thai:** Currently it is not working. If you know how to fix it, please comment on [this](https://github.com/dzcpy/transliteration/issues/67) issue.
 
-* __Cyrillic:__ Cyrillic characters are overlapped between a few languages. The result might be inaccurate in some specific languages, for example Bulgarian.
+- **Cyrillic:** Cyrillic characters are overlapped between a few languages. The result might be inaccurate in some specific languages, for example Bulgarian.
 
@@ -339,0 +326,0 @@ If you find any other issues, please raise a ticket.

New alerts

License Policy Violation

License

This package is not allowed per your license policy. Review the package's license to ensure compliance.

Found 1 instance in 1 package

Fixed alerts

License Policy Violation

License

This package is not allowed per your license policy. Review the package's license to ensure compliance.

Found 1 instance in 1 package

Improved metrics

Total package byte prevSize

1535795

increased by1.56%

Number of package files

22

increased by4.76%

Lines of code

3797

increased by1.99%

Worsened metrics

Number of lines in readme file

320

decreased by-3.9%

Dependency changes

• + Addedcliui@7.0.4(transitive)

• + Addedescalade@3.2.0(transitive)

• + Addedwrap-ansi@7.0.0(transitive)

• + Addedy18n@5.0.8(transitive)

• + Addedyargs@16.2.0(transitive)

• + Addedyargs-parser@20.2.9(transitive)

• - Removedcamelcase@5.3.1(transitive)

• - Removedcliui@6.0.0(transitive)

• - Removeddecamelize@1.2.0(transitive)

• - Removedfind-up@4.1.0(transitive)

• - Removedlocate-path@5.0.0(transitive)

• - Removedp-limit@2.3.0(transitive)

• - Removedp-locate@4.1.0(transitive)

• - Removedp-try@2.2.0(transitive)

• - Removedpath-exists@4.0.0(transitive)

• - Removedrequire-main-filename@2.0.0(transitive)

• - Removedset-blocking@2.0.0(transitive)

• - Removedwhich-module@2.0.1(transitive)

• - Removedwrap-ansi@6.2.0(transitive)

• - Removedy18n@4.0.3(transitive)

• - Removedyargs@15.4.1(transitive)

• - Removedyargs-parser@18.1.3(transitive)

• Updatedyargs@^16.1.0