@jscpd/tokenizer - npm Package Compare versions

Comparing version 3.3.0-rc.3 to 3.3.0-rc.4

dist/formats.js

@@ -78,2 +78,5 @@ "use strict";
     },
+    comments: {
+        exts: []
+    },
     cpp: {
@@ -80,0 +83,0 @@ exts: ['cpp', 'c++', 'cc', 'cxx'],

dist/prism.d.ts

@@ -1,5 +0,4 @@
-import { IToken } from './interfaces';
 import { TokensMap } from './token-map';
-import { IOptions } from '@jscpd/core';
+import { IOptions, IToken } from '@jscpd/core';
 export declare function tokenize(code: string, language: string): IToken[];
 export declare function createTokenMapBasedOnCode(id: string, data: string, format: string, options?: Partial<IOptions>): TokensMap[];

dist/prism.js

 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
-const prismjs_1 = require("prismjs");
+const Prism = require("prismjs");
 const formats_1 = require("./formats");
 const token_map_1 = require("./token-map");
-const loadLanguages = require('prismjs/components/');
-const ignore = {
-    ignore: [
-        {
-            pattern: /(jscpd:ignore-start)[\s\S]*?(?=jscpd:ignore-end)/,
-            lookbehind: true,
-            greedy: true,
-        },
-        {
-            pattern: /jscpd:ignore-start/,
-            greedy: false,
-        },
-        {
-            pattern: /jscpd:ignore-end/,
-            greedy: false,
-        },
-    ],
+const grammar_loader_1 = require("./grammar-loader");
+const initializeFormats = () => {
+    grammar_loader_1.loadLanguages();
+    const ignore = {
+        ignore: [
+            {
+                pattern: /(jscpd:ignore-start)[\s\S]*?(?=jscpd:ignore-end)/,
+                lookbehind: true,
+                greedy: true,
+            },
+            {
+                pattern: /jscpd:ignore-start/,
+                greedy: false,
+            },
+            {
+                pattern: /jscpd:ignore-end/,
+                greedy: false,
+            },
+        ],
+    };
+    const punctuation = {
+        // eslint-disable-next-line @typescript-eslint/camelcase
+        new_line: /\n/,
+        empty: /\s+/,
+    };
+    // eslint-disable-next-line @typescript-eslint/no-explicit-any
+    Prism.languages.markup.script.inside = Object.assign(Object.assign(Object.assign({}, ignore), Prism.languages.markup.script.inside), punctuation);
+    // eslint-disable-next-line @typescript-eslint/no-explicit-any
+    Prism.languages.markup.style.inside = Object.assign(Object.assign(Object.assign({}, ignore), Prism.languages.markup.style.inside), punctuation);
+    Object.keys(Prism.languages).forEach((lang) => {
+        Prism.languages[lang] =
+            typeof Prism.languages[lang] === 'object' ? Object.assign(Object.assign(Object.assign({}, ignore), Prism.languages[lang]), punctuation) :
+                Prism.languages[lang];
+    });
 };
-const punctuation = {
-    // eslint-disable-next-line @typescript-eslint/camelcase
-    new_line: /\n/,
-    empty: /\s+/,
-};
-// eslint-disable-next-line @typescript-eslint/no-explicit-any
-prismjs_1.languages.markup.script.inside = Object.assign(Object.assign(Object.assign({}, ignore), prismjs_1.languages.markup.script.inside), punctuation);
-// eslint-disable-next-line @typescript-eslint/no-explicit-any
-prismjs_1.languages.markup.style.inside = Object.assign(Object.assign(Object.assign({}, ignore), prismjs_1.languages.markup.style.inside), punctuation);
+initializeFormats();
 function getLanguagePrismName(lang) {
@@ -39,10 +48,2 @@ if (lang in formats_1.FORMATS && formats_1.FORMATS[lang].parent) {
 }
-function initLanguages(langs) {
-    const langToLoad = langs.map(getLanguagePrismName);
-    loadLanguages(langToLoad);
-    Object.keys(prismjs_1.languages).forEach((lang) => {
-        prismjs_1.languages[lang] =
-            typeof prismjs_1.languages[lang] === 'object' ? Object.assign(Object.assign(Object.assign({}, ignore), prismjs_1.languages[lang]), punctuation) : prismjs_1.languages[lang];
-    });
-}
 function tokenize(code, language) {
@@ -52,3 +53,2 @@ let length = 0;
     let column = 1;
-    initLanguages([language]);
     function sanitizeLangName(name) {
@@ -97,6 +97,6 @@ return name && name.replace ? name.replace('language-', '') : 'unknown';
     function createTokens(token, lang) {
-        if (token instanceof prismjs_1.Token && typeof token.content === 'string') {
+        if (token instanceof Prism.Token && typeof token.content === 'string') {
             return createTokenFromFlatToken(token, lang);
         }
-        if (token instanceof prismjs_1.Token && Array.isArray(token.content)) {
+        if (token instanceof Prism.Token && Array.isArray(token.content)) {
             let res = [];
@@ -109,10 +109,8 @@ token.content.forEach((t) => (res = res.concat(createTokens(t, token.alias ? sanitizeLangName(token.alias) : lang))));
     let tokens = [];
-    prismjs_1.tokenize(code, prismjs_1.languages[getLanguagePrismName(language)]).forEach((t) => (tokens = tokens.concat(createTokens(t, language))));
-    return tokens.map(calculateLocation).filter((t) => {
-        return t.format !== 'important' && t.format !== 'property' && t.format !== 'url' && t.format !== 'class-name';
-    });
+    Prism.tokenize(code, Prism.languages[getLanguagePrismName(language)])
+        .forEach((t) => (tokens = tokens.concat(createTokens(t, language))));
+    return tokens.map(calculateLocation).filter((t) => t.format in formats_1.FORMATS);
 }
 exports.tokenize = tokenize;
 function createTokenMapBasedOnCode(id, data, format, options = {}) {
-    initLanguages([format]);
     const { mode, ignoreCase } = options;
@@ -119,0 +117,0 @@ const tokens = tokenize(data, format)

dist/token-map.d.ts

@@ -1,3 +0,2 @@
-import { IMapFrame, IToken } from '@jscpd/core';
-import { ITokensMap } from '@jscpd/core/src/interfaces/tokens-map.inerface';
+import { IMapFrame, IToken, ITokensMap } from '@jscpd/core';
 export declare class TokensMap implements ITokensMap, Iterator<IMapFrame | boolean>, Iterable<IMapFrame | boolean> {
@@ -4,0 +3,0 @@ private readonly id;

package.json

 {
   "name": "@jscpd/tokenizer",
-  "version": "3.3.0-rc.3",
+  "version": "3.3.0-rc.4",
   "description": "tokenizer of source code for jscpd",
@@ -28,4 +28,4 @@ "author": "Andrey Kucherenko <[e-mail]>",
     "compile": "tsc -p tsconfig.build.json",
-    "prepare": "ts-node ../publish-util.ts ./package.json dist/index",
-    "postpublish": "ts-node ../publish-util.ts ./package.json src/index",
+    "prepare": "ts-node ../../build-utils/publish-util.ts ./package.json dist/index",
+    "postpublish": "ts-node ../../build-utils/publish-util.ts ./package.json src/index",
     "prepublishOnly": "yarn build"
@@ -38,7 +38,7 @@ },
   "dependencies": {
-    "@jscpd/core": "^3.3.0-rc.3",
+    "@jscpd/core": "^3.3.0-rc.4",
     "prismjs": "^1.20.0",
     "spark-md5": "^3.0.1"
   },
-  "gitHead": "9f388ffc21fc8a6cbb407ee84b225b6e11e18e99"
+  "gitHead": "b5ed8e89d18005ef26d0100d039efd95a6d226db"
 }

New alerts

Dynamic require

Supply chain risk

Dynamic require can indicate the package is performing dangerous or unsafe dynamic code execution.

Found 1 instance in 1 package

Improved metrics

Total package byte prevSize

42671

increased by10.7%

Number of package files

31

increased by14.81%

Lines of code

822

increased by6.89%

Worsened metrics

Number of low supply chain risk alerts

1

increased byInfinity%

Dependency changes

• Updated@jscpd/core@^3.3.0-rc.4