linguist-js - npm Package Compare versions

Comparing version 2.7.1 to 2.8.0

dist/cli.js

@@ -27,2 +27,3 @@ "use strict";
     .option('-o|--offline [bool]', 'Use packaged data files instead of fetching latest from GitHub', false)
+    .option('-L|--calculateLines [bool]', 'Calculate lines of code totals', true)
     .option('-V|--keepVendored [bool]', 'Prevent skipping over vendored/generated files', false)
@@ -33,2 +34,3 @@ .option('-B|--keepBinary [bool]', 'Prevent skipping over binary files', false)
     .option('-I|--checkIgnored [bool]', 'Force the checking of gitignore files', true)
+    .option('-D|--checkDetected [bool]', 'Force files marked with linguist-detectable to always appear in output', true)
     .option('-H|--checkHeuristics [bool]', 'Apply heuristics to ambiguous languages', true)
@@ -84,3 +86,3 @@ .option('-S|--checkShebang [bool]', 'Check shebang lines for explicit classification', true)
             // List parsed results
-            for (const [lang, { bytes, color }] of sortedEntries) {
+            for (const [lang, { bytes, lines, color }] of sortedEntries) {
                 const percent = (bytes) => bytes / (totalBytes || 1) * 100;
@@ -92,5 +94,6 @@ const fmtd = {
                     bytes: bytes.toLocaleString().padStart(10, ' '),
+                    loc: lines.code.toLocaleString().padStart(10, ' '),
                     icon: colouredMsg(hexToRgb(color !== null && color !== void 0 ? color : '#ededed'), '\u2588'),
                 };
-                console.log(`  ${fmtd.index}. ${fmtd.icon} ${fmtd.lang} ${fmtd.percent}% ${fmtd.bytes} B`);
+                console.log(`  ${fmtd.index}. ${fmtd.icon} ${fmtd.lang} ${fmtd.percent}% ${fmtd.bytes} B ${fmtd.loc} LOC`);
                 // If using `listFiles` option, list all files tagged as this language
@@ -97,0 +100,0 @@ if (args.listFiles) {

dist/helpers/parse-gitattributes.d.ts

@@ -6,2 +6,3 @@ import * as T from '../types';
     'documentation': boolean | null;
+    'detectable': boolean | null;
     'binary': boolean | null;
@@ -8,0 +9,0 @@ 'language': T.LanguageResult;

dist/helpers/parse-gitattributes.js

@@ -24,6 +24,8 @@ "use strict";
         const hasFalseParts = (str) => falseParts(str).length > 0;
+        const boolOrNullVal = (str) => hasTrueParts(str) ? true : hasFalseParts(str) ? false : null;
         const attrs = {
-            'generated': hasTrueParts('linguist-generated') ? true : hasFalseParts('linguist-generated') ? false : null,
-            'vendored': hasTrueParts('linguist-vendored') ? true : hasFalseParts('linguist-vendored') ? false : null,
-            'documentation': hasTrueParts('linguist-documentation') ? true : hasFalseParts('linguist-documentation') ? false : null,
+            'generated': boolOrNullVal('linguist-generated'),
+            'vendored': boolOrNullVal('linguist-vendored'),
+            'documentation': boolOrNullVal('linguist-documentation'),
+            'detectable': boolOrNullVal('linguist-detectable'),
             'binary': hasTrueParts('binary') || hasFalseParts('text') ? true : hasFalseParts('binary') || hasTrueParts('text') ? false : null,
@@ -30,0 +32,0 @@ 'language': (_b = (_a = trueParts('linguist-language').at(-1)) === null || _a === void 0 ? void 0 : _a.split('=')[1]) !== null && _b !== void 0 ? _b : null,

dist/helpers/read-file.d.ts

@@ -5,2 +5,2 @@ /**
  */
-export default function readFile(filename: string, onlyFirstLine?: boolean): Promise<string>;
+export default function readFileChunk(filename: string, onlyFirstLine?: boolean): Promise<string>;

dist/helpers/read-file.js

@@ -11,3 +11,3 @@ "use strict";
  */
-async function readFile(filename, onlyFirstLine = false) {
+async function readFileChunk(filename, onlyFirstLine = false) {
     const chunkSize = 100;
@@ -24,2 +24,2 @@ const stream = fs_1.default.createReadStream(filename, { highWaterMark: chunkSize });
 }
-exports.default = readFile;
+exports.default = readFileChunk;

dist/index.js

@@ -42,4 +42,4 @@ "use strict";
 async function analyse(rawPaths, opts = {}) {
-    var _a, _b, _c, _d, _e, _f, _g, _h, _j, _k, _l, _m, _o, _p, _q;
-    var _r, _s;
+    var _a, _b, _c, _d, _e, _f, _g, _h, _j, _k, _l, _m, _o, _p, _q, _r, _s, _t;
+    var _u, _v;
     const useRawContent = opts.fileContent !== undefined;
@@ -50,3 +50,5 @@ const input = [rawPaths !== null && rawPaths !== void 0 ? rawPaths : []].flat();
     opts = {
+        calculateLines: (_b = opts.calculateLines) !== null && _b !== void 0 ? _b : true,
         checkIgnored: !opts.quick,
+        checkDetected: !opts.quick,
         checkAttributes: !opts.quick,
@@ -70,5 +72,5 @@ checkHeuristics: !opts.quick,
     const results = {
-        files: { count: 0, bytes: 0, results: {}, alternatives: {} },
-        languages: { count: 0, bytes: 0, results: {} },
-        unknown: { count: 0, bytes: 0, extensions: {}, filenames: {} },
+        files: { count: 0, bytes: 0, lines: { total: 0, content: 0, code: 0 }, results: {}, alternatives: {} },
+        languages: { count: 0, bytes: 0, lines: { total: 0, content: 0, code: 0 }, results: {} },
+        unknown: { count: 0, bytes: 0, lines: { total: 0, content: 0, code: 0 }, extensions: {}, filenames: {} },
     };
@@ -87,3 +89,3 @@ // Set a common root path so that vendor paths do not incorrectly match parent folders
     ignored.add('.git/');
-    ignored.add((_b = opts.ignoredFiles) !== null && _b !== void 0 ? _b : []);
+    ignored.add((_c = opts.ignoredFiles) !== null && _c !== void 0 ? _c : []);
     const regexIgnores = opts.keepVendored ? [] : vendorPaths.map(path => RegExp(path, 'i'));
@@ -183,3 +185,3 @@ // Load file paths and folders
     // Ignore specific languages
-    for (const lang of (_c = opts.ignoredLanguages) !== null && _c !== void 0 ? _c : []) {
+    for (const lang of (_d = opts.ignoredLanguages) !== null && _d !== void 0 ? _d : []) {
         for (const key in langData) {
@@ -237,3 +239,3 @@ if (lang.toLowerCase() === key.toLowerCase()) {
         if (useRawContent) {
-            firstLine = (_e = (_d = manualFileContent[files.indexOf(file)]) === null || _d === void 0 ? void 0 : _d.split('\n')[0]) !== null && _e !== void 0 ? _e : null;
+            firstLine = (_f = (_e = manualFileContent[files.indexOf(file)]) === null || _e === void 0 ? void 0 : _e.split('\n')[0]) !== null && _f !== void 0 ? _f : null;
         }
@@ -257,3 +259,3 @@ else if (fs_1.default.existsSync(file) && !fs_1.default.lstatSync(file).isDirectory()) {
                 if (opts.checkShebang && hasShebang) {
-                    const matchesInterpretor = (_f = data.interpreters) === null || _f === void 0 ? void 0 : _f.some(interpreter => firstLine.match(`\\b${interpreter}\\b`));
+                    const matchesInterpretor = (_g = data.interpreters) === null || _g === void 0 ? void 0 : _g.some(interpreter => firstLine.match(`\\b${interpreter}\\b`));
                     if (matchesInterpretor)
@@ -266,3 +268,3 @@ matches.push(lang);
                     const matchesLang = modelineText.match(langMatcher(lang));
-                    const matchesAlias = (_g = data.aliases) === null || _g === void 0 ? void 0 : _g.some(lang => modelineText.match(langMatcher(lang)));
+                    const matchesAlias = (_h = data.aliases) === null || _h === void 0 ? void 0 : _h.some(lang => modelineText.match(langMatcher(lang)));
                     if (matchesLang || matchesAlias)
@@ -286,3 +288,3 @@ matches.push(lang);
         for (const lang in langData) {
-            const matchesName = (_h = langData[lang].filenames) === null || _h === void 0 ? void 0 : _h.some(name => path_1.default.basename(file.toLowerCase()) === name.toLowerCase());
+            const matchesName = (_j = langData[lang].filenames) === null || _j === void 0 ? void 0 : _j.some(name => path_1.default.basename(file.toLowerCase()) === name.toLowerCase());
             if (matchesName) {
@@ -297,3 +299,3 @@ addResult(file, lang);
             for (const lang in langData) {
-                const extMatches = (_j = langData[lang].extensions) === null || _j === void 0 ? void 0 : _j.filter(ext => file.toLowerCase().endsWith(ext.toLowerCase()));
+                const extMatches = (_k = langData[lang].extensions) === null || _k === void 0 ? void 0 : _k.filter(ext => file.toLowerCase().endsWith(ext.toLowerCase()));
                 if (extMatches === null || extMatches === void 0 ? void 0 : extMatches.length) {
@@ -344,3 +346,3 @@ for (const ext of extMatches)
                     // Make sure the results includes this language
-                    const languageGroup = (_k = langData[heuristic.language]) === null || _k === void 0 ? void 0 : _k.group;
+                    const languageGroup = (_l = langData[heuristic.language]) === null || _l === void 0 ? void 0 : _l.group;
                     const matchesLang = fileAssociations[file].includes(heuristic.language);
@@ -390,13 +392,19 @@ const matchesParent = languageGroup && fileAssociations[file].includes(languageGroup);
     // Skip specified categories
-    if ((_l = opts.categories) === null || _l === void 0 ? void 0 : _l.length) {
+    if ((_m = opts.categories) === null || _m === void 0 ? void 0 : _m.length) {
         const categories = ['data', 'markup', 'programming', 'prose'];
         const hiddenCategories = categories.filter(cat => !opts.categories.includes(cat));
         for (const [file, lang] of Object.entries(results.files.results)) {
-            if (!hiddenCategories.some(cat => { var _a; return lang && ((_a = langData[lang]) === null || _a === void 0 ? void 0 : _a.type) === cat; })) {
+            // Skip if language is not hidden
+            if (!hiddenCategories.some(cat => { var _a; return lang && ((_a = langData[lang]) === null || _a === void 0 ? void 0 : _a.type) === cat; }))
                 continue;
+            // Skip if language is forced as detectable
+            if (opts.checkDetected) {
+                const detectable = (0, ignore_1.default)().add(getFlaggedGlobs('detectable', true));
+                if (detectable.ignores(relPath(file)))
+                    continue;
             }
+            // Delete result otherwise
             delete results.files.results[file];
-            if (lang) {
+            if (lang)
                 delete results.languages.results[lang];
-            }
         }
@@ -427,23 +435,56 @@ for (const category of hiddenCategories) {
             continue;
-        const fileSize = (_o = (_m = manualFileContent[files.indexOf(file)]) === null || _m === void 0 ? void 0 : _m.length) !== null && _o !== void 0 ? _o : fs_1.default.statSync(file).size;
+        // Calculate file size
+        const fileSize = (_p = (_o = manualFileContent[files.indexOf(file)]) === null || _o === void 0 ? void 0 : _o.length) !== null && _p !== void 0 ? _p : fs_1.default.statSync(file).size;
+        // Calculate lines of code
+        const loc = { total: 0, content: 0, code: 0 };
+        if (opts.calculateLines) {
+            const fileContent = (_r = ((_q = manualFileContent[files.indexOf(file)]) !== null && _q !== void 0 ? _q : fs_1.default.readFileSync(file).toString())) !== null && _r !== void 0 ? _r : '';
+            const allLines = fileContent.split(/\r?\n/gm);
+            loc.total = allLines.length;
+            loc.content = allLines.filter(line => line.trim().length > 0).length;
+            const codeLines = fileContent
+                .replace(/^\s*(\/\/|# |;|--).+/gm, '')
+                .replace(/\/\*.+\*\/|<!--.+-->/sg, '');
+            loc.code = codeLines.split(/\r?\n/gm).filter(line => line.trim().length > 0).length;
+        }
+        // Apply to files totals
         results.files.bytes += fileSize;
-        // If no language found, add extension in other section
-        if (!lang) {
+        results.files.lines.total += loc.total;
+        results.files.lines.content += loc.content;
+        results.files.lines.code += loc.code;
+        // Add results to 'languages' section if language match found, or 'unknown' section otherwise
+        if (lang) {
+            const { type } = langData[lang];
+            // set default if unset
+            (_s = (_u = results.languages.results)[lang]) !== null && _s !== void 0 ? _s : (_u[lang] = { type, bytes: 0, lines: { total: 0, content: 0, code: 0 }, color: langData[lang].color });
+            // apply results to 'languages' section
+            if (opts.childLanguages) {
+                results.languages.results[lang].parent = langData[lang].group;
+            }
+            results.languages.results[lang].bytes += fileSize;
+            results.languages.bytes += fileSize;
+            results.languages.results[lang].lines.total += loc.total;
+            results.languages.results[lang].lines.content += loc.content;
+            results.languages.results[lang].lines.code += loc.code;
+            results.languages.lines.total += loc.total;
+            results.languages.lines.content += loc.content;
+            results.languages.lines.code += loc.code;
+        }
+        else {
             const ext = path_1.default.extname(file);
-            const unknownType = ext === '' ? 'filenames' : 'extensions';
-            const name = ext === '' ? path_1.default.basename(file) : ext;
-            (_p = (_r = results.unknown[unknownType])[name]) !== null && _p !== void 0 ? _p : (_r[name] = 0);
+            const unknownType = ext ? 'extensions' : 'filenames';
+            const name = ext || path_1.default.basename(file);
+            // apply results to 'unknown' section
+            (_t = (_v = results.unknown[unknownType])[name]) !== null && _t !== void 0 ? _t : (_v[name] = 0);
             results.unknown[unknownType][name] += fileSize;
             results.unknown.bytes += fileSize;
-            continue;
+            results.unknown.lines.total += loc.total;
+            results.unknown.lines.content += loc.content;
+            results.unknown.lines.code += loc.code;
         }
-        // Add language and bytes data to corresponding section
-        const { type } = langData[lang];
-        (_q = (_s = results.languages.results)[lang]) !== null && _q !== void 0 ? _q : (_s[lang] = { type, bytes: 0, color: langData[lang].color });
-        if (opts.childLanguages) {
-            results.languages.results[lang].parent = langData[lang].group;
-        }
-        results.languages.results[lang].bytes += fileSize;
-        results.languages.bytes += fileSize;
     }
+    // Set lines output to NaN when line calculation is disabled
+    if (opts.calculateLines === false) {
+        results.files.lines = { total: NaN, content: NaN, code: NaN };
+    }
     // Set counts
@@ -450,0 +491,0 @@ results.files.count = Object.keys(results.files.results).length;

dist/types.d.ts

@@ -22,3 +22,5 @@ export type LanguageResult = string | null;
     offline?: boolean;
+    calculateLines?: boolean;
     checkIgnored?: boolean;
+    checkDetected?: boolean;
     checkAttributes?: boolean;
@@ -33,2 +35,7 @@ checkHeuristics?: boolean;
         bytes: Bytes;
+        lines: {
+            total: Integer;
+            content: Integer;
+            code: Integer;
+        };
         /** Note: Results use slashes as delimiters even on Windows. */
@@ -41,4 +48,14 @@ results: Record<FilePath, LanguageResult>;
         bytes: Bytes;
+        lines: {
+            total: Integer;
+            content: Integer;
+            code: Integer;
+        };
         results: Record<Language, {
             bytes: Bytes;
+            lines: {
+                total: Integer;
+                content: Integer;
+                code: Integer;
+            };
             type: Category;
@@ -52,2 +69,7 @@ parent?: Language;
         bytes: Bytes;
+        lines: {
+            total: Integer;
+            content: Integer;
+            code: Integer;
+        };
         extensions: Record<string, Bytes>;
@@ -54,0 +76,0 @@ filenames: Record<string, Bytes>;

package.json

 {
 	"name": "linguist-js",
-	"version": "2.7.1",
+	"version": "2.8.0",
 	"description": "Analyse languages used in a folder. Powered by GitHub Linguist, although it doesn't need to be installed.",
@@ -14,3 +14,3 @@ "main": "dist/index.js",
 	"scripts": {
-		"download-files": "npx tsx@4 build/download-files",
+		"download-files": "npx tsx@3 build/download-files",
 		"pre-publish": "npm run download-files && npm test && npm run perf",
@@ -42,7 +42,7 @@ "perf": "tsc && node test/perf",
 	"dependencies": {
-		"binary-extensions": "^2.2.0 <3",
+		"binary-extensions": "^2.3.0 <3",
 		"commander": "^9.5.0 <10",
 		"common-path-prefix": "^3.0.0",
 		"cross-fetch": "^3.1.8 <4",
-		"ignore": "^5.3.1",
+		"ignore": "^5.3.2",
 		"isbinaryfile": "^4.0.10 <5",
@@ -49,0 +49,0 @@ "js-yaml": "^4.1.0",

readme.md

@@ -53,2 +53,7 @@ [![Latest version](https://img.shields.io/github/v/release/Nixinova/Linguist?label=latest%20version&style=flat-square)](https://github.com/Nixinova/Linguist/releases)
     "bytes": 6020,
+	"lines": {
+		"total": 100,
+		"content": 90,
+		"code": 80,
+	},
     "results": {
@@ -68,7 +73,12 @@ "/src/index.ts": "TypeScript",
     "bytes": 6010,
+	"lines": {
+		"total": 90,
+		"content": 80,
+		"code": 70,
+	},
     "results": {
-        "JavaScript": { "type": "programming", "bytes": 1000, "color": "#f1e05a" },
-        "Markdown": { "type": "prose", "bytes": 3000, "color": "#083fa1" },
-        "Ruby": { "type": "programming", "bytes": 10, "color": "#701516" },
-        "TypeScript": { "type": "programming", "bytes": 2000, "color": "#2b7489" },
+        "JavaScript": { "type": "programming", "bytes": 1000, "lines": { "total": 49, "content": 49, "code": 44 }, "color": "#f1e05a" },
+        "Markdown": { "type": "prose", "bytes": 3000, "lines": { "total": 10, "content": 5, "code": 5 }, "color": "#083fa1" },
+        "Ruby": { "type": "programming", "bytes": 10, "lines": { "total": 1, "content": 1, "code": 1 }, "color": "#701516" },
+        "TypeScript": { "type": "programming", "bytes": 2000, "lines": { "total": 30, "content": 25, "code": 20 }, "color": "#2b7489" },
     },
@@ -79,2 +89,7 @@ },
     "bytes": 10,
+	"lines": {
+		"total": 10,
+		"content": 10,
+		"code": 10,
+	},
     "filenames": {
@@ -133,5 +148,7 @@ "no-lang": 10,
       Whether to skip complex language analysis such as the checking of heuristics and gitattributes statements (defaults to `false`).
-      Alias for `checkAttributes:false, checkIgnored:false, checkHeuristics:false, checkShebang:false, checkModeline:false`.
+      Alias for `checkAttributes:false, checkIgnored:false, checkDetected:false, checkHeuristics:false, checkShebang:false, checkModeline:false`.
     - `offline` (boolean):
       Whether to use pre-packaged metadata files instead of fetching them from GitHub at runtime (defaults to `false`).
+    - `calculateLines` (boolean):
+      Whether to calculate line of code totals (defaults to `true`).
     - `keepVendored` (boolean):
@@ -150,2 +167,4 @@ Whether to keep vendored files (dependencies, etc) (defaults to `false`).
       Does nothing when `fileContent` is set.
+    - `checkDetected` (boolean):
+      Force files marked with `linguist-detectable` to show up in the output, even if the file is not part of the declared `categories`.
     - `checkHeuristics` (boolean):
@@ -195,2 +214,4 @@ Apply heuristics to ambiguous languages (defaults to `true` unless `quick` is set).
     Use pre-packaged metadata files instead of fetching them from GitHub at runtime.
+  - `--calculateLines`:
+    Calculate line of code totals from files.
   - `--keepVendored`:
@@ -208,2 +229,5 @@ Include vendored files (auto-generated files, dependencies folder, etc) in the output.
     Use alongside `--quick` to override it disabling this option.
+  - `--checkDetected`:
+    Force files marked with `linguist-detectable` to show up in the output, even if the file is not part of the declared `--categories`.
+    Use alongside `--quick` to override it disabling this option.
   - `--checkHeuristics`:
@@ -210,0 +234,0 @@ Apply heuristics to ambiguous languages.

New alerts

License Policy Violation

License

This package is not allowed per your license policy. Review the package's license to ensure compliance.

Found 1 instance in 1 package

Fixed alerts

License Policy Violation

License

This package is not allowed per your license policy. Review the package's license to ensure compliance.

Found 1 instance in 1 package

Improved metrics

Total package byte prevSize

239410

increased by4.15%

Lines of code

1050

increased by7.03%

Number of lines in readme file

239

increased by11.16%

Dependency changes

• Updatedbinary-extensions@^2.3.0 <3

• Updatedignore@^5.3.2