@textlint/text-to-ast - npm Package Compare versions

Comparing version 3.1.6 to 3.1.7

CHANGELOG.md

@@ -6,2 +6,27 @@ # Change Log
 
+<a name="3.1.7"></a>
+## [3.1.7](https://github.com/textlint/textlint/compare/@textlint/text-to-ast@3.1.6...@textlint/text-to-ast@3.1.7) (2020-02-01)
+
+
+### Bug Fixes
+
+* **text-to-ast:** support CRLF line break ([dec1b39](https://github.com/textlint/textlint/commit/dec1b39))
+
+
+### Chores
+
+* **text-to-ast:** force convert to crlf ([14673ed](https://github.com/textlint/textlint/commit/14673ed))
+* remove unused modules ([6c8bb3f](https://github.com/textlint/textlint/commit/6c8bb3f))
+* **deps:** remove string.prototype.matchall ([9aa5de3](https://github.com/textlint/textlint/commit/9aa5de3))
+
+
+### Tests
+
+* **text-to-ast:** add \r\n test case ([c853e6f](https://github.com/textlint/textlint/commit/c853e6f))
+* **text-to-ast:** support empty text ([6195e85](https://github.com/textlint/textlint/commit/6195e85))
+
+
+
+
+
 <a name="3.1.6"></a>
@@ -8,0 +33,0 @@ ## [3.1.6](https://github.com/textlint/textlint/compare/@textlint/text-to-ast@3.1.5...@textlint/text-to-ast@3.1.6) (2019-10-14)

lib/plaintext-parser.js

@@ -5,3 +5,3 @@ // LICENSE : MIT
 var Syntax = require("./plaintext-syntax");
-var LINEBREAKE_MARK = /\r?\n/g;
+
 function parseLine(lineText, lineNumber, startIndex) {
@@ -27,2 +27,3 @@ // Inline Node have `value`. It it not part of TxtNode.
 }
+
 /**
@@ -32,8 +33,8 @@ * create BreakNode next to StrNode
  */
-function createEndedBRNode(prevNode) {
+function createEndedBRNode(prevNode, lineBreakText) {
     return {
         type: Syntax.Break,
-        raw: "\n",
-        value: "\n",
-        range: [prevNode.range[1], prevNode.range[1] + 1],
+        raw: lineBreakText,
+        value: lineBreakText,
+        range: [prevNode.range[1], prevNode.range[1] + lineBreakText.length],
         loc: {
@@ -46,3 +47,3 @@ start: {
                 line: prevNode.loc.end.line,
-                column: prevNode.loc.end.column + 1
+                column: prevNode.loc.end.column + lineBreakText.length
             }
@@ -52,2 +53,3 @@ }
 }
+
 /**
@@ -73,2 +75,3 @@ * create BreakNode next to StrNode
 }
+
 /**
@@ -102,2 +105,24 @@ * create paragraph node from TxtNodes
 
+function splitTextByLine(text) {
+    var LINEBREAKE_MARK_PATTERN = /\r?\n/g;
+    var results = [];
+    var match = null;
+    var prevMatchIndex = 0;
+    while ((match = LINEBREAKE_MARK_PATTERN.exec(text)) !== null) {
+        var slicedText = text.slice(prevMatchIndex, match.index);
+        results.push({
+            text: prevMatchIndex === match.index ? "" : slicedText,
+            lineBreak: match[0]
+        });
+        prevMatchIndex = match.index + match[0].length;
+    }
+    if (text.length !== prevMatchIndex) {
+        results.push({
+            text: text.slice(prevMatchIndex, text.length),
+            lineBreak: null
+        });
+    }
+    return results;
+}
+
 /**
@@ -109,15 +134,15 @@ * parse text and return ast mapped location info.
 function parse(text) {
-    var textLineByLine = text.split(LINEBREAKE_MARK);
+    var textLineByLine = splitTextByLine(text);
     // it should be alternately Str and Break
     var startIndex = 0;
     var lastLineIndex = textLineByLine.length - 1;
-    var isLasEmptytLine = function isLasEmptytLine(line, index) {
-        return index === lastLineIndex && line === "";
+    var isLastEmptyLine = function isLastEmptyLine(line, index) {
+        return index === lastLineIndex && line.text === "";
     };
     var isEmptyLine = function isEmptyLine(line, index) {
-        return index !== lastLineIndex && line === "";
+        return index !== lastLineIndex && line.text === "";
     };
     var children = textLineByLine.reduce(function (result, currentLine, index) {
         var lineNumber = index + 1;
-        if (isLasEmptytLine(currentLine, index)) {
+        if (isLastEmptyLine(currentLine, index)) {
             return result;
@@ -134,8 +159,11 @@ }
         // (Paragraph > Str) -> Br?
-        var strNode = parseLine(currentLine, lineNumber, startIndex);
+        var strNode = parseLine(currentLine.text, lineNumber, startIndex);
         var paragraph = createParagraph([strNode]);
         startIndex += paragraph.raw.length;
         result.push(paragraph);
-        if (index !== lastLineIndex) {
-            var breakNode = createEndedBRNode(paragraph);
+        // add Break node with actual line break value
+        // It should support CRLF
+        // https://github.com/textlint/textlint/issues/656
+        if (currentLine.lineBreak !== null) {
+            var breakNode = createEndedBRNode(paragraph, currentLine.lineBreak);
             startIndex += breakNode.raw.length;
@@ -146,2 +174,21 @@ result.push(breakNode);
     }, []);
+    var lastLine = textLineByLine[textLineByLine.length - 1];
+    if (lastLine === undefined) {
+        return {
+            type: Syntax.Document,
+            raw: "",
+            range: [0, 0],
+            loc: {
+                start: {
+                    line: 1,
+                    column: 0
+                },
+                end: {
+                    line: 1,
+                    column: 0
+                }
+            },
+            children: children
+        };
+    }
     return {
@@ -156,5 +203,10 @@ type: Syntax.Document,
             },
-            end: {
+            end:
+            // if Last Line has line break
+            lastLine.lineBreak !== null ? {
+                line: textLineByLine.length + 1,
+                column: 0
+            } : {
                 line: textLineByLine.length,
-                column: textLineByLine[textLineByLine.length - 1].length
+                column: lastLine.text.length
             }
@@ -165,3 +217,4 @@ },
 }
+
 module.exports = parse;
 //# sourceMappingURL=plaintext-parser.js.map

package.json

 {
   "name": "@textlint/text-to-ast",
-  "version": "3.1.6",
+  "version": "3.1.7",
   "description": "Parse plain text to AST with location info.",
@@ -34,2 +34,4 @@ "keywords": [
     "prepublish": "npm run --if-present build",
+    "crlf": "eolConverter crlf test/snapshots/crlf/input.txt",
+    "pretest": "npm run crlf",
     "test": "mocha \"test/**/*.{js,ts}\"",
@@ -49,2 +51,3 @@ "watch": "babel src --out-dir lib --watch --source-maps"
     "cross-env": "^6.0.3",
+    "eol-converter-cli": "^1.0.8",
     "mocha": "^6.2.1",
@@ -57,3 +60,3 @@ "power-assert": "^1.6.1",
   },
-  "gitHead": "ec9110a9ee0d3fc5b1af93abcc083b447e3c9b78"
+  "gitHead": "1fe466505d1760dcee8d57c91a654f1a7c0f70aa"
 }

src/plaintext-parser.js

 // LICENSE : MIT
 "use strict";
 const Syntax = require("./plaintext-syntax");
-const LINEBREAKE_MARK = /\r?\n/g;
+
 function parseLine(lineText, lineNumber, startIndex) {
@@ -25,2 +25,3 @@ // Inline Node have `value`. It it not part of TxtNode.
 }
+
 /**
@@ -30,8 +31,8 @@ * create BreakNode next to StrNode
  */
-function createEndedBRNode(prevNode) {
+function createEndedBRNode(prevNode, lineBreakText) {
     return {
         type: Syntax.Break,
-        raw: "\n",
-        value: "\n",
-        range: [prevNode.range[1], prevNode.range[1] + 1],
+        raw: lineBreakText,
+        value: lineBreakText,
+        range: [prevNode.range[1], prevNode.range[1] + lineBreakText.length],
         loc: {
@@ -44,3 +45,3 @@ start: {
                 line: prevNode.loc.end.line,
-                column: prevNode.loc.end.column + 1
+                column: prevNode.loc.end.column + lineBreakText.length
             }
@@ -50,2 +51,3 @@ }
 }
+
 /**
@@ -71,2 +73,3 @@ * create BreakNode next to StrNode
 }
+
 /**
@@ -102,2 +105,24 @@ * create paragraph node from TxtNodes
 
+function splitTextByLine(text) {
+    const LINEBREAKE_MARK_PATTERN = /\r?\n/g;
+    const results = [];
+    let match = null;
+    let prevMatchIndex = 0;
+    while ((match = LINEBREAKE_MARK_PATTERN.exec(text)) !== null) {
+        const slicedText = text.slice(prevMatchIndex, match.index);
+        results.push({
+            text: prevMatchIndex === match.index ? "" : slicedText,
+            lineBreak: match[0]
+        });
+        prevMatchIndex = match.index + match[0].length;
+    }
+    if (text.length !== prevMatchIndex) {
+        results.push({
+            text: text.slice(prevMatchIndex, text.length),
+            lineBreak: null
+        });
+    }
+    return results;
+}
+
 /**
@@ -109,15 +134,15 @@ * parse text and return ast mapped location info.
 function parse(text) {
-    const textLineByLine = text.split(LINEBREAKE_MARK);
+    const textLineByLine = splitTextByLine(text);
     // it should be alternately Str and Break
     let startIndex = 0;
     const lastLineIndex = textLineByLine.length - 1;
-    const isLasEmptytLine = (line, index) => {
-        return index === lastLineIndex && line === "";
+    const isLastEmptyLine = (line, index) => {
+        return index === lastLineIndex && line.text === "";
     };
     const isEmptyLine = (line, index) => {
-        return index !== lastLineIndex && line === "";
+        return index !== lastLineIndex && line.text === "";
     };
     const children = textLineByLine.reduce(function(result, currentLine, index) {
         const lineNumber = index + 1;
-        if (isLasEmptytLine(currentLine, index)) {
+        if (isLastEmptyLine(currentLine, index)) {
             return result;
@@ -134,8 +159,11 @@ }
         // (Paragraph > Str) -> Br?
-        const strNode = parseLine(currentLine, lineNumber, startIndex);
+        const strNode = parseLine(currentLine.text, lineNumber, startIndex);
         const paragraph = createParagraph([strNode]);
         startIndex += paragraph.raw.length;
         result.push(paragraph);
-        if (index !== lastLineIndex) {
-            const breakNode = createEndedBRNode(paragraph);
+        // add Break node with actual line break value
+        // It should support CRLF
+        // https://github.com/textlint/textlint/issues/656
+        if (currentLine.lineBreak !== null) {
+            const breakNode = createEndedBRNode(paragraph, currentLine.lineBreak);
             startIndex += breakNode.raw.length;
@@ -146,2 +174,21 @@ result.push(breakNode);
     }, []);
+    const lastLine = textLineByLine[textLineByLine.length - 1];
+    if (lastLine === undefined) {
+        return {
+            type: Syntax.Document,
+            raw: "",
+            range: [0, 0],
+            loc: {
+                start: {
+                    line: 1,
+                    column: 0
+                },
+                end: {
+                    line: 1,
+                    column: 0
+                }
+            },
+            children
+        };
+    }
     return {
@@ -156,6 +203,13 @@ type: Syntax.Document,
             },
-            end: {
-                line: textLineByLine.length,
-                column: textLineByLine[textLineByLine.length - 1].length
-            }
+            end:
+                // if Last Line has line break
+                lastLine.lineBreak !== null
+                    ? {
+                          line: textLineByLine.length + 1,
+                          column: 0
+                      }
+                    : {
+                          line: textLineByLine.length,
+                          column: lastLine.text.length
+                      }
         },
@@ -165,2 +219,3 @@ children
 }
+
 module.exports = parse;

No alert changes

Improved metrics

Total package byte prevSize

36995

increased by11.99%

Lines of code

435

increased by12.99%

Worsened metrics

Dev dependency count

11

increased by10%

Number of package files

13

decreased by-18.75%

No dependency changes