jsonrepair - npm Package Compare versions

Comparing version 3.4.1 to 3.5.0

bin/cli.js

 #!/usr/bin/env node
-import { createReadStream, createWriteStream, readFileSync } from 'fs'
-import { dirname, join } from 'path'
-import { fileURLToPath } from 'url'
-import { jsonrepair } from '../lib/esm/jsonrepair.js'
+import { createReadStream, createWriteStream, readFileSync, renameSync } from 'node:fs'
+import { pipeline as pipelineCallback } from 'node:stream'
+import { dirname, join } from 'node:path'
+import { fileURLToPath } from 'node:url'
+import { promisify } from 'node:util'
+import { jsonrepairTransform } from '../lib/esm/stream.js'
 
+const pipeline = promisify(pipelineCallback)
 const __filename = fileURLToPath(import.meta.url)
 const __dirname = dirname(__filename)
 
-function outputVersion() {
-  const file = join(__dirname, '../package.json')
-  const pkg = JSON.parse(String(readFileSync(file, 'utf-8')))
-
-  console.log(pkg.version)
-}
-
-function outputHelp() {
-  console.log('jsonrepair')
-  console.log('https://github.com/josdejong/jsonrepair')
-  console.log()
-  console.log(
-    'Repair invalid JSON documents. When a document could not be repaired, the output will be left unchanged.'
-  )
-  console.log()
-  console.log('Usage:')
-  console.log('    jsonrepair [filename] {OPTIONS}')
-  console.log()
-  console.log('Options:')
-  console.log('    --version, -v       Show application version')
-  console.log('    --help,    -h       Show this message')
-  console.log()
-  console.log('Example usage:')
-  console.log(
-    '    jsonrepair broken.json                        # Repair a file, output to console'
-  )
-  console.log('    jsonrepair broken.json > repaired.json        # Repair a file, output to file')
-  console.log(
-    '    jsonrepair broken.json --overwrite            # Repair a file, replace the file itself'
-  )
-  console.log(
-    '    cat broken.json | jsonrepair                  # Repair data from an input stream'
-  )
-  console.log(
-    '    cat broken.json | jsonrepair > repaired.json  # Repair data from an input stream, output to file'
-  )
-  console.log()
-}
-
-function streamToString(readableStream) {
-  return new Promise((resolve, reject) => {
-    let text = ''
-
-    readableStream.on('data', (chunk) => {
-      text += String(chunk)
-    })
-    readableStream.on('end', () => {
-      readableStream.destroy()
-      resolve(text)
-    })
-    readableStream.on('error', (err) => reject(err))
-  })
-}
-
 function processArgs(args) {
@@ -69,7 +18,12 @@ const options = {
     overwrite: false,
-    inputFile: null
+    bufferSize: undefined,
+    inputFile: null,
+    outputFile: null
   }
 
   // we skip the first two args, since they contain node and the script path
-  args.slice(2).forEach(function (arg) {
+  let i = 2
+  while (i < args.length) {
+    const arg = args[i]
+
     switch (arg) {
@@ -90,2 +44,13 @@ case '-v':
 
+      case '--buffer':
+        i++
+        options.bufferSize = parseSize(args[i])
+        break
+
+      case '-o':
+      case '--output':
+        i++
+        options.outputFile = args[i]
+        break
+
       default:
@@ -98,30 +63,119 @@ if (options.inputFile == null) {
     }
-  })
 
+    i++
+  }
+
   return options
 }
 
-const options = processArgs(process.argv)
+async function run(options) {
+  if (options.version) {
+    outputVersion()
+    return
+  }
 
-if (options.version) {
-  outputVersion()
-} else if (options.help) {
-  outputHelp()
-} else if (options.inputFile != null) {
+  if (options.help) {
+    outputHelp()
+    return
+  }
+
   if (options.overwrite) {
-    streamToString(createReadStream(options.inputFile))
-      .then((text) => {
-        const outputStream = createWriteStream(options.inputFile)
-        outputStream.write(jsonrepair(text))
-      })
-      .catch((err) => process.stderr.write(err.toString()))
-  } else {
-    streamToString(createReadStream(options.inputFile))
-      .then((text) => process.stdout.write(jsonrepair(text)))
-      .catch((err) => process.stderr.write(err.toString()))
+    if (!options.inputFile) {
+      console.error('Error: cannot use --overwrite: no input file provided')
+      process.exit(1)
+    }
+    if (options.outputFile) {
+      console.error('Error: cannot use --overwrite: there is also an --output provided')
+      process.exit(1)
+    }
+
+    const tempFileSuffix = '.repair-' + new Date().toISOString().replace(/\W/g, '-') + '.json'
+    const tempFile = options.inputFile + tempFileSuffix
+
+    try {
+      const readStream = createReadStream(options.inputFile)
+      const writeStream = createWriteStream(tempFile)
+      await pipeline(
+        readStream,
+        jsonrepairTransform({ bufferSize: options.bufferSize }),
+        writeStream
+      )
+      renameSync(tempFile, options.inputFile)
+    } catch (err) {
+      process.stderr.write(err.toString())
+      process.exit(1)
+    }
+
+    return
   }
-} else {
-  streamToString(process.stdin)
-    .then((text) => process.stdout.write(jsonrepair(text)))
-    .catch((err) => process.stderr.write(err.toString()))
+
+  try {
+    const readStream = options.inputFile ? createReadStream(options.inputFile) : process.stdin
+    const writeStream = options.outputFile ? createWriteStream(options.outputFile) : process.stdout
+    await pipeline(readStream, jsonrepairTransform({ bufferSize: options.bufferSize }), writeStream)
+  } catch (err) {
+    process.stderr.write(err.toString())
+    process.exit(1)
+  }
 }
+
+function outputVersion() {
+  const file = join(__dirname, '../package.json')
+  const pkg = JSON.parse(String(readFileSync(file, 'utf-8')))
+
+  console.log(pkg.version)
+}
+
+function parseSize(size) {
+  // match
+  const match = size.match(/^(\d+)([KMG]?)$/)
+  if (!match) {
+    throw new Error(`Buffer size "${size}" not recognized. Examples: 65536, 512K, 2M`)
+  }
+
+  const num = parseInt(match[1])
+  const suffix = match[2] // K, M, or G
+
+  switch (suffix) {
+    case 'K':
+      return num * 1024
+    case 'M':
+      return num * 1024 * 1024
+    case 'G':
+      return num * 1024 * 1024 * 1024
+    default:
+      return num
+  }
+}
+
+const help = `
+jsonrepair
+https://github.com/josdejong/jsonrepair
+
+Repair invalid JSON documents. When a document could not be repaired, the output will be left unchanged.
+
+Usage:
+    jsonrepair [filename] {OPTIONS}
+
+Options:
+    --version, -v       Show application version
+    --help,    -h       Show this message
+    --output,  -o       Output file
+    --overwrite         Overwrite the input file
+    --buffer            Buffer size in bytes, for example 64K (default) or 1M
+
+Example usage:
+    jsonrepair broken.json                        # Repair a file, output to console
+    jsonrepair broken.json > repaired.json        # Repair a file, output to file
+    jsonrepair broken.json --output repaired.json # Repair a file, output to file
+    jsonrepair broken.json --overwrite            # Repair a file, replace the file itself
+    cat broken.json | jsonrepair                  # Repair data from an input stream
+    cat broken.json | jsonrepair > repaired.json  # Repair data from an input stream, output to file
+`
+
+function outputHelp() {
+  console.log(help)
+}
+
+const options = processArgs(process.argv)
+await run(options)

lib/cjs/index.js

@@ -18,4 +18,4 @@ "use strict";
 });
-var _jsonrepair = require("./jsonrepair.js");
-var _JSONRepairError = require("./JSONRepairError.js");
+var _jsonrepair = require("./regular/jsonrepair.js");
+var _JSONRepairError = require("./utils/JSONRepairError.js");
 //# sourceMappingURL=index.js.map

lib/esm/index.js

@@ -1,3 +0,4 @@
-export { jsonrepair } from './jsonrepair.js';
-export { JSONRepairError } from './JSONRepairError.js';
+// Cross-platform, non-streaming JavaScript API
+export { jsonrepair } from './regular/jsonrepair.js';
+export { JSONRepairError } from './utils/JSONRepairError.js';
 //# sourceMappingURL=index.js.map

lib/types/index.d.ts

@@ -1,3 +0,3 @@
-export { jsonrepair } from './jsonrepair.js';
-export { JSONRepairError } from './JSONRepairError.js';
+export { jsonrepair } from './regular/jsonrepair.js';
+export { JSONRepairError } from './utils/JSONRepairError.js';
 //# sourceMappingURL=index.d.ts.map

lib/umd/jsonrepair.js

@@ -14,3 +14,2 @@ (function (global, factory) {
 
-  // TODO: sort the codes
   const codeBackslash = 0x5c; // "\"
@@ -35,4 +34,4 @@ const codeSlash = 0x2f; // "/"
   const codeQuote = 0x27; // "'"
-  const codeZero = 0x30;
-  const codeNine = 0x39;
+  const codeZero = 0x30; // "0"
+  const codeNine = 0x39; // "9"
   const codeComma = 0x2c; // ","
@@ -71,3 +70,3 @@ const codeDot = 0x2e; // "." (dot, period)
   function isDelimiter(char) {
-    return regexDelimiter.test(char) || char && isQuote(char.charCodeAt(0));
+    return regexDelimiter.test(char) || isQuote(char.charCodeAt(0));
   }
@@ -142,2 +141,3 @@ const regexDelimiter = /^[,:[\]{}()\n+]$/;
   }
+
   /**
@@ -485,11 +485,5 @@ * Strip last occurrence of textToStrip from text
         // or any single-quote-like start with a single-quote-like end
-        const isEndQuote = isDoubleQuote(text.charCodeAt(i)) ? isDoubleQuote : isSingleQuote(text.charCodeAt(i)) ? isSingleQuote // eslint-disable-line indent
-        : isSingleQuoteLike(text.charCodeAt(i)) // eslint-disable-line indent
-        ? isSingleQuoteLike // eslint-disable-line indent
-        : isDoubleQuoteLike; // eslint-disable-line indent
-
+        const isEndQuote = isDoubleQuote(text.charCodeAt(i)) ? isDoubleQuote : isSingleQuote(text.charCodeAt(i)) ? isSingleQuote : isSingleQuoteLike(text.charCodeAt(i)) ? isSingleQuoteLike : isDoubleQuoteLike;
         const iBefore = i;
-        const outputBefore = output; // we may need to revert
-
-        output += '"';
+        let str = '"';
         i++;
@@ -502,3 +496,3 @@ const isEndOfString = stopAtDelimiter ? i => isDelimiter(text[i]) : i => isEndQuote(text.charCodeAt(i));
             if (escapeChar !== undefined) {
-              output += text.slice(i, i + 2);
+              str += text.slice(i, i + 2);
               i += 2;
@@ -511,5 +505,5 @@ } else if (char === 'u') {
               if (j === 6) {
-                output += text.slice(i, i + 6);
+                str += text.slice(i, i + 6);
                 i += 6;
-              } else if (j < 6 && i + j >= text.length) {
+              } else if (i + j >= text.length) {
                 // repair invalid or truncated unicode char at the end of the text
@@ -519,7 +513,7 @@ // by removing the unicode char and ending the string here
               } else {
-                throwInvalidUnicodeCharacter(i);
+                throwInvalidUnicodeCharacter();
               }
             } else {
               // repair invalid escape character: remove it
-              output += char;
+              str += char;
               i += 2;
@@ -532,7 +526,7 @@ }
               // repair unescaped double quote
-              output += '\\' + char;
+              str += '\\' + char;
               i++;
             } else if (isControlCharacter(code)) {
               // unescaped control character
-              output += controlCharacters[char];
+              str += controlCharacters[char];
               i++;
@@ -543,3 +537,3 @@ } else {
               }
-              output += char;
+              str += char;
               i++;
@@ -561,12 +555,12 @@ }
           i = iBefore;
-          output = outputBefore;
           return parseString(true);
         }
         if (hasEndQuote) {
-          output += '"';
+          str += '"';
           i++;
         } else {
           // repair missing quote
-          output = insertBeforeLastWhitespace(output, '"');
+          str = insertBeforeLastWhitespace(str, '"');
         }
+        output += str;
         parseConcatenatedString();
@@ -592,6 +586,10 @@ return true;
         const start = output.length;
-        parseString();
-
-        // repair: remove the start quote of the second string
-        output = removeAtIndex(output, start, 1);
+        const parsedStr = parseString();
+        if (parsedStr) {
+          // repair: remove the start quote of the second string
+          output = removeAtIndex(output, start, 1);
+        } else {
+          // repair: remove the + because it is not followed by a string
+          output = insertBeforeLastWhitespace(output, '"');
+        }
       }
@@ -746,3 +744,3 @@ return processed;
     }
-    function throwInvalidUnicodeCharacter(i) {
+    function throwInvalidUnicodeCharacter() {
       const chars = text.slice(i, i + 6);
@@ -749,0 +747,0 @@ throw new JSONRepairError(`Invalid unicode character "${chars}"`, i);

lib/umd/jsonrepair.min.js

@@ -1,3 +0,3 @@
-!function(t,e){"object"==typeof exports&&"undefined"!=typeof module?e(exports):"function"==typeof define&&define.amd?define(["exports"],e):e((t="undefined"!=typeof globalThis?globalThis:t||self).JSONRepair={})}(this,function(t){"use strict";class p extends Error{constructor(t,e){super(t+" at position "+e),this.position=e}}const a=125,e=32,w=10,x=9,y=13,$=8,O=12,N=34,r=39,J=48,S=57,j=65,I=97,k=70,m=102,h=160,d=8192,l=8202,T=8239,E=8287,R=12288,n=8220,o=8221,c=8216,i=8217,f=96,u=180;function U(t){return t>=J&&t<=S}function F(t){return s.test(t)||t&&B(t.charCodeAt(0))}const s=/^[,:[\]{}()\n+]$/;function q(t){return A.test(t)||t&&B(t.charCodeAt(0))}const A=/^[[{\w-]$/;function z(t){return t===e||t===w||t===x||t===y}function B(t){return D(t)||H(t)}function D(t){return t===N||t===n||t===o}function G(t){return t===N}function H(t){return t===r||t===c||t===i||t===f||t===u}function K(t){return t===r}function L(t,e,r){r=2<arguments.length&&void 0!==r&&r,e=t.lastIndexOf(e);return-1!==e?t.substring(0,e)+(r?"":t.substring(e+1)):t}function M(t,e){let r=t.length;if(!z(t.charCodeAt(r-1)))return t+e;for(;z(t.charCodeAt(r-1));)r--;return t.substring(0,r)+e+t.substring(r)}const P={"\b":"\\b","\f":"\\f","\n":"\\n","\r":"\\r","\t":"\\t"},Q={'"':'"',"\\":"\\","/":"/",b:"\b",f:"\f",n:"\n",r:"\r",t:"\t"};t.JSONRepairError=p,t.jsonrepair=function(s){let A=0,C="";if(!c())throw new p("Unexpected end of json string",s.length);var t=i(44);if(t&&g(),q(s[A])&&/[,\n][ \t\r]*$/.test(C)){t||(C=M(C,","));{let t=!0,e=!0;for(;e;)t?t=!1:i(44)||(C=M(C,",")),e=c();e||(C=L(C,","));C=`[
+!function(t,e){"object"==typeof exports&&"undefined"!=typeof module?e(exports):"function"==typeof define&&define.amd?define(["exports"],e):e((t="undefined"!=typeof globalThis?globalThis:t||self).JSONRepair={})}(this,function(t){"use strict";class p extends Error{constructor(t,e){super(t+" at position "+e),this.position=e}}const a=125,e=32,w=10,x=9,y=13,$=8,O=12,N=34,r=39,J=48,S=57,j=65,I=97,k=70,m=102,h=160,d=8192,l=8202,T=8239,E=8287,R=12288,n=8220,o=8221,c=8216,i=8217,f=96,u=180;function U(t){return t>=J&&t<=S}function F(t){return s.test(t)||B(t.charCodeAt(0))}const s=/^[,:[\]{}()\n+]$/;function q(t){return A.test(t)||t&&B(t.charCodeAt(0))}const A=/^[[{\w-]$/;function z(t){return t===e||t===w||t===x||t===y}function B(t){return D(t)||H(t)}function D(t){return t===N||t===n||t===o}function G(t){return t===N}function H(t){return t===r||t===c||t===i||t===f||t===u}function K(t){return t===r}function L(t,e,r){r=2<arguments.length&&void 0!==r&&r,e=t.lastIndexOf(e);return-1!==e?t.substring(0,e)+(r?"":t.substring(e+1)):t}function M(t,e){let r=t.length;if(!z(t.charCodeAt(r-1)))return t+e;for(;z(t.charCodeAt(r-1));)r--;return t.substring(0,r)+e+t.substring(r)}const P={"\b":"\\b","\f":"\\f","\n":"\\n","\r":"\\r","\t":"\\t"},Q={'"':'"',"\\":"\\","/":"/",b:"\b",f:"\f",n:"\n",r:"\r",t:"\t"};t.JSONRepairError=p,t.jsonrepair=function(s){let A=0,C="";if(!c())throw new p("Unexpected end of json string",s.length);var t=i(44);if(t&&g(),q(s[A])&&/[,\n][ \t\r]*$/.test(C)){t||(C=M(C,","));{let t=!0,e=!0;for(;e;)t?t=!1:i(44)||(C=M(C,",")),e=c();e||(C=L(C,","));C=`[
 ${C}
-]`}}else t&&(C=L(C,","));for(;s.charCodeAt(A)===a||93===s.charCodeAt(A);)A++,g();if(A>=s.length)return C;throw new p("Unexpected character "+JSON.stringify(s[A]),A);function c(){g();var t=function(){if(123!==s.charCodeAt(A))return!1;{C+="{",A++,g();let e=!0;for(;A<s.length&&s.charCodeAt(A)!==a;){let t;if(e?(t=!0,e=!1):((t=i(44))||(C=M(C,",")),g()),!(b()||f())){s.charCodeAt(A)===a||123===s.charCodeAt(A)||93===s.charCodeAt(A)||91===s.charCodeAt(A)||void 0===s[A]?C=L(C,","):function(){throw new p("Object key expected",A)}();break}g();var r=i(58),n=A>=s.length,o=(r||(q(s[A])||n?C=M(C,":"):u()),c());o||(r||n?C+="null":u())}return s.charCodeAt(A)===a?(C+="}",A++):C=M(C,"}"),!0}}()||function(){if(91!==s.charCodeAt(A))return!1;{C+="[",A++,g();let t=!0;for(;A<s.length&&93!==s.charCodeAt(A);){t?t=!1:i(44)||(C=M(C,","));var e=c();if(!e){C=L(C,",");break}}return 93===s.charCodeAt(A)?(C+="]",A++):C=M(C,"]"),!0}}()||b()||function(){var t=A;if(45===s.charCodeAt(A)&&(A++,o(t)))return!0;for(;U(s.charCodeAt(A));)A++;if(46===s.charCodeAt(A)){if(A++,o(t))return!0;for(;U(s.charCodeAt(A));)A++}if(101===s.charCodeAt(A)||69===s.charCodeAt(A)){if(A++,45!==s.charCodeAt(A)&&43!==s.charCodeAt(A)||A++,o(t))return!0;for(;U(s.charCodeAt(A));)A++}{var e;if(A>t)return t=s.slice(t,A),e=/^0\d/.test(t),C+=e?`"${t}"`:t,!0}return!1}()||r("true","true")||r("false","false")||r("null","null")||r("True","true")||r("False","false")||r("None","null")||f();return g(),t}function g(){A;let t=e();for(;t=(t=function(){if(47===s.charCodeAt(A)&&42===s.charCodeAt(A+1)){for(;A<s.length&&!function(t,e){return"*"===t[e]&&"/"===t[e+1]}(s,A);)A++;A+=2}else{if(47!==s.charCodeAt(A)||47!==s.charCodeAt(A+1))return!1;for(;A<s.length&&s.charCodeAt(A)!==w;)A++}return!0}())&&e(););A}function e(){let t="";for(var e,r;(e=z(s.charCodeAt(A)))||(r=s.charCodeAt(A))===h||r>=d&&r<=l||r===T||r===E||r===R;)t+=e?s[A]:" ",A++;return 0<t.length&&(C+=t,!0)}function i(t){return s.charCodeAt(A)===t&&(C+=s[A],A++,!0)}function v(){92===s.charCodeAt(A)&&A++}function b(t){var e,t=0<arguments.length&&void 0!==t&&t;let r=92===s.charCodeAt(A);if(r&&(A++,r=!0),B(s.charCodeAt(A))){const l=G(s.charCodeAt(A))?G:K(s.charCodeAt(A))?K:H(s.charCodeAt(A))?H:D;for(var n=A,o=C,c=(C+='"',A++,t?t=>F(s[t]):t=>l(s.charCodeAt(t)));A<s.length&&!c(A);){if(92===s.charCodeAt(A)){var i=s.charAt(A+1);if(void 0!==Q[i])C+=s.slice(A,A+2),A+=2;else if("u"===i){let t=2;for(;t<6&&((e=s.charCodeAt(A+t))>=J&&e<=S||e>=j&&e<=k||e>=I&&e<=m);)t++;if(6===t)C+=s.slice(A,A+6),A+=6;else{if(!(t<6&&A+t>=s.length))throw u=a=void 0,a=A,u=s.slice(a,a+6),new p(`Invalid unicode character "${u}"`,a);A=s.length}}else C+=i,A+=2}else{var f,u=s.charAt(A),a=s.charCodeAt(A);if(a===N&&92!==s.charCodeAt(A-1))C+="\\"+u;else if((i=a)===w||i===y||i===x||i===$||i===O)C+=P[u];else{if(!(32<=(f=a)&&f<=1114111))throw f=void 0,f=u,new p("Invalid character "+JSON.stringify(f),A);C+=u}A++}r&&v()}var h=B(s.charCodeAt(A));if(!(h&&(A+1>=s.length||F(function(t,e){let r=e;for(;z(t.charCodeAt(r));)r++;return t.charAt(r)}(s,A+1))))&&!t)return A=n,C=o,b(!0);h?(C+='"',A++):C=M(C,'"');{let t=!1;g();for(;43===s.charCodeAt(A);){t=!0,A++,g();var d=(C=L(C,'"',!0)).length;b(),C=function(t,e,r){return t.substring(0,e)+t.substring(e+r)}(C,d,1)}t}return!0}return!1}function r(t,e){return s.slice(A,A+t.length)===t&&(C+=e,A+=t.length,!0)}function f(){for(var t=A;A<s.length&&!F(s[A]);)A++;if(A>t){if(40===s.charCodeAt(A))A++,c(),41===s.charCodeAt(A)&&(A++,59===s.charCodeAt(A))&&A++;else{for(;z(s.charCodeAt(A-1))&&0<A;)A--;t=s.slice(t,A);C+="undefined"===t?"null":JSON.stringify(t),s.charCodeAt(A)===N&&A++}return!0}}function n(t){if(!U(s.charCodeAt(A)))throw t=s.slice(t,A),new p(`Invalid number '${t}', expecting a digit `+(s[A]?`but got '${s[A]}'`:"but reached end of input"),A)}function o(t){if(A>=s.length)return C+=s.slice(t,A)+"0",1;n(t)}function u(){throw new p("Colon expected",A)}}});
+]`}}else t&&(C=L(C,","));for(;s.charCodeAt(A)===a||93===s.charCodeAt(A);)A++,g();if(A>=s.length)return C;throw new p("Unexpected character "+JSON.stringify(s[A]),A);function c(){g();var t=function(){if(123!==s.charCodeAt(A))return!1;{C+="{",A++,g();let e=!0;for(;A<s.length&&s.charCodeAt(A)!==a;){let t;if(e?(t=!0,e=!1):((t=i(44))||(C=M(C,",")),g()),!(b()||f())){s.charCodeAt(A)===a||123===s.charCodeAt(A)||93===s.charCodeAt(A)||91===s.charCodeAt(A)||void 0===s[A]?C=L(C,","):function(){throw new p("Object key expected",A)}();break}g();var r=i(58),n=A>=s.length,o=(r||(q(s[A])||n?C=M(C,":"):u()),c());o||(r||n?C+="null":u())}return s.charCodeAt(A)===a?(C+="}",A++):C=M(C,"}"),!0}}()||function(){if(91!==s.charCodeAt(A))return!1;{C+="[",A++,g();let t=!0;for(;A<s.length&&93!==s.charCodeAt(A);){t?t=!1:i(44)||(C=M(C,","));var e=c();if(!e){C=L(C,",");break}}return 93===s.charCodeAt(A)?(C+="]",A++):C=M(C,"]"),!0}}()||b()||function(){var t=A;if(45===s.charCodeAt(A)&&(A++,o(t)))return!0;for(;U(s.charCodeAt(A));)A++;if(46===s.charCodeAt(A)){if(A++,o(t))return!0;for(;U(s.charCodeAt(A));)A++}if(101===s.charCodeAt(A)||69===s.charCodeAt(A)){if(A++,45!==s.charCodeAt(A)&&43!==s.charCodeAt(A)||A++,o(t))return!0;for(;U(s.charCodeAt(A));)A++}{var e;if(A>t)return t=s.slice(t,A),e=/^0\d/.test(t),C+=e?`"${t}"`:t,!0}return!1}()||r("true","true")||r("false","false")||r("null","null")||r("True","true")||r("False","false")||r("None","null")||f();return g(),t}function g(){A;let t=e();for(;t=(t=function(){if(47===s.charCodeAt(A)&&42===s.charCodeAt(A+1)){for(;A<s.length&&!function(t,e){return"*"===t[e]&&"/"===t[e+1]}(s,A);)A++;A+=2}else{if(47!==s.charCodeAt(A)||47!==s.charCodeAt(A+1))return!1;for(;A<s.length&&s.charCodeAt(A)!==w;)A++}return!0}())&&e(););A}function e(){let t="";for(var e,r;(e=z(s.charCodeAt(A)))||(r=s.charCodeAt(A))===h||r>=d&&r<=l||r===T||r===E||r===R;)t+=e?s[A]:" ",A++;return 0<t.length&&(C+=t,!0)}function i(t){return s.charCodeAt(A)===t&&(C+=s[A],A++,!0)}function v(){92===s.charCodeAt(A)&&A++}function b(t){var r,t=0<arguments.length&&void 0!==t&&t;let n=92===s.charCodeAt(A);if(n&&(A++,n=!0),B(s.charCodeAt(A))){const l=G(s.charCodeAt(A))?G:K(s.charCodeAt(A))?K:H(s.charCodeAt(A))?H:D;var o=A;let e='"';A++;for(var c=t?t=>F(s[t]):t=>l(s.charCodeAt(t));A<s.length&&!c(A);){if(92===s.charCodeAt(A)){var i=s.charAt(A+1);if(void 0!==Q[i])e+=s.slice(A,A+2),A+=2;else if("u"===i){let t=2;for(;t<6&&((r=s.charCodeAt(A+t))>=J&&r<=S||r>=j&&r<=k||r>=I&&r<=m);)t++;if(6===t)e+=s.slice(A,A+6),A+=6;else{if(!(A+t>=s.length))throw u=void 0,u=s.slice(A,A+6),new p(`Invalid unicode character "${u}"`,A);A=s.length}}else e+=i,A+=2}else{var f,u=s.charAt(A),i=s.charCodeAt(A);if(i===N&&92!==s.charCodeAt(A-1))e+="\\"+u;else if((f=i)===w||f===y||f===x||f===$||f===O)e+=P[u];else{if(!(32<=(f=i)&&f<=1114111))throw f=void 0,f=u,new p("Invalid character "+JSON.stringify(f),A);e+=u}A++}n&&v()}var a=B(s.charCodeAt(A));if(!(a&&(A+1>=s.length||F(function(t,e){let r=e;for(;z(t.charCodeAt(r));)r++;return t.charAt(r)}(s,A+1))))&&!t)return A=o,b(!0);a?(e+='"',A++):e=M(e,'"'),C+=e;{let t=!1;g();for(;43===s.charCodeAt(A);){t=!0,A++,g();var h=(C=L(C,'"',!0)).length,d=b();C=d?function(t,e,r){return t.substring(0,e)+t.substring(e+r)}(C,h,1):M(C,'"')}t}return!0}return!1}function r(t,e){return s.slice(A,A+t.length)===t&&(C+=e,A+=t.length,!0)}function f(){for(var t=A;A<s.length&&!F(s[A]);)A++;if(A>t){if(40===s.charCodeAt(A))A++,c(),41===s.charCodeAt(A)&&(A++,59===s.charCodeAt(A))&&A++;else{for(;z(s.charCodeAt(A-1))&&0<A;)A--;t=s.slice(t,A);C+="undefined"===t?"null":JSON.stringify(t),s.charCodeAt(A)===N&&A++}return!0}}function n(t){if(!U(s.charCodeAt(A)))throw t=s.slice(t,A),new p(`Invalid number '${t}', expecting a digit `+(s[A]?`but got '${s[A]}'`:"but reached end of input"),A)}function o(t){if(A>=s.length)return C+=s.slice(t,A)+"0",1;n(t)}function u(){throw new p("Colon expected",A)}}});

package.json

 {
   "name": "jsonrepair",
-  "version": "3.4.1",
+  "version": "3.5.0",
   "description": "Repair broken JSON documents",
@@ -20,2 +20,7 @@ "repository": {
       "types": "./lib/types/index.d.ts"
+    },
+    "./stream": {
+      "import": "./lib/esm/stream.js",
+      "require": "./lib/cjs/stream.js",
+      "types": "./lib/types/stream.d.ts"
     }
@@ -28,3 +33,5 @@ },
     "fix",
-    "invalid"
+    "invalid",
+    "stream",
+    "streaming"
   ],
@@ -65,19 +72,19 @@ "bin": {
   "devDependencies": {
-    "@babel/cli": "7.23.0",
-    "@babel/core": "7.23.2",
-    "@babel/plugin-transform-typescript": "7.22.15",
-    "@babel/preset-env": "7.23.2",
-    "@babel/preset-typescript": "7.23.2",
-    "@commitlint/cli": "18.2.0",
-    "@commitlint/config-conventional": "18.1.0",
-    "@types/node": "20.8.10",
-    "@typescript-eslint/eslint-plugin": "6.9.1",
-    "@typescript-eslint/parser": "6.9.1",
+    "@babel/cli": "7.23.4",
+    "@babel/core": "7.23.5",
+    "@babel/plugin-transform-typescript": "7.23.5",
+    "@babel/preset-env": "7.23.5",
+    "@babel/preset-typescript": "7.23.3",
+    "@commitlint/cli": "18.4.3",
+    "@commitlint/config-conventional": "18.4.3",
+    "@types/node": "20.10.4",
+    "@typescript-eslint/eslint-plugin": "6.13.2",
+    "@typescript-eslint/parser": "6.13.2",
     "benchmark": "2.1.4",
     "cpy-cli": "5.0.0",
     "del-cli": "5.1.0",
-    "eslint": "8.52.0",
+    "eslint": "8.55.0",
     "eslint-config-standard": "17.1.0",
     "eslint-plugin-import": "2.29.0",
-    "eslint-plugin-n": "16.2.0",
+    "eslint-plugin-n": "16.3.1",
     "eslint-plugin-node": "11.1.0",
@@ -87,10 +94,10 @@ "eslint-plugin-promise": "6.1.1",
     "npm-run-all": "4.1.5",
-    "prettier": "3.0.3",
-    "rollup": "4.2.0",
+    "prettier": "3.1.0",
+    "rollup": "4.6.1",
     "standard-version": "9.5.0",
     "ts-node": "10.9.1",
-    "typescript": "5.2.2",
+    "typescript": "5.3.3",
     "uglify-js": "3.17.4",
-    "vitest": "0.34.6"
+    "vitest": "1.0.1"
   }
 }

README.md

@@ -34,3 +34,3 @@ # jsonrepair
 
-Note that in practice, the `jsonrepair` library can handle file sizes up to 512 MB.
+The `jsonrepair` library has streaming support and can handle infinitely large documents.
 
@@ -48,3 +48,3 @@ ## Install
 
-Use with an ES modules import (recommended):
+Use the `jsonrepair` function using an ES modules import:
 
@@ -68,2 +68,28 @@ ```js
 
+Use the streaming API in Node.js:
+
+```js
+import { createReadStream, createWriteStream } from 'node:fs'
+import { pipeline } from 'node:stream'
+import { jsonrepairTransform } from 'jsonrepair/stream'
+
+const inputStream = createReadStream('./data/broken.json')
+const outputStream = createWriteStream('./data/repaired.json')
+
+pipeline(inputStream, jsonrepairTransform(), outputStream, (err) => {
+  if (err) {
+    console.error(err)
+  } else {
+    console.log('done')
+  }
+})
+
+// or using .pipe() instead of pipeline():
+// inputStream
+//   .pipe(jsonrepairTransform())
+//   .pipe(outputStream)
+//   .on('error', (err) => console.error(err))
+//   .on('finish', () => console.log('done'))
+```
+
 Use in CommonJS (not recommended):
@@ -91,2 +117,6 @@
 
+#### Regular API
+
+You can use `jsonrepair` as a function or as a streaming transform. Broken JSON is passed to the function, and the function either returns the repaired JSON, or throws an `JSONRepairError` exception when an issue is encountered which could not be solved.
+
 ```ts
@@ -97,5 +127,14 @@ // @throws JSONRepairError
 
-The function `jsonrepair` throws an exception `JSONRepairError` when an issue is encountered which could not be solved. When no error is thrown, the output will be valid JSON.
+#### Streaming API
 
+The streaming API is availabe in `jsonrepair/stream` and can be used in a [Node.js stream](https://nodejs.org/api/stream.html). It consists of a transform function that can be used in a stream pipeline.
 
+```ts
+jsonrepairTransform(options?: { chunkSize?: number, bufferSize?: number }) : Transform
+```
+
+The option `chunkSize` determines the size of the chunks that the transform outputs, and is `65536` bytes by default. Changing `chunkSize` can influcence the performance. 
+
+The option `bufferSize` determines how many bytes of the input and output stream are kept in memory and is also `65536` bytes by default. This buffer is used as a "moving window" on the input and output. This is necessary because `jsonrepair` must look ahead or look back to see what to fix, and it must sometimes walk back the generated output to insert a missing comma for example. The `bufferSize` must be larger than the length of the largest string and whitespace in the JSON data, otherwise, and error is thrown when processing the data. Making `bufferSize` very large will result in more memory usage and less performance.
+
 ### Command Line Interface (CLI)
@@ -119,3 +158,6 @@
 --version, -v       Show application version
---help,    -h       Show help
+--help,    -h       Show this message
+--output,  -o       Output file
+--overwrite         Overwrite the input file
+--buffer            Buffer size in bytes, for example 64K (default) or 1M
 ```
@@ -126,7 +168,8 @@
 ```
-$ jsonrepair broken.json                         # Repair a file, output to console
-$ jsonrepair broken.json > repaired.json         # Repair a file, output to file
-$ jsonrepair broken.json --overwrite             # Repair a file, replace the file itself
-$ cat broken.json | jsonrepair                   # Repair data from an input stream
-$ cat broken.json | jsonrepair > repaired.json   # Repair data from an input stream, output to file
+$ jsonrepair broken.json                        # Repair a file, output to console
+$ jsonrepair broken.json > repaired.json        # Repair a file, output to file
+$ jsonrepair broken.json --output repaired.json # Repair a file, output to file
+$ jsonrepair broken.json --overwrite            # Repair a file, replace the file itself
+$ cat broken.json | jsonrepair                  # Repair data from an input stream
+$ cat broken.json | jsonrepair > repaired.json  # Repair data from an input stream, output to file
 ```
@@ -142,2 +185,9 @@
 
+When implementing a fix or a new feature, it important to know that there are currently two implementations:
+
+- `src/regular` This is a non-streaming implementation. The code is small and works for files up to 512MB, ideal for usage in the browser.
+- `src/streaming` A streaming implementation that can be used in Node.js. The code is larger and more complex, and the implementation uses a configurable `bufferSize` and `chunkSize`. When the parsed document contains a string or number that is longer than the configured `bufferSize`, the library will throw an "Index out of range" error since it cannot hold the full string in the buffer. When configured with an infinite buffer size, the streaming implementation works the same as the regular implementation. In that case this out of range error cannot occur, but it makes the performance worse and the application can run out of memory when repairing large documents.
+
+Both implementations are tested against the same suite of unit tests in `src/index.test.ts`.
+
 To build the library (ESM, CommonJs, and UMD output in the folder `lib`):
@@ -144,0 +194,0 @@

New alerts

License Policy Violation

License

This package is not allowed per your license policy. Review the package's license to ensure compliance.

Found 1 instance in 1 package

Major refactor

Supply chain risk

Package has recently undergone a major refactor. It may be unstable or indicate significant internal changes. Use caution when updating to versions that include significant changes.

Found 1 instance in 1 package

Fixed alerts

License Policy Violation

License

This package is not allowed per your license policy. Review the package's license to ensure compliance.

Found 1 instance in 1 package

Filesystem access

Supply chain risk

Accesses the file system, and could potentially read sensitive data.

Found 1 instance in 1 package

Improved metrics

Total package byte prevSize

449315

increased by72.16%

Number of package files

70

increased by105.88%

Lines of code

4351

increased by80.17%

Number of lines in readme file

243

increased by25.91%

Number of low supply chain risk alerts

0

decreased by-100%

Worsened metrics

Number of medium supply chain risk alerts

1

increased byInfinity%

No dependency changes