jsonarrayfs - npm Package Compare versions

Comparing version 1.0.1 to 1.1.0

dist/index.d.mts

@@ -21,2 +21,3 @@ type BufferEncoding$1 = "ascii" | "utf8" | "utf-8" | "utf16le" | "utf-16le" | "ucs2" | "ucs-2" | "base64" | "base64url" | "latin1" | "binary" | "hex";
     private chunkGenerator;
+    private addToResult;
     private resetParser;
@@ -26,3 +27,3 @@ private stringElementParser;
     private containerElementParser;
-    stream<T>(chunkSize: number): AsyncGenerator<T_1[], T_1[], unknown>;
+    stream(chunkSize: number, filter?: (element: T) => boolean): AsyncGenerator<T[], T[], unknown>;
     private static sanitizeReadStreamOptions;
@@ -29,0 +30,0 @@ static create: <T_1>(path: string, options?: ReadStreamOptions) => Promise<JsonArrayStreamer<T_1>>;

dist/index.d.ts

@@ -21,2 +21,3 @@ type BufferEncoding$1 = "ascii" | "utf8" | "utf-8" | "utf16le" | "utf-16le" | "ucs2" | "ucs-2" | "base64" | "base64url" | "latin1" | "binary" | "hex";
     private chunkGenerator;
+    private addToResult;
     private resetParser;
@@ -26,3 +27,3 @@ private stringElementParser;
     private containerElementParser;
-    stream<T>(chunkSize: number): AsyncGenerator<T_1[], T_1[], unknown>;
+    stream(chunkSize: number, filter?: (element: T) => boolean): AsyncGenerator<T[], T[], unknown>;
     private static sanitizeReadStreamOptions;
@@ -29,0 +30,0 @@ static create: <T_1>(path: string, options?: ReadStreamOptions) => Promise<JsonArrayStreamer<T_1>>;

dist/index.js

@@ -1,2 +0,2 @@
-"use strict";var S=Object.defineProperty;var L=Object.getOwnPropertyDescriptor;var I=Object.getOwnPropertyNames,k=Object.getOwnPropertySymbols;var A=Object.prototype.hasOwnProperty,$=Object.prototype.propertyIsEnumerable;var P=(s,e)=>(e=Symbol[s])?e:Symbol.for("Symbol."+s);var y=(s,e,t)=>e in s?S(s,e,{enumerable:!0,configurable:!0,writable:!0,value:t}):s[e]=t,v=(s,e)=>{for(var t in e||(e={}))A.call(e,t)&&y(s,t,e[t]);if(k)for(var t of k(e))$.call(e,t)&&y(s,t,e[t]);return s};var K=(s,e)=>{for(var t in e)S(s,t,{get:e[t],enumerable:!0})},M=(s,e,t,i)=>{if(e&&typeof e=="object"||typeof e=="function")for(let a of I(e))!A.call(s,a)&&a!==t&&S(s,a,{get:()=>e[a],enumerable:!(i=L(e,a))||i.enumerable});return s};var z=s=>M(S({},"__esModule",{value:!0}),s);var O=(s,e,t)=>new Promise((i,a)=>{var h=r=>{try{l(t.next(r))}catch(u){a(u)}},o=r=>{try{l(t.throw(r))}catch(u){a(u)}},l=r=>r.done?i(r.value):Promise.resolve(r.value).then(h,o);l((t=t.apply(s,e)).next())}),C=function(s,e){this[0]=s,this[1]=e},g=(s,e,t)=>{var i=(o,l,r,u)=>{try{var m=t[o](l),p=(l=m.value)instanceof C,c=m.done;Promise.resolve(p?l[0]:l).then(f=>p?i(o==="return"?o:"next",l[1]?{done:f.done,value:f.value}:f,r,u):r({value:f,done:c})).catch(f=>i("throw",f,r,u))}catch(f){u(f)}},a=o=>h[o]=l=>new Promise((r,u)=>i(o,l,r,u)),h={};return t=t.apply(s,e),h[P("asyncIterator")]=()=>h,a("next"),a("throw"),a("return"),h};var R=(s,e,t)=>(e=s[P("asyncIterator")])?e.call(s):(s=s[P("iterator")](),e={},t=(i,a)=>(a=s[i])&&(e[i]=h=>new Promise((o,l,r)=>(h=a.call(s,h),r=h.done,Promise.resolve(h.value).then(u=>o({value:u,done:r}),l)))),t("next"),t("return"),e);var x={};K(x,{appendFile:()=>D,createReadStream:()=>H});module.exports=z(x);var N=require("fs"),b=require("events");var n={BRACKET:{OPEN:"[",CLOSE:"]"},BRACE:{OPEN:"{",CLOSE:"}"},QUOTE:'"',ESCAPE:"\\",SPACE:" ",COMMA:",",NEW_LINE:`
-`};var d=class d{constructor(e,t){this.readStream=(0,N.createReadStream)(e,d.sanitizeReadStreamOptions(t)),this.rootDetected=!1,this.elementDetected=!1,this.elementType="others",this.elementParser=null,this.elementEnclosureCount=0,this.isCharInsideQuotes=!1,this.isCharEscaped=!1,this.chunkBuffer="",this.resultBuffer=[]}chunkGenerator(){return g(this,null,function*(){if(!this.readStream)throw new Error("Stream not initialized");try{for(var e=R(this.readStream),t,i,a;t=!(i=yield new C(e.next())).done;t=!1){let h=i.value;yield h}}catch(i){a=[i]}finally{try{t&&(i=e.return)&&(yield new C(i.call(e)))}finally{if(a)throw a[0]}}})}resetParser(){this.elementDetected=!1,this.elementType="others",this.elementParser=null,this.elementEnclosureCount=0,this.isCharInsideQuotes=!1,this.isCharEscaped=!1,this.chunkBuffer=""}stringElementParser(e){if(this.chunkBuffer=`${this.chunkBuffer}${e}`,e===n.QUOTE)if(this.isCharInsideQuotes&&!this.isCharEscaped){let t=JSON.parse(this.chunkBuffer);this.resultBuffer.push(t),this.resetParser()}else this.chunkBuffer===n.QUOTE?this.isCharInsideQuotes=!0:this.isCharEscaped&&(this.isCharEscaped=!1);else e===n.ESCAPE?this.isCharEscaped=!this.isCharEscaped:this.isCharEscaped&&(this.isCharEscaped=!1)}primitiveElementParser(e){if([n.COMMA,n.BRACKET.CLOSE].includes(e)){let t=JSON.parse(this.chunkBuffer);this.resultBuffer.push(t),this.resetParser()}else this.chunkBuffer=`${this.chunkBuffer}${e}`}containerElementParser(e){let t=this.elementType==="array"?n.BRACKET:n.BRACE;if(e===t.OPEN&&!this.isCharInsideQuotes)this.chunkBuffer=`${this.chunkBuffer}${e}`,this.elementEnclosureCount+=1;else if(e===t.CLOSE&&!this.isCharInsideQuotes){if(this.chunkBuffer=`${this.chunkBuffer}${e}`,this.elementEnclosureCount-=1,this.elementEnclosureCount===0){let i=JSON.parse(this.chunkBuffer);this.resultBuffer.push(i),this.resetParser()}}else this.chunkBuffer.length&&(this.chunkBuffer=`${this.chunkBuffer}${e}`,e===n.ESCAPE?this.isCharEscaped=!this.isCharEscaped:e===n.QUOTE&&!this.isCharEscaped?this.isCharInsideQuotes=!this.isCharInsideQuotes:this.isCharEscaped&&(this.isCharEscaped=!1))}stream(e){return g(this,null,function*(){var t,i,a,h,m;try{for(var o=R(this.chunkGenerator()),l,r,u;l=!(r=yield new C(o.next())).done;l=!1){let p=r.value;for(let c of p){if(!this.rootDetected){this.rootDetected=c===n.BRACKET.OPEN;continue}this.elementDetected||(this.elementDetected=![n.SPACE,n.COMMA,n.NEW_LINE,n.BRACKET.CLOSE].includes(c)),this.elementDetected&&(this.elementParser||(c===n.BRACKET.OPEN?(this.elementType="array",this.elementParser=this.containerElementParser):c===n.BRACE.OPEN?(this.elementType="object",this.elementParser=this.containerElementParser):c===n.QUOTE?(this.elementType="string",this.elementParser=this.stringElementParser):(this.elementType="others",this.elementParser=this.primitiveElementParser)),this.elementParser(c),this.resultBuffer.length===e&&((t=this.readStream)!=null&&t.closed||(i=this.readStream)==null||i.pause(),yield this.resultBuffer.splice(0,e),(a=this.readStream)!=null&&a.closed||(h=this.readStream)==null||h.resume()))}}}catch(r){u=[r]}finally{try{l&&(r=o.return)&&(yield new C(r.call(o)))}finally{if(u)throw u[0]}}if((m=this.readStream)==null||m.close(),this.readStream=null,this.chunkBuffer.length){let p=JSON.parse(this.chunkBuffer);this.resultBuffer.push(p),this.resetParser()}return this.resultBuffer.length&&(yield this.resultBuffer.splice(0)),this.resultBuffer})}};d.sanitizeReadStreamOptions=e=>{let t=v({},e||{});return t.encoding||(t.encoding="utf-8"),Object.keys(t).forEach(i=>{["signal","encoding","highWatermark"].includes(i)||delete t[i]}),t},d.create=(e,t)=>O(d,null,function*(){let i=new d(e,t);return yield(0,b.once)(i.readStream,"readable"),i});var w=d,Q=w;var T=require("fs/promises");var U=(s,e,...t)=>O(void 0,null,function*(){let i=null;try{let{size:a}=yield(0,T.stat)(s),h=Buffer.byteLength(n.BRACKET.CLOSE,e),o=Math.min(h,a),l=Buffer.alloc(o),r=a;i=yield(0,T.open)(s,"a+");e:for(;r;){r-=o,yield i.read(l,0,o,r);let f=l.toString(e);for(let E=f.length-1;E>=0;E--)if(f[E]===n.BRACKET.CLOSE){r+=E===0?0:Buffer.byteLength(f.slice(0,E),e);break e}}let u=Buffer.alloc(a-r);yield i.read(u,0,u.length,r);let m=t.map(f=>JSON.stringify(f)).join(",");if(r){let f=r;e:for(;;){f-=o,yield i.read(l,0,o,f);let E=l.toString(e);for(let B=E.length-1;B>=0;B--)if(![n.NEW_LINE,n.SPACE].includes(E[B])){E[B]!==n.BRACKET.OPEN&&(m=`,${m}`);break e}}}else m=`[${m}]`;let p=Buffer.from(m,e),c=Buffer.alloc(p.length+u.length);p.copy(c),u.copy(c,p.length),yield i.write(c,0,c.length,r)}catch(a){throw a}finally{yield i==null?void 0:i.close()}}),D=U;var H=Q.create;0&&(module.exports={appendFile,createReadStream});
+"use strict";var T=Object.defineProperty;var L=Object.getOwnPropertyDescriptor;var I=Object.getOwnPropertyNames,y=Object.getOwnPropertySymbols;var A=Object.prototype.hasOwnProperty,$=Object.prototype.propertyIsEnumerable;var R=(i,e)=>(e=Symbol[i])?e:Symbol.for("Symbol."+i);var k=(i,e,t)=>e in i?T(i,e,{enumerable:!0,configurable:!0,writable:!0,value:t}):i[e]=t,b=(i,e)=>{for(var t in e||(e={}))A.call(e,t)&&k(i,t,e[t]);if(y)for(var t of y(e))$.call(e,t)&&k(i,t,e[t]);return i};var K=(i,e)=>{for(var t in e)T(i,t,{get:e[t],enumerable:!0})},M=(i,e,t,s)=>{if(e&&typeof e=="object"||typeof e=="function")for(let r of I(e))!A.call(i,r)&&r!==t&&T(i,r,{get:()=>e[r],enumerable:!(s=L(e,r))||s.enumerable});return i};var z=i=>M(T({},"__esModule",{value:!0}),i);var S=(i,e,t)=>new Promise((s,r)=>{var o=a=>{try{f(t.next(a))}catch(u){r(u)}},h=a=>{try{f(t.throw(a))}catch(u){r(u)}},f=a=>a.done?s(a.value):Promise.resolve(a.value).then(o,h);f((t=t.apply(i,e)).next())}),C=function(i,e){this[0]=i,this[1]=e},P=(i,e,t)=>{var s=(h,f,a,u)=>{try{var E=t[h](f),d=(f=E.value)instanceof C,c=E.done;Promise.resolve(d?f[0]:f).then(n=>d?s(h==="return"?h:"next",f[1]?{done:n.done,value:n.value}:n,a,u):a({value:n,done:c})).catch(n=>s("throw",n,a,u))}catch(n){u(n)}},r=h=>o[h]=f=>new Promise((a,u)=>s(h,f,a,u)),o={};return t=t.apply(i,e),o[R("asyncIterator")]=()=>o,r("next"),r("throw"),r("return"),o};var g=(i,e,t)=>(e=i[R("asyncIterator")])?e.call(i):(i=i[R("iterator")](),e={},t=(s,r)=>(r=i[s])&&(e[s]=o=>new Promise((h,f,a)=>(o=r.call(i,o),a=o.done,Promise.resolve(o.value).then(u=>h({value:u,done:a}),f)))),t("next"),t("return"),e);var x={};K(x,{appendFile:()=>D,createReadStream:()=>H});module.exports=z(x);var v=require("fs"),N=require("events");var l={BRACKET:{OPEN:"[",CLOSE:"]"},BRACE:{OPEN:"{",CLOSE:"}"},QUOTE:'"',ESCAPE:"\\",SPACE:" ",COMMA:",",NEW_LINE:`
+`};var p=class p{constructor(e,t){this.readStream=(0,v.createReadStream)(e,p.sanitizeReadStreamOptions(t)),this.rootDetected=!1,this.elementDetected=!1,this.elementType="others",this.elementParser=null,this.elementEnclosureCount=0,this.isCharInsideQuotes=!1,this.isCharEscaped=!1,this.chunkBuffer="",this.resultBuffer=[]}chunkGenerator(){return P(this,null,function*(){if(!this.readStream)throw new Error("Stream not initialized");try{for(var e=g(this.readStream),t,s,r;t=!(s=yield new C(e.next())).done;t=!1){let o=s.value;yield o}}catch(s){r=[s]}finally{try{t&&(s=e.return)&&(yield new C(s.call(e)))}finally{if(r)throw r[0]}}})}addToResult(e,t){if(!t)this.resultBuffer.push(e);else try{t(e)&&this.resultBuffer.push(e)}catch(s){}}resetParser(){this.elementDetected=!1,this.elementType="others",this.elementParser=null,this.elementEnclosureCount=0,this.isCharInsideQuotes=!1,this.isCharEscaped=!1,this.chunkBuffer=""}stringElementParser(e,t){if(this.chunkBuffer=`${this.chunkBuffer}${e}`,e===l.QUOTE)if(this.isCharInsideQuotes&&!this.isCharEscaped){let s=JSON.parse(this.chunkBuffer);this.addToResult(s,t),this.resetParser()}else this.chunkBuffer===l.QUOTE?this.isCharInsideQuotes=!0:this.isCharEscaped&&(this.isCharEscaped=!1);else e===l.ESCAPE?this.isCharEscaped=!this.isCharEscaped:this.isCharEscaped&&(this.isCharEscaped=!1)}primitiveElementParser(e,t){if([l.COMMA,l.BRACKET.CLOSE].includes(e)){let s=JSON.parse(this.chunkBuffer);this.addToResult(s,t),this.resetParser()}else this.chunkBuffer=`${this.chunkBuffer}${e}`}containerElementParser(e,t){let s=this.elementType==="array"?l.BRACKET:l.BRACE;if(e===s.OPEN&&!this.isCharInsideQuotes)this.chunkBuffer=`${this.chunkBuffer}${e}`,this.elementEnclosureCount+=1;else if(e===s.CLOSE&&!this.isCharInsideQuotes){if(this.chunkBuffer=`${this.chunkBuffer}${e}`,this.elementEnclosureCount-=1,this.elementEnclosureCount===0){let r=JSON.parse(this.chunkBuffer);this.addToResult(r,t),this.resetParser()}}else this.chunkBuffer.length&&(this.chunkBuffer=`${this.chunkBuffer}${e}`,e===l.ESCAPE?this.isCharEscaped=!this.isCharEscaped:e===l.QUOTE&&!this.isCharEscaped?this.isCharInsideQuotes=!this.isCharInsideQuotes:this.isCharEscaped&&(this.isCharEscaped=!1))}stream(e,t){return P(this,null,function*(){var s,r,o,h,d;try{for(var f=g(this.chunkGenerator()),a,u,E;a=!(u=yield new C(f.next())).done;a=!1){let c=u.value;for(let n of c){if(!this.rootDetected){this.rootDetected=n===l.BRACKET.OPEN;continue}this.elementDetected||(this.elementDetected=![l.SPACE,l.COMMA,l.NEW_LINE,l.BRACKET.CLOSE].includes(n)),this.elementDetected&&(this.elementParser||(n===l.BRACKET.OPEN?(this.elementType="array",this.elementParser=this.containerElementParser):n===l.BRACE.OPEN?(this.elementType="object",this.elementParser=this.containerElementParser):n===l.QUOTE?(this.elementType="string",this.elementParser=this.stringElementParser):(this.elementType="others",this.elementParser=this.primitiveElementParser)),this.elementParser(n,t),this.resultBuffer.length===e&&((s=this.readStream)!=null&&s.closed||(r=this.readStream)==null||r.pause(),yield this.resultBuffer.splice(0,e),(o=this.readStream)!=null&&o.closed||(h=this.readStream)==null||h.resume()))}}}catch(u){E=[u]}finally{try{a&&(u=f.return)&&(yield new C(u.call(f)))}finally{if(E)throw E[0]}}if((d=this.readStream)==null||d.close(),this.readStream=null,this.chunkBuffer.length){let c=JSON.parse(this.chunkBuffer);this.addToResult(c,t),this.resetParser()}return this.resultBuffer.length&&(yield this.resultBuffer.splice(0)),this.resultBuffer})}};p.sanitizeReadStreamOptions=e=>{let t=b({},e||{});return t.encoding||(t.encoding="utf-8"),Object.keys(t).forEach(s=>{["signal","encoding","highWatermark"].includes(s)||delete t[s]}),t},p.create=(e,t)=>S(p,null,function*(){let s=new p(e,t);return yield(0,N.once)(s.readStream,"readable"),s});var w=p,Q=w;var O=require("fs/promises");var U=(i,e,...t)=>S(void 0,null,function*(){let s=null;try{let{size:r}=yield(0,O.stat)(i),o=Buffer.byteLength(l.BRACKET.CLOSE,e),h=Math.min(o,r),f=Buffer.alloc(h),a=r;s=yield(0,O.open)(i,"a+");e:for(;a;){a-=h,yield s.read(f,0,h,a);let n=f.toString(e);for(let m=n.length-1;m>=0;m--)if(n[m]===l.BRACKET.CLOSE){a+=m===0?0:Buffer.byteLength(n.slice(0,m),e);break e}}let u=Buffer.alloc(r-a);yield s.read(u,0,u.length,a);let E=t.map(n=>JSON.stringify(n)).join(",");if(a){let n=a;e:for(;;){n-=h,yield s.read(f,0,h,n);let m=f.toString(e);for(let B=m.length-1;B>=0;B--)if(![l.NEW_LINE,l.SPACE].includes(m[B])){m[B]!==l.BRACKET.OPEN&&(E=`,${E}`);break e}}}else E=`[${E}]`;let d=Buffer.from(E,e),c=Buffer.alloc(d.length+u.length);d.copy(c),u.copy(c,d.length),yield s.write(c,0,c.length,a)}catch(r){throw r}finally{yield s==null?void 0:s.close()}}),D=U;var H=Q.create;0&&(module.exports={appendFile,createReadStream});

dist/index.mjs

@@ -1,2 +0,2 @@
-var A=Object.defineProperty;var R=Object.getOwnPropertySymbols;var v=Object.prototype.hasOwnProperty,N=Object.prototype.propertyIsEnumerable;var O=(i,e)=>(e=Symbol[i])?e:Symbol.for("Symbol."+i);var w=(i,e,t)=>e in i?A(i,e,{enumerable:!0,configurable:!0,writable:!0,value:t}):i[e]=t,k=(i,e)=>{for(var t in e||(e={}))v.call(e,t)&&w(i,t,e[t]);if(R)for(var t of R(e))N.call(e,t)&&w(i,t,e[t]);return i};var S=(i,e,t)=>new Promise((s,n)=>{var h=r=>{try{l(t.next(r))}catch(u){n(u)}},o=r=>{try{l(t.throw(r))}catch(u){n(u)}},l=r=>r.done?s(r.value):Promise.resolve(r.value).then(h,o);l((t=t.apply(i,e)).next())}),C=function(i,e){this[0]=i,this[1]=e},T=(i,e,t)=>{var s=(o,l,r,u)=>{try{var m=t[o](l),p=(l=m.value)instanceof C,c=m.done;Promise.resolve(p?l[0]:l).then(f=>p?s(o==="return"?o:"next",l[1]?{done:f.done,value:f.value}:f,r,u):r({value:f,done:c})).catch(f=>s("throw",f,r,u))}catch(f){u(f)}},n=o=>h[o]=l=>new Promise((r,u)=>s(o,l,r,u)),h={};return t=t.apply(i,e),h[O("asyncIterator")]=()=>h,n("next"),n("throw"),n("return"),h};var P=(i,e,t)=>(e=i[O("asyncIterator")])?e.call(i):(i=i[O("iterator")](),e={},t=(s,n)=>(n=i[s])&&(e[s]=h=>new Promise((o,l,r)=>(h=n.call(i,h),r=h.done,Promise.resolve(h.value).then(u=>o({value:u,done:r}),l)))),t("next"),t("return"),e);import{createReadStream as b}from"fs";import{once as Q}from"events";var a={BRACKET:{OPEN:"[",CLOSE:"]"},BRACE:{OPEN:"{",CLOSE:"}"},QUOTE:'"',ESCAPE:"\\",SPACE:" ",COMMA:",",NEW_LINE:`
-`};var d=class d{constructor(e,t){this.readStream=b(e,d.sanitizeReadStreamOptions(t)),this.rootDetected=!1,this.elementDetected=!1,this.elementType="others",this.elementParser=null,this.elementEnclosureCount=0,this.isCharInsideQuotes=!1,this.isCharEscaped=!1,this.chunkBuffer="",this.resultBuffer=[]}chunkGenerator(){return T(this,null,function*(){if(!this.readStream)throw new Error("Stream not initialized");try{for(var e=P(this.readStream),t,s,n;t=!(s=yield new C(e.next())).done;t=!1){let h=s.value;yield h}}catch(s){n=[s]}finally{try{t&&(s=e.return)&&(yield new C(s.call(e)))}finally{if(n)throw n[0]}}})}resetParser(){this.elementDetected=!1,this.elementType="others",this.elementParser=null,this.elementEnclosureCount=0,this.isCharInsideQuotes=!1,this.isCharEscaped=!1,this.chunkBuffer=""}stringElementParser(e){if(this.chunkBuffer=`${this.chunkBuffer}${e}`,e===a.QUOTE)if(this.isCharInsideQuotes&&!this.isCharEscaped){let t=JSON.parse(this.chunkBuffer);this.resultBuffer.push(t),this.resetParser()}else this.chunkBuffer===a.QUOTE?this.isCharInsideQuotes=!0:this.isCharEscaped&&(this.isCharEscaped=!1);else e===a.ESCAPE?this.isCharEscaped=!this.isCharEscaped:this.isCharEscaped&&(this.isCharEscaped=!1)}primitiveElementParser(e){if([a.COMMA,a.BRACKET.CLOSE].includes(e)){let t=JSON.parse(this.chunkBuffer);this.resultBuffer.push(t),this.resetParser()}else this.chunkBuffer=`${this.chunkBuffer}${e}`}containerElementParser(e){let t=this.elementType==="array"?a.BRACKET:a.BRACE;if(e===t.OPEN&&!this.isCharInsideQuotes)this.chunkBuffer=`${this.chunkBuffer}${e}`,this.elementEnclosureCount+=1;else if(e===t.CLOSE&&!this.isCharInsideQuotes){if(this.chunkBuffer=`${this.chunkBuffer}${e}`,this.elementEnclosureCount-=1,this.elementEnclosureCount===0){let s=JSON.parse(this.chunkBuffer);this.resultBuffer.push(s),this.resetParser()}}else this.chunkBuffer.length&&(this.chunkBuffer=`${this.chunkBuffer}${e}`,e===a.ESCAPE?this.isCharEscaped=!this.isCharEscaped:e===a.QUOTE&&!this.isCharEscaped?this.isCharInsideQuotes=!this.isCharInsideQuotes:this.isCharEscaped&&(this.isCharEscaped=!1))}stream(e){return T(this,null,function*(){var t,s,n,h,m;try{for(var o=P(this.chunkGenerator()),l,r,u;l=!(r=yield new C(o.next())).done;l=!1){let p=r.value;for(let c of p){if(!this.rootDetected){this.rootDetected=c===a.BRACKET.OPEN;continue}this.elementDetected||(this.elementDetected=![a.SPACE,a.COMMA,a.NEW_LINE,a.BRACKET.CLOSE].includes(c)),this.elementDetected&&(this.elementParser||(c===a.BRACKET.OPEN?(this.elementType="array",this.elementParser=this.containerElementParser):c===a.BRACE.OPEN?(this.elementType="object",this.elementParser=this.containerElementParser):c===a.QUOTE?(this.elementType="string",this.elementParser=this.stringElementParser):(this.elementType="others",this.elementParser=this.primitiveElementParser)),this.elementParser(c),this.resultBuffer.length===e&&((t=this.readStream)!=null&&t.closed||(s=this.readStream)==null||s.pause(),yield this.resultBuffer.splice(0,e),(n=this.readStream)!=null&&n.closed||(h=this.readStream)==null||h.resume()))}}}catch(r){u=[r]}finally{try{l&&(r=o.return)&&(yield new C(r.call(o)))}finally{if(u)throw u[0]}}if((m=this.readStream)==null||m.close(),this.readStream=null,this.chunkBuffer.length){let p=JSON.parse(this.chunkBuffer);this.resultBuffer.push(p),this.resetParser()}return this.resultBuffer.length&&(yield this.resultBuffer.splice(0)),this.resultBuffer})}};d.sanitizeReadStreamOptions=e=>{let t=k({},e||{});return t.encoding||(t.encoding="utf-8"),Object.keys(t).forEach(s=>{["signal","encoding","highWatermark"].includes(s)||delete t[s]}),t},d.create=(e,t)=>S(d,null,function*(){let s=new d(e,t);return yield Q(s.readStream,"readable"),s});var g=d,y=g;import{stat as D,open as L}from"fs/promises";var I=(i,e,...t)=>S(void 0,null,function*(){let s=null;try{let{size:n}=yield D(i),h=Buffer.byteLength(a.BRACKET.CLOSE,e),o=Math.min(h,n),l=Buffer.alloc(o),r=n;s=yield L(i,"a+");e:for(;r;){r-=o,yield s.read(l,0,o,r);let f=l.toString(e);for(let E=f.length-1;E>=0;E--)if(f[E]===a.BRACKET.CLOSE){r+=E===0?0:Buffer.byteLength(f.slice(0,E),e);break e}}let u=Buffer.alloc(n-r);yield s.read(u,0,u.length,r);let m=t.map(f=>JSON.stringify(f)).join(",");if(r){let f=r;e:for(;;){f-=o,yield s.read(l,0,o,f);let E=l.toString(e);for(let B=E.length-1;B>=0;B--)if(![a.NEW_LINE,a.SPACE].includes(E[B])){E[B]!==a.BRACKET.OPEN&&(m=`,${m}`);break e}}}else m=`[${m}]`;let p=Buffer.from(m,e),c=Buffer.alloc(p.length+u.length);p.copy(c),u.copy(c,p.length),yield s.write(c,0,c.length,r)}catch(n){throw n}finally{yield s==null?void 0:s.close()}}),$=I;var Z=y.create;export{$ as appendFile,Z as createReadStream};
+var A=Object.defineProperty;var g=Object.getOwnPropertySymbols;var b=Object.prototype.hasOwnProperty,v=Object.prototype.propertyIsEnumerable;var S=(i,e)=>(e=Symbol[i])?e:Symbol.for("Symbol."+i);var w=(i,e,t)=>e in i?A(i,e,{enumerable:!0,configurable:!0,writable:!0,value:t}):i[e]=t,y=(i,e)=>{for(var t in e||(e={}))b.call(e,t)&&w(i,t,e[t]);if(g)for(var t of g(e))v.call(e,t)&&w(i,t,e[t]);return i};var T=(i,e,t)=>new Promise((s,l)=>{var o=r=>{try{f(t.next(r))}catch(u){l(u)}},h=r=>{try{f(t.throw(r))}catch(u){l(u)}},f=r=>r.done?s(r.value):Promise.resolve(r.value).then(o,h);f((t=t.apply(i,e)).next())}),C=function(i,e){this[0]=i,this[1]=e},O=(i,e,t)=>{var s=(h,f,r,u)=>{try{var E=t[h](f),d=(f=E.value)instanceof C,c=E.done;Promise.resolve(d?f[0]:f).then(a=>d?s(h==="return"?h:"next",f[1]?{done:a.done,value:a.value}:a,r,u):r({value:a,done:c})).catch(a=>s("throw",a,r,u))}catch(a){u(a)}},l=h=>o[h]=f=>new Promise((r,u)=>s(h,f,r,u)),o={};return t=t.apply(i,e),o[S("asyncIterator")]=()=>o,l("next"),l("throw"),l("return"),o};var R=(i,e,t)=>(e=i[S("asyncIterator")])?e.call(i):(i=i[S("iterator")](),e={},t=(s,l)=>(l=i[s])&&(e[s]=o=>new Promise((h,f,r)=>(o=l.call(i,o),r=o.done,Promise.resolve(o.value).then(u=>h({value:u,done:r}),f)))),t("next"),t("return"),e);import{createReadStream as N}from"fs";import{once as Q}from"events";var n={BRACKET:{OPEN:"[",CLOSE:"]"},BRACE:{OPEN:"{",CLOSE:"}"},QUOTE:'"',ESCAPE:"\\",SPACE:" ",COMMA:",",NEW_LINE:`
+`};var p=class p{constructor(e,t){this.readStream=N(e,p.sanitizeReadStreamOptions(t)),this.rootDetected=!1,this.elementDetected=!1,this.elementType="others",this.elementParser=null,this.elementEnclosureCount=0,this.isCharInsideQuotes=!1,this.isCharEscaped=!1,this.chunkBuffer="",this.resultBuffer=[]}chunkGenerator(){return O(this,null,function*(){if(!this.readStream)throw new Error("Stream not initialized");try{for(var e=R(this.readStream),t,s,l;t=!(s=yield new C(e.next())).done;t=!1){let o=s.value;yield o}}catch(s){l=[s]}finally{try{t&&(s=e.return)&&(yield new C(s.call(e)))}finally{if(l)throw l[0]}}})}addToResult(e,t){if(!t)this.resultBuffer.push(e);else try{t(e)&&this.resultBuffer.push(e)}catch(s){}}resetParser(){this.elementDetected=!1,this.elementType="others",this.elementParser=null,this.elementEnclosureCount=0,this.isCharInsideQuotes=!1,this.isCharEscaped=!1,this.chunkBuffer=""}stringElementParser(e,t){if(this.chunkBuffer=`${this.chunkBuffer}${e}`,e===n.QUOTE)if(this.isCharInsideQuotes&&!this.isCharEscaped){let s=JSON.parse(this.chunkBuffer);this.addToResult(s,t),this.resetParser()}else this.chunkBuffer===n.QUOTE?this.isCharInsideQuotes=!0:this.isCharEscaped&&(this.isCharEscaped=!1);else e===n.ESCAPE?this.isCharEscaped=!this.isCharEscaped:this.isCharEscaped&&(this.isCharEscaped=!1)}primitiveElementParser(e,t){if([n.COMMA,n.BRACKET.CLOSE].includes(e)){let s=JSON.parse(this.chunkBuffer);this.addToResult(s,t),this.resetParser()}else this.chunkBuffer=`${this.chunkBuffer}${e}`}containerElementParser(e,t){let s=this.elementType==="array"?n.BRACKET:n.BRACE;if(e===s.OPEN&&!this.isCharInsideQuotes)this.chunkBuffer=`${this.chunkBuffer}${e}`,this.elementEnclosureCount+=1;else if(e===s.CLOSE&&!this.isCharInsideQuotes){if(this.chunkBuffer=`${this.chunkBuffer}${e}`,this.elementEnclosureCount-=1,this.elementEnclosureCount===0){let l=JSON.parse(this.chunkBuffer);this.addToResult(l,t),this.resetParser()}}else this.chunkBuffer.length&&(this.chunkBuffer=`${this.chunkBuffer}${e}`,e===n.ESCAPE?this.isCharEscaped=!this.isCharEscaped:e===n.QUOTE&&!this.isCharEscaped?this.isCharInsideQuotes=!this.isCharInsideQuotes:this.isCharEscaped&&(this.isCharEscaped=!1))}stream(e,t){return O(this,null,function*(){var s,l,o,h,d;try{for(var f=R(this.chunkGenerator()),r,u,E;r=!(u=yield new C(f.next())).done;r=!1){let c=u.value;for(let a of c){if(!this.rootDetected){this.rootDetected=a===n.BRACKET.OPEN;continue}this.elementDetected||(this.elementDetected=![n.SPACE,n.COMMA,n.NEW_LINE,n.BRACKET.CLOSE].includes(a)),this.elementDetected&&(this.elementParser||(a===n.BRACKET.OPEN?(this.elementType="array",this.elementParser=this.containerElementParser):a===n.BRACE.OPEN?(this.elementType="object",this.elementParser=this.containerElementParser):a===n.QUOTE?(this.elementType="string",this.elementParser=this.stringElementParser):(this.elementType="others",this.elementParser=this.primitiveElementParser)),this.elementParser(a,t),this.resultBuffer.length===e&&((s=this.readStream)!=null&&s.closed||(l=this.readStream)==null||l.pause(),yield this.resultBuffer.splice(0,e),(o=this.readStream)!=null&&o.closed||(h=this.readStream)==null||h.resume()))}}}catch(u){E=[u]}finally{try{r&&(u=f.return)&&(yield new C(u.call(f)))}finally{if(E)throw E[0]}}if((d=this.readStream)==null||d.close(),this.readStream=null,this.chunkBuffer.length){let c=JSON.parse(this.chunkBuffer);this.addToResult(c,t),this.resetParser()}return this.resultBuffer.length&&(yield this.resultBuffer.splice(0)),this.resultBuffer})}};p.sanitizeReadStreamOptions=e=>{let t=y({},e||{});return t.encoding||(t.encoding="utf-8"),Object.keys(t).forEach(s=>{["signal","encoding","highWatermark"].includes(s)||delete t[s]}),t},p.create=(e,t)=>T(p,null,function*(){let s=new p(e,t);return yield Q(s.readStream,"readable"),s});var P=p,k=P;import{stat as D,open as L}from"fs/promises";var I=(i,e,...t)=>T(void 0,null,function*(){let s=null;try{let{size:l}=yield D(i),o=Buffer.byteLength(n.BRACKET.CLOSE,e),h=Math.min(o,l),f=Buffer.alloc(h),r=l;s=yield L(i,"a+");e:for(;r;){r-=h,yield s.read(f,0,h,r);let a=f.toString(e);for(let m=a.length-1;m>=0;m--)if(a[m]===n.BRACKET.CLOSE){r+=m===0?0:Buffer.byteLength(a.slice(0,m),e);break e}}let u=Buffer.alloc(l-r);yield s.read(u,0,u.length,r);let E=t.map(a=>JSON.stringify(a)).join(",");if(r){let a=r;e:for(;;){a-=h,yield s.read(f,0,h,a);let m=f.toString(e);for(let B=m.length-1;B>=0;B--)if(![n.NEW_LINE,n.SPACE].includes(m[B])){m[B]!==n.BRACKET.OPEN&&(E=`,${E}`);break e}}}else E=`[${E}]`;let d=Buffer.from(E,e),c=Buffer.alloc(d.length+u.length);d.copy(c),u.copy(c,d.length),yield s.write(c,0,c.length,r)}catch(l){throw l}finally{yield s==null?void 0:s.close()}}),$=I;var Z=k.create;export{$ as appendFile,Z as createReadStream};

package.json

 {
   "name": "jsonarrayfs",
-  "version": "1.0.1",
-  "description": "A Node.js library to stream JSON array elements from files in custom-sized chunks and effortlessly append data to existing arrays.",
+  "version": "1.1.0",
+  "description": "Efficiently handle JSON array files in Node.js with minimal memory usage. Perfect for efficiently processing large data volumes without worrying about memory limitations.",
   "exports": {
@@ -24,6 +24,13 @@ ".": {
     "stream",
-    "json array",
     "chunk",
     "batch",
-    "append json"
+    "json file handling",
+    "json array",
+    "json stream",
+    "stream filter",
+    "json append",
+    "stream processing",
+    "on the fly filtering",
+    "direct appends",
+    "memory efficient"
   ],
@@ -30,0 +37,0 @@ "author": "Akash S Panickar",

README.md

 # jsonarrayfs <img src="https://img.shields.io/npm/dm/jsonarrayfs" />
 
-jsonarrayfs is a Node.js library designed to facilitate efficient handling of JSON array data stored in files. It offers functionality for streaming JSON array elements in batches, allowing for reduced memory consumption when working with large datasets. Additionally, jsonarrayfs provides a method for appending new data to existing JSON array files without loading the entire dataset into memory.
+"jsonarrayfs" is a Node.js library crafted for robust and memory-efficient management of massive JSON array files. It enables seamless handling of JSON arrays without the need to load the entire file into memory, making it perfect for efficiently managing large datasets without overwhelming system resources.
 
+## Key Features
+
+- **Stream Processing**: Read JSON array in manageable chunks (eg: 50k elements at a time) using stream.
+- **On-the-Fly Filtering**: Apply filter to the stream to fetch only relevant data, reducing the data you handle even further.
+- **Direct Appends**: Append new elements directly to the JSON array file, avoiding unnecessary loading, modification and rewriting.
+
+## Benefits
+
+- **Memory Optimization**: Process JSON array files with minimal memory usage, making it ideal for resource-constrained environments.
+- **Handles Large Datasets**: Efficiently manage massive JSON array files without memory limitations.
+- **Improved Performance**: Faster processing times due to efficient streaming, filtering and appending capabilities.
+- **Enhanced Scalability**: Scales seamlessly with growing datasets, ensuring smooth performance.
+
 ## Installation
@@ -13,11 +26,11 @@
 
-## Features
+## Usage
 
-- Stream JSON array elements:
+- Stream Processing:
 
-```js
+```ts
 import { createReadStream } from "jsonarrayfs";
 
 // Create a streamer to read JSON array elements from a file
-const streamer = await createReadStream("./data.json", { encoding: 'utf-8' });
+const streamer = await createReadStream("./data.json", { encoding: "utf-8" });
 
@@ -30,5 +43,24 @@ // Stream JSON array elements in batches of 100
 
+- On-the-Fly Filtering:
+
+```ts
+import { createReadStream } from "jsonarrayfs";
+
+const streamer = await createReadStream<{ offer: boolean; price: number }>(
+  "./data.json",
+  { encoding: "utf-8" }
+);
+
+// Add filter to the stream to fetch only relevant elements
+for await (const chunk of streamer.stream(
+  100,
+  (element) => element.price < 500 || element.offer
+)) {
+  // Your processing logic here
+}
+```
+
 - Append data to existing JSON array:
 
-```js
+```ts
 import { appendFile } from "jsonarrayfs";
@@ -38,8 +70,8 @@
 const newData = [
-  { id: 1, name: "JavaScript" },
-  { id: 2, name: "Go" }
+  { id: 1, name: "Earth", price: 1000, offer: true },
+  { id: 2, name: "Moon", price: 500, offer: false },
 ];
 
 // Append new data to the existing JSON array file
-await appendFile("./data.json", 'utf-8', ...newData);
+await appendFile("./data.json", "utf-8", ...newData);
 ```
@@ -53,2 +85,2 @@
 
-[MIT License ](https://github.com/mochatek/jsonarrayfs/blob/main/LICENSE)
+[MIT License ](https://github.com/mochatek/jsonarrayfs/blob/main/LICENSE)

New alerts

Filesystem access

Supply chain risk

Accesses the file system, and could potentially read sensitive data.

Found 2 instances in 1 package

Long strings

Supply chain risk

Contains long string literals, which may be a sign of obfuscated or packed code.

Found 1 instance in 1 package

Fixed alerts

Filesystem access

Supply chain risk

Accesses the file system, and could potentially read sensitive data.

Found 2 instances in 1 package

Long strings

Supply chain risk

Contains long string literals, which may be a sign of obfuscated or packed code.

Found 1 instance in 1 package

Improved metrics

Total package byte prevSize

20393

9.12%

Lines of code

83

3.75%

Number of lines in readme file

83

66%

No dependency changes