@diotoborg/assumenda-rem-iure
Advanced tools
A tool for JSON parse and stringify by stream and web worker.
A tool for JSON parse and stringify by stream and web worker.
To view documentation or get support, visit docs.
Note: @diotoborg/assumenda-rem-iure is mainly dependent on
from2andjsonparse.
npm i @diotoborg/assumenda-rem-iure
Link: [dev source code]
async function testSmall() {
let obj = {
a: 123,
b: 145.67,
c: 'test中文1',
d: true,
f: [11, 'xyz', false, [166, 197, 215]],
g: {
ga: 223,
gb: 245.67,
gc: 'test中文2',
gd: [66, 97, 115],
},
}
let arr = [
{
a: 123,
b: 145.67,
c: 'test中文1',
d: true,
},
{
f: [11, 'xyz', false, [166, 197, 215]],
g: {
ga: 223,
gb: 245.67,
gc: 'test中文2',
gd: [66, 97, 115],
},
},
]
let jobj = await json.stringify(obj)
console.log(jobj)
// => {"a":123,"b":145.67,"c":"test中文1","d":true,"f":[11,"xyz",false,[166,197,215]],"g":{"ga":223,"gb":245.67,"gc":"test中文2","gd":[66,97,115]}}
let jarr = await json.stringify(arr)
console.log(jarr)
// => [{"a":123,"b":145.67,"c":"test中文1","d":true},{"f":[11,"xyz",false,[166,197,215]],"g":{"ga":223,"gb":245.67,"gc":"test中文2","gd":[66,97,115]}}]
let robj = await json.parse(jobj)
console.log(robj)
// => {
// a: 123,
// b: 145.67,
// c: 'test中文1',
// d: true,
// f: [ 11, 'xyz', false, [ 166, 197, 215 ] ],
// g: { ga: 223, gb: 245.67, gc: 'test中文2', gd: [ 66, 97, 115 ] }
// }
let rarr = await json.parse(jarr)
console.log(rarr)
// => [
// { a: 123, b: 145.67, c: 'test中文1', d: true },
// {
// f: [ 11, 'xyz', false, [Array] ],
// g: { ga: 223, gb: 245.67, gc: 'test中文2', gd: [Array] }
// }
// ]
}
testSmall()
.catch((err) => {
console.log(err)
})
Link: [dev source code]
async function testLarge() {
let n = 700 //7000, 9000
let lgRows = []
for (let i = 1; i <= 1000; i++) {
lgRows.push([i, 23.4, '567', 'abc', true, false, [], {}, [8, 9.01], { a: 12.34, b: 'xyz', i }])
}
let lgArr = []
for (let i = 1; i <= n; i++) {
lgArr.push({
key: 'k' + i,
value: {
a: 123,
b: 145.67,
c: 'test中文1',
d: true,
f: [11, 'xyz', false, [166, 197, 215]],
g: {
ga: 223,
gb: 245.67,
gc: 'test中文2',
gd: [66, 97, 115],
ge: JSON.parse(JSON.stringify(lgRows)),
},
}
})
}
if (true) {
try {
let res = JSON.stringify(lgArr)
console.log('JSON.stringify(lgArr)', res.length, res.substr(0, 200) + '...')
}
catch (err) {
console.log('JSON.stringify(lgArr) catch', err)
// n=7000 => catch RangeError: Invalid string length => Out Of Memory
// n=9000 => FATAL ERROR: MarkCompactCollector: young object promotion failed Allocation failed - JavaScript heap out of memory
}
}
if (true) {
await json.stringify(lgArr)
.then((res) => {
console.log('json.stringify(lgArr) then', res.length, res.substr(0, 200) + '...')
})
.catch((err) => {
console.log('json.stringify(lgArr) catch', err)
// n=7000 => FATAL ERROR: Ineffective mark-compacts near heap limit Allocation failed - JavaScript heap out of memory
})
}
}
testLarge()
.catch((err) => {
console.log(err)
})
Link: [dev source code]
async function testStream() {
async function testStringify() {
return new Promise((resolve, reject) => {
//fp
let fp = './temp-stringify.json'
//arr
let n = 4000 //4000000
let arr = []
for (let i = 1; i <= n; i++) {
arr.push({
key: 'k' + i,
value: {
a: 123,
b: 145.67,
c: 'test中文1',
d: true,
f: [11, 'xyz', false, [166, 197, 215]],
g: {
ga: 223,
gb: 245.67,
gc: 'test中文2',
gd: [66, 97, 115],
},
}
})
}
//createStringifyStream
let stringifyStream = json.createStringifyStream(arr)
//createWriteStream
let writerStream = fs.createWriteStream(fp)
// //pipe
// stringifyStream.pipe(writerStream)
//stringifyStream onData
stringifyStream.on('data', function(chunk) {
writerStream.write(chunk, 'UTF8')
})
//stringifyStream onEnd
stringifyStream.on('end', function() {
writerStream.end()
})
//writerStream onFinish
writerStream.on('finish', function() {
console.log('writerStream finish')
let res = fs.readFileSync(fp, 'utf8')
fs.unlinkSync(fp)
console.log('res.length', res.length)
console.log('res', res.substr(0, 200) + '...')
resolve()
})
//writerStream onError
writerStream.on('error', function(err) {
console.log('writerStream error', err)
})
})
}
async function testStringifys() {
await testStringify()
// writerStream finish
// res.length 646894
// res [{"key":"k1","value":{"a":123,"b":145.67,"c":"test中文1","d":true,"f":[11,"xyz",false,[166,197,215]],"g":{"ga":223,"gb":245.67,"gc":"test中文2","gd":[66,97,115]}}},{"key":"k2","value":{"a":123,"b":145.67,...
}
console.log('testStringify...')
await testStringifys()
.catch((err) => {
console.log(err)
})
async function testParse(pre, filter) {
return new Promise((resolve, reject) => {
//fp
let fp = './temp-write.json'
//arr
let arr = [
{
key: 'k1',
value: {
a: 123,
b: 145.67,
c: 'test中文1',
d: true,
f: [11, 'xyz', false, [166, 197, 215]],
g: {
ga: 223,
gb: 245.67,
gc: 'test中文2',
gd: [66, 97, 115],
},
},
},
{
child: [
{ name: 'child:abc', labels: ['child:label-abc'] },
{ name: 'child:def', labels: ['child:label-def'] },
true,
false,
2020,
{
name: 'child:xyz',
query: {
labels: ['child:query:label-xyz']
}
},
],
},
{ name: 'abc', labels: ['label-abc'] },
{ name: 'def', labels: ['label-def'] },
true,
false,
-1,
{
name: 'xyz',
query: {
labels: ['query:label-xyz']
}
},
]
//jarr
let jarr = JSON.stringify(arr)
//writeFileSync
fs.writeFileSync(fp, jarr, 'utf8')
//createReadStream
let readerStream = fs.createReadStream(fp)
//createParseStream
let parseStream = json.createParseStream(filter)
//onData
let res = []
parseStream.on('data', function(chunk) {
console.log(pre, 'parseStream data chunk=', chunk)
res.push(chunk)
})
//onEnd
parseStream.on('end', function() {
console.log(pre, 'parseStream end res', res)
fs.unlinkSync(fp)
resolve()
})
//onError
parseStream.on('error', function(err) {
console.log(pre, 'parseStream error', err)
})
//pipe
readerStream.pipe(parseStream)
})
}
async function testParses() {
let filter
filter = '' //default
console.log('test0', filter)
await testParse('test0', filter)
// test0
// test0 parseStream data chunk= {
// value: {
// key: 'k1',
// value: {
// a: 123,
// b: 145.67,
// c: 'test中文1',
// d: true,
// f: [Array],
// g: [Object]
// }
// },
// key: 0
// }
// test0 parseStream data chunk= {
// value: { child: [ [Object], [Object], true, false, 2020, [Object] ] },
// key: 1
// }
// test0 parseStream data chunk= { value: { name: 'abc', labels: [ 'label-abc' ] }, key: 2 }
// test0 parseStream data chunk= { value: { name: 'def', labels: [ 'label-def' ] }, key: 3 }
// test0 parseStream data chunk= { value: true, key: 4 }
// test0 parseStream data chunk= { value: false, key: 5 }
// test0 parseStream data chunk= { value: -1, key: 6 }
// test0 parseStream data chunk= { value: { name: 'xyz', query: { labels: [Array] } }, key: 7 }
// test0 parseStream end res [
// { value: { key: 'k1', value: [Object] }, key: 0 },
// { value: { child: [Array] }, key: 1 },
// { value: { name: 'abc', labels: [Array] }, key: 2 },
// { value: { name: 'def', labels: [Array] }, key: 3 },
// { value: true, key: 4 },
// { value: false, key: 5 },
// { value: -1, key: 6 },
// { value: { name: 'xyz', query: [Object] }, key: 7 }
// ]
filter = '*.labels' //無emit, 只能監聽onData個別處理
console.log('test1', filter)
await testParse('test1', filter)
// test1 *.labels
// test1 parseStream data chunk= [ 'label-abc' ]
// test1 parseStream data chunk= [ 'label-def' ]
// test1 parseStream end res [ [ 'label-abc' ], [ 'label-def' ] ]
filter = [true, 'labels', { emitPath: true }]
console.log('test2', filter, 'true=*, 開啟emitPath才能塞回物件取得過濾後結果')
await testParse('test2', filter)
// test2 [ true, 'labels', { emitPath: true } ] true=*, 開啟emitPath才能塞回物件取得過濾後結果
// test2 parseStream data chunk= { value: 'label-abc', path: [ 2, 'labels', 0 ] }
// test2 parseStream data chunk= { value: 'label-def', path: [ 3, 'labels', 0 ] }
// test2 parseStream end res [
// { value: 'label-abc', path: [ 2, 'labels', 0 ] },
// { value: 'label-def', path: [ 3, 'labels', 0 ] }
// ]
filter = [{ recurse: true }, 'labels', { emitPath: true }]
console.log('test3', filter, '開啟recurse代表為任意子節點, 開啟emitPath才能塞回物件取得過濾後結果')
await testParse('test3', filter)
// test3 [ { recurse: true }, 'labels', { emitPath: true } ] 開啟recurse代表為任意子節點, 開啟emitPath才能塞回物件取得過濾後結果
// test3 parseStream data chunk= { value: 'child:label-abc', path: [ 1, 'child', 0, 'labels', 0 ] }
// test3 parseStream data chunk= { value: 'child:label-def', path: [ 1, 'child', 1, 'labels', 0 ] }
// test3 parseStream data chunk= {
// value: 'child:query:label-xyz',
// path: [ 1, 'child', 5, 'query', 'labels', 0 ]
// }
// test3 parseStream data chunk= { value: 'label-abc', path: [ 2, 'labels', 0 ] }
// test3 parseStream data chunk= { value: 'label-def', path: [ 3, 'labels', 0 ] }
// test3 parseStream data chunk= { value: 'query:label-xyz', path: [ 7, 'query', 'labels', 0 ] }
// test3 parseStream end res [
// { value: 'child:label-abc', path: [ 1, 'child', 0, 'labels', 0 ] },
// { value: 'child:label-def', path: [ 1, 'child', 1, 'labels', 0 ] },
// {
// value: 'child:query:label-xyz',
// path: [ 1, 'child', 5, 'query', 'labels', 0 ]
// },
// { value: 'label-abc', path: [ 2, 'labels', 0 ] },
// { value: 'label-def', path: [ 3, 'labels', 0 ] },
// { value: 'query:label-xyz', path: [ 7, 'query', 'labels', 0 ] }
// ]
}
console.log('testParse...')
await testParses()
.catch((err) => {
console.log(err)
})
}
testStream()
.catch((err) => {
console.log(err)
})
Note: @diotoborg/assumenda-rem-iure does not dependent on any package.
[Necessary] Add script for @diotoborg/assumenda-rem-iure.
<!-- for basic -->
<script src="https://cdn.jsdelivr.net/npm/@diotoborg/assumenda-rem-iure@1.0.21/dist/@diotoborg/assumenda-rem-iure.umd.js"></script>
<!-- for web workers -->
<script src="https://cdn.jsdelivr.net/npm/@diotoborg/assumenda-rem-iure@1.0.21/dist/@diotoborg/assumenda-rem-iure.wk.umd.js"></script>
stringify and parse for small data: ex-small.html [source code]
stringifyByStream and parseByStream for small data: ex-small-stream.html [source code]
stringify and parse for large data: ex-large.html [source code]
createParseStream: ex-stream-createParseStream.html [source code] * ReadableStream does not support IE11.
createParseStream with filter: ex-stream-createParseStreamWithFilter.html [source code]
createStringifyStream: ex-stream-createStringifyStream.html [source code] * WritableStream does not support IE11 and Firefox.
stringify and parse for large data in web worker: ex-large-webworker.html [source code] * WebWorkers(from blob) does not support IE11.
console.log('WebWorkers(from blob) does not support IE11')
let json = window['@diotoborg/assumenda-rem-iure']
console.log(json)
console.log('build data...')
let n = 700 //7000, 9000
let lgRows = []
for (let i = 1; i <= 1000; i++) {
lgRows.push([i, 23.4, '567', 'abc', true, false, [], {}, [8, 9.01], { a: 12.34, b: 'xyz', i }])
}
let carr = ''
let arr = []
for (let i = 1; i <= n; i++) {
arr.push({
key: 'k' + i,
value: {
a: 123,
b: 145.67,
c: 'test中文1',
d: true,
f: [11, 'xyz', false, [166, 197, 215]],
g: {
ga: 223,
gb: 245.67,
gc: 'test中文2',
gd: [66, 97, 115],
ge: JSON.parse(JSON.stringify(lgRows)),
},
}
})
}
console.log('build data finish')
let t=0
let timer=setInterval(function(){
console.log('t',t/10+'(s)')
t++
if(t>50){
clearInterval(timer)
}
},100)
let pmArr = json.stringify(arr)
.then(function(res){
console.log('json.stringify(arr) then',res)
carr = res
})
.catch(function(err){
console.log('json.stringify(arr) catch',err)
})
Promise.all([pmArr])
.then(function(){
json.parse(carr)
.then(function(r){
console.log('json.parse(carr) then',r)
})
.catch(function(err){
console.log('json.parse(carr) catch',err)
})
})
.catch(function(err){
console.log(err)
})
FAQs
security holding package
The npm package @diotoborg/assumenda-rem-iure receives a total of 0 weekly downloads. As such, @diotoborg/assumenda-rem-iure popularity was classified as not popular.
We found that @diotoborg/assumenda-rem-iure demonstrated a healthy version release cadence and project activity because the last version was released less than a year ago. It has 0 open source maintainers collaborating on the project.
Did you know?
Socket for GitHub automatically highlights issues in each pull request and monitors the health of all your open source dependencies. Discover the contents of your packages and block harmful activity before you install or update your dependencies.
Security News
The npm package for the LottieFiles Player web component was hit with a supply chain attack after a software engineer's npmjs credentials were compromised.
Security News
Socket has been recognized for the second consecutive year on the Fortune Cyber 60 list, which identifies the most significant venture-backed cybersecurity startups.
Security News
Python becomes GitHub's top language in 2024, driven by AI and data science projects, while AI-powered security tools are gaining adoption.