@naturalcycles/nodejs-lib
Advanced tools
Comparing version 13.23.0 to 13.23.1
@@ -7,3 +7,3 @@ /// <reference types="node" /> | ||
import { DumpOptions } from 'js-yaml'; | ||
import { ReadableTyped, WritableTyped } from '../stream/stream.model'; | ||
import { ReadableTyped, TransformTyped } from '../stream/stream.model'; | ||
/** | ||
@@ -81,3 +81,3 @@ * fs2 conveniently groups filesystem functions together. | ||
createReadStreamAsNDJSON<ROW = any>(inputPath: string): ReadableTyped<ROW>; | ||
createWriteStreamAsNDJSON(outputPath: string): WritableTyped<any>; | ||
createWriteStreamAsNDJSON(outputPath: string): TransformTyped<any, any>[]; | ||
} | ||
@@ -84,0 +84,0 @@ export declare const fs2: FS2; |
@@ -314,3 +314,4 @@ "use strict"; | ||
/* | ||
Returns a Writable. | ||
Returns an array of Transforms, so that you can ...destructure them at | ||
the end of the _pipeline. | ||
@@ -324,13 +325,13 @@ Replaces a list of operations: | ||
this.ensureFile(outputPath); | ||
const transform1 = (0, transformToNDJson_1.transformToNDJson)(); | ||
let transform = transform1; | ||
if (outputPath.endsWith('.gz')) { | ||
transform = transform.pipe((0, node_zlib_1.createGzip)({ | ||
// chunkSize: 64 * 1024, // no observed speedup | ||
})); | ||
} | ||
transform.pipe(node_fs_1.default.createWriteStream(outputPath, { | ||
// highWaterMark: 64 * 1024, // no observed speedup | ||
})); | ||
return transform1; | ||
return [ | ||
(0, transformToNDJson_1.transformToNDJson)(), | ||
outputPath.endsWith('.gz') | ||
? (0, node_zlib_1.createGzip)({ | ||
// chunkSize: 64 * 1024, // no observed speedup | ||
}) | ||
: undefined, | ||
node_fs_1.default.createWriteStream(outputPath, { | ||
// highWaterMark: 64 * 1024, // no observed speedup | ||
}), | ||
].filter(js_lib_1._isTruthy); | ||
} | ||
@@ -337,0 +338,0 @@ } |
@@ -29,5 +29,5 @@ "use strict"; | ||
(0, __1.transformLogProgress)({ metric: 'saved', logEvery: logEveryOutput }), | ||
__1.fs2.createWriteStreamAsNDJSON(outputFilePath), | ||
...__1.fs2.createWriteStreamAsNDJSON(outputFilePath), | ||
]); | ||
} | ||
exports.ndjsonMap = ndjsonMap; |
{ | ||
"name": "@naturalcycles/nodejs-lib", | ||
"version": "13.23.0", | ||
"version": "13.23.1", | ||
"scripts": { | ||
@@ -5,0 +5,0 @@ "prepare": "husky", |
@@ -22,6 +22,6 @@ /* | ||
import { createGzip, createUnzip } from 'node:zlib' | ||
import { _jsonParse } from '@naturalcycles/js-lib' | ||
import { _isTruthy, _jsonParse } from '@naturalcycles/js-lib' | ||
import yaml, { DumpOptions } from 'js-yaml' | ||
import { transformToNDJson } from '../stream/ndjson/transformToNDJson' | ||
import { ReadableTyped, WritableTyped } from '../stream/stream.model' | ||
import { ReadableTyped, TransformTyped } from '../stream/stream.model' | ||
import { transformSplitOnNewline } from '../stream/transform/transformSplit' | ||
@@ -357,3 +357,4 @@ import { requireFileToExist } from '../util/env.util' | ||
/* | ||
Returns a Writable. | ||
Returns an array of Transforms, so that you can ...destructure them at | ||
the end of the _pipeline. | ||
@@ -365,20 +366,16 @@ Replaces a list of operations: | ||
*/ | ||
createWriteStreamAsNDJSON(outputPath: string): WritableTyped<any> { | ||
createWriteStreamAsNDJSON(outputPath: string): TransformTyped<any, any>[] { | ||
this.ensureFile(outputPath) | ||
const transform1 = transformToNDJson() | ||
let transform = transform1 | ||
if (outputPath.endsWith('.gz')) { | ||
transform = transform.pipe( | ||
createGzip({ | ||
// chunkSize: 64 * 1024, // no observed speedup | ||
}), | ||
) | ||
} | ||
transform.pipe( | ||
return [ | ||
transformToNDJson(), | ||
outputPath.endsWith('.gz') | ||
? createGzip({ | ||
// chunkSize: 64 * 1024, // no observed speedup | ||
}) | ||
: undefined, | ||
fs.createWriteStream(outputPath, { | ||
// highWaterMark: 64 * 1024, // no observed speedup | ||
}), | ||
) | ||
return transform1 | ||
].filter(_isTruthy) as TransformTyped<any, any>[] | ||
} | ||
@@ -385,0 +382,0 @@ } |
@@ -63,4 +63,4 @@ import { AbortableAsyncMapper, ErrorMode } from '@naturalcycles/js-lib' | ||
transformLogProgress({ metric: 'saved', logEvery: logEveryOutput }), | ||
fs2.createWriteStreamAsNDJSON(outputFilePath), | ||
...fs2.createWriteStreamAsNDJSON(outputFilePath), | ||
]) | ||
} |
464864
13240