Huge News!Announcing our $40M Series B led by Abstract Ventures.Learn More →
@quintype/migration-helpers
Advanced tools
@quintype/migration-helpers - npm Package Compare versions
Comparing version 1.0.1 to 1.1.0
@@ -1,3 +0,3 @@ | ||
| /** @private */ | ||
| export declare function partitionAll<T>(stream: AsyncIterableIterator<T>, size?: number): AsyncIterableIterator<T[]>; | ||
| /// <reference types="node" /> | ||
| import { Readable } from 'stream'; | ||
| export interface GenerateToFileOptions { | ||
@@ -9,2 +9,2 @@ readonly batchSize?: number; | ||
| /** @private */ | ||
| export declare function writeToFiles<T>(stream: AsyncIterableIterator<T>, { batchSize, directory, filePrefix }: GenerateToFileOptions): Promise<void>; | ||
| export declare function writeToFiles<T>(source: AsyncIterableIterator<T> | Readable, { batchSize, directory, filePrefix }: GenerateToFileOptions): Promise<void>; |
| "use strict"; | ||
| var __asyncValues = (this && this.__asyncValues) || function (o) { | ||
| if (!Symbol.asyncIterator) throw new TypeError("Symbol.asyncIterator is not defined."); | ||
| var m = o[Symbol.asyncIterator], i; | ||
| return m ? m.call(o) : (o = typeof __values === "function" ? __values(o) : o[Symbol.iterator](), i = {}, verb("next"), verb("throw"), verb("return"), i[Symbol.asyncIterator] = function () { return this; }, i); | ||
| function verb(n) { i[n] = o[n] && function (v) { return new Promise(function (resolve, reject) { v = o[n](v), settle(resolve, reject, v.done, v.value); }); }; } | ||
| function settle(resolve, reject, d, v) { Promise.resolve(v).then(function(v) { resolve({ value: v, done: d }); }, reject); } | ||
| }; | ||
| var __await = (this && this.__await) || function (v) { return this instanceof __await ? (this.v = v, this) : new __await(v); } | ||
| var __asyncGenerator = (this && this.__asyncGenerator) || function (thisArg, _arguments, generator) { | ||
| if (!Symbol.asyncIterator) throw new TypeError("Symbol.asyncIterator is not defined."); | ||
| var g = generator.apply(thisArg, _arguments || []), i, q = []; | ||
| return i = {}, verb("next"), verb("throw"), verb("return"), i[Symbol.asyncIterator] = function () { return this; }, i; | ||
| function verb(n) { if (g[n]) i[n] = function (v) { return new Promise(function (a, b) { q.push([n, v, a, b]) > 1 || resume(n, v); }); }; } | ||
| function resume(n, v) { try { step(g[n](v)); } catch (e) { settle(q[0][3], e); } } | ||
| function step(r) { r.value instanceof __await ? Promise.resolve(r.value.v).then(fulfill, reject) : settle(q[0][2], r); } | ||
| function fulfill(value) { resume("next", value); } | ||
| function reject(value) { resume("throw", value); } | ||
| function settle(f, v) { if (f(v), q.shift(), q.length) resume(q[0][0], q[0][1]); } | ||
| }; | ||
| Object.defineProperty(exports, "__esModule", { value: true }); | ||
@@ -27,53 +8,70 @@ // tslint:disable:no-expression-statement readonly-array no-if-statement | ||
| /** @private */ | ||
| function partitionAll(stream, size = 1000) { | ||
| return __asyncGenerator(this, arguments, function* partitionAll_1() { | ||
| var e_1, _a; | ||
| let batch = []; | ||
| try { | ||
| for (var stream_2 = __asyncValues(stream), stream_2_1; stream_2_1 = yield __await(stream_2.next()), !stream_2_1.done;) { | ||
| const item = stream_2_1.value; | ||
| batch.push(item); | ||
| if (batch.length === size) { | ||
| yield yield __await(batch); | ||
| batch = []; | ||
| } | ||
| function asyncToStream(generator) { | ||
| return new stream_1.Readable({ | ||
| objectMode: true, | ||
| async read() { | ||
| const { done, value } = await generator.next(); | ||
| if (done) { | ||
| this.push(null); | ||
| } | ||
| } | ||
| catch (e_1_1) { e_1 = { error: e_1_1 }; } | ||
| finally { | ||
| try { | ||
| if (stream_2_1 && !stream_2_1.done && (_a = stream_2.return)) yield __await(_a.call(stream_2)); | ||
| else { | ||
| this.push(value); | ||
| } | ||
| finally { if (e_1) throw e_1.error; } | ||
| } | ||
| if (batch.length) { | ||
| yield yield __await(batch); | ||
| } | ||
| }); | ||
| } | ||
| exports.partitionAll = partitionAll; | ||
| /** @private */ | ||
| async function writeToFiles(stream, { batchSize, directory = '.', filePrefix = 'c' }) { | ||
| var e_2, _a; | ||
| let fileNum = 1; | ||
| try { | ||
| for (var _b = __asyncValues(partitionAll(stream, batchSize)), _c; _c = await _b.next(), !_c.done;) { | ||
| const batch = _c.value; | ||
| await writeBatchToFile(batch, `${directory}/${filePrefix}-${String(fileNum).padStart(5, '0')}.txt.gz`); | ||
| fileNum++; | ||
| } | ||
| function batchStream(size = 1000) { | ||
| let batch = []; | ||
| let batchNumber = 1; | ||
| function emitBatch(transform) { | ||
| transform.push({ batchNumber, batch }); | ||
| batchNumber++; | ||
| batch = []; | ||
| } | ||
| catch (e_2_1) { e_2 = { error: e_2_1 }; } | ||
| finally { | ||
| try { | ||
| if (_c && !_c.done && (_a = _b.return)) await _a.call(_b); | ||
| return new stream_1.Transform({ | ||
| objectMode: true, | ||
| transform(data, _, callback) { | ||
| batch = batch.concat(data); | ||
| if (batch.length >= size) { | ||
| emitBatch(this); | ||
| } | ||
| callback(); | ||
| }, | ||
| flush() { | ||
| if (batch.length > 0) { | ||
| emitBatch(this); | ||
| } | ||
| this.push(null); | ||
| } | ||
| finally { if (e_2) throw e_2.error; } | ||
| } | ||
| }); | ||
| } | ||
| /** @private */ | ||
| function writeToFiles(source, { batchSize, directory = '.', filePrefix = 'c' }) { | ||
| return new Promise((resolve, reject) => { | ||
| const stream = source instanceof stream_1.Readable ? source : asyncToStream(source); | ||
| stream | ||
| .pipe(batchStream(batchSize)) | ||
| .pipe(writeBatchToFile(directory, filePrefix)) | ||
| .on('finish', resolve) | ||
| .on('error', reject); | ||
| }); | ||
| } | ||
| exports.writeToFiles = writeToFiles; | ||
| /** @private */ | ||
| function writeBatchToFile(batch, file) { | ||
| function writeBatchToFile(directory, filePrefix) { | ||
| return new stream_1.Writable({ | ||
| objectMode: true, | ||
| write({ batchNumber, batch }, _, callback) { | ||
| createJSONStream(batch) | ||
| .pipe(zlib_1.createGzip()) | ||
| .pipe(fs_1.createWriteStream(`${directory}/${filePrefix}-${String(batchNumber).padStart(5, '0')}.txt.gz`)) | ||
| .on('finish', callback); | ||
| } | ||
| }); | ||
| } | ||
| /** @private */ | ||
| function createJSONStream(batch) { | ||
| let numberRead = 0; | ||
| const stream = new stream_1.Readable({ | ||
| return new stream_1.Readable({ | ||
| read() { | ||
@@ -93,9 +91,3 @@ const slice = batch.slice(numberRead, numberRead + 16); | ||
| }); | ||
| return new Promise(resolve => { | ||
| stream | ||
| .pipe(zlib_1.createGzip()) | ||
| .pipe(fs_1.createWriteStream(file)) | ||
| .on('finish', resolve); | ||
| }); | ||
| } | ||
| //# sourceMappingURL=data:application/json;base64,eyJ2ZXJzaW9uIjozLCJmaWxlIjoiYXN5bmMtd3JpdGVyLmpzIiwic291cmNlUm9vdCI6IiIsInNvdXJjZXMiOlsiLi4vLi4vLi4vc3JjL2xpYi9hc3luYy13cml0ZXIudHMiXSwibmFtZXMiOltdLCJtYXBwaW5ncyI6Ijs7Ozs7Ozs7Ozs7Ozs7Ozs7Ozs7O0FBQUEsd0VBQXdFO0FBQ3hFLDJCQUF1QztBQUN2QyxtQ0FBa0M7QUFDbEMsK0JBQWtDO0FBRWxDLGVBQWU7QUFDZixTQUF1QixZQUFZLENBQ2pDLE1BQWdDLEVBQ2hDLE9BQWUsSUFBSTs7O1FBRW5CLElBQUksS0FBSyxHQUFRLEVBQUUsQ0FBQzs7WUFDcEIsS0FBeUIsSUFBQSxXQUFBLGNBQUEsTUFBTSxDQUFBLFlBQUE7Z0JBQXBCLE1BQU0sSUFBSSxtQkFBQSxDQUFBO2dCQUNuQixLQUFLLENBQUMsSUFBSSxDQUFDLElBQUksQ0FBQyxDQUFDO2dCQUNqQixJQUFJLEtBQUssQ0FBQyxNQUFNLEtBQUssSUFBSSxFQUFFO29CQUN6QixvQkFBTSxLQUFLLENBQUEsQ0FBQztvQkFDWixLQUFLLEdBQUcsRUFBRSxDQUFDO2lCQUNaO2FBQ0Y7Ozs7Ozs7OztRQUNELElBQUksS0FBSyxDQUFDLE1BQU0sRUFBRTtZQUNoQixvQkFBTSxLQUFLLENBQUEsQ0FBQztTQUNiO0lBQ0gsQ0FBQztDQUFBO0FBZkQsb0NBZUM7QUFRRCxlQUFlO0FBQ1IsS0FBSyxVQUFVLFlBQVksQ0FDaEMsTUFBZ0MsRUFDaEMsRUFBRSxTQUFTLEVBQUUsU0FBUyxHQUFHLEdBQUcsRUFBRSxVQUFVLEdBQUcsR0FBRyxFQUF5Qjs7SUFFdkUsSUFBSSxPQUFPLEdBQUcsQ0FBQyxDQUFDOztRQUNoQixLQUEwQixJQUFBLEtBQUEsY0FBQSxZQUFZLENBQUMsTUFBTSxFQUFFLFNBQVMsQ0FBQyxDQUFBLElBQUE7WUFBOUMsTUFBTSxLQUFLLFdBQUEsQ0FBQTtZQUNwQixNQUFNLGdCQUFnQixDQUFDLEtBQUssRUFBRSxHQUFHLFNBQVMsSUFBSSxVQUFVLElBQUksTUFBTSxDQUFDLE9BQU8sQ0FBQyxDQUFDLFFBQVEsQ0FBQyxDQUFDLEVBQUUsR0FBRyxDQUFDLFNBQVMsQ0FBQyxDQUFDO1lBQ3ZHLE9BQU8sRUFBRSxDQUFDO1NBQ1g7Ozs7Ozs7OztBQUNILENBQUM7QUFURCxvQ0FTQztBQUVELGVBQWU7QUFDZixTQUFTLGdCQUFnQixDQUFJLEtBQVUsRUFBRSxJQUFZO0lBQ25ELElBQUksVUFBVSxHQUFHLENBQUMsQ0FBQztJQUNuQixNQUFNLE1BQU0sR0FBRyxJQUFJLGlCQUFRLENBQUM7UUFDMUIsSUFBSTtZQUNGLE1BQU0sS0FBSyxHQUFHLEtBQUssQ0FBQyxLQUFLLENBQUMsVUFBVSxFQUFFLFVBQVUsR0FBRyxFQUFFLENBQUMsQ0FBQztZQUN2RCxVQUFVLElBQUksRUFBRSxDQUFDO1lBQ2pCLElBQUksS0FBSyxDQUFDLE1BQU0sS0FBSyxDQUFDLEVBQUU7Z0JBQ3RCLElBQUksQ0FBQyxJQUFJLENBQUMsSUFBSSxDQUFDLENBQUM7YUFDakI7aUJBQU07Z0JBQ0wsS0FBSyxNQUFNLElBQUksSUFBSSxLQUFLLEVBQUU7b0JBQ3hCLElBQUksQ0FBQyxJQUFJLENBQUMsSUFBSSxDQUFDLFNBQVMsQ0FBQyxJQUFJLENBQUMsQ0FBQyxDQUFDO29CQUNoQyxJQUFJLENBQUMsSUFBSSxDQUFDLElBQUksQ0FBQyxDQUFDO2lCQUNqQjthQUNGO1FBQ0gsQ0FBQztLQUNGLENBQUMsQ0FBQztJQUVILE9BQU8sSUFBSSxPQUFPLENBQUMsT0FBTyxDQUFDLEVBQUU7UUFDM0IsTUFBTTthQUNILElBQUksQ0FBQyxpQkFBVSxFQUFFLENBQUM7YUFDbEIsSUFBSSxDQUFDLHNCQUFpQixDQUFDLElBQUksQ0FBQyxDQUFDO2FBQzdCLEVBQUUsQ0FBQyxRQUFRLEVBQUUsT0FBTyxDQUFDLENBQUM7SUFDM0IsQ0FBQyxDQUFDLENBQUM7QUFDTCxDQUFDIn0= | ||
| //# sourceMappingURL=data:application/json;base64,eyJ2ZXJzaW9uIjozLCJmaWxlIjoiYXN5bmMtd3JpdGVyLmpzIiwic291cmNlUm9vdCI6IiIsInNvdXJjZXMiOlsiLi4vLi4vLi4vc3JjL2xpYi9hc3luYy13cml0ZXIudHMiXSwibmFtZXMiOltdLCJtYXBwaW5ncyI6Ijs7QUFBQSx3RUFBd0U7QUFDeEUsMkJBQXVDO0FBQ3ZDLG1DQUF1RDtBQUN2RCwrQkFBa0M7QUFFbEMsZUFBZTtBQUNmLFNBQVMsYUFBYSxDQUFJLFNBQW1DO0lBQzNELE9BQU8sSUFBSSxpQkFBUSxDQUFDO1FBQ2xCLFVBQVUsRUFBRSxJQUFJO1FBRWhCLEtBQUssQ0FBQyxJQUFJO1lBQ1IsTUFBTSxFQUFFLElBQUksRUFBRSxLQUFLLEVBQUUsR0FBRyxNQUFNLFNBQVMsQ0FBQyxJQUFJLEVBQUUsQ0FBQztZQUMvQyxJQUFJLElBQUksRUFBRTtnQkFDUixJQUFJLENBQUMsSUFBSSxDQUFDLElBQUksQ0FBQyxDQUFDO2FBQ2pCO2lCQUFNO2dCQUNMLElBQUksQ0FBQyxJQUFJLENBQUMsS0FBSyxDQUFDLENBQUM7YUFDbEI7UUFDSCxDQUFDO0tBQ0YsQ0FBQyxDQUFDO0FBQ0wsQ0FBQztBQUVELGVBQWU7QUFDZixTQUFTLFdBQVcsQ0FBSSxPQUFlLElBQUk7SUFDekMsSUFBSSxLQUFLLEdBQVEsRUFBRSxDQUFDO0lBQ3BCLElBQUksV0FBVyxHQUFHLENBQUMsQ0FBQztJQUVwQixTQUFTLFNBQVMsQ0FBQyxTQUFvQjtRQUNyQyxTQUFTLENBQUMsSUFBSSxDQUFDLEVBQUUsV0FBVyxFQUFFLEtBQUssRUFBRSxDQUFDLENBQUM7UUFDdkMsV0FBVyxFQUFFLENBQUM7UUFDZCxLQUFLLEdBQUcsRUFBRSxDQUFDO0lBQ2IsQ0FBQztJQUVELE9BQU8sSUFBSSxrQkFBUyxDQUFDO1FBQ25CLFVBQVUsRUFBRSxJQUFJO1FBRWhCLFNBQVMsQ0FBQyxJQUFJLEVBQUUsQ0FBQyxFQUFFLFFBQVE7WUFDekIsS0FBSyxHQUFHLEtBQUssQ0FBQyxNQUFNLENBQUMsSUFBSSxDQUFDLENBQUM7WUFDM0IsSUFBSSxLQUFLLENBQUMsTUFBTSxJQUFJLElBQUksRUFBRTtnQkFDeEIsU0FBUyxDQUFDLElBQUksQ0FBQyxDQUFDO2FBQ2pCO1lBQ0QsUUFBUSxFQUFFLENBQUM7UUFDYixDQUFDO1FBRUQsS0FBSztZQUNILElBQUksS0FBSyxDQUFDLE1BQU0sR0FBRyxDQUFDLEVBQUU7Z0JBQ3BCLFNBQVMsQ0FBQyxJQUFJLENBQUMsQ0FBQzthQUNqQjtZQUNELElBQUksQ0FBQyxJQUFJLENBQUMsSUFBSSxDQUFDLENBQUM7UUFDbEIsQ0FBQztLQUNGLENBQUMsQ0FBQztBQUNMLENBQUM7QUFRRCxlQUFlO0FBQ2YsU0FBZ0IsWUFBWSxDQUMxQixNQUEyQyxFQUMzQyxFQUFFLFNBQVMsRUFBRSxTQUFTLEdBQUcsR0FBRyxFQUFFLFVBQVUsR0FBRyxHQUFHLEVBQXlCO0lBRXZFLE9BQU8sSUFBSSxPQUFPLENBQUMsQ0FBQyxPQUFPLEVBQUUsTUFBTSxFQUFFLEVBQUU7UUFDckMsTUFBTSxNQUFNLEdBQUcsTUFBTSxZQUFZLGlCQUFRLENBQUMsQ0FBQyxDQUFDLE1BQU0sQ0FBQyxDQUFDLENBQUMsYUFBYSxDQUFDLE1BQU0sQ0FBQyxDQUFDO1FBQzNFLE1BQU07YUFDSCxJQUFJLENBQUMsV0FBVyxDQUFDLFNBQVMsQ0FBQyxDQUFDO2FBQzVCLElBQUksQ0FBQyxnQkFBZ0IsQ0FBQyxTQUFTLEVBQUUsVUFBVSxDQUFDLENBQUM7YUFDN0MsRUFBRSxDQUFDLFFBQVEsRUFBRSxPQUFPLENBQUM7YUFDckIsRUFBRSxDQUFDLE9BQU8sRUFBRSxNQUFNLENBQUMsQ0FBQztJQUN6QixDQUFDLENBQUMsQ0FBQztBQUNMLENBQUM7QUFaRCxvQ0FZQztBQUVELGVBQWU7QUFDZixTQUFTLGdCQUFnQixDQUFDLFNBQWlCLEVBQUUsVUFBa0I7SUFDN0QsT0FBTyxJQUFJLGlCQUFRLENBQUM7UUFDbEIsVUFBVSxFQUFFLElBQUk7UUFFaEIsS0FBSyxDQUFDLEVBQUUsV0FBVyxFQUFFLEtBQUssRUFBRSxFQUFFLENBQUMsRUFBRSxRQUFRO1lBQ3ZDLGdCQUFnQixDQUFDLEtBQUssQ0FBQztpQkFDcEIsSUFBSSxDQUFDLGlCQUFVLEVBQUUsQ0FBQztpQkFDbEIsSUFBSSxDQUFDLHNCQUFpQixDQUFDLEdBQUcsU0FBUyxJQUFJLFVBQVUsSUFBSSxNQUFNLENBQUMsV0FBVyxDQUFDLENBQUMsUUFBUSxDQUFDLENBQUMsRUFBRSxHQUFHLENBQUMsU0FBUyxDQUFDLENBQUM7aUJBQ3BHLEVBQUUsQ0FBQyxRQUFRLEVBQUUsUUFBUSxDQUFDLENBQUM7UUFDNUIsQ0FBQztLQUNGLENBQUMsQ0FBQztBQUNMLENBQUM7QUFFRCxlQUFlO0FBQ2YsU0FBUyxnQkFBZ0IsQ0FBSSxLQUFVO0lBQ3JDLElBQUksVUFBVSxHQUFHLENBQUMsQ0FBQztJQUNuQixPQUFPLElBQUksaUJBQVEsQ0FBQztRQUNsQixJQUFJO1lBQ0YsTUFBTSxLQUFLLEdBQUcsS0FBSyxDQUFDLEtBQUssQ0FBQyxVQUFVLEVBQUUsVUFBVSxHQUFHLEVBQUUsQ0FBQyxDQUFDO1lBQ3ZELFVBQVUsSUFBSSxFQUFFLENBQUM7WUFDakIsSUFBSSxLQUFLLENBQUMsTUFBTSxLQUFLLENBQUMsRUFBRTtnQkFDdEIsSUFBSSxDQUFDLElBQUksQ0FBQyxJQUFJLENBQUMsQ0FBQzthQUNqQjtpQkFBTTtnQkFDTCxLQUFLLE1BQU0sSUFBSSxJQUFJLEtBQUssRUFBRTtvQkFDeEIsSUFBSSxDQUFDLElBQUksQ0FBQyxJQUFJLENBQUMsU0FBUyxDQUFDLElBQUksQ0FBQyxDQUFDLENBQUM7b0JBQ2hDLElBQUksQ0FBQyxJQUFJLENBQUMsSUFBSSxDQUFDLENBQUM7aUJBQ2pCO2FBQ0Y7UUFDSCxDQUFDO0tBQ0YsQ0FBQyxDQUFDO0FBQ0wsQ0FBQyJ9 |
@@ -14,9 +14,9 @@ interface StoryMandatoryFields { | ||
| readonly 'published-at': number; | ||
| /** The type of the story. Use 'text' for a normal story */ | ||
| readonly "story-template": string; | ||
| /** The type of the story. Use `'text'` for a normal story */ | ||
| readonly 'story-template': string; | ||
| } | ||
| interface StoryHeroImageFields { | ||
| /** A URL to pull the hero image from. This can be absolute, or relative starting with '/' */ | ||
| readonly "temporary-hero-image-url"?: string; | ||
| readonly "hero-image-caption"?: string; | ||
| readonly 'temporary-hero-image-url'?: string; | ||
| readonly 'hero-image-caption'?: string; | ||
| } | ||
@@ -45,3 +45,3 @@ interface StoryMetadataFields { | ||
| interface StoryBody { | ||
| /** An HTML blob of the story. Should look like "<p>Para1</p><p>Para2</p>" */ | ||
| /** An HTML blob of the story. Should look like `"<p>Para1</p><p>Para2</p>""` */ | ||
| readonly body: string; | ||
@@ -48,0 +48,0 @@ } |
@@ -0,1 +1,3 @@ | ||
| /// <reference types="node" /> | ||
| import { Readable } from 'stream'; | ||
| import { GenerateToFileOptions } from './async-writer'; | ||
@@ -22,6 +24,27 @@ import { Story } from './editor-types'; | ||
| * | ||
| * @param stream An Async Generator which yields stories | ||
| * You may also use a stream to produce stories | ||
| * | ||
| * ```ts | ||
| * import { Story, writeStories } from '@quintype/migration-helpers'; | ||
| * import { Transform } from 'stream' | ||
| * | ||
| * const stream = conn.query("select * from stories").stream(); | ||
| * | ||
| * const transformSqlRowToStories = new Transform({ | ||
| * objectMode: true, | ||
| * | ||
| * transform(rows, _, callback) { | ||
| * for(const story of rows) { | ||
| * this.push(rowToStory(story)) | ||
| * } | ||
| * } | ||
| * }) | ||
| * | ||
| * writeStories(stream.pipe(transformSqlRowToStories)); | ||
| * ``` | ||
| * | ||
| * @param stream An Async Generator or Readable which yields stories | ||
| * @param source A string describing where the stories come from. ex: interviews | ||
| * @param opts Control some fine grained tuning | ||
| */ | ||
| export declare function writeStories(stream: AsyncIterableIterator<Story>, source?: string, opts?: GenerateToFileOptions): Promise<void>; | ||
| export declare function writeStories(stream: AsyncIterableIterator<Story> | Readable, source?: string, opts?: GenerateToFileOptions): Promise<void>; |
@@ -23,3 +23,24 @@ "use strict"; | ||
| * | ||
| * @param stream An Async Generator which yields stories | ||
| * You may also use a stream to produce stories | ||
| * | ||
| * ```ts | ||
| * import { Story, writeStories } from '@quintype/migration-helpers'; | ||
| * import { Transform } from 'stream' | ||
| * | ||
| * const stream = conn.query("select * from stories").stream(); | ||
| * | ||
| * const transformSqlRowToStories = new Transform({ | ||
| * objectMode: true, | ||
| * | ||
| * transform(rows, _, callback) { | ||
| * for(const story of rows) { | ||
| * this.push(rowToStory(story)) | ||
| * } | ||
| * } | ||
| * }) | ||
| * | ||
| * writeStories(stream.pipe(transformSqlRowToStories)); | ||
| * ``` | ||
| * | ||
| * @param stream An Async Generator or Readable which yields stories | ||
| * @param source A string describing where the stories come from. ex: interviews | ||
@@ -33,2 +54,2 @@ * @param opts Control some fine grained tuning | ||
| exports.writeStories = writeStories; | ||
| //# sourceMappingURL=data:application/json;base64,eyJ2ZXJzaW9uIjozLCJmaWxlIjoic3Rvcmllcy13cml0ZXIuanMiLCJzb3VyY2VSb290IjoiIiwic291cmNlcyI6WyIuLi8uLi8uLi9zcmMvbGliL3N0b3JpZXMtd3JpdGVyLnRzIl0sIm5hbWVzIjpbXSwibWFwcGluZ3MiOiI7O0FBQUEsaURBQXFFO0FBR3JFOzs7Ozs7Ozs7Ozs7Ozs7Ozs7Ozs7O0dBc0JHO0FBQ0gsU0FBZ0IsWUFBWSxDQUMxQixNQUFvQyxFQUNwQyxTQUFpQixRQUFRLEVBQ3pCLE9BQThCLEVBQUU7SUFFaEMsTUFBTSxVQUFVLEdBQUcsSUFBSSxDQUFDLFVBQVUsQ0FBQyxDQUFDLENBQUMsU0FBUyxNQUFNLElBQUksSUFBSSxDQUFDLFVBQVUsRUFBRSxDQUFDLENBQUMsQ0FBQyxTQUFTLE1BQU0sRUFBRSxDQUFDO0lBQzlGLE9BQU8sMkJBQVksQ0FBQyxNQUFNLGtCQUFJLFVBQVUsSUFBSyxJQUFJLEVBQUcsQ0FBQztBQUN2RCxDQUFDO0FBUEQsb0NBT0MifQ== | ||
| //# sourceMappingURL=data:application/json;base64,eyJ2ZXJzaW9uIjozLCJmaWxlIjoic3Rvcmllcy13cml0ZXIuanMiLCJzb3VyY2VSb290IjoiIiwic291cmNlcyI6WyIuLi8uLi8uLi9zcmMvbGliL3N0b3JpZXMtd3JpdGVyLnRzIl0sIm5hbWVzIjpbXSwibWFwcGluZ3MiOiI7O0FBRUEsaURBQXFFO0FBR3JFOzs7Ozs7Ozs7Ozs7Ozs7Ozs7Ozs7Ozs7Ozs7Ozs7Ozs7Ozs7Ozs7Ozs7O0dBMkNHO0FBQ0gsU0FBZ0IsWUFBWSxDQUMxQixNQUErQyxFQUMvQyxTQUFpQixRQUFRLEVBQ3pCLE9BQThCLEVBQUU7SUFFaEMsTUFBTSxVQUFVLEdBQUcsSUFBSSxDQUFDLFVBQVUsQ0FBQyxDQUFDLENBQUMsU0FBUyxNQUFNLElBQUksSUFBSSxDQUFDLFVBQVUsRUFBRSxDQUFDLENBQUMsQ0FBQyxTQUFTLE1BQU0sRUFBRSxDQUFDO0lBQzlGLE9BQU8sMkJBQVksQ0FBQyxNQUFNLGtCQUFJLFVBQVUsSUFBSyxJQUFJLEVBQUcsQ0FBQztBQUN2RCxDQUFDO0FBUEQsb0NBT0MifQ== |
@@ -1,3 +0,3 @@ | ||
| /** @private */ | ||
| export declare function partitionAll<T>(stream: AsyncIterableIterator<T>, size?: number): AsyncIterableIterator<T[]>; | ||
| /// <reference types="node" /> | ||
| import { Readable } from 'stream'; | ||
| export interface GenerateToFileOptions { | ||
@@ -9,2 +9,2 @@ readonly batchSize?: number; | ||
| /** @private */ | ||
| export declare function writeToFiles<T>(stream: AsyncIterableIterator<T>, { batchSize, directory, filePrefix }: GenerateToFileOptions): Promise<void>; | ||
| export declare function writeToFiles<T>(source: AsyncIterableIterator<T> | Readable, { batchSize, directory, filePrefix }: GenerateToFileOptions): Promise<void>; |
| // tslint:disable:no-expression-statement readonly-array no-if-statement | ||
| import { createWriteStream } from 'fs'; | ||
| import { Readable } from 'stream'; | ||
| import { Readable, Transform, Writable } from 'stream'; | ||
| import { createGzip } from 'zlib'; | ||
| /** @private */ | ||
| export async function* partitionAll(stream, size = 1000) { | ||
| let batch = []; | ||
| for await (const item of stream) { | ||
| batch.push(item); | ||
| if (batch.length === size) { | ||
| yield batch; | ||
| batch = []; | ||
| function asyncToStream(generator) { | ||
| return new Readable({ | ||
| objectMode: true, | ||
| async read() { | ||
| const { done, value } = await generator.next(); | ||
| if (done) { | ||
| this.push(null); | ||
| } | ||
| else { | ||
| this.push(value); | ||
| } | ||
| } | ||
| } | ||
| if (batch.length) { | ||
| yield batch; | ||
| } | ||
| }); | ||
| } | ||
| /** @private */ | ||
| export async function writeToFiles(stream, { batchSize, directory = '.', filePrefix = 'c' }) { | ||
| let fileNum = 1; | ||
| for await (const batch of partitionAll(stream, batchSize)) { | ||
| await writeBatchToFile(batch, `${directory}/${filePrefix}-${String(fileNum).padStart(5, '0')}.txt.gz`); | ||
| fileNum++; | ||
| function batchStream(size = 1000) { | ||
| let batch = []; | ||
| let batchNumber = 1; | ||
| function emitBatch(transform) { | ||
| transform.push({ batchNumber, batch }); | ||
| batchNumber++; | ||
| batch = []; | ||
| } | ||
| return new Transform({ | ||
| objectMode: true, | ||
| transform(data, _, callback) { | ||
| batch = batch.concat(data); | ||
| if (batch.length >= size) { | ||
| emitBatch(this); | ||
| } | ||
| callback(); | ||
| }, | ||
| flush() { | ||
| if (batch.length > 0) { | ||
| emitBatch(this); | ||
| } | ||
| this.push(null); | ||
| } | ||
| }); | ||
| } | ||
| /** @private */ | ||
| function writeBatchToFile(batch, file) { | ||
| export function writeToFiles(source, { batchSize, directory = '.', filePrefix = 'c' }) { | ||
| return new Promise((resolve, reject) => { | ||
| const stream = source instanceof Readable ? source : asyncToStream(source); | ||
| stream | ||
| .pipe(batchStream(batchSize)) | ||
| .pipe(writeBatchToFile(directory, filePrefix)) | ||
| .on('finish', resolve) | ||
| .on('error', reject); | ||
| }); | ||
| } | ||
| /** @private */ | ||
| function writeBatchToFile(directory, filePrefix) { | ||
| return new Writable({ | ||
| objectMode: true, | ||
| write({ batchNumber, batch }, _, callback) { | ||
| createJSONStream(batch) | ||
| .pipe(createGzip()) | ||
| .pipe(createWriteStream(`${directory}/${filePrefix}-${String(batchNumber).padStart(5, '0')}.txt.gz`)) | ||
| .on('finish', callback); | ||
| } | ||
| }); | ||
| } | ||
| /** @private */ | ||
| function createJSONStream(batch) { | ||
| let numberRead = 0; | ||
| const stream = new Readable({ | ||
| return new Readable({ | ||
| read() { | ||
@@ -45,9 +87,3 @@ const slice = batch.slice(numberRead, numberRead + 16); | ||
| }); | ||
| return new Promise(resolve => { | ||
| stream | ||
| .pipe(createGzip()) | ||
| .pipe(createWriteStream(file)) | ||
| .on('finish', resolve); | ||
| }); | ||
| } | ||
| //# sourceMappingURL=data:application/json;base64,eyJ2ZXJzaW9uIjozLCJmaWxlIjoiYXN5bmMtd3JpdGVyLmpzIiwic291cmNlUm9vdCI6IiIsInNvdXJjZXMiOlsiLi4vLi4vLi4vc3JjL2xpYi9hc3luYy13cml0ZXIudHMiXSwibmFtZXMiOltdLCJtYXBwaW5ncyI6IkFBQUEsd0VBQXdFO0FBQ3hFLE9BQU8sRUFBRSxpQkFBaUIsRUFBRSxNQUFNLElBQUksQ0FBQztBQUN2QyxPQUFPLEVBQUUsUUFBUSxFQUFFLE1BQU0sUUFBUSxDQUFDO0FBQ2xDLE9BQU8sRUFBRSxVQUFVLEVBQUUsTUFBTSxNQUFNLENBQUM7QUFFbEMsZUFBZTtBQUNmLE1BQU0sQ0FBQyxLQUFLLFNBQVMsQ0FBQyxDQUFDLFlBQVksQ0FDakMsTUFBZ0MsRUFDaEMsT0FBZSxJQUFJO0lBRW5CLElBQUksS0FBSyxHQUFRLEVBQUUsQ0FBQztJQUNwQixJQUFJLEtBQUssRUFBRSxNQUFNLElBQUksSUFBSSxNQUFNLEVBQUU7UUFDL0IsS0FBSyxDQUFDLElBQUksQ0FBQyxJQUFJLENBQUMsQ0FBQztRQUNqQixJQUFJLEtBQUssQ0FBQyxNQUFNLEtBQUssSUFBSSxFQUFFO1lBQ3pCLE1BQU0sS0FBSyxDQUFDO1lBQ1osS0FBSyxHQUFHLEVBQUUsQ0FBQztTQUNaO0tBQ0Y7SUFDRCxJQUFJLEtBQUssQ0FBQyxNQUFNLEVBQUU7UUFDaEIsTUFBTSxLQUFLLENBQUM7S0FDYjtBQUNILENBQUM7QUFRRCxlQUFlO0FBQ2YsTUFBTSxDQUFDLEtBQUssVUFBVSxZQUFZLENBQ2hDLE1BQWdDLEVBQ2hDLEVBQUUsU0FBUyxFQUFFLFNBQVMsR0FBRyxHQUFHLEVBQUUsVUFBVSxHQUFHLEdBQUcsRUFBeUI7SUFFdkUsSUFBSSxPQUFPLEdBQUcsQ0FBQyxDQUFDO0lBQ2hCLElBQUksS0FBSyxFQUFFLE1BQU0sS0FBSyxJQUFJLFlBQVksQ0FBQyxNQUFNLEVBQUUsU0FBUyxDQUFDLEVBQUU7UUFDekQsTUFBTSxnQkFBZ0IsQ0FBQyxLQUFLLEVBQUUsR0FBRyxTQUFTLElBQUksVUFBVSxJQUFJLE1BQU0sQ0FBQyxPQUFPLENBQUMsQ0FBQyxRQUFRLENBQUMsQ0FBQyxFQUFFLEdBQUcsQ0FBQyxTQUFTLENBQUMsQ0FBQztRQUN2RyxPQUFPLEVBQUUsQ0FBQztLQUNYO0FBQ0gsQ0FBQztBQUVELGVBQWU7QUFDZixTQUFTLGdCQUFnQixDQUFJLEtBQVUsRUFBRSxJQUFZO0lBQ25ELElBQUksVUFBVSxHQUFHLENBQUMsQ0FBQztJQUNuQixNQUFNLE1BQU0sR0FBRyxJQUFJLFFBQVEsQ0FBQztRQUMxQixJQUFJO1lBQ0YsTUFBTSxLQUFLLEdBQUcsS0FBSyxDQUFDLEtBQUssQ0FBQyxVQUFVLEVBQUUsVUFBVSxHQUFHLEVBQUUsQ0FBQyxDQUFDO1lBQ3ZELFVBQVUsSUFBSSxFQUFFLENBQUM7WUFDakIsSUFBSSxLQUFLLENBQUMsTUFBTSxLQUFLLENBQUMsRUFBRTtnQkFDdEIsSUFBSSxDQUFDLElBQUksQ0FBQyxJQUFJLENBQUMsQ0FBQzthQUNqQjtpQkFBTTtnQkFDTCxLQUFLLE1BQU0sSUFBSSxJQUFJLEtBQUssRUFBRTtvQkFDeEIsSUFBSSxDQUFDLElBQUksQ0FBQyxJQUFJLENBQUMsU0FBUyxDQUFDLElBQUksQ0FBQyxDQUFDLENBQUM7b0JBQ2hDLElBQUksQ0FBQyxJQUFJLENBQUMsSUFBSSxDQUFDLENBQUM7aUJBQ2pCO2FBQ0Y7UUFDSCxDQUFDO0tBQ0YsQ0FBQyxDQUFDO0lBRUgsT0FBTyxJQUFJLE9BQU8sQ0FBQyxPQUFPLENBQUMsRUFBRTtRQUMzQixNQUFNO2FBQ0gsSUFBSSxDQUFDLFVBQVUsRUFBRSxDQUFDO2FBQ2xCLElBQUksQ0FBQyxpQkFBaUIsQ0FBQyxJQUFJLENBQUMsQ0FBQzthQUM3QixFQUFFLENBQUMsUUFBUSxFQUFFLE9BQU8sQ0FBQyxDQUFDO0lBQzNCLENBQUMsQ0FBQyxDQUFDO0FBQ0wsQ0FBQyJ9 | ||
| //# sourceMappingURL=data:application/json;base64,eyJ2ZXJzaW9uIjozLCJmaWxlIjoiYXN5bmMtd3JpdGVyLmpzIiwic291cmNlUm9vdCI6IiIsInNvdXJjZXMiOlsiLi4vLi4vLi4vc3JjL2xpYi9hc3luYy13cml0ZXIudHMiXSwibmFtZXMiOltdLCJtYXBwaW5ncyI6IkFBQUEsd0VBQXdFO0FBQ3hFLE9BQU8sRUFBRSxpQkFBaUIsRUFBRSxNQUFNLElBQUksQ0FBQztBQUN2QyxPQUFPLEVBQUUsUUFBUSxFQUFFLFNBQVMsRUFBRSxRQUFRLEVBQUUsTUFBTSxRQUFRLENBQUM7QUFDdkQsT0FBTyxFQUFFLFVBQVUsRUFBRSxNQUFNLE1BQU0sQ0FBQztBQUVsQyxlQUFlO0FBQ2YsU0FBUyxhQUFhLENBQUksU0FBbUM7SUFDM0QsT0FBTyxJQUFJLFFBQVEsQ0FBQztRQUNsQixVQUFVLEVBQUUsSUFBSTtRQUVoQixLQUFLLENBQUMsSUFBSTtZQUNSLE1BQU0sRUFBRSxJQUFJLEVBQUUsS0FBSyxFQUFFLEdBQUcsTUFBTSxTQUFTLENBQUMsSUFBSSxFQUFFLENBQUM7WUFDL0MsSUFBSSxJQUFJLEVBQUU7Z0JBQ1IsSUFBSSxDQUFDLElBQUksQ0FBQyxJQUFJLENBQUMsQ0FBQzthQUNqQjtpQkFBTTtnQkFDTCxJQUFJLENBQUMsSUFBSSxDQUFDLEtBQUssQ0FBQyxDQUFDO2FBQ2xCO1FBQ0gsQ0FBQztLQUNGLENBQUMsQ0FBQztBQUNMLENBQUM7QUFFRCxlQUFlO0FBQ2YsU0FBUyxXQUFXLENBQUksT0FBZSxJQUFJO0lBQ3pDLElBQUksS0FBSyxHQUFRLEVBQUUsQ0FBQztJQUNwQixJQUFJLFdBQVcsR0FBRyxDQUFDLENBQUM7SUFFcEIsU0FBUyxTQUFTLENBQUMsU0FBb0I7UUFDckMsU0FBUyxDQUFDLElBQUksQ0FBQyxFQUFFLFdBQVcsRUFBRSxLQUFLLEVBQUUsQ0FBQyxDQUFDO1FBQ3ZDLFdBQVcsRUFBRSxDQUFDO1FBQ2QsS0FBSyxHQUFHLEVBQUUsQ0FBQztJQUNiLENBQUM7SUFFRCxPQUFPLElBQUksU0FBUyxDQUFDO1FBQ25CLFVBQVUsRUFBRSxJQUFJO1FBRWhCLFNBQVMsQ0FBQyxJQUFJLEVBQUUsQ0FBQyxFQUFFLFFBQVE7WUFDekIsS0FBSyxHQUFHLEtBQUssQ0FBQyxNQUFNLENBQUMsSUFBSSxDQUFDLENBQUM7WUFDM0IsSUFBSSxLQUFLLENBQUMsTUFBTSxJQUFJLElBQUksRUFBRTtnQkFDeEIsU0FBUyxDQUFDLElBQUksQ0FBQyxDQUFDO2FBQ2pCO1lBQ0QsUUFBUSxFQUFFLENBQUM7UUFDYixDQUFDO1FBRUQsS0FBSztZQUNILElBQUksS0FBSyxDQUFDLE1BQU0sR0FBRyxDQUFDLEVBQUU7Z0JBQ3BCLFNBQVMsQ0FBQyxJQUFJLENBQUMsQ0FBQzthQUNqQjtZQUNELElBQUksQ0FBQyxJQUFJLENBQUMsSUFBSSxDQUFDLENBQUM7UUFDbEIsQ0FBQztLQUNGLENBQUMsQ0FBQztBQUNMLENBQUM7QUFRRCxlQUFlO0FBQ2YsTUFBTSxVQUFVLFlBQVksQ0FDMUIsTUFBMkMsRUFDM0MsRUFBRSxTQUFTLEVBQUUsU0FBUyxHQUFHLEdBQUcsRUFBRSxVQUFVLEdBQUcsR0FBRyxFQUF5QjtJQUV2RSxPQUFPLElBQUksT0FBTyxDQUFDLENBQUMsT0FBTyxFQUFFLE1BQU0sRUFBRSxFQUFFO1FBQ3JDLE1BQU0sTUFBTSxHQUFHLE1BQU0sWUFBWSxRQUFRLENBQUMsQ0FBQyxDQUFDLE1BQU0sQ0FBQyxDQUFDLENBQUMsYUFBYSxDQUFDLE1BQU0sQ0FBQyxDQUFDO1FBQzNFLE1BQU07YUFDSCxJQUFJLENBQUMsV0FBVyxDQUFDLFNBQVMsQ0FBQyxDQUFDO2FBQzVCLElBQUksQ0FBQyxnQkFBZ0IsQ0FBQyxTQUFTLEVBQUUsVUFBVSxDQUFDLENBQUM7YUFDN0MsRUFBRSxDQUFDLFFBQVEsRUFBRSxPQUFPLENBQUM7YUFDckIsRUFBRSxDQUFDLE9BQU8sRUFBRSxNQUFNLENBQUMsQ0FBQztJQUN6QixDQUFDLENBQUMsQ0FBQztBQUNMLENBQUM7QUFFRCxlQUFlO0FBQ2YsU0FBUyxnQkFBZ0IsQ0FBQyxTQUFpQixFQUFFLFVBQWtCO0lBQzdELE9BQU8sSUFBSSxRQUFRLENBQUM7UUFDbEIsVUFBVSxFQUFFLElBQUk7UUFFaEIsS0FBSyxDQUFDLEVBQUUsV0FBVyxFQUFFLEtBQUssRUFBRSxFQUFFLENBQUMsRUFBRSxRQUFRO1lBQ3ZDLGdCQUFnQixDQUFDLEtBQUssQ0FBQztpQkFDcEIsSUFBSSxDQUFDLFVBQVUsRUFBRSxDQUFDO2lCQUNsQixJQUFJLENBQUMsaUJBQWlCLENBQUMsR0FBRyxTQUFTLElBQUksVUFBVSxJQUFJLE1BQU0sQ0FBQyxXQUFXLENBQUMsQ0FBQyxRQUFRLENBQUMsQ0FBQyxFQUFFLEdBQUcsQ0FBQyxTQUFTLENBQUMsQ0FBQztpQkFDcEcsRUFBRSxDQUFDLFFBQVEsRUFBRSxRQUFRLENBQUMsQ0FBQztRQUM1QixDQUFDO0tBQ0YsQ0FBQyxDQUFDO0FBQ0wsQ0FBQztBQUVELGVBQWU7QUFDZixTQUFTLGdCQUFnQixDQUFJLEtBQVU7SUFDckMsSUFBSSxVQUFVLEdBQUcsQ0FBQyxDQUFDO0lBQ25CLE9BQU8sSUFBSSxRQUFRLENBQUM7UUFDbEIsSUFBSTtZQUNGLE1BQU0sS0FBSyxHQUFHLEtBQUssQ0FBQyxLQUFLLENBQUMsVUFBVSxFQUFFLFVBQVUsR0FBRyxFQUFFLENBQUMsQ0FBQztZQUN2RCxVQUFVLElBQUksRUFBRSxDQUFDO1lBQ2pCLElBQUksS0FBSyxDQUFDLE1BQU0sS0FBSyxDQUFDLEVBQUU7Z0JBQ3RCLElBQUksQ0FBQyxJQUFJLENBQUMsSUFBSSxDQUFDLENBQUM7YUFDakI7aUJBQU07Z0JBQ0wsS0FBSyxNQUFNLElBQUksSUFBSSxLQUFLLEVBQUU7b0JBQ3hCLElBQUksQ0FBQyxJQUFJLENBQUMsSUFBSSxDQUFDLFNBQVMsQ0FBQyxJQUFJLENBQUMsQ0FBQyxDQUFDO29CQUNoQyxJQUFJLENBQUMsSUFBSSxDQUFDLElBQUksQ0FBQyxDQUFDO2lCQUNqQjthQUNGO1FBQ0gsQ0FBQztLQUNGLENBQUMsQ0FBQztBQUNMLENBQUMifQ== |
@@ -14,9 +14,9 @@ interface StoryMandatoryFields { | ||
| readonly 'published-at': number; | ||
| /** The type of the story. Use 'text' for a normal story */ | ||
| readonly "story-template": string; | ||
| /** The type of the story. Use `'text'` for a normal story */ | ||
| readonly 'story-template': string; | ||
| } | ||
| interface StoryHeroImageFields { | ||
| /** A URL to pull the hero image from. This can be absolute, or relative starting with '/' */ | ||
| readonly "temporary-hero-image-url"?: string; | ||
| readonly "hero-image-caption"?: string; | ||
| readonly 'temporary-hero-image-url'?: string; | ||
| readonly 'hero-image-caption'?: string; | ||
| } | ||
@@ -45,3 +45,3 @@ interface StoryMetadataFields { | ||
| interface StoryBody { | ||
| /** An HTML blob of the story. Should look like "<p>Para1</p><p>Para2</p>" */ | ||
| /** An HTML blob of the story. Should look like `"<p>Para1</p><p>Para2</p>""` */ | ||
| readonly body: string; | ||
@@ -48,0 +48,0 @@ } |
@@ -0,1 +1,3 @@ | ||
| /// <reference types="node" /> | ||
| import { Readable } from 'stream'; | ||
| import { GenerateToFileOptions } from './async-writer'; | ||
@@ -22,6 +24,27 @@ import { Story } from './editor-types'; | ||
| * | ||
| * @param stream An Async Generator which yields stories | ||
| * You may also use a stream to produce stories | ||
| * | ||
| * ```ts | ||
| * import { Story, writeStories } from '@quintype/migration-helpers'; | ||
| * import { Transform } from 'stream' | ||
| * | ||
| * const stream = conn.query("select * from stories").stream(); | ||
| * | ||
| * const transformSqlRowToStories = new Transform({ | ||
| * objectMode: true, | ||
| * | ||
| * transform(rows, _, callback) { | ||
| * for(const story of rows) { | ||
| * this.push(rowToStory(story)) | ||
| * } | ||
| * } | ||
| * }) | ||
| * | ||
| * writeStories(stream.pipe(transformSqlRowToStories)); | ||
| * ``` | ||
| * | ||
| * @param stream An Async Generator or Readable which yields stories | ||
| * @param source A string describing where the stories come from. ex: interviews | ||
| * @param opts Control some fine grained tuning | ||
| */ | ||
| export declare function writeStories(stream: AsyncIterableIterator<Story>, source?: string, opts?: GenerateToFileOptions): Promise<void>; | ||
| export declare function writeStories(stream: AsyncIterableIterator<Story> | Readable, source?: string, opts?: GenerateToFileOptions): Promise<void>; |
@@ -21,3 +21,24 @@ import { writeToFiles } from './async-writer'; | ||
| * | ||
| * @param stream An Async Generator which yields stories | ||
| * You may also use a stream to produce stories | ||
| * | ||
| * ```ts | ||
| * import { Story, writeStories } from '@quintype/migration-helpers'; | ||
| * import { Transform } from 'stream' | ||
| * | ||
| * const stream = conn.query("select * from stories").stream(); | ||
| * | ||
| * const transformSqlRowToStories = new Transform({ | ||
| * objectMode: true, | ||
| * | ||
| * transform(rows, _, callback) { | ||
| * for(const story of rows) { | ||
| * this.push(rowToStory(story)) | ||
| * } | ||
| * } | ||
| * }) | ||
| * | ||
| * writeStories(stream.pipe(transformSqlRowToStories)); | ||
| * ``` | ||
| * | ||
| * @param stream An Async Generator or Readable which yields stories | ||
| * @param source A string describing where the stories come from. ex: interviews | ||
@@ -30,2 +51,2 @@ * @param opts Control some fine grained tuning | ||
| } | ||
| //# sourceMappingURL=data:application/json;base64,eyJ2ZXJzaW9uIjozLCJmaWxlIjoic3Rvcmllcy13cml0ZXIuanMiLCJzb3VyY2VSb290IjoiIiwic291cmNlcyI6WyIuLi8uLi8uLi9zcmMvbGliL3N0b3JpZXMtd3JpdGVyLnRzIl0sIm5hbWVzIjpbXSwibWFwcGluZ3MiOiJBQUFBLE9BQU8sRUFBeUIsWUFBWSxFQUFFLE1BQU0sZ0JBQWdCLENBQUM7QUFHckU7Ozs7Ozs7Ozs7Ozs7Ozs7Ozs7Ozs7R0FzQkc7QUFDSCxNQUFNLFVBQVUsWUFBWSxDQUMxQixNQUFvQyxFQUNwQyxTQUFpQixRQUFRLEVBQ3pCLE9BQThCLEVBQUU7SUFFaEMsTUFBTSxVQUFVLEdBQUcsSUFBSSxDQUFDLFVBQVUsQ0FBQyxDQUFDLENBQUMsU0FBUyxNQUFNLElBQUksSUFBSSxDQUFDLFVBQVUsRUFBRSxDQUFDLENBQUMsQ0FBQyxTQUFTLE1BQU0sRUFBRSxDQUFDO0lBQzlGLE9BQU8sWUFBWSxDQUFDLE1BQU0sRUFBRSxFQUFFLFVBQVUsRUFBRSxHQUFHLElBQUksRUFBRSxDQUFDLENBQUM7QUFDdkQsQ0FBQyJ9 | ||
| //# sourceMappingURL=data:application/json;base64,eyJ2ZXJzaW9uIjozLCJmaWxlIjoic3Rvcmllcy13cml0ZXIuanMiLCJzb3VyY2VSb290IjoiIiwic291cmNlcyI6WyIuLi8uLi8uLi9zcmMvbGliL3N0b3JpZXMtd3JpdGVyLnRzIl0sIm5hbWVzIjpbXSwibWFwcGluZ3MiOiJBQUVBLE9BQU8sRUFBeUIsWUFBWSxFQUFFLE1BQU0sZ0JBQWdCLENBQUM7QUFHckU7Ozs7Ozs7Ozs7Ozs7Ozs7Ozs7Ozs7Ozs7Ozs7Ozs7Ozs7Ozs7Ozs7Ozs7R0EyQ0c7QUFDSCxNQUFNLFVBQVUsWUFBWSxDQUMxQixNQUErQyxFQUMvQyxTQUFpQixRQUFRLEVBQ3pCLE9BQThCLEVBQUU7SUFFaEMsTUFBTSxVQUFVLEdBQUcsSUFBSSxDQUFDLFVBQVUsQ0FBQyxDQUFDLENBQUMsU0FBUyxNQUFNLElBQUksSUFBSSxDQUFDLFVBQVUsRUFBRSxDQUFDLENBQUMsQ0FBQyxTQUFTLE1BQU0sRUFBRSxDQUFDO0lBQzlGLE9BQU8sWUFBWSxDQUFDLE1BQU0sRUFBRSxFQUFFLFVBQVUsRUFBRSxHQUFHLElBQUksRUFBRSxDQUFDLENBQUM7QUFDdkQsQ0FBQyJ9 |
@@ -5,3 +5,13 @@ # Change Log | ||
| <a name="1.1.0"></a> | ||
| # [1.1.0](https://github.com/gja/@quintype/migration-helpers/compare/v1.0.1...v1.1.0) (2019-04-04) | ||
| ### Features | ||
| * writeStories can now take a stream of stories ([03112a0](https://github.com/gja/@quintype/migration-helpers/commit/03112a0)) | ||
| <a name="1.0.1"></a> | ||
| ## 1.0.1 (2019-04-03) |
| { | ||
| "name": "@quintype/migration-helpers", | ||
| "version": "1.0.1", | ||
| "version": "1.1.0", | ||
| "description": "Helper Functions for Migrations", | ||
@@ -28,4 +28,4 @@ "main": "build/main/index.js", | ||
| "doc": "run-s doc:html && opn build/docs/index.html", | ||
| "doc:html": "typedoc src/ --exclude **/*.spec.ts --target ES6 --mode file --out build/docs", | ||
| "doc:json": "typedoc src/ --exclude **/*.spec.ts --target ES6 --mode file --json build/docs/typedoc.json", | ||
| "doc:html": "typedoc src/ --exclude **/*.spec.ts --target ES6 --excludeExternals --mode file --readme Documentation.md --out build/docs", | ||
| "doc:json": "typedoc src/ --exclude **/*.spec.ts --target ES6 --mode file --readme Documentation.md --json build/docs/typedoc.json", | ||
| "doc:publish": "gh-pages -m \"[ci skip] Updates\" -d build/docs", | ||
@@ -74,4 +74,4 @@ "version": "standard-version", | ||
| "tslint-immutable": "^5.0.0", | ||
| "typedoc": "^0.13.0", | ||
| "typescript": "^3.1.6" | ||
| "typedoc": "^0.14.2", | ||
| "typescript": "^3.4.1" | ||
| }, | ||
@@ -78,0 +78,0 @@ "ava": { |
New alerts
License Policy Violation
LicenseThis package is not allowed per your license policy. Review the package's license to ensure compliance.
Found 1 instance in 1 package
Fixed alerts
License Policy Violation
LicenseThis package is not allowed per your license policy. Review the package's license to ensure compliance.
Found 1 instance in 1 package
Improved metrics
- Total package byte prevSize
- increased by15.14%
37042
- Number of package files
- increased by3.7%
28
- Lines of code
- increased by26.85%
548
No dependency changes