template-replace-stream
Advanced tools
Comparing version
@@ -11,3 +11,3 @@ import { TransformOptions, Readable, Transform, TransformCallback } from 'node:stream'; | ||
* Default: `false`. If true, the stream throws an error when a template variable has no | ||
* replacement value | ||
* replacement value. Takes precedence over `removeUnmatchedTemplate`. | ||
*/ | ||
@@ -94,4 +94,6 @@ throwOnUnmatchedTemplate: boolean; | ||
* @param stringSource The source to write to the output stream | ||
* @param callback The callback to call when the source was written | ||
*/ | ||
private writeToOutput; | ||
private writeStreamToOutput; | ||
private toBuffer; | ||
@@ -98,0 +100,0 @@ } |
@@ -128,4 +128,7 @@ "use strict"; | ||
if (value) { | ||
yield this.writeToOutput(value); | ||
this.writeToOutput(value, callback); | ||
this._stack = this._stack.subarray(this._stackIndex); | ||
this._stackIndex = 0; | ||
if (this._state === 3 /* PIPING_STREAM */) | ||
return; | ||
} else { | ||
@@ -146,3 +149,4 @@ this.releaseStack(this._stackIndex); | ||
} | ||
callback(); | ||
if (this._state !== 3 /* PIPING_STREAM */) | ||
callback(); | ||
}); | ||
@@ -184,25 +188,27 @@ } | ||
findVariableEnd() { | ||
for (; this._stackIndex < this._options.maxVariableNameLength + this._startPattern.length; this._stackIndex++) { | ||
if (this._stackIndex >= this._stack.length) | ||
const nextEndIndex = this._stack.indexOf(this._endPattern[0], this._stackIndex); | ||
const nextStartIndex = this._stack.indexOf(this._startPattern[0], this._stackIndex); | ||
if (nextEndIndex === -1 && nextStartIndex === -1) { | ||
this._matchCount += this._stack.length - this._stackIndex; | ||
if (this._matchCount < this._options.maxVariableNameLength) { | ||
this._stackIndex = this._stack.length; | ||
return; | ||
const char = this._stack[this._stackIndex]; | ||
if (char === this._endPattern[0]) { | ||
this._state = 2 /* SEARCHING_END_PATTERN */; | ||
this._matchCount = 1; | ||
this._stackIndex++; | ||
return; | ||
} else if (char === this._startPattern[0]) { | ||
this._state = 0 /* SEARCHING_START_PATTERN */; | ||
this._matchCount = 1; | ||
this._stackIndex++; | ||
this.releaseStack(this._stackIndex - this._matchCount); | ||
return; | ||
} | ||
this._state = 0 /* SEARCHING_START_PATTERN */; | ||
if (this._options.throwOnUnmatchedTemplate) | ||
throw new Error("Variable name processing reached limit"); | ||
if (this._options.log) | ||
console.debug("Variable name processing reached limit, skipping"); | ||
this.releaseStack(this._stack.length); | ||
return; | ||
} | ||
if (this._options.throwOnUnmatchedTemplate) | ||
throw new Error("Variable name processing reached limit"); | ||
if (this._options.log) | ||
console.debug("Variable name processing reached limit, skipping"); | ||
this._state = 0 /* SEARCHING_START_PATTERN */; | ||
this.releaseStack(this._stackIndex); | ||
if (nextStartIndex === -1 || nextStartIndex > nextEndIndex) { | ||
this._state = 2 /* SEARCHING_END_PATTERN */; | ||
this._stackIndex = nextEndIndex + 1; | ||
} else { | ||
this._state = 0 /* SEARCHING_START_PATTERN */; | ||
this._stackIndex = nextStartIndex + 1; | ||
this.releaseStack(nextStartIndex); | ||
} | ||
this._matchCount = 1; | ||
} | ||
@@ -216,2 +222,3 @@ /** | ||
findEndPattern() { | ||
let match = true; | ||
for (; this._matchCount < this._endPattern.length; this._matchCount++ & this._stackIndex++) { | ||
@@ -222,9 +229,9 @@ if (this._stackIndex >= this._stack.length) | ||
this.releaseStack(this._stackIndex); | ||
this._matchCount = 0; | ||
this._state = 0 /* SEARCHING_START_PATTERN */; | ||
return false; | ||
match = false; | ||
break; | ||
} | ||
} | ||
this._matchCount = 0; | ||
this._state = 0 /* SEARCHING_START_PATTERN */; | ||
return true; | ||
return match; | ||
} | ||
@@ -258,7 +265,6 @@ /** | ||
if (value === void 0) { | ||
if (this._options.throwOnUnmatchedTemplate) { | ||
if (this._options.throwOnUnmatchedTemplate) | ||
throw new Error(`Variable "${variableName}" not found in the variable map`); | ||
} else if (this._options.log) { | ||
if (this._options.log) | ||
console.debug(`Unmatched variable "${variableName}"`); | ||
} | ||
} else { | ||
@@ -275,24 +281,35 @@ if (this._options.log) | ||
* @param stringSource The source to write to the output stream | ||
* @param callback The callback to call when the source was written | ||
*/ | ||
writeToOutput(stringSource) { | ||
writeToOutput(stringSource, callback) { | ||
if (stringSource instanceof import_node_stream.Readable) { | ||
this._state = 3 /* PIPING_STREAM */; | ||
this.writeStreamToOutput(stringSource).then(() => callback()).catch(callback); | ||
} else { | ||
this.push(this.toBuffer(stringSource)); | ||
} | ||
} | ||
writeStreamToOutput(stream) { | ||
return __async(this, null, function* () { | ||
if (stringSource instanceof import_node_stream.Readable) { | ||
try { | ||
for (var iter = __forAwait(stream), more, temp, error; more = !(temp = yield iter.next()).done; more = false) { | ||
const chunk = temp.value; | ||
if (!this.push(chunk)) { | ||
yield new Promise((resolve, reject) => { | ||
this.once("drain", resolve); | ||
this.once("error", reject); | ||
}); | ||
} | ||
} | ||
} catch (temp) { | ||
error = [temp]; | ||
} finally { | ||
try { | ||
for (var iter = __forAwait(stringSource), more, temp, error; more = !(temp = yield iter.next()).done; more = false) { | ||
const chunk = temp.value; | ||
this.push(chunk); | ||
} | ||
} catch (temp) { | ||
error = [temp]; | ||
more && (temp = iter.return) && (yield temp.call(iter)); | ||
} finally { | ||
try { | ||
more && (temp = iter.return) && (yield temp.call(iter)); | ||
} finally { | ||
if (error) | ||
throw error[0]; | ||
} | ||
if (error) | ||
throw error[0]; | ||
} | ||
} else { | ||
this.push(this.toBuffer(stringSource)); | ||
} | ||
this._state = 0 /* SEARCHING_START_PATTERN */; | ||
}); | ||
@@ -299,0 +316,0 @@ } |
{ | ||
"name": "template-replace-stream", | ||
"version": "2.0.0", | ||
"version": "2.1.0", | ||
"description": "A high performance template replace stream working on binary or string streams", | ||
@@ -5,0 +5,0 @@ "main": "dist/index.js", |
@@ -5,3 +5,3 @@ # template-replace-stream | ||
This module is written in pure TypeScript, consists of only 278 lines of code and has no other dependencies. It is flexible and allows replacing an arbitrary wide range of template variables while being extremely fast (see [Benchmarks](#benchmarks)). | ||
This module is written in pure TypeScript, consists of only 189 lines of code (including type definitions) and has no other dependencies. It is flexible and allows replacing an arbitrary wide range of template variables while being extremely fast (we reached over 20GiB/s, see [Benchmarks](#benchmarks)). | ||
@@ -21,3 +21,3 @@ ## Install | ||
```js | ||
const {TemplateReplaceStream} = require("template-replace-stream"); | ||
const {TemplateReplaceStream} = require("../dist"); | ||
const fs = require("node:fs"); | ||
@@ -42,17 +42,3 @@ const path = require("node:path"); | ||
```ts | ||
import {TemplateReplaceStream} from "template-replace-stream"; | ||
import fs from "node:fs"; | ||
import path from "node:path"; | ||
// create a map of variables to replace. This will replace "{{replace-me}}" with "really fast" | ||
const variables = new Map([["replace-me", "really fast"]]); | ||
// create the streams | ||
const readStream = fs.createReadStream(path.join(__dirname, "template.txt")); | ||
const writeStream = fs.createWriteStream(path.join(__dirname, "example.txt")); | ||
const templateReplaceStream = new TemplateReplaceStream(variables); | ||
// connect the streams and put the template replace stream in the middle | ||
readStream.pipe(templateReplaceStream).pipe(writeStream); | ||
writeStream.on("finish", () => console.log("Finished writing example.txt")); | ||
{{ typescript-example.ts }} | ||
``` | ||
@@ -69,43 +55,3 @@ | ||
```ts | ||
import {StringSource, TemplateReplaceStream} from "template-replace-stream"; | ||
import fs from "fs"; | ||
import path from "path"; | ||
import sloc from "sloc" | ||
const rootDir = path.join(__dirname, ".."); | ||
const exampleFiles = ["javascript-example.js", "typescript-example.ts", "create-readme.ts"]; | ||
const loc = sloc(fs.readFileSync(path.join(rootDir, "src", "template-replace-stream.ts"), "utf8"), "ts").total; | ||
/** | ||
* Opens a file stream and replaces the import paths in the examples. This is used to | ||
* have module imports in the README but still local imports in the examples. | ||
* | ||
* @param file The file to read. | ||
*/ | ||
function openExampleStream(file: string) { | ||
const replaceStream = new TemplateReplaceStream( | ||
new Map([ | ||
[`../src`, `"template-replace-stream"`], | ||
[`../dist`, `"template-replace-stream"`] | ||
]), | ||
{ | ||
startPattern: '"', | ||
endPattern: '"' | ||
} | ||
); | ||
return fs.createReadStream(path.join(__dirname, file)).pipe(replaceStream); | ||
} | ||
// the map of example files and their read streams and further template variables | ||
const templateMap = new Map<string, StringSource>(exampleFiles.map((file) => [file, openExampleStream(file)])); | ||
templateMap.set("loc", loc.toString()); | ||
// create the streams | ||
const readmeReadStream = fs.createReadStream(path.join(rootDir, "README.template.md")); | ||
const readmeWriteStream = fs.createWriteStream(path.join(rootDir, "README.md")); | ||
// connect the streams and put the template replace stream in the middle | ||
readmeReadStream.pipe(new TemplateReplaceStream(templateMap)).pipe(readmeWriteStream); | ||
readmeWriteStream.on("finish", () => console.log("Finished writing README.md")); | ||
{{ create-readme.ts }} | ||
``` | ||
@@ -143,7 +89,9 @@ </details> | ||
The benchmarks were run on my MacBook Pro with an Apple M1 Pro Chip and an on-board SSD. The "native" data refers to reading a files from disk without doing anything else with it (native `fs.Readable` streams). So they are the absolute highest possible. | ||
The benchmarks were run on my MacBook Pro with an Apple M1 Pro Chip. The data source were virtual files generated from- and to memory to omit any bottleneck due to the file system. The "native" data refers to reading a files from disk without doing anything else with it (native `fs.Readable` streams). So they are the absolute highest possible. | ||
## Replacing a single Template Variable in a large File | ||
 | ||
Like the raw file system stream, a `TemplateReplaceStream` becomes exponentially faster with an increasing source file size. It is more than 20x faster than the `replace-stream` when processing large files. The throughput of the `TemplateReplaceStream` was almost 20GiB/s when replacing a single variable in a 100MiB file. | ||
Like the raw file system stream, a `TemplateReplaceStream` becomes faster with an increasing source file size. It is more than 20x faster than the `replace-stream` when processing large files. The throughput of the `TemplateReplaceStream` was more than 20GiB/s when replacing a single variable in a 100MiB file. | ||
@@ -154,2 +102,27 @@  | ||
We will provide more benchmarks with the next release, especially with replacing a lot of variables. | ||
## Replacing 10 thousand Template Variables in a large File | ||
 | ||
You can see that the performance declines when working with more replacements. Note that one reason is the virtually generated workload (see "native" in the graph). `TemplateReplaceStream` still reaches 10GiB/s. | ||
 | ||
To replace ten thousand template variables in a 100MiB file, the `TemplateReplaceStream` takes around 10ms. Since this duration is similar for smaller file sizes, we can see that it does not perform too well in the 1MiB file. We will keep optimizing for that. | ||
## Changelog | ||
### 2.1.0 | ||
- Further improve performance by using `Buffer.indexOf()` to find the end of a template variable, too | ||
- Add more benchmarks | ||
### 2.0.0 | ||
- Drastically improve performance (by ~10x) by using `Buffer.indexOf()` instead of iterating over the buffer myself | ||
- Rename option `throwOnMissingVariable` to `throwOnUnmatchedTemplate` | ||
- Add benchmarks | ||
## 1.0.1 | ||
- Update README | ||
## 1.0.0 | ||
- Initial Release |
@@ -5,3 +5,3 @@ # template-replace-stream | ||
This module is written in pure TypeScript, consists of only {{loc}} lines of code and has no other dependencies. It is flexible and allows replacing an arbitrary wide range of template variables while being extremely fast (see [Benchmarks](#benchmarks)). | ||
This module is written in pure TypeScript, consists of only {{loc}} lines of code (including type definitions) and has no other dependencies. It is flexible and allows replacing an arbitrary wide range of template variables while being extremely fast (we reached over 20GiB/s, see [Benchmarks](#benchmarks)). | ||
@@ -72,7 +72,9 @@ ## Install | ||
The benchmarks were run on my MacBook Pro with an Apple M1 Pro Chip and an on-board SSD. The "native" data refers to reading a files from disk without doing anything else with it (native `fs.Readable` streams). So they are the absolute highest possible. | ||
The benchmarks were run on my MacBook Pro with an Apple M1 Pro Chip. The data source were virtual files generated from- and to memory to omit any bottleneck due to the file system. The "native" data refers to reading a files from disk without doing anything else with it (native `fs.Readable` streams). So they are the absolute highest possible. | ||
## Replacing a single Template Variable in a large File | ||
 | ||
Like the raw file system stream, a `TemplateReplaceStream` becomes exponentially faster with an increasing source file size. It is more than 20x faster than the `replace-stream` when processing large files. The throughput of the `TemplateReplaceStream` was almost 20GiB/s when replacing a single variable in a 100MiB file. | ||
Like the raw file system stream, a `TemplateReplaceStream` becomes faster with an increasing source file size. It is more than 20x faster than the `replace-stream` when processing large files. The throughput of the `TemplateReplaceStream` was more than 20GiB/s when replacing a single variable in a 100MiB file. | ||
@@ -83,2 +85,27 @@  | ||
We will provide more benchmarks with the next release, especially with replacing a lot of variables. | ||
## Replacing 10 thousand Template Variables in a large File | ||
 | ||
You can see that the performance declines when working with more replacements. Note that one reason is the virtually generated workload (see "native" in the graph). `TemplateReplaceStream` still reaches 10GiB/s. | ||
 | ||
To replace ten thousand template variables in a 100MiB file, the `TemplateReplaceStream` takes around 10ms. Since this duration is similar for smaller file sizes, we can see that it does not perform too well in the 1MiB file. We will keep optimizing for that. | ||
## Changelog | ||
### 2.1.0 | ||
- Further improve performance by using `Buffer.indexOf()` to find the end of a template variable, too | ||
- Add more benchmarks | ||
### 2.0.0 | ||
- Drastically improve performance (by ~10x) by using `Buffer.indexOf()` instead of iterating over the buffer myself | ||
- Rename option `throwOnMissingVariable` to `throwOnUnmatchedTemplate` | ||
- Add benchmarks | ||
## 1.0.1 | ||
- Update README | ||
## 1.0.0 | ||
- Initial Release |
Sorry, the diff of this file is not supported yet
Sorry, the diff of this file is not supported yet
Sorry, the diff of this file is not supported yet
Sorry, the diff of this file is not supported yet
79144
6.22%707
5.37%122
-18.12%