You're Invited:Meet the Socket Team at BlackHat and DEF CON in Las Vegas, Aug 4-6.RSVP →
Socket
Book a DemoInstallSign in
Socket
Book a DemoInstallSign in

template-replace-stream

Package Overview
Dependencies
Maintainers
1
Versions
6
Alerts
File Explorer

Advanced tools

License
Socket logo

Install Socket

Detect and block malicious and high-risk dependencies

Install

template-replace-stream - npm Package Compare versions

Comparing version

2.0.0
to
2.1.0

4

dist/index.d.ts

@@ -11,3 +11,3 @@ import { TransformOptions, Readable, Transform, TransformCallback } from 'node:stream';

* Default: `false`. If true, the stream throws an error when a template variable has no
* replacement value
* replacement value. Takes precedence over `removeUnmatchedTemplate`.
*/

@@ -94,4 +94,6 @@ throwOnUnmatchedTemplate: boolean;

* @param stringSource The source to write to the output stream
* @param callback The callback to call when the source was written
*/
private writeToOutput;
private writeStreamToOutput;
private toBuffer;

@@ -98,0 +100,0 @@ }

107

dist/index.js

@@ -128,4 +128,7 @@ "use strict";

if (value) {
yield this.writeToOutput(value);
this.writeToOutput(value, callback);
this._stack = this._stack.subarray(this._stackIndex);
this._stackIndex = 0;
if (this._state === 3 /* PIPING_STREAM */)
return;
} else {

@@ -146,3 +149,4 @@ this.releaseStack(this._stackIndex);

}
callback();
if (this._state !== 3 /* PIPING_STREAM */)
callback();
});

@@ -184,25 +188,27 @@ }

findVariableEnd() {
for (; this._stackIndex < this._options.maxVariableNameLength + this._startPattern.length; this._stackIndex++) {
if (this._stackIndex >= this._stack.length)
const nextEndIndex = this._stack.indexOf(this._endPattern[0], this._stackIndex);
const nextStartIndex = this._stack.indexOf(this._startPattern[0], this._stackIndex);
if (nextEndIndex === -1 && nextStartIndex === -1) {
this._matchCount += this._stack.length - this._stackIndex;
if (this._matchCount < this._options.maxVariableNameLength) {
this._stackIndex = this._stack.length;
return;
const char = this._stack[this._stackIndex];
if (char === this._endPattern[0]) {
this._state = 2 /* SEARCHING_END_PATTERN */;
this._matchCount = 1;
this._stackIndex++;
return;
} else if (char === this._startPattern[0]) {
this._state = 0 /* SEARCHING_START_PATTERN */;
this._matchCount = 1;
this._stackIndex++;
this.releaseStack(this._stackIndex - this._matchCount);
return;
}
this._state = 0 /* SEARCHING_START_PATTERN */;
if (this._options.throwOnUnmatchedTemplate)
throw new Error("Variable name processing reached limit");
if (this._options.log)
console.debug("Variable name processing reached limit, skipping");
this.releaseStack(this._stack.length);
return;
}
if (this._options.throwOnUnmatchedTemplate)
throw new Error("Variable name processing reached limit");
if (this._options.log)
console.debug("Variable name processing reached limit, skipping");
this._state = 0 /* SEARCHING_START_PATTERN */;
this.releaseStack(this._stackIndex);
if (nextStartIndex === -1 || nextStartIndex > nextEndIndex) {
this._state = 2 /* SEARCHING_END_PATTERN */;
this._stackIndex = nextEndIndex + 1;
} else {
this._state = 0 /* SEARCHING_START_PATTERN */;
this._stackIndex = nextStartIndex + 1;
this.releaseStack(nextStartIndex);
}
this._matchCount = 1;
}

@@ -216,2 +222,3 @@ /**

findEndPattern() {
let match = true;
for (; this._matchCount < this._endPattern.length; this._matchCount++ & this._stackIndex++) {

@@ -222,9 +229,9 @@ if (this._stackIndex >= this._stack.length)

this.releaseStack(this._stackIndex);
this._matchCount = 0;
this._state = 0 /* SEARCHING_START_PATTERN */;
return false;
match = false;
break;
}
}
this._matchCount = 0;
this._state = 0 /* SEARCHING_START_PATTERN */;
return true;
return match;
}

@@ -258,7 +265,6 @@ /**

if (value === void 0) {
if (this._options.throwOnUnmatchedTemplate) {
if (this._options.throwOnUnmatchedTemplate)
throw new Error(`Variable "${variableName}" not found in the variable map`);
} else if (this._options.log) {
if (this._options.log)
console.debug(`Unmatched variable "${variableName}"`);
}
} else {

@@ -275,24 +281,35 @@ if (this._options.log)

* @param stringSource The source to write to the output stream
* @param callback The callback to call when the source was written
*/
writeToOutput(stringSource) {
writeToOutput(stringSource, callback) {
if (stringSource instanceof import_node_stream.Readable) {
this._state = 3 /* PIPING_STREAM */;
this.writeStreamToOutput(stringSource).then(() => callback()).catch(callback);
} else {
this.push(this.toBuffer(stringSource));
}
}
writeStreamToOutput(stream) {
return __async(this, null, function* () {
if (stringSource instanceof import_node_stream.Readable) {
try {
for (var iter = __forAwait(stream), more, temp, error; more = !(temp = yield iter.next()).done; more = false) {
const chunk = temp.value;
if (!this.push(chunk)) {
yield new Promise((resolve, reject) => {
this.once("drain", resolve);
this.once("error", reject);
});
}
}
} catch (temp) {
error = [temp];
} finally {
try {
for (var iter = __forAwait(stringSource), more, temp, error; more = !(temp = yield iter.next()).done; more = false) {
const chunk = temp.value;
this.push(chunk);
}
} catch (temp) {
error = [temp];
more && (temp = iter.return) && (yield temp.call(iter));
} finally {
try {
more && (temp = iter.return) && (yield temp.call(iter));
} finally {
if (error)
throw error[0];
}
if (error)
throw error[0];
}
} else {
this.push(this.toBuffer(stringSource));
}
this._state = 0 /* SEARCHING_START_PATTERN */;
});

@@ -299,0 +316,0 @@ }

2

package.json
{
"name": "template-replace-stream",
"version": "2.0.0",
"version": "2.1.0",
"description": "A high performance template replace stream working on binary or string streams",

@@ -5,0 +5,0 @@ "main": "dist/index.js",

95

README.md

@@ -5,3 +5,3 @@ # template-replace-stream

This module is written in pure TypeScript, consists of only 278 lines of code and has no other dependencies. It is flexible and allows replacing an arbitrary wide range of template variables while being extremely fast (see [Benchmarks](#benchmarks)).
This module is written in pure TypeScript, consists of only 189 lines of code (including type definitions) and has no other dependencies. It is flexible and allows replacing an arbitrary wide range of template variables while being extremely fast (we reached over 20GiB/s, see [Benchmarks](#benchmarks)).

@@ -21,3 +21,3 @@ ## Install

```js
const {TemplateReplaceStream} = require("template-replace-stream");
const {TemplateReplaceStream} = require("../dist");
const fs = require("node:fs");

@@ -42,17 +42,3 @@ const path = require("node:path");

```ts
import {TemplateReplaceStream} from "template-replace-stream";
import fs from "node:fs";
import path from "node:path";
// create a map of variables to replace. This will replace "{{replace-me}}" with "really fast"
const variables = new Map([["replace-me", "really fast"]]);
// create the streams
const readStream = fs.createReadStream(path.join(__dirname, "template.txt"));
const writeStream = fs.createWriteStream(path.join(__dirname, "example.txt"));
const templateReplaceStream = new TemplateReplaceStream(variables);
// connect the streams and put the template replace stream in the middle
readStream.pipe(templateReplaceStream).pipe(writeStream);
writeStream.on("finish", () => console.log("Finished writing example.txt"));
{{ typescript-example.ts }}
```

@@ -69,43 +55,3 @@

```ts
import {StringSource, TemplateReplaceStream} from "template-replace-stream";
import fs from "fs";
import path from "path";
import sloc from "sloc"
const rootDir = path.join(__dirname, "..");
const exampleFiles = ["javascript-example.js", "typescript-example.ts", "create-readme.ts"];
const loc = sloc(fs.readFileSync(path.join(rootDir, "src", "template-replace-stream.ts"), "utf8"), "ts").total;
/**
* Opens a file stream and replaces the import paths in the examples. This is used to
* have module imports in the README but still local imports in the examples.
*
* @param file The file to read.
*/
function openExampleStream(file: string) {
const replaceStream = new TemplateReplaceStream(
new Map([
[`../src`, `"template-replace-stream"`],
[`../dist`, `"template-replace-stream"`]
]),
{
startPattern: '"',
endPattern: '"'
}
);
return fs.createReadStream(path.join(__dirname, file)).pipe(replaceStream);
}
// the map of example files and their read streams and further template variables
const templateMap = new Map<string, StringSource>(exampleFiles.map((file) => [file, openExampleStream(file)]));
templateMap.set("loc", loc.toString());
// create the streams
const readmeReadStream = fs.createReadStream(path.join(rootDir, "README.template.md"));
const readmeWriteStream = fs.createWriteStream(path.join(rootDir, "README.md"));
// connect the streams and put the template replace stream in the middle
readmeReadStream.pipe(new TemplateReplaceStream(templateMap)).pipe(readmeWriteStream);
readmeWriteStream.on("finish", () => console.log("Finished writing README.md"));
{{ create-readme.ts }}
```

@@ -143,7 +89,9 @@ </details>

The benchmarks were run on my MacBook Pro with an Apple M1 Pro Chip and an on-board SSD. The "native" data refers to reading a files from disk without doing anything else with it (native `fs.Readable` streams). So they are the absolute highest possible.
The benchmarks were run on my MacBook Pro with an Apple M1 Pro Chip. The data source were virtual files generated from- and to memory to omit any bottleneck due to the file system. The "native" data refers to reading a files from disk without doing anything else with it (native `fs.Readable` streams). So they are the absolute highest possible.
## Replacing a single Template Variable in a large File
![Throughput vs. File Size when replacing a single Variable](benchmarks/plots/throughput-vs-data-size-with-one-replacement.png)
Like the raw file system stream, a `TemplateReplaceStream` becomes exponentially faster with an increasing source file size. It is more than 20x faster than the `replace-stream` when processing large files. The throughput of the `TemplateReplaceStream` was almost 20GiB/s when replacing a single variable in a 100MiB file.
Like the raw file system stream, a `TemplateReplaceStream` becomes faster with an increasing source file size. It is more than 20x faster than the `replace-stream` when processing large files. The throughput of the `TemplateReplaceStream` was more than 20GiB/s when replacing a single variable in a 100MiB file.

@@ -154,2 +102,27 @@ ![Duration vs File Size when replacing a single Variable](benchmarks/plots/size-vs-duration-with-one-replacement.png)

We will provide more benchmarks with the next release, especially with replacing a lot of variables.
## Replacing 10 thousand Template Variables in a large File
![Throughput vs. File Size when replacing a 10K Variables](benchmarks/plots/throughput-vs-data-size-with-10k-replacement.png)
You can see that the performance declines when working with more replacements. Note that one reason is the virtually generated workload (see "native" in the graph). `TemplateReplaceStream` still reaches 10GiB/s.
![Duration vs File Size when replacing a 10K Variables](benchmarks/plots/size-vs-duration-with-10k-replacement.png)
To replace ten thousand template variables in a 100MiB file, the `TemplateReplaceStream` takes around 10ms. Since this duration is similar for smaller file sizes, we can see that it does not perform too well in the 1MiB file. We will keep optimizing for that.
## Changelog
### 2.1.0
- Further improve performance by using `Buffer.indexOf()` to find the end of a template variable, too
- Add more benchmarks
### 2.0.0
- Drastically improve performance (by ~10x) by using `Buffer.indexOf()` instead of iterating over the buffer myself
- Rename option `throwOnMissingVariable` to `throwOnUnmatchedTemplate`
- Add benchmarks
## 1.0.1
- Update README
## 1.0.0
- Initial Release

35

README.template.md

@@ -5,3 +5,3 @@ # template-replace-stream

This module is written in pure TypeScript, consists of only {{loc}} lines of code and has no other dependencies. It is flexible and allows replacing an arbitrary wide range of template variables while being extremely fast (see [Benchmarks](#benchmarks)).
This module is written in pure TypeScript, consists of only {{loc}} lines of code (including type definitions) and has no other dependencies. It is flexible and allows replacing an arbitrary wide range of template variables while being extremely fast (we reached over 20GiB/s, see [Benchmarks](#benchmarks)).

@@ -72,7 +72,9 @@ ## Install

The benchmarks were run on my MacBook Pro with an Apple M1 Pro Chip and an on-board SSD. The "native" data refers to reading a files from disk without doing anything else with it (native `fs.Readable` streams). So they are the absolute highest possible.
The benchmarks were run on my MacBook Pro with an Apple M1 Pro Chip. The data source were virtual files generated from- and to memory to omit any bottleneck due to the file system. The "native" data refers to reading a files from disk without doing anything else with it (native `fs.Readable` streams). So they are the absolute highest possible.
## Replacing a single Template Variable in a large File
![Throughput vs. File Size when replacing a single Variable](benchmarks/plots/throughput-vs-data-size-with-one-replacement.png)
Like the raw file system stream, a `TemplateReplaceStream` becomes exponentially faster with an increasing source file size. It is more than 20x faster than the `replace-stream` when processing large files. The throughput of the `TemplateReplaceStream` was almost 20GiB/s when replacing a single variable in a 100MiB file.
Like the raw file system stream, a `TemplateReplaceStream` becomes faster with an increasing source file size. It is more than 20x faster than the `replace-stream` when processing large files. The throughput of the `TemplateReplaceStream` was more than 20GiB/s when replacing a single variable in a 100MiB file.

@@ -83,2 +85,27 @@ ![Duration vs File Size when replacing a single Variable](benchmarks/plots/size-vs-duration-with-one-replacement.png)

We will provide more benchmarks with the next release, especially with replacing a lot of variables.
## Replacing 10 thousand Template Variables in a large File
![Throughput vs. File Size when replacing a 10K Variables](benchmarks/plots/throughput-vs-data-size-with-10k-replacement.png)
You can see that the performance declines when working with more replacements. Note that one reason is the virtually generated workload (see "native" in the graph). `TemplateReplaceStream` still reaches 10GiB/s.
![Duration vs File Size when replacing a 10K Variables](benchmarks/plots/size-vs-duration-with-10k-replacement.png)
To replace ten thousand template variables in a 100MiB file, the `TemplateReplaceStream` takes around 10ms. Since this duration is similar for smaller file sizes, we can see that it does not perform too well in the 1MiB file. We will keep optimizing for that.
## Changelog
### 2.1.0
- Further improve performance by using `Buffer.indexOf()` to find the end of a template variable, too
- Add more benchmarks
### 2.0.0
- Drastically improve performance (by ~10x) by using `Buffer.indexOf()` instead of iterating over the buffer myself
- Rename option `throwOnMissingVariable` to `throwOnUnmatchedTemplate`
- Add benchmarks
## 1.0.1
- Update README
## 1.0.0
- Initial Release
dist/index.d.mts

Sorry, the diff of this file is not supported yet

dist/index.js.map

Sorry, the diff of this file is not supported yet

dist/index.mjs

Sorry, the diff of this file is not supported yet

dist/index.mjs.map

Sorry, the diff of this file is not supported yet